test(validation): migrate canonical release checks to auto-testing

.github/workflows/kms-script-tests.yml

@@ -5,12 +5,14 @@ on:
     paths:
       - rustfs-kms-test.sh
       - test_rustfs_kms_presign.py
+      - test_validation_kms.sh
       - .github/workflows/kms-script-tests.yml
   push:
     branches: [main]
     paths:
       - rustfs-kms-test.sh
       - test_rustfs_kms_presign.py
+      - test_validation_kms.sh
       - .github/workflows/kms-script-tests.yml
 
 permissions:
@@ -28,3 +30,5 @@ jobs:
         run: bash -n rustfs-kms-test.sh
       - name: Test presigned KMS case without a server
         run: python3 -m unittest test_rustfs_kms_presign.py
+      - name: Test canonical KMS round trip without a server
+        run: bash test_validation_kms.sh

.github/workflows/s3-script-tests.yml

@@ -6,13 +6,15 @@ on:
       - rustfs-s3-compat-test.sh
       - test_rustfs_s3_usage.py
       - test_rustfs_s3_signing.py
+      - test_validation_s3_basic.sh
       - .github/workflows/s3-script-tests.yml
   push:
     branches: [main]
     paths:
       - rustfs-s3-compat-test.sh
       - test_rustfs_s3_usage.py
       - test_rustfs_s3_signing.py
+      - test_validation_s3_basic.sh
       - .github/workflows/s3-script-tests.yml
 
 permissions:
@@ -32,3 +34,5 @@ jobs:
         run: python3 -m pip install botocore==1.43.19
       - name: Test S3 cases and signatures without a cluster
         run: python3 -m unittest test_rustfs_s3_usage.py test_rustfs_s3_signing.py
+      - name: Test validation S3 round trip without a cluster
+        run: bash test_validation_s3_basic.sh

.github/workflows/validation-catalog-tests.yml

@@ -0,0 +1,185 @@
+name: Validation Catalog Tests
+
+on:
+  pull_request:
+    paths:
+      - validation-cases.json
+      - validation-migration-review.json
+      - scripts/validation_migration_review.py
+      - scripts/validation_catalog.py
+      - scripts/validation_report.py
+      - scripts/validation_result.py
+      - test_validation_migration.py
+      - test_validation_ilm.py
+      - scripts/validation_ilm/**
+      - test_validation_pool.py
+      - scripts/validation_pool/**
+      - rustfs_pool_expand.sh
+      - test_validation_replication.py
+      - scripts/validation_replication/**
+      - test_validation_heal.py
+      - test_validation_heal*.py
+      - test_validation_heal_recovery.py
+      - test_validation_heal_authorization.py
+      - test_validation_heal_full_restart.py
+      - test_validation_heal_mrf.py
+      - test_validation_heal_unformatted.py
+      - test_validation_heal_performance.py
+      - test_validation_heal_multi_pool.py
+      - test_validation_heal_scopes.py
+      - test_validation_heal_bucket_metadata.py
+      - test_validation_heal_write_quorum.py
+      - test_validation_heal_coordinator_crash.py
+      - test_validation_heal_flapping.py
+      - test_validation_heal_capacity.py
+      - test_validation_heal_network.py
+      - test_validation_heal_encrypted.py
+      - test_validation_heal_schema.py
+      - test_validation_heal_native.py
+      - test_validation_heal_lock_successor.py
+      - test_validation_heal_race_native.py
+      - test_validation_heal_incarnation_native.py
+      - test_validation_heal_manager_native.py
+      - test_validation_heal_manager_fixture.py
+      - test_validation_heal_journal_fault.py
+      - test_validation_heal_checkpoint.py
+      - test_validation_heal_authority.py
+      - test_validation_heal_orphan.py
+      - test_validation_heal_options.py
+      - test_validation_heal_environment.py
+      - test_validation_heal_queue_pressure.py
+      - test_validation_heal_nolock.py
+      - test_validation_heal_race.py
+      - test_validation_heal_g06_source.py
+      - test_validation_heal_environment_gate.py
+      - test_validation_heal_outcome.py
+      - test_validation_heal_outcome_grace.py
+      - test_validation_heal_persistence.py
+      - test_validation_heal_persistence_mrf.py
+      - test_validation_heal_incarnation.py
+      - test_validation_heal_incarnation_replay.py
+      - scripts/validation_heal/**
+      - testdata/i05-*.ndjson
+      - testdata/validation/i05-*.zip
+      - docs/validation-heal-*.md
+      - rustfs_heal_test.sh
+      - test_validation_oidc.py
+      - scripts/validation_oidc.py
+      - scripts/validation_oidc/**
+      - test_validation_webhooks.py
+      - scripts/validation_webhooks.py
+      - scripts/run_validation_case.sh
+      - rustfs-replication-test.sh
+      - rustfs-storage-test.sh
+      - test_validation_scanner_usage.sh
+      - .github/workflows/validation-catalog-tests.yml
+  push:
+    branches: [main]
+    paths:
+      - validation-cases.json
+      - validation-migration-review.json
+      - scripts/validation_migration_review.py
+      - scripts/validation_catalog.py
+      - scripts/validation_report.py
+      - scripts/validation_result.py
+      - test_validation_migration.py
+      - test_validation_ilm.py
+      - scripts/validation_ilm/**
+      - test_validation_pool.py
+      - scripts/validation_pool/**
+      - rustfs_pool_expand.sh
+      - test_validation_replication.py
+      - scripts/validation_replication/**
+      - test_validation_heal.py
+      - test_validation_heal*.py
+      - test_validation_heal_recovery.py
+      - test_validation_heal_authorization.py
+      - test_validation_heal_full_restart.py
+      - test_validation_heal_mrf.py
+      - test_validation_heal_unformatted.py
+      - test_validation_heal_performance.py
+      - test_validation_heal_multi_pool.py
+      - test_validation_heal_scopes.py
+      - test_validation_heal_bucket_metadata.py
+      - test_validation_heal_write_quorum.py
+      - test_validation_heal_coordinator_crash.py
+      - test_validation_heal_flapping.py
+      - test_validation_heal_capacity.py
+      - test_validation_heal_network.py
+      - test_validation_heal_encrypted.py
+      - test_validation_heal_schema.py
+      - test_validation_heal_native.py
+      - test_validation_heal_lock_successor.py
+      - test_validation_heal_race_native.py
+      - test_validation_heal_incarnation_native.py
+      - test_validation_heal_manager_native.py
+      - test_validation_heal_manager_fixture.py
+      - test_validation_heal_journal_fault.py
+      - test_validation_heal_checkpoint.py
+      - test_validation_heal_authority.py
+      - test_validation_heal_orphan.py
+      - test_validation_heal_options.py
+      - test_validation_heal_environment.py
+      - test_validation_heal_queue_pressure.py
+      - test_validation_heal_nolock.py
+      - test_validation_heal_race.py
+      - test_validation_heal_g06_source.py
+      - test_validation_heal_environment_gate.py
+      - test_validation_heal_outcome.py
+      - test_validation_heal_outcome_grace.py
+      - test_validation_heal_persistence.py
+      - test_validation_heal_persistence_mrf.py
+      - test_validation_heal_incarnation.py
+      - test_validation_heal_incarnation_replay.py
+      - scripts/validation_heal/**
+      - testdata/i05-*.ndjson
+      - testdata/validation/i05-*.zip
+      - docs/validation-heal-*.md
+      - rustfs_heal_test.sh
+      - test_validation_oidc.py
+      - scripts/validation_oidc.py
+      - scripts/validation_oidc/**
+      - test_validation_webhooks.py
+      - scripts/validation_webhooks.py
+      - scripts/run_validation_case.sh
+      - rustfs-replication-test.sh
+      - rustfs-storage-test.sh
+      - test_validation_scanner_usage.sh
+      - .github/workflows/validation-catalog-tests.yml
+
+permissions:
+  contents: read
+
+jobs:
+  contract-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - run: python3 scripts/validation_catalog.py check
+      - run: python3 -B scripts/validation_migration_review.py --require-complete
+      - run: python3 -B -m unittest -v test_validation_migration.py
+      - run: python3 -B -m unittest -v test_validation_ilm.py
+      - run: python3 -B -m unittest -v test_validation_pool.py
+      - run: python3 -B -m unittest -v test_validation_replication.py
+      - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt
+      - name: Install network namespace test dependency
+        run: |
+          if ! command -v nft >/dev/null; then
+            sudo apt-get update
+            sudo apt-get install --no-install-recommends -y nftables
+          fi
+      - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py
+      - run: python3 -B -m unittest -v test_validation_heal_timeout_retention.py test_validation_heal_race_retention.py test_validation_heal_retained_scene.py test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_source.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_lock_successor.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py
+      - run: python3 -B scripts/validation_heal/matrix.py check
+      - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py
+      - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py
+      - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py
+      - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py
+      - run: python3 -B -m unittest -v test_validation_heal_race_native.py test_validation_heal_g06_source.py
+      - run: python3 -B -m unittest -v test_validation_heal_g02_source.py test_validation_heal_endpoint_resolution.py
+      - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh
+      - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh
+      - run: bash test_validation_scanner_usage.sh

README.md

@@ -1,6 +1,84 @@
 # auto-testing
 Auto testing scripts
 
+## Release validation migration
+
+`validation-cases.json` snapshots all 158 canonical cases from
+`rustfs-release-validation/scenarios/catalog.yaml` (66 implemented, 84
+procedure-ready, 8 planned). The source status is historical context. A run
+result carries `migration_status: verified` only after its assertions, topology,
+and live evidence are checked in this repository. `reuse_candidates` names
+existing suites to inspect and never grants coverage by itself.
+
+```bash
+python3 scripts/validation_catalog.py check
+python3 scripts/validation_catalog.py summary
+python3 -B -m unittest -v test_validation_migration.py
+```
+
+To refresh after an intentional source catalog change:
+
+```bash
+python3 scripts/validation_catalog.py sync \
+  --source /absolute/path/to/rustfs-release-validation/scenarios/catalog.yaml
+```
+
+`scripts/validation_report.py` accepts one JSON result per canonical case ID.
+Results must separately report source status, migration status, verdict,
+cleanup, candidate/rc provenance, and evidence for each source assertion.
+Pass `--gate-status implemented` only after the 66 migrated executors are
+ready; missing or unverified cases fail that gate. Without a gate status, the
+report shows migration progress and still rejects malformed or false PASS
+results.
+
+The executable canonical cases currently include `s3.basic-object-round-trip`,
+`scanner.usage-convergence`, `kms.sse-kms-round-trip`,
+`tiering.transition-and-restore`, `events.delivery`,
+`audit.request-correlation`, `iam.policy-enforcement`, and
+`scanner.status-readiness`. The S3, KMS, and IAM
+cases use single-disk
+deployment. The scanner case uses the storage suite's single-node four-disk
+deployment, waits for a post-upload successful scanner cycle, and requires a
+complete usage snapshot with exactly 20 objects and 20,690 logical bytes.
+Each requires a candidate manifest with `source_sha`, `package_url`, and
+`package_sha256`. The installed binary
+SHA-256 is measured on the test node and included in result provenance:
+
+```bash
+./rustfs-s3-compat-test.sh \
+  --validation-case s3.basic-object-round-trip \
+  --candidate-manifest /absolute/path/candidate.json \
+  --testing-commit "$(git rev-parse HEAD)" \
+  --validation-artifacts-dir /absolute/path/new-artifacts-dir \
+  --yes
+
+./rustfs-storage-test.sh \
+  --validation-case scanner.usage-convergence \
+  --candidate-manifest /absolute/path/candidate.json \
+  --testing-commit "$(git rev-parse HEAD)" \
+  --validation-artifacts-dir /absolute/path/another-new-artifacts-dir \
+  --yes
+
+./rustfs-kms-test.sh \
+  --validation-case kms.sse-kms-round-trip \
+  --candidate-manifest /absolute/path/candidate.json \
+  --testing-commit "$(git rev-parse HEAD)" \
+  --validation-artifacts-dir /absolute/path/kms-new-artifacts-dir \
+  --yes
+
+python3 scripts/validation_report.py \
+  --results-dir /absolute/path/new-artifacts-dir/cases \
+  --output /absolute/path/new-artifacts-dir/report.md
+```
+
+The report's full implemented-case gate is intentionally not enabled by these
+runs. Run it with `--gate-status implemented` only after all 66
+implemented source cases have verified executors and live results.
+
+S3, scanner, KMS, tiering, and IAM have live PASS evidence against nightly
+build `37958901959` attempt `1` on the current auto-testing fleet. Events and
+audit have local executor tests but still need live verification.
+
 ## Fault-tolerance suite
 
 `rustfs-fault-tolerance-test.sh` writes a machine-readable completion result