diff --git a/.github/workflows/rustfs-validation-heal-sweep.yml b/.github/workflows/rustfs-validation-heal-sweep.yml new file mode 100644 index 0000000..a9e4bd3 --- /dev/null +++ b/.github/workflows/rustfs-validation-heal-sweep.yml @@ -0,0 +1,519 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: RustFS Validation Heal Sweep + +# Nightly full-catalog Heal validation. Every run resolves the newest +# successful nightly candidate, triggers a source-bound validation-tools +# build on rustfs/rustfs for that exact candidate, then executes every +# runnable heal.* catalog case (procedure-ready minus the 13 +# operator-assisted environment scenarios) through the same +# run_validation_case.sh entry the interactive validation job uses. +# The environment scenarios and diagnostic entrances cannot run +# unattended on this fleet; the report lists them as BLOCKED. + +on: + schedule: + # After the nightly chain (build ~01:00 UTC + chain with the full + # validation lane) has normally released the shared fleet. + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + case: + description: 'Single heal case id (overrides the full list), e.g. heal.admin-healthy-noop' + type: string + required: false + default: '' + limit: + description: 'Maximum cases (0 = every runnable heal case)' + type: number + required: false + default: 0 + build_run_id: + description: 'Successful nightly package run; empty resolves the latest' + type: string + required: false + default: '' + build_run_attempt: + description: 'Attempt of that package run' + type: string + required: false + default: '1' + skip_tools: + description: 'Reuse the newest existing tools run instead of building (debug only)' + type: boolean + required: false + default: false + +permissions: + contents: read + +concurrency: + group: rustfs-validation-heal-sweep + cancel-in-progress: false + +env: + RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }} + RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }} + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + +jobs: + heal-sweep: + if: github.repository == 'rustfs/auto-testing' + runs-on: smoke-testing + concurrency: + group: rustfs-shared-functional-tests-v2 + cancel-in-progress: false + timeout-minutes: 600 + steps: + - name: Checkout chain tooling + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + persist-credentials: false + + - name: Resolve nightly candidate + id: candidate + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + BUILD_RUN_ID: ${{ inputs.build_run_id }} + BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }} + CHAIN_OUTPUT: ${{ runner.temp }}/validation-heal-candidate.json + run: | + set -euo pipefail + # Scheduled runs carry no inputs; resolve the newest successful + # nightly build explicitly before invoking the resolver. + if [ -z "${BUILD_RUN_ID:-}" ]; then + LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')" + BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')" + BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')" + export BUILD_RUN_ID BUILD_RUN_ATTEMPT + echo "heal sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}" + fi + python3 scripts/resolve_functional_candidate.py + python3 - <<'PY' + import json + import os + from pathlib import Path + root = Path(os.environ["RUNNER_TEMP"]) / "validation-heal" + root.mkdir(parents=True, exist_ok=True) + chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-heal-candidate.json").read_text()) + manifest = chain["candidate"]["manifest"] + (root / "candidate.json").write_text(json.dumps({ + "package_url": manifest["package_url"], + "package_sha256": manifest["package_sha256"], + "source_sha": manifest["source_sha"], + }, sort_keys=True) + "\n") + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write("testing_sha=" + chain["testing_sha"] + "\n") + output.write("package_url=" + manifest["package_url"] + "\n") + output.write("source_sha=" + manifest["source_sha"] + "\n") + PY + + - name: Build source-bound validation tools + id: tools + timeout-minutes: 60 + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + CANDIDATE_SHA: ${{ steps.candidate.outputs.source_sha }} + SKIP_TOOLS: ${{ inputs.skip_tools }} + run: | + set -euo pipefail + if [ "${SKIP_TOOLS}" = "true" ]; then + RUN="$(gh api "repos/rustfs/rustfs/actions/workflows/rustfs-validation-tools.yml/runs?status=success&per_page=1" --jq '.workflow_runs[0]')" + else + BEFORE="$(gh api "repos/rustfs/rustfs/actions/workflows/rustfs-validation-tools.yml/runs?per_page=1" --jq '.workflow_runs[0].id')" + # native/manager builds are pinned to reviewed reference + # candidates (REVIEWED_CANDIDATES allowlist) and fail on any + # other source SHA; the nightly sweep only needs the inspector + # and the shared fixture encoder, which build for any candidate. + gh api --method POST "repos/rustfs/rustfs/actions/workflows/rustfs-validation-tools.yml/dispatches" \ + -F ref="main" \ + -f "inputs[candidate_sha]=${CANDIDATE_SHA}" \ + -f "inputs[auto_testing_sha]=${GITHUB_SHA}" \ + -f "inputs[build_shared_heal]=true" >/dev/null + echo "dispatched validation-tools build for candidate ${CANDIDATE_SHA}" + # Wait for the dispatched run to appear, then for it to finish. + RUN="" + for _ in $(seq 1 60); do + RUN="$(gh api "repos/rustfs/rustfs/actions/workflows/rustfs-validation-tools.yml/runs?event=workflow_dispatch&per_page=10" \ + --arg before "${BEFORE}" --jq '[.workflow_runs[] | select(.id > ($before | tonumber))][0] // empty')" + [ -n "${RUN}" ] && break + sleep 10 + done + [ -n "${RUN}" ] || { echo "validation-tools run did not appear"; exit 1; } + RUN_ID="$(printf '%s' "${RUN}" | jq -r '.id')" + for _ in $(seq 1 240); do + STATUS="$(gh api "repos/rustfs/rustfs/actions/runs/${RUN_ID}" --jq '.status')" + [ "${STATUS}" = "completed" ] && break + sleep 15 + done + [ "${STATUS}" = "completed" ] || { echo "validation-tools run ${RUN_ID} did not complete in time"; exit 1; } + RUN="$(gh api "repos/rustfs/rustfs/actions/runs/${RUN_ID}/attempts/$(printf '%s' "${RUN}" | jq -r '.run_attempt')")" + fi + printf '%s' "${RUN}" | jq -e '.conclusion == "success"' >/dev/null \ + || { echo "validation-tools build failed"; printf '%s' "${RUN}" | jq '{id, conclusion, html_url}'; exit 1; } + printf '%s' "${RUN}" | jq -e '.path == ".github/workflows/rustfs-validation-tools.yml"' >/dev/null + TOOL_RUN_ID="$(printf '%s' "${RUN}" | jq -r '.id')" + TOOL_ATTEMPT="$(printf '%s' "${RUN}" | jq -r '.run_attempt')" + echo "validation-tools run: ${TOOL_RUN_ID} attempt ${TOOL_ATTEMPT}" + { + echo "run_id=${TOOL_RUN_ID}" + echo "attempt=${TOOL_ATTEMPT}" + } >> "${GITHUB_OUTPUT}" + + - name: Download tool artifacts + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + TOOL_RUN_ID: ${{ steps.tools.outputs.run_id }} + TOOL_ATTEMPT: ${{ steps.tools.outputs.attempt }} + run: | + set -euo pipefail + ROOT="${RUNNER_TEMP}/validation-heal-tools" + mkdir -p "${ROOT}" + for KIND in tools shared; do + NAME="rustfs-validation-${KIND}-${TOOL_RUN_ID}-${TOOL_ATTEMPT}" + DEST="${ROOT}/${KIND}" + gh api "repos/rustfs/rustfs/actions/runs/${TOOL_RUN_ID}/artifacts?per_page=100" \ + --arg name "${NAME}" --jq '.artifacts[] | select(.name == $name) | .id' | head -1 > /tmp/artifact-id + AID="$(cat /tmp/artifact-id)" + [ -n "${AID}" ] || { echo "missing artifact ${NAME}"; exit 1; } + curl -fsSL -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/rustfs/rustfs/actions/artifacts/${AID}/zip" -o "/tmp/${KIND}.zip" + mkdir -p "${DEST}" && unzip -oq "/tmp/${KIND}.zip" -d "${DEST}" + echo "${KIND}: $(ls "${DEST}" | head -5 | tr '\n' ' ')" + done + + - name: Verify tool binding to the candidate + env: + VALIDATION_ROOT: ${{ runner.temp }}/validation-heal + run: | + set -euo pipefail + python3 -B scripts/validation_heal/inspector.py \ + --directory "${RUNNER_TEMP}/validation-heal-tools/tools" \ + --candidate "${VALIDATION_ROOT}/candidate.json" \ + > "${VALIDATION_ROOT}/tool-manifest.json" + { + echo "VALIDATION_HEAL_TOOL_DIRECTORY=${RUNNER_TEMP}/validation-heal-tools/tools" + echo "VALIDATION_HEAL_SHARED_DIRECTORY=${RUNNER_TEMP}/validation-heal-tools/shared" + } >> "${GITHUB_ENV}" + + - name: Prepare validation Python runtime + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install validation Python requirements + run: | + set -euo pipefail + validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + test ! -e "${validation_python}" + python3 -m venv "${validation_python}" + "${validation_python}/bin/python" -m pip install \ + -r scripts/validation_ilm/requirements.txt + printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}" + + - name: Select heal cases + id: select + env: + SELECTED_CASE: ${{ inputs.case }} + CASE_LIMIT: ${{ inputs.limit }} + HEAL_ROOT: ${{ runner.temp }}/validation-heal + run: | + set -euo pipefail + export SELECTED_CASE CASE_LIMIT HEAL_ROOT + python3 - <<'PY' + import json, os + from pathlib import Path + # Environment scenarios from run_case.sh's environment_gate list: + # they need operator-assisted resources (forced restart hosts, + # dedicated topologies) and never run unattended. + env_gated = { + "heal.mixed-version-upgrade-rollback", "heal.dedicated-host-forced-restart", + "heal.race-ilm-expiry-and-transition", "heal.race-replication-resync", + "heal.with-expansion-rebalance", "heal.with-rebalance-stop-resume", + "heal.with-decommission", "heal.with-decommission-cancel-clear", + "heal.after-retired-pool-removal", "heal.ec84-multi-set-restart", + "heal.slow-io-and-eio", "heal.kms-and-tier-outage", "heal.long-soak", + } + # Candidate-pinned races/diagnostics: their executors hard-require + # the reviewed reference candidate (source_contract allowlists in + # race.py / I05 / F07) and cannot run against arbitrary nightlies. + candidate_pinned = { + "heal.race-unversioned-overwrite", "heal.race-versioned-put", + "heal.race-version-delete", "heal.race-markers-and-batch-delete", + "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update", + "heal.admin-nolock-safety", "heal.lock-timeout-and-caller-abort", + "heal.checkpoint-and-mrf-corruption", "heal.outcome-counters-and-error-truth", + "heal.bucket-incarnation-recreation", + } + catalog = json.loads(Path("validation-cases.json").read_text()) + selected = [c["id"] for m in catalog["modules"] if m["id"] in ("heal-scanner", "heal-extended") + for c in m["cases"] if c.get("source_status") == "procedure-ready"] + excluded = env_gated | candidate_pinned + runnable = [i for i in selected if i not in excluded] + blocked = sorted(i for i in selected if i in env_gated) + pinned = sorted(i for i in selected if i in candidate_pinned) + one = os.environ.get("SELECTED_CASE", "").strip() + if one: + if one not in runnable: + raise SystemExit(f"unknown or not runnable heal case: {one}") + runnable = [one] + limit = int(os.environ.get("CASE_LIMIT") or 0) + if limit > 0: + runnable = runnable[:limit] + root = Path(os.environ["HEAL_ROOT"]) + (root / "heal-cases.txt").write_text("\n".join(runnable) + "\n") + (root / "blocked-cases.txt").write_text("\n".join(blocked) + "\n") + (root / "pinned-cases.txt").write_text("\n".join(pinned) + "\n") + print(f"selected {len(runnable)} runnable heal case(s); " + f"{len(blocked)} environment-gated + {len(pinned)} candidate-pinned BLOCKED") + PY + + - name: Admit clean shared environment + id: environment + timeout-minutes: 5 + env: + FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-heal.json + run: python3 scripts/functional_environment.py enter --suite heal + + - name: Run heal validation cases + id: run_cases + timeout-minutes: 480 + env: + AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }} + HEAL_ROOT: ${{ runner.temp }}/validation-heal + run: | + set -uo pipefail + mkdir -p "${HEAL_ROOT}/results" + : > "${HEAL_ROOT}/results.txt" + while read -r CASE_ID; do + [ -n "${CASE_ID}" ] || continue + echo "::group::case ${CASE_ID}" + rm -rf "${HEAL_ROOT}/run" + rm -f "${HEAL_ROOT}/results/${CASE_ID//\//_}.timeout" + rc=0 + timeout 900 ./scripts/run_validation_case.sh \ + "${CASE_ID}" "${HEAL_ROOT}/candidate.json" \ + "${AUTO_TESTING_SHA}" "${HEAL_ROOT}/run" \ + < /dev/null \ + > "${HEAL_ROOT}/results/${CASE_ID//\//_}.log" 2>&1 || rc=$? + # The case verdict is authoritative from the result JSON: drivers + # may self-report BLOCKED (diagnostic gates) with a nonzero exit; + # only FAIL / INFRA_ERROR / TIMEOUT keep the sweep red. + VERDICT="" + if [ -f "${HEAL_ROOT}/run/cases/${CASE_ID}.json" ]; then + VERDICT="$(jq -r '.verdict // empty' "${HEAL_ROOT}/run/cases/${CASE_ID}.json" 2>/dev/null || true)" + cp "${HEAL_ROOT}/run/cases/${CASE_ID}.json" "${HEAL_ROOT}/results/${CASE_ID//\//_}.result.json" 2>/dev/null || true + fi + STATUS="FAIL" + if [ "${VERDICT}" = "PASS" ] && [ "${rc}" -eq 0 ]; then + STATUS="PASS" + elif [ "${VERDICT}" = "BLOCKED" ]; then + STATUS="BLOCKED" + elif [ "${rc}" -eq 124 ]; then + STATUS="TIMEOUT" + elif [ "${VERDICT}" = "INFRA_ERROR" ]; then + STATUS="FAIL" + fi + printf '%s\t%s\n' "${CASE_ID}" "${STATUS}" >> "${HEAL_ROOT}/results.txt" + echo "case ${CASE_ID}: ${STATUS} (rc=${rc} verdict=${VERDICT:-none})" + if [ -f "${HEAL_ROOT}/run/report.md" ]; then + cp "${HEAL_ROOT}/run/report.md" "${HEAL_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true + fi + echo "::endgroup::" + # Fleet hygiene between cases (ssh -n keeps the case list intact). + for node in ${RUSTFS_NODES}; do + ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" ' + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -f /etc/default/rustfs + ' || echo "inter-case cleanup failed on ${node} (continuing)" + done + done < "${HEAL_ROOT}/heal-cases.txt" + echo "heal sweep finished: $(wc -l < "${HEAL_ROOT}/results.txt") case(s) executed" + + - name: Generate heal sweep report + if: ${{ always() && steps.environment.outcome == 'success' }} + env: + HEAL_ROOT: ${{ runner.temp }}/validation-heal + PACKAGE_URL: ${{ steps.candidate.outputs.package_url }} + SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }} + AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }} + TOOL_RUN_ID: ${{ steps.tools.outputs.run_id }} + run: | + set -euo pipefail + python3 - <<'PY' + import json, os + from pathlib import Path + root = Path(os.environ["HEAL_ROOT"]) + catalog = json.loads(Path("validation-cases.json").read_text()) + module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]} + rows = [] + for line in (root / "results.txt").read_text().splitlines(): + if line: + case_id, status = line.split("\t") + rows.append((case_id, module_of.get(case_id, "heal"), status)) + passed = sum(1 for _, _, s in rows if s == "PASS") + failed = sum(1 for _, _, s in rows if s == "FAIL") + timed_out = sum(1 for _, _, s in rows if s == "TIMEOUT") + blocked = (root / "blocked-cases.txt").read_text().split() + pinned = (root / "pinned-cases.txt").read_text().split() + run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/" + f"{os.environ.get('GITHUB_REPOSITORY')}/actions/runs/{os.environ.get('GITHUB_RUN_ID')}") + report = [ + "# RustFS validation heal sweep report", "", + f"- Run: {run_url}", + f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}", + f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}", + f"- Package: {os.environ.get('PACKAGE_URL', '')}", + f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}", + f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", + f"- validation-tools run: {os.environ.get('TOOL_RUN_ID', '')}", "", + ] + by_module = {} + order = [] + for case_id, module, status in rows: + by_module.setdefault(module, []).append((case_id, status)) + if module not in order: + order.append(module) + report += ["## Case Summary", ""] + for module in order: + cases = by_module[module] + mp = sum(1 for _, s in cases if s == "PASS") + report += [f"### {module} ({len(cases)})", "", "| Case | Status |", "| --- | --- |"] + report += [f"| {c} | {s} |" for c, s in cases] + report += ["", f"- Subtotal: {mp} passed, {len(cases) - mp} not passed", ""] + report += [ + "## Environment-gated (BLOCKED)", "", + "These cases require operator-assisted resources (forced-restart hosts,", + "dedicated multi-set topologies) and never run unattended:", "", + ] + report += [f"- {c}: BLOCKED" for c in blocked] + report += [ + "", "## Candidate-pinned races/diagnostics (BLOCKED)", "", + "These executors hard-require the reviewed reference candidate", + "(native contract allowlists); they run only on interactive", "validation dispatches:", "", + ] + report += [f"- {c}: BLOCKED" for c in pinned] + report += [ + "", "## Totals", "", + f"- Total: {len(rows)}", + f"- PASS: {passed}", + f"- FAIL: {failed}", + f"- TIMEOUT: {timed_out}", + f"- BLOCKED: {len(blocked) + len(pinned) + sum(1 for _, _, s in rows if s == 'BLOCKED')}", "", + "Failing cases keep this workflow red only as a report; the sweep artifact", + "retains every case log and per-case report for diagnosis.", "", + ] + (root / "report.md").write_text("\n".join(report)) + (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n") + PY + cat "${HEAL_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload functional report to dashboard + if: ${{ always() && steps.environment.outcome == 'success' }} + timeout-minutes: 2 + continue-on-error: true + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + REPORT_FILE: ${{ runner.temp }}/validation-heal/report.md + SUITE: validation-heal + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" + exit 0 + fi + DATE="$(date -u +%Y-%m-%d)" + REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" + SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${SHA}" ]; then + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + else + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + fi + echo "dashboard updated: ${REPORT_PATH}" + + - name: Retain heal sweep evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rustfs-validation-heal-sweep-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/validation-heal/report.md + ${{ runner.temp }}/validation-heal/results.txt + ${{ runner.temp }}/validation-heal/heal-cases.txt + ${{ runner.temp }}/validation-heal/blocked-cases.txt + ${{ runner.temp }}/validation-heal/candidate.json + ${{ runner.temp }}/validation-heal/results/ + if-no-files-found: warn + retention-days: 30 + + - name: Cleanup environment (after) + id: environment_cleanup + timeout-minutes: 5 + if: ${{ always() && steps.environment.outcome == 'success' }} + run: | + set -uo pipefail + cleanup_status=0 + for node in ${RUSTFS_NODES}; do + ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -f /etc/default/rustfs + ' || cleanup_status=$? + done + exit "${cleanup_status}" + + - name: Release environment only after successful cleanup + if: ${{ always() && steps.environment.outcome == 'success' }} + timeout-minutes: 5 + env: + FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-heal.json + CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }} + run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME" + + - name: Notify on failure + if: failure() + run: echo "RustFS validation heal sweep failed (harness/environment breakdown); see the sweep artifact for case logs." + + - name: Enforce heal sweep result + if: ${{ always() && steps.environment.outcome == 'success' }} + env: + HEAL_ROOT: ${{ runner.temp }}/validation-heal + run: | + set -euo pipefail + if grep -qE $'\t(FAIL|TIMEOUT)$' "${HEAL_ROOT}/results.txt"; then + echo "validation heal sweep has failing or timed-out cases:" + grep -E $'\t(FAIL|TIMEOUT)$' "${HEAL_ROOT}/results.txt" + exit 1 + fi