From 345e69ad3602f991f25197774c547dd098331a90 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 17:43:01 +0300
Subject: [PATCH 01/12] feat(ci): report clusteralerts fired during release
 rollover

The release e2e pipeline had no visibility into alerts of the module
itself: nothing in the repo ever looked at ClusterAlert objects, and the
rules in monitoring/prometheus-rules were only covered by promtool unit
tests.

A dedicated job now watches the nested cluster for the whole test and
upgrade sequence and collects every firing D8Virtualization* alert. A
ClusterAlert object exists only while the alert fires, so the watch polls
instead of taking a single snapshot at the end. It stops on a ConfigMap
marker placed in the nested cluster by a separate always-running job:
runners share no filesystem, and the marker must also appear when
test-new-release never started.

The report job renders a table into the job summary, emits a warning
annotation per alert and exits non-zero, which paints it red in the UI.
Being continue-on-error, it leaves the workflow conclusion successful.
Alerts are attributed to pre-upgrade, upgrade or post-upgrade by
comparing the observation time with the upgrade window, so an alert that
was already firing before the rollover is not mistaken for its result.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/report-clusteralerts.sh  | 111 ++++++++++++++++
 .../e2e/signal-clusteralerts-watch-stop.sh    |  39 ++++++
 .../scripts/bash/e2e/watch-clusteralerts.sh   |  91 +++++++++++++
 .../e2e-test-releases-reusable-pipeline.yml   | 122 +++++++++++++++++-
 4 files changed, 358 insertions(+), 5 deletions(-)
 create mode 100644 .github/scripts/bash/e2e/report-clusteralerts.sh
 create mode 100644 .github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
 create mode 100644 .github/scripts/bash/e2e/watch-clusteralerts.sh

diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
new file mode 100644
index 0000000000..e0b9d2ac27
--- /dev/null
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+require_env CLUSTERALERTS_DIR
+
+alerts_dir="${CLUSTERALERTS_DIR:-}"
+alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
+fail_on_alerts="${FAIL_ON_ALERTS:-true}"
+summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
+
+# The watch runs as a single job and does not know which pipeline phase it is
+# observing, so the phase is derived here from the upgrade timestamps. A zero
+# means the corresponding job never reported one.
+started="${RELEASE_UPGRADE_STARTED_AT:-0}"
+finished="${RELEASE_UPGRADE_FINISHED_AT:-0}"
+[[ "${started}" =~ ^[0-9]+$ ]] || started=0
+[[ "${finished}" =~ ^[0-9]+$ ]] || finished=0
+
+# shellcheck disable=SC2016 # $started and $finished are jq variables, passed in via --argjson
+phase_program='
+def phase_of(upgrade_started; upgrade_finished):
+  if upgrade_started == 0 or .observedAt < upgrade_started
+  then { phase: "pre-upgrade", order: 0 }
+  elif upgrade_finished == 0 or .observedAt < upgrade_finished
+  then { phase: "upgrade", order: 1 }
+  else { phase: "post-upgrade", order: 2 }
+  end;
+map(. + phase_of($started; $finished))
+| unique_by([.phase, .name, .id])
+| sort_by([.order, .name])
+'
+
+shopt -s nullglob
+logs=("${alerts_dir}"/*.jsonl)
+shopt -u nullglob
+
+if [ "${#logs[@]}" -eq 0 ]; then
+  echo "[WARN] No ClusterAlerts logs found in ${alerts_dir}"
+  alerts='[]'
+else
+  echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}"
+  echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
+  alerts="$(jq -s \
+    --argjson started "${started}" \
+    --argjson finished "${finished}" \
+    "${phase_program}" "${logs[@]}")"
+fi
+
+count="$(jq 'length' <<< "${alerts}")"
+
+# Alert summaries are markdown ending with a newline; flatten them for the
+# table and for annotations.
+oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");'
+
+{
+  echo "## ClusterAlerts in the nested cluster"
+  echo
+} >> "${summary_file}"
+
+if [ "${count}" -eq 0 ]; then
+  echo "No \`${alert_prefix}*\` alerts were firing during the release rollover." >> "${summary_file}"
+  echo "[INFO] No ${alert_prefix}* alerts were firing during the release rollover"
+  exit 0
+fi
+
+{
+  echo "| Phase | Alert | Severity | First seen | Summary |"
+  echo "|---|---|---|---|---|"
+  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
+  echo
+  echo "<details><summary>Alert details</summary>"
+  echo
+  jq -r '.[] | "#### \(.name) — \(.phase)\n\n- severity level: \(.severityLevel)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
+  echo "</details>"
+} >> "${summary_file}"
+
+# Annotations put the alerts on top of the run page, not only in the summary.
+jq -r "${oneline}"' .[] | "::warning title=ClusterAlert \(.name)::[\(.phase)] \(.summary | oneline)"' <<< "${alerts}"
+
+echo "[INFO] Firing alerts:"
+jq -r '.[] | "  [\(.phase)] \(.name) (severity \(.severityLevel))"' <<< "${alerts}"
+
+if [ "${fail_on_alerts}" != "true" ]; then
+  echo "[INFO] FAIL_ON_ALERTS is not 'true', not failing the job"
+  exit 0
+fi
+
+# Failing here is what paints this job red; the job itself is
+# continue-on-error, so the workflow conclusion stays successful.
+echo "[ERROR] ${count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
+trap - ERR
+exit 1
diff --git a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
new file mode 100644
index 0000000000..05029e67a9
--- /dev/null
+++ b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Tells the ClusterAlerts watch that the pipeline is over. The marker is a
+# ConfigMap in the nested cluster because the watch runs on its own runner and
+# runners share no filesystem.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
+stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+
+echo "[INFO] Creating stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
+
+# Never fail the pipeline over the marker: if it cannot be created, the watch
+# ends on its own timeout instead.
+if kubectl -n "${stop_namespace}" create configmap "${stop_configmap}" \
+  --from-literal=run_id="${GITHUB_RUN_ID:-unknown}"; then
+  echo "[INFO] Stop marker created"
+else
+  echo "[WARN] Failed to create the stop marker, the watch will end on its own timeout"
+fi
diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
new file mode 100644
index 0000000000..50cfc86b89
--- /dev/null
+++ b/.github/scripts/bash/e2e/watch-clusteralerts.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Collects firing ClusterAlerts of the virtualization module from the cluster
+# the current kubeconfig points at, until the stop marker appears in that same
+# cluster (see signal-clusteralerts-watch-stop.sh) or the timeout is reached.
+#
+# The marker lives in the cluster rather than on disk on purpose: the watch runs
+# on its own runner, and runners share no filesystem.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+require_env CLUSTERALERTS_LOG
+
+alerts_log="${CLUSTERALERTS_LOG:-}"
+alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
+poll_interval="${POLL_INTERVAL:-15}"
+# Keep this below the job timeout, otherwise the job is cancelled before the
+# collected alerts can be uploaded.
+timeout_seconds="${TIMEOUT_SECONDS:-17400}"
+stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
+stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+
+mkdir -p "$(dirname -- "${alerts_log}")"
+: > "${alerts_log}"
+
+deadline=$(( $(date +%s) + timeout_seconds ))
+
+# A marker left over from a previous run against the same cluster (a re-run of
+# failed jobs, for example) would stop the watch immediately.
+kubectl -n "${stop_namespace}" delete configmap "${stop_configmap}" --ignore-not-found
+
+echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
+echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
+echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
+
+# A ClusterAlert object exists only while the alert is firing, so the log
+# accumulates one line per poll per firing alert. Deduplication and the split
+# into pipeline phases happen in report-clusteralerts.sh.
+while [ "$(date +%s)" -lt "${deadline}" ]; do
+  if kubectl -n "${stop_namespace}" get configmap "${stop_configmap}" >/dev/null 2>&1; then
+    echo "[INFO] Stop marker found, ending the watch"
+    exit 0
+  fi
+
+  # The nested API server can blink while the module is being rolled over,
+  # so a failed poll must never end the watch.
+  if ! snapshot="$(kubectl get clusteralerts -o json 2>&1)"; then
+    echo "[WARN] Failed to read ClusterAlerts, retrying in ${poll_interval}s: ${snapshot}"
+    sleep "${poll_interval}"
+    continue
+  fi
+
+  printf '%s' "${snapshot}" | jq -c \
+    --arg prefix "${alert_prefix}" \
+    --argjson observedAt "$(date +%s)" \
+    '.items[]
+     | select((.alert.name // "") | startswith($prefix))
+     | {
+         observedAt: $observedAt,
+         name: .alert.name,
+         severityLevel: (.alert.severityLevel // ""),
+         summary: (.alert.summary // ""),
+         description: (.alert.description // ""),
+         labels: (.alert.labels // {}),
+         id: .metadata.name,
+         firstSeen: (.status.startsAt // .metadata.creationTimestamp // "")
+       }' >> "${alerts_log}" \
+    || echo "[WARN] Failed to parse the ClusterAlerts snapshot, skipping this poll"
+
+  sleep "${poll_interval}"
+done
+
+echo "[WARN] Watch timeout of ${timeout_seconds}s reached before the stop marker appeared"
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 649b84c161..ce314c0066 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -145,6 +145,7 @@ env:
   K8S_VERSION: ${{ inputs.cluster_config_k8s_version }}
   STORAGE_TYPE: ${{ inputs.storage_type }}
   E2E_START_TIME: ${{ inputs.date_start }}
+  CLUSTERALERTS_DIR: ${{ github.workspace }}/clusteralerts
 
 defaults:
   run:
@@ -536,6 +537,7 @@ jobs:
         with:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
+
       - name: Authenticate go module fetches to the fox kubevirt mirror
         env:
           FOX_TOKEN: ${{ secrets.FOX_TOKEN }}
@@ -600,6 +602,7 @@ jobs:
       - test-current-release
     outputs:
       upgrade_started_at: ${{ steps.patch-modulepulloverride.outputs.upgrade_started_at }}
+      upgrade_finished_at: ${{ steps.upgrade-finished.outputs.upgrade_finished_at }}
     steps:
       - uses: actions/checkout@v6
 
@@ -614,6 +617,7 @@ jobs:
         with:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
+
       - name: Show current MPO state
         run: |
           echo "[INFO] Current ModulePullOverride before patching:"
@@ -655,6 +659,13 @@ jobs:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
         run: bash "${E2E_SCRIPT_DIR}/wait-vmops-migration-terminal.sh"
 
+      # Closes the upgrade window for the ClusterAlerts report; runs even on
+      # failure so alerts are still attributed to the right phase.
+      - name: Mark the end of the upgrade window
+        id: upgrade-finished
+        if: always()
+        run: echo "upgrade_finished_at=$(date +%s)" >> "$GITHUB_OUTPUT"
+
   test-new-release:
     name: "E2E test (new-release: ${{ inputs.new_release }})"
     runs-on: ubuntu-latest
@@ -671,6 +682,12 @@ jobs:
           checkout: "false"
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Setup kubeconfig
+        uses: ./.github/actions/use-nested-kubeconfig
+        with:
+          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
+          check-api: "false"
+
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -700,11 +717,6 @@ jobs:
           echo "Download dependencies"
           go mod download
 
-      - name: Setup kubeconfig
-        uses: ./.github/actions/use-nested-kubeconfig
-        with:
-          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
-          check-api: "false"
       - name: "Run E2E tests on new-release"
         env:
           NEW_RELEASE: ${{ env.NEW_RELEASE }}
@@ -715,3 +727,103 @@ jobs:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
           RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
         run: bash "${E2E_SCRIPT_DIR}/run-release-e2e.sh"
+
+  # Runs alongside the whole test and upgrade sequence: starts together with
+  # test-current-release and ends when signal-watch-stop places the marker in
+  # the nested cluster.
+  watch-clusteralerts:
+    name: Watch ClusterAlerts in nested cluster
+    runs-on: ubuntu-latest
+    needs:
+      - bootstrap
+      - configure-virtualization
+    # Safety net only: the watch itself stops earlier, on its own timeout, so
+    # that the collected alerts are still uploaded.
+    timeout-minutes: 300
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup E2E toolchain
+        uses: ./.github/actions/setup-e2e-toolchain
+        with:
+          checkout: "false"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup kubeconfig
+        uses: ./.github/actions/use-nested-kubeconfig
+        with:
+          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
+          check-api: "false"
+
+      - name: Watch ClusterAlerts
+        env:
+          CLUSTERALERTS_LOG: ${{ env.CLUSTERALERTS_DIR }}/clusteralerts.jsonl
+        run: bash "${E2E_SCRIPT_DIR}/watch-clusteralerts.sh"
+
+      - name: Upload collected ClusterAlerts
+        uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: clusteralerts-${{ github.run_id }}
+          path: ${{ env.CLUSTERALERTS_DIR }}/*.jsonl
+          if-no-files-found: ignore
+          retention-days: 3
+
+  # Separate job rather than a final step of test-new-release: it must also run
+  # when that job never started because an earlier one failed, otherwise the
+  # watch would keep polling until its timeout.
+  signal-watch-stop:
+    name: Stop the ClusterAlerts watch
+    runs-on: ubuntu-latest
+    needs:
+      - bootstrap
+      - test-new-release
+    # Without a bootstrapped cluster there is no watch to stop and no
+    # kubeconfig to reach it with.
+    if: always() && needs.bootstrap.result == 'success'
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup E2E toolchain
+        uses: ./.github/actions/setup-e2e-toolchain
+        with:
+          checkout: "false"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup kubeconfig
+        uses: ./.github/actions/use-nested-kubeconfig
+        with:
+          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
+          check-api: "false"
+
+      - name: Signal the ClusterAlerts watch to stop
+        run: bash "${E2E_SCRIPT_DIR}/signal-clusteralerts-watch-stop.sh"
+
+  report-clusteralerts:
+    name: ClusterAlerts in nested cluster
+    runs-on: ubuntu-latest
+    needs:
+      - patch-modulepulloverride
+      - watch-clusteralerts
+    if: always()
+    # A firing alert must be visible but must not fail the pipeline: this job
+    # exits non-zero (turning red in the UI) while the workflow conclusion
+    # stays successful.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Download collected ClusterAlerts
+        uses: actions/download-artifact@v8
+        # Nothing to download when the pipeline failed before the watch ran;
+        # that must not be mistaken for a firing alert.
+        continue-on-error: true
+        with:
+          name: clusteralerts-${{ github.run_id }}
+          path: ${{ env.CLUSTERALERTS_DIR }}
+
+      - name: Report ClusterAlerts
+        env:
+          RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
+          RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
+        run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"

From 3a59b3a8aa60229313028d73877682033d19146a Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 18:40:49 +0300
Subject: [PATCH 02/12] fix(ci): wait for the whole rollover before reporting
 clusteralerts

The report job only depended on the upgrade job and on the watch, so in
the run graph it sat next to the new-release tests instead of after them.
Worse, if the watch job died early - on the kubeconfig step, say - the
report started while the new-release tests were still running and
declared that no alerts had been firing.

It now depends on the test jobs as well, and an empty result is reported
as "not monitored" instead of "nothing was firing" when the watch job did
not succeed.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/report-clusteralerts.sh          | 8 ++++++++
 .github/workflows/e2e-test-releases-reusable-pipeline.yml | 6 ++++++
 2 files changed, 14 insertions(+)

diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
index e0b9d2ac27..412ae8759e 100644
--- a/.github/scripts/bash/e2e/report-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -25,6 +25,7 @@ require_env CLUSTERALERTS_DIR
 alerts_dir="${CLUSTERALERTS_DIR:-}"
 alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
 fail_on_alerts="${FAIL_ON_ALERTS:-true}"
+watch_result="${WATCH_RESULT:-}"
 summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
 
 # The watch runs as a single job and does not know which pipeline phase it is
@@ -77,6 +78,13 @@ oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");'
 } >> "${summary_file}"
 
 if [ "${count}" -eq 0 ]; then
+  # An empty report means "nothing was firing" only if the watch actually ran.
+  if [ -n "${watch_result}" ] && [ "${watch_result}" != "success" ]; then
+    echo "The watch job did not complete (result: \`${watch_result}\`), so alerts were **not** monitored." >> "${summary_file}"
+    echo "::warning title=ClusterAlerts were not monitored::The watch job result is '${watch_result}'"
+    exit 0
+  fi
+
   echo "No \`${alert_prefix}*\` alerts were firing during the release rollover." >> "${summary_file}"
   echo "[INFO] No ${alert_prefix}* alerts were firing during the release rollover"
   exit 0
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index ce314c0066..2d88a44cf6 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -802,8 +802,13 @@ jobs:
   report-clusteralerts:
     name: ClusterAlerts in nested cluster
     runs-on: ubuntu-latest
+    # Depends on the test jobs as well, not only on the watch: should the watch
+    # job die early, the report must still wait for the whole rollover instead
+    # of declaring "no alerts" while the tests are still running.
     needs:
+      - test-current-release
       - patch-modulepulloverride
+      - test-new-release
       - watch-clusteralerts
     if: always()
     # A firing alert must be visible but must not fail the pipeline: this job
@@ -826,4 +831,5 @@ jobs:
         env:
           RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
           RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
+          WATCH_RESULT: ${{ needs.watch-clusteralerts.result }}
         run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"

From 3eeef5f5b885ee8ee056fe209ee9cc54bd2dc55a Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 19:22:09 +0300
Subject: [PATCH 03/12] chore(ci): temporary tweaks for testing the
 clusteralerts report

Pin the release-upgrade defaults to the versions under test and delete the
nested cluster right after the ClusterAlerts report, so a test run does not
hold the cluster until the nightly cleanup.

Drop this commit before merge.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../e2e-test-releases-reusable-pipeline.yml   | 26 +++++++++++++++++++
 .github/workflows/e2e-test-releases.yml       |  4 +--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 2d88a44cf6..c5ccf60c65 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -833,3 +833,29 @@ jobs:
           RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
           WATCH_RESULT: ${{ needs.watch-clusteralerts.result }}
         run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"
+
+  # TEMPORARY: drop before merge. Frees the nested cluster right after the
+  # ClusterAlerts report instead of waiting for the nightly cleanup.
+  delete-nested-cluster:
+    name: Delete nested cluster (temporary)
+    runs-on: ubuntu-latest
+    # report-clusteralerts already waits for the whole test and upgrade
+    # sequence, so this runs once nothing needs the cluster any more.
+    needs:
+      - bootstrap
+      - report-clusteralerts
+    if: always()
+    steps:
+      - name: Configure kubectl via azure/k8s-set-context@v4
+        uses: azure/k8s-set-context@v4
+        with:
+          method: kubeconfig
+          context: e2e-cluster-nightly-e2e-virt-sa
+          kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }}
+
+      - name: Delete the namespace and the cluster-scoped VirtualMachineClass
+        env:
+          NAMESPACE: ${{ needs.bootstrap.outputs.namespace }}
+        run: |
+          kubectl delete namespace "${NAMESPACE}" --timeout=300s || true
+          kubectl delete vmclass "${NAMESPACE}-cpu" --timeout=300s || true
diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index b209a88abd..dc0cdfae03 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -20,12 +20,12 @@ on:
       current-release:
         description: "Current release tag like v1.4.1, or PR reference like pr2034/2034"
         required: false # before merge to main, set to true
-        default: "v1.6.3-rc.3" # before merge to main, remove
+        default: "v1.9.6" # before merge to main, remove
         type: string
       next-release:
         description: "Next release like v1.5.0, or PR reference like pr2034/2034"
         required: false # before merge to main, set to true
-        default: "v1.7.0" # before merge to main, remove
+        default: "v1.9.7-rc.0" # before merge to main, remove
         type: string
       enableBuild:
         description: "Build release images before E2E tests"

From f3ab7eb46c4300adc816d5a2d6589520552646db Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Mon, 10 Aug 2026 23:30:06 +0300
Subject: [PATCH 04/12] fix(ci): pick virtualization feature gates by release
 version

The release e2e config hardcoded a feature gate that the 1.9 release line
never shipped, so ModulePullOverride validation failed and the module was
never installed: the pipeline only reported that it timed out waiting for
virtualization to become ready.

Gates are now derived from the release under test. The upgrade narrows the
list to what both releases support before switching the image tag, then
enables everything the new release supports once its images are running, so
each release is tested with all the gates it has.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh            | 22 +++++++
 .../e2e/configure-virtualization-release.sh   | 11 +++-
 .../e2e/patch-virtualization-feature-gates.sh | 58 +++++++++++++++++++
 .../e2e-test-releases-reusable-pipeline.yml   | 12 ++++
 4 files changed, 100 insertions(+), 3 deletions(-)
 create mode 100644 .github/scripts/bash/e2e/patch-virtualization-feature-gates.sh

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 245eb8c877..e9189e974e 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -47,6 +47,28 @@ modules_repo_for_registry() {
   fi
 }
 
+# Echoes the virtualization feature gates supported by every given release, one
+# per line. A gate the pulled module does not know fails ModulePullOverride
+# validation and leaves the module uninstalled, so a gate is only listed when
+# none of the releases predate it. In-place resize never shipped in the 1.9
+# line. Anything that is not a release tag (a PR reference, a build off main)
+# carries the current gates.
+# Usage: virtualization_feature_gates [release]...
+virtualization_feature_gates() {
+  local release
+
+  echo "HotplugCPUWithLiveMigration"
+  echo "HotplugMemoryWithLiveMigration"
+
+  for release in "$@"; do
+    if [[ "${release}" =~ ^v([0-9]+)\.([0-9]+)\. ]] && (( BASH_REMATCH[1] == 1 && BASH_REMATCH[2] < 10 )); then
+      return 0
+    fi
+  done
+
+  echo "HotplugCPUAndMemoryWithInPlaceResize"
+}
+
 # Reads a manifest from stdin and applies it with retries.
 # Usage: kubectl_apply_with_retry [count] [delay] [diag_fn]
 # diag_fn is an optional function name invoked on each failed attempt.
diff --git a/.github/scripts/bash/e2e/configure-virtualization-release.sh b/.github/scripts/bash/e2e/configure-virtualization-release.sh
index b39ff8a7ad..d8fc6d86da 100644
--- a/.github/scripts/bash/e2e/configure-virtualization-release.sh
+++ b/.github/scripts/bash/e2e/configure-virtualization-release.sh
@@ -37,6 +37,13 @@ current_release="$(required_env_value CURRENT_RELEASE)"
 
 REGISTRY="$(registry_host_from_docker_cfg "${dev_registry_docker_cfg}")"
 
+# Only the gates this release knows: a gate it does not support fails
+# ModulePullOverride validation and the module never installs. The upgrade
+# revisits the list for the new release (patch-virtualization-feature-gates.sh).
+feature_gates_yaml="$(virtualization_feature_gates "${current_release}" | sed 's/^/      - /')"
+echo "[INFO] Feature gates for ${current_release}:"
+echo "${feature_gates_yaml}"
+
 echo "[INFO] Apply ModuleSource prod config"
 kubectl_apply_with_retry 20 10 show_deckhouse_state <<EOF
 apiVersion: deckhouse.io/v1alpha1
@@ -71,9 +78,7 @@ spec:
     virtualMachineCIDRs:
       - 192.168.10.0/24
     featureGates:
-      - HotplugCPUWithLiveMigration
-      - HotplugMemoryWithLiveMigration
-      - HotplugCPUAndMemoryWithInPlaceResize
+${feature_gates_yaml}
   source: deckhouse-dev
   version: 1
 ---
diff --git a/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
new file mode 100644
index 0000000000..bb675a88a2
--- /dev/null
+++ b/.github/scripts/bash/e2e/patch-virtualization-feature-gates.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sets the virtualization feature gates of the module config to those supported
+# by every given release, and verifies the result.
+#
+# Usage: patch-virtualization-feature-gates.sh <release>...
+#
+# During a release upgrade this runs twice. Before the image tag is patched it
+# is called with both releases, which drops the gates the new release does not
+# know - otherwise the new module fails validation and never installs. After the
+# upgrade it is called with the new release alone, which enables the gates only
+# that release supports.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+if [ "$#" -eq 0 ]; then
+  echo "[ERROR] Usage: $(basename -- "${BASH_SOURCE[0]}") <release>..." >&2
+  exit 1
+fi
+
+gates_json="$(virtualization_feature_gates "$@" | jq -Rsc 'split("\n") | map(select(length > 0))')"
+current_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')"
+
+echo "[INFO] Feature gates supported by $*: ${gates_json}"
+
+if [ "${current_json}" = "${gates_json}" ]; then
+  echo "[INFO] Module config already lists exactly these gates, nothing to patch"
+  exit 0
+fi
+
+echo "[INFO] Patching feature gates: ${current_json:-none} -> ${gates_json}"
+kubectl patch mc virtualization --type merge -p "{\"spec\":{\"settings\":{\"featureGates\":${gates_json}}}}"
+
+patched_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')"
+if [ "${patched_json}" != "${gates_json}" ]; then
+  echo "[ERROR] Feature gates were not applied: expected ${gates_json}, got ${patched_json:-none}" >&2
+  exit 1
+fi
+
+echo "[INFO] Feature gates in effect: ${patched_json}"
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index c5ccf60c65..559d34e53e 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -623,6 +623,13 @@ jobs:
           echo "[INFO] Current ModulePullOverride before patching:"
           kubectl get mpo virtualization -o yaml
 
+      # Gates the new release does not know would fail its validation, so they
+      # go before the image tag is switched, while the old module is still live.
+      - name: Drop feature gates the new release does not support
+        run: |
+          bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" \
+            "${CURRENT_RELEASE}" "${NEW_RELEASE}"
+
       - name: "Patch ModulePullOverride to new-release: ${{ env.NEW_RELEASE }}"
         id: patch-modulepulloverride
         run: |
@@ -647,6 +654,11 @@ jobs:
           NEW_RELEASE: ${{ env.NEW_RELEASE }}
         run: bash "${E2E_SCRIPT_DIR}/verify-image-digests.sh"
 
+      # Now that the new images are running, the gates only that release knows
+      # can be enabled, so the new release is tested with all of them on.
+      - name: Enable every feature gate the new release supports
+        run: bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" "${NEW_RELEASE}"
+
       - name: Show ModulePullOverride state after upgrade
         run: |
           echo "[INFO] ModulePullOverride after upgrade:"

From da7b5a4bae041469954901ebee7c82425bd84aa8 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 01:08:57 +0300
Subject: [PATCH 05/12] fix(ci): skip the migration wait when the upgrade
 cannot migrate vms

The release pipeline waited for one Evict operation per running virtual
machine after every upgrade, and failed after 20 minutes when none appeared.
Two releases that ship the same virt-handler and virt-launcher never move a
virtual machine, so the wait could not be satisfied and reported a timeout
instead of an upgrade that simply had nothing to migrate.

The expectation is now derived from the workload image digests of both
releases: unchanged images end the step with a message, and anything that
cannot be determined keeps the previous waiting behaviour.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh            |  9 +++++
 .../scripts/bash/e2e/verify-image-digests.sh  |  3 +-
 .../bash/e2e/wait-vmops-migration-terminal.sh | 37 +++++++++++++++++++
 .../e2e-test-releases-reusable-pipeline.yml   |  1 +
 4 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index e9189e974e..92fc259176 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -69,6 +69,15 @@ virtualization_feature_gates() {
   echo "HotplugCPUAndMemoryWithInPlaceResize"
 }
 
+# Echoes images_digests.json packaged in the module image of a given release.
+# Usage: module_images_digests <module_source> <release>
+module_images_digests() {
+  local module_source="$1"
+  local release="$2"
+
+  crane export "${module_source}/virtualization:${release}" - | tar -Oxf - images_digests.json
+}
+
 # Reads a manifest from stdin and applies it with retries.
 # Usage: kubectl_apply_with_retry [count] [delay] [diag_fn]
 # diag_fn is an optional function name invoked on each failed attempt.
diff --git a/.github/scripts/bash/e2e/verify-image-digests.sh b/.github/scripts/bash/e2e/verify-image-digests.sh
index d161eebfe9..73bd57f07e 100644
--- a/.github/scripts/bash/e2e/verify-image-digests.sh
+++ b/.github/scripts/bash/e2e/verify-image-digests.sh
@@ -33,9 +33,8 @@ required_env_value() {
 new_release="$(required_env_value NEW_RELEASE)"
 dev_module_source="$(required_env_value DEV_MODULE_SOURCE)"
 
-MODULE_IMAGE="${dev_module_source}/virtualization:${new_release}"
 echo "[INFO] Extracting images_digests.json from virtualization:${new_release}"
-images_hash="$(crane export "${MODULE_IMAGE}" - | tar -Oxf - images_digests.json)"
+images_hash="$(module_images_digests "${dev_module_source}" "${new_release}")"
 echo "[INFO] Expected image digests:"
 echo "::group::images_digests.json"
 echo "${images_hash}" | jq .
diff --git a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
index 6eda51d2fc..6eb19395c1 100644
--- a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
+++ b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
@@ -33,6 +33,43 @@ release_namespace="$(required_env_value RELEASE_NAMESPACE)"
 
 sleep_interval="${SLEEP_INTERVAL:-10}"
 timeout_seconds="${TIMEOUT_SECONDS:-1200}"
+# Virtual machines are only moved when the workload images change: a new
+# virt-handler drains the VMs off its node, a new virt-launcher makes the
+# workload updater migrate the running ones. Releases that leave both untouched
+# never trigger a migration, so there would be nothing to wait for.
+migration_expected() {
+  local module_source="${DEV_MODULE_SOURCE:-}"
+  local current="${CURRENT_RELEASE:-}"
+  local new="${NEW_RELEASE:-}"
+  local current_digests new_digests image
+
+  if [ -z "${module_source}" ] || [ -z "${current}" ] || [ -z "${new}" ]; then
+    echo "[WARN] DEV_MODULE_SOURCE, CURRENT_RELEASE or NEW_RELEASE is not set, cannot tell whether the upgrade migrates VMs; waiting anyway"
+    return 0
+  fi
+
+  if ! current_digests="$(module_images_digests "${module_source}" "${current}")" ||
+     ! new_digests="$(module_images_digests "${module_source}" "${new}")"; then
+    echo "[WARN] Failed to read the image digests of ${current} or ${new}, cannot tell whether the upgrade migrates VMs; waiting anyway"
+    return 0
+  fi
+
+  for image in virtHandler virtLauncher; do
+    if [ "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${current_digests}")" \
+      != "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${new_digests}")" ]; then
+      echo "[INFO] The ${image} image differs between ${current} and ${new}, virtual machines will be migrated"
+      return 0
+    fi
+  done
+
+  return 1
+}
+
+if ! migration_expected; then
+  echo "[INFO] ${CURRENT_RELEASE} and ${NEW_RELEASE} ship the same virt-handler and virt-launcher: the upgrade does not migrate virtual machines, nothing to wait for"
+  exit 0
+fi
+
 deadline=$(( $(date +%s) + timeout_seconds ))
 
 # The number of Running VMs at the start is the number of Evict VMOPs we
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 559d34e53e..dc7277d2c3 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -669,6 +669,7 @@ jobs:
       - name: Wait for all migrations to reach a terminal phase
         env:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
+          DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
         run: bash "${E2E_SCRIPT_DIR}/wait-vmops-migration-terminal.sh"
 
       # Closes the upgrade window for the ClusterAlerts report; runs even on

From 6fb8388580bda613f8cd4fea3e0c19599537d5b2 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 01:15:10 +0300
Subject: [PATCH 06/12] chore(ci): bump actions off the deprecated node 20
 runtime

Runners force these actions onto Node.js 24 already and annotate every run
with a deprecation warning. Moved each one to its first major built for
Node.js 24: cache v4 to v5, setup-go v5 to v6, github-script v7 to v8,
setup-kubectl and k8s-set-context v4 to v5.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/actions/setup-e2e-toolchain/action.yml       |  4 ++--
 .github/workflows/dev_module_build.yml               | 12 ++++++------
 .github/workflows/dev_validation.yaml                |  8 ++++----
 .github/workflows/e2e-nightly-reusable-pipeline.yml  | 12 ++++++------
 .github/workflows/e2e-nightly.yml                    | 10 +++++-----
 .../e2e-test-releases-reusable-pipeline.yml          | 12 ++++++------
 .github/workflows/e2e-test-releases.yml              |  2 +-
 .../workflows/release_module_release-channels.yml    |  4 ++--
 8 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/.github/actions/setup-e2e-toolchain/action.yml b/.github/actions/setup-e2e-toolchain/action.yml
index 1fb7ac9b91..2a38a65580 100644
--- a/.github/actions/setup-e2e-toolchain/action.yml
+++ b/.github/actions/setup-e2e-toolchain/action.yml
@@ -40,7 +40,7 @@ runs:
 
     - name: Restore d8 cache
       id: d8-cache
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: /opt/deckhouse/bin/d8
         key: d8-${{ inputs.d8-version }}-${{ runner.os }}
@@ -57,7 +57,7 @@ runs:
 
     - name: Install kubectl CLI
       if: inputs.install-kubectl == 'true'
-      uses: azure/setup-kubectl@v4
+      uses: azure/setup-kubectl@v5
 
     - name: Install htpasswd utility
       if: inputs.install-htpasswd == 'true'
diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 12f9bef117..cf570f2629 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -63,7 +63,7 @@ jobs:
     steps:
       - name: Get Pull Request Labels
         id: get-labels
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             function processDelveLabels(labelList) {
@@ -212,7 +212,7 @@ jobs:
     name: Run go linter
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -314,7 +314,7 @@ jobs:
     name: Run unit test
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -464,7 +464,7 @@ jobs:
     needs: set_vars
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -644,7 +644,7 @@ jobs:
     steps:
       - name: Create Initial PR Comment
         id: create_comment
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
           script: |
@@ -696,7 +696,7 @@ jobs:
         uses: ./.github/actions/install-d8
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
diff --git a/.github/workflows/dev_validation.yaml b/.github/workflows/dev_validation.yaml
index 9376e293d3..070482c97b 100644
--- a/.github/workflows/dev_validation.yaml
+++ b/.github/workflows/dev_validation.yaml
@@ -58,7 +58,7 @@ jobs:
     name: Validation no-cyrillic
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -83,7 +83,7 @@ jobs:
     name: Validation doc-changes
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -108,7 +108,7 @@ jobs:
     name: Validation go-work
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -220,7 +220,7 @@ jobs:
     steps:
       - name: Setup Go ${{ matrix.components.go-version }}
         if: matrix.components.component != 'vm-route-forge' || needs.paths_filter.outputs.vm_route_forge == 'true'
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ matrix.components.go-version }}
 
diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml
index 3e44dac1a6..b00559753d 100644
--- a/.github/workflows/e2e-nightly-reusable-pipeline.yml
+++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml
@@ -237,8 +237,8 @@ jobs:
         with:
           docker_cfg: ${{ secrets.REGISTRY_DOCKER_CFG }}
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -584,7 +584,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -696,7 +696,7 @@ jobs:
 
       - name: Determine failed stage and prepare report
         id: determine-stage
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         env:
           STORAGE_TYPE: ${{ inputs.storage_type }}
           PIPELINE_JOB_NAME: ${{ inputs.pipeline_job_name }}
@@ -763,8 +763,8 @@ jobs:
             "$artifact_path"
           unzip -o "${RUNNER_TEMP}/${ARTIFACT_NAME}.zip" -d "${{ env.SETUP_CLUSTER_TYPE_PATH }}"
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml
index 8c1a4626b8..65e31c5dc7 100644
--- a/.github/workflows/e2e-nightly.yml
+++ b/.github/workflows/e2e-nightly.yml
@@ -36,8 +36,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v6
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -60,8 +60,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v6
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -240,7 +240,7 @@ jobs:
 
       - name: Send results to channel
         id: render-report
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         env:
           EXPECTED_STORAGE_TYPES: '["replicated","nfs","ceph"]'
           LOOP_API_BASE_URL: ${{ secrets.LOOP_API_BASE_URL }}
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index dc7277d2c3..5f8fc41cbc 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -191,8 +191,8 @@ jobs:
         with:
           docker_cfg: ${{ secrets.REGISTRY_DOCKER_CFG }}
 
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
@@ -522,7 +522,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -702,7 +702,7 @@ jobs:
           check-api: "false"
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -859,8 +859,8 @@ jobs:
       - report-clusteralerts
     if: always()
     steps:
-      - name: Configure kubectl via azure/k8s-set-context@v4
-        uses: azure/k8s-set-context@v4
+      - name: Configure kubectl via azure/k8s-set-context@v5
+        uses: azure/k8s-set-context@v5
         with:
           method: kubeconfig
           context: e2e-cluster-nightly-e2e-virt-sa
diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index dc0cdfae03..10e23de868 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -76,7 +76,7 @@ jobs:
     steps:
       - name: Resolve release refs
         id: resolve
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         env:
           CURRENT_RELEASE_INPUT: ${{ github.event.inputs.current-release }}
           NEXT_RELEASE_INPUT: ${{ github.event.inputs.next-release }}
diff --git a/.github/workflows/release_module_release-channels.yml b/.github/workflows/release_module_release-channels.yml
index a14434a935..d2fe7063bd 100644
--- a/.github/workflows/release_module_release-channels.yml
+++ b/.github/workflows/release_module_release-channels.yml
@@ -142,7 +142,7 @@ jobs:
 
       - name: Set up Go ${{ env.GO_VERSION }}
         if: ${{ !inputs.skip_requirements_check }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 
@@ -449,7 +449,7 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "${{ env.GO_VERSION }}"
 

From 158cca9ed8bd0a0e3e60ac984d1d5b5414e552d6 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 11:23:59 +0300
Subject: [PATCH 07/12] fix(ci): run the e2e pipelines on the project go
 version

setup-go v6 pins GOTOOLCHAIN to the requested version instead of letting Go
fetch a newer one, and the pipelines asked for a Go older than the go.work of
the sources they test, so every Go build failed. Both the release and the
nightly pipelines now ask for the version the project builds with, and keep
the toolchain automatic for refs that need another one.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/workflows/e2e-nightly-reusable-pipeline.yml       | 2 ++
 .github/workflows/e2e-nightly.yml                         | 4 ++--
 .github/workflows/e2e-test-releases-reusable-pipeline.yml | 2 ++
 .github/workflows/e2e-test-releases.yml                   | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml
index b00559753d..8c40f4d24d 100644
--- a/.github/workflows/e2e-nightly-reusable-pipeline.yml
+++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml
@@ -187,6 +187,8 @@ env:
   DECKHOUSE_VERSION: ${{ inputs.deckhouse_version }}
   DEFAULT_USER: ${{ inputs.default_user }}
   GO_VERSION: ${{ inputs.go_version }}
+  # setup-go v6 would pin the toolchain; a checked out ref may need a newer one.
+  GOTOOLCHAIN: auto
   SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster
   E2E_SCRIPT_DIR: ${{ github.workspace }}/.github/scripts/bash/e2e
   K8S_VERSION: ${{ inputs.cluster_config_k8s_version }}
diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml
index 65e31c5dc7..1547a7e0db 100644
--- a/.github/workflows/e2e-nightly.yml
+++ b/.github/workflows/e2e-nightly.yml
@@ -146,7 +146,7 @@ jobs:
       deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }}
       registry_profile: ${{ needs.set-vars.outputs.registry_profile }}
       default_user: cloud
-      go_version: "1.24.13"
+      go_version: "1.25.12"
       e2e_timeout: "3.5h"
       e2e_image_base_url: ${{ needs.set-vars.outputs.e2e_image_base_url }}
       date_start: ${{ needs.set-vars.outputs.date_start }}
@@ -180,7 +180,7 @@ jobs:
       deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }}
       registry_profile: ${{ needs.set-vars.outputs.registry_profile }}
       default_user: cloud
-      go_version: "1.24.13"
+      go_version: "1.25.12"
       e2e_timeout: "3.5h"
       e2e_focus_tests: "VirtualDiskProvisioning|VirtualDiskSnapshots|VirtualImageCreation|VirtualDiskResizing|DiskAttachment|BlockDeviceHotplug|Migration|StorageClassMigration|RWOVirtualDiskMigration|VMSOP|Restore|DataExports"
       e2e_image_base_url: ${{ needs.set-vars.outputs.e2e_image_base_url }}
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index 5f8fc41cbc..eb8b102aed 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -140,6 +140,8 @@ env:
   DECKHOUSE_VERSION: ${{ inputs.deckhouse_version }}
   DEFAULT_USER: ${{ inputs.default_user }}
   GO_VERSION: ${{ inputs.go_version }}
+  # setup-go v6 would pin the toolchain; a checked out ref may need a newer one.
+  GOTOOLCHAIN: auto
   SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster
   E2E_SCRIPT_DIR: ${{ github.workspace }}/.github/scripts/bash/e2e
   K8S_VERSION: ${{ inputs.cluster_config_k8s_version }}
diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index 10e23de868..3018b68250 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -229,7 +229,7 @@ jobs:
       deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }}
       registry_profile: ${{ needs.set-vars.outputs.registry_profile }}
       default_user: cloud
-      go_version: "1.25.8"
+      go_version: "1.25.12"
       date_start: ${{ needs.set-vars.outputs.date_start }}
       randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }}
       cluster_config_workers_memory: "9Gi"

From d36db1e0a21f102cd9312941215ae54f6777c36e Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 12:48:08 +0300
Subject: [PATCH 08/12] delete before merge, ci concurrency - group

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/workflows/e2e-test-releases.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml
index 3018b68250..5555e68de0 100644
--- a/.github/workflows/e2e-test-releases.yml
+++ b/.github/workflows/e2e-test-releases.yml
@@ -34,7 +34,7 @@ on:
         type: boolean
 
 concurrency:
-  group: "${{ github.workflow }}-${{ github.event.inputs.current-release }}-${{ github.event.inputs.next-release || 'no-next' }}"
+  group: "${{ github.workflow }}- ${{ github.ref }}"
   cancel-in-progress: true
 
 defaults:

From 5fe8d2ed9d1da004ff07ff3820995360416dd081 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 15:35:14 +0300
Subject: [PATCH 09/12] fix(ci): skip the migration window check when the
 upgrade cannot migrate vms

The release smoke test compares the iperf window against the migration of
the iperf server, and failed when no migration operation existed at all.
Releases that ship the same virt-handler and virt-launcher never move a
virtual machine, so the pipeline now passes its own verdict to the tests and
the comparison is skipped when there is nothing to compare against. An unset
value still expects a migration, so a missing one keeps failing the test.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../bash/e2e/wait-vmops-migration-terminal.sh    | 10 ++++++++++
 .../e2e-test-releases-reusable-pipeline.yml      |  3 +++
 test/e2e/release/current_release_smoke.go        | 16 ++++++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
index 6eb19395c1..da14b7520e 100644
--- a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
+++ b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh
@@ -65,11 +65,21 @@ migration_expected() {
   return 1
 }
 
+# The verdict is published so the new-release tests can tell a missing migration
+# from one that was never going to happen.
+publish_verdict() {
+  [ -n "${GITHUB_OUTPUT:-}" ] || return 0
+  echo "migrates_vms=$1" >> "${GITHUB_OUTPUT}"
+}
+
 if ! migration_expected; then
+  publish_verdict false
   echo "[INFO] ${CURRENT_RELEASE} and ${NEW_RELEASE} ship the same virt-handler and virt-launcher: the upgrade does not migrate virtual machines, nothing to wait for"
   exit 0
 fi
 
+publish_verdict true
+
 deadline=$(( $(date +%s) + timeout_seconds ))
 
 # The number of Running VMs at the start is the number of Evict VMOPs we
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index eb8b102aed..daeb1c00d8 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -605,6 +605,7 @@ jobs:
     outputs:
       upgrade_started_at: ${{ steps.patch-modulepulloverride.outputs.upgrade_started_at }}
       upgrade_finished_at: ${{ steps.upgrade-finished.outputs.upgrade_finished_at }}
+      migrates_vms: ${{ steps.wait-migrations.outputs.migrates_vms }}
     steps:
       - uses: actions/checkout@v6
 
@@ -669,6 +670,7 @@ jobs:
           kubectl get modules virtualization
 
       - name: Wait for all migrations to reach a terminal phase
+        id: wait-migrations
         env:
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
           DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }}
@@ -741,6 +743,7 @@ jobs:
           RELEASE_TEST_PHASE: post-upgrade
           RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }}
           RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
+          RELEASE_UPGRADE_MIGRATES_VMS: ${{ needs.patch-modulepulloverride.outputs.migrates_vms }}
         run: bash "${E2E_SCRIPT_DIR}/run-release-e2e.sh"
 
   # Runs alongside the whole test and upgrade sequence: starts together with
diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go
index 9e6516829a..fb23f86aa6 100644
--- a/test/e2e/release/current_release_smoke.go
+++ b/test/e2e/release/current_release_smoke.go
@@ -42,6 +42,7 @@ const (
 	releaseTestPhasePostUpgrade  = "post-upgrade"
 	releaseUpgradeContextPathEnv = "RELEASE_UPGRADE_CONTEXT_PATH"
 	releaseNamespaceEnv          = "RELEASE_NAMESPACE"
+	releaseUpgradeMigratesVMsEnv = "RELEASE_UPGRADE_MIGRATES_VMS"
 )
 
 var _ = Describe("CurrentReleaseSmoke", func() {
@@ -205,6 +206,14 @@ func (t *currentReleaseSmokeTest) verifyIPerfContinuityAfterUpgrade() {
 	report := getIPerfClientReport(t.framework, t.iperfClient.vm, releaseIPerfReportPath)
 	Expect(isExpectedIPerfReportError(report.Error)).To(BeTrue(), "iperf3 report contains an unexpected error: %q", report.Error)
 
+	if !upgradeMigratesVMs() {
+		By("Skipping the migration window checks: the upgrade does not migrate virtual machines")
+		Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data")
+		Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput")
+
+		return
+	}
+
 	By("Verifying the iperf test brackets the migration window (started before, stopped after)")
 	migration := getMigrationWindow(t.framework, t.iperfServer.vm.Name, t.iperfServer.vm.Namespace)
 
@@ -278,6 +287,13 @@ func getReleaseTestPhase() string {
 	return releaseTestPhasePreUpgrade
 }
 
+// Upgrades between releases that ship the same virt-handler and virt-launcher
+// never move a virtual machine. An unset value means the pipeline could not tell,
+// so a migration is still expected.
+func upgradeMigratesVMs() bool {
+	return os.Getenv(releaseUpgradeMigratesVMsEnv) != "false"
+}
+
 func mustGetEnv(name string) string {
 	value := os.Getenv(name)
 	Expect(value).NotTo(BeEmpty(), "environment variable %s must be set", name)

From a96cd45c093907e89ac4385d341c6838ecc233d8 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 15:35:21 +0300
Subject: [PATCH 10/12] fix(ci): scope the clusteralerts watch stop marker to
 the run attempt

The watch deleted the stop marker on start to ignore leftovers from earlier
runs, which could delete the marker another job had just created for it: the
watch then kept polling until its own timeout hours later, with the report
job waiting on it. The marker name now carries the run and attempt, so
foreign markers are invisible and none has to be deleted.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .github/scripts/bash/e2e/common.sh                        | 8 ++++++++
 .../scripts/bash/e2e/signal-clusteralerts-watch-stop.sh   | 2 +-
 .github/scripts/bash/e2e/watch-clusteralerts.sh           | 6 +-----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index 92fc259176..b3a0eee73a 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -69,6 +69,14 @@ virtualization_feature_gates() {
   echo "HotplugCPUAndMemoryWithInPlaceResize"
 }
 
+# Echoes the name of the ConfigMap that stops the ClusterAlerts watch. The name
+# carries the run attempt so a marker left by another run, or by a previous
+# attempt of this one, can never stop this watch - and this watch never has to
+# delete a marker that another job may have just created for it.
+watch_stop_configmap_name() {
+  printf 'e2e-clusteralerts-watch-stop-%s-%s' "${GITHUB_RUN_ID:-local}" "${GITHUB_RUN_ATTEMPT:-1}"
+}
+
 # Echoes images_digests.json packaged in the module image of a given release.
 # Usage: module_images_digests <module_source> <release>
 module_images_digests() {
diff --git a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
index 05029e67a9..e5a672e492 100644
--- a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
+++ b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
@@ -25,7 +25,7 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPT_DIR}/common.sh"
 
 stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+stop_configmap="$(watch_stop_configmap_name)"
 
 echo "[INFO] Creating stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
 
diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
index 50cfc86b89..9558bc698b 100644
--- a/.github/scripts/bash/e2e/watch-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/watch-clusteralerts.sh
@@ -36,17 +36,13 @@ poll_interval="${POLL_INTERVAL:-15}"
 # collected alerts can be uploaded.
 timeout_seconds="${TIMEOUT_SECONDS:-17400}"
 stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="${WATCH_STOP_CONFIGMAP:-e2e-clusteralerts-watch-stop}"
+stop_configmap="$(watch_stop_configmap_name)"
 
 mkdir -p "$(dirname -- "${alerts_log}")"
 : > "${alerts_log}"
 
 deadline=$(( $(date +%s) + timeout_seconds ))
 
-# A marker left over from a previous run against the same cluster (a re-run of
-# failed jobs, for example) would stop the watch immediately.
-kubectl -n "${stop_namespace}" delete configmap "${stop_configmap}" --ignore-not-found
-
 echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
 echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
 echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"

From d9f15eef375e673dce0a86c17ad0add4c10ad85a Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 15:48:18 +0300
Subject: [PATCH 11/12] feat(ci): archive the object of every collected
 clusteralert

The report shows a fixed set of fields, and the ClusterAlert itself disappears
from the cluster as soon as the alert stops firing - together with the cluster
the pipeline deletes. Every matching alert is now also archived as the whole
object in YAML, as first seen, next to the collected log.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/watch-clusteralerts.sh   | 32 ++++++++++++++++++-
 .../e2e-test-releases-reusable-pipeline.yml   |  2 +-
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
index 9558bc698b..7e67824b3e 100644
--- a/.github/scripts/bash/e2e/watch-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/watch-clusteralerts.sh
@@ -38,7 +38,9 @@ timeout_seconds="${TIMEOUT_SECONDS:-17400}"
 stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
 stop_configmap="$(watch_stop_configmap_name)"
 
-mkdir -p "$(dirname -- "${alerts_log}")"
+alerts_dir="$(dirname -- "${alerts_log}")"
+
+mkdir -p "${alerts_dir}"
 : > "${alerts_log}"
 
 deadline=$(( $(date +%s) + timeout_seconds ))
@@ -47,6 +49,32 @@ echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
 echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
 echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
 
+# Keeps the whole object of every matching alert, as first seen, next to the log:
+# the ClusterAlert is gone from the cluster once the alert stops firing, and the
+# cluster itself does not outlive the pipeline.
+dump_alert_objects() {
+  local snapshot="$1"
+  local id name dump
+
+  while IFS=$'\t' read -r id name; do
+    [ -n "${id}" ] || continue
+    dump="${alerts_dir}/${name:-clusteralert}-${id}.yaml"
+    [ -e "${dump}" ] && continue
+
+    if ! printf '%s' "${snapshot}" \
+      | jq --arg id "${id}" '.items[] | select(.metadata.name == $id)' \
+      | yq -p=json -o=yaml > "${dump}"; then
+      echo "[WARN] Failed to dump the object of ClusterAlert ${id}"
+      rm -f "${dump}"
+    fi
+  done < <(printf '%s' "${snapshot}" | jq -r \
+    --arg prefix "${alert_prefix}" \
+    '.items[]
+     | select((.alert.name // "") | startswith($prefix))
+     | [.metadata.name, (.alert.name // "")]
+     | @tsv')
+}
+
 # A ClusterAlert object exists only while the alert is firing, so the log
 # accumulates one line per poll per firing alert. Deduplication and the split
 # into pipeline phases happen in report-clusteralerts.sh.
@@ -81,6 +109,8 @@ while [ "$(date +%s)" -lt "${deadline}" ]; do
        }' >> "${alerts_log}" \
     || echo "[WARN] Failed to parse the ClusterAlerts snapshot, skipping this poll"
 
+  dump_alert_objects "${snapshot}"
+
   sleep "${poll_interval}"
 done
 
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index daeb1c00d8..d89843a15f 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -783,7 +783,7 @@ jobs:
         if: always()
         with:
           name: clusteralerts-${{ github.run_id }}
-          path: ${{ env.CLUSTERALERTS_DIR }}/*.jsonl
+          path: ${{ env.CLUSTERALERTS_DIR }}
           if-no-files-found: ignore
           retention-days: 3
 

From ec5eaf989ac7752dca7fb866be74507e63efe916 Mon Sep 17 00:00:00 2001
From: Nikita Korolev <nikita.korolev@flant.com>
Date: Tue, 11 Aug 2026 17:38:15 +0300
Subject: [PATCH 12/12] refactor(ci): collect clusteralerts from prometheus in
 a single step

Watching the ClusterAlerts objects took a job that lived as long as the whole
pipeline, a stop marker in the nested cluster to end it, and an artifact round
trip to hand the collected log to the report - because a ClusterAlert exists
only while its alert is active.

The ALERTS series keep that history, so one range query at the end of the
pipeline replaces all of it. The phase now comes from the intersection of the
firing interval with the upgrade window instead of the moment a poll happened
to catch the alert, and pending alerts are reported as well: with a 'for'
clause they are what a short rollover produces. Only a fired alert paints the
job red, or components restarting would make that the norm.

The rendered summary and description are kept: their templates come from the
alerting rules of the same Prometheus and the labels of the series render
them.

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>
---
 .../scripts/bash/e2e/collect-clusteralerts.sh | 237 ++++++++++++++++++
 .github/scripts/bash/e2e/common.sh            |   8 -
 .../scripts/bash/e2e/report-clusteralerts.sh  | 112 +++++----
 .../e2e/signal-clusteralerts-watch-stop.sh    |  39 ---
 .../scripts/bash/e2e/watch-clusteralerts.sh   | 117 ---------
 .../e2e-test-releases-reusable-pipeline.yml   | 111 +++-----
 6 files changed, 330 insertions(+), 294 deletions(-)
 create mode 100644 .github/scripts/bash/e2e/collect-clusteralerts.sh
 delete mode 100644 .github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
 delete mode 100644 .github/scripts/bash/e2e/watch-clusteralerts.sh

diff --git a/.github/scripts/bash/e2e/collect-clusteralerts.sh b/.github/scripts/bash/e2e/collect-clusteralerts.sh
new file mode 100644
index 0000000000..7ede3ce77e
--- /dev/null
+++ b/.github/scripts/bash/e2e/collect-clusteralerts.sh
@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Flant JSC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Collects the alerts of the virtualization module that were active in the
+# nested cluster during the release rollover. Runs once, at the end of the
+# pipeline, against the nested kubeconfig.
+#
+# Prometheus is the source rather than the ClusterAlerts objects: a ClusterAlert
+# exists only while its alert is active, so a single late look at the API server
+# would see nothing of what happened during the upgrade, while the ALERTS series
+# keep the whole history of the observation window.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/bash/e2e/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+require_env CLUSTERALERTS_DIR
+require_env CLUSTERALERTS_WINDOW_STARTED_AT
+
+alerts_dir="${CLUSTERALERTS_DIR:-}"
+alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
+prometheus_namespace="${PROMETHEUS_NAMESPACE:-d8-monitoring}"
+prometheus_selector="${PROMETHEUS_SELECTOR:-prometheus=main}"
+# Port of the prometheus container inside the pod, discovered below when not
+# pinned. port-forward joins the pod network namespace, so a listener bound to
+# localhost there is reachable too.
+prometheus_port="${PROMETHEUS_PORT:-}"
+local_port="${PROMETHEUS_LOCAL_PORT:-19090}"
+query_step="${QUERY_STEP:-30}"
+ready_attempts="${READY_ATTEMPTS:-30}"
+ready_delay="${READY_DELAY:-2}"
+
+# The window starts when virtualization was configured, not when the pipeline
+# started: before that the module does not exist, and its alerts cannot either.
+window_start="${CLUSTERALERTS_WINDOW_STARTED_AT:-}"
+window_end="$(date +%s)"
+
+# Same semantics as in report-clusteralerts.sh: a zero means the corresponding
+# job never reported a timestamp.
+started="${RELEASE_UPGRADE_STARTED_AT:-0}"
+finished="${RELEASE_UPGRADE_FINISHED_AT:-0}"
+[[ "${started}" =~ ^[0-9]+$ ]] || started=0
+[[ "${finished}" =~ ^[0-9]+$ ]] || finished=0
+
+if [[ ! "${window_start}" =~ ^[0-9]+$ ]] || [ "${window_start}" -eq 0 ]; then
+  echo "[ERROR] CLUSTERALERTS_WINDOW_STARTED_AT must be a unix timestamp, got: '${window_start}'" >&2
+  exit 1
+fi
+
+range_file="${alerts_dir}/clusteralerts-query-range.json"
+rules_file="${alerts_dir}/clusteralerts-rules.json"
+alerts_log="${alerts_dir}/clusteralerts.jsonl"
+port_forward_log="${alerts_dir}/clusteralerts-port-forward.log"
+
+port_forward_pid=""
+
+cleanup() {
+  [ -n "${port_forward_pid}" ] || return 0
+  kill "${port_forward_pid}" 2>/dev/null || true
+  wait "${port_forward_pid}" 2>/dev/null || true
+  port_forward_pid=""
+}
+
+prom_api() {
+  local path="$1"
+  shift
+  curl -sS -f --max-time 120 -G "http://127.0.0.1:${local_port}${path}" "$@"
+}
+
+# port-forward plus curl on the runner, not kubectl exec plus curl in the
+# container: the Prometheus image ships no shell tools to query itself with.
+start_port_forward() {
+  local pod="$1" attempt
+
+  echo "[INFO] Forwarding 127.0.0.1:${local_port} to ${pod}:${prometheus_port} in ${prometheus_namespace}"
+  kubectl -n "${prometheus_namespace}" port-forward \
+    "pod/${pod}" "${local_port}:${prometheus_port}" > "${port_forward_log}" 2>&1 &
+  port_forward_pid=$!
+  trap cleanup EXIT
+
+  # The tunnel needs a moment, and probing the query API instead of /-/ready
+  # checks exactly what the collection is about to use.
+  for ((attempt = 1; attempt <= ready_attempts; attempt++)); do
+    if prom_api /api/v1/query --data-urlencode 'query=1' > /dev/null 2>&1; then
+      echo "[INFO] Prometheus API is reachable after ${attempt} attempt(s)"
+      return 0
+    fi
+
+    if ! kill -0 "${port_forward_pid}" 2>/dev/null; then
+      echo "[ERROR] kubectl port-forward exited early:" >&2
+      cat "${port_forward_log}" >&2 || true
+      return 1
+    fi
+
+    sleep "${ready_delay}"
+  done
+
+  echo "[ERROR] Prometheus API did not become reachable after ${ready_attempts} attempt(s):" >&2
+  cat "${port_forward_log}" >&2 || true
+  return 1
+}
+
+# Annotation templates live in the rules, not in the series, so they are fetched
+# separately and rendered per series below.
+fetch_rules() {
+  if prom_api /api/v1/rules --data-urlencode 'type=alert' > "${rules_file}" &&
+    [ "$(jq -r '.status // ""' "${rules_file}")" = "success" ]; then
+    return 0
+  fi
+
+  # A missing summary must never sink the report: the alert itself is the news.
+  echo "[WARN] Failed to fetch alerting rules, alerts will be reported without a summary"
+  printf '%s\n' '{"status":"error","data":{"groups":[]}}' > "${rules_file}"
+}
+
+templates_program='
+[ .data.groups[]?.rules[]?
+  | select(.type == "alerting")
+  | { key: .name,
+      value: {
+        summary: (.annotations.summary // ""),
+        description: (.annotations.description // "")
+      }
+    }
+] | from_entries
+'
+
+# shellcheck disable=SC2016 # $started, $finished and $templates are jq variables, passed in via --argjson
+records_program='
+def phase_of($t):
+  if $started == 0 or $t < $started then { phase: "pre-upgrade", order: 0 }
+  elif $finished == 0 or $t < $finished then { phase: "upgrade", order: 1 }
+  else { phase: "post-upgrade", order: 2 }
+  end;
+
+# The annotations are Go templates that reference $labels only (verified over
+# monitoring/prometheus-rules), so replacing every label by its value renders
+# them. A reference to a label the series does not carry is left as it is.
+def render($labels):
+  reduce ($labels | to_entries[]) as $l
+    (.; gsub("\\{\\{\\s*\\$labels\\." + $l.key + "\\s*\\}\\}"; $l.value));
+
+[ .data.result[]
+  | . as $series
+  | ($series.metric.alertname // "") as $name
+  | ($series.metric | del(.__name__, .alertname, .alertstate)) as $labels
+  # One record per phase the series touches: an alert that spans the upgrade is
+  # news in every phase it was active in, and grouping the samples by phase
+  # intersects its active interval with the upgrade window.
+  | [ $series.values[] | .[0] | floor | { t: ., ph: phase_of(.) } ]
+  | group_by(.ph.order)
+  | .[]
+  | { phase: .[0].ph.phase,
+      order: .[0].ph.order,
+      name: $name,
+      alertstate: ($series.metric.alertstate // ""),
+      severityLevel: ($series.metric.severity_level // ""),
+      labels: $labels,
+      firstSeen: (map(.t) | min | todate),
+      lastSeen: (map(.t) | max | todate),
+      summary: (($templates[$name].summary // "") | render($labels)),
+      description: (($templates[$name].description // "") | render($labels)),
+      id: ([$name, ($series.metric.alertstate // ""), ($labels | tojson)] | join("|"))
+    }
+]
+| unique_by([.order, .id])
+| sort_by([.order, .name, .alertstate])
+| .[]
+'
+
+mkdir -p "${alerts_dir}"
+
+echo "[INFO] Collecting alerts matching '${alert_prefix}*' from Prometheus in ${prometheus_namespace}"
+echo "[INFO] Observation window: ${window_start}..${window_end} ($(( window_end - window_start ))s), step ${query_step}s"
+echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
+
+pod="$(kubectl -n "${prometheus_namespace}" get pod \
+  -l "${prometheus_selector}" \
+  --field-selector=status.phase=Running \
+  -o jsonpath='{.items[0].metadata.name}')"
+
+if [ -z "${pod}" ]; then
+  echo "[ERROR] No Running pod matching '${prometheus_selector}' in namespace ${prometheus_namespace}" >&2
+  exit 1
+fi
+
+# Asking the pod which port carries the API beats assuming one: an authenticating
+# sidecar may well be the container that owns 9090 there.
+if [ -z "${prometheus_port}" ]; then
+  prometheus_port="$(kubectl -n "${prometheus_namespace}" get pod "${pod}" \
+    -o jsonpath='{.spec.containers[?(@.name=="prometheus")].ports[?(@.name=="web")].containerPort}' || true)"
+  prometheus_port="${prometheus_port:-9090}"
+fi
+
+start_port_forward "${pod}"
+
+# query_range and not query: an instant query would only see what is still
+# active now, while the report is about what was active during the rollover.
+prom_api /api/v1/query_range \
+  --data-urlencode "query=ALERTS{alertname=~\"${alert_prefix}.*\"}" \
+  --data-urlencode "start=${window_start}" \
+  --data-urlencode "end=${window_end}" \
+  --data-urlencode "step=${query_step}" > "${range_file}"
+
+if [ "$(jq -r '.status // ""' "${range_file}")" != "success" ]; then
+  echo "[ERROR] Prometheus rejected the range query: $(jq -c '.' "${range_file}")" >&2
+  exit 1
+fi
+
+fetch_rules
+
+templates="$(jq "${templates_program}" "${rules_file}")"
+
+jq -c \
+  --argjson started "${started}" \
+  --argjson finished "${finished}" \
+  --argjson templates "${templates}" \
+  "${records_program}" "${range_file}" > "${alerts_log}"
+
+count="$(grep -c . "${alerts_log}" || true)"
+echo "[INFO] Collected ${count} alert record(s) into ${alerts_log}"
+jq -r '"  [\(.phase)] \(.name) \(.alertstate) (\(.firstSeen) .. \(.lastSeen))"' "${alerts_log}"
diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh
index b3a0eee73a..92fc259176 100644
--- a/.github/scripts/bash/e2e/common.sh
+++ b/.github/scripts/bash/e2e/common.sh
@@ -69,14 +69,6 @@ virtualization_feature_gates() {
   echo "HotplugCPUAndMemoryWithInPlaceResize"
 }
 
-# Echoes the name of the ConfigMap that stops the ClusterAlerts watch. The name
-# carries the run attempt so a marker left by another run, or by a previous
-# attempt of this one, can never stop this watch - and this watch never has to
-# delete a marker that another job may have just created for it.
-watch_stop_configmap_name() {
-  printf 'e2e-clusteralerts-watch-stop-%s-%s' "${GITHUB_RUN_ID:-local}" "${GITHUB_RUN_ATTEMPT:-1}"
-}
-
 # Echoes images_digests.json packaged in the module image of a given release.
 # Usage: module_images_digests <module_source> <release>
 module_images_digests() {
diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh
index 412ae8759e..d202e3125b 100644
--- a/.github/scripts/bash/e2e/report-clusteralerts.sh
+++ b/.github/scripts/bash/e2e/report-clusteralerts.sh
@@ -25,87 +25,91 @@ require_env CLUSTERALERTS_DIR
 alerts_dir="${CLUSTERALERTS_DIR:-}"
 alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
 fail_on_alerts="${FAIL_ON_ALERTS:-true}"
-watch_result="${WATCH_RESULT:-}"
+collect_result="${COLLECT_RESULT:-}"
 summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
 
-# The watch runs as a single job and does not know which pipeline phase it is
-# observing, so the phase is derived here from the upgrade timestamps. A zero
-# means the corresponding job never reported one.
-started="${RELEASE_UPGRADE_STARTED_AT:-0}"
-finished="${RELEASE_UPGRADE_FINISHED_AT:-0}"
-[[ "${started}" =~ ^[0-9]+$ ]] || started=0
-[[ "${finished}" =~ ^[0-9]+$ ]] || finished=0
-
-# shellcheck disable=SC2016 # $started and $finished are jq variables, passed in via --argjson
-phase_program='
-def phase_of(upgrade_started; upgrade_finished):
-  if upgrade_started == 0 or .observedAt < upgrade_started
-  then { phase: "pre-upgrade", order: 0 }
-  elif upgrade_finished == 0 or .observedAt < upgrade_finished
-  then { phase: "upgrade", order: 1 }
-  else { phase: "post-upgrade", order: 2 }
-  end;
-map(. + phase_of($started; $finished))
-| unique_by([.phase, .name, .id])
-| sort_by([.order, .name])
-'
+{
+  echo "## ClusterAlerts in the nested cluster"
+  echo
+} >> "${summary_file}"
+
+# Renders the "we could not check" verdict: an empty report is only good news
+# when the collection itself succeeded.
+not_collected() {
+  local reason="$1"
+
+  echo "${reason}" >> "${summary_file}"
+  echo "::warning title=ClusterAlerts were not collected::${reason}"
+  exit 0
+}
 
 shopt -s nullglob
 logs=("${alerts_dir}"/*.jsonl)
 shopt -u nullglob
 
+if [ -n "${collect_result}" ] && [ "${collect_result}" != "success" ]; then
+  not_collected "The collection step did not complete (result: \`${collect_result}\`), so alerts were **not** checked."
+fi
+
+# collect-clusteralerts.sh always writes its log, empty or not, so a missing one
+# means the collection never got that far - a step that never ran reports no
+# result at all.
 if [ "${#logs[@]}" -eq 0 ]; then
-  echo "[WARN] No ClusterAlerts logs found in ${alerts_dir}"
-  alerts='[]'
-else
-  echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}"
-  echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}"
-  alerts="$(jq -s \
-    --argjson started "${started}" \
-    --argjson finished "${finished}" \
-    "${phase_program}" "${logs[@]}")"
+  echo "[WARN] No collected ClusterAlerts found in ${alerts_dir}"
+  not_collected "No collected alerts were found, so alerts were **not** checked."
 fi
 
+echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}"
+# The phase, the state and the firing interval come from the collector; sorting
+# is repeated here only to keep the order stable across several log files.
+alerts="$(jq -s 'sort_by([.order, .name, .alertstate])' "${logs[@]}")"
+
 count="$(jq 'length' <<< "${alerts}")"
 
 # Alert summaries are markdown ending with a newline; flatten them for the
 # table and for annotations.
 oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");'
 
-{
-  echo "## ClusterAlerts in the nested cluster"
-  echo
-} >> "${summary_file}"
-
 if [ "${count}" -eq 0 ]; then
-  # An empty report means "nothing was firing" only if the watch actually ran.
-  if [ -n "${watch_result}" ] && [ "${watch_result}" != "success" ]; then
-    echo "The watch job did not complete (result: \`${watch_result}\`), so alerts were **not** monitored." >> "${summary_file}"
-    echo "::warning title=ClusterAlerts were not monitored::The watch job result is '${watch_result}'"
-    exit 0
-  fi
-
-  echo "No \`${alert_prefix}*\` alerts were firing during the release rollover." >> "${summary_file}"
-  echo "[INFO] No ${alert_prefix}* alerts were firing during the release rollover"
+  echo "No \`${alert_prefix}*\` alerts were active during the release rollover." >> "${summary_file}"
+  echo "[INFO] No ${alert_prefix}* alerts were active during the release rollover"
   exit 0
 fi
 
 {
-  echo "| Phase | Alert | Severity | First seen | Summary |"
-  echo "|---|---|---|---|---|"
-  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
+  echo "| Phase | Alert | State | Severity | First seen | Summary |"
+  echo "|---|---|---|---|---|---|"
+  jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.alertstate) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}"
   echo
   echo "<details><summary>Alert details</summary>"
   echo
-  jq -r '.[] | "#### \(.name) — \(.phase)\n\n- severity level: \(.severityLevel)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
+  jq -r '.[] | "#### \(.name) — \(.phase) (\(.alertstate))\n\n- severity level: \(.severityLevel)\n- active: \(.firstSeen) .. \(.lastSeen)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}"
   echo "</details>"
 } >> "${summary_file}"
 
 # Annotations put the alerts on top of the run page, not only in the summary.
-jq -r "${oneline}"' .[] | "::warning title=ClusterAlert \(.name)::[\(.phase)] \(.summary | oneline)"' <<< "${alerts}"
-
-echo "[INFO] Firing alerts:"
-jq -r '.[] | "  [\(.phase)] \(.name) (severity \(.severityLevel))"' <<< "${alerts}"
+# A pending alert is a notice rather than a warning: it did not hold long enough
+# to be one.
+jq -r "${oneline}"' .[]
+  | (if .alertstate == "firing" then "::warning" else "::notice" end)
+    + " title=ClusterAlert \(.name)::[\(.phase)/\(.alertstate)] \(.summary | oneline)"' <<< "${alerts}"
+
+echo "[INFO] Active alerts:"
+jq -r '.[] | "  [\(.phase)] \(.name) \(.alertstate) (severity \(.severityLevel))"' <<< "${alerts}"
+
+firing_count="$(jq '[.[] | select(.alertstate == "firing")] | length' <<< "${alerts}")"
+
+# Only a fired alert paints the job red. Components restart during a rollover,
+# so rules with a `for` clause go pending on almost every run: failing on those
+# would make a red job the norm and tell the reviewer nothing.
+if [ "${firing_count}" -eq 0 ]; then
+  {
+    echo
+    echo "No \`${alert_prefix}*\` alert reached the firing state; ${count} were pending only."
+  } >> "${summary_file}"
+  echo "[INFO] No ${alert_prefix}* alert reached the firing state, ${count} were pending only"
+  exit 0
+fi
 
 if [ "${fail_on_alerts}" != "true" ]; then
   echo "[INFO] FAIL_ON_ALERTS is not 'true', not failing the job"
@@ -114,6 +118,6 @@ fi
 
 # Failing here is what paints this job red; the job itself is
 # continue-on-error, so the workflow conclusion stays successful.
-echo "[ERROR] ${count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
+echo "[ERROR] ${firing_count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2
 trap - ERR
 exit 1
diff --git a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh b/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
deleted file mode 100644
index e5a672e492..0000000000
--- a/.github/scripts/bash/e2e/signal-clusteralerts-watch-stop.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2026 Flant JSC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Tells the ClusterAlerts watch that the pipeline is over. The marker is a
-# ConfigMap in the nested cluster because the watch runs on its own runner and
-# runners share no filesystem.
-
-set -Eeuo pipefail
-
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=.github/scripts/bash/e2e/common.sh
-source "${SCRIPT_DIR}/common.sh"
-
-stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="$(watch_stop_configmap_name)"
-
-echo "[INFO] Creating stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
-
-# Never fail the pipeline over the marker: if it cannot be created, the watch
-# ends on its own timeout instead.
-if kubectl -n "${stop_namespace}" create configmap "${stop_configmap}" \
-  --from-literal=run_id="${GITHUB_RUN_ID:-unknown}"; then
-  echo "[INFO] Stop marker created"
-else
-  echo "[WARN] Failed to create the stop marker, the watch will end on its own timeout"
-fi
diff --git a/.github/scripts/bash/e2e/watch-clusteralerts.sh b/.github/scripts/bash/e2e/watch-clusteralerts.sh
deleted file mode 100644
index 7e67824b3e..0000000000
--- a/.github/scripts/bash/e2e/watch-clusteralerts.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2026 Flant JSC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Collects firing ClusterAlerts of the virtualization module from the cluster
-# the current kubeconfig points at, until the stop marker appears in that same
-# cluster (see signal-clusteralerts-watch-stop.sh) or the timeout is reached.
-#
-# The marker lives in the cluster rather than on disk on purpose: the watch runs
-# on its own runner, and runners share no filesystem.
-
-set -Eeuo pipefail
-
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=.github/scripts/bash/e2e/common.sh
-source "${SCRIPT_DIR}/common.sh"
-
-require_env CLUSTERALERTS_LOG
-
-alerts_log="${CLUSTERALERTS_LOG:-}"
-alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}"
-poll_interval="${POLL_INTERVAL:-15}"
-# Keep this below the job timeout, otherwise the job is cancelled before the
-# collected alerts can be uploaded.
-timeout_seconds="${TIMEOUT_SECONDS:-17400}"
-stop_namespace="${WATCH_STOP_NAMESPACE:-default}"
-stop_configmap="$(watch_stop_configmap_name)"
-
-alerts_dir="$(dirname -- "${alerts_log}")"
-
-mkdir -p "${alerts_dir}"
-: > "${alerts_log}"
-
-deadline=$(( $(date +%s) + timeout_seconds ))
-
-echo "[INFO] Watching ClusterAlerts matching '${alert_prefix}*'"
-echo "[INFO] Poll interval: ${poll_interval}s, watch timeout: ${timeout_seconds}s, log: ${alerts_log}"
-echo "[INFO] Stop marker: configmap ${stop_configmap} in namespace ${stop_namespace}"
-
-# Keeps the whole object of every matching alert, as first seen, next to the log:
-# the ClusterAlert is gone from the cluster once the alert stops firing, and the
-# cluster itself does not outlive the pipeline.
-dump_alert_objects() {
-  local snapshot="$1"
-  local id name dump
-
-  while IFS=$'\t' read -r id name; do
-    [ -n "${id}" ] || continue
-    dump="${alerts_dir}/${name:-clusteralert}-${id}.yaml"
-    [ -e "${dump}" ] && continue
-
-    if ! printf '%s' "${snapshot}" \
-      | jq --arg id "${id}" '.items[] | select(.metadata.name == $id)' \
-      | yq -p=json -o=yaml > "${dump}"; then
-      echo "[WARN] Failed to dump the object of ClusterAlert ${id}"
-      rm -f "${dump}"
-    fi
-  done < <(printf '%s' "${snapshot}" | jq -r \
-    --arg prefix "${alert_prefix}" \
-    '.items[]
-     | select((.alert.name // "") | startswith($prefix))
-     | [.metadata.name, (.alert.name // "")]
-     | @tsv')
-}
-
-# A ClusterAlert object exists only while the alert is firing, so the log
-# accumulates one line per poll per firing alert. Deduplication and the split
-# into pipeline phases happen in report-clusteralerts.sh.
-while [ "$(date +%s)" -lt "${deadline}" ]; do
-  if kubectl -n "${stop_namespace}" get configmap "${stop_configmap}" >/dev/null 2>&1; then
-    echo "[INFO] Stop marker found, ending the watch"
-    exit 0
-  fi
-
-  # The nested API server can blink while the module is being rolled over,
-  # so a failed poll must never end the watch.
-  if ! snapshot="$(kubectl get clusteralerts -o json 2>&1)"; then
-    echo "[WARN] Failed to read ClusterAlerts, retrying in ${poll_interval}s: ${snapshot}"
-    sleep "${poll_interval}"
-    continue
-  fi
-
-  printf '%s' "${snapshot}" | jq -c \
-    --arg prefix "${alert_prefix}" \
-    --argjson observedAt "$(date +%s)" \
-    '.items[]
-     | select((.alert.name // "") | startswith($prefix))
-     | {
-         observedAt: $observedAt,
-         name: .alert.name,
-         severityLevel: (.alert.severityLevel // ""),
-         summary: (.alert.summary // ""),
-         description: (.alert.description // ""),
-         labels: (.alert.labels // {}),
-         id: .metadata.name,
-         firstSeen: (.status.startsAt // .metadata.creationTimestamp // "")
-       }' >> "${alerts_log}" \
-    || echo "[WARN] Failed to parse the ClusterAlerts snapshot, skipping this poll"
-
-  dump_alert_objects "${snapshot}"
-
-  sleep "${poll_interval}"
-done
-
-echo "[WARN] Watch timeout of ${timeout_seconds}s reached before the stop marker appeared"
diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
index d89843a15f..a4adf9fecf 100644
--- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml
+++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml
@@ -478,6 +478,8 @@ jobs:
     needs:
       - bootstrap
       - configure-storage
+    outputs:
+      configured_at: ${{ steps.mark-configured.outputs.configured_at }}
     steps:
       - uses: actions/checkout@v6
 
@@ -512,6 +514,13 @@ jobs:
           echo "[INFO] Checking virt-handler pods "
           virt_handler_ready
 
+      # Opens the observation window of the ClusterAlerts report. Runs even on
+      # failure, so a module that never became ready is still observed.
+      - name: Mark the start of the ClusterAlerts observation window
+        id: mark-configured
+        if: always()
+        run: echo "configured_at=$(date +%s)" >> "$GITHUB_OUTPUT"
+
   test-current-release:
     name: "E2E test (current-release: ${{ inputs.current_release }})"
     runs-on: ubuntu-latest
@@ -746,18 +755,22 @@ jobs:
           RELEASE_UPGRADE_MIGRATES_VMS: ${{ needs.patch-modulepulloverride.outputs.migrates_vms }}
         run: bash "${E2E_SCRIPT_DIR}/run-release-e2e.sh"
 
-  # Runs alongside the whole test and upgrade sequence: starts together with
-  # test-current-release and ends when signal-watch-stop places the marker in
-  # the nested cluster.
-  watch-clusteralerts:
-    name: Watch ClusterAlerts in nested cluster
+  report-clusteralerts:
+    name: ClusterAlerts in nested cluster
     runs-on: ubuntu-latest
+    # Waits for the whole test and upgrade sequence: Prometheus is queried once,
+    # at the end, so everything that could fire must have happened by then.
     needs:
       - bootstrap
       - configure-virtualization
-    # Safety net only: the watch itself stops earlier, on its own timeout, so
-    # that the collected alerts are still uploaded.
-    timeout-minutes: 300
+      - test-current-release
+      - patch-modulepulloverride
+      - test-new-release
+    if: always()
+    # A firing alert must be visible but must not fail the pipeline: this job
+    # exits non-zero (turning red in the UI) while the workflow conclusion
+    # stays successful.
+    continue-on-error: true
     steps:
       - uses: actions/checkout@v6
 
@@ -773,10 +786,21 @@ jobs:
           kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
           check-api: "false"
 
-      - name: Watch ClusterAlerts
+      - name: Collect ClusterAlerts
+        id: collect
         env:
-          CLUSTERALERTS_LOG: ${{ env.CLUSTERALERTS_DIR }}/clusteralerts.jsonl
-        run: bash "${E2E_SCRIPT_DIR}/watch-clusteralerts.sh"
+          CLUSTERALERTS_WINDOW_STARTED_AT: ${{ needs.configure-virtualization.outputs.configured_at }}
+          RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
+          RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
+        run: bash "${E2E_SCRIPT_DIR}/collect-clusteralerts.sh"
+
+      - name: Report ClusterAlerts
+        # Runs even when the collection failed: the report is the only place
+        # that says so, and a silent job would read as "no alerts".
+        if: always()
+        env:
+          COLLECT_RESULT: ${{ steps.collect.outcome }}
+        run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"
 
       - name: Upload collected ClusterAlerts
         uses: actions/upload-artifact@v7
@@ -787,71 +811,6 @@ jobs:
           if-no-files-found: ignore
           retention-days: 3
 
-  # Separate job rather than a final step of test-new-release: it must also run
-  # when that job never started because an earlier one failed, otherwise the
-  # watch would keep polling until its timeout.
-  signal-watch-stop:
-    name: Stop the ClusterAlerts watch
-    runs-on: ubuntu-latest
-    needs:
-      - bootstrap
-      - test-new-release
-    # Without a bootstrapped cluster there is no watch to stop and no
-    # kubeconfig to reach it with.
-    if: always() && needs.bootstrap.result == 'success'
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Setup E2E toolchain
-        uses: ./.github/actions/setup-e2e-toolchain
-        with:
-          checkout: "false"
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Setup kubeconfig
-        uses: ./.github/actions/use-nested-kubeconfig
-        with:
-          kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }}
-          check-api: "false"
-
-      - name: Signal the ClusterAlerts watch to stop
-        run: bash "${E2E_SCRIPT_DIR}/signal-clusteralerts-watch-stop.sh"
-
-  report-clusteralerts:
-    name: ClusterAlerts in nested cluster
-    runs-on: ubuntu-latest
-    # Depends on the test jobs as well, not only on the watch: should the watch
-    # job die early, the report must still wait for the whole rollover instead
-    # of declaring "no alerts" while the tests are still running.
-    needs:
-      - test-current-release
-      - patch-modulepulloverride
-      - test-new-release
-      - watch-clusteralerts
-    if: always()
-    # A firing alert must be visible but must not fail the pipeline: this job
-    # exits non-zero (turning red in the UI) while the workflow conclusion
-    # stays successful.
-    continue-on-error: true
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Download collected ClusterAlerts
-        uses: actions/download-artifact@v8
-        # Nothing to download when the pipeline failed before the watch ran;
-        # that must not be mistaken for a firing alert.
-        continue-on-error: true
-        with:
-          name: clusteralerts-${{ github.run_id }}
-          path: ${{ env.CLUSTERALERTS_DIR }}
-
-      - name: Report ClusterAlerts
-        env:
-          RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }}
-          RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }}
-          WATCH_RESULT: ${{ needs.watch-clusteralerts.result }}
-        run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh"
-
   # TEMPORARY: drop before merge. Frees the nested cluster right after the
   # ClusterAlerts report instead of waiting for the nightly cleanup.
   delete-nested-cluster: