diff --git a/.github/actions/setup-e2e-toolchain/action.yml b/.github/actions/setup-e2e-toolchain/action.yml index 1fb7ac9b91..2a38a65580 100644 --- a/.github/actions/setup-e2e-toolchain/action.yml +++ b/.github/actions/setup-e2e-toolchain/action.yml @@ -40,7 +40,7 @@ runs: - name: Restore d8 cache id: d8-cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: /opt/deckhouse/bin/d8 key: d8-${{ inputs.d8-version }}-${{ runner.os }} @@ -57,7 +57,7 @@ runs: - name: Install kubectl CLI if: inputs.install-kubectl == 'true' - uses: azure/setup-kubectl@v4 + uses: azure/setup-kubectl@v5 - name: Install htpasswd utility if: inputs.install-htpasswd == 'true' diff --git a/.github/scripts/bash/e2e/collect-clusteralerts.sh b/.github/scripts/bash/e2e/collect-clusteralerts.sh new file mode 100644 index 0000000000..7ede3ce77e --- /dev/null +++ b/.github/scripts/bash/e2e/collect-clusteralerts.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Collects the alerts of the virtualization module that were active in the +# nested cluster during the release rollover. Runs once, at the end of the +# pipeline, against the nested kubeconfig. +# +# Prometheus is the source rather than the ClusterAlerts objects: a ClusterAlert +# exists only while its alert is active, so a single late look at the API server +# would see nothing of what happened during the upgrade, while the ALERTS series +# keep the whole history of the observation window. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +require_env CLUSTERALERTS_DIR +require_env CLUSTERALERTS_WINDOW_STARTED_AT + +alerts_dir="${CLUSTERALERTS_DIR:-}" +alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}" +prometheus_namespace="${PROMETHEUS_NAMESPACE:-d8-monitoring}" +prometheus_selector="${PROMETHEUS_SELECTOR:-prometheus=main}" +# Port of the prometheus container inside the pod, discovered below when not +# pinned. port-forward joins the pod network namespace, so a listener bound to +# localhost there is reachable too. +prometheus_port="${PROMETHEUS_PORT:-}" +local_port="${PROMETHEUS_LOCAL_PORT:-19090}" +query_step="${QUERY_STEP:-30}" +ready_attempts="${READY_ATTEMPTS:-30}" +ready_delay="${READY_DELAY:-2}" + +# The window starts when virtualization was configured, not when the pipeline +# started: before that the module does not exist, and its alerts cannot either. +window_start="${CLUSTERALERTS_WINDOW_STARTED_AT:-}" +window_end="$(date +%s)" + +# Same semantics as in report-clusteralerts.sh: a zero means the corresponding +# job never reported a timestamp. +started="${RELEASE_UPGRADE_STARTED_AT:-0}" +finished="${RELEASE_UPGRADE_FINISHED_AT:-0}" +[[ "${started}" =~ ^[0-9]+$ ]] || started=0 +[[ "${finished}" =~ ^[0-9]+$ ]] || finished=0 + +if [[ ! "${window_start}" =~ ^[0-9]+$ ]] || [ "${window_start}" -eq 0 ]; then + echo "[ERROR] CLUSTERALERTS_WINDOW_STARTED_AT must be a unix timestamp, got: '${window_start}'" >&2 + exit 1 +fi + +range_file="${alerts_dir}/clusteralerts-query-range.json" +rules_file="${alerts_dir}/clusteralerts-rules.json" +alerts_log="${alerts_dir}/clusteralerts.jsonl" +port_forward_log="${alerts_dir}/clusteralerts-port-forward.log" + +port_forward_pid="" + +cleanup() { + [ -n "${port_forward_pid}" ] || return 0 + kill "${port_forward_pid}" 2>/dev/null || true + wait "${port_forward_pid}" 2>/dev/null || true + port_forward_pid="" +} + +prom_api() { + local path="$1" + shift + curl -sS -f --max-time 120 -G "http://127.0.0.1:${local_port}${path}" "$@" +} + +# port-forward plus curl on the runner, not kubectl exec plus curl in the +# container: the Prometheus image ships no shell tools to query itself with. +start_port_forward() { + local pod="$1" attempt + + echo "[INFO] Forwarding 127.0.0.1:${local_port} to ${pod}:${prometheus_port} in ${prometheus_namespace}" + kubectl -n "${prometheus_namespace}" port-forward \ + "pod/${pod}" "${local_port}:${prometheus_port}" > "${port_forward_log}" 2>&1 & + port_forward_pid=$! + trap cleanup EXIT + + # The tunnel needs a moment, and probing the query API instead of /-/ready + # checks exactly what the collection is about to use. + for ((attempt = 1; attempt <= ready_attempts; attempt++)); do + if prom_api /api/v1/query --data-urlencode 'query=1' > /dev/null 2>&1; then + echo "[INFO] Prometheus API is reachable after ${attempt} attempt(s)" + return 0 + fi + + if ! kill -0 "${port_forward_pid}" 2>/dev/null; then + echo "[ERROR] kubectl port-forward exited early:" >&2 + cat "${port_forward_log}" >&2 || true + return 1 + fi + + sleep "${ready_delay}" + done + + echo "[ERROR] Prometheus API did not become reachable after ${ready_attempts} attempt(s):" >&2 + cat "${port_forward_log}" >&2 || true + return 1 +} + +# Annotation templates live in the rules, not in the series, so they are fetched +# separately and rendered per series below. +fetch_rules() { + if prom_api /api/v1/rules --data-urlencode 'type=alert' > "${rules_file}" && + [ "$(jq -r '.status // ""' "${rules_file}")" = "success" ]; then + return 0 + fi + + # A missing summary must never sink the report: the alert itself is the news. + echo "[WARN] Failed to fetch alerting rules, alerts will be reported without a summary" + printf '%s\n' '{"status":"error","data":{"groups":[]}}' > "${rules_file}" +} + +templates_program=' +[ .data.groups[]?.rules[]? + | select(.type == "alerting") + | { key: .name, + value: { + summary: (.annotations.summary // ""), + description: (.annotations.description // "") + } + } +] | from_entries +' + +# shellcheck disable=SC2016 # $started, $finished and $templates are jq variables, passed in via --argjson +records_program=' +def phase_of($t): + if $started == 0 or $t < $started then { phase: "pre-upgrade", order: 0 } + elif $finished == 0 or $t < $finished then { phase: "upgrade", order: 1 } + else { phase: "post-upgrade", order: 2 } + end; + +# The annotations are Go templates that reference $labels only (verified over +# monitoring/prometheus-rules), so replacing every label by its value renders +# them. A reference to a label the series does not carry is left as it is. +def render($labels): + reduce ($labels | to_entries[]) as $l + (.; gsub("\\{\\{\\s*\\$labels\\." + $l.key + "\\s*\\}\\}"; $l.value)); + +[ .data.result[] + | . as $series + | ($series.metric.alertname // "") as $name + | ($series.metric | del(.__name__, .alertname, .alertstate)) as $labels + # One record per phase the series touches: an alert that spans the upgrade is + # news in every phase it was active in, and grouping the samples by phase + # intersects its active interval with the upgrade window. + | [ $series.values[] | .[0] | floor | { t: ., ph: phase_of(.) } ] + | group_by(.ph.order) + | .[] + | { phase: .[0].ph.phase, + order: .[0].ph.order, + name: $name, + alertstate: ($series.metric.alertstate // ""), + severityLevel: ($series.metric.severity_level // ""), + labels: $labels, + firstSeen: (map(.t) | min | todate), + lastSeen: (map(.t) | max | todate), + summary: (($templates[$name].summary // "") | render($labels)), + description: (($templates[$name].description // "") | render($labels)), + id: ([$name, ($series.metric.alertstate // ""), ($labels | tojson)] | join("|")) + } +] +| unique_by([.order, .id]) +| sort_by([.order, .name, .alertstate]) +| .[] +' + +mkdir -p "${alerts_dir}" + +echo "[INFO] Collecting alerts matching '${alert_prefix}*' from Prometheus in ${prometheus_namespace}" +echo "[INFO] Observation window: ${window_start}..${window_end} ($(( window_end - window_start ))s), step ${query_step}s" +echo "[INFO] Upgrade window: started_at=${started}, finished_at=${finished}" + +pod="$(kubectl -n "${prometheus_namespace}" get pod \ + -l "${prometheus_selector}" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}')" + +if [ -z "${pod}" ]; then + echo "[ERROR] No Running pod matching '${prometheus_selector}' in namespace ${prometheus_namespace}" >&2 + exit 1 +fi + +# Asking the pod which port carries the API beats assuming one: an authenticating +# sidecar may well be the container that owns 9090 there. +if [ -z "${prometheus_port}" ]; then + prometheus_port="$(kubectl -n "${prometheus_namespace}" get pod "${pod}" \ + -o jsonpath='{.spec.containers[?(@.name=="prometheus")].ports[?(@.name=="web")].containerPort}' || true)" + prometheus_port="${prometheus_port:-9090}" +fi + +start_port_forward "${pod}" + +# query_range and not query: an instant query would only see what is still +# active now, while the report is about what was active during the rollover. +prom_api /api/v1/query_range \ + --data-urlencode "query=ALERTS{alertname=~\"${alert_prefix}.*\"}" \ + --data-urlencode "start=${window_start}" \ + --data-urlencode "end=${window_end}" \ + --data-urlencode "step=${query_step}" > "${range_file}" + +if [ "$(jq -r '.status // ""' "${range_file}")" != "success" ]; then + echo "[ERROR] Prometheus rejected the range query: $(jq -c '.' "${range_file}")" >&2 + exit 1 +fi + +fetch_rules + +templates="$(jq "${templates_program}" "${rules_file}")" + +jq -c \ + --argjson started "${started}" \ + --argjson finished "${finished}" \ + --argjson templates "${templates}" \ + "${records_program}" "${range_file}" > "${alerts_log}" + +count="$(grep -c . "${alerts_log}" || true)" +echo "[INFO] Collected ${count} alert record(s) into ${alerts_log}" +jq -r '" [\(.phase)] \(.name) \(.alertstate) (\(.firstSeen) .. \(.lastSeen))"' "${alerts_log}" diff --git a/.github/scripts/bash/e2e/common.sh b/.github/scripts/bash/e2e/common.sh index 245eb8c877..92fc259176 100644 --- a/.github/scripts/bash/e2e/common.sh +++ b/.github/scripts/bash/e2e/common.sh @@ -47,6 +47,37 @@ modules_repo_for_registry() { fi } +# Echoes the virtualization feature gates supported by every given release, one +# per line. A gate the pulled module does not know fails ModulePullOverride +# validation and leaves the module uninstalled, so a gate is only listed when +# none of the releases predate it. In-place resize never shipped in the 1.9 +# line. Anything that is not a release tag (a PR reference, a build off main) +# carries the current gates. +# Usage: virtualization_feature_gates [release]... +virtualization_feature_gates() { + local release + + echo "HotplugCPUWithLiveMigration" + echo "HotplugMemoryWithLiveMigration" + + for release in "$@"; do + if [[ "${release}" =~ ^v([0-9]+)\.([0-9]+)\. ]] && (( BASH_REMATCH[1] == 1 && BASH_REMATCH[2] < 10 )); then + return 0 + fi + done + + echo "HotplugCPUAndMemoryWithInPlaceResize" +} + +# Echoes images_digests.json packaged in the module image of a given release. +# Usage: module_images_digests +module_images_digests() { + local module_source="$1" + local release="$2" + + crane export "${module_source}/virtualization:${release}" - | tar -Oxf - images_digests.json +} + # Reads a manifest from stdin and applies it with retries. # Usage: kubectl_apply_with_retry [count] [delay] [diag_fn] # diag_fn is an optional function name invoked on each failed attempt. diff --git a/.github/scripts/bash/e2e/configure-virtualization-release.sh b/.github/scripts/bash/e2e/configure-virtualization-release.sh index b39ff8a7ad..d8fc6d86da 100644 --- a/.github/scripts/bash/e2e/configure-virtualization-release.sh +++ b/.github/scripts/bash/e2e/configure-virtualization-release.sh @@ -37,6 +37,13 @@ current_release="$(required_env_value CURRENT_RELEASE)" REGISTRY="$(registry_host_from_docker_cfg "${dev_registry_docker_cfg}")" +# Only the gates this release knows: a gate it does not support fails +# ModulePullOverride validation and the module never installs. The upgrade +# revisits the list for the new release (patch-virtualization-feature-gates.sh). +feature_gates_yaml="$(virtualization_feature_gates "${current_release}" | sed 's/^/ - /')" +echo "[INFO] Feature gates for ${current_release}:" +echo "${feature_gates_yaml}" + echo "[INFO] Apply ModuleSource prod config" kubectl_apply_with_retry 20 10 show_deckhouse_state <... +# +# During a release upgrade this runs twice. Before the image tag is patched it +# is called with both releases, which drops the gates the new release does not +# know - otherwise the new module fails validation and never installs. After the +# upgrade it is called with the new release alone, which enables the gates only +# that release supports. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +if [ "$#" -eq 0 ]; then + echo "[ERROR] Usage: $(basename -- "${BASH_SOURCE[0]}") ..." >&2 + exit 1 +fi + +gates_json="$(virtualization_feature_gates "$@" | jq -Rsc 'split("\n") | map(select(length > 0))')" +current_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')" + +echo "[INFO] Feature gates supported by $*: ${gates_json}" + +if [ "${current_json}" = "${gates_json}" ]; then + echo "[INFO] Module config already lists exactly these gates, nothing to patch" + exit 0 +fi + +echo "[INFO] Patching feature gates: ${current_json:-none} -> ${gates_json}" +kubectl patch mc virtualization --type merge -p "{\"spec\":{\"settings\":{\"featureGates\":${gates_json}}}}" + +patched_json="$(kubectl get mc virtualization -o jsonpath='{.spec.settings.featureGates}')" +if [ "${patched_json}" != "${gates_json}" ]; then + echo "[ERROR] Feature gates were not applied: expected ${gates_json}, got ${patched_json:-none}" >&2 + exit 1 +fi + +echo "[INFO] Feature gates in effect: ${patched_json}" diff --git a/.github/scripts/bash/e2e/report-clusteralerts.sh b/.github/scripts/bash/e2e/report-clusteralerts.sh new file mode 100644 index 0000000000..d202e3125b --- /dev/null +++ b/.github/scripts/bash/e2e/report-clusteralerts.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/bash/e2e/common.sh +source "${SCRIPT_DIR}/common.sh" + +require_env CLUSTERALERTS_DIR + +alerts_dir="${CLUSTERALERTS_DIR:-}" +alert_prefix="${CLUSTERALERTS_PREFIX:-D8Virtualization}" +fail_on_alerts="${FAIL_ON_ALERTS:-true}" +collect_result="${COLLECT_RESULT:-}" +summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +{ + echo "## ClusterAlerts in the nested cluster" + echo +} >> "${summary_file}" + +# Renders the "we could not check" verdict: an empty report is only good news +# when the collection itself succeeded. +not_collected() { + local reason="$1" + + echo "${reason}" >> "${summary_file}" + echo "::warning title=ClusterAlerts were not collected::${reason}" + exit 0 +} + +shopt -s nullglob +logs=("${alerts_dir}"/*.jsonl) +shopt -u nullglob + +if [ -n "${collect_result}" ] && [ "${collect_result}" != "success" ]; then + not_collected "The collection step did not complete (result: \`${collect_result}\`), so alerts were **not** checked." +fi + +# collect-clusteralerts.sh always writes its log, empty or not, so a missing one +# means the collection never got that far - a step that never ran reports no +# result at all. +if [ "${#logs[@]}" -eq 0 ]; then + echo "[WARN] No collected ClusterAlerts found in ${alerts_dir}" + not_collected "No collected alerts were found, so alerts were **not** checked." +fi + +echo "[INFO] Reading collected ClusterAlerts from: ${logs[*]}" +# The phase, the state and the firing interval come from the collector; sorting +# is repeated here only to keep the order stable across several log files. +alerts="$(jq -s 'sort_by([.order, .name, .alertstate])' "${logs[@]}")" + +count="$(jq 'length' <<< "${alerts}")" + +# Alert summaries are markdown ending with a newline; flatten them for the +# table and for annotations. +oneline='def oneline: gsub("\\s+"; " ") | sub("^ "; "") | sub(" $"; "");' + +if [ "${count}" -eq 0 ]; then + echo "No \`${alert_prefix}*\` alerts were active during the release rollover." >> "${summary_file}" + echo "[INFO] No ${alert_prefix}* alerts were active during the release rollover" + exit 0 +fi + +{ + echo "| Phase | Alert | State | Severity | First seen | Summary |" + echo "|---|---|---|---|---|---|" + jq -r "${oneline}"' .[] | "| \(.phase) | \(.name) | \(.alertstate) | \(.severityLevel) | \(.firstSeen) | \(.summary | oneline | gsub("\\|"; "\\|")) |"' <<< "${alerts}" + echo + echo "
Alert details" + echo + jq -r '.[] | "#### \(.name) — \(.phase) (\(.alertstate))\n\n- severity level: \(.severityLevel)\n- active: \(.firstSeen) .. \(.lastSeen)\n- labels: `\(.labels | tojson)`\n\n\(.description)\n"' <<< "${alerts}" + echo "
" +} >> "${summary_file}" + +# Annotations put the alerts on top of the run page, not only in the summary. +# A pending alert is a notice rather than a warning: it did not hold long enough +# to be one. +jq -r "${oneline}"' .[] + | (if .alertstate == "firing" then "::warning" else "::notice" end) + + " title=ClusterAlert \(.name)::[\(.phase)/\(.alertstate)] \(.summary | oneline)"' <<< "${alerts}" + +echo "[INFO] Active alerts:" +jq -r '.[] | " [\(.phase)] \(.name) \(.alertstate) (severity \(.severityLevel))"' <<< "${alerts}" + +firing_count="$(jq '[.[] | select(.alertstate == "firing")] | length' <<< "${alerts}")" + +# Only a fired alert paints the job red. Components restart during a rollover, +# so rules with a `for` clause go pending on almost every run: failing on those +# would make a red job the norm and tell the reviewer nothing. +if [ "${firing_count}" -eq 0 ]; then + { + echo + echo "No \`${alert_prefix}*\` alert reached the firing state; ${count} were pending only." + } >> "${summary_file}" + echo "[INFO] No ${alert_prefix}* alert reached the firing state, ${count} were pending only" + exit 0 +fi + +if [ "${fail_on_alerts}" != "true" ]; then + echo "[INFO] FAIL_ON_ALERTS is not 'true', not failing the job" + exit 0 +fi + +# Failing here is what paints this job red; the job itself is +# continue-on-error, so the workflow conclusion stays successful. +echo "[ERROR] ${firing_count} ClusterAlert(s) were firing in the nested cluster, see the job summary" >&2 +trap - ERR +exit 1 diff --git a/.github/scripts/bash/e2e/verify-image-digests.sh b/.github/scripts/bash/e2e/verify-image-digests.sh index d161eebfe9..73bd57f07e 100644 --- a/.github/scripts/bash/e2e/verify-image-digests.sh +++ b/.github/scripts/bash/e2e/verify-image-digests.sh @@ -33,9 +33,8 @@ required_env_value() { new_release="$(required_env_value NEW_RELEASE)" dev_module_source="$(required_env_value DEV_MODULE_SOURCE)" -MODULE_IMAGE="${dev_module_source}/virtualization:${new_release}" echo "[INFO] Extracting images_digests.json from virtualization:${new_release}" -images_hash="$(crane export "${MODULE_IMAGE}" - | tar -Oxf - images_digests.json)" +images_hash="$(module_images_digests "${dev_module_source}" "${new_release}")" echo "[INFO] Expected image digests:" echo "::group::images_digests.json" echo "${images_hash}" | jq . diff --git a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh index 6eda51d2fc..da14b7520e 100644 --- a/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh +++ b/.github/scripts/bash/e2e/wait-vmops-migration-terminal.sh @@ -33,6 +33,53 @@ release_namespace="$(required_env_value RELEASE_NAMESPACE)" sleep_interval="${SLEEP_INTERVAL:-10}" timeout_seconds="${TIMEOUT_SECONDS:-1200}" +# Virtual machines are only moved when the workload images change: a new +# virt-handler drains the VMs off its node, a new virt-launcher makes the +# workload updater migrate the running ones. Releases that leave both untouched +# never trigger a migration, so there would be nothing to wait for. +migration_expected() { + local module_source="${DEV_MODULE_SOURCE:-}" + local current="${CURRENT_RELEASE:-}" + local new="${NEW_RELEASE:-}" + local current_digests new_digests image + + if [ -z "${module_source}" ] || [ -z "${current}" ] || [ -z "${new}" ]; then + echo "[WARN] DEV_MODULE_SOURCE, CURRENT_RELEASE or NEW_RELEASE is not set, cannot tell whether the upgrade migrates VMs; waiting anyway" + return 0 + fi + + if ! current_digests="$(module_images_digests "${module_source}" "${current}")" || + ! new_digests="$(module_images_digests "${module_source}" "${new}")"; then + echo "[WARN] Failed to read the image digests of ${current} or ${new}, cannot tell whether the upgrade migrates VMs; waiting anyway" + return 0 + fi + + for image in virtHandler virtLauncher; do + if [ "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${current_digests}")" \ + != "$(jq -r --arg i "${image}" '.[$i] // ""' <<< "${new_digests}")" ]; then + echo "[INFO] The ${image} image differs between ${current} and ${new}, virtual machines will be migrated" + return 0 + fi + done + + return 1 +} + +# The verdict is published so the new-release tests can tell a missing migration +# from one that was never going to happen. +publish_verdict() { + [ -n "${GITHUB_OUTPUT:-}" ] || return 0 + echo "migrates_vms=$1" >> "${GITHUB_OUTPUT}" +} + +if ! migration_expected; then + publish_verdict false + echo "[INFO] ${CURRENT_RELEASE} and ${NEW_RELEASE} ship the same virt-handler and virt-launcher: the upgrade does not migrate virtual machines, nothing to wait for" + exit 0 +fi + +publish_verdict true + deadline=$(( $(date +%s) + timeout_seconds )) # The number of Running VMs at the start is the number of Evict VMOPs we diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml index 12f9bef117..cf570f2629 100644 --- a/.github/workflows/dev_module_build.yml +++ b/.github/workflows/dev_module_build.yml @@ -63,7 +63,7 @@ jobs: steps: - name: Get Pull Request Labels id: get-labels - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | function processDelveLabels(labelList) { @@ -212,7 +212,7 @@ jobs: name: Run go linter steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -314,7 +314,7 @@ jobs: name: Run unit test steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -464,7 +464,7 @@ jobs: needs: set_vars steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -644,7 +644,7 @@ jobs: steps: - name: Create Initial PR Comment id: create_comment - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: github-token: ${{secrets.RELEASE_PLEASE_TOKEN}} script: | @@ -696,7 +696,7 @@ jobs: uses: ./.github/actions/install-d8 - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" diff --git a/.github/workflows/dev_validation.yaml b/.github/workflows/dev_validation.yaml index 9376e293d3..070482c97b 100644 --- a/.github/workflows/dev_validation.yaml +++ b/.github/workflows/dev_validation.yaml @@ -58,7 +58,7 @@ jobs: name: Validation no-cyrillic steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -83,7 +83,7 @@ jobs: name: Validation doc-changes steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -108,7 +108,7 @@ jobs: name: Validation go-work steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -220,7 +220,7 @@ jobs: steps: - name: Setup Go ${{ matrix.components.go-version }} if: matrix.components.component != 'vm-route-forge' || needs.paths_filter.outputs.vm_route_forge == 'true' - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ matrix.components.go-version }} diff --git a/.github/workflows/e2e-nightly-reusable-pipeline.yml b/.github/workflows/e2e-nightly-reusable-pipeline.yml index 3e44dac1a6..8c40f4d24d 100644 --- a/.github/workflows/e2e-nightly-reusable-pipeline.yml +++ b/.github/workflows/e2e-nightly-reusable-pipeline.yml @@ -187,6 +187,8 @@ env: DECKHOUSE_VERSION: ${{ inputs.deckhouse_version }} DEFAULT_USER: ${{ inputs.default_user }} GO_VERSION: ${{ inputs.go_version }} + # setup-go v6 would pin the toolchain; a checked out ref may need a newer one. + GOTOOLCHAIN: auto SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster E2E_SCRIPT_DIR: ${{ github.workspace }}/.github/scripts/bash/e2e K8S_VERSION: ${{ inputs.cluster_config_k8s_version }} @@ -237,8 +239,8 @@ jobs: with: docker_cfg: ${{ secrets.REGISTRY_DOCKER_CFG }} - - name: Configure kubectl via azure/k8s-set-context@v4 - uses: azure/k8s-set-context@v4 + - name: Configure kubectl via azure/k8s-set-context@v5 + uses: azure/k8s-set-context@v5 with: method: kubeconfig context: e2e-cluster-nightly-e2e-virt-sa @@ -584,7 +586,7 @@ jobs: - uses: actions/checkout@v6 - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -696,7 +698,7 @@ jobs: - name: Determine failed stage and prepare report id: determine-stage - uses: actions/github-script@v7 + uses: actions/github-script@v8 env: STORAGE_TYPE: ${{ inputs.storage_type }} PIPELINE_JOB_NAME: ${{ inputs.pipeline_job_name }} @@ -763,8 +765,8 @@ jobs: "$artifact_path" unzip -o "${RUNNER_TEMP}/${ARTIFACT_NAME}.zip" -d "${{ env.SETUP_CLUSTER_TYPE_PATH }}" - - name: Configure kubectl via azure/k8s-set-context@v4 - uses: azure/k8s-set-context@v4 + - name: Configure kubectl via azure/k8s-set-context@v5 + uses: azure/k8s-set-context@v5 with: method: kubeconfig context: e2e-cluster-nightly-e2e-virt-sa diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 8c1a4626b8..1547a7e0db 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -36,8 +36,8 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - - name: Configure kubectl via azure/k8s-set-context@v4 - uses: azure/k8s-set-context@v4 + - name: Configure kubectl via azure/k8s-set-context@v5 + uses: azure/k8s-set-context@v5 with: method: kubeconfig context: e2e-cluster-nightly-e2e-virt-sa @@ -60,8 +60,8 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - - name: Configure kubectl via azure/k8s-set-context@v4 - uses: azure/k8s-set-context@v4 + - name: Configure kubectl via azure/k8s-set-context@v5 + uses: azure/k8s-set-context@v5 with: method: kubeconfig context: e2e-cluster-nightly-e2e-virt-sa @@ -146,7 +146,7 @@ jobs: deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }} registry_profile: ${{ needs.set-vars.outputs.registry_profile }} default_user: cloud - go_version: "1.24.13" + go_version: "1.25.12" e2e_timeout: "3.5h" e2e_image_base_url: ${{ needs.set-vars.outputs.e2e_image_base_url }} date_start: ${{ needs.set-vars.outputs.date_start }} @@ -180,7 +180,7 @@ jobs: deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }} registry_profile: ${{ needs.set-vars.outputs.registry_profile }} default_user: cloud - go_version: "1.24.13" + go_version: "1.25.12" e2e_timeout: "3.5h" e2e_focus_tests: "VirtualDiskProvisioning|VirtualDiskSnapshots|VirtualImageCreation|VirtualDiskResizing|DiskAttachment|BlockDeviceHotplug|Migration|StorageClassMigration|RWOVirtualDiskMigration|VMSOP|Restore|DataExports" e2e_image_base_url: ${{ needs.set-vars.outputs.e2e_image_base_url }} @@ -240,7 +240,7 @@ jobs: - name: Send results to channel id: render-report - uses: actions/github-script@v7 + uses: actions/github-script@v8 env: EXPECTED_STORAGE_TYPES: '["replicated","nfs","ceph"]' LOOP_API_BASE_URL: ${{ secrets.LOOP_API_BASE_URL }} diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 649b84c161..cce43a7190 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -140,11 +140,14 @@ env: DECKHOUSE_VERSION: ${{ inputs.deckhouse_version }} DEFAULT_USER: ${{ inputs.default_user }} GO_VERSION: ${{ inputs.go_version }} + # setup-go v6 would pin the toolchain; a checked out ref may need a newer one. + GOTOOLCHAIN: auto SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster E2E_SCRIPT_DIR: ${{ github.workspace }}/.github/scripts/bash/e2e K8S_VERSION: ${{ inputs.cluster_config_k8s_version }} STORAGE_TYPE: ${{ inputs.storage_type }} E2E_START_TIME: ${{ inputs.date_start }} + CLUSTERALERTS_DIR: ${{ github.workspace }}/clusteralerts defaults: run: @@ -190,8 +193,8 @@ jobs: with: docker_cfg: ${{ secrets.REGISTRY_DOCKER_CFG }} - - name: Configure kubectl via azure/k8s-set-context@v4 - uses: azure/k8s-set-context@v4 + - name: Configure kubectl via azure/k8s-set-context@v5 + uses: azure/k8s-set-context@v5 with: method: kubeconfig context: e2e-cluster-nightly-e2e-virt-sa @@ -475,6 +478,8 @@ jobs: needs: - bootstrap - configure-storage + outputs: + configured_at: ${{ steps.mark-configured.outputs.configured_at }} steps: - uses: actions/checkout@v6 @@ -509,6 +514,13 @@ jobs: echo "[INFO] Checking virt-handler pods " virt_handler_ready + # Opens the observation window of the ClusterAlerts report. Runs even on + # failure, so a module that never became ready is still observed. + - name: Mark the start of the ClusterAlerts observation window + id: mark-configured + if: always() + run: echo "configured_at=$(date +%s)" >> "$GITHUB_OUTPUT" + test-current-release: name: "E2E test (current-release: ${{ inputs.current_release }})" runs-on: ubuntu-latest @@ -521,7 +533,7 @@ jobs: - uses: actions/checkout@v6 - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -536,6 +548,7 @@ jobs: with: kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} check-api: "false" + - name: Authenticate go module fetches to the fox kubevirt mirror env: FOX_TOKEN: ${{ secrets.FOX_TOKEN }} @@ -600,6 +613,8 @@ jobs: - test-current-release outputs: upgrade_started_at: ${{ steps.patch-modulepulloverride.outputs.upgrade_started_at }} + upgrade_finished_at: ${{ steps.upgrade-finished.outputs.upgrade_finished_at }} + migrates_vms: ${{ steps.wait-migrations.outputs.migrates_vms }} steps: - uses: actions/checkout@v6 @@ -614,11 +629,19 @@ jobs: with: kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} check-api: "false" + - name: Show current MPO state run: | echo "[INFO] Current ModulePullOverride before patching:" kubectl get mpo virtualization -o yaml + # Gates the new release does not know would fail its validation, so they + # go before the image tag is switched, while the old module is still live. + - name: Drop feature gates the new release does not support + run: | + bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" \ + "${CURRENT_RELEASE}" "${NEW_RELEASE}" + - name: "Patch ModulePullOverride to new-release: ${{ env.NEW_RELEASE }}" id: patch-modulepulloverride run: | @@ -643,6 +666,11 @@ jobs: NEW_RELEASE: ${{ env.NEW_RELEASE }} run: bash "${E2E_SCRIPT_DIR}/verify-image-digests.sh" + # Now that the new images are running, the gates only that release knows + # can be enabled, so the new release is tested with all of them on. + - name: Enable every feature gate the new release supports + run: bash "${E2E_SCRIPT_DIR}/patch-virtualization-feature-gates.sh" "${NEW_RELEASE}" + - name: Show ModulePullOverride state after upgrade run: | echo "[INFO] ModulePullOverride after upgrade:" @@ -651,10 +679,19 @@ jobs: kubectl get modules virtualization - name: Wait for all migrations to reach a terminal phase + id: wait-migrations env: RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }} + DEV_MODULE_SOURCE: ${{ vars.DEV_MODULE_SOURCE }} run: bash "${E2E_SCRIPT_DIR}/wait-vmops-migration-terminal.sh" + # Closes the upgrade window for the ClusterAlerts report; runs even on + # failure so alerts are still attributed to the right phase. + - name: Mark the end of the upgrade window + id: upgrade-finished + if: always() + run: echo "upgrade_finished_at=$(date +%s)" >> "$GITHUB_OUTPUT" + test-new-release: name: "E2E test (new-release: ${{ inputs.new_release }})" runs-on: ubuntu-latest @@ -671,8 +708,14 @@ jobs: checkout: "false" github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup kubeconfig + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" + - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -700,11 +743,6 @@ jobs: echo "Download dependencies" go mod download - - name: Setup kubeconfig - uses: ./.github/actions/use-nested-kubeconfig - with: - kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} - check-api: "false" - name: "Run E2E tests on new-release" env: NEW_RELEASE: ${{ env.NEW_RELEASE }} @@ -714,4 +752,93 @@ jobs: RELEASE_TEST_PHASE: post-upgrade RELEASE_NAMESPACE: ${{ needs.test-current-release.outputs.release_namespace }} RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }} + RELEASE_UPGRADE_MIGRATES_VMS: ${{ needs.patch-modulepulloverride.outputs.migrates_vms }} run: bash "${E2E_SCRIPT_DIR}/run-release-e2e.sh" + + report-clusteralerts: + name: ClusterAlerts in nested cluster + runs-on: ubuntu-latest + # Waits for the whole test and upgrade sequence: Prometheus is queried once, + # at the end, so everything that could fire must have happened by then. + needs: + - bootstrap + - configure-virtualization + - test-current-release + - patch-modulepulloverride + - test-new-release + if: always() + # A firing alert must be visible but must not fail the pipeline: this job + # exits non-zero (turning red in the UI) while the workflow conclusion + # stays successful. + continue-on-error: true + steps: + - uses: actions/checkout@v6 + + - name: Setup E2E toolchain + uses: ./.github/actions/setup-e2e-toolchain + with: + checkout: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} + + # Red must mean "an alert fired" and nothing else, so neither a missing + # cluster nor a failed query is allowed to fail this job: the report step + # turns both into a "not checked" verdict. + - name: Setup kubeconfig + continue-on-error: true + uses: ./.github/actions/use-nested-kubeconfig + with: + kubeconfig: ${{ needs.bootstrap.outputs.kubeconfig }} + check-api: "false" + + - name: Collect ClusterAlerts + id: collect + if: always() + continue-on-error: true + env: + CLUSTERALERTS_WINDOW_STARTED_AT: ${{ needs.configure-virtualization.outputs.configured_at }} + RELEASE_UPGRADE_STARTED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_started_at }} + RELEASE_UPGRADE_FINISHED_AT: ${{ needs.patch-modulepulloverride.outputs.upgrade_finished_at }} + run: bash "${E2E_SCRIPT_DIR}/collect-clusteralerts.sh" + + - name: Report ClusterAlerts + # Runs even when the collection failed: the report is the only place + # that says so, and a silent job would read as "no alerts". + if: always() + env: + COLLECT_RESULT: ${{ steps.collect.outcome }} + run: bash "${E2E_SCRIPT_DIR}/report-clusteralerts.sh" + + - name: Upload collected ClusterAlerts + uses: actions/upload-artifact@v7 + if: always() + with: + name: clusteralerts-${{ github.run_id }} + path: ${{ env.CLUSTERALERTS_DIR }} + if-no-files-found: ignore + retention-days: 3 + + # TEMPORARY: drop before merge. Frees the nested cluster right after the + # ClusterAlerts report instead of waiting for the nightly cleanup. + delete-nested-cluster: + name: Delete nested cluster (temporary) + runs-on: ubuntu-latest + # report-clusteralerts already waits for the whole test and upgrade + # sequence, so this runs once nothing needs the cluster any more. + needs: + - bootstrap + - report-clusteralerts + if: always() + steps: + - name: Configure kubectl via azure/k8s-set-context@v5 + uses: azure/k8s-set-context@v5 + with: + method: kubeconfig + context: e2e-cluster-nightly-e2e-virt-sa + kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + + - name: Delete the namespace and the cluster-scoped VirtualMachineClass + env: + NAMESPACE: ${{ needs.bootstrap.outputs.namespace }} + run: | + kubectl delete namespace "${NAMESPACE}" --timeout=300s || true + kubectl delete vmclass "${NAMESPACE}-cpu" --timeout=300s || true diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index b209a88abd..5555e68de0 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -20,12 +20,12 @@ on: current-release: description: "Current release tag like v1.4.1, or PR reference like pr2034/2034" required: false # before merge to main, set to true - default: "v1.6.3-rc.3" # before merge to main, remove + default: "v1.9.6" # before merge to main, remove type: string next-release: description: "Next release like v1.5.0, or PR reference like pr2034/2034" required: false # before merge to main, set to true - default: "v1.7.0" # before merge to main, remove + default: "v1.9.7-rc.0" # before merge to main, remove type: string enableBuild: description: "Build release images before E2E tests" @@ -34,7 +34,7 @@ on: type: boolean concurrency: - group: "${{ github.workflow }}-${{ github.event.inputs.current-release }}-${{ github.event.inputs.next-release || 'no-next' }}" + group: "${{ github.workflow }}- ${{ github.ref }}" cancel-in-progress: true defaults: @@ -76,7 +76,7 @@ jobs: steps: - name: Resolve release refs id: resolve - uses: actions/github-script@v7 + uses: actions/github-script@v8 env: CURRENT_RELEASE_INPUT: ${{ github.event.inputs.current-release }} NEXT_RELEASE_INPUT: ${{ github.event.inputs.next-release }} @@ -229,7 +229,7 @@ jobs: deckhouse_version: ${{ needs.set-vars.outputs.deckhouse_version }} registry_profile: ${{ needs.set-vars.outputs.registry_profile }} default_user: cloud - go_version: "1.25.8" + go_version: "1.25.12" date_start: ${{ needs.set-vars.outputs.date_start }} randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} cluster_config_workers_memory: "9Gi" diff --git a/.github/workflows/release_module_release-channels.yml b/.github/workflows/release_module_release-channels.yml index a14434a935..d2fe7063bd 100644 --- a/.github/workflows/release_module_release-channels.yml +++ b/.github/workflows/release_module_release-channels.yml @@ -142,7 +142,7 @@ jobs: - name: Set up Go ${{ env.GO_VERSION }} if: ${{ !inputs.skip_requirements_check }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" @@ -449,7 +449,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "${{ env.GO_VERSION }}" diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 9e6516829a..fb23f86aa6 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -42,6 +42,7 @@ const ( releaseTestPhasePostUpgrade = "post-upgrade" releaseUpgradeContextPathEnv = "RELEASE_UPGRADE_CONTEXT_PATH" releaseNamespaceEnv = "RELEASE_NAMESPACE" + releaseUpgradeMigratesVMsEnv = "RELEASE_UPGRADE_MIGRATES_VMS" ) var _ = Describe("CurrentReleaseSmoke", func() { @@ -205,6 +206,14 @@ func (t *currentReleaseSmokeTest) verifyIPerfContinuityAfterUpgrade() { report := getIPerfClientReport(t.framework, t.iperfClient.vm, releaseIPerfReportPath) Expect(isExpectedIPerfReportError(report.Error)).To(BeTrue(), "iperf3 report contains an unexpected error: %q", report.Error) + if !upgradeMigratesVMs() { + By("Skipping the migration window checks: the upgrade does not migrate virtual machines") + Expect(report.End.SumSent.Bytes).To(BeNumerically(">", 0), "iperf3 client should send data") + Expect(report.End.SumSent.BitsPerSecond).To(BeNumerically(">", 0), "iperf3 client should report throughput") + + return + } + By("Verifying the iperf test brackets the migration window (started before, stopped after)") migration := getMigrationWindow(t.framework, t.iperfServer.vm.Name, t.iperfServer.vm.Namespace) @@ -278,6 +287,13 @@ func getReleaseTestPhase() string { return releaseTestPhasePreUpgrade } +// Upgrades between releases that ship the same virt-handler and virt-launcher +// never move a virtual machine. An unset value means the pipeline could not tell, +// so a migration is still expected. +func upgradeMigratesVMs() bool { + return os.Getenv(releaseUpgradeMigratesVMsEnv) != "false" +} + func mustGetEnv(name string) string { value := os.Getenv(name) Expect(value).NotTo(BeEmpty(), "environment variable %s must be set", name)