test: compare published platform images in kind

.github/workflows/platform-image-canary.yml

@@ -0,0 +1,77 @@
+name: Platform image canary
+
+on:
+  workflow_dispatch:
+    inputs:
+      component:
+        description: Component to compare
+        required: true
+        type: choice
+        options:
+          - autoscaler
+          - node-recovery
+          - pull-secret-renewer
+      old_image:
+        description: Previously published image pinned by digest
+        required: true
+        type: string
+      new_image:
+        description: Candidate published image pinned by digest
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  canary:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Validate image references
+        env:
+          COMPONENT: ${{ inputs.component }}
+          OLD_IMAGE: ${{ inputs.old_image }}
+          NEW_IMAGE: ${{ inputs.new_image }}
+        run: |
+          set -euo pipefail
+          for image in "$OLD_IMAGE" "$NEW_IMAGE"; do
+            [[ "$image" =~ ^ghcr\.io/exadev/github-runner-${COMPONENT}@sha256:[0-9a-f]{64}$ ]] || { echo "Invalid image reference: $image" >&2; exit 1; }
+          done
+          [[ "$OLD_IMAGE" != "$NEW_IMAGE" ]] || { echo 'Images must differ' >&2; exit 1; }
+      - name: Install Ansible
+        run: |
+          set -euo pipefail
+          python3 -m venv "${RUNNER_TEMP}/venv"
+          "${RUNNER_TEMP}/venv/bin/pip" install --quiet ansible-core pyyaml jinja2
+          echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH"
+      - name: Install Helm
+        if: inputs.component == 'autoscaler'
+        run: |
+          set -euo pipefail
+          archive=helm-v4.3.0-linux-amd64.tar.gz
+          curl -fsSLo "${RUNNER_TEMP}/${archive}" "https://get.helm.sh/${archive}"
+          echo "86584a54def73570558f66f5111cc53dfed56689637ae32c1201205d494f54fb  ${RUNNER_TEMP}/${archive}" | sha256sum -c -
+          tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}"
+          echo "${RUNNER_TEMP}/linux-amd64" >> "$GITHUB_PATH"
+      - name: Compare published images
+        env:
+          COMPONENT: ${{ inputs.component }}
+          OLD_IMAGE: ${{ inputs.old_image }}
+          NEW_IMAGE: ${{ inputs.new_image }}
+        run: |
+          set -euo pipefail
+          for image in "$OLD_IMAGE" "$NEW_IMAGE"; do
+            for run in 1 2; do
+              echo "Testing ${image}, run ${run}"
+              case "$COMPONENT" in
+                autoscaler) GRTEST_AUTOSCALER_IMAGE="$image" PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/autoscaler_ceiling/run.sh ;;
+                node-recovery) GRTEST_NODE_RECOVERY_IMAGE="$image" tests/node_recovery/run.sh ;;
+                pull-secret-renewer) GRTEST_RENEWER_IMAGE="$image" tests/pull_secret_renewal/run.sh ;;
+                *) exit 1 ;;
+              esac
+            done
+          done

tests/autoscaler_ceiling/run.sh

@@ -166,6 +166,32 @@ for namespace in "${pooled_namespaces[@]}"; do targets="$targets $namespace/grte
 
 # One poll, for real (not dry run), its output kept for the checks and for a failure's report.
 poll() {
+  if [ -n "${GRTEST_AUTOSCALER_IMAGE:-}" ]; then
+    kubectl -n "$platform_namespace" delete pod autoscaler-poll --ignore-not-found --wait=true >/dev/null
+    kubectl -n "$platform_namespace" run autoscaler-poll --image="$GRTEST_AUTOSCALER_IMAGE" --restart=Never \
+      --overrides "$(jq -n --arg targets "${targets# }" --arg ns "$platform_namespace" --arg selector "${eligibility_label}=true" '{spec:{serviceAccountName:"autoscaler",containers:[{name:"autoscaler-poll",image:env.GRTEST_AUTOSCALER_IMAGE,command:["/app/scripts/autoscaler.sh"],env:[{name:"AUTOSCALER_TARGETS",value:$targets},{name:"AUTOSCALER_DRY_RUN",value:"false"},{name:"AUTOSCALER_USABLE_BUDGET_GI",value:"1024"},{name:"AUTOSCALER_FLOOR",value:"1"},{name:"AUTOSCALER_NODE_SELECTOR",value:$selector},{name:"AUTOSCALER_STATE_NAMESPACE",value:$ns},{name:"AUTOSCALER_STATUS_CONFIGMAP",value:"autoscaler-status"}]}]}}')" >/dev/null
+    local phase
+    for _ in {1..120}; do
+      phase="$(kubectl -n "$platform_namespace" get pod autoscaler-poll -o jsonpath='{.status.phase}')"
+      case "$phase" in Succeeded|Failed) break ;; esac
+      sleep 2
+    done
+    kubectl -n "$platform_namespace" get pod autoscaler-poll -o json > "$work/autoscaler-pod.json"
+    "$python" - "$repo_root" "$work/autoscaler-pod.json" <<'PY'
+import json, pathlib, sys, yaml
+root = pathlib.Path(sys.argv[1])
+tasks = yaml.safe_load((root / 'roles/github_runner_arc/tasks/install_platform.yml').read_text())
+task = next(t for t in tasks if t.get('name') == 'Platform: install/upgrade the autoscaler Deployment')
+expected = task['kubernetes.core.k8s']['definition']['spec']['template']['spec']
+actual = json.loads(pathlib.Path(sys.argv[2]).read_text())['spec']
+assert actual['serviceAccountName'] == expected['serviceAccountName']
+assert actual.get('securityContext', {}) == expected.get('securityContext', {})
+assert actual['containers'][0].get('securityContext', {}) == expected['containers'][0].get('securityContext', {})
+PY
+    kubectl -n "$platform_namespace" logs autoscaler-poll > "$work/poll.log" || fail "could not read poll logs"
+    [ "$phase" = Succeeded ] || fail "the poll pod ended in $phase"
+    return
+  fi
   PATH="$work/bin:$PATH" \
   AUTOSCALER_TARGETS="${targets# }" \
   AUTOSCALER_DRY_RUN=false \

tests/node_recovery/run.sh

@@ -11,7 +11,7 @@ ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}"
 cluster=grtest-recovery
 platform_namespace=github-runner-platform
 namespace=grtest-listeners
-image=grtest/node-recovery:test
+image="${GRTEST_NODE_RECOVERY_IMAGE:-grtest/node-recovery:test}"
 pod_image=busybox:1.36
 watcher_node="${cluster}-worker"
 doomed_node="${cluster}-worker2"
@@ -153,9 +153,12 @@ server_version="$(kubectl version -o json | jq -r '.serverVersion.gitVersion')"
 log "Server version ${server_version}"
 doomed_uid="$(kubectl get node "$doomed_node" -o jsonpath='{.metadata.uid}')"
 
-log "Building and loading the watcher image and the stand-in's image"
-docker build -q -f "$repo_root/node-recovery/Dockerfile" -t "$image" "$repo_root" >/dev/null
-kind load docker-image "$image" --name "$cluster" >/dev/null
+if [ -z "${GRTEST_NODE_RECOVERY_IMAGE:-}" ]; then
+  log "Building and loading the watcher image"
+  docker build -q -f "$repo_root/node-recovery/Dockerfile" -t "$image" "$repo_root" >/dev/null
+  kind load docker-image "$image" --name "$cluster" >/dev/null
+fi
+log "Loading the stand-in's image"
 # The stand-in's image is pulled by each node's own runtime: a host Docker on the containerd image store pulls a multi-platform index holding only the host platform's blobs, and kind's import of it with all platforms fails on the missing ones.
 for node in $(kind get nodes --name "$cluster"); do
   docker exec "$node" crictl pull "$pod_image" >/dev/null

tests/pull_secret_renewal/run.sh

@@ -8,7 +8,7 @@ repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
 ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}"
 cluster=grtest-renewal
 namespace=grtest-runners
-image=grtest/pull-secret-renewer:test
+image="${GRTEST_RENEWER_IMAGE:-grtest/pull-secret-renewer:test}"
 stub_image=python:3.13-alpine
 pull_secret=ghcr-pull
 renewer="${pull_secret}-renewer"
@@ -63,9 +63,11 @@ run_job() {
 log "Creating kind cluster $cluster"
 kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null
 
-log "Building and loading the renewer image"
-docker build -q -f "$repo_root/pull-secret-renewer/Dockerfile" -t "$image" "$repo_root" >/dev/null
-kind load docker-image "$image" --name "$cluster" >/dev/null
+if [ -z "${GRTEST_RENEWER_IMAGE:-}" ]; then
+  log "Building and loading the renewer image"
+  docker build -q -f "$repo_root/pull-secret-renewer/Dockerfile" -t "$image" "$repo_root" >/dev/null
+  kind load docker-image "$image" --name "$cluster" >/dev/null
+fi
 
 log "Starting the GitHub stand-in"
 kubectl create namespace "$namespace" >/dev/null