CI: Retry spot-terminated jobs in merge queue

.github/workflows/visual-tests-demos-merge-queue.yml

@@ -0,0 +1,182 @@
+name: Demos Visual Tests (merge queue)
+
+on:
+  merge_group:
+
+concurrency:
+  group: wf-${{ github.event.merge_group.head_sha || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: false
+
+jobs:
+  attempt1:
+    name: Attempt 1
+    uses: ./.github/workflows/visual-tests-demos.yml
+    secrets: inherit
+
+  triage:
+    name: Triage attempt 1
+    runs-on: ubuntu-latest
+    needs: [attempt1]
+    if: always()
+    timeout-minutes: 10
+    permissions:
+      actions: read
+    outputs:
+      is_retry: ${{ steps.decide.outputs.is_retry }}
+      real_failure: ${{ steps.decide.outputs.real_failure }}
+      retry_targets: ${{ steps.decide.outputs.retry_targets }}
+    steps:
+      - id: decide
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
+          ATTEMPT1_RESULT: ${{ needs.attempt1.result }}
+        run: |
+          set -euo pipefail
+
+          emit() {
+            echo "is_retry=$1"       >> "$GITHUB_OUTPUT"
+            echo "real_failure=$2"   >> "$GITHUB_OUTPUT"
+            echo "retry_targets=$3"  >> "$GITHUB_OUTPUT"
+          }
+
+          if [ "$ATTEMPT1_RESULT" = "success" ]; then
+            echo "Attempt 1 passed."
+            emit false false '{}'
+            exit 0
+          fi
+          if [ "$ATTEMPT1_RESULT" = "cancelled" ]; then
+            echo "Attempt 1 cancelled; not retrying."
+            emit false true '{}'
+            exit 0
+          fi
+
+          jobs=$(gh api --paginate "/repos/$REPO/actions/runs/$RUN_ID/jobs?per_page=100")
+          failed=$(echo "$jobs" | jq -c '
+            [ .jobs[]
+              | select(.conclusion=="failure")
+              | select(.name | startswith("Attempt 1 /"))
+              | { id, name, steps } ]')
+
+          count=$(echo "$failed" | jq 'length')
+          if [ "$count" -eq 0 ]; then
+            echo "Attempt 1 failed but no per-job failure identified; not retrying."
+            emit false true '{}'
+            exit 0
+          fi
+
+          retry_json='{}'
+          all_retriable=1
+
+          for i in $(seq 0 $((count - 1))); do
+            entry=$(echo "$failed" | jq -c ".[$i]")
+            jid=$(echo   "$entry" | jq -r '.id')
+            jname=$(echo "$entry" | jq -r '.name')
+            child=${jname#"Attempt 1 / "}
+
+            category=""
+            case "$child" in
+              angular\(*\)-screenshots-*)
+                category="testcafe-frameworks-all"
+                ;;
+              *)
+                echo "$child → real failure (not whitelisted for retry)"
+                all_retriable=0
+                continue
+                ;;
+            esac
+
+            # Spot signature: disconnect banner in the log tail, OR last step never
+            # completed (runner died silently, job hit timeout with an in-progress step).
+            spot=0
+            if gh api -H "Accept: application/vnd.github.raw" \
+                 "/repos/$REPO/actions/jobs/$jid/logs" 2>/dev/null \
+                 | tail -400 | grep -q "lost communication with the server"; then
+              spot=1
+            fi
+            if [ $spot -eq 0 ]; then
+              last_status=$(echo     "$entry" | jq -r '.steps[-1].status     // ""')
+              last_conclusion=$(echo "$entry" | jq -r '.steps[-1].conclusion // ""')
+              if [ "$last_status" = "in_progress" ] || [ -z "$last_conclusion" ]; then
+                spot=1
+              fi
+            fi
+
+            if [ $spot -eq 0 ]; then
+              echo "$child → real failure (no spot signature)"
+              all_retriable=0
+              continue
+            fi
+
+            echo "$child → spot-like failure, category=$category"
+            case "$category" in
+              testcafe-frameworks-all)
+                constel=$(echo "$child" | sed -E 's/^(.+)-screenshots-.+$/\1/')
+                theme=$(echo   "$child" | sed -E 's/^.+-screenshots-(.+)$/\1/')
+                retry_json=$(echo "$retry_json" | jq -c \
+                  --arg constel "$constel" --arg theme "$theme" \
+                  '.["testcafe-frameworks-all"].include =
+                    ((.["testcafe-frameworks-all"].include // [])
+                     + [{CONSTEL:$constel, THEME:$theme}])')
+                ;;
+            esac
+          done
+
+          if [ $all_retriable -eq 0 ]; then
+            echo "At least one non-retriable failure; not retrying."
+            emit false true '{}'
+            exit 0
+          fi
+
+          target_count=$(echo "$retry_json" | jq 'to_entries | length')
+          if [ "$target_count" -eq 0 ]; then
+            echo "No retriable targets identified; not retrying."
+            emit false true '{}'
+            exit 0
+          fi
+
+          echo "Attempt 2 will re-run:"
+          echo "$retry_json" | jq .
+          emit true false "$(echo "$retry_json" | jq -c .)"
+
+  attempt2:
+    name: Attempt 2
+    needs: [attempt1, triage]
+    if: needs.triage.outputs.is_retry == 'true'
+    uses: ./.github/workflows/visual-tests-demos.yml
+    with:
+      is_retry: true
+      retry_targets: ${{ needs.triage.outputs.retry_targets }}
+    secrets: inherit
+
+  gate:
+    name: Merge queue gate
+    runs-on: ubuntu-latest
+    needs: [attempt1, triage, attempt2]
+    if: always()
+    timeout-minutes: 5
+    steps:
+      - name: Verdict
+        env:
+          ATTEMPT1: ${{ needs.attempt1.result }}
+          ATTEMPT2: ${{ needs.attempt2.result }}
+          REAL_FAILURE: ${{ needs.triage.outputs.real_failure }}
+        run: |
+          set -e
+          echo "attempt1=$ATTEMPT1  attempt2=$ATTEMPT2  real_failure=$REAL_FAILURE"
+
+          if [ "$REAL_FAILURE" = "true" ]; then
+            echo "Real failure detected in attempt 1."
+            exit 1
+          fi
+          if [ "$ATTEMPT1" = "success" ]; then
+            echo "Attempt 1 passed."
+            exit 0
+          fi
+          if [ "$ATTEMPT2" = "success" ]; then
+            echo "Attempt 1 flaked on spot; attempt 2 recovered."
+            exit 0
+          fi
+          echo "Retry did not recover."
+          exit 1

.github/workflows/visual-tests-demos.yml

@@ -8,7 +8,16 @@ on:
   pull_request:
     paths-ignore:
       - 'apps/**/*.md'
-  merge_group:
+  workflow_call:
+    inputs:
+      is_retry:
+        description: 'Set by the merge-queue wrapper on attempt 2.'
+        type: boolean
+        default: false
+      retry_targets:
+        description: 'JSON map of jobs / matrix entries to re-run on retry.'
+        type: string
+        default: '{}'
   workflow_dispatch:
 
 env:
@@ -142,14 +151,53 @@ jobs:
 
           echo "matrix=$matrix" >> $GITHUB_OUTPUT
 
+  determine-frameworks-all-matrix:
+    runs-on: ubuntu-latest
+    name: Determine testcafe-frameworks-all matrix
+    needs: [check-should-run, determine-framework-tests-scope]
+    if: |
+      always() &&
+      needs.check-should-run.outputs.should-run == 'true' &&
+      needs.determine-framework-tests-scope.result == 'success' &&
+      needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'all'
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Build matrix
+        id: matrix
+        env:
+          IS_RETRY: ${{ inputs.is_retry }}
+          RETRY_TARGETS: ${{ inputs.retry_targets }}
+        run: |
+          set -euo pipefail
+          if [ "$IS_RETRY" = "true" ]; then
+            matrix=$(echo "$RETRY_TARGETS" | jq -c '.["testcafe-frameworks-all"] // {"include":[]}')
+          else
+            matrix=$(cat <<'JSON'
+          {
+            "CONSTEL": [
+              "react(1/3)","react(2/3)","react(3/3)",
+              "vue(1/5)","vue(2/5)","vue(3/5)","vue(4/5)","vue(5/5)",
+              "angular(1/10)","angular(2/10)","angular(3/10)","angular(4/10)","angular(5/10)",
+              "angular(6/10)","angular(7/10)","angular(8/10)","angular(9/10)","angular(10/10)"
+            ],
+            "THEME": ["fluent.blue.light"]
+          }
+          JSON
+          )
+            matrix=$(echo "$matrix" | jq -c .)
+          fi
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+
   build-devextreme:
     runs-on: devextreme-shr2
     name: Build DevExtreme
     needs: [check-should-run, determine-framework-tests-scope]
     if: |
       always() &&
       needs.check-should-run.outputs.should-run == 'true' &&
-      needs.determine-framework-tests-scope.result == 'success'
+      needs.determine-framework-tests-scope.result == 'success' &&
+      inputs.is_retry != true
     env:
       NODE_OPTIONS: --max-old-space-size=8192
     timeout-minutes: 30
@@ -283,7 +331,8 @@ jobs:
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-framework-tests-scope.outputs.framework-tests-scope != 'none' &&
-      needs.build-devextreme.result == 'success'
+      needs.build-devextreme.result == 'success' &&
+      inputs.is_retry != true
     env:
       NODE_OPTIONS: --max-old-space-size=8192
 
@@ -355,7 +404,8 @@ jobs:
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-framework-tests-scope.outputs.framework-tests-scope != 'none' &&
-      needs.build-devextreme.result == 'success'
+      needs.build-devextreme.result == 'success' &&
+      inputs.is_retry != true
 
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -498,7 +548,8 @@ jobs:
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'changed' &&
-      needs.build-devextreme.result == 'success'
+      needs.build-devextreme.result == 'success' &&
+      inputs.is_retry != true
 
     steps:
     - name: Get sources
@@ -611,7 +662,8 @@ jobs:
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'all' &&
-      needs.build-devextreme.result == 'success'
+      needs.build-devextreme.result == 'success' &&
+      inputs.is_retry != true
 
     steps:
     - name: Get sources
@@ -678,7 +730,8 @@ jobs:
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-jquery-test-matrix.result == 'success' &&
-      needs.build-devextreme.result == 'success'
+      needs.build-devextreme.result == 'success' &&
+      inputs.is_retry != true
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.determine-jquery-test-matrix.outputs.matrix) }}
@@ -830,37 +883,20 @@ jobs:
         if-no-files-found: ignore
 
   testcafe-frameworks-all:
-    needs: [check-should-run, determine-framework-tests-scope, build-demos]
+    needs: [check-should-run, determine-framework-tests-scope, build-demos, determine-frameworks-all-matrix]
     if: |
       always() &&
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'all' &&
-      needs.build-demos.result == 'success'
+      needs.determine-frameworks-all-matrix.result == 'success' &&
+      (
+        (inputs.is_retry != true && needs.build-demos.result == 'success') ||
+        (inputs.is_retry == true && fromJson(inputs.retry_targets)['testcafe-frameworks-all'] != null)
+      )
     strategy:
       fail-fast: false
-      matrix:
-        CONSTEL: [
-          react(1/3),
-          react(2/3),
-          react(3/3),
-          vue(1/5),
-          vue(2/5),
-          vue(3/5),
-          vue(4/5),
-          vue(5/5),
-          angular(1/10),
-          angular(2/10),
-          angular(3/10),
-          angular(4/10),
-          angular(5/10),
-          angular(6/10),
-          angular(7/10),
-          angular(8/10),
-          angular(9/10),
-          angular(10/10),
-        ]
-        THEME: ['fluent.blue.light']
+      matrix: ${{ fromJson(needs.determine-frameworks-all-matrix.outputs.matrix) }}
 
     runs-on: devextreme-shr2
     name: ${{ matrix.CONSTEL }}-screenshots-${{ matrix.THEME }}
@@ -976,6 +1012,7 @@ jobs:
         name: screenshots-${{ env.JOB_NAME }}
         path: ${{ github.workspace }}/apps/demos/testing/artifacts/compared-screenshots/**/*
         if-no-files-found: ignore
+        overwrite: true
 
   testcafe-frameworks-changed:
     needs: [check-should-run, determine-framework-tests-scope, build-demos]
@@ -984,7 +1021,8 @@ jobs:
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'changed' &&
-      needs.build-demos.result == 'success'
+      needs.build-demos.result == 'success' &&
+      inputs.is_retry != true
     strategy:
       fail-fast: false
       matrix:
@@ -1144,6 +1182,7 @@ jobs:
           name: screenshots-angular
           pattern: screenshots-angular*
           delete-merged: true
+          overwrite: true
 
       - name: Merge jQuery accessibility reports
         if: needs.testcafe-jquery.result == 'failure'
@@ -1160,7 +1199,8 @@ jobs:
     if: |
       always() &&
       needs.check-should-run.outputs.should-run == 'true' &&
-      needs.build-devextreme.result == 'success'
+      needs.build-devextreme.result == 'success' &&
+      inputs.is_retry != true
     runs-on: devextreme-shr2
     timeout-minutes: 60
     env:
@@ -1235,7 +1275,8 @@ jobs:
       needs.check-should-run.outputs.should-run == 'true' &&
       needs.determine-framework-tests-scope.result == 'success' &&
       needs.determine-framework-tests-scope.outputs.framework-tests-scope != 'none' &&
-      needs.build-devextreme.result == 'success'
+      needs.build-devextreme.result == 'success' &&
+      inputs.is_retry != true
     strategy:
       fail-fast: false
       # Angular AOT bundling is slow, so it is split into parallel shards