diff --git a/.github/workflows/visual-tests-demos-merge-queue.yml b/.github/workflows/visual-tests-demos-merge-queue.yml new file mode 100644 index 000000000000..20f4797f47e7 --- /dev/null +++ b/.github/workflows/visual-tests-demos-merge-queue.yml @@ -0,0 +1,182 @@ +name: Demos Visual Tests (merge queue) + +on: + merge_group: + +concurrency: + group: wf-${{ github.event.merge_group.head_sha || github.sha }}-${{ github.workflow }} + cancel-in-progress: false + +jobs: + attempt1: + name: Attempt 1 + uses: ./.github/workflows/visual-tests-demos.yml + secrets: inherit + + triage: + name: Triage attempt 1 + runs-on: ubuntu-latest + needs: [attempt1] + if: always() + timeout-minutes: 10 + permissions: + actions: read + outputs: + is_retry: ${{ steps.decide.outputs.is_retry }} + real_failure: ${{ steps.decide.outputs.real_failure }} + retry_targets: ${{ steps.decide.outputs.retry_targets }} + steps: + - id: decide + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + ATTEMPT1_RESULT: ${{ needs.attempt1.result }} + run: | + set -euo pipefail + + emit() { + echo "is_retry=$1" >> "$GITHUB_OUTPUT" + echo "real_failure=$2" >> "$GITHUB_OUTPUT" + echo "retry_targets=$3" >> "$GITHUB_OUTPUT" + } + + if [ "$ATTEMPT1_RESULT" = "success" ]; then + echo "Attempt 1 passed." + emit false false '{}' + exit 0 + fi + if [ "$ATTEMPT1_RESULT" = "cancelled" ]; then + echo "Attempt 1 cancelled; not retrying." + emit false true '{}' + exit 0 + fi + + jobs=$(gh api --paginate "/repos/$REPO/actions/runs/$RUN_ID/jobs?per_page=100") + failed=$(echo "$jobs" | jq -c ' + [ .jobs[] + | select(.conclusion=="failure") + | select(.name | startswith("Attempt 1 /")) + | { id, name, steps } ]') + + count=$(echo "$failed" | jq 'length') + if [ "$count" -eq 0 ]; then + echo "Attempt 1 failed but no per-job failure identified; not retrying." + emit false true '{}' + exit 0 + fi + + retry_json='{}' + all_retriable=1 + + for i in $(seq 0 $((count - 1))); do + entry=$(echo "$failed" | jq -c ".[$i]") + jid=$(echo "$entry" | jq -r '.id') + jname=$(echo "$entry" | jq -r '.name') + child=${jname#"Attempt 1 / "} + + category="" + case "$child" in + angular\(*\)-screenshots-*) + category="testcafe-frameworks-all" + ;; + *) + echo "$child → real failure (not whitelisted for retry)" + all_retriable=0 + continue + ;; + esac + + # Spot signature: disconnect banner in the log tail, OR last step never + # completed (runner died silently, job hit timeout with an in-progress step). + spot=0 + if gh api -H "Accept: application/vnd.github.raw" \ + "/repos/$REPO/actions/jobs/$jid/logs" 2>/dev/null \ + | tail -400 | grep -q "lost communication with the server"; then + spot=1 + fi + if [ $spot -eq 0 ]; then + last_status=$(echo "$entry" | jq -r '.steps[-1].status // ""') + last_conclusion=$(echo "$entry" | jq -r '.steps[-1].conclusion // ""') + if [ "$last_status" = "in_progress" ] || [ -z "$last_conclusion" ]; then + spot=1 + fi + fi + + if [ $spot -eq 0 ]; then + echo "$child → real failure (no spot signature)" + all_retriable=0 + continue + fi + + echo "$child → spot-like failure, category=$category" + case "$category" in + testcafe-frameworks-all) + constel=$(echo "$child" | sed -E 's/^(.+)-screenshots-.+$/\1/') + theme=$(echo "$child" | sed -E 's/^.+-screenshots-(.+)$/\1/') + retry_json=$(echo "$retry_json" | jq -c \ + --arg constel "$constel" --arg theme "$theme" \ + '.["testcafe-frameworks-all"].include = + ((.["testcafe-frameworks-all"].include // []) + + [{CONSTEL:$constel, THEME:$theme}])') + ;; + esac + done + + if [ $all_retriable -eq 0 ]; then + echo "At least one non-retriable failure; not retrying." + emit false true '{}' + exit 0 + fi + + target_count=$(echo "$retry_json" | jq 'to_entries | length') + if [ "$target_count" -eq 0 ]; then + echo "No retriable targets identified; not retrying." + emit false true '{}' + exit 0 + fi + + echo "Attempt 2 will re-run:" + echo "$retry_json" | jq . + emit true false "$(echo "$retry_json" | jq -c .)" + + attempt2: + name: Attempt 2 + needs: [attempt1, triage] + if: needs.triage.outputs.is_retry == 'true' + uses: ./.github/workflows/visual-tests-demos.yml + with: + is_retry: true + retry_targets: ${{ needs.triage.outputs.retry_targets }} + secrets: inherit + + gate: + name: Merge queue gate + runs-on: ubuntu-latest + needs: [attempt1, triage, attempt2] + if: always() + timeout-minutes: 5 + steps: + - name: Verdict + env: + ATTEMPT1: ${{ needs.attempt1.result }} + ATTEMPT2: ${{ needs.attempt2.result }} + REAL_FAILURE: ${{ needs.triage.outputs.real_failure }} + run: | + set -e + echo "attempt1=$ATTEMPT1 attempt2=$ATTEMPT2 real_failure=$REAL_FAILURE" + + if [ "$REAL_FAILURE" = "true" ]; then + echo "Real failure detected in attempt 1." + exit 1 + fi + if [ "$ATTEMPT1" = "success" ]; then + echo "Attempt 1 passed." + exit 0 + fi + if [ "$ATTEMPT2" = "success" ]; then + echo "Attempt 1 flaked on spot; attempt 2 recovered." + exit 0 + fi + echo "Retry did not recover." + exit 1 diff --git a/.github/workflows/visual-tests-demos.yml b/.github/workflows/visual-tests-demos.yml index 54c485c1817d..753c6a6e5771 100644 --- a/.github/workflows/visual-tests-demos.yml +++ b/.github/workflows/visual-tests-demos.yml @@ -8,7 +8,16 @@ on: pull_request: paths-ignore: - 'apps/**/*.md' - merge_group: + workflow_call: + inputs: + is_retry: + description: 'Set by the merge-queue wrapper on attempt 2.' + type: boolean + default: false + retry_targets: + description: 'JSON map of jobs / matrix entries to re-run on retry.' + type: string + default: '{}' workflow_dispatch: env: @@ -142,6 +151,44 @@ jobs: echo "matrix=$matrix" >> $GITHUB_OUTPUT + determine-frameworks-all-matrix: + runs-on: ubuntu-latest + name: Determine testcafe-frameworks-all matrix + needs: [check-should-run, determine-framework-tests-scope] + if: | + always() && + needs.check-should-run.outputs.should-run == 'true' && + needs.determine-framework-tests-scope.result == 'success' && + needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'all' + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Build matrix + id: matrix + env: + IS_RETRY: ${{ inputs.is_retry }} + RETRY_TARGETS: ${{ inputs.retry_targets }} + run: | + set -euo pipefail + if [ "$IS_RETRY" = "true" ]; then + matrix=$(echo "$RETRY_TARGETS" | jq -c '.["testcafe-frameworks-all"] // {"include":[]}') + else + matrix=$(cat <<'JSON' + { + "CONSTEL": [ + "react(1/3)","react(2/3)","react(3/3)", + "vue(1/5)","vue(2/5)","vue(3/5)","vue(4/5)","vue(5/5)", + "angular(1/10)","angular(2/10)","angular(3/10)","angular(4/10)","angular(5/10)", + "angular(6/10)","angular(7/10)","angular(8/10)","angular(9/10)","angular(10/10)" + ], + "THEME": ["fluent.blue.light"] + } + JSON + ) + matrix=$(echo "$matrix" | jq -c .) + fi + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + build-devextreme: runs-on: devextreme-shr2 name: Build DevExtreme @@ -149,7 +196,8 @@ jobs: if: | always() && needs.check-should-run.outputs.should-run == 'true' && - needs.determine-framework-tests-scope.result == 'success' + needs.determine-framework-tests-scope.result == 'success' && + inputs.is_retry != true env: NODE_OPTIONS: --max-old-space-size=8192 timeout-minutes: 30 @@ -283,7 +331,8 @@ jobs: needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-framework-tests-scope.outputs.framework-tests-scope != 'none' && - needs.build-devextreme.result == 'success' + needs.build-devextreme.result == 'success' && + inputs.is_retry != true env: NODE_OPTIONS: --max-old-space-size=8192 @@ -355,7 +404,8 @@ jobs: needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-framework-tests-scope.outputs.framework-tests-scope != 'none' && - needs.build-devextreme.result == 'success' + needs.build-devextreme.result == 'success' && + inputs.is_retry != true runs-on: ubuntu-latest timeout-minutes: 60 @@ -498,7 +548,8 @@ jobs: needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'changed' && - needs.build-devextreme.result == 'success' + needs.build-devextreme.result == 'success' && + inputs.is_retry != true steps: - name: Get sources @@ -611,7 +662,8 @@ jobs: needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'all' && - needs.build-devextreme.result == 'success' + needs.build-devextreme.result == 'success' && + inputs.is_retry != true steps: - name: Get sources @@ -678,7 +730,8 @@ jobs: needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-jquery-test-matrix.result == 'success' && - needs.build-devextreme.result == 'success' + needs.build-devextreme.result == 'success' && + inputs.is_retry != true strategy: fail-fast: false matrix: ${{ fromJson(needs.determine-jquery-test-matrix.outputs.matrix) }} @@ -830,37 +883,20 @@ jobs: if-no-files-found: ignore testcafe-frameworks-all: - needs: [check-should-run, determine-framework-tests-scope, build-demos] + needs: [check-should-run, determine-framework-tests-scope, build-demos, determine-frameworks-all-matrix] if: | always() && needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'all' && - needs.build-demos.result == 'success' + needs.determine-frameworks-all-matrix.result == 'success' && + ( + (inputs.is_retry != true && needs.build-demos.result == 'success') || + (inputs.is_retry == true && fromJson(inputs.retry_targets)['testcafe-frameworks-all'] != null) + ) strategy: fail-fast: false - matrix: - CONSTEL: [ - react(1/3), - react(2/3), - react(3/3), - vue(1/5), - vue(2/5), - vue(3/5), - vue(4/5), - vue(5/5), - angular(1/10), - angular(2/10), - angular(3/10), - angular(4/10), - angular(5/10), - angular(6/10), - angular(7/10), - angular(8/10), - angular(9/10), - angular(10/10), - ] - THEME: ['fluent.blue.light'] + matrix: ${{ fromJson(needs.determine-frameworks-all-matrix.outputs.matrix) }} runs-on: devextreme-shr2 name: ${{ matrix.CONSTEL }}-screenshots-${{ matrix.THEME }} @@ -976,6 +1012,7 @@ jobs: name: screenshots-${{ env.JOB_NAME }} path: ${{ github.workspace }}/apps/demos/testing/artifacts/compared-screenshots/**/* if-no-files-found: ignore + overwrite: true testcafe-frameworks-changed: needs: [check-should-run, determine-framework-tests-scope, build-demos] @@ -984,7 +1021,8 @@ jobs: needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-framework-tests-scope.outputs.framework-tests-scope == 'changed' && - needs.build-demos.result == 'success' + needs.build-demos.result == 'success' && + inputs.is_retry != true strategy: fail-fast: false matrix: @@ -1144,6 +1182,7 @@ jobs: name: screenshots-angular pattern: screenshots-angular* delete-merged: true + overwrite: true - name: Merge jQuery accessibility reports if: needs.testcafe-jquery.result == 'failure' @@ -1160,7 +1199,8 @@ jobs: if: | always() && needs.check-should-run.outputs.should-run == 'true' && - needs.build-devextreme.result == 'success' + needs.build-devextreme.result == 'success' && + inputs.is_retry != true runs-on: devextreme-shr2 timeout-minutes: 60 env: @@ -1235,7 +1275,8 @@ jobs: needs.check-should-run.outputs.should-run == 'true' && needs.determine-framework-tests-scope.result == 'success' && needs.determine-framework-tests-scope.outputs.framework-tests-scope != 'none' && - needs.build-devextreme.result == 'success' + needs.build-devextreme.result == 'success' && + inputs.is_retry != true strategy: fail-fast: false # Angular AOT bundling is slow, so it is split into parallel shards