From cc74801810ed588f263f04caf35ccb4350812457 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Fri, 26 Jun 2026 10:53:10 +0530 Subject: [PATCH 01/17] ci(sdk-regression): auto-run on PRs via `run-sdk-regression` label Make SDK regression runnable as an automatic PR check, not only via a manual `RUN_REGRESSION` comment. Adds a `pull_request` trigger gated by the `run-sdk-regression` label; resolves the PR head ref/sha from either event; keeps the comment path and its write/admin permission guard intact. Untrusted head.ref is passed via env (not interpolated into the shell) and is still validated by the existing regex-match step before any downstream workflow is triggered. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 47 +++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 73f86de39..5c57f7187 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -1,7 +1,9 @@ name: SDK Regression on: issue_comment: - types: [created, edited] + types: [created, edited] + pull_request: + types: [labeled, synchronize, reopened] permissions: contents: read jobs: @@ -15,7 +17,14 @@ jobs: contents: read pull-requests: read statuses: write - if: ${{ github.event.issue.pull_request && github.event.comment.body == 'RUN_REGRESSION' }} + # Run when either: + # - a maintainer comments RUN_REGRESSION on a PR (legacy manual path), or + # - the PR carries the `run-sdk-regression` label (auto path / quality gate). + # Both paths are internal-only: only write/admin collaborators can comment-gate + # (enforced by check-access below) or apply a label. + if: >- + (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == 'RUN_REGRESSION') || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) strategy: matrix: # Format: repo@branch (default branch is master) @@ -40,7 +49,10 @@ jobs: - percy-appium-js - gatsby-plugin-percy steps: + # Permission check applies only to the comment path; the label path is + # already gated by GitHub (only write+ collaborators can label a PR). - name: Get user permissions + if: ${{ github.event_name == 'issue_comment' }} uses: actions/github-script@f891eff65186019cbb3f7190c4590bc0a1b76fbc # v4.1.0 id: check-access with: @@ -51,15 +63,34 @@ jobs: return data.permission; result-encoding: string - name: Check Access Level - if: steps.check-access.outputs.result != 'write' && steps.check-access.outputs.result != 'admin' + if: ${{ github.event_name == 'issue_comment' && steps.check-access.outputs.result != 'write' && steps.check-access.outputs.result != 'admin' }} run: exit 1 - uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 - if: ${{ github.event.issue.pull_request }} + if: ${{ github.event_name == 'issue_comment' }} id: comment-branch + # Pass event data via env (never interpolate untrusted head.ref into the + # shell directly). The ref is validated by the regex-match step below + # before it is ever used to trigger a downstream workflow. + - name: Resolve PR head ref and sha + id: pr + env: + EVENT_NAME: ${{ github.event_name }} + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + COMMENT_HEAD_REF: ${{ steps.comment-branch.outputs.head_ref }} + COMMENT_HEAD_SHA: ${{ steps.comment-branch.outputs.head_sha }} + run: | + if [ "$EVENT_NAME" = "pull_request" ]; then + echo "head_ref=$PR_HEAD_REF" >> "$GITHUB_OUTPUT" + echo "head_sha=$PR_HEAD_SHA" >> "$GITHUB_OUTPUT" + else + echo "head_ref=$COMMENT_HEAD_REF" >> "$GITHUB_OUTPUT" + echo "head_sha=$COMMENT_HEAD_SHA" >> "$GITHUB_OUTPUT" + fi - uses: actions-ecosystem/action-regex-match@9e6c4fb3d5e898f505be7a1fb6e7b0a278f6665b # v2.0.2 id: regex-match with: - text: ${{ steps.comment-branch.outputs.head_ref }} + text: ${{ steps.pr.outputs.head_ref }} regex: '^[a-zA-Z0-9_/\-]+$' - name: Break on invalid branch name run: exit 1 @@ -78,7 +109,7 @@ jobs: github-token: ${{ secrets.WORKFLOW_DISPATCH_ACTIONS_TOKEN }} script: | const { owner, repo } = context.repo; - const sha = '${{ steps.comment-branch.outputs.head_sha }}' + const sha = '${{ steps.pr.outputs.head_sha }}' const state = 'pending'; const target_url = '${{ steps.job-url.outputs.html_url }}' const check_name = 'SDK Regression ${{ matrix.repo }}' @@ -105,7 +136,7 @@ jobs: github_token: ${{ secrets.WORKFLOW_DISPATCH_ACTIONS_TOKEN }} workflow_file_name: test.yml ref: ${{ steps.split.outputs._1 || 'master' }} - client_payload: '{ "branch": "${{ steps.comment-branch.outputs.head_ref }}"}' + client_payload: '{ "branch": "${{ steps.pr.outputs.head_ref }}"}' wait_interval: 15 - name: Update Status uses: actions/github-script@f891eff65186019cbb3f7190c4590bc0a1b76fbc # v4.1.0 @@ -113,7 +144,7 @@ jobs: github-token: ${{ secrets.WORKFLOW_DISPATCH_ACTIONS_TOKEN }} script: | const { owner, repo } = context.repo; - const sha = '${{ steps.comment-branch.outputs.head_sha }}' + const sha = '${{ steps.pr.outputs.head_sha }}' const state = '${{ steps.reg-test.outcome }}'; const target_url = '${{ steps.job-url.outputs.html_url }}' const check_name = 'SDK Regression ${{ matrix.repo }}' From 0a6470ff194159ad25e4d1cf2dbebe908bfc657c Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 01:08:20 +0530 Subject: [PATCH 02/17] ci(sdk-regression): also fan out App Percy + POA on Buildkite The same regression trigger (RUN_REGRESSION comment or run-sdk-regression label) now also fires the App Percy + POA suites, which run on Buildkite (real BrowserStack devices/browsers). A new trigger-app-poa job repository_dispatches to percy/percy-automation, whose workflow creates the Buildkite builds against this CLI branch. percy-automation remains the single owner of App/POA-on-Buildkite; this is just the trigger. Internal-only guard (write/admin or label) and env-based, regex-validated branch handling mirror the web job. Requires a PERCY_AUTOMATION_DISPATCH_TOKEN secret. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 76 +++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 5c57f7187..e121c6367 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -148,7 +148,7 @@ jobs: const state = '${{ steps.reg-test.outcome }}'; const target_url = '${{ steps.job-url.outputs.html_url }}' const check_name = 'SDK Regression ${{ matrix.repo }}' - + github.repos.createCommitStatus({ context: check_name, owner, @@ -157,3 +157,77 @@ jobs: state, target_url }); + + # The same regression also fans out to the App Percy + POA suites, which run + # on Buildkite (real BrowserStack devices/browsers). We don't run them here โ€” + # we repository_dispatch to percy/percy-automation, whose workflow turns this + # into Buildkite builds against this CLI branch. percy-automation stays the + # single owner of App/POA-on-Buildkite; this job is just the trigger. + trigger-app-poa: + name: trigger App Percy + POA (Buildkite) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + if: >- + (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == 'RUN_REGRESSION') || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) + steps: + # Same internal-only guard as the web job: comment path must be write/admin. + - name: Check access (comment path only) + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/github-script@f891eff65186019cbb3f7190c4590bc0a1b76fbc # v4.1.0 + with: + script: | + const { owner, repo } = context.repo; + const { login } = context.payload.comment.user; + const { data } = await github.repos.getCollaboratorPermissionLevel({ owner, repo, username: login }); + if (data.permission !== 'write' && data.permission !== 'admin') { + core.setFailed(`@${login} lacks write access for regression`); + } + - uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 + if: ${{ github.event_name == 'issue_comment' }} + id: comment-branch + # Untrusted head.ref flows via env (not interpolated into the shell) and + # is regex-validated before it reaches the dispatch payload. + - name: Resolve PR head ref + id: pr + env: + EVENT_NAME: ${{ github.event_name }} + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + COMMENT_HEAD_REF: ${{ steps.comment-branch.outputs.head_ref }} + run: | + if [ "$EVENT_NAME" = "pull_request" ]; then + echo "head_ref=$PR_HEAD_REF" >> "$GITHUB_OUTPUT" + else + echo "head_ref=$COMMENT_HEAD_REF" >> "$GITHUB_OUTPUT" + fi + - uses: actions-ecosystem/action-regex-match@9e6c4fb3d5e898f505be7a1fb6e7b0a278f6665b # v2.0.2 + id: regex-match + with: + text: ${{ steps.pr.outputs.head_ref }} + regex: '^[a-zA-Z0-9_/\-]+$' + - name: Break on invalid branch name + if: ${{ steps.regex-match.outputs.match == '' }} + run: exit 1 + - name: Dispatch App Percy + POA regression (Buildkite) to percy-automation + env: + # Token (PAT / GitHub App) allowed to create a repository_dispatch on + # percy/percy-automation. That repo's workflow creates the Buildkite + # builds on the app-percy + poa SDK regression suites. + DISPATCH_TOKEN: ${{ secrets.PERCY_AUTOMATION_DISPATCH_TOKEN }} + HEAD_REF: ${{ steps.pr.outputs.head_ref }} + run: | + set -euo pipefail + if [ -z "${DISPATCH_TOKEN:-}" ]; then + echo "::error::PERCY_AUTOMATION_DISPATCH_TOKEN secret is not set"; exit 1 + fi + payload=$(jq -n --arg b "$HEAD_REF" \ + '{event_type:"sdk-regression", client_payload:{cli_branch:$b, suites:"both"}}') + curl -sSf -X POST \ + -H "Authorization: Bearer $DISPATCH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/percy/percy-automation/dispatches \ + -d "$payload" + echo "Dispatched App Percy + POA regression for CLI branch: $HEAD_REF" From 7509fbfc2ad0c9f7a01ad6eaacd55272734e706f Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 01:13:07 +0530 Subject: [PATCH 03/17] ci(sdk-regression): trigger App/POA on Buildkite directly + report table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the percy-automation repository_dispatch hop with a direct Buildkite REST call: the app-poa-regression job creates builds on the app-percy and poa SDK regression suites (CLI built from this branch), polls them to completion, and upserts a per-SDK pass/fail table comment on the PR. - Direct Buildkite trigger (BUILDKITE_API_TOKEN in this repo) โ€” no extra repo hop. - Waits for the builds (bounded by MAX_WAIT_MIN), then posts/edits a marker comment with each suite's per-job result + build links; fails the job if any job failed/canceled. - Internal-only guard + env-based, regex-validated branch handling unchanged. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 114 +++++++++++++++++++++------ 1 file changed, 88 insertions(+), 26 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index e121c6367..a8bd8768a 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -158,17 +158,17 @@ jobs: target_url }); - # The same regression also fans out to the App Percy + POA suites, which run - # on Buildkite (real BrowserStack devices/browsers). We don't run them here โ€” - # we repository_dispatch to percy/percy-automation, whose workflow turns this - # into Buildkite builds against this CLI branch. percy-automation stays the - # single owner of App/POA-on-Buildkite; this job is just the trigger. - trigger-app-poa: - name: trigger App Percy + POA (Buildkite) + # App Percy + POA run on Buildkite (real BrowserStack devices/browsers). This + # job triggers those suites DIRECTLY via the Buildkite REST API against this + # CLI branch, waits for the builds to finish, and posts a per-SDK pass/fail + # table back onto the PR. + app-poa-regression: + name: App Percy + POA regression (Buildkite) runs-on: ubuntu-latest permissions: contents: read - pull-requests: read + issues: write # upsert the result comment (PR comments = issue comments) + pull-requests: write if: >- (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == 'RUN_REGRESSION') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) @@ -189,18 +189,22 @@ jobs: if: ${{ github.event_name == 'issue_comment' }} id: comment-branch # Untrusted head.ref flows via env (not interpolated into the shell) and - # is regex-validated before it reaches the dispatch payload. - - name: Resolve PR head ref + # is regex-validated before it reaches the Buildkite payload. + - name: Resolve PR head ref + number id: pr env: EVENT_NAME: ${{ github.event_name }} PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_NUMBER_PR: ${{ github.event.pull_request.number }} + PR_NUMBER_ISSUE: ${{ github.event.issue.number }} COMMENT_HEAD_REF: ${{ steps.comment-branch.outputs.head_ref }} run: | if [ "$EVENT_NAME" = "pull_request" ]; then echo "head_ref=$PR_HEAD_REF" >> "$GITHUB_OUTPUT" + echo "pr_number=$PR_NUMBER_PR" >> "$GITHUB_OUTPUT" else echo "head_ref=$COMMENT_HEAD_REF" >> "$GITHUB_OUTPUT" + echo "pr_number=$PR_NUMBER_ISSUE" >> "$GITHUB_OUTPUT" fi - uses: actions-ecosystem/action-regex-match@9e6c4fb3d5e898f505be7a1fb6e7b0a278f6665b # v2.0.2 id: regex-match @@ -210,24 +214,82 @@ jobs: - name: Break on invalid branch name if: ${{ steps.regex-match.outputs.match == '' }} run: exit 1 - - name: Dispatch App Percy + POA regression (Buildkite) to percy-automation + - name: Run App Percy + POA on Buildkite, wait, and report env: - # Token (PAT / GitHub App) allowed to create a repository_dispatch on - # percy/percy-automation. That repo's workflow creates the Buildkite - # builds on the app-percy + poa SDK regression suites. - DISPATCH_TOKEN: ${{ secrets.PERCY_AUTOMATION_DISPATCH_TOKEN }} + BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} # scope: write_builds + read_builds + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.pr.outputs.pr_number }} HEAD_REF: ${{ steps.pr.outputs.head_ref }} + BK_ORG: percy + SUITES: "app-percy-sdk-regression-suite poa-sdk-regression-suite" + POLL_INTERVAL: "30" + MAX_WAIT_MIN: "90" run: | set -euo pipefail - if [ -z "${DISPATCH_TOKEN:-}" ]; then - echo "::error::PERCY_AUTOMATION_DISPATCH_TOKEN secret is not set"; exit 1 + [ -n "${BUILDKITE_API_TOKEN:-}" ] || { echo "::error::BUILDKITE_API_TOKEN secret is not set"; exit 1; } + api="https://api.buildkite.com/v2/organizations/$BK_ORG" + auth="Authorization: Bearer $BUILDKITE_API_TOKEN" + nums=(); slugs=(); urls=() + + # 1) Create one build per suite, CLI built from this branch. + for slug in $SUITES; do + body=$(jq -n --arg b "$HEAD_REF" --arg m ":robot: CLI regression for $HEAD_REF (PR #$PR_NUMBER)" \ + '{commit:"HEAD", branch:"main", message:$m, env:{CLI_BRANCH_NAME:$b, CLI_PACKAGE_TYPE:"repo"}}') + resp=$(curl -sSf -X POST -H "$auth" -H "Content-Type: application/json" "$api/pipelines/$slug/builds" -d "$body") + nums+=("$(echo "$resp" | jq -r '.number')") + slugs+=("$slug") + urls+=("$(echo "$resp" | jq -r '.web_url')") + echo "Created $slug build #${nums[-1]}: ${urls[-1]}" + done + + # 2) Poll until every build reaches a terminal state (or we time out). + terminal=" passed failed canceled skipped not_run blocked " + deadline=$(( $(date +%s) + MAX_WAIT_MIN * 60 )) + while :; do + done_all=1 + for i in "${!nums[@]}"; do + st=$(curl -sSf -H "$auth" "$api/pipelines/${slugs[$i]}/builds/${nums[$i]}" | jq -r '.state') + case "$terminal" in *" $st "*) ;; *) done_all=0 ;; esac + done + [ "$done_all" = 1 ] && break + if [ "$(date +%s)" -ge "$deadline" ]; then echo "::warning::timed out after ${MAX_WAIT_MIN}m; reporting partial"; break; fi + sleep "$POLL_INTERVAL" + done + + # 3) Build the per-SDK result table. + { + echo "" + echo "## ๐Ÿค– SDK Regression โ€” App Percy + POA" + echo "" + echo "CLI branch \`$HEAD_REF\` (built from source) ยท web-SDK results post as commit statuses." + echo "" + echo "| Suite | SDK job | Result |" + echo "|---|---|---|" + } > comment.md + overall_fail=0 + for i in "${!nums[@]}"; do + jobs=$(curl -sSf -H "$auth" "$api/pipelines/${slugs[$i]}/builds/${nums[$i]}") + echo "$jobs" | jq -r --arg s "${slugs[$i]}" ' + .jobs[] | select(.type=="script" and .name != null) | + "| \($s) | \(.name) | " + (if .state=="passed" then "โœ… pass" else "โŒ " + .state end) + " |"' >> comment.md + if echo "$jobs" | jq -e '.jobs[] | select(.type=="script") | select(.state=="failed" or .state=="canceled")' >/dev/null; then + overall_fail=1 + fi + done + { + echo "" + for i in "${!nums[@]}"; do echo "- [\`${slugs[$i]}\` build #${nums[$i]}](${urls[$i]})"; done + } >> comment.md + + # 4) Upsert the marker comment on the PR (create, or edit the previous one). + jq -n --rawfile b comment.md '{body:$b}' > payload.json + existing=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" --paginate \ + --jq '.[] | select(.body | startswith("")) | .id' | head -n1 || true) + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$existing" --input payload.json >/dev/null + else + gh api -X POST "repos/$REPO/issues/$PR_NUMBER/comments" --input payload.json >/dev/null fi - payload=$(jq -n --arg b "$HEAD_REF" \ - '{event_type:"sdk-regression", client_payload:{cli_branch:$b, suites:"both"}}') - curl -sSf -X POST \ - -H "Authorization: Bearer $DISPATCH_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/percy/percy-automation/dispatches \ - -d "$payload" - echo "Dispatched App Percy + POA regression for CLI branch: $HEAD_REF" + echo "Reported App/POA results to PR #$PR_NUMBER" + [ "$overall_fail" = 0 ] || { echo "::error::App/POA regression had failures"; exit 1; } From 16b09d0b85046888ae6c6c72ca089f310ad072ef Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 01:15:58 +0530 Subject: [PATCH 04/17] ci(sdk-regression): filter bootstrap/wait jobs from the result table Grounded against real builds of app-percy-sdk-regression-suite: the matrix jobs are named per SDK+device (e.g. 'Python-Android [...]'), but the build also has the bootstrap upload step ('App-Percy-SDK-tests'/'POA-SDK-tests') and an unnamed wait job. Exclude both from the per-SDK pass/fail table and the failure check. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index a8bd8768a..d16e9af10 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -268,12 +268,20 @@ jobs: echo "|---|---|---|" } > comment.md overall_fail=0 + # The bootstrap step that uploads the matrix is named after the pipeline + # ("App-Percy-SDK-tests" / "POA-SDK-tests"); exclude it and the unnamed + # wait job so the table is only the real per-SDK device jobs. for i in "${!nums[@]}"; do jobs=$(curl -sSf -H "$auth" "$api/pipelines/${slugs[$i]}/builds/${nums[$i]}") echo "$jobs" | jq -r --arg s "${slugs[$i]}" ' - .jobs[] | select(.type=="script" and .name != null) | - "| \($s) | \(.name) | " + (if .state=="passed" then "โœ… pass" else "โŒ " + .state end) + " |"' >> comment.md - if echo "$jobs" | jq -e '.jobs[] | select(.type=="script") | select(.state=="failed" or .state=="canceled")' >/dev/null; then + .jobs[] + | select(.type=="script" and .name != null and .name != "") + | select(.name != "App-Percy-SDK-tests" and .name != "POA-SDK-tests") + | "| \($s) | \(.name) | " + (if .state=="passed" then "โœ… pass" else "โŒ " + (.state // "n/a") end) + " |"' >> comment.md + if echo "$jobs" | jq -e ' + .jobs[] + | select(.type=="script" and .name != "App-Percy-SDK-tests" and .name != "POA-SDK-tests") + | select(.state=="failed" or .state=="canceled")' >/dev/null; then overall_fail=1 fi done From 087f919bb8208e2b34076e3781d1249d51ddbc02 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 01:23:25 +0530 Subject: [PATCH 05/17] ci(sdk-regression): add 5 injection-capable SDKs to the fan-out matrix detox, playwright-python, robotframework, playwright-java, playwright-dotnet all support CLI-branch injection in their test.yml but were missing from the matrix, so a CLI change silently skipped them. Added as @main (their default branch) since the split default ref is master. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index d16e9af10..99f596c0f 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -48,6 +48,14 @@ jobs: - percy-capybara - percy-appium-js - gatsby-plugin-percy + # Injection-capable SDKs previously missing from the fan-out. + # Their default branch is `main`, so pin the dispatch ref with @main + # (the split default below is `master`, which would 404 for these). + - percy-detox@main + - percy-playwright-python@main + - percy-robotframework@main + - percy-playwright-java@main + - percy-playwright-dotnet@main steps: # Permission check applies only to the comment path; the label path is # already gated by GitHub (only write+ collaborators can label a PR). From 957c415b39b09c8553594ef4c396ea080c05797e Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 02:17:05 +0530 Subject: [PATCH 06/17] ci(sdk-regression): add workflow_dispatch to run the web fan-out on demand Lets the SDK regression matrix be triggered manually (and on the PR's own branch) against a chosen CLI branch, without a comment/label. The Buildkite App/POA job stays comment/label-only, so a dispatch tests the web fan-out only. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 99f596c0f..40b13fc33 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -4,6 +4,12 @@ on: types: [created, edited] pull_request: types: [labeled, synchronize, reopened] + workflow_dispatch: + inputs: + branch: + description: CLI branch to run all SDK regression against + required: false + default: master permissions: contents: read jobs: @@ -24,7 +30,8 @@ jobs: # (enforced by check-access below) or apply a label. if: >- (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == 'RUN_REGRESSION') || - (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) || + (github.event_name == 'workflow_dispatch') strategy: matrix: # Format: repo@branch (default branch is master) @@ -87,8 +94,13 @@ jobs: PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} COMMENT_HEAD_REF: ${{ steps.comment-branch.outputs.head_ref }} COMMENT_HEAD_SHA: ${{ steps.comment-branch.outputs.head_sha }} + DISPATCH_BRANCH: ${{ github.event.inputs.branch }} + DISPATCH_SHA: ${{ github.sha }} run: | - if [ "$EVENT_NAME" = "pull_request" ]; then + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + echo "head_ref=$DISPATCH_BRANCH" >> "$GITHUB_OUTPUT" + echo "head_sha=$DISPATCH_SHA" >> "$GITHUB_OUTPUT" + elif [ "$EVENT_NAME" = "pull_request" ]; then echo "head_ref=$PR_HEAD_REF" >> "$GITHUB_OUTPUT" echo "head_sha=$PR_HEAD_SHA" >> "$GITHUB_OUTPUT" else From 3afc900eee14210619b461e7c0115a52a246af36 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 02:18:37 +0530 Subject: [PATCH 07/17] ci(sdk-regression): set fail-fast: false on the SDK matrix A regression fan-out must report every SDK's result; with default fail-fast the first SDK failure cancels all other matrix jobs, hiding the rest. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 40b13fc33..351b0ae26 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -33,6 +33,9 @@ jobs: (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) || (github.event_name == 'workflow_dispatch') strategy: + # Don't let one SDK's failure cancel the rest โ€” a regression run must + # report every SDK's result, not abort on the first failure. + fail-fast: false matrix: # Format: repo@branch (default branch is master) repo: From ebdff3b16f0fb31c0d9a3d5a6f3795a2068b341e Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 02:51:40 +0530 Subject: [PATCH 08/17] ci(sdk-regression): dispatch percy-storybook's versioned test workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit storybook has no test.yml โ€” it uses test-storybook-vN.yml โ€” so the fan-out silently failed to trigger it. Dispatch test-storybook-v10.yml for storybook. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 351b0ae26..4c1f5329a 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -157,7 +157,9 @@ jobs: owner: percy repo: ${{ steps.split.outputs._0 }} github_token: ${{ secrets.WORKFLOW_DISPATCH_ACTIONS_TOKEN }} - workflow_file_name: test.yml + # Most SDKs expose `test.yml`; percy-storybook uses versioned + # `test-storybook-vN.yml` files (latest = v10), so dispatch that one. + workflow_file_name: ${{ steps.split.outputs._0 == 'percy-storybook' && 'test-storybook-v10.yml' || 'test.yml' }} ref: ${{ steps.split.outputs._1 || 'master' }} client_payload: '{ "branch": "${{ steps.pr.outputs.head_ref }}"}' wait_interval: 15 From 0f330e689257c220e162e2dca14a602ae381c1dd Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 14:41:00 +0530 Subject: [PATCH 09/17] ci(sdk-regression): add appium-dotnet + styleguidist to the matrix Both now support CLI-branch injection (appium-dotnet#403, styleguidist#25). Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 4c1f5329a..d59492a24 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -66,6 +66,8 @@ jobs: - percy-robotframework@main - percy-playwright-java@main - percy-playwright-dotnet@main + - percy-appium-dotnet@main + - percy-styleguidist@main steps: # Permission check applies only to the comment path; the label path is # already gated by GitHub (only write+ collaborators can label a PR). From 9cb2da6b20632b9a2e667cbdc444ada92c9b9592 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 15:00:42 +0530 Subject: [PATCH 10/17] ci(sdk-regression): add App Percy + remaining SDKs to the matrix Adds appium-python/java/wd/ruby, maestro-web/app, react-native-app, tosca-dotnet, uipath, xcui-swift. react-native-app uses storybook-rn-ci.yml (per-repo workflow filename). Skips puppeteer/ember (per decision) and katalon/espresso (infeasible / needs emulator). Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index d59492a24..1c228d592 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -68,6 +68,17 @@ jobs: - percy-playwright-dotnet@main - percy-appium-dotnet@main - percy-styleguidist@main + # App Percy + remaining SDKs (default branch noted; @main where not master) + - percy-appium-python + - percy-appium-java + - percy-appium-wd + - percy-appium-ruby@main + - percy-maestro-web@main + - percy-maestro-app@main + - percy-react-native-app@main + - percy-tosca-dotnet@main + - percy-uipath@main + - percy-xcui-swift@main steps: # Permission check applies only to the comment path; the label path is # already gated by GitHub (only write+ collaborators can label a PR). @@ -161,7 +172,7 @@ jobs: github_token: ${{ secrets.WORKFLOW_DISPATCH_ACTIONS_TOKEN }} # Most SDKs expose `test.yml`; percy-storybook uses versioned # `test-storybook-vN.yml` files (latest = v10), so dispatch that one. - workflow_file_name: ${{ steps.split.outputs._0 == 'percy-storybook' && 'test-storybook-v10.yml' || 'test.yml' }} + workflow_file_name: ${{ steps.split.outputs._0 == 'percy-storybook' && 'test-storybook-v10.yml' || (steps.split.outputs._0 == 'percy-react-native-app' && 'storybook-rn-ci.yml' || 'test.yml') }} ref: ${{ steps.split.outputs._1 || 'master' }} client_payload: '{ "branch": "${{ steps.pr.outputs.head_ref }}"}' wait_interval: 15 From e37d03e99f6a3cc0e137d6ea6b8811900ee086ef Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 15:01:24 +0530 Subject: [PATCH 11/17] ci(sdk-regression): exclude percy-ember and percy-puppeteer Their tests assert on the PER-7348 readiness-gate contract and fail against an ahead-of-release cli@master until they adapt + bump @percy/sdk-utils. Skip per decision. Part of PER-9772. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 1c228d592..673a7c1a6 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -39,9 +39,10 @@ jobs: matrix: # Format: repo@branch (default branch is master) repo: - - percy-ember + # percy-ember and percy-puppeteer intentionally excluded: their tests + # assert on the PER-7348 readiness-gate contract and fail against an + # ahead-of-release cli@master until they adapt + bump @percy/sdk-utils. - percy-cypress - - percy-puppeteer - percy-storybook - percy-playwright - percy-testcafe From c3593c62bf20c641128ddca7f7137f0b49c5c3d6 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Mon, 29 Jun 2026 20:11:17 +0530 Subject: [PATCH 12/17] ci(sdk-regression): paginate job-id lookup + don't gate on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fan-out matrix has grown past 30 jobs (currently 35). The `Get Current Job Log URL` step (Tiryoh/gha-jobid-action) defaults to per_page=30, so every job on page 2 fails to find itself, resolves job_id to null, and exits 1 *before* dispatching the SDK workflow โ€” producing false reds (appium-js, maestro-app, maestro-web, selenium-ruby) that never actually ran a regression. Set per_page=100 (jobs API max) to cover the whole matrix, and mark the step continue-on-error since it only feeds the commit-status target_url and must never gate the regression itself. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 673a7c1a6..b49f21dbc 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -136,9 +136,16 @@ jobs: - name: Get Current Job Log URL uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 id: job-url + # This step only resolves a target_url for the commit status; a lookup + # failure must never gate the regression itself. + continue-on-error: true with: github_token: ${{ secrets.WORKFLOW_DISPATCH_ACTIONS_TOKEN }} - job_name: "regression (${{ matrix.repo }})" + job_name: "regression (${{ matrix.repo }})" + # The fan-out matrix is >30 jobs (currently 35); the action defaults to + # per_page=30, so jobs on page 2 resolve to null and exit 1. Cover the + # whole matrix (GitHub jobs API max page size is 100). + per_page: 100 - name: Output Current Job Log URL run: echo ${{ steps.jobs.outputs.html_url }} - uses: actions/github-script@f891eff65186019cbb3f7190c4590bc0a1b76fbc # v4.1.0 From 01edff7b46da257a0a5f47784718fec944512c32 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Tue, 30 Jun 2026 12:12:34 +0530 Subject: [PATCH 13/17] ci(sdk-regression): re-add percy-ember and percy-puppeteer to matrix Both were excluded because their tests assert on the PER-7348 readiness-gate contract and red against an ahead-of-release cli@master. Re-adding them as-is: both have a workflow_dispatch trigger on their default branch, so they dispatch and run. Expected to red on master until they adapt to the two-call readiness contract + bump @percy/sdk-utils; we'll fix the reds as they surface. Matrix is now 36 SDKs (job-id lookup already paginated to per_page=100). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index b49f21dbc..678a31806 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -38,10 +38,13 @@ jobs: fail-fast: false matrix: # Format: repo@branch (default branch is master) - repo: - # percy-ember and percy-puppeteer intentionally excluded: their tests - # assert on the PER-7348 readiness-gate contract and fail against an + repo: + # NOTE: percy-ember and percy-puppeteer assert on the PER-7348 + # readiness-gate contract, so they are expected to RED against an # ahead-of-release cli@master until they adapt + bump @percy/sdk-utils. + # Kept in the matrix intentionally (run them, fix the reds as they come). + - percy-ember + - percy-puppeteer - percy-cypress - percy-storybook - percy-playwright From 4211a680e2c370f076e0d0af8228f1a64411e34f Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Tue, 30 Jun 2026 12:22:54 +0530 Subject: [PATCH 14/17] ci(sdk-regression): dispatch ci.yml for tosca-dotnet and uipath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit percy-tosca-dotnet and percy-uipath have no test.yml โ€” their @percy/cli inject step lives in ci.yml (workflow_dispatch + branch input + "Set up @percy/cli from git" cloning the injected branch are all present there). The orchestrator was dispatching test.yml, so both 404'd at the trigger step โ€” previously mis-attributed to a token-access gap. Map both to ci.yml in the workflow_file_name selector so the fan-out reaches their real (correct) inject workflow. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sdk-regression.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 678a31806..1fb2c8c46 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -181,9 +181,12 @@ jobs: owner: percy repo: ${{ steps.split.outputs._0 }} github_token: ${{ secrets.WORKFLOW_DISPATCH_ACTIONS_TOKEN }} - # Most SDKs expose `test.yml`; percy-storybook uses versioned - # `test-storybook-vN.yml` files (latest = v10), so dispatch that one. - workflow_file_name: ${{ steps.split.outputs._0 == 'percy-storybook' && 'test-storybook-v10.yml' || (steps.split.outputs._0 == 'percy-react-native-app' && 'storybook-rn-ci.yml' || 'test.yml') }} + # Most SDKs expose `test.yml`; a few use a differently-named workflow + # that carries the @percy/cli inject step: + # percy-storybook -> versioned test-storybook-vN.yml (latest v10) + # percy-react-native-app -> storybook-rn-ci.yml + # percy-tosca-dotnet / percy-uipath -> ci.yml (they have no test.yml) + workflow_file_name: ${{ steps.split.outputs._0 == 'percy-storybook' && 'test-storybook-v10.yml' || (steps.split.outputs._0 == 'percy-react-native-app' && 'storybook-rn-ci.yml' || ((steps.split.outputs._0 == 'percy-tosca-dotnet' || steps.split.outputs._0 == 'percy-uipath') && 'ci.yml' || 'test.yml')) }} ref: ${{ steps.split.outputs._1 || 'master' }} client_payload: '{ "branch": "${{ steps.pr.outputs.head_ref }}"}' wait_interval: 15 From 879314ced12673083a8ad589eeb03f599143cd89 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Fri, 3 Jul 2026 10:56:09 +0530 Subject: [PATCH 15/17] ci(sdk-regression): drop percy-robotframework and percy-nightmare from matrix percy-robotframework is archived (read-only) so workflow dispatch always 404s; percy-nightmare's downstream run hangs until the 6h job limit. Co-Authored-By: Claude Fable 5 --- .github/workflows/sdk-regression.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 1fb2c8c46..4f6c9c787 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -50,7 +50,6 @@ jobs: - percy-playwright - percy-testcafe - percy-nightwatch - - percy-nightmare - percy-webdriverio - percy-webdriverio@v2 - percy-protractor @@ -67,7 +66,6 @@ jobs: # (the split default below is `master`, which would 404 for these). - percy-detox@main - percy-playwright-python@main - - percy-robotframework@main - percy-playwright-java@main - percy-playwright-dotnet@main - percy-appium-dotnet@main From fb74987b55c762d27da6a8c6abed64f2694a2225 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Fri, 3 Jul 2026 10:57:24 +0530 Subject: [PATCH 16/17] ci(sdk-regression): remove App Percy + POA Buildkite job Co-Authored-By: Claude Fable 5 --- .github/workflows/sdk-regression.yml | 144 --------------------------- 1 file changed, 144 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 4f6c9c787..2201b5eb4 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -207,147 +207,3 @@ jobs: state, target_url }); - - # App Percy + POA run on Buildkite (real BrowserStack devices/browsers). This - # job triggers those suites DIRECTLY via the Buildkite REST API against this - # CLI branch, waits for the builds to finish, and posts a per-SDK pass/fail - # table back onto the PR. - app-poa-regression: - name: App Percy + POA regression (Buildkite) - runs-on: ubuntu-latest - permissions: - contents: read - issues: write # upsert the result comment (PR comments = issue comments) - pull-requests: write - if: >- - (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == 'RUN_REGRESSION') || - (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) - steps: - # Same internal-only guard as the web job: comment path must be write/admin. - - name: Check access (comment path only) - if: ${{ github.event_name == 'issue_comment' }} - uses: actions/github-script@f891eff65186019cbb3f7190c4590bc0a1b76fbc # v4.1.0 - with: - script: | - const { owner, repo } = context.repo; - const { login } = context.payload.comment.user; - const { data } = await github.repos.getCollaboratorPermissionLevel({ owner, repo, username: login }); - if (data.permission !== 'write' && data.permission !== 'admin') { - core.setFailed(`@${login} lacks write access for regression`); - } - - uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 - if: ${{ github.event_name == 'issue_comment' }} - id: comment-branch - # Untrusted head.ref flows via env (not interpolated into the shell) and - # is regex-validated before it reaches the Buildkite payload. - - name: Resolve PR head ref + number - id: pr - env: - EVENT_NAME: ${{ github.event_name }} - PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} - PR_NUMBER_PR: ${{ github.event.pull_request.number }} - PR_NUMBER_ISSUE: ${{ github.event.issue.number }} - COMMENT_HEAD_REF: ${{ steps.comment-branch.outputs.head_ref }} - run: | - if [ "$EVENT_NAME" = "pull_request" ]; then - echo "head_ref=$PR_HEAD_REF" >> "$GITHUB_OUTPUT" - echo "pr_number=$PR_NUMBER_PR" >> "$GITHUB_OUTPUT" - else - echo "head_ref=$COMMENT_HEAD_REF" >> "$GITHUB_OUTPUT" - echo "pr_number=$PR_NUMBER_ISSUE" >> "$GITHUB_OUTPUT" - fi - - uses: actions-ecosystem/action-regex-match@9e6c4fb3d5e898f505be7a1fb6e7b0a278f6665b # v2.0.2 - id: regex-match - with: - text: ${{ steps.pr.outputs.head_ref }} - regex: '^[a-zA-Z0-9_/\-]+$' - - name: Break on invalid branch name - if: ${{ steps.regex-match.outputs.match == '' }} - run: exit 1 - - name: Run App Percy + POA on Buildkite, wait, and report - env: - BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} # scope: write_builds + read_builds - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - PR_NUMBER: ${{ steps.pr.outputs.pr_number }} - HEAD_REF: ${{ steps.pr.outputs.head_ref }} - BK_ORG: percy - SUITES: "app-percy-sdk-regression-suite poa-sdk-regression-suite" - POLL_INTERVAL: "30" - MAX_WAIT_MIN: "90" - run: | - set -euo pipefail - [ -n "${BUILDKITE_API_TOKEN:-}" ] || { echo "::error::BUILDKITE_API_TOKEN secret is not set"; exit 1; } - api="https://api.buildkite.com/v2/organizations/$BK_ORG" - auth="Authorization: Bearer $BUILDKITE_API_TOKEN" - nums=(); slugs=(); urls=() - - # 1) Create one build per suite, CLI built from this branch. - for slug in $SUITES; do - body=$(jq -n --arg b "$HEAD_REF" --arg m ":robot: CLI regression for $HEAD_REF (PR #$PR_NUMBER)" \ - '{commit:"HEAD", branch:"main", message:$m, env:{CLI_BRANCH_NAME:$b, CLI_PACKAGE_TYPE:"repo"}}') - resp=$(curl -sSf -X POST -H "$auth" -H "Content-Type: application/json" "$api/pipelines/$slug/builds" -d "$body") - nums+=("$(echo "$resp" | jq -r '.number')") - slugs+=("$slug") - urls+=("$(echo "$resp" | jq -r '.web_url')") - echo "Created $slug build #${nums[-1]}: ${urls[-1]}" - done - - # 2) Poll until every build reaches a terminal state (or we time out). - terminal=" passed failed canceled skipped not_run blocked " - deadline=$(( $(date +%s) + MAX_WAIT_MIN * 60 )) - while :; do - done_all=1 - for i in "${!nums[@]}"; do - st=$(curl -sSf -H "$auth" "$api/pipelines/${slugs[$i]}/builds/${nums[$i]}" | jq -r '.state') - case "$terminal" in *" $st "*) ;; *) done_all=0 ;; esac - done - [ "$done_all" = 1 ] && break - if [ "$(date +%s)" -ge "$deadline" ]; then echo "::warning::timed out after ${MAX_WAIT_MIN}m; reporting partial"; break; fi - sleep "$POLL_INTERVAL" - done - - # 3) Build the per-SDK result table. - { - echo "" - echo "## ๐Ÿค– SDK Regression โ€” App Percy + POA" - echo "" - echo "CLI branch \`$HEAD_REF\` (built from source) ยท web-SDK results post as commit statuses." - echo "" - echo "| Suite | SDK job | Result |" - echo "|---|---|---|" - } > comment.md - overall_fail=0 - # The bootstrap step that uploads the matrix is named after the pipeline - # ("App-Percy-SDK-tests" / "POA-SDK-tests"); exclude it and the unnamed - # wait job so the table is only the real per-SDK device jobs. - for i in "${!nums[@]}"; do - jobs=$(curl -sSf -H "$auth" "$api/pipelines/${slugs[$i]}/builds/${nums[$i]}") - echo "$jobs" | jq -r --arg s "${slugs[$i]}" ' - .jobs[] - | select(.type=="script" and .name != null and .name != "") - | select(.name != "App-Percy-SDK-tests" and .name != "POA-SDK-tests") - | "| \($s) | \(.name) | " + (if .state=="passed" then "โœ… pass" else "โŒ " + (.state // "n/a") end) + " |"' >> comment.md - if echo "$jobs" | jq -e ' - .jobs[] - | select(.type=="script" and .name != "App-Percy-SDK-tests" and .name != "POA-SDK-tests") - | select(.state=="failed" or .state=="canceled")' >/dev/null; then - overall_fail=1 - fi - done - { - echo "" - for i in "${!nums[@]}"; do echo "- [\`${slugs[$i]}\` build #${nums[$i]}](${urls[$i]})"; done - } >> comment.md - - # 4) Upsert the marker comment on the PR (create, or edit the previous one). - jq -n --rawfile b comment.md '{body:$b}' > payload.json - existing=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" --paginate \ - --jq '.[] | select(.body | startswith("")) | .id' | head -n1 || true) - if [ -n "$existing" ]; then - gh api -X PATCH "repos/$REPO/issues/comments/$existing" --input payload.json >/dev/null - else - gh api -X POST "repos/$REPO/issues/$PR_NUMBER/comments" --input payload.json >/dev/null - fi - echo "Reported App/POA results to PR #$PR_NUMBER" - [ "$overall_fail" = 0 ] || { echo "::error::App/POA regression had failures"; exit 1; } From b07f84c7d1d669171eae7ac1a0811a49d5392275 Mon Sep 17 00:00:00 2001 From: Pranav Zinzurde Date: Fri, 3 Jul 2026 18:21:23 +0530 Subject: [PATCH 17/17] ci(sdk-regression): comment+dispatch triggers only; per-SDK ref overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the run-sdk-regression label (pull_request) trigger โ€” regression now runs only via a RUN_REGRESSION PR comment or workflow_dispatch. Dispatch gains an sdk_refs input (comma-separated repo@branch) to run individual SDKs' workflows from a specific branch (e.g. CLI master + one SDK's feature branch); unlisted SDKs keep their matrix default ref. Overridden refs are validated before flowing into the downstream dispatch. Co-Authored-By: Claude Fable 5 --- .github/workflows/sdk-regression.yml | 49 ++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/.github/workflows/sdk-regression.yml b/.github/workflows/sdk-regression.yml index 2201b5eb4..35f60b754 100644 --- a/.github/workflows/sdk-regression.yml +++ b/.github/workflows/sdk-regression.yml @@ -2,14 +2,19 @@ name: SDK Regression on: issue_comment: types: [created, edited] - pull_request: - types: [labeled, synchronize, reopened] workflow_dispatch: inputs: branch: description: CLI branch to run all SDK regression against required: false default: master + sdk_refs: + description: >- + Per-SDK workflow ref overrides, comma-separated repo@branch + (e.g. percy-cypress@my-fix,percy-ember@feat-x). SDKs not listed + run from their matrix default ref. + required: false + default: '' permissions: contents: read jobs: @@ -24,13 +29,12 @@ jobs: pull-requests: read statuses: write # Run when either: - # - a maintainer comments RUN_REGRESSION on a PR (legacy manual path), or - # - the PR carries the `run-sdk-regression` label (auto path / quality gate). - # Both paths are internal-only: only write/admin collaborators can comment-gate - # (enforced by check-access below) or apply a label. + # - a maintainer comments RUN_REGRESSION on a PR, or + # - the workflow is dispatched manually (CLI branch + optional per-SDK refs). + # Both paths are internal-only: the comment path is gated to write/admin + # collaborators (check-access below); dispatch requires repo write access. if: >- (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == 'RUN_REGRESSION') || - (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-sdk-regression')) || (github.event_name == 'workflow_dispatch') strategy: # Don't let one SDK's failure cancel the rest โ€” a regression run must @@ -108,8 +112,6 @@ jobs: id: pr env: EVENT_NAME: ${{ github.event_name }} - PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} COMMENT_HEAD_REF: ${{ steps.comment-branch.outputs.head_ref }} COMMENT_HEAD_SHA: ${{ steps.comment-branch.outputs.head_sha }} DISPATCH_BRANCH: ${{ github.event.inputs.branch }} @@ -118,9 +120,6 @@ jobs: if [ "$EVENT_NAME" = "workflow_dispatch" ]; then echo "head_ref=$DISPATCH_BRANCH" >> "$GITHUB_OUTPUT" echo "head_sha=$DISPATCH_SHA" >> "$GITHUB_OUTPUT" - elif [ "$EVENT_NAME" = "pull_request" ]; then - echo "head_ref=$PR_HEAD_REF" >> "$GITHUB_OUTPUT" - echo "head_sha=$PR_HEAD_SHA" >> "$GITHUB_OUTPUT" else echo "head_ref=$COMMENT_HEAD_REF" >> "$GITHUB_OUTPUT" echo "head_sha=$COMMENT_HEAD_SHA" >> "$GITHUB_OUTPUT" @@ -172,6 +171,30 @@ jobs: with: msg: ${{ matrix.repo }} separator: '@' + # Resolve the ref the SDK's workflow runs from. Dispatch may override it + # per SDK via the sdk_refs input (comma-separated repo@branch); anything + # not listed keeps the matrix default. The chosen ref is validated before + # use โ€” it flows into a downstream workflow dispatch. + - name: Resolve SDK workflow ref + id: sdk-ref + env: + SDK_REFS: ${{ github.event.inputs.sdk_refs }} + REPO_NAME: ${{ steps.split.outputs._0 }} + DEFAULT_REF: ${{ steps.split.outputs._1 || 'master' }} + run: | + ref="$DEFAULT_REF" + IFS=',' read -ra overrides <<< "${SDK_REFS:-}" + for o in "${overrides[@]}"; do + o="${o#"${o%%[![:space:]]*}"}"; o="${o%"${o##*[![:space:]]}"}" + case "$o" in + "$REPO_NAME"@?*) ref="${o#*@}" ;; + esac + done + if ! printf '%s' "$ref" | grep -Eq '^[a-zA-Z0-9_/.\-]+$'; then + echo "::error::invalid sdk ref '$ref' for $REPO_NAME" + exit 1 + fi + echo "ref=$ref" >> "$GITHUB_OUTPUT" - name: Trigger Workflow & Wait uses: convictional/trigger-workflow-and-wait@f69fa9eedd3c62a599220f4d5745230e237904be # v1.6.5 id: reg-test @@ -185,7 +208,7 @@ jobs: # percy-react-native-app -> storybook-rn-ci.yml # percy-tosca-dotnet / percy-uipath -> ci.yml (they have no test.yml) workflow_file_name: ${{ steps.split.outputs._0 == 'percy-storybook' && 'test-storybook-v10.yml' || (steps.split.outputs._0 == 'percy-react-native-app' && 'storybook-rn-ci.yml' || ((steps.split.outputs._0 == 'percy-tosca-dotnet' || steps.split.outputs._0 == 'percy-uipath') && 'ci.yml' || 'test.yml')) }} - ref: ${{ steps.split.outputs._1 || 'master' }} + ref: ${{ steps.sdk-ref.outputs.ref }} client_payload: '{ "branch": "${{ steps.pr.outputs.head_ref }}"}' wait_interval: 15 - name: Update Status