Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
a128108
CI: make visual test passing rules stricter
EugeniyKiyashko Jun 30, 2026
d4c6617
no message
EugeniyKiyashko Jun 30, 2026
c2c331b
no message
EugeniyKiyashko Jul 1, 2026
aa6e2b0
no message
EugeniyKiyashko Jul 1, 2026
72e7832
no message
EugeniyKiyashko Jul 1, 2026
291788c
no message
EugeniyKiyashko Jul 1, 2026
ac160e1
revert
EugeniyKiyashko Jul 7, 2026
cdfa970
CI: make visual test passing rules stricter
EugeniyKiyashko Jun 30, 2026
e0ac243
no message
EugeniyKiyashko Jun 30, 2026
5db1269
no message
EugeniyKiyashko Jul 1, 2026
83c1bc4
no message
EugeniyKiyashko Jul 1, 2026
8c48ba9
no message
EugeniyKiyashko Jul 1, 2026
af1e974
no message
EugeniyKiyashko Jul 1, 2026
8a5bd99
revert
EugeniyKiyashko Jul 7, 2026
89511a5
Merge branch '26_1_tests_stability' of https://github.com/EugeniyKiya…
EugeniyKiyashko Jul 7, 2026
531ef99
no message
EugeniyKiyashko Jul 7, 2026
0ddcd7d
no message
EugeniyKiyashko Jul 7, 2026
2b375ba
no message
EugeniyKiyashko Jul 7, 2026
a9c2e09
no message
EugeniyKiyashko Jul 8, 2026
e27febc
e2e tests: collect unstable tests statistics
EugeniyKiyashko Jul 7, 2026
a7036f3
no message
EugeniyKiyashko Jul 7, 2026
6c881e6
no message
EugeniyKiyashko Jul 7, 2026
d4519ab
no message
EugeniyKiyashko Jul 8, 2026
d97d28e
no message
EugeniyKiyashko Jul 8, 2026
93bd496
no message
EugeniyKiyashko Jul 8, 2026
359677d
no message
EugeniyKiyashko Jul 8, 2026
ac3a29c
Merge branch '26_1' into 26_1_run_unstable_with_statistics_
EugeniyKiyashko Jul 8, 2026
10716ff
revert changes
EugeniyKiyashko Jul 8, 2026
61ba09a
revert changes
EugeniyKiyashko Jul 8, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
271 changes: 253 additions & 18 deletions .github/workflows/testcafe_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -212,33 +212,33 @@ jobs:
run: |
base_matrix_start=$(cat <<'JSON'
[
{ "componentFolder": "accessibility", "name": "accessibility" }
{ "componentFolder": "accessibility", "name": "accessibility", "concurrency": 6 }
]
JSON
)

base_matrix_end=$(cat <<'JSON'
[
{ "componentFolder": "common", "name": "common", "concurrency": 2 },
{ "componentFolder": "common", "name": "common", "concurrency": 3 },

{ "name": "generic", "theme": "generic.light", "concurrency": 2 },
{ "name": "material", "theme": "material.blue.light", "concurrency": 2 },
{ "name": "material - compact", "theme": "material.blue.light.compact", "concurrency": 2 },

{ "componentFolder": "cardView", "name": "cardView" },
{ "componentFolder": "dataGrid", "name": "dataGrid (1/4)", "indices": "1/4", "cache": true, "concurrency": 6 },
{ "componentFolder": "dataGrid", "name": "dataGrid (2/4)", "indices": "2/4", "cache": true, "concurrency": 6 },
{ "componentFolder": "dataGrid", "name": "dataGrid (3/4)", "indices": "3/4", "cache": true, "concurrency": 6 },
{ "componentFolder": "dataGrid", "name": "dataGrid (4/4)", "indices": "4/4", "cache": true, "concurrency": 6 },
{ "componentFolder": "dataGrid", "name": "dataGrid (1/4)", "indices": "1/4", "cache": true, "concurrency": 5 },
{ "componentFolder": "dataGrid", "name": "dataGrid (2/4)", "indices": "2/4", "cache": true, "concurrency": 5 },
{ "componentFolder": "dataGrid", "name": "dataGrid (3/4)", "indices": "3/4", "cache": true, "concurrency": 5 },
{ "componentFolder": "dataGrid", "name": "dataGrid (4/4)", "indices": "4/4", "cache": true, "concurrency": 5 },

{ "componentFolder": "editors", "name": "editors" },
{ "componentFolder": "navigation", "name": "navigation", "concurrency": 6 },

{ "componentFolder": "scheduler/common", "name": "scheduler / common (1/3)", "indices": "1/3" },
{ "componentFolder": "scheduler/common", "name": "scheduler / common (2/3)", "indices": "2/3" },
{ "componentFolder": "scheduler/common", "name": "scheduler / common (3/3)", "indices": "3/3" },
{ "componentFolder": "scheduler/viewOffset", "name": "scheduler / viewOffset (1/2)", "indices": "1/2" },
{ "componentFolder": "scheduler/viewOffset", "name": "scheduler / viewOffset (2/2)", "indices": "2/2" },
{ "componentFolder": "scheduler/common", "name": "scheduler / common (1/3)", "indices": "1/3", "concurrency": 6 },
{ "componentFolder": "scheduler/common", "name": "scheduler / common (2/3)", "indices": "2/3", "concurrency": 6 },
{ "componentFolder": "scheduler/common", "name": "scheduler / common (3/3)", "indices": "3/3", "concurrency": 6 },
{ "componentFolder": "scheduler/viewOffset", "name": "scheduler / viewOffset (1/2)", "indices": "1/2", "concurrency": 4 },
{ "componentFolder": "scheduler/viewOffset", "name": "scheduler / viewOffset (2/2)", "indices": "2/2", "concurrency": 4 },
{ "componentFolder": "scheduler/timezones", "name": "scheduler / timezones (Europe/Berlin)", "timezone": "Europe/Berlin" },
{ "componentFolder": "scheduler/timezones", "name": "scheduler / timezones (America/Los_Angeles)", "timezone": "America/Los_Angeles" }
]
Expand All @@ -247,10 +247,10 @@ jobs:

style_dependent_matrix=$(cat <<'JSON'
[
{ "componentFolder": "accessibility", "name": "accessibility - fluent.dark", "theme": "fluent.blue.dark" },
{ "componentFolder": "accessibility", "name": "accessibility - material.light", "theme": "material.blue.light" },
{ "componentFolder": "accessibility", "name": "accessibility - material.dark", "theme": "material.blue.dark" }
]
{ "componentFolder": "accessibility", "name": "accessibility - fluent.dark", "theme": "fluent.blue.dark", "concurrency": 6 },
{ "componentFolder": "accessibility", "name": "accessibility - material.light", "theme": "material.blue.light", "concurrency": 6 },
{ "componentFolder": "accessibility", "name": "accessibility - material.dark", "theme": "material.blue.dark", "concurrency": 6 }
]
JSON
)

Expand Down Expand Up @@ -346,14 +346,38 @@ jobs:
[ "${{ matrix.concurrency }}" != "" ] && CONCURRENCY="--concurrency ${{ matrix.concurrency }}"
[ "${{ matrix.platform }}" != "" ] && PLATFORM="--platform ${{ matrix.platform }}"
[ "${{ matrix.cache }}" == "true" ] && CACHE="--cache true"
all_args="--browsers=chrome:devextreme-shr2 $COMPONENT_FOLDER $CONCURRENCY $INDICES $PLATFORM $THEME $CACHE"
REPORT_UNSTABLE="--reportUnstable ./artifacts/unstable-report/report.json"
all_args="--browsers=chrome:devextreme-shr2 $COMPONENT_FOLDER $CONCURRENCY $INDICES $PLATFORM $THEME $CACHE $REPORT_UNSTABLE"
echo "$all_args"
pnpm run test $all_args

- name: Sanitize job name
if: ${{ always() }}
run: echo "JOB_NAME=$(echo "${{ matrix.name }}" | tr '/' '-')" >> $GITHUB_ENV

- name: Add run config to unstable test report
if: ${{ always() }}
working-directory: ./e2e/testcafe-devextreme
run: |
report=artifacts/unstable-report/report.json
if [ -f "$report" ]; then
jq --arg theme "${{ matrix.theme }}" \
--arg componentFolder "${{ matrix.componentFolder }}" \
--arg timezone "${{ matrix.timezone }}" \
--arg platform "${{ matrix.platform }}" \
'. + { theme: $theme, componentFolder: $componentFolder, timezone: $timezone, platform: $platform }' \
"$report" > "$report.tmp" && mv "$report.tmp" "$report"
cat "$report"
fi

- name: Upload unstable test report
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: unstable-report-${{ env.JOB_NAME }}
path: ${{ github.workspace }}/e2e/testcafe-devextreme/artifacts/unstable-report/report.json
if-no-files-found: ignore

- name: Copy compared screenshot artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v7
Expand All @@ -370,9 +394,211 @@ jobs:
path: ${{ github.workspace }}/e2e/testcafe-devextreme/artifacts/failedtests/**/*
if-no-files-found: ignore

testcafe-unstable:
name: unstable
needs: [check-should-run, build, testcafe]
if: |
!cancelled() &&
needs.check-should-run.outputs.should-run == 'true' &&
needs.build.result == 'success'
runs-on: devextreme-shr2
timeout-minutes: 45

steps:
- name: Set machine timezone
run: |
sudo ln -sf /usr/share/zoneinfo/GMT /etc/localtime
date

- name: Get sources
uses: actions/checkout@v6

- name: Download artifacts
uses: actions/download-artifact@v8
with:
name: devextreme-artifacts
path: ./packages/devextreme

- name: Unpack artifacts
working-directory: ./packages/devextreme
run: 7z x artifacts.zip -aoa

- name: Setup Chrome
uses: ./.github/actions/setup-chrome
with:
chrome-version: '149.0.7827.114'

- name: Use Node.js
uses: actions/setup-node@v6
with:
node-version-file: '.node-version'

- uses: pnpm/action-setup@v6
with:
run_install: false

- name: Get pnpm store directory
shell: bash
run: |
echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

- uses: actions/cache/restore@v5
name: Restore pnpm cache
with:
path: ${{ env.STORE_PATH }}
key: ${{ runner.os }}-pnpm-cache-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-cache

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Download unstable test reports
uses: actions/download-artifact@v8
with:
pattern: unstable-report-*
path: ./e2e/testcafe-devextreme/unstable-reports

- name: Run unstable tests
working-directory: ./e2e/testcafe-devextreme
env:
NODE_OPTIONS: --max-old-space-size=8192
run: |
last_failed=0
rerun_failed=0
unstable_failed=0
result=./artifacts/unstable-run-result.json
new_flakes_summary=./artifacts/new-flakes-summary.txt
mkdir -p ./artifacts
: > "$new_flakes_summary"

# Every test gets up to 3 attempts (initial run + 2 retries) and passes
# if at least one attempt succeeds. A reported failure means all attempts failed.
run_tests() {
local rc=0
rm -f "$result"
pnpm run test --browsers=chrome:devextreme-shr2 --concurrency 1 --retryAttempts 2 --forcePageReloads true --reportFailures "$result" "$@" || rc=$?
if [ ! -f "$result" ]; then
echo "Test runner crashed (exit code $rc)"
exit 1
fi
last_failed=$(jq '.failedCount // (.tests | length)' "$result")
echo " > tests that failed all attempts in this run: $last_failed"
}

shopt -s nullglob
for report in unstable-reports/*/report.json; do
job_name=$(basename "$(dirname "$report")")
job_name=${job_name#unstable-report-}

echo ""
echo "============================================================"
echo " [NEW FLAKY] Rerun of test(s) that failed once in job '$job_name'"
echo "============================================================"
cat "$report"

theme=$(jq -r '.theme // empty' "$report")
componentFolder=$(jq -r '.componentFolder // empty' "$report")
timezone=$(jq -r '.timezone // empty' "$report")
platform=$(jq -r '.platform // empty' "$report")

sudo ln -sf "/usr/share/zoneinfo/${timezone:-GMT}" /etc/localtime

extra_args=(--testsFile "$report" --theme "${theme:-fluent.blue.light}")
[ -n "$componentFolder" ] && extra_args+=(--componentFolder "$componentFolder")
[ -n "$platform" ] && extra_args+=(--platform "$platform")

run_tests "${extra_args[@]}"
rerun_failed=$((rerun_failed + last_failed))

while IFS= read -r name; do
if jq -e --arg n "$name" '.tests | index($n)' "$result" > /dev/null; then
echo "FAILED | $name | $job_name" >> "$new_flakes_summary"
else
echo "passed | $name | $job_name" >> "$new_flakes_summary"
fi
done < <(jq -r '.tests[]' "$report")
done

sudo ln -sf /usr/share/zoneinfo/GMT /etc/localtime

echo ""
echo "============================================================"
echo " [KNOWN UNSTABLE] Run of all tests marked with test.meta({ unstable: true })"
echo "============================================================"
run_tests --onlyUnstable true
unstable_failed=$last_failed
meta_failed_names=$(jq -r '.tests[]' "$result")

echo ""
echo "============================================================"
echo " UNSTABLE JOB SUMMARY"
echo "============================================================"
echo " NEW flaky tests (failed once in a regular job, rerun here):"
if [ -s "$new_flakes_summary" ]; then
sed 's/^/ /' "$new_flakes_summary"
else
echo " (none)"
fi
echo ""
echo " KNOWN unstable tests (test.meta({ unstable: true })), failed all attempts: $unstable_failed"
if [ -n "$meta_failed_names" ]; then
echo "$meta_failed_names" | sed 's/^/ FAILED | /'
fi
echo "============================================================"

{
echo "## Unstable job summary"
echo ""
echo "### New flaky tests (forwarded from regular jobs)"
if [ -s "$new_flakes_summary" ]; then
echo ""
echo "| Result | Test | Source job |"
echo "|---|---|---|"
awk -F' \\| ' '{ printf "| %s | %s | %s |\n", $1, $2, $3 }' "$new_flakes_summary"
else
echo ""
echo "_none_"
fi
echo ""
echo "### Known unstable tests (\`test.meta({ unstable: true })\`)"
echo ""
echo "Failed all attempts: **$unstable_failed**"
if [ -n "$meta_failed_names" ]; then
echo ""
echo "$meta_failed_names" | sed 's/^/- /'
fi
} >> "$GITHUB_STEP_SUMMARY"

if [ "$rerun_failed" -gt 0 ]; then
echo "A NEW flaky test failed all attempts."
echo "This is a persistent failure, not a flake - marking the job as failed."
exit 1
fi
if [ "$unstable_failed" -gt 1 ]; then
echo "More than one KNOWN unstable test failed - marking the job as failed."
exit 1
fi

- name: Copy compared screenshot artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: compared-screenshots-unstable
path: ${{ github.workspace }}/e2e/testcafe-devextreme/artifacts/compared-screenshots/**/*
if-no-files-found: ignore

- name: Copy failed test artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: failed-tests-unstable
path: ${{ github.workspace }}/e2e/testcafe-devextreme/artifacts/failedtests/**/*
if-no-files-found: ignore

merge-artifacts:
runs-on: ubuntu-latest
needs: testcafe
needs: [testcafe, testcafe-unstable]
if: ${{ always() }}

steps:
Expand All @@ -392,10 +618,19 @@ jobs:
pattern: failed-tests-*
delete-merged: true

- name: Merge unstable test reports
uses: actions/upload-artifact/merge@v7
continue-on-error: true
with:
name: unstable-reports
pattern: unstable-report-*
separate-directories: true
delete-merged: true

notify:
runs-on: ubuntu-latest
name: Send notifications
needs: [build, testcafe]
needs: [build, testcafe, testcafe-unstable]
if: github.event_name != 'pull_request' && contains(needs.*.result, 'failure')

steps:
Expand Down
2 changes: 1 addition & 1 deletion apps/demos/utils/visual-tests/testcafe-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ async function main() {
.browsers(process.env.BROWSERS || 'chrome --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning')
.concurrency(concurrency || 1)
.run({
quarantineMode: { successThreshold: 1, attemptLimit: 2 },
quarantineMode: { successThreshold: 1, attemptLimit: 3 },
// @ts-expect-error ts-error
hooks: {
test: {
Expand Down
Loading
Loading