From c2b3efdea839cc214f80b2473df03b228914795e Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 2 Jul 2026 19:25:42 +0200 Subject: [PATCH 1/3] ci(macrobenchmark): Run startup benchmark on Sauce Labs (POC) The sentry-uitest-android-macrobenchmark module currently only runs on a locally connected device. This wires it to Sauce Labs so we can evaluate whether the cold-start timeToInitialDisplay benchmark can run on the real-device cloud already used by our other benchmarks. This is a proof-of-concept, gated behind workflow_dispatch and kept off the per-PR path. Device-guard errors are intentionally not suppressed: on non-rooted, unlocked-clock cloud devices the guards (UNLOCKED, DEBUGGABLE, ...) are expected to fire, and seeing which ones fire is the point of the POC. It also tells us whether timeToInitialDisplay can be retrieved from Sauce artifacts (benchmark JSON) or must be parsed from the device log. Co-Authored-By: Claude Opus 4.8 --- .../integration-tests-macrobenchmark.yml | 60 +++++++++++++++++++ .../sentry-uitest-android-macrobenchmark.yml | 43 +++++++++++++ Makefile | 7 ++- 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/integration-tests-macrobenchmark.yml create mode 100644 .sauce/sentry-uitest-android-macrobenchmark.yml diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml new file mode 100644 index 00000000000..5fc25110a83 --- /dev/null +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -0,0 +1,60 @@ +name: 'Integration Tests - Macrobenchmark (POC)' +# Proof-of-concept: run the sentry-uitest-android-macrobenchmark cold-start benchmark on Sauce +# Labs real devices. Manual trigger only — this is not yet a per-PR gate. The goal is to learn +# (a) whether Macrobenchmark runs on non-rooted cloud devices and which device guards fire, and +# (b) whether timeToInitialDisplay can be retrieved from Sauce artifacts. See the module README. +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + macrobenchmark: + name: Macrobenchmark + runs-on: ubuntu-latest + + # we copy the secret to the env variable in order to access it in the workflow + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + + steps: + - name: Git checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + submodules: 'recursive' + + - name: 'Set up Java: 17' + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 + with: + cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + + - name: Make assembleMacrobenchmark + if: env.SAUCE_USERNAME != null + run: make assembleMacrobenchmark + + - name: Run Macrobenchmark in SauceLab + uses: saucelabs/saucectl-run-action@bc81720eb01738d9c664b07fe42621bd0014283f # pin@v3 + if: env.SAUCE_USERNAME != null + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + config-file: .sauce/sentry-uitest-android-macrobenchmark.yml + + - name: Upload Sauce artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: macrobenchmark-artifacts + path: ./artifacts/ + if-no-files-found: warn diff --git a/.sauce/sentry-uitest-android-macrobenchmark.yml b/.sauce/sentry-uitest-android-macrobenchmark.yml new file mode 100644 index 00000000000..6751560dc35 --- /dev/null +++ b/.sauce/sentry-uitest-android-macrobenchmark.yml @@ -0,0 +1,43 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + concurrency: 1 + metadata: + build: sentry-uitest-android-macrobenchmark-$GITHUB_REF-$GITHUB_SHA + tags: + - benchmarks + - android + - macrobenchmark + +defaults: + timeout: 40m + +espresso: + # Target app under test: Macrobenchmark cold-starts sentry-samples-android. It must be + # release-like; the release build type is signed with the debug key so it installs on Sauce. + app: ./sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk + # Instrumentation APK: the self-instrumenting com.android.test macrobenchmark module. + testApp: ./sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk + +suites: + + - name: "Macrobenchmark startup (api 35)" + # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process + # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and + # permissions (it force-stops rather than `pm clear`). + devices: + - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end + +# Grab both the benchmark JSON (if Sauce collects additional test output) and the device log, so +# we can see which path actually yields timeToInitialDisplay. Macrobenchmark also logs guard +# failures (UNLOCKED, DEBUGGABLE, ...) here, which is the expected first-run signal on cloud +# hardware — see the module README. +artifacts: + download: + when: always + match: + - junit.xml + - "*.log" + - "*-benchmarkData.json" + directory: ./artifacts/ diff --git a/Makefile b/Makefile index 3967ff856ad..09413b6af30 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ -.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish +.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleMacrobenchmarkRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish all: stop clean javadocs compile assembleBenchmarks: assembleBenchmarkTestRelease +assembleMacrobenchmark: assembleMacrobenchmarkRelease assembleUiTests: assembleUiTestRelease preMerge: check publish: clean dryRelease @@ -39,6 +40,10 @@ api: assembleBenchmarkTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest +# Assemble the target sample app (release) and the Macrobenchmark instrumentation apk +assembleMacrobenchmarkRelease: + ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark + # Assemble release and Android test apk of the uitest-android module assembleUiTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest From 65c5ff0511e52021232e1626996cf4bd65e3d5f1 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 2 Jul 2026 19:29:43 +0200 Subject: [PATCH 2/3] ci(macrobenchmark): Trigger POC workflow on branch push Temporary scaffolding so the Sauce run executes on this branch without first landing on the default branch (workflow_dispatch is not dispatchable until the workflow exists on the default branch). Remove before merge. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/integration-tests-macrobenchmark.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index 5fc25110a83..5d04a1de969 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -5,6 +5,11 @@ name: 'Integration Tests - Macrobenchmark (POC)' # (b) whether timeToInitialDisplay can be retrieved from Sauce artifacts. See the module README. on: workflow_dispatch: + # Temporary POC scaffolding: run on pushes to this branch so we can see it work on Sauce + # without merging to the default branch first. Remove before merge. + push: + branches: + - no/macrobenchmark-on-sauce-poc concurrency: group: ${{ github.workflow }}-${{ github.ref }} From dcf7e1763c5279501b6f3b3df28077759cb0d71b Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Thu, 2 Jul 2026 19:31:18 +0200 Subject: [PATCH 3/3] ci(macrobenchmark): Call gradle directly instead of a Makefile target Drop the assembleMacrobenchmark Makefile target and invoke the two gradle assemble tasks directly from the workflow. Only one workflow uses them, so the extra indirection isn't worth it. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/integration-tests-macrobenchmark.yml | 4 ++-- Makefile | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml index 5d04a1de969..f9c25497220 100644 --- a/.github/workflows/integration-tests-macrobenchmark.yml +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -42,9 +42,9 @@ jobs: with: cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - name: Make assembleMacrobenchmark + - name: Assemble target app and Macrobenchmark apk if: env.SAUCE_USERNAME != null - run: make assembleMacrobenchmark + run: ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark - name: Run Macrobenchmark in SauceLab uses: saucelabs/saucectl-run-action@bc81720eb01738d9c664b07fe42621bd0014283f # pin@v3 diff --git a/Makefile b/Makefile index 09413b6af30..3967ff856ad 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ -.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleMacrobenchmarkRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish +.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish all: stop clean javadocs compile assembleBenchmarks: assembleBenchmarkTestRelease -assembleMacrobenchmark: assembleMacrobenchmarkRelease assembleUiTests: assembleUiTestRelease preMerge: check publish: clean dryRelease @@ -40,10 +39,6 @@ api: assembleBenchmarkTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -# Assemble the target sample app (release) and the Macrobenchmark instrumentation apk -assembleMacrobenchmarkRelease: - ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark - # Assemble release and Android test apk of the uitest-android module assembleUiTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest