diff --git a/.github/workflows/integration-tests-benchmarks.yml b/.github/workflows/integration-tests-benchmarks.yml deleted file mode 100644 index 66e4498dcb5..00000000000 --- a/.github/workflows/integration-tests-benchmarks.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: 'Integration Tests - Benchmarks' -on: - schedule: - - cron: '0 0 * * *' # every night at midnight UTC - pull_request: - paths: - - '**/sentry/**' - - '**/sentry-android/**' - - '**/sentry-android-core/**' - - '**/sentry-android-ndk/**' - - '**/sentry-android-integration-tests/**' - - '**/.github/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - name: Benchmarks - runs-on: ubuntu-latest - - # we copy the secret to the env variable in order to access it in the workflow - env: - SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - steps: - - name: Git checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: 'recursive' - - - name: 'Set up Java: 17' - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 - with: - distribution: 'temurin' - java-version: '17' - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - # Clean, build and release a test apk, but only if we will run the benchmark - - name: Make assembleBenchmarks - if: env.SAUCE_USERNAME != null - run: make assembleBenchmarks - - - name: Run All Tests in SauceLab - uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v3 - if: github.event_name != 'pull_request' && env.SAUCE_USERNAME != null - env: - GITHUB_TOKEN: ${{ github.token }} - with: - sauce-username: ${{ secrets.SAUCE_USERNAME }} - sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - config-file: .sauce/sentry-uitest-android-benchmark.yml - - - name: Run one test in SauceLab - uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v3 - if: github.event_name == 'pull_request' && env.SAUCE_USERNAME != null - env: - GITHUB_TOKEN: ${{ github.token }} - with: - sauce-username: ${{ secrets.SAUCE_USERNAME }} - sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - config-file: .sauce/sentry-uitest-android-benchmark-lite.yml - - app-metrics: - runs-on: ubuntu-latest - - # we copy the secret to the env variable in order to access it in the workflow - env: - SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - steps: - - name: Git checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: 'recursive' - - - name: 'Set up Java: 17' - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 - with: - distribution: 'temurin' - java-version: '17' - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - id: app-plain-cache - with: - path: sentry-android-integration-tests/test-app-plain/build/outputs/apk/release/test-app-plain-release.apk - key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ hashFiles('sentry-android-integration-tests/test-app-plain/**') }} - - - name: Build Android app plain - if: steps.app-plain-cache.outputs['cache-hit'] != 'true' - run: ./gradlew :sentry-android-integration-tests:test-app-plain:assembleRelease - - - name: Build Android app with Sentry - run: ./gradlew :sentry-android-integration-tests:test-app-sentry:assembleRelease - - - name: Collect app metrics - uses: getsentry/action-app-sdk-overhead-metrics@44fb5489ac4ac252c87d84811972dc93a1e490b8 - with: - config: sentry-android-integration-tests/metrics-test.yml - sauce-user: ${{ secrets.SAUCE_USERNAME }} - sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} - if: env.SAUCE_USERNAME != null diff --git a/.sauce/sentry-uitest-android-benchmark-lite.yml b/.sauce/sentry-uitest-android-benchmark-lite.yml deleted file mode 100644 index fec4a141def..00000000000 --- a/.sauce/sentry-uitest-android-benchmark-lite.yml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1alpha -kind: espresso -sauce: - region: us-west-1 - concurrency: 1 - metadata: - build: sentry-uitest-android-benchmark-lite-$GITHUB_REF-$GITHUB_SHA - tags: - - benchmarks - - android - -defaults: - timeout: 40m - -espresso: - app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk - testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk - -suites: - - - name: "Android 15 Benchmark lite (api 35)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - name: ".*" - platformVersion: "15" - -artifacts: - download: - when: always - match: - - junit.xml - directory: ./artifacts/ diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml deleted file mode 100644 index 12995ea5e07..00000000000 --- a/.sauce/sentry-uitest-android-benchmark.yml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: v1alpha -kind: espresso -sauce: - region: us-west-1 - concurrency: 3 - metadata: - build: sentry-uitest-android-benchmark-$GITHUB_REF-$GITHUB_SHA - tags: - - benchmarks - - android - -defaults: - timeout: 90m - -espresso: - app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk - testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk - -suites: - - # Devices are chosen so that there is a high-end and a low-end device for each api level - - name: "Android 15 (api 35)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end - - id: Samsung_Galaxy_S23_15_real_sjc1 # Samsung Galaxy S23 - api 35 (15) - mid end - - id: Google_Pixel_6a_15_real_sjc1 # Google Pixel 6a - api 35 (15) - low end - - - name: "Android 14 (api 34)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - id: Google_Pixel_9_Pro_XL_real_sjc1 # Google Pixel 9 Pro XL - api 34 (14) - high end - - id: Samsung_Galaxy_A54_real_sjc1 # Samsung Galaxy A54 - api 34 (14) - low end - - - name: "Android 13 (api 33)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - id: Google_Pixel_7_Pro_real_us # Google Pixel 7 Pro - api 33 (13) - high end - - id: Samsung_Galaxy_A32_5G_real_sjc1 # Samsung Galaxy A32 5G - api 33 (13) - low end - -# At the time of writing (August, 13, 2025), the market share per android version is: -# 15.0 = 26.75%, 14.0 = 19.5%, 13 = 15.95% -# Using these 3 versions we cover 62.2% of all devices out there. Currently, this is enough for benchmarking scope - -artifacts: - download: - when: always - match: - - junit.xml - directory: ./artifacts/ diff --git a/Makefile b/Makefile index 3967ff856ad..3941eb93ec3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ -.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish +.PHONY: all clean compile javadocs dryRelease update checkFormat api assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish all: stop clean javadocs compile -assembleBenchmarks: assembleBenchmarkTestRelease assembleUiTests: assembleUiTestRelease preMerge: check publish: clean dryRelease @@ -35,10 +34,6 @@ checkFormat: api: ./gradlew apiDump -# Assemble release and Android test apk of the uitest-android-benchmark module -assembleBenchmarkTestRelease: - ./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest - # Assemble release and Android test apk of the uitest-android module assembleUiTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest diff --git a/build.gradle.kts b/build.gradle.kts index a663628b467..58472fcb3ee 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -77,10 +77,7 @@ apiValidation { "sentry-samples-spring-boot-4-webflux", "sentry-samples-ktor-client", "sentry-uitest-android", - "sentry-uitest-android-benchmark", "sentry-uitest-android-critical", - "test-app-plain", - "test-app-sentry", "test-app-size", "sentry-samples-netflix-dgs", "sentry-samples-console-otlp", diff --git a/sentry-android-integration-tests/README.md b/sentry-android-integration-tests/README.md index ecbd15b34f5..4b3c495a978 100644 --- a/sentry-android-integration-tests/README.md +++ b/sentry-android-integration-tests/README.md @@ -1,8 +1,4 @@ # Android performance-impact and integration tests -* [Sample app without sentry](./test-app-plain) created with Android Studio -> New Project -> Basic Activity -* [Same app, but with Sentry included](./test-app-sentry) - made part of the root project -* [App metrics test specification (yaml)](./metrics-test.yml) -* [Espresso-based benchmarks](./sentry-uitest-android-benchmark) - run within SauceLabs (see /.sauce/*.yml) * [Espresso-based UI tests](./sentry-uitest-android) - run within SauceLabs (see /.sauce/*.yml) * Also used for compatibility test matrix against new AGP versions diff --git a/sentry-android-integration-tests/metrics-test.yml b/sentry-android-integration-tests/metrics-test.yml deleted file mode 100644 index a73ca1ef7c0..00000000000 --- a/sentry-android-integration-tests/metrics-test.yml +++ /dev/null @@ -1,12 +0,0 @@ -apps: - - name: io.sentry.java.tests.perf.appplain - activity: MainActivity - path: ./test-app-plain/build/outputs/apk/release/test-app-plain-release.apk - - name: io.sentry.java.tests.perf.appsentry - activity: MainActivity - path: ./test-app-sentry/build/outputs/apk/release/test-app-sentry-release.apk - -startupTimeTest: - runs: 50 - diffMin: 0 - diffMax: 150 diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore b/sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore deleted file mode 100644 index 796b96d1c40..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro deleted file mode 100644 index b2db365bad4..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro +++ /dev/null @@ -1,38 +0,0 @@ -# Add project specific ProGuard rules here. -# You can control the set of applied configuration files using the -# proguardFiles setting in build.gradle.kts. -# -# For more details, see -# http://developer.android.com/guide/developing/tools/proguard.html - -# If your project uses WebView with JS, uncomment the following -# and specify the fully qualified class name to the JavaScript interface -# class: -#-keepclassmembers class fqcn.of.javascript.interface.for.webview { -# public *; -#} - -# Uncomment this to preserve the line number information for -# debugging stack traces. -#-keepattributes SourceFile,LineNumberTable - -# If you keep the line number information, uncomment this to -# hide the original source file name. -#-renamesourcefileattribute SourceFile - --dontobfuscate -#Shrinking removes annotations and "unused classes" from test apk, so we don't shrink --dontshrink - --keepattributes *Annotation* - --dontnote junit.framework.** --dontnote junit.runner.** - --dontwarn androidx.test.** --dontwarn org.junit.** - --dontwarn androidx.annotation.** --dontwarn com.google.errorprone.** - --ignorewarnings diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts deleted file mode 100644 index c3ca2379a76..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts +++ /dev/null @@ -1,123 +0,0 @@ -import io.gitlab.arturbosch.detekt.Detekt -import net.ltgt.gradle.errorprone.errorprone -import org.jetbrains.kotlin.gradle.dsl.JvmTarget - -plugins { - id("com.android.application") - alias(libs.plugins.kotlin.android) - alias(libs.plugins.errorprone) - alias(libs.plugins.gradle.versions) - alias(libs.plugins.detekt) -} - -android { - compileSdk = libs.versions.compileSdk.get().toInt() - namespace = "io.sentry.uitest.android.benchmark" - - defaultConfig { - applicationId = "io.sentry.uitest.android.benchmark" - minSdk = libs.versions.minSdk.get().toInt() - targetSdk = libs.versions.targetSdk.get().toInt() - versionCode = 1 - versionName = "1.0.0" - - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - // Runs each test in its own instance of Instrumentation. This way they are isolated from - // one another and get their own Application instance. - // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle - // This doesn't work on some devices with Android 11+. Clearing package data resets permissions. - // Check the readme for more info. - testInstrumentationRunnerArguments["clearPackageData"] = "true" - } - - testOptions { execution = "ANDROIDX_TEST_ORCHESTRATOR" } - - buildFeatures { - // Determines whether to support View Binding. - // Note that the viewBinding.enabled property is now deprecated. - viewBinding = true - } - - signingConfigs { - getByName("debug") { - storeFile = rootProject.file("debug.keystore") - storePassword = "android" - keyAlias = "androiddebugkey" - keyPassword = "android" - } - } - - testBuildType = "release" - - buildTypes { - getByName("release") { - isMinifyEnabled = true - isShrinkResources = true - signingConfig = signingConfigs.getByName("debug") // to be able to run release mode - proguardFiles( - getDefaultProguardFile("proguard-android-optimize.txt"), - "benchmark-proguard-rules.pro", - ) - testProguardFiles( - getDefaultProguardFile("proguard-android-optimize.txt"), - "benchmark-proguard-rules.pro", - ) - } - } - - kotlin { compilerOptions.jvmTarget = JvmTarget.JVM_11 } - - lint { - warningsAsErrors = true - checkDependencies = true - - // We run a full lint analysis as build part in CI, so skip vital checks for assemble tasks. - checkReleaseBuilds = false - } - - androidComponents.beforeVariants { - if (it.buildType == "debug") { - it.enable = false - } - } -} - -dependencies { - implementation( - kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION) - ) - implementation(projects.sentryAndroid) - implementation(libs.androidx.appcompat) - implementation(libs.androidx.constraintlayout) - implementation(libs.androidx.core) - implementation(libs.androidx.recyclerview) - implementation(libs.androidx.test.espresso.idling.resource) - - compileOnly(libs.nopen.annotations) - - errorprone(libs.errorprone.core) - errorprone(libs.nopen.checker) - errorprone(libs.nullaway) - - androidTestUtil(libs.androidx.test.orchestrator) - androidTestImplementation(projects.sentryTestSupport) - androidTestImplementation(libs.kotlin.test.junit) - androidTestImplementation(libs.androidx.test.espresso.core) - androidTestImplementation(libs.androidx.test.core.ktx) - androidTestImplementation(libs.androidx.test.ext.junit) - androidTestImplementation(libs.androidx.test.rules) - androidTestImplementation(libs.androidx.test.runner) -} - -tasks.withType().configureEach { - options.errorprone { - check("NullAway", net.ltgt.gradle.errorprone.CheckSeverity.ERROR) - option("NullAway:AnnotatedPackages", "io.sentry") - option("NullAway:UnannotatedSubPackages", "io.sentry.uitest.android.benchmark.databinding") - } -} - -tasks.withType().configureEach { - // Target version of the generated JVM bytecode. It is used for type resolution. - jvmTarget = JavaVersion.VERSION_1_8.toString() -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt deleted file mode 100644 index 1b17fdf0c39..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt +++ /dev/null @@ -1,30 +0,0 @@ -package io.sentry.uitest.android.benchmark - -import android.content.Context -import android.view.Choreographer -import androidx.lifecycle.Lifecycle -import androidx.test.core.app.ApplicationProvider -import androidx.test.core.app.launchActivity -import androidx.test.platform.app.InstrumentationRegistry -import androidx.test.runner.AndroidJUnitRunner -import kotlin.test.BeforeTest - -abstract class BaseBenchmarkTest { - protected lateinit var runner: AndroidJUnitRunner - protected lateinit var context: Context - protected lateinit var choreographer: Choreographer - - @BeforeTest - fun baseSetUp() { - runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner - context = ApplicationProvider.getApplicationContext() - context.cacheDir.deleteRecursively() - // Must run on the main thread to get the main thread choreographer. - runner.runOnMainSync { choreographer = Choreographer.getInstance() } - - // We need the refresh rate, but we can get it only from the activity, so we start and destroy - // one - val benchmarkScenario = launchActivity() - benchmarkScenario.moveToState(Lifecycle.State.DESTROYED) - } -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt deleted file mode 100644 index 05a50895f25..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ /dev/null @@ -1,101 +0,0 @@ -package io.sentry.uitest.android.benchmark - -import android.os.Bundle -import androidx.lifecycle.Lifecycle -import androidx.test.core.app.launchActivity -import androidx.test.espresso.Espresso -import androidx.test.espresso.Espresso.onView -import androidx.test.espresso.IdlingRegistry -import androidx.test.espresso.action.ViewActions.swipeUp -import androidx.test.espresso.matcher.ViewMatchers.withId -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.runner.AndroidJUnitRunner -import io.sentry.ITransaction -import io.sentry.uitest.android.benchmark.util.BenchmarkOperation -import kotlin.test.AfterTest -import kotlin.test.BeforeTest -import kotlin.test.Test -import kotlin.test.assertTrue -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class SentryBenchmarkTest : BaseBenchmarkTest() { - @BeforeTest - fun setUp() { - IdlingRegistry.getInstance().register(BenchmarkActivity.scrollingIdlingResource) - } - - @AfterTest - fun cleanup() { - IdlingRegistry.getInstance().unregister(BenchmarkActivity.scrollingIdlingResource) - } - - @Test - fun benchmarkSameOperation() { - // We compare two operation that are the same. We expect the increases to be negligible, as the - // results - // should be very similar. - val op1 = BenchmarkOperation(choreographer, op = getOperation(runner)) - val op2 = BenchmarkOperation(choreographer, op = getOperation(runner)) - val refreshRate = BenchmarkActivity.refreshRate ?: 60F - // since we benchmark the same operation, warmupIterations = 1 would effectively mean - // 2 warmup runs which should be enough - val comparisonResults = - BenchmarkOperation.compare( - op1, - "Op1", - op2, - "Op2", - refreshRate, - warmupIterations = 1, - measuredIterations = 10, - ) - val comparisonResult = comparisonResults.getSummaryResult() - comparisonResult.printResults() - - // Currently we just want to assert the cpu overhead - assertTrue( - comparisonResult.cpuTimeIncreasePercentage in -2F..2F, - "Expected ${comparisonResult.cpuTimeIncreasePercentage} to be in range -2 < x < 2", - ) - // The fps decrease comparison is skipped, due to approximation: 59.51 and 59.49 fps are - // considered 60 and 59, - // respectively. Also, if the average fps is 20 or 60, a difference of 1 fps becomes 5% or 1.66% - // respectively. - } - - /** - * Operation that will be compared: it launches [BenchmarkActivity], swipe the list and closes it. - * The [transactionBuilder] is used to create the transaction before the swipes. - */ - private fun getOperation( - runner: AndroidJUnitRunner, - transactionBuilder: () -> ITransaction? = { null }, - ): () -> Unit = { - var transaction: ITransaction? = null - // Launch the sentry-uitest-android-benchmark activity - val benchmarkScenario = - launchActivity( - activityOptions = - Bundle().apply { putBoolean(BenchmarkActivity.EXTRA_SUSTAINED_PERFORMANCE_MODE, true) } - ) - // Starts a transaction (it can be null, but we still runOnMainSync to make operations as - // similar as possible) - runner.runOnMainSync { transaction = transactionBuilder() } - // Just swipe the list some times: this is the benchmarked operation - swipeList(2) - // We finish the transaction. We do it on main thread, so there's no need to perform other - // operations after it - runner.runOnMainSync { transaction?.finish() } - - benchmarkScenario.moveToState(Lifecycle.State.DESTROYED) - } - - private fun swipeList(times: Int) { - repeat(times) { - Thread.sleep(100) - onView(withId(R.id.benchmark_transaction_list)).perform(swipeUp()) - Espresso.onIdle() - } - } -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkComparisonResult.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkComparisonResult.kt deleted file mode 100644 index f2e27436286..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkComparisonResult.kt +++ /dev/null @@ -1,165 +0,0 @@ -package io.sentry.uitest.android.benchmark.util - -import java.util.concurrent.TimeUnit - -/** Result of the [BenchmarkOperation] comparison. */ -internal data class BenchmarkComparisonResult( - /** Number of measured iterations. */ - val iterations: Int, - /** Screen refresh rate. */ - val refreshRate: Float, - /** Screen refresh rate. */ - val cores: Int, - /** Name of the first compared operation. */ - val op1Name: String, - /** Name of the second compared operation. */ - val op2Name: String, - /** Raw cpu time in milliseconds of op1. */ - val op1CpuTime: List, - /** Raw cpu time in milliseconds of op2. */ - val op2CpuTime: List, - /** Increase of cpu time in milliseconds. */ - val cpuTimeIncreases: List, - /** Increase of cpu time in percentage. */ - val cpuTimeIncreasePercentages: List, - /** Raw dropped frames of op1. */ - val op1DroppedFrames: List, - /** Raw dropped frames of op2. */ - val op2DroppedFrames: List, - /** Increase of dropped frames. */ - val droppedFramesIncreases: List, - /** Increase of dropped frames in percentage. */ - val droppedFramesIncreasePercentages: List, - /** Raw duration in nanoseconds of op1. */ - val op1Duration: List, - /** Raw duration in nanoseconds of op2. */ - val op2Duration: List, - /** Increase of duration in nanoseconds. If it's low enough, no end user will ever realize it. */ - val durationIncreaseNanos: List, - /** Increase of duration in percentage. */ - val durationIncreasePercentage: List, - /** Raw fps of op1. */ - val op1Fps: List, - /** Raw fps of op2. */ - val op2Fps: List, - /** Decrease of fps. */ - val fpsDecreases: List, - /** Decrease of fps in percentage. */ - val fpsDecreasePercentages: List, -) { - /** - * Prints the raw results of all runs of the comparison. Each printed line is prefixed by - * [prefix], to allow parsers to easily parse log files to read raw values. - */ - fun printAllRuns(prefix: String) { - repeat(iterations) { index -> - println("$prefix ==================== Iteration $index ====================") - - println( - "$prefix [$op2Name]: duration=${op2Duration[index]} ns, cpuTime=${op2CpuTime[index]}, fps=${op2Fps[index]}, droppedFrames=${op2DroppedFrames[index]}" - ) - println( - "$prefix [$op1Name]: duration=${op1Duration[index]} ns, cpuTime=${op1CpuTime[index]}, fps=${op1Fps[index]}, droppedFrames=${op1DroppedFrames[index]}" - ) - println( - "$prefix Duration increase: %.2f%% (%d ns = %d ms)" - .format( - durationIncreasePercentage[index], - durationIncreaseNanos[index], - TimeUnit.NANOSECONDS.toMillis(durationIncreaseNanos[index]), - ) - ) - - println( - "$prefix CPU time overhead, over $cores cores: %.2f%% (%d ms)" - .format( - cpuTimeIncreasePercentages[index], - TimeUnit.NANOSECONDS.toMillis(cpuTimeIncreases[index]), - ) - ) - - println( - "$prefix FPS decrease: %.2f%% (%d fps)" - .format(fpsDecreasePercentages[index], fpsDecreases[index]) - ) - - val expectedFrames = TimeUnit.NANOSECONDS.toMillis(op2Duration[index]) * refreshRate / 1000 - println( - "$prefix Frame drop increase, over $expectedFrames total frames, with $refreshRate hz: %.2f%% (%.2f)" - .format(droppedFramesIncreasePercentages[index], droppedFramesIncreases[index]) - ) - } - } - - fun getSummaryResult() = - BenchmarkSummaryResult( - calculatePercentile(cpuTimeIncreases, 90), - calculatePercentile(cpuTimeIncreasePercentages, 90), - calculatePercentile(droppedFramesIncreases, 90), - calculatePercentile(droppedFramesIncreasePercentages, 90), - calculatePercentile(durationIncreaseNanos, 90), - calculatePercentile(durationIncreasePercentage, 90), - calculatePercentile(fpsDecreases, 90), - calculatePercentile(fpsDecreasePercentages, 90), - ) - - /** Calculate the [percentile] of the [list]. [percentile] should be in the range 0, 100. */ - private fun calculatePercentile(list: List, percentile: Int): T { - if (list.isEmpty()) { - return 0 as T - } - val sortedList = list.sortedBy { it.toDouble() } - val percentileIndex = (list.size * percentile / 100 - 1).coerceIn(0, list.size) - return sortedList[percentileIndex] - } -} - -/** Result of the [BenchmarkOperation] comparison. */ -internal data class BenchmarkSummaryResult( - /** - * Increase of cpu time in nanoseconds. It has no direct impact on performance of the app, but it - * has on battery usage, as the cpu is 'awaken' longer. - */ - val cpuTimeIncreaseNanos: Long, - /** Increase of cpu time in percentage. */ - val cpuTimeIncreasePercentage: Double, - /** - * Increase of dropped frames.Very important, as it weights dropped frames based on the time - * passed between each frame. This is the metric end users can perceive as 'performance' in app - * usage. - */ - val droppedFramesIncrease: Double, - /** Increase of dropped frames in percentage. */ - val droppedFramesIncreasePercentage: Double, - /** Increase of duration in nanoseconds. If it's low enough, no end user will ever realize it. */ - val durationIncreaseNanos: Long, - /** Increase of duration in percentage. */ - val durationIncreasePercentage: Double, - /** - * Decrease of fps. Not really important, as even if fps are the same, the cpu could be doing more - * work in the frame window, and it could be hidden by checking average fps only. - */ - val fpsDecrease: Int, - /** Decrease of fps in percentage. */ - val fpsDecreasePercentage: Double, -) { - /** Prints the summary results of the comparison. */ - fun printResults() { - println( - "Duration increase: %.2f%% (%d ns = %d ms)" - .format( - durationIncreasePercentage, - durationIncreaseNanos, - TimeUnit.NANOSECONDS.toMillis(durationIncreaseNanos), - ) - ) - println( - "CPU time overhead: %.2f%% (%d ms)".format(cpuTimeIncreasePercentage, cpuTimeIncreaseNanos) - ) - println("FPS decrease: %.2f%% (%d fps)".format(fpsDecreasePercentage, fpsDecrease)) - println( - "Frame drop increase: %.2f%% (%.2f)" - .format(droppedFramesIncreasePercentage, droppedFramesIncrease) - ) - } -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt deleted file mode 100644 index 6b209b27559..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt +++ /dev/null @@ -1,186 +0,0 @@ -package io.sentry.uitest.android.benchmark.util - -import android.os.SystemClock -import android.system.Os -import android.system.OsConstants -import android.view.Choreographer -import java.io.File -import java.util.LinkedList -import java.util.concurrent.TimeUnit - -// 60 FPS is the recommended target: https://www.youtube.com/watch?v=CaMTIgxCSqU -private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 - -/** - * Class that allows to benchmark some operations. Create two [BenchmarkOperation] objects and - * compare them using [BenchmarkOperation.compare] to get a [BenchmarkComparisonResult] with - * relative or absolute measured overheads. - */ -internal class BenchmarkOperation( - private val choreographer: Choreographer, - private val before: (() -> Unit)? = null, - private val after: (() -> Unit)? = null, - private val op: () -> Unit, -) { - companion object { - /** - * Running two operations sequentially (running 10 times the first and then 10 times the second) - * results in the first operation to always be slower, so comparing two different operations on - * equal terms is not possible. This method runs [op1] and [op2] in an alternating sequence. - * When [op1] and [op2] are the same, we get (nearly) identical results, as expected. You can - * adjust [warmupIterations] and [measuredIterations]. The lower they are, the faster the - * benchmark, but accuracy decreases. - */ - fun compare( - op1: BenchmarkOperation, - op1Name: String, - op2: BenchmarkOperation, - op2Name: String, - refreshRate: Float, - warmupIterations: Int = 2, - measuredIterations: Int = 20, - ): BenchmarkComparisonResult { - // Android pushes the "installed app" event to other apps and the system itself. - // Let's give it time to do whatever it wants before starting measuring the operations. - Thread.sleep(3000) - // The first operations are the slowest, as the device is still doing things like filling the - // cache. - repeat(warmupIterations) { - op1.warmup() - op2.warmup() - } - - // Now we can measure the operations (in alternating sequence). We change the order of - // measurement to - // avoid issues with the first operation being slower than then last one. - var revertIteration = true - repeat(measuredIterations) { - if (revertIteration) { - op2.iterate(refreshRate) - op1.iterate(refreshRate) - } else { - op1.iterate(refreshRate) - op2.iterate(refreshRate) - } - revertIteration = !revertIteration - } - val op1Result = op1.getResult(op1Name) - val op2Result = op2.getResult(op2Name) - - // Let's print the raw results. - println("=====================================") - println(op1Name) - println(op1Result) - println("=====================================") - println(op2Name) - println(op2Result) - println("=====================================") - - return op2Result.compare(op1Result, measuredIterations, refreshRate) - } - } - - private var lastFrameTimeNanos: Long = 0 - private val cpuDurationNanosList: MutableList = LinkedList() - private val droppedFramesList: MutableList = LinkedList() - private val durationNanosList: MutableList = LinkedList() - private val fpsList: MutableList = LinkedList() - - /** Run the operation without measuring it. */ - private fun warmup() { - before?.invoke() - op() - after?.invoke() - isolate() - } - - /** Run the operation and measure it, updating sentry-uitest-android-benchmark data. */ - private fun iterate(refreshRate: Float) { - before?.invoke() - Thread.sleep(200) - frameCallback.setup(refreshRate) - - val startRealtimeNs = SystemClock.elapsedRealtimeNanos() - val startCpuTimeMs = readProcessorTimeNanos() - lastFrameTimeNanos = startRealtimeNs - - choreographer.postFrameCallback(frameCallback) - op() - choreographer.removeFrameCallback(frameCallback) - - val durationNanos = SystemClock.elapsedRealtimeNanos() - startRealtimeNs - cpuDurationNanosList.add(readProcessorTimeNanos() - startCpuTimeMs) - durationNanosList.add(durationNanos) - droppedFramesList.add(frameCallback.droppedFrames) - // fps = counted frames per seconds converted into frames per nanoseconds, divided by duration - // in nanoseconds - // We don't convert the duration into seconds to avoid issues with rounding and possible - // division by 0 - fpsList.add((frameCallback.frames * TimeUnit.SECONDS.toNanos(1) / durationNanos).toInt()) - - after?.invoke() - isolate() - } - - private fun readProcessorTimeNanos(): Long { - val clockSpeedHz = Os.sysconf(OsConstants._SC_CLK_TCK) - // val numCores = Os.sysconf(OsConstants._SC_NPROCESSORS_CONF) - val nanosecondsPerClockTick = 1_000_000_000 / clockSpeedHz.toDouble() - val selfStat = File("/proc/self/stat") - val stats = selfStat.readText().trim().split("[\n\t\r ]".toRegex()) - if (stats.isNotEmpty()) { - val uTime = stats[13].toLong() - val sTime = stats[14].toLong() - val cuTime = stats[15].toLong() - val csTime = stats[16].toLong() - return ((uTime + sTime + cuTime + csTime) * nanosecondsPerClockTick).toLong() - } - return 0 - } - - /** Return the [BenchmarkOperationComparable] for the operation. */ - private fun getResult(operationName: String): BenchmarkOperationComparable = - BenchmarkOperationComparable( - cpuDurationNanosList, - droppedFramesList, - durationNanosList, - fpsList, - operationName, - ) - - /** - * Helps ensure that operations don't impact one another. Doesn't appear to currently have an - * impact on the benchmark. - */ - private fun isolate() { - Thread.sleep(200) - Runtime.getRuntime().gc() - Thread.sleep(200) - } - - private val frameCallback = - object : Choreographer.FrameCallback { - private var expectedFrameDurationNanos: Float = TimeUnit.SECONDS.toNanos(1) / 60F - var frames = 0 - var droppedFrames = 0.0 - - fun setup(refreshRate: Float) { - frames = 0 - droppedFrames = 0.0 - expectedFrameDurationNanos = TimeUnit.SECONDS.toNanos(1) / refreshRate - } - - override fun doFrame(frameTimeNanos: Long) { - frames++ - val timeSinceLastFrameNanos = frameTimeNanos - lastFrameTimeNanos - if (timeSinceLastFrameNanos > expectedFrameDurationNanos) { - // Fractions of frames dropped are weighted to improve the accuracy of the results. - // For example, 31ms between frames is much worse than 17ms, even though both - // durations are within the "1 frame dropped" range. - droppedFrames += timeSinceLastFrameNanos / expectedFrameDurationNanos - 1 - } - lastFrameTimeNanos = frameTimeNanos - choreographer.postFrameCallback(this) - } - } -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationComparable.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationComparable.kt deleted file mode 100644 index ed2b3ac2eae..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationComparable.kt +++ /dev/null @@ -1,79 +0,0 @@ -package io.sentry.uitest.android.benchmark.util - -import java.util.concurrent.TimeUnit - -/** Stores the results of a single run of [BenchmarkOperation]. */ -internal data class BenchmarkOperationComparable( - val cpuTimeNanos: List, - val droppedFrames: List, - val durationNanos: List, - val fps: List, - val operationName: String, -) { - /** Compare two [BenchmarkOperation], calculating increases of each parameter. */ - fun compare( - other: BenchmarkOperationComparable, - iterations: Int, - refreshRate: Float, - ): BenchmarkComparisonResult { - val cores = Runtime.getRuntime().availableProcessors() - val durationIncreaseNanos = ArrayList() - val durationIncreasePercentage = ArrayList() - val cpuTimeIncreaseNanos = ArrayList() - val cpuTimeOverheadPercentage = ArrayList() - val fpsDecrease = ArrayList() - val fpsDecreasePercentage = ArrayList() - val droppedFramesIncrease = ArrayList() - val droppedFramesIncreasePercentage = ArrayList() - - repeat(iterations) { index -> - // Measure average duration - durationIncreaseNanos.add(durationNanos[index] - other.durationNanos[index]) - durationIncreasePercentage.add( - durationIncreaseNanos[index] * 100.0 / other.durationNanos[index] - ) - - // Measure average cpu time - // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 - // thread only. - cpuTimeIncreaseNanos.add((cpuTimeNanos[index] - other.cpuTimeNanos[index]) / cores) - cpuTimeOverheadPercentage.add(cpuTimeIncreaseNanos[index] * 100.0 / other.cpuTimeNanos[index]) - - // Measure average fps - fpsDecrease.add(other.fps[index] - fps[index]) - fpsDecreasePercentage.add(fpsDecrease[index] * 100.0 / other.fps[index]) - - // Measure average dropped frames - droppedFramesIncrease.add(droppedFrames[index] - other.droppedFrames[index]) - val totalExpectedFrames = - TimeUnit.NANOSECONDS.toMillis(other.durationNanos[index]) * refreshRate / 1000 - droppedFramesIncreasePercentage.add( - droppedFramesIncrease[index] * 100 / (totalExpectedFrames - other.droppedFrames[index]) - ) - } - - return BenchmarkComparisonResult( - iterations, - refreshRate, - cores, - operationName, - other.operationName, - cpuTimeNanos, - other.cpuTimeNanos, - cpuTimeIncreaseNanos, - cpuTimeOverheadPercentage, - droppedFrames, - other.droppedFrames, - droppedFramesIncrease, - droppedFramesIncreasePercentage, - durationNanos, - other.durationNanos, - durationIncreaseNanos, - durationIncreasePercentage, - fps, - other.fps, - fpsDecrease, - fpsDecreasePercentage, - ) - } -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml deleted file mode 100644 index 4bcfd5f3dd5..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt deleted file mode 100644 index 6772a7a1e38..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt +++ /dev/null @@ -1,96 +0,0 @@ -package io.sentry.uitest.android.benchmark - -import android.os.Build -import android.os.Bundle -import androidx.appcompat.app.AppCompatActivity -import androidx.recyclerview.widget.LinearLayoutManager -import androidx.recyclerview.widget.RecyclerView -import androidx.test.espresso.idling.CountingIdlingResource -import io.sentry.uitest.android.benchmark.databinding.ActivityBenchmarkBinding -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors - -/** A simple activity with a list of bitmaps. */ -class BenchmarkActivity : AppCompatActivity() { - companion object { - /** The activity will set this when scrolling. */ - val scrollingIdlingResource = - CountingIdlingResource("sentry-uitest-android-benchmark-activityScrolling") - - /** The refresh rate of the device, set on activity create. */ - var refreshRate: Float? = null - - internal const val EXTRA_SUSTAINED_PERFORMANCE_MODE = "EXTRA_SUSTAINED_PERFORMANCE_MODE" - } - - /** - * Each background thread will run non-stop calculations during the benchmark. One such thread - * seems enough to represent a busy application. This number can be increased to mimic busier - * applications. - */ - private val backgroundThreadPoolSize = 1 - private val executor: ExecutorService = Executors.newFixedThreadPool(backgroundThreadPoolSize) - private var resumed = false - private lateinit var binding: ActivityBenchmarkBinding - - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - - if ( - Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && - savedInstanceState?.getBoolean(EXTRA_SUSTAINED_PERFORMANCE_MODE) == true - ) { - window.setSustainedPerformanceMode(true) - } - - refreshRate = - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { - display?.refreshRate - } else { - windowManager.defaultDisplay.refreshRate - } - binding = ActivityBenchmarkBinding.inflate(layoutInflater) - setContentView(binding.root) - - // We show a simple list that changes the idling resource - binding.benchmarkTransactionList.apply { - layoutManager = LinearLayoutManager(this@BenchmarkActivity) - adapter = BenchmarkTransactionListAdapter() - addOnScrollListener( - object : RecyclerView.OnScrollListener() { - override fun onScrollStateChanged(recyclerView: RecyclerView, newState: Int) { - super.onScrollStateChanged(recyclerView, newState) - if (newState == RecyclerView.SCROLL_STATE_DRAGGING) { - scrollingIdlingResource.increment() - } - if (newState == RecyclerView.SCROLL_STATE_IDLE) { - scrollingIdlingResource.decrement() - } - } - } - ) - } - } - - @Suppress("MagicNumber") - override fun onResume() { - super.onResume() - resumed = true - - // Do operations until the activity is paused. - repeat(backgroundThreadPoolSize) { - executor.execute { - var x = 0 - for (i in 0..1_000_000_000) { - x += i * i - if (!resumed) break - } - } - } - } - - override fun onPause() { - super.onPause() - resumed = false - } -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt deleted file mode 100644 index 822d0db8f55..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt +++ /dev/null @@ -1,49 +0,0 @@ -package io.sentry.uitest.android.benchmark - -import android.annotation.SuppressLint -import android.graphics.Bitmap -import android.graphics.Color -import android.view.LayoutInflater -import android.view.ViewGroup -import android.widget.ImageView -import android.widget.TextView -import androidx.recyclerview.widget.RecyclerView -import io.sentry.uitest.android.benchmark.databinding.BenchmarkItemListBinding -import kotlin.random.Random - -/** Simple [RecyclerView.Adapter] that generates a bitmap and a text to show for each item. */ -internal class BenchmarkTransactionListAdapter : RecyclerView.Adapter() { - override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { - val binding = - BenchmarkItemListBinding.inflate(LayoutInflater.from(parent.context), parent, false) - return ViewHolder(binding) - } - - override fun onBindViewHolder(holder: ViewHolder, position: Int) { - holder.imageView.setImageBitmap(generateBitmap()) - - @SuppressLint("SetTextI18n") - holder.textView.text = "Item $position ${"sentry ".repeat(position)}" - } - - @Suppress("MagicNumber") - private fun generateBitmap(): Bitmap { - val bitmapSize = 100 - val colors = - (0 until (bitmapSize * bitmapSize)) - .map { Color.rgb(Random.nextInt(256), Random.nextInt(256), Random.nextInt(256)) } - .toIntArray() - return Bitmap.createBitmap(colors, bitmapSize, bitmapSize, Bitmap.Config.ARGB_8888) - } - - // Disables view recycling. - override fun getItemViewType(position: Int): Int = position - - override fun getItemCount(): Int = 200 -} - -internal class ViewHolder(binding: BenchmarkItemListBinding) : - RecyclerView.ViewHolder(binding.root) { - val imageView: ImageView = binding.benchmarkItemListImage - val textView: TextView = binding.benchmarkItemListText -} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml deleted file mode 100644 index f70a1c2449f..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml deleted file mode 100644 index a3bc14e3a78..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - diff --git a/sentry-android-integration-tests/sentry-uitest-android/README.md b/sentry-android-integration-tests/sentry-uitest-android/README.md index 08389e0c16c..d6ccf2f1f93 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/README.md +++ b/sentry-android-integration-tests/sentry-uitest-android/README.md @@ -6,17 +6,9 @@ By default the envelopes sent to relay are caught by a mock server which allows # How to use Simply run `./gradlew connectedCheck` to run all ui tests of all modules (requires a connected device, either physical or an emulator). -_Care: the benchmarks need to run the tests multiple times to get reliable results. This means they can take a long time (several minutes)._ -If you don't care about benchmark tests you can run `./gradlew connectedCheck -x :sentry-android-integration-tests:sentry-uitest-android-benchmark:connectedCheck`. -You can run benchmark tests only with `./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:connectedCheck`. # SauceLabs To run on saucelabs execute following commands (need also `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` environment variables): -For Benchmarks: -``` -./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -saucectl run -c .sauce/sentry-uitest-android-benchmark.yml -``` For End 2 End: ``` ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest diff --git a/sentry-android-integration-tests/test-app-plain/.gitignore b/sentry-android-integration-tests/test-app-plain/.gitignore deleted file mode 100644 index 42afabfd2ab..00000000000 --- a/sentry-android-integration-tests/test-app-plain/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build \ No newline at end of file diff --git a/sentry-android-integration-tests/test-app-plain/build.gradle.kts b/sentry-android-integration-tests/test-app-plain/build.gradle.kts deleted file mode 100644 index 9778363ede8..00000000000 --- a/sentry-android-integration-tests/test-app-plain/build.gradle.kts +++ /dev/null @@ -1,51 +0,0 @@ -plugins { id("com.android.application") } - -android { - compileSdk = libs.versions.compileSdk.get().toInt() - namespace = "io.sentry.java.tests.perf.appplain" - - defaultConfig { - applicationId = "io.sentry.java.tests.perf.appplain" - minSdk = libs.versions.minSdk.get().toInt() - targetSdk = libs.versions.targetSdk.get().toInt() - versionCode = 1 - versionName = "1.0" - } - - buildTypes { - getByName("release") { - isMinifyEnabled = true - signingConfig = signingConfigs.getByName("debug") // to be able to run release mode - proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") - ndk { - abiFilters.clear() - abiFilters.add("arm64-v8a") - } - } - } - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - buildFeatures { viewBinding = true } - signingConfigs { - getByName("debug") { - storeFile = rootProject.file("debug.keystore") - storePassword = "android" - keyAlias = "androiddebugkey" - keyPassword = "android" - } - } - - androidComponents.beforeVariants { - it.enable = !Config.Android.shouldSkipDebugVariant(it.buildType) - } -} - -dependencies { - implementation("androidx.appcompat:appcompat:1.3.0") - implementation("com.google.android.material:material:1.4.0") - implementation("androidx.constraintlayout:constraintlayout:2.2.1") - implementation("androidx.navigation:navigation-fragment:2.3.5") - implementation("androidx.navigation:navigation-ui:2.3.5") -} diff --git a/sentry-android-integration-tests/test-app-plain/proguard-rules.pro b/sentry-android-integration-tests/test-app-plain/proguard-rules.pro deleted file mode 100644 index d5b66a5b790..00000000000 --- a/sentry-android-integration-tests/test-app-plain/proguard-rules.pro +++ /dev/null @@ -1,23 +0,0 @@ -# Add project specific ProGuard rules here. -# You can control the set of applied configuration files using the -# proguardFiles setting in build.gradle.kts. -# -# For more details, see -# http://developer.android.com/guide/developing/tools/proguard.html - -# If your project uses WebView with JS, uncomment the following -# and specify the fully qualified class name to the JavaScript interface -# class: -#-keepclassmembers class fqcn.of.javascript.interface.for.webview { -# public *; -#} - -# Uncomment this to preserve the line number information for -# debugging stack traces. -#-keepattributes SourceFile,LineNumberTable - -# If you keep the line number information, uncomment this to -# hide the original source file name. -#-renamesourcefileattribute SourceFile - --keep,allowobfuscation,allowshrinking class * extends androidx.navigation.Navigator diff --git a/sentry-android-integration-tests/test-app-plain/src/main/AndroidManifest.xml b/sentry-android-integration-tests/test-app-plain/src/main/AndroidManifest.xml deleted file mode 100644 index 9b9f7a910cb..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/AndroidManifest.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - diff --git a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/FirstFragment.java b/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/FirstFragment.java deleted file mode 100644 index 473aaf8aa76..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/FirstFragment.java +++ /dev/null @@ -1,42 +0,0 @@ -package io.sentry.java.tests.perf.appplain; - -import android.os.Bundle; -import android.view.LayoutInflater; -import android.view.View; -import android.view.ViewGroup; -import androidx.annotation.NonNull; -import androidx.fragment.app.Fragment; -import androidx.navigation.fragment.NavHostFragment; -import io.sentry.java.tests.perf.appplain.databinding.FragmentFirstBinding; - -public class FirstFragment extends Fragment { - - private FragmentFirstBinding binding; - - @Override - public View onCreateView( - LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { - - binding = FragmentFirstBinding.inflate(inflater, container, false); - return binding.getRoot(); - } - - public void onViewCreated(@NonNull View view, Bundle savedInstanceState) { - super.onViewCreated(view, savedInstanceState); - - binding.buttonFirst.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View view) { - NavHostFragment.findNavController(FirstFragment.this) - .navigate(R.id.action_FirstFragment_to_SecondFragment); - } - }); - } - - @Override - public void onDestroyView() { - super.onDestroyView(); - binding = null; - } -} diff --git a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/MainActivity.java b/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/MainActivity.java deleted file mode 100644 index e37de2be3a3..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/MainActivity.java +++ /dev/null @@ -1,74 +0,0 @@ -package io.sentry.java.tests.perf.appplain; - -import android.os.Bundle; -import android.view.Menu; -import android.view.MenuItem; -import android.view.View; -import androidx.appcompat.app.AppCompatActivity; -import androidx.navigation.NavController; -import androidx.navigation.Navigation; -import androidx.navigation.ui.AppBarConfiguration; -import androidx.navigation.ui.NavigationUI; -import com.google.android.material.snackbar.Snackbar; -import io.sentry.java.tests.perf.appplain.databinding.ActivityMainBinding; - -public class MainActivity extends AppCompatActivity { - - private AppBarConfiguration appBarConfiguration; - private ActivityMainBinding binding; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - binding = ActivityMainBinding.inflate(getLayoutInflater()); - setContentView(binding.getRoot()); - - setSupportActionBar(binding.toolbar); - - NavController navController = - Navigation.findNavController(this, R.id.nav_host_fragment_content_main); - appBarConfiguration = new AppBarConfiguration.Builder(navController.getGraph()).build(); - NavigationUI.setupActionBarWithNavController(this, navController, appBarConfiguration); - - binding.fab.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View view) { - Snackbar.make(view, "Replace with your own action", Snackbar.LENGTH_LONG) - .setAction("Action", null) - .show(); - } - }); - } - - @Override - public boolean onCreateOptionsMenu(Menu menu) { - // Inflate the menu; this adds items to the action bar if it is present. - getMenuInflater().inflate(R.menu.menu_main, menu); - return true; - } - - @Override - public boolean onOptionsItemSelected(MenuItem item) { - // Handle action bar item clicks here. The action bar will - // automatically handle clicks on the Home/Up button, so long - // as you specify a parent activity in AndroidManifest.xml. - int id = item.getItemId(); - - //noinspection SimplifiableIfStatement - if (id == R.id.action_settings) { - return true; - } - - return super.onOptionsItemSelected(item); - } - - @Override - public boolean onSupportNavigateUp() { - NavController navController = - Navigation.findNavController(this, R.id.nav_host_fragment_content_main); - return NavigationUI.navigateUp(navController, appBarConfiguration) - || super.onSupportNavigateUp(); - } -} diff --git a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/SecondFragment.java b/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/SecondFragment.java deleted file mode 100644 index cf04c92febf..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/SecondFragment.java +++ /dev/null @@ -1,42 +0,0 @@ -package io.sentry.java.tests.perf.appplain; - -import android.os.Bundle; -import android.view.LayoutInflater; -import android.view.View; -import android.view.ViewGroup; -import androidx.annotation.NonNull; -import androidx.fragment.app.Fragment; -import androidx.navigation.fragment.NavHostFragment; -import io.sentry.java.tests.perf.appplain.databinding.FragmentSecondBinding; - -public class SecondFragment extends Fragment { - - private FragmentSecondBinding binding; - - @Override - public View onCreateView( - LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { - - binding = FragmentSecondBinding.inflate(inflater, container, false); - return binding.getRoot(); - } - - public void onViewCreated(@NonNull View view, Bundle savedInstanceState) { - super.onViewCreated(view, savedInstanceState); - - binding.buttonSecond.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View view) { - NavHostFragment.findNavController(SecondFragment.this) - .navigate(R.id.action_SecondFragment_to_FirstFragment); - } - }); - } - - @Override - public void onDestroyView() { - super.onDestroyView(); - binding = null; - } -} diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable-v24/ic_launcher_foreground.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/drawable-v24/ic_launcher_foreground.xml deleted file mode 100644 index 2b068d11462..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable-v24/ic_launcher_foreground.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable/ic_launcher_background.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/drawable/ic_launcher_background.xml deleted file mode 100644 index 07d5da9cbf1..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable/ic_launcher_background.xml +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/activity_main.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/layout/activity_main.xml deleted file mode 100644 index 1acd302c3fc..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/activity_main.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/content_main.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/layout/content_main.xml deleted file mode 100644 index e416e1c18d5..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/content_main.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - \ No newline at end of file diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/fragment_first.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/layout/fragment_first.xml deleted file mode 100644 index fb44a3d9176..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/fragment_first.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - -