diff --git a/CHANGELOG.md b/CHANGELOG.md index ef33d0641..a8520dc37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ We use [semantic versioning](http://semver.org/): - PATCH version when you make backwards compatible bug fixes. # Next version +- [fix] _impacted-test-engine_: Tests in a `@ParameterizedClass` are now collected. +- [fix] _impacted-test-engine_: A test failure inside a nested test container (e.g. an invocation of a `@ParameterizedClass`) is no longer swallowed. +- [fix] _teamscale-jacoco-agent_, _teamscale-maven-plugin_: A test that was executed more than once (e.g. once per parameter set of a `@ParameterizedClass`) is now reported once in the testwise coverage report, with the coverage of all of its executions merged, their durations summed up and the most severe of their results. Previously each execution overwrote the previous one. # 38.1.0 - [feature] _agent_: `git-properties-jar` now also accepts a folder, which is searched for `git.properties` files. Previously, a folder was rejected with a warning and no commit was auto-detected. diff --git a/agent/src/main/kotlin/com/teamscale/jacoco/agent/convert/Converter.kt b/agent/src/main/kotlin/com/teamscale/jacoco/agent/convert/Converter.kt index b3e20cada..90d803d11 100644 --- a/agent/src/main/kotlin/com/teamscale/jacoco/agent/convert/Converter.kt +++ b/agent/src/main/kotlin/com/teamscale/jacoco/agent/convert/Converter.kt @@ -94,9 +94,7 @@ class Converter arguments.getOutputFile(), arguments.splitAfter, null ).use { coverageWriter -> - jacocoExecutionDataList.forEach { executionDataFile -> - generator.convertAndConsume(executionDataFile, coverageWriter) - } + generator.convertAndConsumePerTest(jacocoExecutionDataList, coverageWriter) } } } diff --git a/agent/src/main/kotlin/com/teamscale/jacoco/agent/testimpact/CoverageViaHttpStrategy.kt b/agent/src/main/kotlin/com/teamscale/jacoco/agent/testimpact/CoverageViaHttpStrategy.kt index a3c0a4714..ca1245563 100644 --- a/agent/src/main/kotlin/com/teamscale/jacoco/agent/testimpact/CoverageViaHttpStrategy.kt +++ b/agent/src/main/kotlin/com/teamscale/jacoco/agent/testimpact/CoverageViaHttpStrategy.kt @@ -28,8 +28,8 @@ class CoverageViaHttpStrategy( val builder = TestInfoBuilder(test) val dump = controller.dumpAndReset() reportGenerator.updateClassDirCache() - reportGenerator.convert(dump)?.let { builder.setCoverage(it) } - builder.setExecution(testExecution) + reportGenerator.convert(dump)?.let { builder.addCoverage(it) } + builder.addExecution(testExecution) val testInfo = builder.build() logger.debug("Generated test info {}", testInfo) return testInfo diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f670272ae..8ad4cfcc2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -57,6 +57,7 @@ log4j-core = { module = "org.apache.logging.log4j:log4j-core", version = "2.26.1 junit-bom = { module = "org.junit:junit-bom", version = "6.1.3" } junit-jupiter = { module = "org.junit.jupiter:junit-jupiter" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" } +junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine" } junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher" } junit-platform-engine = { module = "org.junit.platform:junit-platform-engine" } junit-platform-commons = { module = "org.junit.platform:junit-platform-commons" } diff --git a/impacted-test-engine/build.gradle.kts b/impacted-test-engine/build.gradle.kts index bca5198d1..747365299 100644 --- a/impacted-test-engine/build.gradle.kts +++ b/impacted-test-engine/build.gradle.kts @@ -32,6 +32,13 @@ dependencies { compileOnly(libs.junit.platform.engine) compileOnly(libs.junit.platform.commons) testImplementation(libs.junit.platform.engine) + testImplementation(libs.junit.platform.launcher) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.junit.jupiter.engine) testImplementation(libs.mockito.kotlin) } + +tasks.test { + // The sample tests are discovered explicitly by JupiterClassTemplateTest and must not be run by Gradle itself. + exclude("**/test_descriptor/samples/**") +} diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/InternalImpactedTestEngine.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/InternalImpactedTestEngine.kt index e1fb9665a..0f8efebeb 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/InternalImpactedTestEngine.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/InternalImpactedTestEngine.kt @@ -3,6 +3,7 @@ package com.teamscale.test_impacted.engine import com.teamscale.test_impacted.commons.LoggerUtils.createLogger import com.teamscale.test_impacted.engine.ImpactedTestEngine.Companion.ENGINE_NAME import com.teamscale.test_impacted.engine.executor.TestwiseCoverageCollectingExecutionListener +import com.teamscale.test_impacted.test_descriptor.ClassTemplateRegistry import com.teamscale.test_impacted.test_descriptor.TestDescriptorResolverRegistry.getTestDescriptorResolver import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.getAvailableTests import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.getTestDescriptorAsString @@ -30,6 +31,12 @@ internal class InternalImpactedTestEngine( private val teamscaleAgentNotifier = configuration.teamscaleAgentNotifier private val testDataWriter = configuration.testDataWriter + /** + * The tests of the `@ParameterizedClass`es in the test tree, recorded during discovery because the JUnit platform + * prunes them from the tree before the tests are executed. + */ + private val classTemplateRegistry = ClassTemplateRegistry() + /** * Performs test discovery by aggregating the result of all [TestEngine]s from the [TestEngineRegistry] * in a single engine [TestDescriptor]. @@ -47,6 +54,7 @@ internal class InternalImpactedTestEngine( ) engineDescriptor.addChild(delegateEngineDescriptor) + classTemplateRegistry.record(delegateEngineDescriptor) } LOG.fine { @@ -64,7 +72,7 @@ internal class InternalImpactedTestEngine( */ fun execute(request: ExecutionRequest) { val rootTestDescriptor = request.rootTestDescriptor - val availableTests = getAvailableTests(rootTestDescriptor) + val availableTests = getAvailableTests(rootTestDescriptor, classTemplateRegistry) LOG.fine { "Starting selection and sorting ${ImpactedTestEngine.ENGINE_ID}:\n${ @@ -72,7 +80,7 @@ internal class InternalImpactedTestEngine( }" } - testSorter.selectAndSort(rootTestDescriptor) + testSorter.selectAndSort(rootTestDescriptor, availableTests) LOG.fine { "Starting execution of request for engine ${ImpactedTestEngine.ENGINE_ID}:\n${ @@ -105,7 +113,8 @@ internal class InternalImpactedTestEngine( TestwiseCoverageCollectingExecutionListener( teamscaleAgentNotifier, testDescriptorResolver, - request.engineExecutionListener + request.engineExecutionListener, + classTemplateRegistry ) testEngine.execute( diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ITestSorter.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ITestSorter.kt index 2fb604a3f..109725401 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ITestSorter.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ITestSorter.kt @@ -7,6 +7,8 @@ interface ITestSorter { /** * Removes any tests from the test descriptor that should not be executed and changes the execution order of the * remaining tests. + * + * @param availableTests The tests contained in the given [testDescriptor], as determined during discovery. */ - fun selectAndSort(testDescriptor: TestDescriptor) + fun selectAndSort(testDescriptor: TestDescriptor, availableTests: AvailableTests) } diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ImpactedTestsSorter.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ImpactedTestsSorter.kt index 37b3812e7..b5e5e8523 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ImpactedTestsSorter.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/ImpactedTestsSorter.kt @@ -2,7 +2,6 @@ package com.teamscale.test_impacted.engine.executor import com.teamscale.client.TestWithClusterId.Companion.fromClusteredTestDetails import com.teamscale.test_impacted.engine.ImpactedTestEngine -import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.getAvailableTests import org.junit.platform.engine.TestDescriptor import java.util.* @@ -12,9 +11,7 @@ import java.util.* */ class ImpactedTestsSorter(private val impactedTestsProvider: ImpactedTestsProvider) : ITestSorter { - override fun selectAndSort(testDescriptor: TestDescriptor) { - val availableTests = getAvailableTests(testDescriptor) - + override fun selectAndSort(testDescriptor: TestDescriptor, availableTests: AvailableTests) { val testClusters = impactedTestsProvider.getImpactedTestsFromTeamscale( availableTests.testList .map { fromClusteredTestDetails(it, impactedTestsProvider.partition) }) diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/NOPTestSorter.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/NOPTestSorter.kt index 07f86acdc..f6f5dfe4c 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/NOPTestSorter.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/NOPTestSorter.kt @@ -7,7 +7,7 @@ import org.junit.platform.engine.TestDescriptor * Teamscale to select or prioritize tests. */ class NOPTestSorter : ITestSorter { - override fun selectAndSort(testDescriptor: TestDescriptor) { + override fun selectAndSort(testDescriptor: TestDescriptor, availableTests: AvailableTests) { // Nothing to do } } diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListener.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListener.kt index 701c2619a..29cdc9d82 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListener.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListener.kt @@ -3,7 +3,9 @@ package com.teamscale.test_impacted.engine.executor import com.teamscale.report.testwise.model.ETestExecutionResult import com.teamscale.report.testwise.model.TestExecution import com.teamscale.test_impacted.commons.LoggerUtils.createLogger +import com.teamscale.test_impacted.test_descriptor.ClassTemplateRegistry import com.teamscale.test_impacted.test_descriptor.ITestDescriptorResolver +import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.isClassTemplate import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.isRepresentative import org.junit.platform.engine.EngineExecutionListener import org.junit.platform.engine.TestDescriptor @@ -24,11 +26,13 @@ import java.io.StringWriter * @param teamscaleAgentNotifier The notifier responsible for signaling test events to the Teamscale JaCoCo agent. * @param testDescriptorResolver A resolver interface used to map [TestDescriptor] objects to uniform paths. * @param delegateEngineExecutionListener The underlying [EngineExecutionListener] to which events are delegated. + * @param classTemplateRegistry The tests of the `@ParameterizedClass`es as recorded during discovery. */ class TestwiseCoverageCollectingExecutionListener( private val teamscaleAgentNotifier: TeamscaleAgentNotifier, private val testDescriptorResolver: ITestDescriptorResolver, - private val delegateEngineExecutionListener: EngineExecutionListener + private val delegateEngineExecutionListener: EngineExecutionListener, + private val classTemplateRegistry: ClassTemplateRegistry ) : EngineExecutionListener { companion object { private val LOG = createLogger() @@ -47,6 +51,14 @@ class TestwiseCoverageCollectingExecutionListener( } override fun executionSkipped(testDescriptor: TestDescriptor, reason: String) { + if (testDescriptor.isClassTemplate()) { + // The tests of a @ParameterizedClass were pruned from the test tree, so report the ones that were + // recorded during discovery instead of descending into the now empty descriptor. They are not forwarded + // to the delegate listener, which only knows the descriptors that are still part of the tree. + testDescriptor.reportSkipped(reason) + delegateEngineExecutionListener.executionSkipped(testDescriptor, reason) + return + } if (!testDescriptor.isRepresentative()) { delegateEngineExecutionListener.executionStarted(testDescriptor) testDescriptor.children.forEach { executionSkipped(it, reason) } @@ -67,6 +79,18 @@ class TestwiseCoverageCollectingExecutionListener( } } + /** Records a [ETestExecutionResult.SKIPPED] execution for every test below the given descriptor. */ + private fun TestDescriptor.reportSkipped(reason: String) { + if (isRepresentative()) { + testDescriptorResolver.getUniformPath(this)?.let { testUniformPath -> + testExecutions.add(TestExecution(testUniformPath, 0L, ETestExecutionResult.SKIPPED, reason)) + } + return + } + val tests = if (isClassTemplate()) classTemplateRegistry.testsOf(this) else children + tests.forEach { it.reportSkipped(reason) } + } + override fun executionStarted(testDescriptor: TestDescriptor) { if (testDescriptor.isRepresentative()) { testDescriptorResolver.getUniformPath(testDescriptor)?.let { testUniformPath -> @@ -90,6 +114,10 @@ class TestwiseCoverageCollectingExecutionListener( val testExecutionResults = testResultCache.computeIfAbsent( testDescriptor.parent.get().uniqueId ) { mutableListOf() } + // Containers may be nested arbitrarily deep below their representative, e.g. a @ParameterizedClass + // contains one invocation per parameter set which in turn contains the test methods. Hand the results + // collected for this container up to its parent so that they reach the representative. + testResultCache.remove(testDescriptor.uniqueId)?.let { testExecutionResults.addAll(it) } testExecutionResults.add(testExecutionResult) } @@ -108,14 +136,16 @@ class TestwiseCoverageCollectingExecutionListener( val message = StringBuilder() var status = TestExecutionResult.Status.SUCCESSFUL testExecutionResults.forEach { executionResult -> - if (message.isNotEmpty()) { - message.append("\n\n") - } - message.append(executionResult.throwable.orElse(null).buildStacktrace()) // Aggregate status here to most severe status according to SUCCESSFUL < ABORTED < FAILED if (status.ordinal < executionResult.status.ordinal) { status = executionResult.status } + + val stacktrace = executionResult.throwable.orElse(null).buildStacktrace() ?: return@forEach + if (message.isNotEmpty()) { + message.append("\n\n") + } + message.append(stacktrace) } return buildTestExecution(testUniformPath, duration, status, message.toString()) diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/ClassTemplateRegistry.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/ClassTemplateRegistry.kt new file mode 100644 index 000000000..20effb342 --- /dev/null +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/ClassTemplateRegistry.kt @@ -0,0 +1,34 @@ +package com.teamscale.test_impacted.test_descriptor + +import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.isClassTemplate +import org.junit.platform.engine.TestDescriptor +import org.junit.platform.engine.UniqueId + +/** + * Remembers the tests of the `@ParameterizedClass`es in the discovered test tree. + * + * The JUnit platform prunes the tests of a `@ParameterizedClass` out of the test tree right after discovery and only + * re-registers them, once per parameter set, while the class is being executed. They are therefore no longer visible + * when the engine collects the available tests, which is why we have to record them while they are still there. + */ +class ClassTemplateRegistry { + private val testsByClassTemplate = mutableMapOf>() + + /** + * Records the tests of all `@ParameterizedClass`es below the given descriptor. Must be called during discovery, + * i.e. before the JUnit platform prunes the test tree. + */ + fun record(testDescriptor: TestDescriptor) { + if (testDescriptor.isClassTemplate() && testDescriptor.children.isNotEmpty()) { + testsByClassTemplate[testDescriptor.uniqueId] = testDescriptor.children.toList() + } + testDescriptor.children.forEach { record(it) } + } + + /** + * Returns the tests that were recorded for the given `@ParameterizedClass` during discovery, falling back to its + * current children if nothing was recorded for it. + */ + fun testsOf(classTemplate: TestDescriptor): Collection = + testsByClassTemplate[classTemplate.uniqueId] ?: classTemplate.children +} diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitClassBasedTestDescriptorResolverBase.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitClassBasedTestDescriptorResolverBase.kt index 4c62eb4bf..84265714d 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitClassBasedTestDescriptorResolverBase.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitClassBasedTestDescriptorResolverBase.kt @@ -12,9 +12,13 @@ abstract class JUnitClassBasedTestDescriptorResolverBase : ITestDescriptorResolv override fun getUniformPath(descriptor: TestDescriptor): String? = descriptor.getClassName()?.let { className -> val dotName = className.replace(".", "/") - "$dotName/${descriptor.legacyReportingName.trim { it <= ' ' }}" + "$dotName/${getTestName(descriptor)}" } + /** Returns the name of the test within its class. */ + protected open fun getTestName(descriptor: TestDescriptor): String = + descriptor.legacyReportingName.trim { it <= ' ' } + override fun getClusterId(descriptor: TestDescriptor): String? { val classSegmentName = descriptor.getClassName() diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolver.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolver.kt index 01e5baf91..2a21858fd 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolver.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolver.kt @@ -1,15 +1,28 @@ package com.teamscale.test_impacted.test_descriptor -import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.getUniqueIdSegment +import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.isInsideClassTemplate import org.junit.platform.engine.TestDescriptor /** Test default test descriptor resolver for the JUnit jupiter [TestEngine]. */ class JUnitJupiterTestDescriptorResolver : JUnitClassBasedTestDescriptorResolverBase() { + override fun getTestName(descriptor: TestDescriptor): String { + val reportingName = super.getTestName(descriptor) + if (!descriptor.isInsideClassTemplate()) { + return reportingName + } + // Within a @ParameterizedClass the jupiter engine appends the index of the enclosing invocation(s) to the + // reporting name, e.g. "testOne()[1]". All invocations run the same test, so the index must not become part + // of the uniform path: whatever the test covered in any of them belongs to that one test. + return reportingName.replace(INVOCATION_INDEXES, "") + } + override fun TestDescriptor.getClassName(): String? { - val classSegment = getUniqueIdSegment(CLASS_SEGMENT_TYPE).orElse(null) ?: return null + val classSegment = uniqueId.segments + .firstOrNull { it.type == CLASS_SEGMENT_TYPE || it.type == CLASS_TEMPLATE_SEGMENT_TYPE } + ?.value ?: return null val nestedClassNames = uniqueId.segments - .filter { it.type == NESTED_CLASS_SEGMENT_TYPE } + .filter { it.type == NESTED_CLASS_SEGMENT_TYPE || it.type == NESTED_CLASS_TEMPLATE_SEGMENT_TYPE } .joinToString("") { "\$${it.value}" } return classSegment + nestedClassNames @@ -19,12 +32,27 @@ class JUnitJupiterTestDescriptorResolver : JUnitClassBasedTestDescriptorResolver get() = "junit-jupiter" companion object { + /** The invocation indexes that the jupiter engine appends to reporting names within a @ParameterizedClass. */ + private val INVOCATION_INDEXES = Regex("(\\[\\d+])+$") + /** The segment type name that the jupiter engine uses for the class descriptor nodes. */ const val CLASS_SEGMENT_TYPE = "class" /** The segment type name that the jupiter engine uses for @Nested inner class descriptor nodes. */ const val NESTED_CLASS_SEGMENT_TYPE = "nested-class" + /** The segment type name that the jupiter engine uses for top-level @ParameterizedClass descriptor nodes. */ + const val CLASS_TEMPLATE_SEGMENT_TYPE = "class-template" + + /** The segment type name that the jupiter engine uses for @Nested @ParameterizedClass descriptor nodes. */ + const val NESTED_CLASS_TEMPLATE_SEGMENT_TYPE = "nested-class-template" + + /** + * The segment type name that the jupiter engine uses for the individual invocations of a @ParameterizedClass. + * These are only registered dynamically during test execution. + */ + const val CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE = "class-template-invocation" + /** The segment type name that the jupiter engine uses for the method descriptor nodes. */ const val METHOD_SEGMENT_TYPE = "method" diff --git a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/TestDescriptorUtils.kt b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/TestDescriptorUtils.kt index 1094e9d56..34921ccc5 100644 --- a/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/TestDescriptorUtils.kt +++ b/impacted-test-engine/src/main/kotlin/com/teamscale/test_impacted/test_descriptor/TestDescriptorUtils.kt @@ -5,6 +5,7 @@ import com.teamscale.test_impacted.commons.IndentingWriter import com.teamscale.test_impacted.commons.LoggerUtils.createLogger import com.teamscale.test_impacted.engine.executor.AvailableTests import org.junit.platform.engine.TestDescriptor +import org.junit.platform.engine.UniqueId import org.junit.platform.engine.support.descriptor.ClassSource import org.junit.platform.engine.support.descriptor.MethodSource import java.util.* @@ -37,10 +38,33 @@ object TestDescriptorUtils { */ fun TestDescriptor.isRepresentative(): Boolean { val isTestTemplateOrTestFactory = isTestTemplateOrTestFactory() - val isNonParameterizedTest = isTest && !parent.get().isTestTemplateOrTestFactory() + val isNonParameterizedTest = isTest && parent.orElse(null)?.isTestTemplateOrTestFactory() != true return isNonParameterizedTest || isTestTemplateOrTestFactory } + /** + * Returns true if a [TestDescriptor] represents a `@ParameterizedClass`. + * + * An example of a [org.junit.platform.engine.UniqueId] of such a [TestDescriptor] is: + * + * + * `[engine:junit-jupiter]/[class:com.example.project.JUnit5Test]/[nested-class-template:WithValueSource]` + */ + fun TestDescriptor.isClassTemplate(): Boolean { + val lastSegmentType = uniqueId.segments.lastOrNull()?.type ?: return false + return JUnitJupiterTestDescriptorResolver.CLASS_TEMPLATE_SEGMENT_TYPE == lastSegmentType + || JUnitJupiterTestDescriptorResolver.NESTED_CLASS_TEMPLATE_SEGMENT_TYPE == lastSegmentType + } + + /** + * Returns true if a [TestDescriptor] is one of the invocations of a `@ParameterizedClass` or is contained in one. + * These are only registered dynamically during test execution. + */ + fun TestDescriptor.isInsideClassTemplate() = + uniqueId.segments.any { + it.type == JUnitJupiterTestDescriptorResolver.CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE + } + /** * Returns true if a [TestDescriptor] represents a test template or a test factory. * @@ -61,13 +85,28 @@ object TestDescriptorUtils { || JUnitJupiterTestDescriptorResolver.TEST_FACTORY_SEGMENT_TYPE == lastSegmentType } - /** Creates a stream of the test representatives contained by the [TestDescriptor]. */ - private fun TestDescriptor.streamTestRepresentatives(): Stream { + /** + * Creates a stream of the test representatives contained by the [TestDescriptor], each together with the + * [UniqueId] that has to be selected in order to execute it. + * + * Both are the same for an ordinary test. The tests of a `@ParameterizedClass` are taken from the + * [ClassTemplateRegistry], because the JUnit platform pruned them from the test tree, and all of them are selected + * via the class template itself, since JUnit can only execute a `@ParameterizedClass` as a whole. + */ + private fun TestDescriptor.streamTestRepresentatives( + classTemplateRegistry: ClassTemplateRegistry, + selectionId: UniqueId? + ): Stream> { + if (isClassTemplate()) { + return classTemplateRegistry.testsOf(this).stream().flatMap { + it.streamTestRepresentatives(classTemplateRegistry, selectionId ?: uniqueId) + } + } if (isRepresentative()) { - return Stream.of(this) + return Stream.of(this to (selectionId ?: uniqueId)) } return children.stream().flatMap { - it.streamTestRepresentatives() + it.streamTestRepresentatives(classTemplateRegistry, selectionId) } } @@ -88,14 +127,19 @@ object TestDescriptorUtils { } } - /** Returns the [AvailableTests] contained within the root [TestDescriptor]. */ + /** + * Returns the [AvailableTests] contained within the root [TestDescriptor], taking the tests of the + * `@ParameterizedClass`es from the given [ClassTemplateRegistry] because the JUnit platform pruned them from the + * test tree. + */ fun getAvailableTests( - rootTestDescriptor: TestDescriptor + rootTestDescriptor: TestDescriptor, + classTemplateRegistry: ClassTemplateRegistry ): AvailableTests { val availableTests = AvailableTests() - rootTestDescriptor.streamTestRepresentatives() - .forEach { testDescriptor -> + rootTestDescriptor.streamTestRepresentatives(classTemplateRegistry, null) + .forEach { (testDescriptor, selectionId) -> val engineId = testDescriptor.uniqueId.engineId if (!engineId.isPresent) { LOG.severe { @@ -131,7 +175,7 @@ object TestDescriptorUtils { null, clusterId ) - availableTests.add(testDescriptor.uniqueId, testDetails) + availableTests.add(selectionId, testDetails) } diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/ImpactedTestEngineTestBase.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/ImpactedTestEngineTestBase.kt index 816207a04..6dd41e687 100644 --- a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/ImpactedTestEngineTestBase.kt +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/ImpactedTestEngineTestBase.kt @@ -34,6 +34,8 @@ abstract class ImpactedTestEngineTestBase { Assertions.assertThat(engineDescriptor.uniqueId) .isEqualTo(UniqueId.forEngine(ImpactedTestEngine.ENGINE_ID)) + afterDiscovery() + whenever(executionRequest.engineExecutionListener) .thenReturn(executionListener) whenever(executionRequest.rootTestDescriptor) @@ -62,6 +64,14 @@ abstract class ImpactedTestEngineTestBase { /** Verifies that the interactions with the executionListener are the ones we would expect. */ abstract fun verifyCallbacks(executionListener: EngineExecutionListener) + /** + * Hook for changes that the JUnit platform applies to the test tree between discovery and execution, e.g. the + * pruning of the tests of a `@ParameterizedClass`. + */ + open fun afterDiscovery() { + // Nothing to do by default + } + private fun createInternalImpactedTestEngine(engines: List): InternalImpactedTestEngine { engines.forEach { engine -> whenever(testEngineRegistry.getTestEngine(eq(engine.id))) diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/ImpactedTestEngineWithParameterizedClassTest.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/ImpactedTestEngineWithParameterizedClassTest.kt new file mode 100644 index 000000000..880468da3 --- /dev/null +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/ImpactedTestEngineWithParameterizedClassTest.kt @@ -0,0 +1,87 @@ +package com.teamscale.test_impacted.engine + +import com.teamscale.client.PrioritizableTest +import com.teamscale.client.PrioritizableTestCluster +import com.teamscale.test_impacted.engine.executor.DummyEngine +import com.teamscale.test_impacted.engine.executor.SimpleTestDescriptor +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_TEMPLATE_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.METHOD_SEGMENT_TYPE +import org.junit.platform.engine.EngineExecutionListener +import org.junit.platform.engine.TestExecutionResult +import org.junit.platform.engine.UniqueId +import org.mockito.kotlin.verify + +/** + * Test setup for a JUnit Jupiter `@ParameterizedClass`. Its tests are visible during discovery, then pruned from the + * test tree by the JUnit platform, and finally registered again during execution, once per parameter set. Each of them + * is one test, but they can only be selected via the class template, since JUnit executes the class as a whole. + * + * The setup contains a second, non-impacted test class, so that the test fails if the engine cannot map the impacted + * test back to a local one and therefore falls back to executing everything. + */ +internal class ImpactedTestEngineWithParameterizedClassTest : ImpactedTestEngineTestBase() { + private val engineRootId = UniqueId.forEngine("junit-jupiter") + + private val classTemplateId = engineRootId.append(CLASS_TEMPLATE_SEGMENT_TYPE, "example.ParameterizedTest") + + /** The test as it is visible while the engine discovers the tests. */ + private val discoveredTestCase = + SimpleTestDescriptor.testCase(classTemplateId.append(METHOD_SEGMENT_TYPE, "testMethod()")) + + private val classTemplate = SimpleTestDescriptor.testContainer(classTemplateId, discoveredTestCase) + + /** A test that is not impacted and must therefore not be executed. */ + private val nonImpactedClassId = engineRootId.append(CLASS_SEGMENT_TYPE, "example.OtherTest") + private val nonImpactedTestCase = + SimpleTestDescriptor.testCase(nonImpactedClassId.append(METHOD_SEGMENT_TYPE, "otherTest()")) + private val nonImpactedClass = SimpleTestDescriptor.testContainer(nonImpactedClassId, nonImpactedTestCase) + + private val testRoot = SimpleTestDescriptor.testContainer(engineRootId, classTemplate, nonImpactedClass) + + /** One invocation per parameter set, each repeating all tests of the class. */ + private val invocations = (1..2).map { invocationIndex -> + val invocationId = classTemplateId.append(CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE, "#$invocationIndex") + SimpleTestDescriptor.dynamicTestContainer( + invocationId, + SimpleTestDescriptor.testCase( + invocationId.append(METHOD_SEGMENT_TYPE, "testMethod()"), "testMethod()[$invocationIndex]" + ) + ) + } + + override fun afterDiscovery() { + classTemplate.removeChild(discoveredTestCase) + classTemplate.dynamicTests.addAll(invocations) + } + + override val engines = listOf(DummyEngine(testRoot)) + + override val impactedTests = + listOf( + PrioritizableTestCluster( + "example.ParameterizedTest", + listOf(PrioritizableTest("example/ParameterizedTest/testMethod()")) + ) + ) + + override fun verifyCallbacks(executionListener: EngineExecutionListener) { + verify(executionListener).executionStarted(testRoot) + verify(executionListener).executionStarted(classTemplate) + + invocations.forEach { invocation -> + verify(executionListener).dynamicTestRegistered(invocation) + verify(executionListener).executionStarted(invocation) + invocation.children.forEach { testCase -> + verify(executionListener).dynamicTestRegistered(testCase) + verify(executionListener).executionStarted(testCase) + verify(executionListener).executionFinished(testCase, TestExecutionResult.successful()) + } + verify(executionListener).executionFinished(invocation, TestExecutionResult.successful()) + } + + verify(executionListener).executionFinished(classTemplate, TestExecutionResult.successful()) + verify(executionListener).executionFinished(testRoot, TestExecutionResult.successful()) + } +} diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/SimpleTestDescriptor.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/SimpleTestDescriptor.kt index 202fe1b6c..586765c57 100644 --- a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/SimpleTestDescriptor.kt +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/SimpleTestDescriptor.kt @@ -55,9 +55,14 @@ class SimpleTestDescriptor private constructor( } companion object { - /** Creates a [TestDescriptor] for a concrete test case without children. */ - fun testCase(uniqueId: UniqueId) = - SimpleTestDescriptor(uniqueId, TestDescriptor.Type.TEST, getSimpleDisplayName(uniqueId)) + /** + * Creates a [TestDescriptor] for a concrete test case without children. The display name doubles as the + * descriptor's legacy reporting name, which the jupiter engine e.g. suffixes with the index of the enclosing + * `@ParameterizedClass` invocation. + */ + @JvmOverloads + fun testCase(uniqueId: UniqueId, displayName: String = getSimpleDisplayName(uniqueId)) = + SimpleTestDescriptor(uniqueId, TestDescriptor.Type.TEST, displayName) private fun getSimpleDisplayName(uniqueId: UniqueId) = uniqueId.segments[uniqueId.segments.size - 1].value @@ -72,6 +77,15 @@ class SimpleTestDescriptor private constructor( return simpleTestDescriptor } + /** + * Creates a [TestDescriptor] for a test container (e.g. a `@ParameterizedClass`) which registers all of its + * children dynamically during test execution. + */ + fun dynamicTestContainer(uniqueId: UniqueId, vararg dynamicChildren: TestDescriptor) = + SimpleTestDescriptor(uniqueId, TestDescriptor.Type.CONTAINER, getSimpleDisplayName(uniqueId)).apply { + dynamicTests.addAll(dynamicChildren) + } + /** * Creates a [TestDescriptor] for a test container (e.g., a test class or test engine) containing other * [TestDescriptor] children. diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListenerTest.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListenerTest.kt index ae1e94016..d932266cb 100644 --- a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListenerTest.kt +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/engine/executor/TestwiseCoverageCollectingExecutionListenerTest.kt @@ -2,9 +2,17 @@ package com.teamscale.test_impacted.engine.executor import com.teamscale.report.testwise.model.ETestExecutionResult import com.teamscale.report.testwise.model.TestExecution +import com.teamscale.test_impacted.test_descriptor.ClassTemplateRegistry import com.teamscale.test_impacted.test_descriptor.ITestDescriptorResolver +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_TEMPLATE_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.METHOD_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.TEST_FACTORY_SEGMENT_TYPE import org.assertj.core.api.Assertions import org.junit.jupiter.api.Test +import org.junit.jupiter.engine.descriptor.TestFactoryTestDescriptor.DYNAMIC_CONTAINER_SEGMENT_TYPE import org.junit.platform.engine.EngineExecutionListener import org.junit.platform.engine.TestExecutionResult import org.junit.platform.engine.UniqueId @@ -17,7 +25,7 @@ internal class TestwiseCoverageCollectingExecutionListenerTest { private val executionListenerMock = mock() private val executionListener = TestwiseCoverageCollectingExecutionListener( - mockApi, resolver, executionListenerMock + mockApi, resolver, executionListenerMock, ClassTemplateRegistry() ) private val rootId = UniqueId.forEngine("dummy") @@ -124,4 +132,173 @@ internal class TestwiseCoverageCollectingExecutionListenerTest { Assertions.assertThat(testExecutions) .allMatch { it.result == ETestExecutionResult.SKIPPED } } + + /** + * A `@ParameterizedClass` dynamically registers one invocation per parameter set, each containing all test methods + * of the class. Every method is one test, so the agent has to see one test per method and parameter set, all + * reported under the method's uniform path without the invocation index, so that the coverage of all parameter + * sets ends up on the same test. + */ + @Test + fun testParameterizedClassIsReportedPerTestMethod() { + val jupiterRootId = UniqueId.forEngine("junit-jupiter") + val classTemplateId = jupiterRootId.append(CLASS_TEMPLATE_SEGMENT_TYPE, "example.ParameterizedTest") + val classTemplate = SimpleTestDescriptor.testContainer(classTemplateId) + val testRoot = SimpleTestDescriptor.testContainer(jupiterRootId, classTemplate) + + val listener = jupiterListener() + simulateClassTemplateExecution(listener, testRoot, classTemplate, createInvocations(classTemplateId)) + + // Both parameter sets report the same uniform path, so that their coverage lands on the same test. + verify(mockApi, times(2)).startTest("example/ParameterizedTest/testA()") + verify(mockApi, times(2)).startTest("example/ParameterizedTest/testB()") + verify(mockApi, times(2)).endTest(eq("example/ParameterizedTest/testA()"), any()) + verify(mockApi, times(2)).endTest(eq("example/ParameterizedTest/testB()"), any()) + verifyNoMoreInteractions(mockApi) + + // Every test method of every parameter set is reported with the result that its parameter set produced. + Assertions.assertThat(listener.testExecutions.map { it.uniformPath to it.result }) + .containsExactly( + "example/ParameterizedTest/testA()" to ETestExecutionResult.FAILURE, + "example/ParameterizedTest/testB()" to ETestExecutionResult.PASSED, + "example/ParameterizedTest/testA()" to ETestExecutionResult.PASSED, + "example/ParameterizedTest/testB()" to ETestExecutionResult.PASSED + ) + Assertions.assertThat(listener.testExecutions.first().message).contains("expected") + } + + /** + * The results that the two simulated parameter sets report for the test methods of the `@ParameterizedClass`, in + * the order in which the methods are executed. + */ + private val resultsPerParameterSet = listOf( + listOf("testA()" to FAILED_RESULT, "testB()" to TestExecutionResult.successful()), + listOf("testA()" to TestExecutionResult.successful(), "testB()" to TestExecutionResult.successful()) + ) + + /** Creates one invocation per parameter set, each containing the test methods of [resultsPerParameterSet]. */ + private fun createInvocations(classTemplateId: UniqueId) = + resultsPerParameterSet.mapIndexed { index, results -> + val invocationIndex = index + 1 + val invocationId = classTemplateId.append(CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE, "#$invocationIndex") + val methods = results.map { (methodName, result) -> + // The jupiter engine appends the index of the enclosing invocation to the reporting name. + SimpleTestDescriptor.testCase( + invocationId.append(METHOD_SEGMENT_TYPE, methodName), "$methodName[$invocationIndex]" + ).result(result) + } + SimpleTestDescriptor.testContainer(invocationId, *methods.toTypedArray()) + } + + /** Simulates the execution of the given invocations of a `@ParameterizedClass`. */ + private fun simulateClassTemplateExecution( + listener: TestwiseCoverageCollectingExecutionListener, + testRoot: SimpleTestDescriptor, + classTemplate: SimpleTestDescriptor, + invocations: List + ) { + listener.executionStarted(testRoot) + listener.executionStarted(classTemplate) + invocations.forEach { invocation -> + // The invocations and their test methods are only registered while the class template is executing. + classTemplate.addChild(invocation) + listener.dynamicTestRegistered(invocation) + listener.executionStarted(invocation) + simulateInvocationExecution(listener, invocation) + listener.executionFinished(invocation, TestExecutionResult.successful()) + } + listener.executionFinished(classTemplate, TestExecutionResult.successful()) + listener.executionFinished(testRoot, TestExecutionResult.successful()) + } + + /** Simulates the execution of all test methods of one invocation of a `@ParameterizedClass`. */ + private fun simulateInvocationExecution( + listener: TestwiseCoverageCollectingExecutionListener, + invocation: SimpleTestDescriptor + ) { + invocation.children.filterIsInstance().forEach { method -> + listener.dynamicTestRegistered(method) + listener.executionStarted(method) + listener.executionFinished(method, method.executionResult) + } + } + + /** + * A `@TestFactory` is one test, however deeply its dynamic containers are nested below it. A failure of one of + * those containers therefore has to be reported for the factory method itself, and the containers that did not + * fail must not contribute any "null" noise to its message. + */ + @Test + fun testFailureInNestedContainerIsReportedForTheTest() { + val jupiterRootId = UniqueId.forEngine("junit-jupiter") + val testClassId = jupiterRootId.append(CLASS_SEGMENT_TYPE, "example.FactoryTest") + val testFactoryId = testClassId.append(TEST_FACTORY_SEGMENT_TYPE, "tests()") + val outerContainerId = testFactoryId.append(DYNAMIC_CONTAINER_SEGMENT_TYPE, "#1") + + val innerContainer = SimpleTestDescriptor.testContainer( + outerContainerId.append(DYNAMIC_CONTAINER_SEGMENT_TYPE, "#1") + ) + val outerContainer = SimpleTestDescriptor.testContainer(outerContainerId, innerContainer) + val testFactory = SimpleTestDescriptor.testContainer(testFactoryId, outerContainer) + val testClass = SimpleTestDescriptor.testContainer(testClassId, testFactory) + val testRoot = SimpleTestDescriptor.testContainer(jupiterRootId, testClass) + + val listener = jupiterListener() + listOf(testRoot, testClass, testFactory, outerContainer, innerContainer) + .forEach { listener.executionStarted(it) } + listener.executionFinished(innerContainer, FAILED_RESULT) + listOf(outerContainer, testFactory, testClass, testRoot) + .forEach { listener.executionFinished(it, TestExecutionResult.successful()) } + + verify(mockApi).startTest("example/FactoryTest/tests()") + verify(mockApi).endTest(eq("example/FactoryTest/tests()"), any()) + verifyNoMoreInteractions(mockApi) + + Assertions.assertThat(listener.testExecutions.map { it.uniformPath to it.result }) + .containsExactly("example/FactoryTest/tests()" to ETestExecutionResult.FAILURE) + // Only the failed container contributes to the message, the successful ones have no stacktrace to report. + Assertions.assertThat(listener.testExecutions.single().message) + .contains("expected") + .doesNotContain("null") + } + + /** The tests of a skipped `@ParameterizedClass` are no longer in the test tree, but must still be reported. */ + @Test + fun testSkipOfParameterizedClass() { + val jupiterRootId = UniqueId.forEngine("junit-jupiter") + val classTemplateId = jupiterRootId.append(CLASS_TEMPLATE_SEGMENT_TYPE, "example.ParameterizedTest") + + val recordedTests = listOf("testA()", "testB()").map { + SimpleTestDescriptor.testCase(classTemplateId.append(METHOD_SEGMENT_TYPE, it)) + } + val classTemplate = SimpleTestDescriptor.testContainer(classTemplateId, *recordedTests.toTypedArray()) + val registry = ClassTemplateRegistry().apply { record(classTemplate) } + // The JUnit platform prunes the tests of the class template away before it is executed. + recordedTests.forEach { classTemplate.removeChild(it) } + + val listener = jupiterListener(registry) + + listener.executionSkipped(classTemplate, "Test class is disabled.") + + verify(executionListenerMock).executionSkipped(classTemplate, "Test class is disabled.") + verifyNoMoreInteractions(executionListenerMock) + verifyNoMoreInteractions(mockApi) + + Assertions.assertThat(listener.testExecutions) + .extracting { it.uniformPath } + .containsExactly("example/ParameterizedTest/testA()", "example/ParameterizedTest/testB()") + Assertions.assertThat(listener.testExecutions) + .allMatch { it.result == ETestExecutionResult.SKIPPED } + } + + /** Creates a listener that resolves the uniform paths of the jupiter engine's test descriptors. */ + private fun jupiterListener(classTemplateRegistry: ClassTemplateRegistry = ClassTemplateRegistry()) = + TestwiseCoverageCollectingExecutionListener( + mockApi, JUnitJupiterTestDescriptorResolver(), executionListenerMock, classTemplateRegistry + ) + + companion object { + /** The result reported for the executions that are meant to fail in a simulation. */ + private val FAILED_RESULT = TestExecutionResult.failed(AssertionError("expected")) + } } diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolverTest.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolverTest.kt index 06e83c6e1..200be5c24 100644 --- a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolverTest.kt +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/JUnitJupiterTestDescriptorResolverTest.kt @@ -2,8 +2,12 @@ package com.teamscale.test_impacted.test_descriptor import com.teamscale.test_impacted.engine.executor.SimpleTestDescriptor import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_TEMPLATE_SEGMENT_TYPE import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.METHOD_SEGMENT_TYPE import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.NESTED_CLASS_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.NESTED_CLASS_TEMPLATE_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.TEST_TEMPLATE_SEGMENT_TYPE import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.platform.engine.UniqueId @@ -44,4 +48,70 @@ internal class JUnitJupiterTestDescriptorResolverTest { val descriptor = SimpleTestDescriptor.testCase(methodId) assertThat(resolver.getUniformPath(descriptor)).isEqualTo("com/example/A\$B\$C/test()") } + + /** + * The tests of a top-level `@ParameterizedClass` are registered below one invocation per parameter set. They all + * belong to the same test, so the invocation must not appear in the uniform path. + */ + @Test + fun testMethodInsideTopLevelParameterizedClassUniformPath() { + val methodId = UniqueId.forEngine("junit-jupiter") + .append(CLASS_TEMPLATE_SEGMENT_TYPE, "com.example.MyTest") + .append(CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE, "#2") + .append(METHOD_SEGMENT_TYPE, "testMethod()") + + val descriptor = SimpleTestDescriptor.testCase(methodId, "testMethod()[2]") + assertThat(resolver.getUniformPath(descriptor)).isEqualTo("com/example/MyTest/testMethod()") + assertThat(resolver.getClusterId(descriptor)).isEqualTo("com.example.MyTest") + } + + /** The same for a `@Nested @ParameterizedClass`, whose nesting must be kept in the class name. */ + @Test + fun testMethodInsideNestedParameterizedClassUniformPath() { + val methodId = UniqueId.forEngine("junit-jupiter") + .append(CLASS_SEGMENT_TYPE, "com.example.OuterTest") + .append(NESTED_CLASS_TEMPLATE_SEGMENT_TYPE, "Inner") + .append(CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE, "#1") + .append(METHOD_SEGMENT_TYPE, "testMethod()") + + val descriptor = SimpleTestDescriptor.testCase(methodId, "testMethod()[1]") + assertThat(resolver.getUniformPath(descriptor)).isEqualTo("com/example/OuterTest\$Inner/testMethod()") + assertThat(resolver.getClusterId(descriptor)).isEqualTo("com.example.OuterTest\$Inner") + } + + /** A `@Nested` class inside a `@ParameterizedClass` must keep both class names. */ + @Test + fun testNestedClassInsideParameterizedClassUniformPath() { + val methodId = UniqueId.forEngine("junit-jupiter") + .append(CLASS_TEMPLATE_SEGMENT_TYPE, "com.example.MyTest") + .append(CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE, "#1") + .append(NESTED_CLASS_SEGMENT_TYPE, "Inner") + .append(METHOD_SEGMENT_TYPE, "testMethod()") + + val descriptor = SimpleTestDescriptor.testCase(methodId, "testMethod()[1]") + assertThat(resolver.getUniformPath(descriptor)).isEqualTo("com/example/MyTest\$Inner/testMethod()") + } + + /** A `@ParameterizedTest` inside a `@ParameterizedClass` keeps its arguments but loses the invocation index. */ + @Test + fun testParameterizedTestInsideParameterizedClassUniformPath() { + val testTemplateId = UniqueId.forEngine("junit-jupiter") + .append(CLASS_TEMPLATE_SEGMENT_TYPE, "com.example.MyTest") + .append(CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE, "#3") + .append(TEST_TEMPLATE_SEGMENT_TYPE, "testMethod(java.lang.String)") + + val descriptor = SimpleTestDescriptor.testCase(testTemplateId, "testMethod(String)[3]") + assertThat(resolver.getUniformPath(descriptor)).isEqualTo("com/example/MyTest/testMethod(String)") + } + + /** Reporting names outside of a `@ParameterizedClass` must be kept as they are. */ + @Test + fun testBracketsOutsideOfParameterizedClassesAreKept() { + val methodId = UniqueId.forEngine("junit-jupiter") + .append(CLASS_SEGMENT_TYPE, "com.example.MyTest") + .append(METHOD_SEGMENT_TYPE, "testMethod()") + + val descriptor = SimpleTestDescriptor.testCase(methodId, "testMethod()[1]") + assertThat(resolver.getUniformPath(descriptor)).isEqualTo("com/example/MyTest/testMethod()[1]") + } } diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/JupiterClassTemplateTest.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/JupiterClassTemplateTest.kt new file mode 100644 index 000000000..dfefb7ea7 --- /dev/null +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/JupiterClassTemplateTest.kt @@ -0,0 +1,85 @@ +package com.teamscale.test_impacted.test_descriptor + +import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.getAvailableTests +import com.teamscale.test_impacted.test_descriptor.samples.SampleParameterizedTestClass +import com.teamscale.test_impacted.test_descriptor.samples.SampleTestClass +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.engine.JupiterTestEngine +import org.junit.jupiter.engine.descriptor.ClassTemplateInvocationTestDescriptor +import org.junit.jupiter.engine.descriptor.ClassTemplateTestDescriptor +import org.junit.jupiter.engine.descriptor.ClassTestDescriptor +import org.junit.jupiter.engine.descriptor.NestedClassTestDescriptor +import org.junit.jupiter.engine.descriptor.TestFactoryTestDescriptor +import org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor +import org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor +import org.junit.platform.engine.TestDescriptor +import org.junit.platform.engine.UniqueId +import org.junit.platform.engine.discovery.DiscoverySelectors.selectClass +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder + +/** + * Tests the handling of `@ParameterizedClass` against the real JUnit Jupiter engine. The unit tests in + * [TestDescriptorUtilsTest] and [JUnitJupiterTestDescriptorResolverTest] construct their unique IDs by hand, so only + * these tests notice when the jupiter engine changes the shape of the test tree it reports. + */ +internal class JupiterClassTemplateTest { + + /** + * A `@ParameterizedClass` registers its invocations and their test methods only while it is being executed, which + * is why the JUnit platform prunes them from the discovered test tree. The class template must therefore be + * reported as an available test itself, otherwise its tests neither show up in the test list uploaded to Teamscale + * nor get any coverage recorded for them. + */ + @Test + fun testParameterizedClassesAreDiscoveredAsAvailableTests() { + val discoveryRequest = LauncherDiscoveryRequestBuilder.request() + .selectors(selectClass(SampleTestClass::class.java), selectClass(SampleParameterizedTestClass::class.java)) + .build() + val jupiterEngine = JupiterTestEngine() + + val rootDescriptor = jupiterEngine.discover(discoveryRequest, UniqueId.forEngine(jupiterEngine.id)) + // The engine records the tests of the @ParameterizedClasses while they are still in the tree, ... + val classTemplateRegistry = ClassTemplateRegistry().apply { record(rootDescriptor) } + // ... because the JUnit platform launcher prunes the tree before handing it to the engines for execution. + rootDescriptor.accept(TestDescriptor::prune) + + val samples = "com/teamscale/test_impacted/test_descriptor/samples" + assertThat(getAvailableTests(rootDescriptor, classTemplateRegistry).testList) + .extracting { it.uniformPath } + .containsExactlyInAnyOrder( + "$samples/SampleTestClass/testOuter()", + "$samples/SampleTestClass\$PlainNested/testPlain()", + "$samples/SampleTestClass\$NestedParameterized/testOne()", + "$samples/SampleTestClass\$NestedParameterized/testTwo()", + "$samples/SampleParameterizedTestClass/testA()", + "$samples/SampleParameterizedTestClass/testB()" + ) + } + + /** + * The segment types we match on are not part of the public JUnit API, so make sure they stay in sync with the ones + * the jupiter engine actually uses. + */ + @Test + fun testSegmentTypesMatchTheOnesUsedByTheJupiterEngine() { + assertThat(JUnitJupiterTestDescriptorResolver.CLASS_SEGMENT_TYPE) + .isEqualTo(ClassTestDescriptor.SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.NESTED_CLASS_SEGMENT_TYPE) + .isEqualTo(NestedClassTestDescriptor.SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.METHOD_SEGMENT_TYPE) + .isEqualTo(TestMethodTestDescriptor.SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.TEST_FACTORY_SEGMENT_TYPE) + .isEqualTo(TestFactoryTestDescriptor.SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.DYNAMIC_TEST_SEGMENT_TYPE) + .isEqualTo(TestFactoryTestDescriptor.DYNAMIC_TEST_SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.TEST_TEMPLATE_SEGMENT_TYPE) + .isEqualTo(TestTemplateTestDescriptor.SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.CLASS_TEMPLATE_SEGMENT_TYPE) + .isEqualTo(ClassTemplateTestDescriptor.STANDALONE_CLASS_SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.NESTED_CLASS_TEMPLATE_SEGMENT_TYPE) + .isEqualTo(ClassTemplateTestDescriptor.NESTED_CLASS_SEGMENT_TYPE) + assertThat(JUnitJupiterTestDescriptorResolver.CLASS_TEMPLATE_INVOCATION_SEGMENT_TYPE) + .isEqualTo(ClassTemplateInvocationTestDescriptor.SEGMENT_TYPE) + } +} diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/TestDescriptorUtilsTest.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/TestDescriptorUtilsTest.kt new file mode 100644 index 000000000..7bacc0a28 --- /dev/null +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/TestDescriptorUtilsTest.kt @@ -0,0 +1,131 @@ +package com.teamscale.test_impacted.test_descriptor + +import com.teamscale.client.PrioritizableTest +import com.teamscale.test_impacted.engine.executor.SimpleTestDescriptor +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.CLASS_TEMPLATE_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.METHOD_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.NESTED_CLASS_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.JUnitJupiterTestDescriptorResolver.Companion.NESTED_CLASS_TEMPLATE_SEGMENT_TYPE +import com.teamscale.test_impacted.test_descriptor.TestDescriptorUtils.getAvailableTests +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.platform.engine.UniqueId + +/** Tests for [TestDescriptorUtils]. */ +internal class TestDescriptorUtilsTest { + + private val engineId = UniqueId.forEngine("junit-jupiter") + + private val outerClassId = engineId.append(CLASS_SEGMENT_TYPE, "example.OuterTest") + private val plainNestedClassId = outerClassId.append(NESTED_CLASS_SEGMENT_TYPE, "PlainNested") + private val nestedClassTemplateId = outerClassId.append(NESTED_CLASS_TEMPLATE_SEGMENT_TYPE, "NestedParameterized") + private val classTemplateId = engineId.append(CLASS_TEMPLATE_SEGMENT_TYPE, "example.ParameterizedTest") + + /** The test tree as it looks during discovery, i.e. before the JUnit platform prunes it. */ + private val testRoot = SimpleTestDescriptor.testContainer( + engineId, + SimpleTestDescriptor.testContainer( + outerClassId, + SimpleTestDescriptor.testCase(outerClassId.append(METHOD_SEGMENT_TYPE, "testOuter()")), + SimpleTestDescriptor.testContainer( + plainNestedClassId, + SimpleTestDescriptor.testCase(plainNestedClassId.append(METHOD_SEGMENT_TYPE, "testPlain()")) + ), + SimpleTestDescriptor.testContainer( + nestedClassTemplateId, + SimpleTestDescriptor.testCase(nestedClassTemplateId.append(METHOD_SEGMENT_TYPE, "testOne()")), + SimpleTestDescriptor.testCase(nestedClassTemplateId.append(METHOD_SEGMENT_TYPE, "testTwo()")) + ) + ), + SimpleTestDescriptor.testContainer( + classTemplateId, + SimpleTestDescriptor.testCase(classTemplateId.append(METHOD_SEGMENT_TYPE, "testA()")) + ) + ) + + /** The tests that the engine recorded while the test tree was still complete. */ + private val registry = ClassTemplateRegistry().apply { record(testRoot) } + + /** + * Simulates the pruning that the JUnit platform applies to a `@ParameterizedClass` after discovery. Must not be + * named `prune`, because [org.junit.platform.engine.TestDescriptor.prune] would shadow it. + */ + private fun simulatePlatformPruning() { + listOf(nestedClassTemplateId, classTemplateId).forEach { classTemplateId -> + testRoot.findByUniqueId(classTemplateId).get().let { classTemplate -> + classTemplate.children.toList().forEach { classTemplate.removeChild(it) } + } + } + } + + /** + * The tests of a `@ParameterizedClass` are pruned from the test tree after discovery, so they have to be taken + * from the [ClassTemplateRegistry] which recorded them while they were still there. + */ + @Test + fun testParameterizedClassTestsAreAvailableTests() { + simulatePlatformPruning() + + assertThat(getAvailableTests(testRoot, registry).testList) + .extracting { it.uniformPath } + .containsExactlyInAnyOrder( + "example/OuterTest/testOuter()", + "example/OuterTest\$PlainNested/testPlain()", + "example/OuterTest\$NestedParameterized/testOne()", + "example/OuterTest\$NestedParameterized/testTwo()", + "example/ParameterizedTest/testA()" + ) + } + + /** Without the recorded tests, everything below a pruned `@ParameterizedClass` is lost. */ + @Test + fun testPrunedParameterizedClassTestsAreLostWithoutRecording() { + simulatePlatformPruning() + + assertThat(getAvailableTests(testRoot, ClassTemplateRegistry()).testList) + .extracting { it.uniformPath } + .containsExactlyInAnyOrder( + "example/OuterTest/testOuter()", + "example/OuterTest\$PlainNested/testPlain()" + ) + } + + /** The cluster ID of the tests of a `@ParameterizedClass` is their class, just like for any other test. */ + @Test + fun testParameterizedClassClusterId() { + simulatePlatformPruning() + + assertThat(getAvailableTests(testRoot, registry).testList) + .filteredOn { it.uniformPath.startsWith("example/ParameterizedTest") } + .extracting { it.clusterId } + .containsExactly("example.ParameterizedTest") + } + + /** + * JUnit can only execute a `@ParameterizedClass` as a whole, so each of its tests has to be selected via the class + * template. Otherwise the impacted tests returned by Teamscale cannot be found in the pruned test tree and the + * engine falls back to executing all tests. + */ + @Test + fun testParameterizedClassTestsAreSelectedViaTheirClass() { + simulatePlatformPruning() + + val availableTests = getAvailableTests(testRoot, registry) + + listOf("example/ParameterizedTest/testA()", "example/OuterTest\$NestedParameterized/testOne()") + .forEach { uniformPath -> + val uniqueId = availableTests.convertToUniqueId(PrioritizableTest(uniformPath)) + assertThat(uniqueId).isPresent() + assertThat(testRoot.findByUniqueId(uniqueId.get())).isPresent() + } + } + + /** As long as nothing was pruned, the tests of a `@ParameterizedClass` are found in the test tree itself. */ + @Test + fun testParameterizedClassTestsAreFoundWithoutRecording() { + assertThat(getAvailableTests(testRoot, ClassTemplateRegistry()).testList) + .extracting { it.uniformPath } + .contains("example/ParameterizedTest/testA()") + } +} diff --git a/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/samples/JupiterSamples.kt b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/samples/JupiterSamples.kt new file mode 100644 index 000000000..eef775905 --- /dev/null +++ b/impacted-test-engine/src/test/kotlin/com/teamscale/test_impacted/test_descriptor/samples/JupiterSamples.kt @@ -0,0 +1,69 @@ +package com.teamscale.test_impacted.test_descriptor.samples + +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.Parameter +import org.junit.jupiter.params.ParameterizedClass +import org.junit.jupiter.params.provider.ValueSource + +/** + * Sample tests that are only discovered explicitly by + * [com.teamscale.test_impacted.test_descriptor.JupiterClassTemplateTest]. They are excluded from this project's own + * test run, see `impacted-test-engine/build.gradle.kts`. + */ +class SampleTestClass { + @Test + fun testOuter() { + // Nothing to do, we are only interested in the shape of the discovered test tree. + } + + /** A plain `@Nested` test class, see [SampleTestClass]. */ + @Nested + inner class PlainNested { + @Test + fun testPlain() { + // See above. + } + } + + /** A `@Nested` `@ParameterizedClass`, see [SampleTestClass]. */ + @Nested + @ParameterizedClass + @ValueSource(strings = ["a", "b"]) + inner class NestedParameterized { + /** The parameter the enclosing class is instantiated with. */ + @Parameter + @JvmField + var value: String = "" + + @Test + fun testOne() { + // See above. + } + + @Test + fun testTwo() { + // See above. + } + } +} + +/** A top-level `@ParameterizedClass`, see [SampleTestClass]. */ +@ParameterizedClass +@ValueSource(ints = [1, 2, 3]) +class SampleParameterizedTestClass { + /** The parameter the enclosing class is instantiated with. */ + @Parameter + @JvmField + var value: Int = 0 + + @Test + fun testA() { + // See above. + } + + @Test + fun testB() { + // See above. + } +} diff --git a/report-generator/src/main/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGenerator.kt b/report-generator/src/main/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGenerator.kt index 3b6b77844..ef28a46d3 100644 --- a/report-generator/src/main/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGenerator.kt +++ b/report-generator/src/main/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGenerator.kt @@ -65,7 +65,67 @@ open class JaCoCoTestwiseReportGenerator( return testCoverageBuilders.singleOrNull() } - /** Converts the given dumps to a report. */ + /** + * Converts the given dumps to a report, merging the coverage of all dumps that belong to the same test before + * passing it on. A test produces more than one dump if it was executed repeatedly, e.g. once per parameter set of + * an enclosing `@ParameterizedClass`. + * + * The dumps of one test are neither adjacent nor necessarily in the same file, so a first pass over the files + * determines which tests were dumped repeatedly. Only the coverage of those has to be held back until all files + * have been read; the coverage of every other test is passed on as soon as its dump was read, so that the consumer + * can write it out instead of keeping the whole test run in memory. + */ + @Throws(IOException::class, CoverageGenerationException::class) + open fun convertAndConsumePerTest(executionDataFiles: List, consumer: Consumer) { + val repeatedTestIds = findTestsWithMultipleDumps(executionDataFiles) + val repeatedTestCoverage = TestwiseCoverage() + executionDataFiles.forEach { executionDataFile -> + convertAndConsume(executionDataFile) { coverage -> + if (coverage.uniformPath in repeatedTestIds) { + repeatedTestCoverage.add(coverage) + } else { + consumer.accept(coverage) + } + } + } + repeatedTestCoverage.tests.values.forEach(consumer::accept) + } + + /** Returns the IDs of the tests for which the given *.exec files contain more than one dump. */ + @Throws(IOException::class) + private fun findTestsWithMultipleDumps(executionDataFiles: List): Set { + val seenTestIds = mutableSetOf() + val repeatedTestIds = mutableSetOf() + executionDataFiles.forEach { executionDataFile -> + readSessionInfos(executionDataFile) { info -> + if (info.id.isNotEmpty() && !seenTestIds.add(info.id)) { + repeatedTestIds.add(info.id) + } + } + } + return repeatedTestIds + } + + /** + * Passes the session infos in the given *.exec file to the given visitor. Only the session infos are of interest + * here, so the execution data itself is read but discarded. + */ + @Throws(IOException::class) + private fun readSessionInfos(executionDataFile: File, sessionInfoVisitor: ISessionInfoVisitor) { + BufferedInputStream(FileInputStream(executionDataFile)).use { input -> + ExecutionDataReader(input).apply { + setExecutionDataVisitor { } + setSessionInfoVisitor(sessionInfoVisitor) + read() + } + } + } + + /** + * Converts the dumps in the given *.exec file to a report, passing on the coverage of each dump as soon as it was + * read. Use [convertAndConsumePerTest] unless the consumer can handle more than one result for the same test, + * since a test that was executed repeatedly produces one dump per execution. + */ @Throws(IOException::class) open fun convertAndConsume(executionDataFile: File, consumer: Consumer) { val dumpConsumer = executionDataReader.buildCoverageConsumer(locationIncludeFilter, consumer) diff --git a/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestInfoBuilder.kt b/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestInfoBuilder.kt index dc8c1c4d5..fd1914a44 100644 --- a/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestInfoBuilder.kt +++ b/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestInfoBuilder.kt @@ -46,15 +46,45 @@ class TestInfoBuilder(val uniformPath: String) { content = details.content } - /** Sets the test execution fields. */ - fun setExecution(execution: TestExecution) { - durationSeconds = execution.durationSeconds - result = execution.result - message = execution.message + /** + * Adds a test execution. A test may be executed more than once, e.g. once per parameter set of an enclosing + * `@ParameterizedClass`, in which case the executions are aggregated: the durations are summed up and the result + * with the highest [severity] wins, so that a failure in any of them is not hidden by a later successful execution. + */ + fun addExecution(execution: TestExecution) { + durationSeconds = (durationSeconds ?: 0.0) + execution.durationSeconds + val executionResult = execution.result + val previousResult = result + if (previousResult == null || (executionResult != null && executionResult.severity > previousResult.severity)) { + result = executionResult + } + message = listOfNotNull(message, execution.message).takeIf { it.isNotEmpty() }?.joinToString("\n\n") } - fun setCoverage(coverage: TestCoverageBuilder) { - this.coverage = coverage + /** + * How severe a result is when the results of multiple executions of the same test are aggregated. This must not be + * derived from the declaration order of [ETestExecutionResult], which is part of the report format and says + * nothing about severity: a test that ran and passed in one execution should not be reported as skipped because + * another execution was, and an inconclusive execution must not hide a failed one. + */ + private val ETestExecutionResult.severity: Int + get() = when (this) { + ETestExecutionResult.IGNORED -> 0 + ETestExecutionResult.SKIPPED -> 1 + ETestExecutionResult.PASSED -> 2 + ETestExecutionResult.INCONCLUSIVE -> 3 + ETestExecutionResult.FAILURE -> 4 + ETestExecutionResult.ERROR -> 5 + } + + /** Adds coverage of the test, merging it with any coverage that was already added for it. */ + fun addCoverage(coverage: TestCoverageBuilder) { + val existingCoverage = this.coverage + if (existingCoverage == null) { + this.coverage = coverage + } else { + existingCoverage.addAll(coverage.files) + } } /** Builds a [TestInfo] object of the data in this container. */ diff --git a/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestwiseCoverageReportBuilder.kt b/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestwiseCoverageReportBuilder.kt index 51413a5e6..b5722b0c1 100644 --- a/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestwiseCoverageReportBuilder.kt +++ b/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/builder/TestwiseCoverageReportBuilder.kt @@ -37,11 +37,11 @@ class TestwiseCoverageReportBuilder { } } testCoverage.forEach { coverage -> - resolveUniformPath(report, coverage.uniformPath)?.setCoverage(coverage) + resolveUniformPath(report, coverage.uniformPath)?.addCoverage(coverage) } testExecutions.forEach { testExecution -> val path = testExecution.uniformPath ?: return@forEach - resolveUniformPath(report, path)?.setExecution(testExecution) + resolveUniformPath(report, path)?.addExecution(testExecution) } return report.build(partial) } diff --git a/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/factory/TestInfoFactory.kt b/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/factory/TestInfoFactory.kt index 7dce4a39f..4f903bddf 100644 --- a/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/factory/TestInfoFactory.kt +++ b/report-generator/src/main/kotlin/com/teamscale/report/testwise/model/factory/TestInfoFactory.kt @@ -16,8 +16,11 @@ class TestInfoFactory(testDetails: List, testExecutions: List() - /** Maps uniform paths to test executions. */ - private val testExecutionsMap = mutableMapOf() + /** + * Maps uniform paths to test executions. A test may have been executed more than once, e.g. once per parameter set + * of an enclosing `@ParameterizedClass`. + */ + private val testExecutionsMap = mutableMapOf>() /** Holds all uniform paths for tests that have been written to the outputFile. */ private val processedTestUniformPaths = mutableSetOf() @@ -28,7 +31,7 @@ class TestInfoFactory(testDetails: List, testExecutions: List testExecution.uniformPath?.let { - testExecutionsMap[it] = testExecution + testExecutionsMap.computeIfAbsent(it) { mutableListOf() }.add(testExecution) } } } @@ -45,12 +48,12 @@ class TestInfoFactory(testDetails: List, testExecutions: List setDetails(testDetails) } ?: System.err.println("No test details found for $resolvedUniformPath") - testExecutionsMap[resolvedUniformPath]?.let { execution -> - setExecution(execution) + testExecutionsMap[resolvedUniformPath]?.forEach { execution -> + addExecution(execution) } ?: System.err.println("No test execution found for $resolvedUniformPath") }.build() } @@ -63,10 +66,10 @@ class TestInfoFactory(testDetails: List, testExecutions: List + testExecutionsMap.values.flatten().forEach { testExecution -> if (processedTestUniformPaths.contains(testExecution.uniformPath)) return@forEach System.err.println( "Test " + testExecution.uniformPath + " was executed but no coverage was found. " + diff --git a/report-generator/src/test/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGeneratorTest.kt b/report-generator/src/test/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGeneratorTest.kt index 8509aedad..b05598ac8 100644 --- a/report-generator/src/test/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGeneratorTest.kt +++ b/report-generator/src/test/kotlin/com/teamscale/report/testwise/jacoco/JaCoCoTestwiseReportGeneratorTest.kt @@ -8,13 +8,16 @@ import com.teamscale.report.testwise.model.ETestExecutionResult import com.teamscale.report.testwise.model.TestExecution import com.teamscale.report.testwise.model.TestwiseCoverage import com.teamscale.report.testwise.model.TestwiseCoverageReport +import com.teamscale.report.testwise.model.builder.TestCoverageBuilder import com.teamscale.report.testwise.model.builder.TestwiseCoverageReportBuilder.Companion.createFrom import com.teamscale.report.util.ClasspathWildcardIncludeFilter import com.teamscale.test.TestDataBase +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.mockito.Mockito import org.skyscreamer.jsonassert.JSONAssert import org.skyscreamer.jsonassert.JSONCompareMode +import java.io.File /** Tests for the [JaCoCoTestwiseReportGenerator] class. */ class JaCoCoTestwiseReportGeneratorTest : TestDataBase() { @@ -39,18 +42,50 @@ class JaCoCoTestwiseReportGeneratorTest : TestDataBase() { JSONAssert.assertEquals(expected, report, JSONCompareMode.STRICT) } + /** + * A test that was dumped more than once, e.g. once per parameter set of an enclosing `@ParameterizedClass`, has to + * be passed on exactly once, with the coverage of all of its dumps merged. The dumps of one test may be spread + * over several *.exec files, which is simulated here by converting the same file twice. + */ + @Test + fun testRepeatedDumpsOfTheSameTestAreMerged() { + val executionDataFile = useTestFile("jacoco/sample/coverage.exec") + + val oneDumpPerTest = convertAndConsumePerTest(listOf(executionDataFile)) + val twoDumpsPerTest = convertAndConsumePerTest(listOf(executionDataFile, executionDataFile)) + + assertThat(twoDumpsPerTest.map { it.uniformPath }) + .containsExactlyInAnyOrderElementsOf(oneDumpPerTest.map { it.uniformPath }) + assertThat(twoDumpsPerTest.asReportString()).isEqualTo(oneDumpPerTest.asReportString()) + } + @Throws(Exception::class) private fun runReportGenerator(testDataFolder: String, execFileName: String): String { - val classFileFolder = useTestFile(testDataFolder) - val includeFilter = ClasspathWildcardIncludeFilter(null, null) - val testwiseCoverage = JaCoCoTestwiseReportGenerator( - listOf(classFileFolder), - includeFilter, EDuplicateClassFileBehavior.IGNORE, - Mockito.mock() - ).convert(useTestFile(execFileName)) + val testwiseCoverage = createReportGenerator(testDataFolder).convert(useTestFile(execFileName)) return getTestwiseCoverageReportAsString(testwiseCoverage.generateDummyReport()) } + /** Collects the coverage that the generator passes on for the tests in the given *.exec files. */ + private fun convertAndConsumePerTest(executionDataFiles: List): List { + val coverage = mutableListOf() + createReportGenerator("jacoco/sample/classes.zip") + .convertAndConsumePerTest(executionDataFiles, coverage::add) + return coverage + } + + private fun List.asReportString(): String { + val testwiseCoverage = TestwiseCoverage() + forEach { testwiseCoverage.add(it) } + return getTestwiseCoverageReportAsString(testwiseCoverage.generateDummyReport()) + } + + private fun createReportGenerator(testDataFolder: String) = + JaCoCoTestwiseReportGenerator( + listOf(useTestFile(testDataFolder)), + ClasspathWildcardIncludeFilter(null, null), EDuplicateClassFileBehavior.IGNORE, + Mockito.mock() + ) + companion object { /** Generates a fake coverage report object that wraps the given [TestwiseCoverage]. */ fun TestwiseCoverage.generateDummyReport(): TestwiseCoverageReport { diff --git a/report-generator/src/test/kotlin/com/teamscale/report/testwise/model/builder/TestwiseCoverageReportBuilderTest.kt b/report-generator/src/test/kotlin/com/teamscale/report/testwise/model/builder/TestwiseCoverageReportBuilderTest.kt new file mode 100644 index 000000000..e73417506 --- /dev/null +++ b/report-generator/src/test/kotlin/com/teamscale/report/testwise/model/builder/TestwiseCoverageReportBuilderTest.kt @@ -0,0 +1,111 @@ +package com.teamscale.report.testwise.model.builder + +import com.teamscale.client.TestDetails +import com.teamscale.report.testwise.model.ETestExecutionResult +import com.teamscale.report.testwise.model.TestExecution +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +/** Tests for [TestwiseCoverageReportBuilder]. */ +internal class TestwiseCoverageReportBuilderTest { + + private val uniformPath = "com/example/ParameterizedTest/testMethod()" + + private fun coverage(fileName: String, vararg lines: Int) = + TestCoverageBuilder(uniformPath).apply { + add(FileCoverageBuilder("com/example", fileName).apply { lines.forEach { addLine(it) } }) + } + + /** Builds a report for the single test [uniformPath] from the given coverage and executions. */ + private fun report( + coverage: List = emptyList(), + executions: List = emptyList() + ) = TestwiseCoverageReportBuilder.createFrom( + listOf(TestDetails(uniformPath, "com/example/ParameterizedTest", null)), coverage, executions, false + ) + + /** + * A test is executed more than once if it is part of a `@ParameterizedClass`, once per parameter set. All of those + * executions belong to the same test, so their coverage has to be merged instead of overwriting each other. + */ + @Test + fun testCoverageOfRepeatedExecutionsIsMerged() { + val report = report(coverage = listOf(coverage("Calculator.java", 3, 6), coverage("Calculator.java", 3, 9))) + + assertThat(report.tests).hasSize(1) + assertThat(report.tests.single().paths.single().files.single().coveredLines).isEqualTo("3,6,9") + } + + /** Coverage of different files reported by repeated executions must all end up on the test. */ + @Test + fun testCoverageOfRepeatedExecutionsInDifferentFilesIsMerged() { + val report = report(coverage = listOf(coverage("Calculator.java", 3), coverage("Multiplier.java", 4))) + + assertThat(report.tests.single().paths.single().files) + .extracting { it.fileName } + .containsExactly("Calculator.java", "Multiplier.java") + } + + /** + * The durations of repeated executions are summed up and the most severe result wins, so that a failure in one + * parameter set is not hidden by another one that passed afterwards. + */ + @Test + fun testRepeatedExecutionsAreAggregated() { + val report = report( + executions = listOf( + TestExecution(uniformPath, 20L, ETestExecutionResult.FAILURE, "boom"), + TestExecution(uniformPath, 30L, ETestExecutionResult.PASSED) + ) + ) + + assertThat(report.tests.single().result).isEqualTo(ETestExecutionResult.FAILURE) + assertThat(report.tests.single().duration).isEqualTo(0.05) + assertThat(report.tests.single().message).contains("boom") + } + + /** A failure in a later parameter set must win as well, and the messages of both executions must be kept. */ + @Test + fun testFailureAfterPassedExecutionIsReported() { + val report = report( + executions = listOf( + TestExecution(uniformPath, 20L, ETestExecutionResult.PASSED, "first parameter set"), + TestExecution(uniformPath, 30L, ETestExecutionResult.FAILURE, "boom") + ) + ) + + assertThat(report.tests.single().result).isEqualTo(ETestExecutionResult.FAILURE) + assertThat(report.tests.single().message).isEqualTo("first parameter set\n\nboom") + } + + /** + * The declaration order of [ETestExecutionResult] is not a severity order, so aggregating must not rely on it: a + * test that ran and passed must not be reported as skipped, and a failure must not be hidden by an execution + * whose result the profiler did not learn. + */ + @Test + fun testAggregationDoesNotFollowTheDeclarationOrderOfTheResults() { + assertThat(aggregatedResultOf(ETestExecutionResult.SKIPPED, ETestExecutionResult.PASSED)) + .isEqualTo(ETestExecutionResult.PASSED) + assertThat(aggregatedResultOf(ETestExecutionResult.FAILURE, ETestExecutionResult.INCONCLUSIVE)) + .isEqualTo(ETestExecutionResult.FAILURE) + } + + /** Returns the result reported for a test that was executed once with each of the given results. */ + private fun aggregatedResultOf(vararg results: ETestExecutionResult) = + report(executions = results.map { TestExecution(uniformPath, 10L, it) }).tests.single().result + + /** Executions of a `@ParameterizedTest` still carry the invocation index, which must be stripped. */ + @Test + fun testExecutionsWithParameterizedTestArgumentsAreResolved() { + val report = report( + executions = listOf( + TestExecution("$uniformPath[1]", 10L, ETestExecutionResult.PASSED), + TestExecution("$uniformPath[2]", 10L, ETestExecutionResult.PASSED) + ) + ) + + assertThat(report.tests).hasSize(1) + assertThat(report.tests.single().duration).isEqualTo(0.02) + } +} diff --git a/system-tests/gradle-multi-module/gradle-project/buildSrc/src/main/kotlin/com.example.java-convention.gradle.kts b/system-tests/gradle-multi-module/gradle-project/buildSrc/src/main/kotlin/com.example.java-convention.gradle.kts index 3730b6f65..67cb4e199 100644 --- a/system-tests/gradle-multi-module/gradle-project/buildSrc/src/main/kotlin/com.example.java-convention.gradle.kts +++ b/system-tests/gradle-multi-module/gradle-project/buildSrc/src/main/kotlin/com.example.java-convention.gradle.kts @@ -30,7 +30,7 @@ testing { } dependencies { - testImplementation(platform("org.junit:junit-bom:5.12.1")) + testImplementation(platform("org.junit:junit-bom:6.1.3")) testImplementation("org.junit.jupiter:junit-jupiter") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } diff --git a/system-tests/gradle-multi-module/gradle-project/lib/src/test/java/com/example/lib/CalculatorParameterizedTest.java b/system-tests/gradle-multi-module/gradle-project/lib/src/test/java/com/example/lib/CalculatorParameterizedTest.java new file mode 100644 index 000000000..aafd6ef97 --- /dev/null +++ b/system-tests/gradle-multi-module/gradle-project/lib/src/test/java/com/example/lib/CalculatorParameterizedTest.java @@ -0,0 +1,29 @@ +package com.example.lib; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Uses @ParameterizedClass. The JUnit platform prunes the tests of such a class from the discovered test tree and only + * registers them once per parameter set while the class is being executed, so the profiler reports each of its test + * methods as one test with the executions of all parameter sets collapsed into it. Only calls methods that are already + * covered by {@link CalculatorTest} to keep the expected line coverage stable. + */ +@ParameterizedClass +@ValueSource(ints = {1, 2}) +class CalculatorParameterizedTest { + + /** The summand that the test methods of this class are executed with. */ + @Parameter + int summand; + + @Test + public void testAddIsCommutative() { + Calculator calculator = new Calculator(); + assertEquals(calculator.add(summand, 3), calculator.add(3, summand)); + } +} diff --git a/system-tests/gradle-multi-module/src/test/kotlin/com/teamscale/tia/TestwiseCoverageGradleSystemTest.kt b/system-tests/gradle-multi-module/src/test/kotlin/com/teamscale/tia/TestwiseCoverageGradleSystemTest.kt index 410a60e06..cd3ef4730 100644 --- a/system-tests/gradle-multi-module/src/test/kotlin/com/teamscale/tia/TestwiseCoverageGradleSystemTest.kt +++ b/system-tests/gradle-multi-module/src/test/kotlin/com/teamscale/tia/TestwiseCoverageGradleSystemTest.kt @@ -35,12 +35,14 @@ class TestwiseCoverageGradleSystemTest { val testwiseReport = teamscaleMockServer.getOnlyTestwiseCoverageReport("System Tests") assertThat(testwiseReport.partial).isEqualTo(false) - assertThat(testwiseReport.tests.first().uniformPath) - .isEqualTo("com/example/app/MainTest/testMain()") - assertThat(testwiseReport.tests.last().uniformPath) - .isEqualTo("com/example/lib/CalculatorTest/testAdd()") - assertThat(testwiseReport.tests.first().paths).isNotEmpty() - assertThat(testwiseReport.tests.last().paths).isNotEmpty() + assertThat(testwiseReport.tests).extracting { it.uniformPath } + .containsExactly( + "com/example/app/MainTest/testMain()", + // Each test method of a @ParameterizedClass is one test, no matter how many parameter sets it ran with. + "com/example/lib/CalculatorParameterizedTest/testAddIsCommutative()", + "com/example/lib/CalculatorTest/testAdd()" + ) + assertThat(testwiseReport.tests).allMatch { it.paths.isNotEmpty() } } @Test @@ -54,12 +56,15 @@ class TestwiseCoverageGradleSystemTest { val testwiseReport = teamscaleMockServer.getOnlyTestwiseCoverageReport("System Tests") assertThat(testwiseReport.partial).isEqualTo(true) - assertThat(testwiseReport.tests.first().uniformPath) - .isEqualTo("com/example/app/MainTest/testMain()") - assertThat(testwiseReport.tests.last().uniformPath) - .isEqualTo("com/example/lib/CalculatorTest/testAdd()") + assertThat(testwiseReport.tests).extracting { it.uniformPath } + .containsExactly( + "com/example/app/MainTest/testMain()", + "com/example/lib/CalculatorParameterizedTest/testAddIsCommutative()", + "com/example/lib/CalculatorTest/testAdd()" + ) + // Only the impacted test was executed, so it is the only one with coverage. assertThat(testwiseReport.tests.first().paths).isNotEmpty() - assertThat(testwiseReport.tests.last().paths).isEmpty() + assertThat(testwiseReport.tests.drop(1)).allMatch { it.paths.isEmpty() } } @Test @@ -69,8 +74,9 @@ class TestwiseCoverageGradleSystemTest { assertThat(result.isSuccess).isTrue() val session = teamscaleMockServer.getOnlySession("Unit Tests") - assertThat(session.getReports()).hasSize(3) - assertThat(session.getReports(EReportFormat.JUNIT)).hasSize(2) + assertThat(session.getReports()).hasSize(4) + // One JUnit report per test class: MainTest, CalculatorTest and CalculatorParameterizedTest. + assertThat(session.getReports(EReportFormat.JUNIT)).hasSize(3) val compactReport = session.getCompactCoverageReport(0)!! @@ -89,8 +95,9 @@ class TestwiseCoverageGradleSystemTest { assertThat(result.isSuccess).isTrue() val session = teamscaleMockServer.getOnlySession("Default Tests") - assertThat(session.getReports()).hasSize(3) - assertThat(session.getReports(EReportFormat.JUNIT)).hasSize(2) + assertThat(session.getReports()).hasSize(4) + // One JUnit report per test class: MainTest, CalculatorTest and CalculatorParameterizedTest. + assertThat(session.getReports(EReportFormat.JUNIT)).hasSize(3) val compactReport = session.getCompactCoverageReport(0)!! assertThat(compactReport.coverage.first().filePath).isEqualTo("com/example/app/Main.java") diff --git a/teamscale-maven-plugin/src/main/java/com/teamscale/maven/tia/TestwiseCoverageReportMojo.java b/teamscale-maven-plugin/src/main/java/com/teamscale/maven/tia/TestwiseCoverageReportMojo.java index dae1aae67..97988fad7 100644 --- a/teamscale-maven-plugin/src/main/java/com/teamscale/maven/tia/TestwiseCoverageReportMojo.java +++ b/teamscale-maven-plugin/src/main/java/com/teamscale/maven/tia/TestwiseCoverageReportMojo.java @@ -6,6 +6,7 @@ import com.teamscale.report.ReportUtils; import com.teamscale.report.testwise.ETestArtifactFormat; import com.teamscale.report.testwise.TestwiseCoverageReportWriter; +import com.teamscale.report.testwise.jacoco.cache.CoverageGenerationException; import com.teamscale.report.testwise.jacoco.JaCoCoTestwiseReportGenerator; import com.teamscale.report.testwise.model.TestExecution; import com.teamscale.report.testwise.model.factory.TestInfoFactory; @@ -122,20 +123,27 @@ private void generateTestwiseCoverageReport(JaCoCoTestwiseReportGenerator genera "Could not create the testwise-coverage report folder " + reportsFolder + ". Check that the parent directory is writable.", e); } - List jacocoExecutionDataList = ReportUtils.listFiles(ETestArtifactFormat.JACOCO, reportFileDirectories); - String reportFilePath = reportsFolder.resolve("testwise-coverage.json").toString(); + writeTestwiseCoverageReport(generator, testInfoFactory, reportFileDirectories, + reportsFolder.resolve("testwise-coverage.json")); + } + /** Converts the JaCoCo execution data found in the given directories into one testwise coverage report. */ + private void writeTestwiseCoverageReport(JaCoCoTestwiseReportGenerator generator, TestInfoFactory testInfoFactory, + List reportFileDirectories, Path reportFile) throws MojoFailureException { + List jacocoExecutionDataList = ReportUtils.listFiles(ETestArtifactFormat.JACOCO, reportFileDirectories); Boolean partial = runImpacted && !runAllTests; try (TestwiseCoverageReportWriter coverageWriter = new TestwiseCoverageReportWriter(testInfoFactory, - new File(reportFilePath), splitAfter, partial)) { - for (File executionDataFile : jacocoExecutionDataList) { - logger.info("Writing execution data for file: " + executionDataFile.getName()); - generator.convertAndConsume(executionDataFile, coverageWriter); - } + reportFile.toFile(), splitAfter, partial)) { + logger.info("Writing execution data for files: " + jacocoExecutionDataList); + generator.convertAndConsumePerTest(jacocoExecutionDataList, coverageWriter); } catch (IOException e) { throw new MojoFailureException( - "Could not write the testwise coverage report to " + reportFilePath + "Could not write the testwise coverage report to " + reportFile + ". Check disk space and that the file is not held open by another process.", e); + } catch (CoverageGenerationException e) { + throw new MojoFailureException( + "Could not generate the testwise coverage for " + jacocoExecutionDataList + + ". Check that the class files given to the plugin match the profiled ones.", e); } }