Improve reporting for parallel instrumentation tests (#307)

Wrap the default RunNotifier with a parallel-aware variant if JUnit Jupiter's parallel test execution is enabled.
It injects itself into the default AndroidX instrumentation and reorders the emission of test events as necessary.
This requires a hefty bit of reflective inspection, therefore guard this access via a helper class.

As for the sample app, enable parallelism for it and update the tests to demonstrate the effect of it further.
Finally, improve the error message when trying to launch a UI test (e.g. Espresso) in parallel,
since that doesn't work.

Resolves #295.

Author: mannodermaus

instrumentation/CHANGELOG.md

@@ -7,6 +7,7 @@ Change Log
 - Add support for inherited tests (#288)
 - Only autoconfigure JUnit 5 for instrumentation tests when the user explicitly adds junit-jupiter-api as a dependency
 - Prevent noisy logs in Logcat complaining about unresolvable annotation classes (#306)
+- Add support for parallel execution of non-UI instrumentation tests (#295)
 
 ## 1.3.0 (2021-09-17)
 

instrumentation/build.gradle.kts

@@ -31,4 +31,6 @@ apiValidation {
   ignoredPackages.add("de.mannodermaus.junit5.internal")
   ignoredPackages.add("de.mannodermaus.junit5.compose.internal")
   ignoredProjects.add("sample")
+  ignoredProjects.add("testutil")
+  ignoredProjects.add("testutil-reflect")
 }

instrumentation/core/src/main/java/de/mannodermaus/junit5/ActivityScenarioExtension.kt

@@ -12,6 +12,7 @@ import org.junit.jupiter.api.extension.ExtensionContext
 import org.junit.jupiter.api.extension.ParameterContext
 import org.junit.jupiter.api.extension.ParameterResolver
 import org.junit.jupiter.api.extension.RegisterExtension
+import org.junit.jupiter.api.parallel.ExecutionMode
 import java.lang.reflect.ParameterizedType
 
 /**
@@ -153,6 +154,13 @@ private constructor(private val scenarioSupplier: () -> ActivityScenario<A>) : B
     /* Methods */
 
     override fun beforeEach(context: ExtensionContext) {
+        require(context.executionMode == ExecutionMode.SAME_THREAD) {
+            "UI tests using ActivityScenarioExtension cannot be executed in ${context.executionMode} mode. " +
+                    "Please change it to ${ExecutionMode.SAME_THREAD}, e.g. via the @Execution annotation! " +
+                    "For more information, you can consult the JUnit 5 User Guide at " +
+                    "https://junit.org/junit5/docs/current/user-guide/#writing-tests-parallel-execution-synchronization."
+        }
+
         _scenario = scenarioSupplier()
     }
 

instrumentation/runner/build.gradle.kts

@@ -83,6 +83,7 @@ configurations.all {
 
 dependencies {
   implementation(libs.androidXTestMonitor)
+  implementation(libs.androidXTestRunner)
   implementation(libs.kotlinStdLib)
   implementation(libs.junit4)
 

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnit5.kt

@@ -5,32 +5,27 @@ import android.util.Log
 import androidx.annotation.VisibleForTesting
 import de.mannodermaus.junit5.internal.LOG_TAG
 import de.mannodermaus.junit5.internal.LibcoreAccess
+import de.mannodermaus.junit5.internal.runners.notification.ParallelRunNotifier
 import org.junit.platform.launcher.core.LauncherFactory
 import org.junit.runner.Runner
 import org.junit.runner.notification.RunNotifier
 
 /**
  * JUnit Runner implementation using the JUnit Platform as its backbone.
  * Serves as an intermediate solution to writing JUnit 5-based instrumentation tests
- * until official support arrives for this. This is in Java because we require access to package-private data,
- * and Kotlin is more strict about that: https://youtrack.jetbrains.com/issue/KT-15315
- *
- *
- * Replacement For:
- * AndroidJUnit4
+ * until official support arrives for this.
  *
  * @see org.junit.platform.runner.JUnitPlatform
  */
 @SuppressLint("NewApi")
 @VisibleForTesting(otherwise = VisibleForTesting.PACKAGE_PRIVATE)
-internal class AndroidJUnit5
-@JvmOverloads constructor(
+internal class AndroidJUnit5(
     private val testClass: Class<*>,
-    private val runnerParams: AndroidJUnit5RunnerParams = createRunnerParams(testClass)
+    private val runnerParams: AndroidJUnit5RunnerParams = createRunnerParams(testClass),
 ) : Runner() {
 
     private val launcher = LauncherFactory.create()
-    private val testTree = generateTestTree(runnerParams)
+    private val testTree by lazy { generateTestTree(runnerParams) }
 
     override fun getDescription() =
         testTree.suiteDescription
@@ -41,7 +36,10 @@ internal class AndroidJUnit5
         registerSystemProperties()
 
         // Finally, launch the test plan on the JUnit Platform
-        launcher.execute(testTree.testPlan, AndroidJUnitPlatformRunnerListener(testTree, notifier))
+        launcher.execute(
+            testTree.testPlan,
+            AndroidJUnitPlatformRunnerListener(testTree, createNotifier(notifier)),
+        )
     }
 
     /* Private */
@@ -66,9 +64,19 @@ internal class AndroidJUnit5
         }
     }
 
-    private fun generateTestTree(params: AndroidJUnit5RunnerParams): AndroidJUnitPlatformTestTree {
-        val discoveryRequest = params.createDiscoveryRequest()
-        val testPlan = launcher.discover(discoveryRequest)
-        return AndroidJUnitPlatformTestTree(testPlan, testClass, params.isIsolatedMethodRun())
-    }
+    private fun generateTestTree(params: AndroidJUnit5RunnerParams) =
+        AndroidJUnitPlatformTestTree(
+            testPlan = launcher.discover(params.createDiscoveryRequest()),
+            testClass = testClass,
+            isIsolatedMethodRun = params.isIsolatedMethodRun,
+            isParallelExecutionEnabled = params.isParallelExecutionEnabled,
+        )
+
+    private fun createNotifier(nextNotifier: RunNotifier) =
+        if (testTree.isParallelExecutionEnabled) {
+            // Wrap the default notifier with a special handler for parallel test execution
+            ParallelRunNotifier(nextNotifier)
+        } else {
+            nextNotifier
+        }
 }

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnit5RunnerParams.kt

@@ -26,9 +26,11 @@ internal data class AndroidJUnit5RunnerParams(
             .configurationParameters(this.configurationParameters)
             .build()
 
-    fun isIsolatedMethodRun(): Boolean {
-        return selectors.size == 1 && selectors.first() is MethodSelector
-    }
+    val isIsolatedMethodRun: Boolean
+        get() = selectors.size == 1 && selectors.first() is MethodSelector
+
+    val isParallelExecutionEnabled: Boolean
+        get() = configurationParameters["junit.jupiter.execution.parallel.enabled"] == "true"
 }
 
 private const val ARG_ENVIRONMENT_VARIABLES = "environmentVariables"

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/AndroidJUnitPlatformTestTree.kt

@@ -22,7 +22,8 @@ import java.util.function.Predicate
 internal class AndroidJUnitPlatformTestTree(
     testPlan: TestPlan,
     testClass: Class<*>,
-    private val isIsolatedMethodRun: Boolean
+    private val isIsolatedMethodRun: Boolean,
+    val isParallelExecutionEnabled: Boolean,
 ) {
 
     private val descriptions = mutableMapOf<TestIdentifier, Description>()
@@ -146,7 +147,8 @@ internal class AndroidJUnitPlatformTestTree(
                     .map(nameExtractor)
                     .orElse("<unrooted>"),
                 /* name = */ name,
-                /* uniqueId = */ identifier.uniqueId
+                // Used to distinguish JU5 from other frameworks (e.g. for parallel execution)
+                /* ...annotations = */ org.junit.jupiter.api.Test(),
             )
         } else {
             Description.createSuiteDescription(name, identifier.uniqueId)

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/notification/FilteredRunListener.kt

@@ -0,0 +1,45 @@
+package de.mannodermaus.junit5.internal.runners.notification
+
+import org.junit.runner.Description
+import org.junit.runner.notification.Failure
+import org.junit.runner.notification.RunListener
+
+/**
+ * A wrapper implementation around JUnit's [RunListener] class
+ * which only works selectively. In other words, this implementation only delegates
+ * to its parameter for test descriptors that pass the given [filter].
+ */
+internal class FilteredRunListener(
+    private val delegate: RunListener,
+    private val filter: (Description) -> Boolean,
+) : RunListener() {
+    override fun testStarted(description: Description) {
+        if (filter(description)) {
+            delegate.testStarted(description)
+        }
+    }
+
+    override fun testIgnored(description: Description) {
+        if (filter(description)) {
+            delegate.testIgnored(description)
+        }
+    }
+
+    override fun testFailure(failure: Failure) {
+        if (filter(failure.description)) {
+            delegate.testFailure(failure)
+        }
+    }
+
+    override fun testAssumptionFailure(failure: Failure) {
+        if (filter(failure.description)) {
+            delegate.testAssumptionFailure(failure)
+        }
+    }
+
+    override fun testFinished(description: Description) {
+        if (filter(description)) {
+            delegate.testFinished(description)
+        }
+    }
+}

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/internal/runners/notification/ParallelRunNotifier.kt

@@ -0,0 +1,184 @@
+package de.mannodermaus.junit5.internal.runners.notification
+
+import android.util.Log
+import androidx.test.internal.runner.listener.InstrumentationResultPrinter
+import de.mannodermaus.junit5.internal.LOG_TAG
+import org.junit.runner.Description
+import org.junit.runner.Result
+import org.junit.runner.notification.Failure
+import org.junit.runner.notification.RunListener
+import org.junit.runner.notification.RunNotifier
+
+/**
+ * Wrapping implementation of JUnit 4's run notifier for parallel test execution
+ * (i.e. when "junit.jupiter.execution.parallel.enabled" is active during the run).
+ * It unpacks the singular 'instrumentation result printer' assigned by Android
+ * into using one instance per test, preventing its mutable internals from being
+ * modified by concurrent threads at the same time.
+ */
+internal class ParallelRunNotifier(private val delegate: RunNotifier) : RunNotifier() {
+    companion object {
+        // Reflective access is available via companion object
+        // to allow for shared storage of data across notifiers
+        private val reflection by lazy {
+            try {
+                Reflection()
+            } catch (e: Throwable) {
+                Log.e(LOG_TAG, "FATAL: Cannot initialize reflective access", e)
+                null
+            }
+        }
+    }
+
+    private val states = mutableMapOf<String, InstrumentationResultPrinter?>()
+
+    // Original printer registered via Android instrumentation
+    private val printer = reflection?.initialize(delegate)
+
+    override fun fireTestSuiteStarted(description: Description) {
+        delegate.fireTestSuiteStarted(description)
+    }
+
+    override fun fireTestRunStarted(description: Description) {
+        delegate.fireTestRunStarted(description)
+    }
+
+    override fun fireTestStarted(description: Description) {
+        synchronized(this) {
+            delegate.fireTestStarted(description)
+
+            // Notify original printer immediately,
+            // then freeze its state for the current method for later
+            printer?.testStarted(description)
+            states[description] = reflection?.copy(printer)
+        }
+    }
+
+    override fun fireTestIgnored(description: Description) {
+        synchronized(this) {
+            delegate.fireTestIgnored(description)
+
+            printer?.testIgnored(description)
+        }
+    }
+
+    override fun fireTestFailure(failure: Failure) {
+        delegate.fireTestFailure(failure)
+
+        states[failure.description]?.testFailure(failure)
+    }
+
+    override fun fireTestAssumptionFailed(failure: Failure) {
+        delegate.fireTestAssumptionFailed(failure)
+
+        states[failure.description]?.testAssumptionFailure(failure)
+    }
+
+    override fun fireTestFinished(description: Description) {
+        synchronized(this) {
+            delegate.fireTestFinished(description)
+
+            states[description]?.testFinished(description)
+            states.remove(description)
+        }
+    }
+
+    override fun fireTestRunFinished(result: Result) {
+        delegate.fireTestRunFinished(result)
+    }
+
+    override fun fireTestSuiteFinished(description: Description) {
+        delegate.fireTestSuiteFinished(description)
+    }
+
+    /* Private */
+
+    private operator fun <T> Map<String, T>.get(key: Description): T? {
+        return get(key.displayName)
+    }
+
+    private operator fun <T> MutableMap<String, T>.set(key: Description, value: T) {
+        put(key.displayName, value)
+    }
+
+    private fun <T> MutableMap<String, T>.remove(key: Description) {
+        remove(key.displayName)
+    }
+
+    @Suppress("UNCHECKED_CAST")
+    private class Reflection {
+        private val synchronizedRunListenerClass =
+            Class.forName("org.junit.runner.notification.SynchronizedRunListener")
+        private val synchronizedListenerDelegateField = synchronizedRunListenerClass
+            .getDeclaredField("listener").also { it.isAccessible = true }
+        private val runNotifierListenersField = RunNotifier::class.java
+            .getDeclaredField("listeners").also { it.isAccessible = true }
+
+        private var cached: InstrumentationResultPrinter? = null
+
+        fun initialize(notifier: RunNotifier): InstrumentationResultPrinter? {
+            try {
+                // The printer needs to be retrieved only once per test run
+                cached?.let { return it }
+
+                // The Android system registers a global listener
+                // for communicating status events back to the instrumentation.
+                // In parallel mode, this communication must be piped through
+                // a custom piece of logic in order to not lose any mutable data
+                // from concurrent method invocations
+                val listeners = runNotifierListenersField.get(notifier) as? List<RunListener>
+
+                // The Android instrumentation may wrap the printer inside another JUnit listener,
+                // so make sure to search for the result inside its toString() representation
+                // (rather than through an 'it is X' check)
+                val candidate = listeners?.firstOrNull {
+                    InstrumentationResultPrinter::class.java.name in it.toString()
+                }
+
+                if (candidate != null) {
+                    // Replace the original listener with a wrapped version of itself,
+                    // which will allow all non-JUnit 5 tests through the normal pipeline
+                    // (tests that actually _are_ JUnit 5 will be handled differently)
+                    notifier.removeListener(candidate)
+                    notifier.addListener(FilteredRunListener(candidate, Description::isNotJUnit5))
+                }
+
+                // The Android instrumentation may wrap the printer inside another JUnit listener,
+                // so make sure to search for the result inside its toString() representation
+                // (rather than through an 'it is X' check)
+                val result = if (synchronizedRunListenerClass.isInstance(candidate)) {
+                    synchronizedListenerDelegateField.get(candidate) as? InstrumentationResultPrinter
+                } else {
+                    candidate as? InstrumentationResultPrinter
+                }
+
+                cached = result
+                return result
+            } catch (e: Throwable) {
+                e.printStackTrace()
+                return null
+            }
+        }
+
+        fun copy(original: InstrumentationResultPrinter?): InstrumentationResultPrinter? = try {
+            if (original != null) {
+                InstrumentationResultPrinter().also { copy ->
+                    copy.instrumentation = original.instrumentation
+
+                    InstrumentationResultPrinter::class.java.declaredFields.forEach { field ->
+                        field.isAccessible = true
+                        field.set(copy, field.get(original))
+                    }
+                }
+            } else {
+                null
+            }
+        } catch (e: Throwable) {
+            e.printStackTrace()
+            null
+        }
+    }
+}
+
+private val Description.isNotJUnit5: Boolean
+    get() = getAnnotation(org.junit.jupiter.api.Test::class.java) == null

instrumentation/sample/build.gradle.kts

@@ -22,6 +22,7 @@ android {
     // Make sure to use the AndroidJUnitRunner (or a sub-class) in order to hook in the JUnit 5 Test Builder
     testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
     testInstrumentationRunnerArguments["runnerBuilder"] = "de.mannodermaus.junit5.AndroidJUnit5Builder"
+    testInstrumentationRunnerArguments["configurationParameters"] = "junit.jupiter.execution.parallel.enabled=true,junit.jupiter.execution.parallel.mode.default=concurrent"
 
     buildConfigField("boolean", "MY_VALUE", "true")
   }