Show JUnit 5 instrumentation tests as ignored on older devices (#154)

Adds a dummy implementation of JUnit 5's instrumentation runner.

This, in conjunction with an overrideLibrary statement in the AndroidManifest,
allows users to write JUnit 5 instrumentation tests & still execute their
suite on lower API levels, without a build failure. At runtime, JUnit 5 tests
will be marked as ignored on these devices.

* Update tests for instrumentation runner, adding support for remaining Test annotations

Now, the plugin can also detect TestTemplate & RepeatedTest properly

Author: Marcel Schnelle

instrumentation/runner/build.gradle.kts

@@ -101,7 +101,7 @@ dependencies {
   compileOnly(Libs.junit_jupiter_params)
   compileOnly(Libs.junit_platform_runner)
 
-  testImplementation(Libs.assertj_core)
+  testImplementation(Libs.truth)
   testImplementation(Libs.mockito_core)
   testImplementation(Libs.junit_jupiter_api)
   testImplementation(Libs.junit_jupiter_params)

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/DummyJUnit5.kt

@@ -0,0 +1,25 @@
+package de.mannodermaus.junit5
+
+import android.util.Log
+import org.junit.runner.Description
+import org.junit.runner.Runner
+import org.junit.runner.notification.RunNotifier
+
+/**
+ * Created by Marcel Schnelle on 2019-03-16.
+ */
+class DummyJUnit5(private val testClass: Class<*>) : Runner() {
+
+  private val testMethods = testClass.jupiterTestMethods()
+
+  override fun run(notifier: RunNotifier) {
+    Log.w(LOG_TAG, "JUnit 5 is not supported on this device. All Jupiter tests will be disabled.")
+
+    for (testMethod in testMethods) {
+      val description = Description.createTestDescription(testClass, testMethod.name)
+      notifier.fireTestIgnored(description)
+    }
+  }
+
+  override fun getDescription(): Description = Description.createSuiteDescription(testClass)
+}

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/Extensions.kt

@@ -0,0 +1,41 @@
+package de.mannodermaus.junit5
+
+import android.util.Log
+import java.lang.reflect.Method
+
+/**
+ * Created by Marcel Schnelle on 2019-03-16.
+ */
+
+const val LOG_TAG = "AndroidJUnit5"
+private val jupiterTestAnnotations = listOf(
+        "org.junit.jupiter.api.Test",
+        "org.junit.jupiter.api.TestFactory",
+        "org.junit.jupiter.api.RepeatedTest",
+        "org.junit.jupiter.api.TestTemplate",
+        "org.junit.jupiter.params.ParameterizedTest")
+
+fun Class<*>.jupiterTestMethods(): List<Method> {
+    val allJupiterMethods = mutableListOf<Method>()
+    try {
+
+        // Check each method in the Class for the presence
+        // of the well-known list of JUnit Jupiter annotations
+        allJupiterMethods += declaredMethods.filter { method ->
+            val annotationClassNames = method.declaredAnnotations.map { it.annotationClass.qualifiedName }
+            jupiterTestAnnotations.firstOrNull { annotation ->
+                annotationClassNames.contains(annotation)
+            } != null
+        }
+
+        // Recursively check inner classes as well
+        declaredClasses.forEach { inner ->
+            allJupiterMethods += inner.jupiterTestMethods()
+        }
+
+    } catch (t: Throwable) {
+        Log.w(LOG_TAG, "${t.javaClass.name} in 'hasJupiterTestMethods()' for $name", t)
+    }
+
+    return allJupiterMethods
+}

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/Runner.kt

@@ -1,5 +1,6 @@
 package de.mannodermaus.junit5
 
+import android.os.Build
 import org.junit.platform.runner.JUnitPlatform
 import org.junit.runner.Runner
 
@@ -18,5 +19,14 @@ internal class AndroidJUnit5(klass: Class<*>) : JUnitPlatform(klass)
 
 /**
  * Since we can't reference AndroidJUnit5 directly, use this factory for instantiation.
+ *
+ * On API 26 and above, delegate to the real implementation to drive JUnit 5 tests.
+ * Below that however, they wouldn't work; for this case, delegate a dummy runner
+ * which will highlight these tests as ignored.
  */
-internal fun createJUnit5Runner(klass: Class<*>): Runner = AndroidJUnit5(klass)
+internal fun createJUnit5Runner(klass: Class<*>): Runner =
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+      AndroidJUnit5(klass)
+    } else {
+      DummyJUnit5(klass)
+    }

instrumentation/runner/src/main/kotlin/de/mannodermaus/junit5/RunnerBuilder.kt

@@ -4,16 +4,6 @@ import android.util.Log
 import org.junit.runner.Runner
 import org.junit.runners.model.RunnerBuilder
 
-/* Constants */
-
-private const val LOG_TAG = "AndroidJUnit5"
-private val jupiterTestAnnotations = listOf(
-    "org.junit.jupiter.api.Test",
-    "org.junit.jupiter.api.TestFactory",
-    "org.junit.jupiter.params.ParameterizedTest")
-
-/* Types */
-
 /**
  * Custom RunnerBuilder hooked into the main Test Instrumentation Runner
  * provided by the Android Test Support Library, which allows to run
@@ -63,9 +53,10 @@ class AndroidJUnit5Builder : RunnerBuilder() {
         return null
       }
 
-      if (!testClass.hasJupiterTestMethods()) {
+      if (testClass.jupiterTestMethods().isEmpty()) {
         return null
       }
+
       return createJUnit5Runner(testClass)
 
     } catch (e: NoClassDefFoundError) {
@@ -79,36 +70,4 @@ class AndroidJUnit5Builder : RunnerBuilder() {
       throw e
     }
   }
-
-  /* Extension Functions */
-
-  private fun Class<*>.hasJupiterTestMethods(): Boolean {
-    try {
-      // Check each method in the Class for the presence
-      // of the well-known list of JUnit Jupiter annotations
-      val testMethod = declaredMethods.firstOrNull { method ->
-        val annotationClassNames = method.declaredAnnotations.map { it.annotationClass.qualifiedName }
-        jupiterTestAnnotations.firstOrNull { annotation ->
-          annotationClassNames.contains(annotation)
-        } != null
-      }
-
-      if (testMethod != null) {
-        Log.i(LOG_TAG, "Jupiter Test Class detected: ${this.name}")
-        return true
-      }
-
-      // Recursively check inner classes as well
-      declaredClasses.forEach { inner ->
-        if (inner.hasJupiterTestMethods()) {
-          return true
-        }
-      }
-
-    } catch (t: Throwable) {
-      Log.w(LOG_TAG, "${t.javaClass.name} in 'hasJupiterTestMethods()' for $name", t)
-    }
-
-    return false
-  }
 }

instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/test/Classes.kt

@@ -1,11 +1,11 @@
 package de.mannodermaus.junit5.test
 
+import org.junit.jupiter.api.*
 import org.junit.jupiter.api.DynamicTest.dynamicTest
-import org.junit.jupiter.api.Nested
-import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.TestFactory
+import org.junit.jupiter.api.extension.*
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.CsvSource
+import java.util.stream.Stream
 
 class DoesntHaveTestMethods
 
@@ -15,6 +15,12 @@ class HasTest {
   }
 }
 
+class HasRepeatedTest {
+  @RepeatedTest(5)
+  fun method(info: RepetitionInfo) {
+  }
+}
+
 class HasTestFactory {
   @TestFactory
   fun method() = listOf(
@@ -23,6 +29,35 @@ class HasTestFactory {
   )
 }
 
+class HasTestTemplate {
+  @TestTemplate
+  @ExtendWith(ExampleInvocationContextProvider::class)
+  fun method(param: String) {
+  }
+
+  class ExampleInvocationContextProvider : TestTemplateInvocationContextProvider {
+    override fun provideTestTemplateInvocationContexts(context: ExtensionContext): Stream<TestTemplateInvocationContext> =
+        listOf("param1", "param2")
+            .map(this::context)
+            .stream()
+
+    override fun supportsTestTemplate(context: ExtensionContext) = true
+
+    private fun context(param: String): TestTemplateInvocationContext =
+        object : TestTemplateInvocationContext {
+          override fun getAdditionalExtensions() = listOf(
+              object : ParameterResolver {
+                override fun supportsParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext) =
+                    parameterContext.parameter.type == String::class.java
+
+                override fun resolveParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext) =
+                    param
+              }
+          )
+        }
+  }
+}
+
 class HasParameterizedTest {
   @ParameterizedTest
   @CsvSource("a", "b")

instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/test/ExtensionsTests.kt

@@ -0,0 +1,50 @@
+@file:Suppress("unused")
+
+package de.mannodermaus.junit5.test
+
+import com.google.common.truth.Truth.assertThat
+import de.mannodermaus.junit5.AndroidJUnit5
+import de.mannodermaus.junit5.jupiterTestMethods
+import org.junit.jupiter.api.DisplayName
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.Arguments
+import org.junit.jupiter.params.provider.MethodSource
+import org.junit.runner.notification.RunNotifier
+
+class ExtensionsTests {
+
+  @ParameterizedTest
+  @MethodSource("jupiterTestMethods")
+  @DisplayName("jupiterTestMethods() has correct values & will execute expected number of tests")
+  fun jupiterTestMethods(klass: Class<*>, expectExecutedTests: Int) {
+    val methods = klass.jupiterTestMethods()
+    if (expectExecutedTests == 0) {
+      assertThat(methods).isEmpty()
+    } else {
+      assertThat(methods).isNotEmpty()
+
+      // Verify number of executed test cases as well
+      val notifier = RunNotifier()
+      val listener = CountingRunListener()
+      notifier.addListener(listener)
+      AndroidJUnit5(klass).run(notifier)
+
+      assertThat(listener.count())
+          .named("Executed ${listener.count()} instead of $expectExecutedTests tests: '${listener.methodNames()}'")
+          .isEqualTo(expectExecutedTests)
+    }
+  }
+
+  companion object {
+    @JvmStatic
+    fun jupiterTestMethods() = listOf(
+        Arguments.of(DoesntHaveTestMethods::class.java, 0),
+        Arguments.of(HasTest::class.java, 1),
+        Arguments.of(HasRepeatedTest::class.java, 5),
+        Arguments.of(HasTestFactory::class.java, 2),
+        Arguments.of(HasTestTemplate::class.java, 2),
+        Arguments.of(HasParameterizedTest::class.java, 2),
+        Arguments.of(HasInnerClassWithTest::class.java, 1)
+    )
+  }
+}

instrumentation/runner/src/test/kotlin/de/mannodermaus/junit5/test/RunnerTests.kt

@@ -0,0 +1,5 @@
+<manifest xmlns:tools="http://schemas.android.com/tools"
+    package="de.mannodermaus.junit5.test">
+
+    <uses-sdk tools:overrideLibrary="de.mannodermaus.junit5" />
+</manifest>