Update to JUnit 5.4.1 & Introduce Firebase Test Lab (#156)

Author: Marcel Schnelle

.circleci/config.yml

@@ -7,11 +7,11 @@ defaults: &defaults
         GRADLE_OPTS: '-Dorg.gradle.jvmargs="-Xmx4g -XX:+HeapDumpOnOutOfMemoryError"'
 
 cache_key: &cache_key
-  key: jars-{{ checksum "build.gradle.kts" }}-{{ checksum "plugin/android-junit5/build.gradle.kts" }}-{{ checksum "plugin/android-junit5-tests/build.gradle.kts" }}-{{ checksum "instrumentation/api/build.gradle.kts" }}-{{ checksum "instrumentation/runner/build.gradle.kts" }}-{{ checksum "instrumentation/sample/build.gradle.kts" }}-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }}
+  key: jars-{{ checksum "build.gradle.kts" }}-{{ checksum "plugin/android-junit5/build.gradle.kts" }}-{{ checksum "plugin/android-junit5-tests/build.gradle.kts" }}-{{ checksum "instrumentation/api/build.gradle.kts" }}-{{ checksum "instrumentation/runner/build.gradle.kts" }}-{{ checksum "instrumentation/sample/build.gradle.kts" }}-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }}-{{ checksum "buildSrc/src/main/kotlin/Artifacts.kt" }}-{{ checksum "buildSrc/src/main/kotlin/Libs.kt" }}-{{ checksum "buildSrc/src/main/kotlin/Versions.kt" }}
 
 version: 2
 jobs:
-  build:
+  build_plugin:
     <<: *defaults
     steps:
       - checkout
@@ -25,39 +25,84 @@ jobs:
           name: (Plugin) Test
           command: cd plugin && ./gradlew :android-junit5-tests:check --stacktrace --no-daemon
 
+      - persist_to_workspace:
+          root: ~/root
+          paths:
+            - project
+      - store_artifacts:
+          path: plugin/android-junit5-tests/build/reports/tests/test
+          destination: plugin
+
+  build_instrumentation:
+    <<: *defaults
+    steps:
+      - checkout
+      - restore_cache:
+          <<: *cache_key
       - run:
           name: (Instrumentation) Download Dependencies
           command: cd instrumentation && ./gradlew androidDependencies --no-daemon
+      - save_cache:
+          <<: *cache_key
+          paths:
+            - ~/.gradle/caches
+            - ~/.gradle/wrapper
       - run:
           name: (Instrumentation) Build
-          command: cd instrumentation && ./gradlew :api:assemble :core:assemble :runner:assemble --stacktrace --no-daemon
+          command: |
+            cd instrumentation
+            ./gradlew :api:assembleDebug \
+            :core:assembleDebug :core:assembleDebugAndroidTest \
+            :runner:assembleDebug \
+            :sample:assembleDebug --stacktrace --no-daemon
       - run:
           name: (Instrumentation) Test
           command: cd instrumentation && ./gradlew :api:check :core:check :runner:check --stacktrace --no-daemon
+      - run:
+          name: Store Google Service Account
+          command: echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
+      - run:
+          name: Authorize gcloud and set config defaults
+          command: |
+            sudo gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+            sudo gcloud --quiet config set project ${GOOGLE_PROJECT_ID}
+      - run:
+          name: Test with Firebase Test Lab
+          command: >
+            sudo gcloud firebase test android run \
+              --type instrumentation \
+              --app instrumentation/sample/build/outputs/apk/debug/sample-debug.apk \
+              --test instrumentation/core/build/outputs/apk/androidTest/debug/android-test-core-debug-androidTest.apk \
+              --environment-variables runnerBuilder=de.mannodermaus.junit5.AndroidJUnit5Builder \
+              --test-runner-class androidx.test.runner.AndroidJUnitRunner \
+              --device model=sailfish,version=28,locale=en_US,orientation=portrait \
+              --results-bucket cloud-test-${GOOGLE_PROJECT_ID} \
+              --timeout 15m
+      - run:
+          name: Install gsutil dependency and copy test results data
+          command: |
+            sudo pip install -U crcmod
+            mkdir test-lab-results
+            sudo gsutil -m cp -r -U `sudo gsutil ls gs://cloud-test-${GOOGLE_PROJECT_ID} | tail -1` test-lab-results/ | true
 
-      - save_cache:
-          <<: *cache_key
-          paths:
-            - ~/.gradle/caches
-            - ~/.gradle/wrapper
       - persist_to_workspace:
           root: ~/root
           paths:
             - project
-      - store_artifacts:
-          path: plugin/android-junit5-tests/build/reports/tests/test
-          destination: plugin
       - store_artifacts:
           path: instrumentation/api/build/reports
           destination: instrumentation-api
       - store_artifacts:
           path: instrumentation/core/build/reports
           destination: instrumentation-core
+      - store_artifacts:
+          path: test-lab-results
+          destination: instrumentation-core/test-lab-results
       - store_artifacts:
           path: instrumentation/runner/build/reports
           destination: instrumentation-runner
 
-  deploy_snapshots:
+  deploy_plugin_snapshots:
     <<: *defaults
     steps:
       - attach_workspace:
@@ -67,6 +112,14 @@ jobs:
       - run:
           name: (Plugin) Deploy SNAPSHOTs
           command: cd plugin && ./gradlew publishLibraryPublicationToSnapshotRepository --stacktrace --no-daemon
+
+  deploy_instrumentation_snapshots:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/root
+      - restore_cache:
+          <<: *cache_key
       - run:
           name: (Instrumentation) Deploy SNAPSHOTs
           command: cd instrumentation && ./gradlew publishLibraryPublicationToSnapshotRepository --stacktrace --no-daemon
@@ -98,24 +151,31 @@ workflows:
 
   build_deploy:
     jobs:
-      - build
-      - deploy_snapshots:
+      - build_plugin
+      - build_instrumentation
+      - deploy_plugin_snapshots:
+          requires:
+            - build_plugin
+          filters:
+            branches:
+              only: master
+      - deploy_instrumentation_snapshots:
           requires:
-            - build
+            - build_instrumentation
           filters:
             branches:
               only: master
       - deploy_plugin_release:
           requires:
-            - build
+            - build_plugin
           filters:
             branches:
               only: master
             tags:
               only: plugin-*
       - deploy_instrumentation_release:
           requires:
-            - build
+            - build_instrumentation
           filters:
             branches:
               only: master

buildSrc/src/main/kotlin/Artifacts.kt

@@ -35,7 +35,7 @@ object Artifacts {
       platform = Java,
       groupId = "de.mannodermaus.gradle.plugins",
       artifactId = "android-junit5",
-      currentVersion = "1.4.0.1-SNAPSHOT",
+      currentVersion = "1.4.1.0-SNAPSHOT",
       latestStableVersion = "1.4.0.0",
       license = license,
       description = "Unit Testing with JUnit 5 for Android."

buildSrc/src/main/kotlin/Tasks.kt

@@ -16,7 +16,7 @@ import java.io.File
 open class WriteClasspathResource : DefaultTask() {
 
   @InputFiles
-  lateinit var inputFiles: FileCollection
+  lateinit var inputFiles: Iterable<File>
   @OutputDirectory
   lateinit var outputDir: File
   lateinit var resourceFileName: String

buildSrc/src/main/kotlin/Versions.kt

@@ -58,11 +58,11 @@ object Versions {
 
     const val junit_pioneer: String = "0.2.2" // available: "0.3.0"
 
-    const val org_junit_jupiter: String = "5.4.0"
+    const val org_junit_jupiter: String = "5.4.1"
 
-    const val org_junit_platform: String = "1.4.0"
+    const val org_junit_platform: String = "1.4.1"
 
-    const val junit_vintage_engine: String = "5.4.0"
+    const val junit_vintage_engine: String = "5.4.1"
 
     const val mockito_core: String = "2.19.0" // available: "2.23.4"
 

instrumentation/core/src/androidTest/java/de/mannodermaus/junit5/BaselineJUnit4Test.kt

@@ -0,0 +1,17 @@
+package de.mannodermaus.junit5
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+/**
+ * This JUnit 4 test class is here to validate
+ * that older device, which ignore JUnit 5 tests,
+ * still execute older ones.
+ */
+class BaselineJUnit4Test {
+
+  @Test
+  fun run() {
+    assertEquals(4, 2 + 2)
+  }
+}

plugin/.idea/runConfigurations/Plugin__Run_Unit_Tests__All_.xml

@@ -88,7 +88,6 @@ dependencies {
   testImplementation(project(":android-junit5"))
   testImplementation(gradleTestKit())
   testImplementation(Libs.kotlin_gradle_plugin)
-  testImplementation(Libs.com_android_tools_build_gradle)
   testImplementation(Libs.commons_io)
   testImplementation(Libs.commons_lang)
   testImplementation(Libs.junit)
@@ -99,6 +98,8 @@ dependencies {
   testImplementation(Libs.assertj_core)
   testImplementation(Libs.mockito_core)
 
+  testCompileOnly(Libs.com_android_tools_build_gradle)
+
   testRuntimeOnly(Libs.junit_jupiter_engine)
   testRuntimeOnly(Libs.junit_vintage_engine)
   testRuntimeOnly(Libs.spek_junit_platform_engine)
@@ -128,7 +129,25 @@ dependencies {
 
 // Resource Writers
 tasks.create("writePluginClasspath", WriteClasspathResource::class) {
-  inputFiles = sourceSets["test"].runtimeClasspath
+  // Exclude "current AGP" dependencies from being packages;
+  // those are added again explicitly, through the functionalTestAgpXXX configurations.
+  val targetVersions = listOf(
+      // Android Gradle Plugin itself
+      Versions.com_android_tools_build_gradle,
+      // Dependent artifacts in the Android space which use major version 26, for some reason
+      // (e.g. com.android.tools.analytics).
+      // TODO This will need updating when AGP 4 comes out at some point
+      Versions.com_android_tools_build_gradle.replaceFirst("3.", "26.")
+  )
+  val targetGroups = listOf(
+      "com.android",
+      "androidx"
+  )
+
+  inputFiles = sourceSets["test"].runtimeClasspath.filterNot { file ->
+    // If a dependency file matches any of the blacklisted artifacts, filter it out
+    targetVersions.any { it in file.absolutePath } && targetGroups.any { it in file.absolutePath }
+  }
   outputDir = File("$buildDir/resources/test")
   resourceFileName = "plugin-classpath.txt"
 }

plugin/.idea/runConfigurations/Plugin__Run_Unit_Tests__No_Functional_.xml

@@ -12,8 +12,7 @@ import org.junit.jupiter.api.DynamicTest.dynamicTest
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestFactory
 import org.junit.jupiter.api.extension.ExtendWith
-import org.junitpioneer.jupiter.TempDirectory
-import org.junitpioneer.jupiter.TempDirectory.TempDir
+import org.junit.jupiter.api.io.TempDir
 import java.io.File
 import java.nio.file.Files
 import java.nio.file.Path
@@ -28,7 +27,6 @@ import java.nio.file.Paths
 data class FunctionalTest(val name: String, val config: String, val gradleVersion: String)
 
 @OnlyOnLocalMachine
-@ExtendWith(TempDirectory::class)
 class FunctionalTests {
 
   companion object {
@@ -39,7 +37,7 @@ class FunctionalTests {
 
         FunctionalTest(name = "3.3.x", config = "functionalTestAgp33X", gradleVersion = "5.0"),
 
-        FunctionalTest(name = "3.4.x", config = "functionalTestAgp34X", gradleVersion = "5.1-milestone-1")
+        FunctionalTest(name = "3.4.x", config = "functionalTestAgp34X", gradleVersion = "5.2.1")
     )
     private val SUPPORTED_LANGUAGES = FileLanguage2.values().toList()
 
@@ -48,6 +46,7 @@ class FunctionalTests {
 
   private lateinit var testProjectDir: File
   private lateinit var buildFile: File
+  // Main classpath for plugins
   private lateinit var pluginFiles: List<File>
 
   // Classpath of the main "functionalTest" configuration
@@ -94,6 +93,7 @@ class FunctionalTests {
       buildscript {
         dependencies {
           classpath files(${pluginFiles.splitClasspath()})
+          classpath files(${functionalTestSpecialFiles.splitClasspath()})
         }
       }
       """)