Force mocking without mock framework installed fix

Added mock listeners for engine.
Added force mock listener.
Added url handler for urls like "#utbot/...".
Added message with link to mock framework configuration to test reports if force mocking happened.

Author: Nikita Vlaev

utbot-framework/src/main/kotlin/org/utbot/engine/Mocks.kt

@@ -18,6 +18,7 @@ import java.util.concurrent.atomic.AtomicInteger
 import kotlin.reflect.KFunction2
 import kotlin.reflect.KFunction5
 import kotlinx.collections.immutable.persistentListOf
+import org.utbot.engine.util.mockListeners.MockListenerController
 import soot.BooleanType
 import soot.RefType
 import soot.Scene
@@ -133,7 +134,8 @@ class Mocker(
     private val strategy: MockStrategy,
     private val classUnderTest: ClassId,
     private val hierarchy: Hierarchy,
-    chosenClassesToMockAlways: Set<ClassId>
+    chosenClassesToMockAlways: Set<ClassId>,
+    internal val mockListenerController: MockListenerController? = null,
 ) {
     /**
      * Creates mocked instance of the [type] using mock info if it should be mocked by the mocker,
@@ -164,6 +166,11 @@ class Mocker(
      * For others, if mock is not a new instance mock, asks mock strategy for decision.
      */
     fun shouldMock(
+        type: RefType,
+        mockInfo: UtMockInfo,
+    ): Boolean = checkIfShouldMock(type, mockInfo).also { if (it) mockListenerController?.onShouldMock(strategy) }
+
+    private fun checkIfShouldMock(
         type: RefType,
         mockInfo: UtMockInfo
     ): Boolean {

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -6,15 +6,17 @@ import kotlinx.collections.immutable.persistentSetOf
 import kotlinx.collections.immutable.toPersistentList
 import kotlinx.collections.immutable.toPersistentMap
 import kotlinx.collections.immutable.toPersistentSet
-import kotlinx.coroutines.currentCoroutineContext
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.Job
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.ensureActive
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.FlowCollector
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.onCompletion
 import kotlinx.coroutines.flow.onStart
 import kotlinx.coroutines.isActive
+import kotlinx.coroutines.job
 import kotlinx.coroutines.yield
 import mu.KotlinLogging
 import org.utbot.analytics.EngineAnalyticsContext
@@ -103,6 +105,8 @@ import org.utbot.engine.symbolic.asHardConstraint
 import org.utbot.engine.symbolic.asSoftConstraint
 import org.utbot.engine.symbolic.asAssumption
 import org.utbot.engine.symbolic.asUpdate
+import org.utbot.engine.util.mockListeners.MockListener
+import org.utbot.engine.util.mockListeners.MockListenerController
 import org.utbot.engine.util.statics.concrete.associateEnumSootFieldsWithConcreteValues
 import org.utbot.engine.util.statics.concrete.isEnumAffectingExternalStatics
 import org.utbot.engine.util.statics.concrete.isEnumValuesFieldName
@@ -330,7 +334,7 @@ class UtBotSymbolicEngine(
 
     private val classUnderTest: ClassId = methodUnderTest.clazz.id
 
-    private val mocker: Mocker = Mocker(mockStrategy, classUnderTest, hierarchy, chosenClassesToMockAlways)
+    private val mocker: Mocker = Mocker(mockStrategy, classUnderTest, hierarchy, chosenClassesToMockAlways, MockListenerController(controller))
 
     private val statesForConcreteExecution: MutableList<ExecutionState> = mutableListOf()
 
@@ -541,6 +545,10 @@ class UtBotSymbolicEngine(
                             } else {
                                 traverseStmt(currentStmt)
                             }
+
+                            // Here job can be cancelled from within traverse, e.g. by using force mocking without Mockito.
+                            // So we need to make it throw CancelledException by method below:
+                            currentCoroutineContext().job.ensureActive()
                         } catch (ex: Throwable) {
                             environment.state.close()
 
@@ -2521,6 +2529,8 @@ class UtBotSymbolicEngine(
         )
     }
 
+    fun attachMockListener(mockListener: MockListener) = mocker.mockListenerController?.attach(mockListener)
+
     private fun staticInvoke(invokeExpr: JStaticInvokeExpr): List<MethodResult> {
         val parameters = resolveParameters(invokeExpr.args, invokeExpr.method.parameterTypes)
         val result = mockMakeSymbolic(invokeExpr) ?: mockStaticMethod(invokeExpr.method, parameters)

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/ForceMockListener.kt

@@ -0,0 +1,21 @@
+package org.utbot.engine.util.mockListeners
+import org.utbot.engine.EngineController
+import org.utbot.engine.MockStrategy
+import org.utbot.engine.util.mockListeners.exceptions.ForceMockCancellationException
+
+/**
+ * Listener for mocker events in [org.utbot.engine.UtBotSymbolicEngine].
+ * If forced mock happened, cancels the engine job.
+ *
+ * Supposed to be created only if Mockito is not installed.
+ */
+class ForceMockListener: MockListener {
+    var forceMockHappened = false
+        private set
+
+    override fun onShouldMock(controller: EngineController, strategy: MockStrategy) {
+        // If force mocking happened -- сancel engine job
+        controller.job?.cancel(ForceMockCancellationException())
+        forceMockHappened = true
+    }
+}

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/MockListener.kt

@@ -0,0 +1,11 @@
+package org.utbot.engine.util.mockListeners
+
+import org.utbot.engine.EngineController
+import org.utbot.engine.MockStrategy
+
+/**
+ * Listener that can be attached using [MockListenerController] to mocker in [org.utbot.engine.UtBotSymbolicEngine].
+ */
+interface MockListener {
+    fun onShouldMock(controller: EngineController, strategy: MockStrategy)
+}

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/MockListenerController.kt

@@ -0,0 +1,19 @@
+package org.utbot.engine.util.mockListeners
+
+import org.utbot.engine.EngineController
+import org.utbot.engine.MockStrategy
+
+/**
+ * Controller that allows to attach listeners to mocker in [org.utbot.engine.UtBotSymbolicEngine].
+ */
+class MockListenerController(private val controller: EngineController) {
+    val listeners = mutableListOf<MockListener>()
+
+    fun attach(listener: MockListener) {
+        listeners += listener
+    }
+
+    fun onShouldMock(strategy: MockStrategy) {
+        listeners.map { it.onShouldMock(controller, strategy) }
+    }
+}

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/exceptions/ForceMockCancellationException.kt

@@ -0,0 +1,8 @@
+package org.utbot.engine.util.mockListeners.exceptions
+
+import kotlinx.coroutines.CancellationException
+
+/**
+ * Exception used in [org.utbot.engine.util.mockListeners.ForceMockListener].
+ */
+class ForceMockCancellationException: CancellationException("Forced mocks without Mockito")

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

@@ -188,6 +188,9 @@ data class TestsGenerationReport(
     val classUnderTest: KClass<*>
         get() = executables.firstOrNull()?.clazz ?: error("No executables found in test report")
 
+    // Initial message is generated lazily to avoid evaluation of classUnderTest
+    var initialMessage: () -> String = { "Unit tests for $classUnderTest were generated successfully." }
+
     fun addMethodErrors(testCase: UtTestCase, errors: Map<String, Int>) {
         this.errors[testCase.method] = errors
     }
@@ -212,7 +215,8 @@ data class TestsGenerationReport(
     }
 
     override fun toString(): String = buildString {
-        appendHtmlLine("Unit tests for $classUnderTest were generated successfully.")
+        appendHtmlLine(initialMessage())
+        appendHtmlLine()
         val testMethodsStatistic = executables.map { it.countTestMethods() }
         val errors = executables.map { it.countErrors() }
         val overallTestMethods = testMethodsStatistic.sumBy { it.count }

utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/UtBotTestCaseGenerator.kt

@@ -58,6 +58,7 @@ object UtBotTestCaseGenerator : TestCaseGenerator {
     private val logger = KotlinLogging.logger {}
     private val timeoutLogger = KotlinLogging.logger(logger.name + ".timeout")
 
+    lateinit var configureEngine: (UtBotSymbolicEngine) -> Unit
     lateinit var isCanceled: () -> Boolean
 
     //properties to save time on soot initialization
@@ -71,8 +72,23 @@ object UtBotTestCaseGenerator : TestCaseGenerator {
         classpath: String?,
         dependencyPaths: String,
         isCanceled: () -> Boolean
+    ) = init(
+        buildDir,
+        classpath,
+        dependencyPaths,
+        configureEngine = {},
+        isCanceled
+    )
+
+    fun init(
+        buildDir: Path,
+        classpath: String?,
+        dependencyPaths: String,
+        configureEngine: (UtBotSymbolicEngine) -> Unit,
+        isCanceled: () -> Boolean
     ) {
         this.isCanceled = isCanceled
+        this.configureEngine = configureEngine
         if (isCanceled()) return
 
         checkFrameworkDependencies(dependencyPaths)
@@ -268,13 +284,15 @@ object UtBotTestCaseGenerator : TestCaseGenerator {
                         //yield one to
                         yield()
 
-                        generate(createSymbolicEngine(
+                        val engine: UtBotSymbolicEngine = createSymbolicEngine(
                             controller,
                             method,
                             mockStrategy,
                             chosenClassesToMockAlways,
                             executionTimeEstimator
-                        )).collect {
+                        ).apply(configureEngine)
+
+                        generate(engine).collect {
                             when (it) {
                                 is UtExecution -> method2executions.getValue(method) += it
                                 is UtError -> method2errors.getValue(method).merge(it.description, 1, Int::plus)
@@ -364,7 +382,6 @@ object UtBotTestCaseGenerator : TestCaseGenerator {
         val executions = mutableListOf<UtExecution>()
         val errors = mutableMapOf<String, Int>()
 
-
         runIgnoringCancellationException {
             runBlockingWithCancellationPredicate(isCanceled) {
                 generateAsync(EngineController(), method, mockStrategy).collect {

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/CodeGenerator.kt

@@ -14,6 +14,7 @@ import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.project.Project
 import com.intellij.psi.PsiMethod
 import com.intellij.refactoring.util.classMembers.MemberInfo
+import org.utbot.engine.UtBotSymbolicEngine
 import java.nio.file.Path
 import java.nio.file.Paths
 import kotlin.reflect.KClass
@@ -32,14 +33,15 @@ class CodeGenerator(
     buildDir: String,
     classpath: String,
     pluginJarsPath: String,
-    isCanceled: () -> Boolean
+    configureEngine: (UtBotSymbolicEngine) -> Unit = {},
+    isCanceled: () -> Boolean,
 ) {
     init {
         UtSettings.testMinimizationStrategyType = TestSelectionStrategyType.COVERAGE_STRATEGY
     }
 
-    private val generator = project.service<Settings>().testCasesGenerator.apply {
-        init(Paths.get(buildDir), classpath, pluginJarsPath, isCanceled)
+    val generator = (project.service<Settings>().testCasesGenerator as UtBotTestCaseGenerator).apply {
+        init(Paths.get(buildDir), classpath, pluginJarsPath, configureEngine, isCanceled)
     }
 
     private val settingsState = project.service<Settings>().state
@@ -49,7 +51,7 @@ class CodeGenerator(
     fun generateForSeveralMethods(methods: List<UtMethod<*>>, timeout:Long = UtSettings.utBotGenerationTimeoutInMillis): List<UtTestCase> {
         logger.info("Tests generating parameters $settingsState")
 
-        return (generator as UtBotTestCaseGenerator)
+        return generator
             .generateForSeveralMethods(methods, mockStrategy, chosenClassesToMockAlways, methodsGenerationTimeout = timeout)
             .map { it.summarize(searchDirectory) }
     }

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/TestGenerator.kt

@@ -15,10 +15,6 @@ import org.utbot.framework.plugin.api.UtTestCase
 import org.utbot.intellij.plugin.sarif.SarifReportIdea
 import org.utbot.intellij.plugin.sarif.SourceFindingStrategyIdea
 import org.utbot.intellij.plugin.settings.Settings
-import org.utbot.intellij.plugin.ui.GenerateTestsModel
-import org.utbot.intellij.plugin.ui.SarifReportNotifier
-import org.utbot.intellij.plugin.ui.TestsReportNotifier
-import org.utbot.intellij.plugin.ui.packageName
 import org.utbot.intellij.plugin.ui.utils.getOrCreateSarifReportsPath
 import org.utbot.intellij.plugin.ui.utils.getOrCreateTestResourcesPath
 import org.utbot.sarif.SarifReport
@@ -63,6 +59,11 @@ import org.jetbrains.kotlin.psi.psiUtil.endOffset
 import org.jetbrains.kotlin.psi.psiUtil.startOffset
 import org.jetbrains.kotlin.scripting.resolve.classId
 import org.utbot.intellij.plugin.error.showErrorDialogLater
+import org.utbot.intellij.plugin.ui.GenerateTestsModel
+import org.utbot.intellij.plugin.ui.SarifReportNotifier
+import org.utbot.intellij.plugin.ui.TestReportUrlOpeningListener
+import org.utbot.intellij.plugin.ui.TestsReportNotifier
+import org.utbot.intellij.plugin.ui.packageName
 
 object TestGenerator {
     fun generateTests(model: GenerateTestsModel, testCases: Map<PsiClass, List<UtTestCase>>) {
@@ -317,6 +318,17 @@ object TestGenerator {
         VfsUtil.createDirectories(parent.toString())
         resultedReportedPath.toFile().writeText(testsCodeWithTestReport.testsGenerationReport.getFileContent())
 
+        if (model.forceMockHappened) {
+            testsCodeWithTestReport.testsGenerationReport.apply {
+                initialMessage = {
+                    """
+                    Unit tests for $classUnderTest were generated partially.<br>
+                    <b>Warning</b>: Some test cases were ignored, because no mocking framework is installed in the project.<br>
+                    Better results could be achieved by <a href="${TestReportUrlOpeningListener.prefix}${TestReportUrlOpeningListener.mockitoSuffix}">installing mocking framework</a>.
+                """.trimIndent()
+                }
+            }
+        }
         val notifyMessage = buildString {
             appendHtmlLine(testsCodeWithTestReport.testsGenerationReport.toString())
             appendHtmlLine()
@@ -334,7 +346,7 @@ object TestGenerator {
             """.trimIndent()
             appendHtmlLine(savedFileMessage)
         }
-        TestsReportNotifier.notify(notifyMessage)
+        TestsReportNotifier.notify(notifyMessage, model.project, model.testModule)
     }
 
     @Suppress("unused")