Skip to content

Commit 9d358d7

Browse files
tamarinvs19tamarinvs19
authored
Socket connection in python fuzzing (#1844)
1 parent 29fdb41 commit 9d358d7

28 files changed

+1214
-922
lines changed

‎utbot-python/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ tasks {
2121
}
2222

2323
dependencies {
24-
api(project(":utbot-fuzzers"))
24+
api(project(":utbot-fuzzing"))
2525
api(project(":utbot-framework"))
2626
api(project(":utbot-python-parser"))
2727
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1")

‎utbot-python/samples/easy_samples/annotation_tests.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
class A(Generic[XXX]):
1111
self_: XXX
1212

13-
def f(self, a, b: A[int]):
13+
def f(self, a, b: 'A'[int]):
1414
self.y = b
1515
self.self_.x = b
1616
pass

‎utbot-python/samples/easy_samples/boruvka.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ def boruvka(self) -> None:
5858
component_size = []
5959
mst_weight = 0
6060

61-
minimum_weight_edge: list[Any] = [-1] * self.m_num_of_nodes
61+
minimum_weight_edge: list = [-1] * self.m_num_of_nodes
6262

6363
# A list of components (initialized to all of the nodes)
6464
for node in range(self.m_num_of_nodes):

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt‎

Lines changed: 134 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -3,47 +3,27 @@ package org.utbot.python
33
import kotlinx.coroutines.flow.Flow
44
import kotlinx.coroutines.flow.flow
55
import mu.KotlinLogging
6-
import org.utbot.framework.plugin.api.DocRegularStmt
7-
import org.utbot.framework.plugin.api.EnvironmentModels
8-
import org.utbot.framework.plugin.api.Instruction
9-
import org.utbot.framework.plugin.api.UtError
10-
import org.utbot.framework.plugin.api.UtExecutionResult
11-
import org.utbot.framework.plugin.api.UtExecutionSuccess
12-
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
13-
import org.utbot.framework.plugin.api.UtModel
14-
import org.utbot.fuzzer.FuzzedValue
6+
import org.utbot.framework.plugin.api.*
157
import org.utbot.fuzzer.UtFuzzedExecution
168
import org.utbot.fuzzing.Control
179
import org.utbot.fuzzing.fuzz
1810
import org.utbot.fuzzing.utils.Trie
19-
import org.utbot.python.code.MemoryDump
20-
import org.utbot.python.code.toPythonTree
21-
import org.utbot.python.evaluation.PythonCodeExecutor
22-
import org.utbot.python.evaluation.PythonCodeExecutorImpl
23-
import org.utbot.python.evaluation.PythonEvaluationError
24-
import org.utbot.python.evaluation.PythonEvaluationSuccess
25-
import org.utbot.python.evaluation.PythonEvaluationTimeout
11+
import org.utbot.python.evaluation.*
12+
import org.utbot.python.evaluation.serialiation.MemoryDump
13+
import org.utbot.python.evaluation.serialiation.toPythonTree
2614
import org.utbot.python.framework.api.python.PythonTreeModel
27-
import org.utbot.python.fuzzing.PythonFeedback
28-
import org.utbot.python.fuzzing.PythonFuzzedConcreteValue
29-
import org.utbot.python.fuzzing.PythonFuzzedValue
30-
import org.utbot.python.fuzzing.PythonFuzzing
31-
import org.utbot.python.fuzzing.PythonMethodDescription
15+
import org.utbot.python.framework.api.python.PythonTreeWrapper
16+
import org.utbot.python.fuzzing.*
3217
import org.utbot.python.newtyping.PythonTypeStorage
3318
import org.utbot.python.newtyping.general.Type
3419
import org.utbot.python.newtyping.pythonModules
3520
import org.utbot.python.newtyping.pythonTypeRepresentation
3621
import org.utbot.python.utils.camelToSnakeCase
3722
import org.utbot.summary.fuzzer.names.TestSuggestedInfo
23+
import java.net.ServerSocket
3824

3925
private val logger = KotlinLogging.logger {}
4026

41-
sealed interface FuzzingExecutionFeedback
42-
class ValidExecution(val utFuzzedExecution: UtFuzzedExecution): FuzzingExecutionFeedback
43-
class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback
44-
class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
45-
class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
46-
4727
class PythonEngine(
4828
private val methodUnderTest: PythonMethod,
4929
private val directoriesForSysPath: Set<String>,
@@ -55,6 +35,8 @@ class PythonEngine(
5535
private val pythonTypeStorage: PythonTypeStorage,
5636
) {
5737

38+
private val cache = EvaluationCache()
39+
5840
private fun suggestExecutionName(
5941
description: PythonMethodDescription,
6042
executionResult: UtExecutionResult
@@ -146,109 +128,155 @@ class PythonEngine(
146128
return ValidExecution(utFuzzedExecution)
147129
}
148130

149-
private fun constructEvaluationInput(arguments: List<PythonFuzzedValue>, additionalModules: List<String>): PythonCodeExecutor {
150-
val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
151-
152-
val (thisObject, modelList) =
153-
if (methodUnderTest.hasThisArgument)
154-
Pair(argumentValues[0], argumentValues.drop(1))
155-
else
156-
Pair(null, argumentValues)
157-
158-
val argumentModules = argumentValues
159-
.flatMap { it.allContainingClassIds }
160-
.map { it.moduleName }
161-
.filterNot { it.startsWith(moduleToImport) }
162-
val localAdditionalModules = (additionalModules + argumentModules + moduleToImport).toSet()
163-
164-
return PythonCodeExecutorImpl(
131+
private fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor {
132+
return PythonCodeSocketExecutor(
165133
methodUnderTest,
166-
FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames),
167-
argumentValues.map { FuzzedValue(it) },
168134
moduleToImport,
169-
localAdditionalModules,
170135
pythonPath,
171136
directoriesForSysPath,
172137
timeoutForRun,
138+
pythonWorker,
173139
)
174140
}
175141

176142
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, until: Long): Flow<FuzzingExecutionFeedback> = flow {
177143
val additionalModules = parameters.flatMap { it.pythonModules() }
178144
val coveredLines = initialCoveredLines.toMutableSet()
179145

180-
suspend fun fuzzingResultHandler(description: PythonMethodDescription, arguments: List<PythonFuzzedValue>): PythonFeedback {
181-
val codeExecutor = constructEvaluationInput(arguments, additionalModules)
182-
return when (val evaluationResult = codeExecutor.run()) {
183-
is PythonEvaluationError -> {
184-
val utError = UtError(
185-
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
186-
Throwable(evaluationResult.stackTrace.joinToString("\n"))
187-
)
188-
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
189-
emit(InvalidExecution(utError))
190-
PythonFeedback(control = Control.PASS)
191-
}
146+
ServerSocket(0).use { serverSocket ->
147+
logger.info { "Server port: ${serverSocket.localPort}" }
148+
val manager = PythonWorkerManager(
149+
serverSocket,
150+
pythonPath,
151+
until,
152+
{ constructEvaluationInput(it) },
153+
timeoutForRun.toInt()
154+
)
155+
logger.info { "Executor manager was created successfully" }
192156

193-
is PythonEvaluationTimeout -> {
194-
val utError = UtError(evaluationResult.message, Throwable())
195-
emit(InvalidExecution(utError))
196-
PythonFeedback(control = Control.PASS)
197-
}
157+
fun fuzzingResultHandler(
158+
description: PythonMethodDescription,
159+
arguments: List<PythonFuzzedValue>
160+
): PythonExecutionResult? {
161+
val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
162+
logger.debug(argumentValues.map { it.tree } .toString())
163+
val argumentModules = argumentValues
164+
.flatMap { it.allContainingClassIds }
165+
.map { it.moduleName }
166+
.filterNot { it.startsWith(moduleToImport) }
167+
val localAdditionalModules = (additionalModules + argumentModules + moduleToImport).toSet()
198168

199-
is PythonEvaluationSuccess -> {
200-
val coveredInstructions = evaluationResult.coverage.coveredInstructions
201-
coveredInstructions.forEach { coveredLines.add(it.lineNumber) }
169+
val (thisObject, modelList) =
170+
if (methodUnderTest.hasThisArgument)
171+
Pair(argumentValues[0], argumentValues.drop(1))
172+
else
173+
Pair(null, argumentValues)
174+
val functionArguments = FunctionArguments(
175+
thisObject,
176+
methodUnderTest.thisObjectName,
177+
modelList,
178+
methodUnderTest.argumentsNames
179+
)
180+
try {
181+
return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) {
182+
is PythonEvaluationError -> {
183+
val utError = UtError(
184+
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
185+
Throwable(evaluationResult.stackTrace.joinToString("\n"))
186+
)
187+
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
188+
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
189+
}
202190

203-
val summary = arguments
204-
.zip(methodUnderTest.arguments)
205-
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
191+
is PythonEvaluationTimeout -> {
192+
val utError = UtError(evaluationResult.message, Throwable())
193+
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
194+
}
206195

207-
val hasThisObject = codeExecutor.methodArguments.thisObject != null
196+
is PythonEvaluationSuccess -> {
197+
val coveredInstructions = evaluationResult.coverage.coveredInstructions
198+
coveredInstructions.forEach { coveredLines.add(it.lineNumber) }
208199

209-
when (val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)) {
210-
is ValidExecution -> {
211-
logger.debug { arguments }
212-
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
213-
emit(result)
214-
PythonFeedback(control = Control.CONTINUE, result = trieNode)
215-
}
216-
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
217-
emit(result)
218-
PythonFeedback(control = Control.PASS)
219-
}
220-
is InvalidExecution -> {
221-
emit(result)
222-
PythonFeedback(control = Control.CONTINUE)
200+
val summary = arguments
201+
.zip(methodUnderTest.arguments)
202+
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
203+
204+
val hasThisObject = methodUnderTest.hasThisArgument
205+
206+
when (val result = handleSuccessResult(
207+
parameters,
208+
evaluationResult,
209+
description,
210+
hasThisObject,
211+
summary
212+
)) {
213+
is ValidExecution -> {
214+
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
215+
PythonExecutionResult(
216+
result,
217+
PythonFeedback(control = Control.CONTINUE, result = trieNode)
218+
)
219+
}
220+
221+
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
222+
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
223+
}
224+
225+
is InvalidExecution -> {
226+
PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
227+
}
228+
}
223229
}
224230
}
231+
} catch (_: TimeoutException) {
232+
return null
225233
}
226234
}
227-
}
228235

229-
val pmd = PythonMethodDescription(
230-
methodUnderTest.name,
231-
parameters,
232-
fuzzedConcreteValues,
233-
pythonTypeStorage,
234-
Trie(Instruction::id)
235-
)
236+
val pmd = PythonMethodDescription(
237+
methodUnderTest.name,
238+
parameters,
239+
fuzzedConcreteValues,
240+
pythonTypeStorage,
241+
Trie(Instruction::id)
242+
)
236243

237-
if (parameters.isEmpty()) {
238-
fuzzingResultHandler(pmd, emptyList())
239-
} else {
240-
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
241-
if (isCancelled()) {
242-
logger.info { "Fuzzing process was interrupted" }
243-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
244-
}
245-
if (System.currentTimeMillis() >= until) {
246-
logger.info { "Fuzzing process was interrupted by timeout" }
247-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
248-
}
244+
if (parameters.isEmpty()) {
245+
fuzzingResultHandler(pmd, emptyList())
246+
manager.disconnect()
247+
} else {
248+
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
249+
if (isCancelled()) {
250+
logger.info { "Fuzzing process was interrupted" }
251+
manager.disconnect()
252+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
253+
}
254+
if (System.currentTimeMillis() >= until) {
255+
logger.info { "Fuzzing process was interrupted by timeout" }
256+
manager.disconnect()
257+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
258+
}
249259

250-
return@PythonFuzzing fuzzingResultHandler(description, arguments)
251-
}.fuzz(pmd)
260+
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
261+
val mem = cache.get(pair)
262+
if (mem != null) {
263+
logger.debug("Repeat in fuzzing")
264+
emit(mem.fuzzingExecutionFeedback)
265+
return@PythonFuzzing mem.fuzzingPlatformFeedback
266+
}
267+
val result = fuzzingResultHandler(description, arguments)
268+
if (result == null) { // timeout
269+
logger.info { "Fuzzing process was interrupted by timeout" }
270+
manager.disconnect()
271+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
272+
}
273+
274+
cache.add(pair, result)
275+
emit(result.fuzzingExecutionFeedback)
276+
return@PythonFuzzing result.fuzzingPlatformFeedback
277+
278+
}.fuzz(pmd)
279+
}
252280
}
253281
}
254282
}

0 commit comments

Comments
 (0)