Skip to content

Commit d1b7285

Browse files
MarkoutteMarkoutte
authored
Improve fuzzing for domain specific API (#2256)
1 parent e6c479a commit d1b7285

File tree

14 files changed

+301
-145
lines changed

14 files changed

+301
-145
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt‎

Lines changed: 27 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,8 @@ import org.utbot.framework.util.graph
4040
import org.utbot.framework.util.sootMethod
4141
import org.utbot.fuzzer.*
4242
import org.utbot.fuzzing.*
43-
import org.utbot.fuzzing.providers.AutowiredValueProvider
44-
import org.utbot.fuzzing.type.factories.SimpleFuzzedTypeFactory
45-
import org.utbot.fuzzing.type.factories.SpringFuzzedTypeFactory
43+
import org.utbot.fuzzing.providers.ObjectValueProvider
44+
import org.utbot.fuzzing.spring.SpringBeanValueProvider
4645
import org.utbot.fuzzing.utils.Trie
4746
import org.utbot.instrumentation.ConcreteExecutor
4847
import org.utbot.instrumentation.instrumentation.Instrumentation
@@ -344,44 +343,41 @@ class UtBotSymbolicEngine(
344343
classId != Method::class.java.id && // causes the instrumented process crash at invocation
345344
classId != Class::class.java.id // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method)
346345
}
347-
val hasMethodUnderTestParametersToFuzz = methodUnderTest.parameters.isNotEmpty()
348-
if (!isFuzzable || !hasMethodUnderTestParametersToFuzz && methodUnderTest.isStatic) {
349-
// Currently, fuzzer doesn't work with static methods with empty parameters
346+
if (!isFuzzable) {
350347
return@flow
351348
}
352349
val errorStackTraceTracker = Trie(StackTraceElement::toString)
353350
var attempts = 0
354351
val attemptsLimit = UtSettings.fuzzingMaxAttempts
355352
val names = graph.body.method.tags.filterIsInstance<ParamNamesTag>().firstOrNull()?.names ?: emptyList()
356353
var testEmittedByFuzzer = 0
354+
val valueProviders = ValueProvider.of(defaultValueProviders(defaultIdGenerator))
355+
.letIf(applicationContext is SpringApplicationContext
356+
&& applicationContext.typeReplacementApproach is TypeReplacementApproach.ReplaceIfPossible
357+
) { provider ->
358+
// spring should try to generate bean values, but if it fails, then object value provider is used for it
359+
val springBeanValueProvider = SpringBeanValueProvider(
360+
defaultIdGenerator,
361+
beanProvider = { classId ->
362+
(applicationContext as SpringApplicationContext).beanDefinitions
363+
.filter { it.beanTypeFqn == classId.name }
364+
.map { it.beanName }
365+
},
366+
autowiredModelOriginCreator = { beanName ->
367+
runBlocking {
368+
logger.info { "Getting bean: $beanName" }
369+
concreteExecutor.withProcess { getBean(beanName) }
370+
}
371+
}).withFallback(ObjectValueProvider(defaultIdGenerator))
372+
373+
provider.except { p -> p is ObjectValueProvider }.with(springBeanValueProvider)
374+
}.let(transform)
357375
runJavaFuzzing(
358376
defaultIdGenerator,
359377
methodUnderTest,
360-
collectConstantsForFuzzer(graph),
361-
names,
362-
listOf(transform(ValueProvider.of(defaultValueProviders(defaultIdGenerator)))),
363-
fuzzedTypeFactory = when (applicationContext) {
364-
is SpringApplicationContext -> when (applicationContext.typeReplacementApproach) {
365-
is TypeReplacementApproach.ReplaceIfPossible -> SpringFuzzedTypeFactory(
366-
autowiredValueProvider = AutowiredValueProvider(
367-
defaultIdGenerator,
368-
autowiredModelOriginCreator = { beanName ->
369-
runBlocking {
370-
logger.info { "Getting bean: $beanName" }
371-
concreteExecutor.withProcess { getBean(beanName) }
372-
}
373-
}
374-
),
375-
beanNamesFinder = { classId ->
376-
applicationContext.beanDefinitions
377-
.filter { it.beanTypeFqn == classId.name }
378-
.map { it.beanName }
379-
}
380-
)
381-
is TypeReplacementApproach.DoNotReplace -> SimpleFuzzedTypeFactory()
382-
}
383-
else -> SimpleFuzzedTypeFactory()
384-
},
378+
constants = collectConstantsForFuzzer(graph),
379+
names = names,
380+
providers = listOf(valueProviders),
385381
) { thisInstance, descr, values ->
386382
if (thisInstance?.model is UtNullModel) {
387383
// We should not try to run concretely any models with null-this.

‎utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -156,16 +156,20 @@ private object ConstantsFromCast: ConstantsFinder {
156156
val const = next.useBoxes.findFirstInstanceOf<Constant>()
157157
if (const != null) {
158158
val op = (nextDirectUnit(graph, next) as? JIfStmt)?.let(::sootIfToFuzzedOp) ?: FuzzedContext.Unknown
159-
val exactValue = const.plainValue as Number
160-
return listOfNotNull(
161-
when (value.op.type) {
162-
is ByteType -> FuzzedConcreteValue(byteClassId, exactValue.toByte(), op)
163-
is ShortType -> FuzzedConcreteValue(shortClassId, exactValue.toShort(), op)
164-
is IntType -> FuzzedConcreteValue(intClassId, exactValue.toInt(), op)
165-
is FloatType -> FuzzedConcreteValue(floatClassId, exactValue.toFloat(), op)
166-
else -> null
159+
when (val exactValue = const.plainValue) {
160+
is Number -> return listOfNotNull(
161+
when (value.op.type) {
162+
is ByteType -> FuzzedConcreteValue(byteClassId, exactValue.toByte(), op)
163+
is ShortType -> FuzzedConcreteValue(shortClassId, exactValue.toShort(), op)
164+
is IntType -> FuzzedConcreteValue(intClassId, exactValue.toInt(), op)
165+
is FloatType -> FuzzedConcreteValue(floatClassId, exactValue.toFloat(), op)
166+
else -> null
167+
}
168+
)
169+
is String -> {
170+
return listOfNotNull(FuzzedConcreteValue(stringClassId, exactValue, op))
167171
}
168-
)
172+
}
169173
}
170174
}
171175
return emptyList()

‎utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt‎

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,15 @@ private val logger by lazy { KotlinLogging.logger {} }
2020
* @see [org.utbot.fuzzing.demo.JsonFuzzingKt]
2121
*/
2222
interface Fuzzing<TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<TYPE, RESULT>> {
23+
24+
/**
25+
* Before producing seeds, this method is called to recognize,
26+
* whether seeds should be generated especially.
27+
*
28+
* [Description.clone] method must be overridden, or it throws an exception if the scope is changed.
29+
*/
30+
fun enrich(description: DESCRIPTION, type: TYPE, scope: Scope) {}
31+
2332
/**
2433
* Generates seeds for a concrete type.
2534
*
@@ -57,7 +66,7 @@ interface Fuzzing<TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feed
5766
* Checks whether the fuzzer should stop.
5867
*/
5968
suspend fun isCancelled(description: DESCRIPTION, stats: Statistic<TYPE, RESULT>): Boolean {
60-
return description.parameters.isEmpty()
69+
return false
6170
}
6271

6372
suspend fun beforeIteration(description: DESCRIPTION, statistics: Statistic<TYPE, RESULT>) { }
@@ -71,6 +80,35 @@ open class Description<TYPE>(
7180
parameters: List<TYPE>
7281
) {
7382
val parameters: List<TYPE> = parameters.toList()
83+
84+
open fun clone(scope: Scope): Description<TYPE> {
85+
error("Scope was changed for $this, but method clone is not specified")
86+
}
87+
}
88+
89+
class Scope(
90+
val parameterIndex: Int,
91+
val recursionDepth: Int,
92+
private val properties: MutableMap<ScopeProperty<*>, Any?> = hashMapOf(),
93+
) {
94+
fun <T> putProperty(param: ScopeProperty<T>, value: T) {
95+
properties[param] = value
96+
}
97+
98+
fun <T> getProperty(param: ScopeProperty<T>): T? {
99+
@Suppress("UNCHECKED_CAST")
100+
return properties[param] as? T
101+
}
102+
103+
fun isNotEmpty(): Boolean = properties.isNotEmpty()
104+
}
105+
106+
class ScopeProperty<T>(
107+
val description: String
108+
) {
109+
fun getValue(scope: Scope): T? {
110+
return scope.getProperty(this)
111+
}
74112
}
75113

76114
/**
@@ -335,13 +373,19 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
335373
state: State<TYPE, RESULT>,
336374
): Node<TYPE, RESULT> {
337375
val typeCache = mutableMapOf<TYPE, MutableList<Result<TYPE, RESULT>>>()
338-
val result = parameters.map { type ->
376+
val result = parameters.mapIndexed { index, type ->
339377
val results = typeCache.computeIfAbsent(type) { mutableListOf() }
340378
if (results.isNotEmpty() && random.flipCoin(configuration.probReuseGeneratedValueForSameType)) {
341379
// we need to check cases when one value is passed for different arguments
342380
results.random(random)
343381
} else {
344-
produce(type, fuzzing, description, random, configuration, state).also {
382+
produce(type, fuzzing, description, random, configuration, State(
383+
state.recursionTreeDepth,
384+
state.cache,
385+
state.missedTypes,
386+
state.iterations,
387+
index
388+
)).also {
345389
results += it
346390
}
347391
}
@@ -358,7 +402,22 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
358402
configuration: Configuration,
359403
state: State<TYPE, RESULT>,
360404
): Result<TYPE, RESULT> {
361-
val candidates = state.cache.computeIfAbsent(type) { fuzzing.generate(description, type).toList() }.map {
405+
val scope = Scope(state.parameterIndex, state.recursionTreeDepth).apply {
406+
fuzzing.enrich(description, type, this)
407+
}
408+
@Suppress("UNCHECKED_CAST")
409+
val seeds = when {
410+
scope.isNotEmpty() -> {
411+
fuzzing.generate(description.clone(scope) as DESCRIPTION, type).toList()
412+
}
413+
else -> state.cache.computeIfAbsent(type) {
414+
fuzzing.generate(description, it).toList()
415+
}
416+
}
417+
if (seeds.isEmpty()) {
418+
throw NoSeedValueException(type)
419+
}
420+
val candidates = seeds.map {
362421
@Suppress("UNCHECKED_CAST")
363422
when (it) {
364423
is Seed.Simple<TYPE, RESULT> -> Result.Simple(it.value, it.mutation)
@@ -367,9 +426,6 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
367426
is Seed.Collection<TYPE, RESULT> -> reduce(it, fuzzing, description, random, configuration, state)
368427
}
369428
}
370-
if (candidates.isEmpty()) {
371-
throw NoSeedValueException(type)
372-
}
373429
return candidates.random(random)
374430
}
375431

@@ -618,6 +674,7 @@ private class State<TYPE, RESULT>(
618674
val cache: MutableMap<TYPE, List<Seed<TYPE, RESULT>>>,
619675
val missedTypes: MissedSeed<TYPE, RESULT>,
620676
val iterations: Int = -1,
677+
val parameterIndex: Int = -1,
621678
)
622679

623680
/**

‎utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ class BaseFuzzing<T, R, D : Description<T>, F : Feedback<T, R>>(
3131

3232
constructor(vararg providers: ValueProvider<T, R, D>, exec: suspend (description: D, values: List<R>) -> F) : this(providers.toList(), exec)
3333

34+
override fun enrich(description: D, type: T, scope: Scope) {
35+
providers.asSequence().forEach {
36+
it.enrich(description, type, scope)
37+
}
38+
}
39+
3440
override fun generate(description: D, type: T): Sequence<Seed<T, R>> {
3541
return providers.asSequence().flatMap { provider ->
3642
try {
@@ -55,6 +61,9 @@ class BaseFuzzing<T, R, D : Description<T>, F : Feedback<T, R>>(
5561
* Value provider generates [Seed] and has other methods to combine providers.
5662
*/
5763
fun interface ValueProvider<T, R, D : Description<T>> {
64+
65+
fun enrich(description: D, type: T, scope: Scope) {}
66+
5867
/**
5968
* Generate a sequence of [Seed] that is merged with values generated by other provider.
6069
*/
@@ -104,29 +113,45 @@ fun interface ValueProvider<T, R, D : Description<T>> {
104113
}
105114
}
106115

116+
/**
117+
* Uses fallback value provider in case when 'this' one failed to generate any value.
118+
*/
107119
fun withFallback(fallback: ValueProvider<T, R, D>) : ValueProvider<T, R, D> {
108120
val thisProvider = this
109-
return ValueProvider { description, type ->
110-
val default = if (accept(type)) thisProvider.generate(description, type) else emptySequence()
111-
if (default.iterator().hasNext()) {
112-
default
113-
} else if (fallback.accept(type)) {
114-
fallback.generate(description, type)
115-
} else {
116-
emptySequence()
121+
return object : ValueProvider<T, R, D> {
122+
override fun enrich(description: D, type: T, scope: Scope) {
123+
thisProvider.enrich(description, type, scope)
124+
// Enriching scope by fallback value provider in this point is not quite right,
125+
// but it doesn't look as a problem right now.
126+
fallback.enrich(description, type, scope)
127+
}
128+
129+
override fun generate(description: D, type: T): Sequence<Seed<T, R>> {
130+
val default = if (thisProvider.accept(type)) thisProvider.generate(description, type) else emptySequence()
131+
return if (default.iterator().hasNext()) {
132+
default
133+
} else if (fallback.accept(type)) {
134+
fallback.generate(description, type)
135+
} else {
136+
emptySequence()
137+
}
117138
}
118139
}
119140
}
120141

121142
/**
122-
* Creates new value provider that creates default value if no values are generated by this provider.
143+
* Creates a new value provider that creates default value if no values are generated by this provider.
123144
*/
124145
fun withFallback(fallbackSupplier: (T) -> Seed<T, R>) : ValueProvider<T, R, D> {
125146
return withFallback { _, type ->
126147
sequenceOf(fallbackSupplier(type))
127148
}
128149
}
129150

151+
fun letIf(flag: Boolean, block: (ValueProvider<T, R, D>) -> ValueProvider<T, R, D>) : ValueProvider<T, R, D> {
152+
return if (flag) block(this) else this
153+
}
154+
130155
/**
131156
* Wrapper class that delegates implementation to the [providers].
132157
*/
@@ -143,6 +168,10 @@ fun interface ValueProvider<T, R, D : Description<T>> {
143168
}
144169
}
145170

171+
override fun enrich(description: D, type: T, scope: Scope) {
172+
providers.forEach { it.enrich(description, type, scope) }
173+
}
174+
146175
override fun accept(type: T): Boolean {
147176
return providers.any { it.accept(type) }
148177
}

‎utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/AutowiredFuzzedType.kt‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/FuzzedMethodDescription.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ open class FuzzedMethodDescription(
4444
*/
4545
var isNested: Boolean = false
4646

47+
/**
48+
* True, if method is static, false if it's not and null if it is not defined.
49+
*/
50+
var isStatic: Boolean? = null
51+
4752
/**
4853
* Returns parameter name by its index in the signature
4954
*/

0 commit comments

Comments
 (0)