Refactoring fuzzing mutations

Author: Markoutte

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

@@ -276,13 +276,6 @@ object UtSettings {
      */
     var fuzzingMaxAttempts: Int by getIntProperty(Int.MAX_VALUE)
 
-    /**
-     * Fuzzing tries to mutate values using this factor.
-     *
-     * If any mutation is successful then counter is reset.
-     */
-    var fuzzingRandomMutationsFactor: Int by getIntProperty(10_000)
-
     /**
      * Fuzzer tries to generate and run tests during this time.
      */

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -83,16 +83,18 @@ import org.utbot.framework.plugin.api.util.description
 import org.utbot.framework.util.jimpleBody
 import org.utbot.framework.plugin.api.util.voidClassId
 import org.utbot.fuzzer.FallbackModelProvider
-import org.utbot.fuzzer.FrequencyRandom
 import org.utbot.fuzzer.FuzzedMethodDescription
 import org.utbot.fuzzer.FuzzedValue
 import org.utbot.fuzzer.ModelProvider
 import org.utbot.fuzzer.ReferencePreservingIntIdGenerator
 import org.utbot.fuzzer.Trie
+import org.utbot.fuzzer.TrieBasedFuzzerStatistics
 import org.utbot.fuzzer.UtFuzzedExecution
+import org.utbot.fuzzer.withMutations
 import org.utbot.fuzzer.collectConstantsForFuzzer
 import org.utbot.fuzzer.defaultModelProviders
-import org.utbot.fuzzer.findNextMutatedValue
+import org.utbot.fuzzer.defaultModelMutators
+import org.utbot.fuzzer.flipCoin
 import org.utbot.fuzzer.fuzz
 import org.utbot.fuzzer.providers.ObjectModelProvider
 import org.utbot.instrumentation.ConcreteExecutor
@@ -394,6 +396,7 @@ class UtBotSymbolicEngine(
         val fallbackModelProvider = FallbackModelProvider(defaultIdGenerator)
         val constantValues = collectConstantsForFuzzer(graph)
 
+        val random = Random(0)
         val thisInstance = when {
             methodUnderTest.isStatic -> null
             methodUnderTest.isConstructor -> if (
@@ -407,7 +410,7 @@ class UtBotSymbolicEngine(
             else -> {
                 ObjectModelProvider(defaultIdGenerator).withFallback(fallbackModelProvider).generate(
                     FuzzedMethodDescription("thisInstance", voidClassId, listOf(methodUnderTest.clazz.id), constantValues)
-                ).take(10).shuffled(Random(0)).map { it.value.model }.first().apply {
+                ).take(10).shuffled(random).map { it.value.model }.first().apply {
                     if (this is UtNullModel) { // it will definitely fail because of NPE,
                         return@flow
                     }
@@ -438,35 +441,10 @@ class UtBotSymbolicEngine(
             fuzz(thisMethodDescription, ObjectModelProvider(defaultIdGenerator).apply {
                 limitValuesCreatedByFieldAccessors = 500
             })
-        }
-
-        val frequencyRandom = FrequencyRandom(Random(221))
-        val factor = maxOf(0, UtSettings.fuzzingRandomMutationsFactor)
-        val fuzzedValueSequence: Sequence<List<FuzzedValue>> = sequence {
-            val fvi = fuzzedValues.iterator()
-            while (fvi.hasNext()) {
-                yield(fvi.next())
-                // if we stuck switch to "next + several mutations" mode
-                if ((attempts + 1) % (factor / 100) == 0 && coveredInstructionValues.isNotEmpty()) {
-                    for (i in 0 until (factor / 100)) {
-                        findNextMutatedValue(frequencyRandom, coveredInstructionValues, methodUnderTestDescription)?.let { yield(it) }
-                    }
-                }
-            }
-            // try mutations if fuzzer tried all combinations
-            var tryLocal = factor
-            while (tryLocal-- >= 0) {
-                val value = findNextMutatedValue(frequencyRandom, coveredInstructionValues, methodUnderTestDescription)
-                if (value != null) {
-                    val before = coveredInstructionValues.size
-                    yield(value)
-                    if (coveredInstructionValues.size != before) {
-                        tryLocal = factor
-                    }
-                }
-            }
-        }
-        fuzzedValueSequence.forEach { values ->
+        }.withMutations(
+            TrieBasedFuzzerStatistics(coveredInstructionValues), methodUnderTestDescription, *defaultModelMutators().toTypedArray()
+        )
+        fuzzedValues.forEach { values ->
             if (controller.job?.isActive == false || System.currentTimeMillis() >= until) {
                 logger.info { "Fuzzing overtime: $methodUnderTest" }
                 return@flow
@@ -510,7 +488,7 @@ class UtBotSymbolicEngine(
                     }
                     // Update the seeded values sometimes
                     // This is necessary because some values cannot do a good values in mutation in any case
-                    if (frequencyRandom.random.nextInt(1, 101) <= 50) {
+                    if (random.flipCoin(probability = 50)) {
                         coveredInstructionValues[coverageKey] = values
                     }
                     return@forEach

utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt

@@ -11,7 +11,6 @@ import org.utbot.framework.plugin.api.util.longClassId
 import org.utbot.framework.plugin.api.util.shortClassId
 import org.utbot.framework.plugin.api.util.stringClassId
 import mu.KotlinLogging
-import org.utbot.framework.plugin.api.Instruction
 import soot.BooleanType
 import soot.ByteType
 import soot.CharType
@@ -43,7 +42,6 @@ import soot.jimple.internal.JNeExpr
 import soot.jimple.internal.JTableSwitchStmt
 import soot.jimple.internal.JVirtualInvokeExpr
 import soot.toolkits.graph.ExceptionalUnitGraph
-import kotlin.math.pow
 
 private val logger = KotlinLogging.logger {}
 
@@ -253,22 +251,3 @@ private fun sootIfToFuzzedOp(unit: JIfStmt) = when (unit.condition) {
 }
 
 private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first()
-
-fun findNextMutatedValue(
-    frequencyRandom: FrequencyRandom,
-    coverage: LinkedHashMap<Trie.Node<Instruction>, List<FuzzedValue>>,
-    description: FuzzedMethodDescription,
-): List<FuzzedValue>? {
-    if (coverage.isEmpty()) return null
-    frequencyRandom.prepare(coverage.map { it.key.count }) { 1 / it.toDouble().pow(2) }
-    val values = coverage.values.drop(frequencyRandom.nextIndex()).firstOrNull()
-    if (values != null) {
-        val mutatedValue = defaultMutators().fold(values) { v, mut ->
-            mut.mutate(description, v, frequencyRandom.random)
-        }
-        if (mutatedValue != values) {
-            return mutatedValue
-        }
-    }
-    return null
-}

utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FrequencyRandom.kt

@@ -9,7 +9,6 @@ import org.utbot.framework.plugin.api.UtModel
 class FuzzedValue(
     val model: UtModel,
     val createdBy: ModelProvider? = null,
-    val mutatedBy: List<ModelMutator> = emptyList(),
 ) {
 
     /**

utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt

@@ -96,6 +96,9 @@ class ReferencePreservingIntIdGenerator(lowerBound: Int = DEFAULT_LOWER_BOUND) :
     }
 }
 
+/**
+ * Generated by fuzzer sequence of values which can be passed into the method.
+ */
 fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvider): Sequence<List<FuzzedValue>> {
     if (modelProviders.isEmpty()) {
         throw IllegalArgumentException("At least one model provider is required")
@@ -117,6 +120,29 @@ fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvi
     return CartesianProduct(values, Random(0L)).asSequence()
 }
 
+/**
+ * Wraps sequence of values, iterates through them and mutates.
+ *
+ * Mutation when possible is generated after every value of source sequence and then supplies values until it needed.
+ * [statistics] should is not updated by this method, but can be changed by caller.
+ */
+fun <T> Sequence<List<FuzzedValue>>.withMutations(statistics: FuzzerStatistics<T>, description: FuzzedMethodDescription, vararg mutators: ModelMutator) = sequence {
+    val fvi = iterator()
+    val mutatorList = mutators.toList()
+    val random = Random(0L)
+    while (fvi.hasNext()) {
+        // Takes a value that was generated by model providers and submits it
+        yield(fvi.next())
+        // Fuzzing can generate values that doesn't recover new paths.
+        // So, fuzzing tries to mutate values on each loop
+        // if there are too much attempts to find new paths without mutations.
+        yieldMutated(statistics, description, mutatorList, random)
+    }
+    // try mutations if fuzzer tried all combinations if any seeds are available
+    @Suppress("ControlFlowWithEmptyBody")
+    while (yieldMutated(statistics, description, mutatorList, random)) {}
+}
+
 /**
  * Creates a model provider from a list of default providers.
  */
@@ -150,4 +176,50 @@ fun objectModelProviders(idGenerator: IdentityPreservingIdGenerator<Int>): Model
     )
 }
 
-fun defaultMutators(): List<ModelMutator> = listOf(StringRandomMutator(50), NumberRandomMutator(50))
+fun defaultModelMutators(): List<ModelMutator> = listOf(StringRandomMutator, NumberRandomMutator)
+
+/**
+ * Tries to mutate a random value from the seed.
+ *
+ * Returns `null` if didn't try to do any mutation.
+ */
+fun <T> mutateRandomValueOrNull(
+    statistics: FuzzerStatistics<T>,
+    description: FuzzedMethodDescription,
+    mutators: List<ModelMutator> = defaultModelMutators(),
+    random: Random = Random,
+): List<FuzzedValue>? {
+    if (mutators.isEmpty()) return null
+    val values = statistics.takeIf { it.isNotEmpty() }?.randomValues(random) ?: return null
+    val newValues = values.toMutableList()
+    mutators.asSequence()
+        .forEach { mut ->
+            mut.mutate(description, values, random).forEach { (index, value) ->
+                newValues[index] = value
+            }
+        }
+    return newValues.takeIf { it != values }
+}
+
+/**
+ * Run mutations and yields values into the sequence.
+ *
+ * Mutations are supplied infinitely until [repeat] returns true. [repeat] is run before mutation.
+ *
+ * @param statistics coverage-based seed
+ * @param description method description
+ * @param mutators mutators which are applied to the random value
+ * @param random instance that is used to choose random index from the [statistics]
+ */
+suspend fun <T> SequenceScope<List<FuzzedValue>>.yieldMutated(
+    statistics: FuzzerStatistics<T>,
+    description: FuzzedMethodDescription,
+    mutators: List<ModelMutator>,
+    random: Random
+) : Boolean {
+    mutateRandomValueOrNull(statistics, description, mutators, random)?.let {
+        yield(it)
+        return true
+    }
+    return false
+}

utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt

@@ -0,0 +1,71 @@
+package org.utbot.fuzzer
+
+import kotlin.math.pow
+import kotlin.random.Random
+
+/**
+ * Stores information that can be useful for fuzzing such as coverage, run count, etc.
+ */
+interface FuzzerStatistics<K> {
+
+    val seeds: Collection<K>
+
+    /**
+     * Returns a random seed to process.
+     */
+    fun randomSeed(random: Random): K?
+
+    fun randomValues(random: Random): List<FuzzedValue>?
+
+    fun executions(seed: K): Int
+
+    operator fun get(seed: K): List<FuzzedValue>?
+
+    fun isEmpty(): Boolean
+
+    fun isNotEmpty(): Boolean {
+        return !isEmpty()
+    }
+}
+
+class TrieBasedFuzzerStatistics<V>(
+    private val values: LinkedHashMap<Trie.Node<V>, List<FuzzedValue>> = linkedMapOf()
+) : FuzzerStatistics<Trie.Node<V>> {
+
+    override val seeds: Collection<Trie.Node<V>>
+        get() = values.keys
+
+    override fun randomSeed(random: Random): Trie.Node<V>? {
+        return values.keys.elementAtOrNull(randomIndex(random))
+    }
+
+    override fun isEmpty(): Boolean {
+        return values.isEmpty()
+    }
+
+    override fun isNotEmpty(): Boolean {
+        return values.isNotEmpty()
+    }
+
+    override fun randomValues(random: Random): List<FuzzedValue>? {
+        return values.values.elementAtOrNull(randomIndex(random))
+    }
+
+    private fun randomIndex(random: Random): Int {
+        val frequencies = DoubleArray(values.size).also { f ->
+            values.keys.forEachIndexed { index, key ->
+                f[index] = 1 / key.count.toDouble().pow(2)
+            }
+        }
+        return random.chooseOne(frequencies)
+    }
+
+    override fun get(seed: Trie.Node<V>): List<FuzzedValue>? {
+        return values[seed]
+    }
+
+    override fun executions(seed: Trie.Node<V>): Int {
+        return seed.count
+    }
+
+}