diff --git a/docs/jlearch/execution-state-changes.md b/docs/jlearch/execution-state-changes.md new file mode 100644 index 0000000000..b3a88d3984 --- /dev/null +++ b/docs/jlearch/execution-state-changes.md @@ -0,0 +1,41 @@ +# Changes in Execution State + +```mermaid +classDiagram + class StateAnalyticsProperties{ + +int depth + +int visitedAfterLastFork + +int visitedBeforeLastFork + +int stmtsSinceLastCovered + +ExecutionState? parent + +long executingTime + +double reward + +List~Double~ features + -boolean isFork + -boolean isVisitedNew + -int successorDepth + -int successorVisitedAfterLastFork + -int successorVisitedBeforeLastFork + -int successorStmtSinceLastCovered + + +updateIsVisitedNew() + +updateIsFork() + } + ExecutionState o-- StateAnalyticsProperties +``` + +`StateAnalyticsProperties` maintains properties of `ExecutionState`, which don't need for symbolic execution, but need for `JLearch`. + +* `depth: Int` - number of forks on the state's path excluded current state, if it is fork. In this case, fork is a state with more than one successor excluded implicit `NPE` branches. +* `visitedAfterLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path after the last fork in first time. +* `visitedBeforeLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path before the last fork in first time. +* `stmtsSinceLastCovered: Int` - number of `states` on this state's path after the last state that visited any `stmt` in first time. +* `parent: ExecutionState?` - parent of current `state`. If `UtSettings.featureProcess == false`, then it is always null, because we don't need this field in this case. If it is not null, then we can't delete `state` until all successors of this state will be deleted, which may cause memory issue. +* `executingTime: Long` - amount of time, during which this state was traversed. +* `reward: Double?` - calculated reward of this state +* `features: List` - list of extracted features for this state + +Field with `successor` prefix is used for a constructor of successor properties. + +* `updateIsFork()` - set `isFork` on true. This method is called when traversing of `stmt` produces more than one explicit state. Now it may be during the traversing of `IfStmt`, `SwitchStmt`, `AssignStmt` or `InvokeStmt`. +* `updateIsCoveredNew()` - set `isVisitedNew` on true, set `stmtsSinceLastCovered` on zero and increase `visitedAfterLastFork` on 1. This method is called in `UtBotSymbolicEngine` after new state `s` is polled and `s.stmt` was not visited yet. diff --git a/docs/jlearch/features.md b/docs/jlearch/features.md new file mode 100644 index 0000000000..1b3ca05a00 --- /dev/null +++ b/docs/jlearch/features.md @@ -0,0 +1,17 @@ +# Collecting features + +Now we collect 13 features, that will be described in original [paper](https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf), except constraint representation, but it can be extended. + +* `stack` - size of state’s current call stack. +* `successor` - number of successors of state’s current basic block. +* `testCase` - number of test cases generated so far +* `coverageByBranch` - number of instructions, which was covered first time on our last branch +* `coverageByPath` - number of instructions, which was covered first time on our path +* `depth` - number of forks already performed along state’s path. +* `cpicnt` - number of instructions visited in state's current function. +* `icnt` - number of times for which st ate’s current instruction has + been visited +* `covNew` - number of instructions executed by st ate since the last + time a new instruction is covered +* `subpath` - number of times for which st ate’s subpaths have been + visited. The length of the subpaths can be 1, 2, 4, or 8 respectively diff --git a/docs/jlearch/jlearch-architecture.md b/docs/jlearch/jlearch-architecture.md new file mode 100644 index 0000000000..2263c4013d --- /dev/null +++ b/docs/jlearch/jlearch-architecture.md @@ -0,0 +1,152 @@ +# JLearch architecture + +# Global Class Diagram + +```mermaid +classDiagram + class FeatureProcessor{ + dumpFeatures() + } + <> FeatureProcessor + class TraverseGraphStatistics{ + onVisit(ExecutionState) + onTraversed(ExecutionState) + } + class InterproceduralUnitGraph + class FeatureExtractorFactory + <> FeatureExtractorFactory + class FeatureProcessorFactory + <> FeatureProcessorFactory + class EngineAnalyticsContext + class UtBotSymbolicEngine + class NNRewardGuidedSelectorFactory + <> NNRewardGuidedSelectorFactory + class FeatureExtractor{ + extractFeatures(ExecutionState) + } + <> FeatureExtractor + + UtBotSymbolicEngine ..> EngineAnalyticsContext + EngineAnalyticsContext o-- FeatureProcessorFactory + EngineAnalyticsContext o-- FeatureExtractorFactory + EngineAnalyticsContext o-- NNRewardGuidedSelectorFactory + + FeatureProcessor --|> TraverseGraphStatistics + InterproceduralUnitGraph o-- TraverseGraphStatistics + UtBotSymbolicEngine *-- FeatureProcessor + UtBotSymbolicEngine *-- InterproceduralUnitGraph + + class Predictors + class NNStateRewardPredictor + class NNRewardGuidedSelector + + + class GreedySearch + + class BasePathSelector + + GreedySearch --|> BasePathSelector + NNRewardGuidedSelector --|> GreedySearch + + UtBotSymbolicEngine *-- BasePathSelector + + Predictors o-- NNStateRewardPredictor + NNRewardGuidedSelector ..> Predictors + NNRewardGuidedSelector *-- FeatureExtractor + + NNStateRewardPredictorSmile --|> NNStateRewardPredictor + NNStateRewardPredictorTorch --|> NNStateRewardPredictor + + NNStateRewardGuidedSelectorWithRecalculationWeight --|> NNRewardGuidedSelector + NNStateRewardGuidedSelectorWithoutRecalculationWeight --|> NNRewardGuidedSelector +``` + +This diagram doesn't illustrate some details, so read them below. + +# FeatureProcessor + +It is interface in framework-module, that allows to use implementation from analytics module. + +* `dumpFeatures(state: ExecutionState)` - dump features and rewards in some format on disk. Called at the end of traverse in `UtBotSymbolicEngine` + +## Implementation class diagram + +```mermaid +classDiagram + class FeatureProcessorWithStatesRepetition{ + -Map~Int, FeatureList~ dumpedStates + -Set~Stmt~ visitedStmts + -List~TestCase~ testCases + -int generatedTestCases + dumpFeatures() + } + + class FeatureExtractor{ + extractFeatures(ExecutionState) + } + + class TraverseGraphStatistics{ + onVisit(ExecutionState) + onTraversed(ExecutionState) + } + + class RewardEstimator{ + calculateRewards(List~TestCase~) + } + + class TestCase{ + +List states + +int newCoverage + +int testIndex + } + + FeatureProcessorWithStatesRepetition --|> TraverseGraphStatistics + FeatureProcessorWithStatesRepetition o-- FeatureExtractor + FeatureProcessorWithStatesRepetition o-- RewardEstimator + FeatureProcessorWithStatesRepetition ..> EngineAnalyticsContext + +``` + +`State = Pair` + +`FeatureList = List): Map` - calculates `coverage` for each state and `time` for each state. `Coverage` - sum of `newCoverage` by `TestCase` that contains its state. `Time` - sum of `state.executingTime` by all states, that has this state on its path. Then calculates `reward(coverage, time)`. + +## FeatureProcessorWithStatesRepetition + +* `onVisit(state: ExecutionState)` - extractFeatures for state +* `onTraversed(state: ExecutionState)` - create `TestCase`, so we go from `state` to `state.parent` while it is not root, for each `state` on path add its features to `dumpedStates`, calculate coverage of its `TestCase`, increment `generatedTestCases` on 1 and add new `TestCase` in `testCases`. +* `dumpFeatures()` - call `RewardEstimator.calculateRewards()` and write `csv` file for each `TestCase` in format: `newCov,features,reward` for each `state` in it. `newCov` - flag that indicates whether this `TestCase` cover something new or not. So in this approach, each `state` will be written as many times as the number of `TestCase` that has it. +For creating `FeatureExtractor`, it uses `FeatureExtractorFactory` from `EngineAnalyticsContext`. + +# FeatureExtractor + +It is interface in framework-module, that allows to use implementation from analytics module. +* `extractFeatures(state: ExecutionState)` - create features list for state and store it in `state.features`. Now we extract all features, which were described in [paper](https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf). In feature, we can extend the feature list by other features, for example, NeuroSMT. + +# NNStateRewardPredictor + +Interface for reward predictors. Now it has two implementations in `analytics` module: + +* `NNStateRewardPredictorSmile`: it uses our own format to store feedforward neural network, and it uses `Smile` library to do multiplication of matrix. +* `NNStateRewardPredictorTorch`: it assumed that a model is any type of model in `pt` format. It uses the `Deep Java library` to use such models. + +It should be created at the beginning of work and stored at `Predictors` class to be used in `NNRewardGuidedSelector` from the `framework` module. + + +# NNStateRewardGuidedSelector + +It uses an `EngineAnalyticsContext` to create `FeatureExtractor`. +We override `ExecutionState.weight` as `NNStateRewardPredictor.predict(this.features)`. +We have two different implementantions: +* `NNStateRewardGuidedSelectorWithRecalculation`: we recalculate reward every time, so in `ExecutionState.weight` we extract features and call predict. +* `NNStateRewardGuidedSlectorWithoutRecalculation`: we extract features in `offerImpl`, calculate `reward` and store it in `ExecutionState.reward` without recalculation it every time. + +# EngineAnalyticsContext + +It is an object that should be filled by factories in the beginning of work to allow objects from the `framework` module using objects from `analytics` module. diff --git a/docs/jlearch/new-heuristical-path-selectors.md b/docs/jlearch/new-heuristical-path-selectors.md new file mode 100644 index 0000000000..756abb2c0e --- /dev/null +++ b/docs/jlearch/new-heuristical-path-selectors.md @@ -0,0 +1,93 @@ +# GreedySearch + +```mermaid +classDiagram + class GreedySearch{ + -Set~ExecutionState~ states + +ExecutionState.weight + } + GreedySearch --|> BasePathSelector +``` +Base methods such as `offer` or `remove` is implemented pretty simple and just a delegation to `states`. + +In `peekImpl` we find the set of `states` with maximum `weight` and peek random among them, so to use this class in implementation of some `pathSelector`, you just need to override an `ExecutionState.weight`. + +# SubpathStatistics + +```mermaid +classDiagram + class SubpathStatistics{ + +int index + -Map~Subpath, Int~ subpathCount + subpathCount(ExecutionState) + } + class TraverseGraphStatistics{ + onVisit(ExecutionState) + } + + SubpathStatistics --|> TraverseGraphStatistics + TraverseGraphStatistics o-- InterProceduralUnitGraph +``` +`Subpath` = `List` + +This class maintains frequency of each subpath with length `2^index`, which is presented as `List`, in a certain instance of `InterproceduralUnitGraph` + +* `onVisit(state: ExecutionState)` - we calculate subpath of this state and increment its frequency on `1` +* `subpathCount(state: ExecutionState)` - we calculate subpath of this state and return its frequency + +# SubpathGuidedSelector + +```mermaid +classDiagram + SubpathGuidedSelector o-- SubpathStatistics + SubpathGuidedSelector --|> GreedySearch +``` + +Inspired by [paper](http://pxzhang.cn/paper/concolic_testing/oopsla13-pgse.pdf). + +We override `ExecutionState.weight` as `-StatementStatistics.subpathCount(this)`, so we pick the `state`, which `subpath` is less traveled. + +# StatementStatistics + +```mermaid +classDiagram + class StatementStatistics{ + -Map~Stmt, Int~ statementsCount + -Map~SootMethod, Int~ statementsInMethodCount + +statementCount(ExecutionState) + +statementsInMethodCount(ExecutionState) + } + + class TraverseGraphStatistics{ + onVisit(ExecutionState) + } + + StatementStatistics --|> TraverseGraphStatistics + TraverseGraphStatistics o-- InterProceduralUnitGraph +``` + +This class maintains frequency of each `Stmt` and number of `Stmt`, that was visited in some `SootMethod`, on a certain instance of `InterproceduralUnitGraph`. + +* `onVisit(state: ExecutionState)` - increment frequency of state's `stmt` on 1. If we visit this `stmt` for the first time, then increment number of `Stmt`, that we visit in the current state's `method`, on 1. +* `statementCount(state: ExecutionState)` - get a frequency of state's `stmt` +* `statementsInMethodCount(state: ExecutionState)` - get number of `stmt`, that was visited in the current state's `method`. + +# CPInstSelector + +```mermaid +classDiagram + CPInstSelector o-- StatementStatistics + CPInstSelector --|> NonUniformRandomSearch +``` + +Override `ExecutionState.cost` as `StatementStatistics.statementInMethodCount(this)`, so we are more likely to explore the least explored `method`. + +# ForkDepthSelector + +```mermaid +classDiagram + ForkDepthSelector --|> NonUniformRandomSearch +``` + +Override `ExecutionState.cost` as `ExecutionState.depth`, so we are more likely to explore the least deep `state` in terms of the number of forks on its path. + diff --git a/gradle.properties b/gradle.properties index 8c7c2ceb21..749730f691 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,4 +35,8 @@ eclipse_aether_version=1.1.0 maven_wagon_version=3.5.1 maven_plugin_api_version=3.8.5 maven_plugin_tools_version=3.6.4 +javacpp_version=1.5.3 +jsoup_version=1.7.2 +djl_api_version=0.17.0 +pytorch_native_version=1.9.1 # soot also depends on asm, so there could be two different versions \ No newline at end of file diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index fd34a6eb97..26c56fdcee 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -12,6 +12,7 @@ String classifier = osName + "-x86_64" evaluationDependsOn(':utbot-framework') compileTestJava.dependsOn tasks.getByPath(':utbot-framework:testClasses') + dependencies { implementation(project(":utbot-framework")) implementation(project(':utbot-instrumentation')) @@ -29,6 +30,7 @@ dependencies { implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" implementation group: 'tech.tablesaw', name: 'tablesaw-core', version: '0.38.2' implementation group: 'tech.tablesaw', name: 'tablesaw-jsplot', version: '0.38.2' @@ -37,13 +39,19 @@ dependencies { implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.22.1' + implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version + + implementation "ai.djl:api:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" + testCompile project(':utbot-framework').sourceSets.test.output } test { - useJUnitPlatform{ - excludeTags 'Summary' + useJUnitPlatform { + excludeTags 'Summary' } } @@ -54,4 +62,17 @@ processResources { into "models" } } +} + +jar { + dependsOn classes + manifest { + attributes 'Main-Class': 'org.utbot.QualityAnalysisKt' + } + + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorFactoryImpl.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorFactoryImpl.kt new file mode 100644 index 0000000000..9def22a3dc --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorFactoryImpl.kt @@ -0,0 +1,12 @@ +package org.utbot.features + +import org.utbot.analytics.FeatureExtractor +import org.utbot.analytics.FeatureExtractorFactory +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Implementation of feature extractor factory + */ +class FeatureExtractorFactoryImpl : FeatureExtractorFactory { + override operator fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor = FeatureExtractorImpl(graph) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt new file mode 100644 index 0000000000..6786c9847a --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt @@ -0,0 +1,50 @@ +package org.utbot.features + +import org.utbot.analytics.FeatureExtractor +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.StatementsStatistics +import org.utbot.engine.selectors.strategies.SubpathStatistics + +/** + * Implementation of feature extractor. + * Extract features for state and stores it in features vector of this state. + * + * @param graph execution graph of current symbolic traverse + */ +class FeatureExtractorImpl(private val graph: InterProceduralUnitGraph) : FeatureExtractor { + companion object { + /** + * Indexes for [SubpathStatistics], with which we want to collect our features + */ + private val subpathGuidedSelectorIndexes = listOf(0, 1, 2, 3) + } + + private fun MutableList.add(value: T) = add(value.toDouble()) + private fun > MutableList.add(value: T) = add(value.size) + + private val subpathStatistics = subpathGuidedSelectorIndexes.map { SubpathStatistics(graph, it) } + private val statementStatistics = StatementsStatistics(graph) + + override fun extractFeatures(executionState: ExecutionState, generatedTestCases: Int) { + with(executionState.features) { + if (isNotEmpty()) { + clear() + } + + add(executionState.executionStack) // stack + add(graph.succs(executionState.stmt)) // successor + add(generatedTestCases) // testCase + add(executionState.visitedAfterLastFork) // coverage by branch + add(executionState.visitedBeforeLastFork + executionState.visitedAfterLastFork) // coverage by path + add(executionState.depth) // depth + add(statementStatistics.statementInMethodCount(executionState)) // cpicnt + add(statementStatistics.statementCount(executionState)) // icnt + add(executionState.stmtsSinceLastCovered) // covNew + + subpathStatistics.forEach { + add(it.subpathCount(executionState)) // sgs_i + } + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt new file mode 100644 index 0000000000..3515db7527 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt @@ -0,0 +1,164 @@ +package org.utbot.features + +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.FeatureProcessor +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.framework.UtSettings +import soot.jimple.Stmt +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Paths +import kotlin.math.pow + +/** + * Implementation of feature processor, in which we dump each test, so there will be several copies of each state. + * Goal is make weighted dataset, where more value for states, which generated more tests. + * Extract features for state when this state will be marked visited in graph. + * Add test case, when last state of it will be traversed. + * + * @param graph execution graph of current symbolic traverse + * @param saveDir directory in which we will store features and rewards of [ExecutionState] + */ +class FeatureProcessorWithStatesRepetition( + graph: InterProceduralUnitGraph, + private val saveDir: String = UtSettings.featurePath +) : FeatureProcessor(graph) { + init { + File(saveDir).mkdirs() + } + + companion object { + private const val featureFile = "jlearch_features" + private val featureKeys = Companion::class.java.classLoader.getResourceAsStream(featureFile) + ?.bufferedReader().use { + it?.readText()?.split(System.lineSeparator()) ?: emptyList() + } + } + + private var generatedTestCases = 0 + private val featureExtractor = EngineAnalyticsContext.featureExtractorFactory(graph) + private val rewardEstimator = RewardEstimator() + + private val dumpedStates = mutableMapOf>() + private val visitedStmts = mutableSetOf() + private val testCases = mutableListOf() + + private fun extractFeatures(executionState: ExecutionState) { + featureExtractor.extractFeatures(executionState, generatedTestCases) + } + + private fun addTestCase(executionState: ExecutionState) { + val states = mutableListOf>() + var newCoverage = 0 + + generateSequence(executionState) { currentState -> + val stateHashCode = currentState.hashCode() + + if (currentState.features.isEmpty()) { + extractFeatures(currentState) + } + + states += stateHashCode to currentState.executingTime + dumpedStates[stateHashCode] = currentState.features + + currentState.stmt.let { + if (it !in visitedStmts && !currentState.isInNestedMethod()) { + visitedStmts += it + newCoverage++ + } + } + + currentState.parent + } + + generatedTestCases++ + testCases += TestCase(states, newCoverage, generatedTestCases) + } + + override fun dumpFeatures() { + val rewards = rewardEstimator.calculateRewards(testCases) + + testCases.forEach { ts -> + val outputFile = Paths.get(saveDir, "${UtSettings.testCounter++}.csv").toFile() + FileOutputStream(outputFile, true) + .bufferedWriter() + .use { out -> + out.appendLine("newCov,reward,${featureKeys.joinToString(separator = ",")}") + val reversedStates = ts.states.asReversed() + + reversedStates.forEach { (state, _) -> + val isCoveredNew = ts.newCoverage != 0 + val reward = rewards[state] + val features = dumpedStates[state]?.joinToString(separator = ",") + + out.appendLine("$isCoveredNew,$reward,$features") + } + + out.flush() + } + } + } + + override fun onTraversed(executionState: ExecutionState) { + addTestCase(executionState) + } + + override fun onVisit(executionState: ExecutionState) { + extractFeatures(executionState) + } +} + +internal class RewardEstimator { + + fun calculateRewards(testCases: List): Map { + val rewards = mutableMapOf() + val coverages = mutableMapOf() + val stateToExecutingTime = mutableMapOf() + + testCases.forEach { ts -> + var allTime = 0L + ts.states.forEach { (stateHash, time) -> + coverages.compute(stateHash) { _, v -> + ts.newCoverage + (v ?: 0) + } + val isNewState = stateHash !in stateToExecutingTime + stateToExecutingTime.compute(stateHash) { _, v -> + allTime + (v ?: time) + } + if (isNewState) { + allTime += time + } + } + } + + coverages.forEach { (state, coverage) -> + rewards[state] = reward(coverage.toDouble(), stateToExecutingTime.getValue(state).toDouble()) + } + + return rewards + } + + companion object { + /** + * Threshold for time: executingTime less than that we don't distinct. We are not expiremented with changing it yet, + * now it is just minimal positive value distinct from 0. + */ + private const val minTime = 1.0 + + /** + * Just degree of reward to make it smaller if it more than 1 and bigger if it less than 1. + */ + private const val rewardDegree = 0.5 + + fun reward(coverage: Double, time: Double): Double = (coverage / maxOf(time, minTime)).pow(rewardDegree) + } +} + +/** + * Class that represents test case. + * @param states pairs from stateHash and executingTime, created from each state of this test case + * @param newCoverage number of instructions, that was visited in first time by [states] + * @param testIndex number of test case, that was created before + */ +data class TestCase(val states: List>, val newCoverage: Int, val testIndex: Int) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt new file mode 100644 index 0000000000..51744e6918 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt @@ -0,0 +1,13 @@ +package org.utbot.features + +import org.utbot.analytics.FeatureProcessor +import org.utbot.analytics.FeatureProcessorFactory +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Implementation of feature processor factory, which creates FeatureProcessorWithStatesRepetition. + * See [FeatureProcessorWithStatesRepetition]. + */ +class FeatureProcessorWithStatesRepetitionFactory : FeatureProcessorFactory { + override fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor = FeatureProcessorWithStatesRepetition(graph) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/FeedForwardNetwork.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/FeedForwardNetwork.kt new file mode 100644 index 0000000000..d392038bea --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/FeedForwardNetwork.kt @@ -0,0 +1,36 @@ +package org.utbot.predictors + +import smile.math.matrix.Matrix +import kotlin.math.max + +private object ActivationFunctions { + const val ReLU = "reLU" +} + +data class FeedForwardNetwork(val operations: List<(DoubleArray) -> DoubleArray>) + +private fun reLU(input: DoubleArray): DoubleArray { + return input.map { max(0.0, it) }.toDoubleArray() +} + +internal fun buildModel(nnJson: NNJson): FeedForwardNetwork { + val weights = nnJson.linearLayers.map { Matrix(it) } + val biases = nnJson.biases.map { Matrix(it) } + val operations = mutableListOf<(DoubleArray) -> DoubleArray>() + + nnJson.linearLayers.indices.forEach { i -> + operations.add { + weights[i].mm(Matrix(it)).add(biases[i]).col(0) + } + if (i != nnJson.linearLayers.lastIndex) { + operations.add { + when (nnJson.activationLayers[i]) { + ActivationFunctions.ReLU -> reLU(it) + else -> error("Unsupported activation") + } + } + } + } + + return FeedForwardNetwork(operations) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt new file mode 100644 index 0000000000..9898cadd63 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -0,0 +1,54 @@ +package org.utbot.predictors + +import mu.KotlinLogging +import org.utbot.analytics.UtBotAbstractPredictor +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings +import smile.math.matrix.Matrix +import java.io.File + +private const val DEFAULT_WEIGHT_PATH = "linear.txt" + +private val logger = KotlinLogging.logger {} + +/** + * Last weight is bias + */ +private fun loadWeights(path: String): Matrix { + val weightsFile = File("${UtSettings.rewardModelPath}/${path}") + + if (!weightsFile.exists()) { + error("There is no file with weights with path: ${weightsFile.absolutePath}") + } + + val weightsArray = weightsFile.readText().splitByCommaIntoDoubleArray() + + return Matrix(weightsArray) +} + +class LinearStateRewardPredictor(weightsPath: String = DEFAULT_WEIGHT_PATH) : + UtBotAbstractPredictor>, List> { + private lateinit var weights: Matrix + + init { + try { + weights = loadWeights(weightsPath) + } catch (e: Exception) { + logger.info(e) { + "Error while initialization of LinearStateRewardPredictor. Changing pathSelectorType on INHERITORS_SELECTOR" + } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + } + } + + override fun predict(input: List>): List { + // add 1 to each feature vector + val matrixValues = input + .map { (it + 1.0).toDoubleArray() } + .toTypedArray() + + val X = Matrix(matrixValues) + + return X.mm(weights).col(0).toList() + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt new file mode 100644 index 0000000000..925bcd3751 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt @@ -0,0 +1,42 @@ +package org.utbot.predictors + +import com.google.gson.Gson +import org.utbot.framework.UtSettings +import java.io.FileReader +import java.nio.file.Paths + +data class NNJson( + val linearLayers: Array> = emptyArray(), + val activationLayers: Array = emptyArray(), + val biases: Array = emptyArray() +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as NNJson + + if (!linearLayers.contentDeepEquals(other.linearLayers)) return false + if (!activationLayers.contentEquals(other.activationLayers)) return false + if (!biases.contentDeepEquals(other.biases)) return false + + return true + } + + override fun hashCode(): Int { + var result = linearLayers.contentDeepHashCode() + result = 31 * result + activationLayers.contentHashCode() + result = 31 * result + biases.contentDeepHashCode() + return result + } +} + +internal fun loadModel(path: String): NNJson { + val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() + val nnJson: NNJson = + Gson().fromJson(FileReader(modelFile), NNJson::class.java) ?: run { + error("Empty model") + } + + return nnJson +} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt new file mode 100644 index 0000000000..06a3ebe04d --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt @@ -0,0 +1,8 @@ +package org.utbot.predictors + +import org.utbot.analytics.UtBotAbstractPredictor + +/** + * Interface, which should predict reward for state by features list. + */ +interface NNStateRewardPredictor : UtBotAbstractPredictor, Double> \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt new file mode 100644 index 0000000000..1d9d5dac6d --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt @@ -0,0 +1,43 @@ +package org.utbot.predictors + +import mu.KotlinLogging +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings +import smile.math.matrix.Matrix + +private const val DEFAULT_MODEL_PATH = "nn.json" +private const val DEFAULT_SCALER_PATH = "scaler.txt" + +private val logger = KotlinLogging.logger {} + +private fun getModel(path: String) = buildModel(loadModel(path)) + +class NNStateRewardPredictorBase(modelPath: String = DEFAULT_MODEL_PATH, scalerPath: String = DEFAULT_SCALER_PATH) : + NNStateRewardPredictor { + private lateinit var nn: FeedForwardNetwork + private lateinit var scaler: StandardScaler + + init { + try { + nn = getModel(modelPath) + scaler = loadScaler(scalerPath) + } catch (e: Exception) { + logger.info(e) { + "Error while initialization of NNStateRewardPredictorBase. Changing pathSelectorType on INHERITORS_SELECTOR" + } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + } + } + + override fun predict(input: List): Double { + var inputArray = input.toDoubleArray() + inputArray = Matrix(inputArray).sub(scaler.mean).div(scaler.variance).col(0) + + nn.operations.forEach { + inputArray = it(inputArray) + } + + check(inputArray.size == 1) { "Neural network have several outputs" } + return inputArray[0] + } +} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt new file mode 100644 index 0000000000..b92eb856c6 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt @@ -0,0 +1,37 @@ +package org.utbot.predictors + +import ai.djl.Model +import ai.djl.inference.Predictor +import ai.djl.ndarray.NDArray +import ai.djl.ndarray.NDList +import ai.djl.translate.Translator +import ai.djl.translate.TranslatorContext +import org.utbot.framework.UtSettings +import java.io.Closeable +import java.nio.file.Paths + +class NNStateRewardPredictorTorch : NNStateRewardPredictor, Closeable { + val model: Model = Model.newInstance("model") + + init { + model.load(Paths.get(UtSettings.rewardModelPath, "model.pt1")) + } + + private val predictor: Predictor, Float> = model.newPredictor(object : Translator, Float> { + override fun processInput(ctx: TranslatorContext, input: List): NDList { + val array: NDArray = ctx.ndManager.create(input.toFloatArray()) + return NDList(array) + } + + override fun processOutput(ctx: TranslatorContext, list: NDList): Float = list[0].getFloat() + }) + + override fun predict(input: List): Double { + val reward: Float = predictor.predict(input.map { it.toFloat() }.toList()) + return reward.toDouble() + } + + override fun close() { + predictor.close() + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt new file mode 100644 index 0000000000..c50ffdc7c1 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt @@ -0,0 +1,14 @@ +package org.utbot.predictors + +import org.utbot.framework.UtSettings +import smile.math.matrix.Matrix +import java.nio.file.Paths + +data class StandardScaler(val mean: Matrix?, val variance: Matrix?) + +internal fun loadScaler(path: String): StandardScaler = + Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { + val mean = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not mean in $path") + val variance = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not variance in $path") + StandardScaler(Matrix(mean), Matrix(variance)) + } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt new file mode 100644 index 0000000000..f0f0664642 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt @@ -0,0 +1,8 @@ +package org.utbot.predictors + +fun String.splitByCommaIntoDoubleArray() = + try { + split(',').map(String::toDouble).toDoubleArray() + } catch (e: NumberFormatException) { + error("Wrong format in $this, expect doubles separated by commas") + } \ No newline at end of file diff --git a/utbot-analytics/src/main/resources/jlearch_features b/utbot-analytics/src/main/resources/jlearch_features new file mode 100644 index 0000000000..0a852928bf --- /dev/null +++ b/utbot-analytics/src/main/resources/jlearch_features @@ -0,0 +1,13 @@ +stack +successor +testCase +coverageByBranch +coverageByPath +depth +cpicnt +icnt +covNew +subpath1 +subpath2 +subpath4 +subpath8 \ No newline at end of file diff --git a/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt new file mode 100644 index 0000000000..e847d7a057 --- /dev/null +++ b/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt @@ -0,0 +1,73 @@ +package org.utbot.features + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class FeatureProcessorWithRepetitionTest { + companion object { + fun reward(coverage: Double, time: Double) = RewardEstimator.reward(coverage, time) + } + + @Test + fun testDumpFeatures() { + val statesToInt = mapOf( + "a0" to 1, + "c0" to 2, + "f0" to 3, + "g0" to 4, + "c1" to 5, + "f1" to 6, + "g1" to 7, + "b0" to 8, + "d0" to 9 + ) + + val expectedRewards: Map = mapOf( + "a0" to reward(6.0, 15.0), + "c0" to reward(4.0, 10.0), + "f0" to reward(4.0, 8.0), + "g0" to reward(4.0, 2.0), + "c1" to reward(0.0, 4.0), + "f1" to reward(0.0, 3.0), + "g1" to reward(0.0, 2.0), + "b0" to reward(2.0, 4.0), + "d0" to reward(2.0, 2.0) + ) + + val rewardEstimator = RewardEstimator() + val testCases = listOf( + TestCase( + listOf( + statesToInt["g0"]!! to 2L, + statesToInt["f0"]!! to 2L, + statesToInt["c0"]!! to 2L, + statesToInt["a0"]!! to 1L + ), + newCoverage = 4, testIndex = 0 + ), + TestCase( + listOf( + statesToInt["g1"]!! to 2L, + statesToInt["f1"]!! to 1L, + statesToInt["c1"]!! to 1L, + statesToInt["f0"]!! to 2L, + statesToInt["c0"]!! to 2L, + statesToInt["a0"]!! to 1L + ), + newCoverage = 0, + testIndex = 1 + ), + TestCase( + listOf(statesToInt["d0"]!! to 2L, statesToInt["b0"]!! to 2L, statesToInt["a0"]!! to 1L), + newCoverage = 2, + testIndex = 2 + ) + ) + + val rewards = rewardEstimator.calculateRewards(testCases) + Assertions.assertEquals(9, rewards.size) + expectedRewards.forEach { + Assertions.assertEquals(it.value, rewards[statesToInt[it.key]]) + } + } +} diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt new file mode 100644 index 0000000000..1de5c0cb75 --- /dev/null +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -0,0 +1,34 @@ +package org.utbot.predictors + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.utbot.examples.withPathSelectorType +import org.utbot.examples.withRewardModelPath +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings + +class LinearStateRewardPredictorTest { + @Test + fun simpleTest() { + withRewardModelPath("src/test/resources") { + val pred = LinearStateRewardPredictor() + + val features = listOf( + listOf(2.0, 3.0), + listOf(2.0, 3.0) + ) + + assertEquals(listOf(6.0, 6.0), pred.predict(features)) + } + } + + @Test + fun wrongFormatTest() { + withRewardModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + LinearStateRewardPredictor("wrong_format_linear.txt") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) + } + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt new file mode 100644 index 0000000000..510660d7ca --- /dev/null +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -0,0 +1,85 @@ +package org.utbot.predictors + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test +import org.utbot.examples.withPathSelectorType +import org.utbot.examples.withRewardModelPath +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings +import kotlin.system.measureNanoTime + +class NNStateRewardPredictorTest { + @Test + fun simpleTest() { + withRewardModelPath("src/test/resources") { + val pred = NNStateRewardPredictorBase() + + val features = listOf(0.0, 0.0) + + assertEquals(5.0, pred.predict(features)) + } + } + + @Disabled("Just to see the performance of predictors") + @Test + fun performanceTest() { + val features = (1..13).map { 1.0 }.toList() + withRewardModelPath("models\\test\\0") { + val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorBase, 100, features) + println(averageTime) + } + + + withRewardModelPath("models") { + val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorTorch, 100, features) + println(averageTime) + } + } + + private fun calcAverageTimeForModelPredict( + model: () -> NNStateRewardPredictor, + iterations: Int, + features: List + ): Double { + val pred = model() + + (1..iterations).map { + pred.predict(features) + } + + return (1..iterations) + .map { measureNanoTime { pred.predict(features) } } + .average() + } + + @Test + fun corruptedModelFileTest() { + withRewardModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + NNStateRewardPredictorBase(modelPath = "corrupted_nn.json") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) + } + } + } + + @Test + fun emptyModelFileTest() { + withRewardModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + NNStateRewardPredictorBase(modelPath = "empty_nn.json") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) + } + } + } + + @Test + fun corruptedScalerTest() { + withRewardModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + NNStateRewardPredictorBase(scalerPath = "corrupted_scaler.txt") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) + } + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/corrupted_nn.json b/utbot-analytics/src/test/resources/corrupted_nn.json new file mode 100644 index 0000000000..ffc3ef7976 --- /dev/null +++ b/utbot-analytics/src/test/resources/corrupted_nn.json @@ -0,0 +1,25 @@ +{ + dsfds + "linearLayers": [ + [ + [1, 0], + [0, 1] + ], + [ + [1, 0], + [0, 1] + ], + [ + [1, 1] + ] + ], + "activationLayers": [ + "reLU", + "reLU" + ], + "biases": [ + [1, 1], + [1, 1], + [1] + ] +} \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/corrupted_scaler.txt b/utbot-analytics/src/test/resources/corrupted_scaler.txt new file mode 100644 index 0000000000..42ff95f905 --- /dev/null +++ b/utbot-analytics/src/test/resources/corrupted_scaler.txt @@ -0,0 +1 @@ +1,asada \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/empty_nn.json b/utbot-analytics/src/test/resources/empty_nn.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utbot-analytics/src/test/resources/linear.txt b/utbot-analytics/src/test/resources/linear.txt new file mode 100644 index 0000000000..8a6627b46a --- /dev/null +++ b/utbot-analytics/src/test/resources/linear.txt @@ -0,0 +1 @@ +1,1,1 \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/nn.json b/utbot-analytics/src/test/resources/nn.json new file mode 100644 index 0000000000..945b53aef8 --- /dev/null +++ b/utbot-analytics/src/test/resources/nn.json @@ -0,0 +1,24 @@ +{ + "linearLayers": [ + [ + [1, 0], + [0, 1] + ], + [ + [1, 0], + [0, 1] + ], + [ + [1, 1] + ] + ], + "activationLayers": [ + "reLU", + "reLU" + ], + "biases": [ + [1, 1], + [1, 1], + [1] + ] +} \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/scaler.txt b/utbot-analytics/src/test/resources/scaler.txt new file mode 100644 index 0000000000..b241d25f33 --- /dev/null +++ b/utbot-analytics/src/test/resources/scaler.txt @@ -0,0 +1,2 @@ +0,0 +1,1 diff --git a/utbot-analytics/src/test/resources/wrong_format_linear.txt b/utbot-analytics/src/test/resources/wrong_format_linear.txt new file mode 100644 index 0000000000..bf4d5a1b19 --- /dev/null +++ b/utbot-analytics/src/test/resources/wrong_format_linear.txt @@ -0,0 +1 @@ +1,asdasdsa,asdasd \ No newline at end of file diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 39da879aa6..76d1ed0c97 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -1,12 +1,12 @@ package org.utbot.framework +import mu.KotlinLogging import org.utbot.common.PathUtil.toPath import java.io.FileInputStream import java.io.IOException import java.util.Properties import kotlin.properties.PropertyDelegateProvider import kotlin.reflect.KProperty -import mu.KotlinLogging private val logger = KotlinLogging.logger {} @@ -68,7 +68,8 @@ object UtSettings { private fun getIntProperty(defaultValue: Int) = getProperty(defaultValue, String::toInt) private fun getLongProperty(defaultValue: Long) = getProperty(defaultValue, String::toLong) private fun getStringProperty(defaultValue: String) = getProperty(defaultValue) { it } - private inline fun > getEnumProperty(defaultValue: T) = getProperty(defaultValue) { enumValueOf(it) } + private inline fun > getEnumProperty(defaultValue: T) = + getProperty(defaultValue) { enumValueOf(it) } /** @@ -108,6 +109,11 @@ object UtSettings { */ var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.INHERITORS_SELECTOR) + /** + * Type of nnRewardGuidedSelector + */ + var nnRewardGuidedSelectorType: NNRewardGuidedSelectorType by getEnumProperty(NNRewardGuidedSelectorType.WITHOUT_RECALCULATION) + /** * Steps limit for path selector. */ @@ -152,7 +158,7 @@ object UtSettings { /* * Activate or deactivate tests on comments && names/displayNames * */ - var testSummary by getBooleanProperty( true) + var testSummary by getBooleanProperty(true) var testName by getBooleanProperty(true) var testDisplayName by getBooleanProperty(true) @@ -264,7 +270,9 @@ object UtSettings { /** * Timeout for specific concrete execution (in milliseconds). */ - var concreteExecutionTimeoutInChildProcess: Long by getLongProperty(DEFAULT_CONCRETE_EXECUTION_TIMEOUT_IN_CHILD_PROCESS_MS) + var concreteExecutionTimeoutInChildProcess: Long by getLongProperty( + DEFAULT_CONCRETE_EXECUTION_TIMEOUT_IN_CHILD_PROCESS_MS + ) /** * Number of branch instructions using for clustering executions in the test minimization phase. @@ -286,6 +294,47 @@ object UtSettings { */ var enableUnsatCoreCalculationForHardConstraints by getBooleanProperty(false) + /** + * 2^{this} will be the length of observed subpath. + * See [SubpathGuidedSelector] + */ + var subpathGuidedSelectorIndex by getIntProperty(1) + + /** + * Set of indexes, which will use [SubpathGuidedSelector] in not single mode + */ + var subpathGuidedSelectorIndexes = listOf(0, 1, 2, 3) + + /** + * Flag that indicates whether feature processing for execution states enabled or not + */ + var enableFeatureProcess by getBooleanProperty(false) + + /** + * Path to deserialized reward models + */ + var rewardModelPath by getStringProperty("../models/0") + + /** + * Number of model iterations that will be used during ContestEstimator + */ + var iterations by getIntProperty(1) + + /** + * Path for state features dir + */ + var featurePath by getStringProperty("eval/secondFeatures/antlr/INHERITORS_SELECTOR") + + /** + * Counter for tests during testGeneration for one project in ContestEstimator + */ + var testCounter by getIntProperty(0) + + /** + * Flag for Subpath and NN selectors whether they are combined (Subpath use several indexes, NN use several models) + */ + var singleSelector by getBooleanProperty(true) + override fun toString(): String = properties .entries @@ -293,12 +342,67 @@ object UtSettings { .joinToString(separator = System.lineSeparator()) { "\t${it.key}=${it.value}" } } +/** + * Type of [BasePathSelector]. For each value see class in comment + */ enum class PathSelectorType { + /** + * [CoveredNewSelector] + */ COVERED_NEW_SELECTOR, - INHERITORS_SELECTOR + + /** + * [InheritorsSelector] + */ + INHERITORS_SELECTOR, + + /** + * [SubpathGuidedSelector] + */ + SUBPATH_GUIDED_SELECTOR, + + /** + * [CPInstSelector] + */ + CPI_SELECTOR, + + /** + * [ForkDepthSelector] + */ + FORK_DEPTH_SELECTOR, + + /** + * [NNRewardGuidedSelector] + */ + NN_REWARD_GUIDED_SELECTOR, + + /** + * [RandomSelector] + */ + RANDOM_SELECTOR, + + /** + * [RandomPathSelector] + */ + RANDOM_PATH_SELECTOR } enum class TestSelectionStrategyType { DO_NOT_MINIMIZE_STRATEGY, // Always adds new test COVERAGE_STRATEGY // Adds new test only if it increases coverage } + +/** + * Enum to specify [NNRewardGuidedSelector], see implementations for more details + */ +enum class NNRewardGuidedSelectorType { + /** + * [NNRewardGuidedSelectorWithRecalculation] + */ + WITH_RECALCULATION, + + /** + * [NNRewardGuidedSelectorWithoutRecalculation] + */ + WITHOUT_RECALCULATION +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt new file mode 100644 index 0000000000..0baf0c72cb --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -0,0 +1,30 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.NNRewardGuidedSelectorFactory +import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationFactory +import org.utbot.engine.selectors.NNRewardGuidedSelectorWithoutRecalculationFactory +import org.utbot.framework.NNRewardGuidedSelectorType +import org.utbot.framework.UtSettings + +/** + * Class that stores all objects that need for work analytics module during symbolic execution + */ +object EngineAnalyticsContext { + var featureProcessorFactory: FeatureProcessorFactory = object : FeatureProcessorFactory { + override fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor { + error("Feature processor factory is not provided.") + } + } + + var featureExtractorFactory: FeatureExtractorFactory = object : FeatureExtractorFactory { + override fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor { + error("Feature extractor factory is not provided.") + } + } + + val nnRewardGuidedSelectorFactory: NNRewardGuidedSelectorFactory = when (UtSettings.nnRewardGuidedSelectorType) { + NNRewardGuidedSelectorType.WITHOUT_RECALCULATION -> NNRewardGuidedSelectorWithoutRecalculationFactory() + NNRewardGuidedSelectorType.WITH_RECALCULATION -> NNRewardGuidedSelectorWithRecalculationFactory() + } +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt new file mode 100644 index 0000000000..c86cad1b54 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt @@ -0,0 +1,14 @@ +package org.utbot.analytics + +import org.utbot.engine.ExecutionState + +/** + * Class that encapsulates work with FeatureExtractor during symbolic execution. + */ +interface FeatureExtractor { + /** + * Extract features and store in it [ExecutionState.features] + * @param generatedTestCases number of generated tests so far + */ + fun extractFeatures(executionState: ExecutionState, generatedTestCases: Int) +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt new file mode 100644 index 0000000000..6ad7a916a6 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt @@ -0,0 +1,10 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Class that can create FeatureExtractor. See [FeatureExtractor]. + */ +interface FeatureExtractorFactory { + operator fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt new file mode 100644 index 0000000000..e69e783a97 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt @@ -0,0 +1,14 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.TraverseGraphStatistics + +/** + * Interface that encapsulates work with FeatureProcessor and can only dumpFeatures at the end of symbolic execution + */ +abstract class FeatureProcessor(graph: InterProceduralUnitGraph) : TraverseGraphStatistics(graph) { + /** + * Dump features and rewards of all states in collected test cases. + */ + abstract fun dumpFeatures() +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessorFactory.kt new file mode 100644 index 0000000000..d012aa7e89 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessorFactory.kt @@ -0,0 +1,10 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Class that can create FeatureProcessor. See [FeatureProcessor] + */ +interface FeatureProcessorFactory { + operator fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt index d7876049cb..562d6db7c3 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt @@ -1,7 +1,6 @@ package org.utbot.analytics import org.utbot.engine.pc.UtExpression -import kotlinx.collections.immutable.PersistentList import soot.jimple.Stmt /** @@ -18,9 +17,16 @@ object Predictors { * Predict z3's [Solver.check()] execution time in nanoseconds */ var smt: UtBotNanoTimePredictor> = object : UtBotNanoTimePredictor> {} - var smtIncremental: UtBotNanoTimePredictor = object : UtBotNanoTimePredictor {} + var smtIncremental: UtBotNanoTimePredictor = object : UtBotNanoTimePredictor {} var testName: UtBotAbstractPredictor, String> = object : UtBotAbstractPredictor, String> { override fun predict(input: Iterable): String = "stubName" } + + var stateRewardPredictor: UtBotAbstractPredictor, Double> = + object : UtBotAbstractPredictor, Double> { + override fun predict(input: List): Double { + error("stateRewardPredictor is not provided.") + } + } } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt index 79ab953a89..7e034050e7 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt @@ -1,10 +1,5 @@ package org.utbot.engine -import org.utbot.common.md5 -import org.utbot.engine.pc.UtSolver -import org.utbot.engine.symbolic.SymbolicState -import org.utbot.engine.symbolic.SymbolicStateUpdate -import org.utbot.framework.plugin.api.Step import kotlinx.collections.immutable.PersistentList import kotlinx.collections.immutable.PersistentMap import kotlinx.collections.immutable.PersistentSet @@ -12,9 +7,16 @@ import kotlinx.collections.immutable.persistentHashMapOf import kotlinx.collections.immutable.persistentHashSetOf import kotlinx.collections.immutable.persistentListOf import kotlinx.collections.immutable.plus +import org.utbot.common.md5 +import org.utbot.engine.pc.UtSolver import org.utbot.engine.pc.UtSolverStatusUNDEFINED +import org.utbot.engine.symbolic.SymbolicState +import org.utbot.engine.symbolic.SymbolicStateUpdate +import org.utbot.framework.UtSettings +import org.utbot.framework.plugin.api.Step import soot.SootMethod import soot.jimple.Stmt +import java.util.Objects const val RETURN_DECISION_NUM = -1 const val CALL_DECISION_NUM = -2 @@ -40,6 +42,55 @@ data class ExecutionStackElement( this.copy(localVariableMemory = localVariableMemory.update(memoryUpdate), doesntThrow = doesntThrow) } +/** + * Class that store all information about execution state that needed only for analytics module + */ +data class StateAnalyticsProperties( + /** + * Number of forks already performed along state's path, where fork is statement with multiple successors + */ + val depth: Int = 0, + var visitedAfterLastFork: Int = 0, + var visitedBeforeLastFork: Int = 0, + var stmtsSinceLastCovered: Int = 0, + val parent: ExecutionState? = null, +) { + var executingTime: Long = 0 + var reward: Double? = null + val features: MutableList = mutableListOf() + + /** + * Flag that indicates whether this state is fork or not. Fork here means that we have more than one successor + */ + private var isFork: Boolean = false + + fun definitelyFork() { + isFork = true + } + + private var isVisitedNew: Boolean = false + + fun updateIsVisitedNew() { + isVisitedNew = true + stmtsSinceLastCovered = 0 + visitedAfterLastFork++ + } + + private val successorDepth: Int get() = depth + if (isFork) 1 else 0 + + private val successorVisitedAfterLastFork: Int get() = if (!isFork) visitedAfterLastFork else 0 + private val successorVisitedBeforeLastFork: Int get() = visitedBeforeLastFork + if (isFork) visitedAfterLastFork else 0 + private val successorStmtSinceLastCovered: Int get() = 1 + stmtsSinceLastCovered + + fun successorProperties(parent: ExecutionState) = StateAnalyticsProperties( + successorDepth, + successorVisitedAfterLastFork, + successorVisitedBeforeLastFork, + successorStmtSinceLastCovered, + if (UtSettings.enableFeatureProcess) parent else null + ) +} + /** * [visitedStatementsHashesToCountInPath] is a map representing how many times each instruction from the [path] * has occurred. It is required to calculate priority of the branches and decrease the priority for branches leading @@ -61,12 +112,14 @@ data class ExecutionState( val lastMethod: SootMethod? = null, val methodResult: MethodResult? = null, val exception: SymbolicFailure? = null, - private var outgoingEdges: Int = 0 + private var stateAnalyticsProperties: StateAnalyticsProperties = StateAnalyticsProperties() ) : AutoCloseable { val solver: UtSolver by symbolicState::solver val memory: Memory by symbolicState::memory + private var outgoingEdges = 0 + fun isInNestedMethod() = executionStack.size > 1 val localVariableMemory @@ -107,7 +160,8 @@ data class ExecutionState( pathLength = pathLength + 1, lastEdge = edge, lastMethod = executionStack.last().method, - exception = exception + exception = exception, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -123,14 +177,18 @@ data class ExecutionState( symbolicState = symbolicState, executionStack = executionStack.removeAt(executionStack.lastIndex), path = path + stmt, - visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put(stmtHashcode, stmtCountInPath), + visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put( + stmtHashcode, + stmtCountInPath + ), decisionPath = decisionPath + edge.decisionNum, edges = edges + edge, stmts = stmts.putIfAbsent(stmt, pathLength), pathLength = pathLength + 1, lastEdge = edge, lastMethod = executionStack.last().method, - methodResult + methodResult, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -158,13 +216,17 @@ data class ExecutionState( symbolicState = symbolicState.stateForNestedMethod() + update, executionStack = executionStack + stackElement, path = path + this.stmt, - visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put(stmtHashCode, stmtCountInPath), + visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put( + stmtHashCode, + stmtCountInPath + ), decisionPath = decisionPath + edge.decisionNum, edges = edges + edge, stmts = stmts.putIfAbsent(this.stmt, pathLength), pathLength = pathLength + 1, lastEdge = edge, - lastMethod = stackElement.method + lastMethod = stackElement.method, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -173,7 +235,10 @@ data class ExecutionState( ): ExecutionState { val last = executionStack.last() val stackElement = last.update(stateUpdate.localMemoryUpdates) - return copy(symbolicState = symbolicState + stateUpdate, executionStack = executionStack.set(executionStack.lastIndex, stackElement)) + return copy( + symbolicState = symbolicState + stateUpdate, + executionStack = executionStack.set(executionStack.lastIndex, stackElement) + ) } fun update( @@ -196,13 +261,17 @@ data class ExecutionState( symbolicState = symbolicState + symbolicStateUpdate, executionStack = executionStack.set(executionStack.lastIndex, stackElement), path = path + stmt, - visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put(stmtHashCode, stmtCountInPath), + visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put( + stmtHashCode, + stmtCountInPath + ), decisionPath = decisionPath + edge.decisionNum, edges = edges + edge, stmts = stmts.putIfAbsent(stmt, pathLength), pathLength = pathLength + 1, lastEdge = edge, - lastMethod = stackElement.method + lastMethod = stackElement.method, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -253,12 +322,61 @@ data class ExecutionState( val path = fullPath() val prettifiedPath = path.joinToString(separator = "\n") { (stmt, depth, decision) -> val prefix = when (decision) { - CALL_DECISION_NUM -> "call[${depth}] - " + "".padEnd(2*depth, ' ') - RETURN_DECISION_NUM -> " ret[${depth - 1}] - " + "".padEnd(2*depth, ' ') - else -> " "+"".padEnd(2*depth, ' ') + CALL_DECISION_NUM -> "call[${depth}] - " + "".padEnd(2 * depth, ' ') + RETURN_DECISION_NUM -> " ret[${depth - 1}] - " + "".padEnd(2 * depth, ' ') + else -> " " + "".padEnd(2 * depth, ' ') } "$prefix$stmt" } return " MD5(path)=${prettifiedPath.md5()}\n$prettifiedPath" } + + fun definitelyFork() { + stateAnalyticsProperties.definitelyFork() + } + + fun updateIsVisitedNew() { + stateAnalyticsProperties.updateIsVisitedNew() + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as ExecutionState + + if (stmt != other.stmt) return false + if (symbolicState != other.symbolicState) return false + if (executionStack != other.executionStack) return false + if (path != other.path) return false + if (visitedStatementsHashesToCountInPath != other.visitedStatementsHashesToCountInPath) return false + if (decisionPath != other.decisionPath) return false + if (edges != other.edges) return false + if (stmts != other.stmts) return false + if (pathLength != other.pathLength) return false + if (lastEdge != other.lastEdge) return false + if (lastMethod != other.lastMethod) return false + if (methodResult != other.methodResult) return false + if (exception != other.exception) return false + + return true + } + + private val hashCode by lazy { + Objects.hash( + stmt, symbolicState, executionStack, path, visitedStatementsHashesToCountInPath, decisionPath, + edges, stmts, pathLength, lastEdge, lastMethod, methodResult, exception + ) + } + + override fun hashCode(): Int = hashCode + + var reward by stateAnalyticsProperties::reward + val features by stateAnalyticsProperties::features + var executingTime by stateAnalyticsProperties::executingTime + val depth by stateAnalyticsProperties::depth + var visitedBeforeLastFork by stateAnalyticsProperties::visitedBeforeLastFork + var visitedAfterLastFork by stateAnalyticsProperties::visitedAfterLastFork + var stmtsSinceLastCovered by stateAnalyticsProperties::stmtsSinceLastCovered + val parent by stateAnalyticsProperties::parent } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 8d081900c6..9d203ca991 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -6,9 +6,9 @@ import kotlinx.collections.immutable.persistentSetOf import kotlinx.collections.immutable.toPersistentList import kotlinx.collections.immutable.toPersistentMap import kotlinx.collections.immutable.toPersistentSet +import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Job -import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.FlowCollector import kotlinx.coroutines.flow.flow @@ -17,6 +17,8 @@ import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.isActive import kotlinx.coroutines.yield import mu.KotlinLogging +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.FeatureProcessor import org.utbot.analytics.Predictors import org.utbot.common.WorkaroundReason.HACK import org.utbot.common.WorkaroundReason.REMOVE_ANONYMOUS_CLASSES @@ -80,11 +82,18 @@ import org.utbot.engine.pc.mkOr import org.utbot.engine.pc.select import org.utbot.engine.pc.store import org.utbot.engine.selectors.PathSelector +import org.utbot.engine.selectors.StrategyOption import org.utbot.engine.selectors.coveredNewSelector +import org.utbot.engine.selectors.cpInstSelector +import org.utbot.engine.selectors.forkDepthSelector import org.utbot.engine.selectors.inheritorsSelector +import org.utbot.engine.selectors.nnRewardGuidedSelector import org.utbot.engine.selectors.nurs.NonUniformRandomSearch import org.utbot.engine.selectors.pollUntilFastSAT +import org.utbot.engine.selectors.randomPathSelector +import org.utbot.engine.selectors.randomSelector import org.utbot.engine.selectors.strategies.GraphViz +import org.utbot.engine.selectors.subpathGuidedSelector import org.utbot.engine.symbolic.HardConstraint import org.utbot.engine.symbolic.SoftConstraint import org.utbot.engine.symbolic.SymbolicState @@ -101,6 +110,7 @@ import org.utbot.engine.util.statics.concrete.makeSymbolicValuesFromEnumConcrete import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import org.utbot.framework.UtSettings.checkSolverTimeoutMillis +import org.utbot.framework.UtSettings.enableFeatureProcess import org.utbot.framework.UtSettings.pathSelectorStepsLimit import org.utbot.framework.UtSettings.pathSelectorType import org.utbot.framework.UtSettings.preferredCexOption @@ -143,6 +153,15 @@ import org.utbot.fuzzer.collectConstantsForFuzzer import org.utbot.fuzzer.defaultModelProviders import org.utbot.fuzzer.fuzz import org.utbot.instrumentation.ConcreteExecutor +import java.lang.reflect.ParameterizedType +import kotlin.collections.plus +import kotlin.collections.plusAssign +import kotlin.math.max +import kotlin.math.min +import kotlin.reflect.full.instanceParameter +import kotlin.reflect.full.valueParameters +import kotlin.reflect.jvm.javaType +import kotlin.system.measureTimeMillis import soot.ArrayType import soot.BooleanType import soot.ByteType @@ -233,14 +252,6 @@ import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl import sun.reflect.generics.reflectiveObjects.TypeVariableImpl import sun.reflect.generics.reflectiveObjects.WildcardTypeImpl import java.lang.reflect.Method -import java.lang.reflect.ParameterizedType -import kotlin.collections.plus -import kotlin.collections.plusAssign -import kotlin.math.max -import kotlin.math.min -import kotlin.reflect.full.instanceParameter -import kotlin.reflect.full.valueParameters -import kotlin.reflect.jvm.javaType private val logger = KotlinLogging.logger {} val pathLogger = KotlinLogging.logger(logger.name + ".path") @@ -269,7 +280,24 @@ private fun pathSelector(graph: InterProceduralUnitGraph, typeRegistry: TypeRegi PathSelectorType.INHERITORS_SELECTOR -> inheritorsSelector(graph, typeRegistry) { withStepsLimit(pathSelectorStepsLimit) } - else -> error("Unknown type") + PathSelectorType.SUBPATH_GUIDED_SELECTOR -> subpathGuidedSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.CPI_SELECTOR -> cpInstSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.FORK_DEPTH_SELECTOR -> forkDepthSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.NN_REWARD_GUIDED_SELECTOR -> nnRewardGuidedSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.RANDOM_SELECTOR -> randomSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.RANDOM_PATH_SELECTOR -> randomPathSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } } class UtBotSymbolicEngine( @@ -284,6 +312,7 @@ class UtBotSymbolicEngine( ) : UtContextInitializer() { private val methodUnderAnalysisStmts: Set = graph.stmts.toSet() + private val visitedStmts: MutableSet = mutableSetOf() private val globalGraph = InterProceduralUnitGraph(graph) internal val typeRegistry: TypeRegistry = TypeRegistry() private val pathSelector: PathSelector = pathSelector(globalGraph, typeRegistry) @@ -333,6 +362,9 @@ class UtBotSymbolicEngine( private var queuedSymbolicStateUpdates = SymbolicStateUpdate() + private val featureProcessor: FeatureProcessor? = + if (enableFeatureProcess) EngineAnalyticsContext.featureProcessorFactory(globalGraph) else null + private val insideStaticInitializer get() = environment.state.executionStack.any { it.method.isStaticInitializer } @@ -355,6 +387,7 @@ class UtBotSymbolicEngine( logger.error(e) { "Closing resource failed" } } trackableResources.clear() + featureProcessor?.dumpFeatures() } private suspend fun preTraverse() { @@ -364,7 +397,9 @@ class UtBotSymbolicEngine( stateSelectedCount = 0 } - fun traverse(): Flow = traverseImpl().onStart { preTraverse() }.onCompletion { postTraverse() } + fun traverse(): Flow = traverseImpl() + .onStart { preTraverse() } + .onCompletion { postTraverse() } private fun traverseImpl(): Flow = flow { @@ -402,7 +437,8 @@ class UtBotSymbolicEngine( stateSelectedCount++ pathLogger.trace { "traverse<$methodUnderTest>: choosing next state($stateSelectedCount), " + - "queue size=${(pathSelector as? NonUniformRandomSearch)?.size ?: -1}" } + "queue size=${(pathSelector as? NonUniformRandomSearch)?.size ?: -1}" + } if (controller.executeConcretely || statesForConcreteExecution.isNotEmpty()) { val state = pathSelector.pollUntilFastSAT() ?: statesForConcreteExecution.pollUntilSat() ?: break @@ -457,6 +493,7 @@ class UtBotSymbolicEngine( } else { val state = pathSelector.poll() + // state is null in case states queue is empty // or path selector exceed some limits (steps limit, for example) if (state == null) { @@ -476,32 +513,39 @@ class UtBotSymbolicEngine( } } - environment.state = state + state.executingTime += measureTimeMillis { + environment.state = state - val currentStmt = environment.state.stmt + val currentStmt = environment.state.stmt - environment.method = globalGraph.method(currentStmt) + if (currentStmt !in visitedStmts) { + environment.state.updateIsVisitedNew() + visitedStmts += currentStmt + } - environment.state.lastEdge?.let { - globalGraph.visitEdge(it) - } + environment.method = globalGraph.method(currentStmt) - try { - val exception = environment.state.exception - if (exception != null) { - traverseException(currentStmt, exception) - } else { - traverseStmt(currentStmt) + environment.state.lastEdge?.let { + globalGraph.visitEdge(it) } - } catch (ex: Throwable) { - environment.state.close() - if (ex !is CancellationException) { - logger.error(ex) { "Test generation failed on stmt $currentStmt, symbolic stack trace:\n$symbolicStackTrace" } - // TODO: enrich with nice description for known issues - emit(UtError(ex.description, ex)) - } else { - logger.debug(ex) { "Cancellation happened" } + try { + val exception = environment.state.exception + if (exception != null) { + traverseException(currentStmt, exception) + } else { + traverseStmt(currentStmt) + } + } catch (ex: Throwable) { + environment.state.close() + + if (ex !is CancellationException) { + logger.error(ex) { "Test generation failed on stmt $currentStmt, symbolic stack trace:\n$symbolicStackTrace" } + // TODO: enrich with nice description for known issues + emit(UtError(ex.description, ex)) + } else { + logger.debug(ex) { "Cancellation happened" } + } } } } @@ -1323,6 +1367,10 @@ class UtBotSymbolicEngine( val (positiveCaseSoftConstraint, negativeCaseSoftConstraint) = resolvedCondition.softConstraints val negativeCasePathConstraint = mkNot(positiveCasePathConstraint) + if (positiveCaseEdge != null) { + environment.state.definitelyFork() + } + positiveCaseEdge?.let { edge -> environment.state.expectUndefined() val positiveCaseState = environment.state.updateQueued( @@ -1392,7 +1440,9 @@ class UtBotSymbolicEngine( } if (successors.size > 1) { environment.state.expectUndefined() + environment.state.definitelyFork() } + successors.forEach { (target, expr) -> pathSelector.offer( environment.state.updateQueued( @@ -2600,6 +2650,10 @@ class UtBotSymbolicEngine( // If so, return the result of the override if (artificialMethodOverride.success) { + if (artificialMethodOverride.results.size > 1) { + environment.state.definitelyFork() + } + return mutableListOf().apply { for (result in artificialMethodOverride.results) { when (result) { @@ -2627,6 +2681,10 @@ class UtBotSymbolicEngine( // that will override the invocation. val overrideResults = targets.map { it to overrideInvocation(invocation, it) } + if (overrideResults.sumOf { (_, overriddenResult) -> overriddenResult.results.size } > 1) { + environment.state.definitelyFork() + } + // Separate targets for which invocation should be overridden // from the targets that should be processed regularly. val (overridden, original) = overrideResults.partition { it.second.success } @@ -3289,12 +3347,12 @@ class UtBotSymbolicEngine( mkNot(UtSubNoOverflowExpression(left.expr, right.expr)) } is JMulExpr -> when (sort.type) { - is ByteType -> lowerIntMulOverflowCheck(left, right, Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()) - is ShortType -> lowerIntMulOverflowCheck(left, right, Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) - is IntType -> higherIntMulOverflowCheck(left, right, Int.SIZE_BITS, Int.MIN_VALUE.toLong()) { it: UtExpression -> it.toIntValue() } - is LongType -> higherIntMulOverflowCheck(left, right, Long.SIZE_BITS, Long.MIN_VALUE) { it: UtExpression -> it.toLongValue() } - else -> null - } + is ByteType -> lowerIntMulOverflowCheck(left, right, Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()) + is ShortType -> lowerIntMulOverflowCheck(left, right, Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) + is IntType -> higherIntMulOverflowCheck(left, right, Int.SIZE_BITS, Int.MIN_VALUE.toLong()) { it: UtExpression -> it.toIntValue() } + is LongType -> higherIntMulOverflowCheck(left, right, Long.SIZE_BITS, Long.MIN_VALUE) { it: UtExpression -> it.toLongValue() } + else -> null + } else -> null } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt index 2d3110908a..571034332e 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt @@ -2,9 +2,9 @@ package org.utbot.engine.selectors import org.utbot.engine.ExecutionState import org.utbot.engine.pathLogger -import org.utbot.engine.pc.UtSolverStatusUNSAT import org.utbot.engine.pc.UtSolver import org.utbot.engine.pc.UtSolverStatusKind.SAT +import org.utbot.engine.pc.UtSolverStatusUNSAT import org.utbot.engine.selectors.strategies.ChoosingStrategy import org.utbot.engine.selectors.strategies.StoppingStrategy import org.utbot.framework.UtSettings @@ -59,7 +59,8 @@ abstract class BasePathSelector( /** * @return true if [utSolver] constraints are satisfiable */ - private fun checkUnsat(utSolver: UtSolver): Boolean = utSolver.assertions.isNotEmpty() && utSolver.check(respectSoft = false).statusKind != SAT + private fun checkUnsat(utSolver: UtSolver): Boolean = + utSolver.assertions.isNotEmpty() && utSolver.check(respectSoft = false).statusKind != SAT /** * check fast unsat on forks diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt new file mode 100644 index 0000000000..c4f37c2924 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt @@ -0,0 +1,73 @@ +package org.utbot.engine.selectors + +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.Predictors +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.nurs.GreedySearch +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * @see Learch + * + * Calculates reward using neural network, when state is offered, and then peeks state with maximum reward + * + * @see choosingStrategy [ChossingStrategy] for [GreedySearch] + * + * [GreedySearch] + */ +abstract class NNRewardGuidedSelector( + protected val generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph +) : GreedySearch(choosingStrategy, stoppingStrategy, seed) { + protected val featureExtractor = EngineAnalyticsContext.featureExtractorFactory(graph) + + override val name: String + get() = "NNRewardGuidedSelector" +} + +/** + * Calculate weight of execution state only when it is offered. It has advantage, because it works faster, + * than with recalculation but disadvantage is that some features of execution state can change. + */ +class NNRewardGuidedSelectorWithoutWeightsRecalculation( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph +) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { + override fun offerImpl(state: ExecutionState) { + super.offerImpl(state) + featureExtractor.extractFeatures(state, generatedTestCountingStatistics.generatedTestsCount) + } + + override val ExecutionState.weight: Double + get() { + reward = reward ?: Predictors.stateRewardPredictor.predict(features) + return reward as Double + } +} + +/** + * Calculate weight of execution state every time when it needed. It works slower, + * than without recalculation but features are always relevant + */ +class NNRewardGuidedSelectorWithWeightsRecalculation( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph +) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { + override val ExecutionState.weight: Double + get() { + featureExtractor.extractFeatures(this, generatedTestCountingStatistics.generatedTestsCount) + return Predictors.stateRewardPredictor.predict(features) + } +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt new file mode 100644 index 0000000000..cad9632381 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt @@ -0,0 +1,49 @@ +package org.utbot.engine.selectors + +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * Creates [NNRewardGuidedSelector] + */ +interface NNRewardGuidedSelectorFactory { + operator fun invoke( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph + ): NNRewardGuidedSelector +} + +/** + * Creates [NNRewardGuidedSelectorWithWeightsRecalculation] + */ +class NNRewardGuidedSelectorWithRecalculationFactory : NNRewardGuidedSelectorFactory { + override fun invoke( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int, + graph: InterProceduralUnitGraph + ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithWeightsRecalculation( + generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph + ) +} + +/** + * Creates [NNRewardGuidedSelectorWithoutWeightsRecalculation] + */ +class NNRewardGuidedSelectorWithoutRecalculationFactory : NNRewardGuidedSelectorFactory { + override fun invoke( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int, + graph: InterProceduralUnitGraph + ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithoutWeightsRecalculation( + generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph + ) +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt index 8647995743..1d5fb681d8 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt @@ -2,22 +2,29 @@ package org.utbot.engine.selectors +import org.utbot.analytics.EngineAnalyticsContext import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.TypeRegistry import org.utbot.engine.selectors.StrategyOption.DISTANCE import org.utbot.engine.selectors.StrategyOption.VISIT_COUNTING +import org.utbot.engine.selectors.nurs.CPInstSelector import org.utbot.engine.selectors.nurs.CoveredNewSelector import org.utbot.engine.selectors.nurs.DepthSelector +import org.utbot.engine.selectors.nurs.ForkDepthSelector +import org.utbot.engine.selectors.nurs.InheritorsSelector import org.utbot.engine.selectors.nurs.MinimalDistanceToUncovered import org.utbot.engine.selectors.nurs.NeuroSatSelector -import org.utbot.engine.selectors.nurs.InheritorsSelector import org.utbot.engine.selectors.nurs.RPSelector +import org.utbot.engine.selectors.nurs.SubpathGuidedSelector import org.utbot.engine.selectors.nurs.VisitCountingSelector import org.utbot.engine.selectors.strategies.ChoosingStrategy import org.utbot.engine.selectors.strategies.DistanceStatistics import org.utbot.engine.selectors.strategies.EdgeVisitCountingStatistics +import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics +import org.utbot.engine.selectors.strategies.StatementsStatistics import org.utbot.engine.selectors.strategies.StepsLimitStoppingStrategy import org.utbot.engine.selectors.strategies.StoppingStrategy +import org.utbot.engine.selectors.strategies.SubpathStatistics import org.utbot.framework.UtSettings.seedInPathSelector /** @@ -57,6 +64,34 @@ fun inheritorsSelector( builder: InheritorsSelectorBuilder.() -> (Unit) ) = InheritorsSelectorBuilder(graph, typeRegistry).apply(builder).build() + +/** + * build [SubpathGuidedSelector] using [SubpathGuidedSelectorBuilder] + */ +fun subpathGuidedSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: SubpathGuidedSelectorBuilder.() -> (Unit) +) = SubpathGuidedSelectorBuilder(graph, strategy).apply(builder).build() + +/** + * build [CPInstSelector] using [CPInstSelectorBuilder] + */ +fun cpInstSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: CPInstSelectorBuilder.() -> (Unit) +) = CPInstSelectorBuilder(graph, strategy).apply(builder).build() + +/** + * build [ForkDepthSelector] using [ForkDepthSelectorBuilder] + */ +fun forkDepthSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: ForkDepthSelectorBuilder.() -> (Unit) +) = ForkDepthSelectorBuilder(graph, strategy).apply(builder).build() + /** * build [DepthSelector] using [DepthSelectorBuilder] */ @@ -133,10 +168,22 @@ fun visitCountingSelector( fun interleavedSelector(graph: InterProceduralUnitGraph, builder: InterleavedSelectorBuilder.() -> (Unit)) = InterleavedSelectorBuilder(graph).apply(builder).build() +/** + * build [NNRewardGuidedSelector] using [NNRewardGuidedSelectorBuilder] + */ +fun nnRewardGuidedSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: NNRewardGuidedSelectorBuilder.() -> Unit +) = NNRewardGuidedSelectorBuilder(graph, strategy).apply(builder).build() + data class PathSelectorContext( val graph: InterProceduralUnitGraph, var distanceStatistics: DistanceStatistics? = null, + var subpathStatistics: SubpathStatistics? = null, + var statementsStatistics: StatementsStatistics? = null, var countingStatistics: EdgeVisitCountingStatistics? = null, + var generatedTestCountingStatistics: GeneratedTestCountingStatistics? = null, var stoppingStrategy: StoppingStrategy? = null, ) @@ -192,6 +239,59 @@ class InheritorsSelectorBuilder internal constructor( ) } +/** + * Builder for [SubpathGuidedSelector]. Used in [subpathGuidedSelector] + */ +class SubpathGuidedSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + private val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph) +) : PathSelectorBuilder(graph, context) { + val seed: Int? = seedInPathSelector + + override fun build() = SubpathGuidedSelector( + withSubpathStatistics(), + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed!! + ) +} + +/** + * Builder for [CPInstSelectorBuilder]. Used in [cpInstSelector] + */ +class CPInstSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + private val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph) +) : PathSelectorBuilder(graph, context) { + val seed: Int = 42 + + override fun build() = CPInstSelector( + withStatementsStatistics(), + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed + ) +} + +/** + * Builder for [ForkDepthSelector]. Used in [forkDepthSelector] + */ +class ForkDepthSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph) +) : PathSelectorBuilder(graph, context) { + val seed: Int = 42 + + override fun build() = ForkDepthSelector( + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed + ) +} + /** * Builder for [DepthSelector]. Used in [depthSelector] * @@ -424,6 +524,24 @@ class InterleavedSelectorBuilder internal constructor( override fun build() = InterleavedSelector(builders.map { it.build() }) } +/** + * Builder for [NNRewardGuidedSelector]. Used in [] + */ +class NNRewardGuidedSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + private val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph), +) : PathSelectorBuilder(graph, context) { + private val seed = seedInPathSelector + override fun build() = EngineAnalyticsContext.nnRewardGuidedSelectorFactory( + withGeneratedTestCountingStatistics(), + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed!!, + graph + ) +} + /** * Base pathSelectorBuilder that maintains context to attach necessary statistics to graph */ @@ -460,6 +578,36 @@ sealed class PathSelectorBuilder( return context.distanceStatistics!! } + /** + * Create new [SubpathStatistics] and attach it to graph or use created one + */ + protected fun withSubpathStatistics(): SubpathStatistics { + if (context.subpathStatistics == null) { + context.subpathStatistics = SubpathStatistics(graph) + } + return context.subpathStatistics!! + } + + /** + * Create new [StatementsStatistics] and attach it to graph or use created one + */ + protected fun withStatementsStatistics(): StatementsStatistics { + if (context.statementsStatistics == null) { + context.statementsStatistics = StatementsStatistics(graph) + } + return context.statementsStatistics!! + } + + /** + * Create new [GeneratedTestCountingStatistics] and attach it to graph or use created one + */ + protected fun withGeneratedTestCountingStatistics(): GeneratedTestCountingStatistics { + if (context.generatedTestCountingStatistics == null) { + context.generatedTestCountingStatistics = GeneratedTestCountingStatistics(graph) + } + return context.generatedTestCountingStatistics!! + } + /** * Create new [EdgeVisitCountingStatistics] and attach ti to graph or use created one */ diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt new file mode 100644 index 0000000000..261dfc2fc2 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt @@ -0,0 +1,29 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StatementsStatistics +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * Execution states are weighted as 1.0 / StatementStatistics.statementsInMethodCount(exState), + * where StatementStatistics.statementsInMethodCount(exState) + * is number of visited instructions in the current function of execution state + * + * @see @see Klee analog + * + * [NonUniformRandomSearch] + */ +class CPInstSelector( + private val statementStatistics: StatementsStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int? = 42 +) : NonUniformRandomSearch(choosingStrategy, stoppingStrategy, seed) { + + override val name: String + get() = "NURS:CPICnt" + + override val ExecutionState.cost: Double + get() = statementStatistics.statementInMethodCount(this).toDouble() +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/ForkDepthSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/ForkDepthSelector.kt new file mode 100644 index 0000000000..7c5e27aaf7 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/ForkDepthSelector.kt @@ -0,0 +1,24 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * Execution states are weighted as 1.0 / (exState.depth + 1), + * where exState.depth is number of forks on the executionState's path + * + * @see [NonUniformRandomSearch] + */ +class ForkDepthSelector( + policy: ChoosingStrategy, + stoppingPolicy: StoppingStrategy, + seed: Int? = 42 +) : NonUniformRandomSearch(policy, stoppingPolicy, seed) { + + override val ExecutionState.cost: Double + get() = depth.toDouble() + + override val name: String + get() = "ForkDepth" +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt new file mode 100644 index 0000000000..b00d67e871 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt @@ -0,0 +1,78 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.BasePathSelector +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StoppingStrategy +import org.utbot.engine.selectors.strategies.StrategyObserver +import kotlin.properties.Delegates +import kotlin.random.Random + +/** + * Selects ExecutionState with maximum weight. + * If there are several states with equal maximum weight, than selects random from them. + */ +abstract class GreedySearch( + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42 +) : BasePathSelector(choosingStrategy, stoppingStrategy), StrategyObserver { + private val states = mutableSetOf() + + private val randomGen: Random = Random(seed) + + override fun update() { + // do nothing by default + } + + override fun offerImpl(state: ExecutionState) { + states += state + } + + override fun pollImpl(): ExecutionState? = peekImpl()?.also { remove(it) } + + /** + * Creates set of states with maximum weight and then pick random + */ + override fun peekImpl(): ExecutionState? { + if (isEmpty()) { + return null + } + + val candidates = mutableListOf() + var bestWeight by Delegates.notNull() + + states.forEach { + val weight = it.weight + + if (candidates.isEmpty()) { + bestWeight = weight + candidates += it + } else { + if (bestWeight <= weight) { + if (bestWeight < weight) { + bestWeight = weight + candidates.clear() + } + candidates += it + } + } + } + + return candidates.random(randomGen) + } + + override fun removeImpl(state: ExecutionState): Boolean { + return states.remove(state) + } + + override fun isEmpty() = states.isEmpty() + + override fun close() { + states.forEach { + it.close() + } + } + + protected abstract val ExecutionState.weight: Double +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt new file mode 100644 index 0000000000..a17e94a6a2 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt @@ -0,0 +1,30 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StoppingStrategy +import org.utbot.engine.selectors.strategies.SubpathStatistics + +/** + * Peeks state with minimum frequency of relative subpath. + * See https://github.com/eth-sri/learch/blob/master/klee/lib/Core/Searcher.cpp#L738 + * + * @see [GreedySearch] + */ +class SubpathGuidedSelector( + private val subpathStatistics: SubpathStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42 +) : GreedySearch(choosingStrategy, stoppingStrategy, seed) { + + + override val name + get() = "NURS:SubpathGuidedSearch" + + /** + * Use - to find minimum + */ + override val ExecutionState.weight: Double + get() = -subpathStatistics.subpathCount(this).toDouble() +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/GeneratedTestCountingStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/GeneratedTestCountingStatistics.kt new file mode 100644 index 0000000000..315673e042 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/GeneratedTestCountingStatistics.kt @@ -0,0 +1,15 @@ +package org.utbot.engine.selectors.strategies + +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph + +class GeneratedTestCountingStatistics( + graph: InterProceduralUnitGraph +) : TraverseGraphStatistics(graph) { + var generatedTestsCount = 0 + private set + + override fun onTraversed(executionState: ExecutionState) { + generatedTestsCount++ + } +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/StatementsStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/StatementsStatistics.kt new file mode 100644 index 0000000000..375462191c --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/StatementsStatistics.kt @@ -0,0 +1,37 @@ +package org.utbot.engine.selectors.strategies + +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import soot.SootMethod +import soot.jimple.Stmt + +/** + * Calculates + * - number of times for which state’s current instruction has been visited in [statementsCount] + * - number of instructions visited in state’s current method in [statementsInMethodCount] + */ +class StatementsStatistics( + graph: InterProceduralUnitGraph +) : TraverseGraphStatistics(graph) { + private val statementsCount = mutableMapOf() + private val statementsInMethodCount = mutableMapOf() + + override fun onVisit(executionState: ExecutionState) { + statementsCount.compute(executionState.stmt) { _, v -> + v?.plus(1) ?: 1 + } + + if (statementsCount[executionState.stmt] == 1) { + executionState.lastMethod?.let { + statementsInMethodCount.compute(it) { _, v -> + v?.plus(1) ?: 1 + } + } + } + } + + fun statementCount(executionState: ExecutionState) = statementsCount.getOrDefault(executionState.stmt, 1) + + fun statementInMethodCount(executionState: ExecutionState) = + executionState.lastMethod?.let { statementsInMethodCount.getOrDefault(it, 1) } ?: 0 +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt new file mode 100644 index 0000000000..0110cf03c9 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt @@ -0,0 +1,51 @@ +package org.utbot.engine.selectors.strategies + +import org.utbot.engine.CALL_DECISION_NUM +import org.utbot.engine.Edge +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.framework.UtSettings +import kotlin.math.pow + +/** + * Calculates frequency of subpathes of statements with length equal to 2^{index}. + */ +class SubpathStatistics( + graph: InterProceduralUnitGraph, + index: Int = UtSettings.subpathGuidedSelectorIndex +) : TraverseGraphStatistics(graph) { + private val subpathCount = mutableMapOf, Int>() + private val length: Int = 2.0.pow(index).toInt() + + /** + * Take length last edges from state's path and handle exception edges + */ + private fun ExecutionState.getSubpath(length: Int): List { + val subpath = mutableListOf() + val actualLength = if (pathLength >= length) length else pathLength + + var current = stmt + var exceptionNumber = 0 + (0 until actualLength).forEach { + val decision = decisionPath[decisionPath.size - it - 1] + if (decision < CALL_DECISION_NUM) { + exceptionNumber++ + } + val i = path.size - it - 1 + exceptionNumber + val prev = if (i == path.size) stmt else path[i] + subpath += Edge(prev, current, decision) + current = prev + } + + return subpath + } + + override fun onVisit(executionState: ExecutionState) { + subpathCount.compute(executionState.getSubpath(length)) { _, v -> + v?.plus(1) ?: 1 + } + } + + fun subpathCount(executionState: ExecutionState): Int = + subpathCount.getOrPut(executionState.getSubpath(length)) { 1 } +} \ No newline at end of file diff --git a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt index 1587b5661e..e2318ae026 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt @@ -67,6 +67,7 @@ import kotlin.reflect.KFunction4 import kotlin.reflect.KFunction5 import mu.KotlinLogging import org.junit.jupiter.api.Assertions.assertTrue +import org.utbot.framework.PathSelectorType val logger = KotlinLogging.logger {} @@ -1607,7 +1608,7 @@ abstract class AbstractTestCaseGeneratorTest( summaryNameChecks: List<(String?) -> Boolean> = listOf(), summaryDisplayNameChecks: List<(String?) -> Boolean> = listOf() ) = internalCheck( - method, mockStrategy, branches, matchers, coverage, T1::class, T2::class, T3::class, T4::class , + method, mockStrategy, branches, matchers, coverage, T1::class, T2::class, T3::class, T4::class, arguments = ::withParamsMutationsAndResult, additionalDependencies = additionalDependencies, summaryTextChecks = summaryTextChecks, @@ -2625,7 +2626,9 @@ fun withStaticsBefore(ex: UtValueExecution<*>) = ex.paramsBefore + ex.staticsBef fun withStaticsAfter(ex: UtValueExecution<*>) = ex.paramsBefore + ex.staticsAfter + ex.evaluatedResult fun withThisAndStaticsAfter(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.staticsAfter + ex.evaluatedResult fun withThisAndResult(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.evaluatedResult -fun withThisStaticsBeforeAndResult(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.staticsBefore + ex.evaluatedResult +fun withThisStaticsBeforeAndResult(ex: UtValueExecution<*>) = + listOf(ex.callerBefore) + ex.paramsBefore + ex.staticsBefore + ex.evaluatedResult + fun withThisAndException(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.returnValue fun withMocks(ex: UtValueExecution<*>) = ex.paramsBefore + listOf(ex.mocks) + ex.evaluatedResult fun withMocksAndInstrumentation(ex: UtValueExecution<*>) = @@ -2735,7 +2738,10 @@ inline fun withoutMinimization(block: () -> T): T { } } -inline fun withSettingsFromTestFrameworkConfiguration(config: TestFrameworkConfiguration, block: () -> T): T { +inline fun withSettingsFromTestFrameworkConfiguration( + config: TestFrameworkConfiguration, + block: () -> T +): T { val substituteStaticsWithSymbolicVariable = UtSettings.substituteStaticsWithSymbolicVariable UtSettings.substituteStaticsWithSymbolicVariable = config.resetNonFinalFieldsAfterClinit try { @@ -2750,8 +2756,7 @@ inline fun withoutSubstituteStaticsWithSymbolicVariable(block: () -> T) { UtSettings.substituteStaticsWithSymbolicVariable = false try { block() - } - finally { + } finally { UtSettings.substituteStaticsWithSymbolicVariable = substituteStaticsWithSymbolicVariable } } @@ -2775,3 +2780,23 @@ inline fun withTreatingOverflowAsError(block: () -> T): T { UtSettings.treatOverflowAsError = prev } } + +inline fun withRewardModelPath(rewardModelPath: String, block: () -> T): T { + val prev = UtSettings.rewardModelPath + UtSettings.rewardModelPath = rewardModelPath + try { + return block() + } finally { + UtSettings.rewardModelPath = prev + } +} + +inline fun withPathSelectorType(pathSelectorType: PathSelectorType, block: () -> T): T { + val prev = UtSettings.pathSelectorType + UtSettings.pathSelectorType = pathSelectorType + try { + return block() + } finally { + UtSettings.pathSelectorType = prev + } +} diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index cb8488684d..85ff761b17 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -1,5 +1,7 @@ package org.utbot.contest +import mu.KotlinLogging +import org.utbot.analytics.EngineAnalyticsContext import org.utbot.common.FileUtil import org.utbot.common.bracket import org.utbot.common.info @@ -13,6 +15,8 @@ import org.utbot.contest.Paths.evosuiteReportFile import org.utbot.contest.Paths.jarsDir import org.utbot.contest.Paths.moduleTestDir import org.utbot.contest.Paths.outputDir +import org.utbot.features.FeatureExtractorFactoryImpl +import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.withUtContext @@ -29,7 +33,6 @@ import java.util.zip.ZipInputStream import kotlin.concurrent.thread import kotlin.math.min import kotlin.system.exitProcess -import mu.KotlinLogging import org.utbot.framework.JdkPathService private val logger = KotlinLogging.logger {} @@ -320,6 +323,9 @@ fun runEstimator( // Predictors.smt = UtBotTimePredictor() // Predictors.smtIncremental = UtBotTimePredictorIncremental() // Predictors.testName = StatementUniquenessPredictor() +// Predictors.stateRewardPredictor = NNStateRewardPredictorBase() + EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() + EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() // fix for CTRL-ALT-SHIFT-C from IDEA, which copies in class#method form