From 744418a1697d6664a3c07029188d607e2a3b4add Mon Sep 17 00:00:00 2001 From: Denis Fokin Date: Wed, 25 May 2022 13:50:34 +0300 Subject: [PATCH 01/37] Buildable --- utbot-analytics/build.gradle | 20 + .../kotlin/org/utbot/ClassifierTrainer.kt | 4 +- .../main/kotlin/org/utbot/QualityAnalysis.kt | 167 +++++ .../kotlin/org/utbot/QualityAnalysisConfig.kt | 20 + .../kotlin/org/utbot/QualityAnalysisV2.kt | 67 ++ .../kotlin/org/utbot/StmtCoverageReport.kt | 172 +++++ .../features/FeatureExtractorFactoryImpl.kt | 12 + .../utbot/features/FeatureExtractorImpl.kt | 42 ++ .../FeatureProcessorWithStatesRepetition.kt | 153 +++++ ...ureProcessorWithStatesRepetitionFactory.kt | 12 + .../features/UtExpressionGraphExtraction.kt | 586 ++++++++++++++++++ .../predictors/LinearStateRewardPredictor.kt | 25 + .../predictors/NNStateRewardPredictor.kt | 5 + .../predictors/NNStateRewardPredictorSmile.kt | 73 +++ .../predictors/NNStateRewardPredictorTorch.kt | 37 ++ .../org/utbot/predictors/UtBotSatPredictor.kt | 41 +- .../main/kotlin/org/utbot/utils/BinaryUtil.kt | 29 + .../main/kotlin/org/utbot/utils/MatrixUtil.kt | 129 ++++ .../utbot/utils/layers/MessageAggregation.kt | 27 + .../org/utbot/utils/layers/MessagePassing.kt | 27 + .../org/utbot/utils/layers/SimpleDecoder.kt | 53 ++ .../org/utbot/visual/AbstractHtmlReport.kt | 2 +- .../utbot/visual/ClassificationHtmlReport.kt | 72 +-- .../kotlin/org/utbot/visual/FigureBuilders.kt | 34 +- .../kotlin/org/utbot/visual/HtmlBuilder.kt | 83 ++- .../org/utbot/visual/TracePathReport.kt | 167 ----- .../src/main/resources/config.properties | 3 + .../src/main/resources/css/coverage.css | 168 +++++ .../src/main/resources/css/highlight-idea.css | 153 +++++ .../FeatureProcessorWithRepetitionTest.kt | 61 ++ .../LinearStateRewardPredictorTest.kt | 21 + .../predictors/NNStateRewardPredictorTest.kt | 53 ++ utbot-analytics/src/test/resources/linear.txt | 1 + utbot-analytics/src/test/resources/nn.json | 24 + utbot-analytics/src/test/resources/scaler.txt | 2 + .../kotlin/org/utbot/framework/UtSettings.kt | 60 +- .../org/utbot/analytics/CoverageStatistics.kt | 49 ++ .../utbot/analytics/EngineAnalyticsContext.kt | 32 + .../org/utbot/analytics/FeatureExtractor.kt | 10 + .../analytics/FeatureExtractorFactory.kt | 10 + .../org/utbot/analytics/FeatureProcessor.kt | 11 + .../analytics/FeatureProcessorFactory.kt | 10 + .../kotlin/org/utbot/analytics/Predictors.kt | 10 + .../org/utbot/analytics/UtBotPredictor.kt | 9 +- .../kotlin/org/utbot/engine/ExecutionState.kt | 111 +++- .../org/utbot/engine/UtBotSymbolicEngine.kt | 137 +++- .../engine/selectors/BasePathSelector.kt | 3 +- .../selectors/NNRewardGuidedSelector.kt | 70 +++ .../NNRewardGuidedSelectorFactory.kt | 49 ++ .../engine/selectors/PathSelectorBuilder.kt | 156 ++++- .../engine/selectors/nurs/CPInstSelector.kt | 29 + .../selectors/nurs/ForkDepthSelector.kt | 24 + .../engine/selectors/nurs/GreedySearch.kt | 75 +++ .../selectors/nurs/SubpathGuidedSelector.kt | 26 + .../GeneratedTestCountingStatistics.kt | 15 + .../strategies/StatementsStatistics.kt | 37 ++ .../selectors/strategies/SubpathStatistics.kt | 47 ++ .../examples/AbstractTestCaseGeneratorTest.kt | 10 + .../org/utbot/contest/ContestEstimator.kt | 5 + 59 files changed, 3235 insertions(+), 305 deletions(-) create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorFactoryImpl.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt create mode 100644 utbot-analytics/src/main/resources/config.properties create mode 100644 utbot-analytics/src/main/resources/css/coverage.css create mode 100644 utbot-analytics/src/main/resources/css/highlight-idea.css create mode 100644 utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt create mode 100644 utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt create mode 100644 utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt create mode 100644 utbot-analytics/src/test/resources/linear.txt create mode 100644 utbot-analytics/src/test/resources/nn.json create mode 100644 utbot-analytics/src/test/resources/scaler.txt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessorFactory.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/ForkDepthSelector.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/GeneratedTestCountingStatistics.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/StatementsStatistics.kt create mode 100644 utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index fd34a6eb97..b41d264b9d 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -12,6 +12,7 @@ String classifier = osName + "-x86_64" evaluationDependsOn(':utbot-framework') compileTestJava.dependsOn tasks.getByPath(':utbot-framework:testClasses') + dependencies { implementation(project(":utbot-framework")) implementation(project(':utbot-instrumentation')) @@ -29,6 +30,7 @@ dependencies { implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'javacpp', version: '1.5.3', classifier: "$classifier" implementation group: 'tech.tablesaw', name: 'tablesaw-core', version: '0.38.2' implementation group: 'tech.tablesaw', name: 'tablesaw-jsplot', version: '0.38.2' @@ -37,6 +39,12 @@ dependencies { implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.22.1' + implementation group: 'org.jsoup', name: 'jsoup', version: '1.7.2' + + implementation "ai.djl:api:0.16.0" + implementation "ai.djl.pytorch:pytorch-engine:0.16.0" + implementation "ai.djl.pytorch:pytorch-native-auto:1.9.1" + testCompile project(':utbot-framework').sourceSets.test.output } @@ -54,4 +62,16 @@ processResources { into "models" } } +} + +jar { dependsOn classes + manifest { + attributes 'Main-Class': 'org.utbot.QualityAnalysisKt' + } + + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt index c3bba8ff70..3da7a15ab6 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt @@ -6,7 +6,7 @@ import org.utbot.metrics.ClassificationMetrics import org.utbot.models.ClassifierModel import org.utbot.models.loadModelFromJson import org.utbot.models.save -import org.utbot.visual.ClassificationHtmlReport +import org.utbot.visual.ClassificationHTMLReport import smile.classification.Classifier import smile.data.CategoricalEncoder import smile.data.DataFrame @@ -66,7 +66,7 @@ class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = } override fun visualize() { - val report = ClassificationHtmlReport() + val report = ClassificationHTMLReport() report.run { addHeader(classifierModel.name, properties) addDataDistribution(formula.y(data).toDoubleArray()) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt new file mode 100644 index 0000000000..deaafb5b81 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt @@ -0,0 +1,167 @@ +package org.utbot + +import org.utbot.QualityAnalysisConfig +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.select.Elements +import java.io.File +import java.nio.file.Paths + + +data class Coverage(val misInstructions: Int, val covInstruction: Int, val misBranches: Int, val covBranches: Int, val time: Double = 0.0) { + fun getInstructionCoverage(): Double = when { + (this.misInstructions == 0 && this.covInstruction == 0) -> 0.0 + (this.misInstructions == 0) -> 1.0 + else -> this.covInstruction.toDouble() / (this.covInstruction + this.misInstructions) + } + + fun getBranchesCoverage(): Double = when { + (this.misBranches == 0 && this.covBranches == 0) -> 0.0 + (this.misBranches == 0) -> 1.0 + else -> this.covBranches.toDouble() / (this.covBranches + this.misBranches) + } + + fun getInstructions() = misInstructions + covInstruction +} + + +fun parseJacocoReport(path: String, classes: Set): Pair, Map> { + val perClassResult = mutableMapOf() + val perMethodResult = mutableMapOf() + val contestDocument: Document = Jsoup.parse(File("$path/index.html"), null) + val pkgRows: Elements = contestDocument.select("table")[0].select("tr") + + for (i in 2 until pkgRows.size) { + val pkgHref = pkgRows[i].select("td")[0].select("a").attr("href") + val packageDocument: Document = Jsoup.parse(File("$path/$pkgHref"), null) + val classRows: Elements = packageDocument.select("table")[0].select("tr") + + for (j in 2 until classRows.size) { + val classHref = classRows[j].select("td")[0].select("a").attr("href") + val className = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0] + if (!classes.contains(className)) { + continue + } + + val classDocument: Document = Jsoup.parse(File("$path/${pkgHref.replace("index.html", classHref)}"), null) + val methodRows: Elements = classDocument.select("table")[0].select("tr") + + val coverageInfos = methodRows[1].select("td") + val (misInstructions, covInstructions) = coverageInfos[1].text()?.replace(",", "")?.split(" of ")?.let { + val misInstructions = it[0].toInt() + val allInstructions = it[1].toInt() + misInstructions to (allInstructions - misInstructions) + } ?: (0 to 0) + + val (misBranches, covBranches) = coverageInfos[3].text()?.replace(",", "")?.split(" of ")?.let { + val misBranches = it[0].toInt() + val allBranches = it[1].toInt() + misBranches to (allBranches - misBranches) + } ?: (0 to 0) + + val name = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0] + perClassResult[name] = Coverage(misInstructions, covInstructions, misBranches, covBranches) + + for (k in 2 until methodRows.size) { + + val cols = methodRows[k].select("td") + val methodHref = methodRows[k].select("td")[0].select("a").attr("href") + val methodName = classHref.replace("/index.html", "." + methodHref.replace("html", cols[0].text())) + + val instructions = cols[1].select("img") + val methodMisInstructions = instructions.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt() + ?: 0 + val methodCovInstructions = instructions.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt() + ?: 0 + + val branches = cols[3].select("img") + val methodMisBranches = branches.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0 + val methodCovBranches = branches.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0 + + if (methodMisInstructions == 0 && methodCovBranches == 0 && methodCovInstructions == 0 && methodMisBranches == 0) continue + perMethodResult[methodName] = Coverage(methodMisInstructions, methodCovInstructions, methodMisBranches, methodCovBranches) + } + } + } + + return perClassResult to perMethodResult +} + + +fun main() { + val htmlBuilder = HtmlBuilder() + + val classes = mutableSetOf() + File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } + + // Parse data + val jacocoCoverage = QualityAnalysisConfig.selectors.map { + it to parseJacocoReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}",classes).first + } + + // Instruction coverage report (sum coverages percentages / classNum) + val instructionMetrics = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.getInstructionCoverage() } + } + htmlBuilder.addHeader("Instructions coverage (sum coverages percentages / classNum)") + instructionMetrics.forEach { + htmlBuilder.addText("Mean(Instruction (${it.first} model)) =${it.second.sum() / it.second.size}") + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + instructionMetrics.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten().toTypedArray(), + instructionMetrics.map { it.second }.flatten().toDoubleArray(), + title = "Coverage", + xLabel = "Instructions", + yLabel = "Count" + ) + ) + + // Instruction coverage report (sum covered instructions / sum instructions) + htmlBuilder.addHeader("Instructions coverage(sum covered instructions / sum instructions)") + val covInstruction = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.covInstruction } + } + val instructions = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.getInstructions() } + }.toMap() + covInstruction.forEach { + htmlBuilder.addText("Mean(Instruction (${it.first} model)) =${it.second.sum().toDouble() / (instructions[it.first]?.sum()?.toDouble() ?: 0.0)}") + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + covInstruction.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten().toTypedArray(), + covInstruction.map { it.second }.flatten().map { it.toDouble() }.toDoubleArray(), + title = "Coverage", + xLabel = "Instructions", + yLabel = "Count" + ) + ) + + // Branches coverage report + val branchesMetrics = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.getBranchesCoverage() } + } + htmlBuilder.addHeader("Branches coverage") + branchesMetrics.forEach { + htmlBuilder.addText("Mean(Branches (${it.first} model)) =${it.second.sum() / it.second.size}") + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + branchesMetrics.map { it.second.map { it2 -> "Branches (${it.first} model)" } }.flatten().toTypedArray(), + branchesMetrics.map { it.second }.flatten().toDoubleArray(), + title = "Coverage", + xLabel = "Branches", + yLabel = "Count" + ) + ) + + // Save report + htmlBuilder.saveHTML(Paths.get( + QualityAnalysisConfig.outputDir, + QualityAnalysisConfig.project, + "test.html" + ).toFile().absolutePath) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt new file mode 100644 index 0000000000..9d3fae8e09 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt @@ -0,0 +1,20 @@ +package org.utbot + +import java.io.FileInputStream +import java.util.Properties + +object QualityAnalysisConfig { + const val configPath = "utbot-analytics/src/main/resources/config.properties" + private val properties = Properties().also { props -> + FileInputStream(configPath).use { inputStream -> + props.load(inputStream) + } + } + + val project: String = properties.getProperty("project") + val selectors: List = properties.getProperty("selectors").split(",") + val covStatistics: List = properties.getProperty("covStatistics").split(",") + + val outputDir: String = "eval/res" + val classesList: String = "utbot-junit-contest/src/main/resources/classes/${project}/list" +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt new file mode 100644 index 0000000000..abb75b9c19 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt @@ -0,0 +1,67 @@ +package org.utbot + +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.select.Elements +import java.io.File +import java.nio.file.Paths + + +fun parseReport(path: String, classes: Set): Map { + val result = mutableMapOf() + val contestDocument: Document = Jsoup.parse(File("$path\\index.html"), null) + val packageRows: Elements = contestDocument.select("table")[1].select("tr") + for (i in 1 until packageRows.size) { + val packageHref = packageRows[i].select("td")[0].select("a").attr("href") + val packageDocument: Document = Jsoup.parse(File("$path\\$packageHref"), null) + val clsRows: Elements = packageDocument.select("table")[1].select("tr") + + for (j in 1 until clsRows.size) { + val clsName = clsRows[j].select("td")[0].select("a").attr("href").replace(".classes/", "").replace(".html", "") + val fullName = packageHref.replace("/index.html", ".$clsName") + + if (classes.contains(fullName)) { + result.put(fullName, clsRows[j].select("td")[3].select("span")[0].text().replace("%", "").toDouble()) + } + } + } + + return result +} + + +fun main() { + val htmlBuilder = HtmlBuilder() + + val classes = mutableSetOf() + File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } + + // Parse data + val jacocoCoverage = QualityAnalysisConfig.selectors.map { + it to parseReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}", classes) + } + + // Coverage report + htmlBuilder.addHeader("Line coverage") + jacocoCoverage.forEach { + htmlBuilder.addText("Mean(Line (${it.first} model)) =" + it.second.map { it.value }.sum() / it.second.size) + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + jacocoCoverage.map { it.second.map { it2 -> "Line (${it.first} model)" } }.flatten().toTypedArray(), + jacocoCoverage.map { it.second.map { it.value } }.flatten().toDoubleArray(), + title = "Coverage", + xLabel = "Instructions", + yLabel = "Count" + ) + ) + + // Save report + htmlBuilder.saveHTML(Paths.get( + QualityAnalysisConfig.outputDir, + QualityAnalysisConfig.project, + QualityAnalysisConfig.selectors.joinToString("_") + ).toFile().absolutePath) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt new file mode 100644 index 0000000000..a43bf2604a --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt @@ -0,0 +1,172 @@ +package org.utbot + +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder +import java.io.File +import java.nio.file.Paths +import kotlin.random.Random +import org.apache.commons.io.FileUtils +import smile.read + + +data class Statistics( + val perMethod: Map>, + val perClass: Map>, + val perProject: List +) + + +fun getStatistics(path: String, classes: Set): Statistics { + var projectTotalStmts = 0.0 + var projectMinStartTime = Double.MAX_VALUE + + val rawStatistics = File(path).listFiles()?.filter { it.extension == "txt" && it.readLines().size > 1 }?.map { + val data = read.csv(it.absolutePath) + it.nameWithoutExtension to data.toArray() + }?.toMap() ?: emptyMap() + + val statisticsPerMethod = rawStatistics.map { methodData -> + val startTime = methodData.value[0][0] + val value = methodData.value.map { + doubleArrayOf(it[0] - startTime, it[1] / it[2]) // time, percent + }.sortedBy { it[0] } + + methodData.key to value + }.toMap() + + val statisticsPerClass = classes.map { cls -> + var minStartTime = Double.MAX_VALUE + var totalStmts = 0.0 + val filteredStatistics = rawStatistics.filter { it.key.contains(cls) } + + filteredStatistics.forEach { + minStartTime = minOf(minStartTime, it.value[0][0]) + totalStmts += it.value.getOrNull(1)?.get(2) ?: 0.0 + } + projectTotalStmts += totalStmts + projectMinStartTime = minOf(minStartTime, projectMinStartTime) + + var prevCov = 0.0 + cls to filteredStatistics.toList().sortedBy { it.second[0][0] }.flatMap { methodData -> + val value = methodData.second.map { + doubleArrayOf(it[0] - minStartTime, (it[1] + prevCov) / totalStmts) + } + + prevCov += methodData.second.last()[1] + value + }.sortedBy { it[0] } + }.toMap() + + var prevCov = 0.0 + val statisticsPerProject = rawStatistics + .filter { classes.any { it2 -> it.key.contains(it2) } } + .toList() + .sortedBy { it.second[0][0] } + .flatMap { methodData -> + val value = methodData.second.map { + doubleArrayOf(it[0] - projectMinStartTime, (it[1] + prevCov) / projectTotalStmts) // time, percent + } + + prevCov += methodData.second.last()[1] + value + }.sortedBy { it[0] } + + return Statistics(statisticsPerMethod, statisticsPerClass, statisticsPerProject) +} + +fun main() { + // Processed classes + val classes = mutableSetOf() + File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } + + // Prepare folder + val projectDataPath = "${QualityAnalysisConfig.outputDir}/report/${QualityAnalysisConfig.project}/${QualityAnalysisConfig.selectors.joinToString("_")}" + File(projectDataPath).deleteRecursively() + classes.forEach { File(projectDataPath, it).mkdirs() } + File(projectDataPath, "css").mkdirs() + listOf("css/coverage.css", "css/highlight-idea.css").forEach { + FileUtils.copyInputStreamToFile( + Statistics::class.java.classLoader.getResourceAsStream(it), + Paths.get(projectDataPath, it).toFile() + ) + } + + // Parse statistics + val selectors = QualityAnalysisConfig.selectors + val statistics = QualityAnalysisConfig.covStatistics.map { getStatistics(it, classes) } + val colors = QualityAnalysisConfig.selectors.map { + val rnd = Random.Default + "rgb(${rnd.nextInt(256)}, ${rnd.nextInt(256)}, ${rnd.nextInt(256)})" + } + + val htmlBuilderForProject = HtmlBuilder(pathToStyle = listOf("css/coverage.css", "css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js")) + htmlBuilderForProject.addTable( + statistics.map { it.perClass.map { it.key to (it.value.lastOrNull()?.lastOrNull() ?: 0.0) * 100 }.toMap() }, + selectors, + colors, + "class" + ) + htmlBuilderForProject.addFigure( + FigureBuilders.buildSeveralLinesPlot( + statistics.map { it.perProject.map { it[0] / 1e9 }.toDoubleArray() }, + statistics.map { it.perProject.map { it[1] * 100 }.toDoubleArray() }, + colors, + selectors, + xLabel = "Time (sec)", + yLabel = "Coverage (%)", + title = QualityAnalysisConfig.project + ) + ) + htmlBuilderForProject.saveHTML(File(projectDataPath, "index.html").toString()) + + + classes.forEach { cls -> + val outputDir = File(projectDataPath, cls) + val htmlBuilderForCls = HtmlBuilder(pathToStyle = listOf("../css/coverage.css", "../css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js")) + val filteredStatistics = statistics.map { + it.perMethod.filter { it.key.contains(cls) } + } + + htmlBuilderForCls.addTable( + filteredStatistics.map { it.map { it.key to it.value.last().last() * 100 }.toMap() }, + selectors, + colors, + "method" + ) + htmlBuilderForCls.addFigure( + FigureBuilders.buildSeveralLinesPlot( + statistics.map { it.perClass[cls]?.map { it[0] / 1e9 }?.toDoubleArray() ?: doubleArrayOf() }, + statistics.map { it.perClass[cls]?.map { it[1] * 100 }?.toDoubleArray() ?: doubleArrayOf() }, + colors, + selectors, + xLabel = "Time (sec)", + yLabel = "Coverage (%)", + title = cls + ) + ) + + File(outputDir.toString()).mkdir() + htmlBuilderForCls.saveHTML(File(outputDir, "index.html").toString()) + + filteredStatistics.first().keys.forEach { method -> + val htmlBuilderForMethod = HtmlBuilder(pathToStyle = listOf("../../css/coverage.css", "../../css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js")) + htmlBuilderForMethod.addFigure( + FigureBuilders.buildSeveralLinesPlot( + filteredStatistics.map { it[method]?.map { it[0] / 1e9 }?.toDoubleArray() ?: doubleArrayOf() }, + filteredStatistics.map { it[method]?.map { it[1] * 100 }?.toDoubleArray() ?: doubleArrayOf() }, + colors, + selectors, + xLabel = "Time (sec)", + yLabel = "Coverage (%)", + title = method + ) + ) + + File(outputDir, method).mkdirs() + htmlBuilderForMethod.saveHTML(File(outputDir, "${method}/index.html").toString()) + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorFactoryImpl.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorFactoryImpl.kt new file mode 100644 index 0000000000..9def22a3dc --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorFactoryImpl.kt @@ -0,0 +1,12 @@ +package org.utbot.features + +import org.utbot.analytics.FeatureExtractor +import org.utbot.analytics.FeatureExtractorFactory +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Implementation of feature extractor factory + */ +class FeatureExtractorFactoryImpl : FeatureExtractorFactory { + override operator fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor = FeatureExtractorImpl(graph) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt new file mode 100644 index 0000000000..cd9362aa6f --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt @@ -0,0 +1,42 @@ +package org.utbot.features + +import org.utbot.analytics.FeatureExtractor +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.StatementsStatistics +import org.utbot.engine.selectors.strategies.SubpathStatistics + +/** + * Implementation of feature extractor. + * Extract features for state and stores it in features vector of this state. + */ +class FeatureExtractorImpl(private val graph: InterProceduralUnitGraph) : FeatureExtractor { + companion object { + private val subpathGuidedSelectorIndexes = listOf(0, 1, 2, 3) + } + + private fun MutableList.add(value: T) = add(value.toDouble()) + private fun > MutableList.add(value: T) = add(value.size) + + private val subpathStatistics = subpathGuidedSelectorIndexes.map { SubpathStatistics(graph, it) } + private val statementStatistics = StatementsStatistics(graph) + + override fun extractFeatures(executionState: ExecutionState, generatedTestCases: Int) { + if (executionState.features.isNotEmpty()) { + executionState.features.clear() + } + + executionState.features.add(executionState.executionStack) // stack + executionState.features.add(graph.succs(executionState.stmt)) // successor + executionState.features.add(generatedTestCases) // testCase + executionState.features.add(executionState.visitedAfterLastFork) // coverage by branch + executionState.features.add(executionState.visitedBeforeLastFork + executionState.visitedAfterLastFork) // coverage by path + executionState.features.add(executionState.depth) // depth + executionState.features.add(statementStatistics.statementInMethodCount(executionState)) // cpicnt + executionState.features.add(statementStatistics.statementCount(executionState)) // icnt + executionState.features.add(executionState.stmtsSinceLastCovered) // covNew + subpathStatistics.forEach { + executionState.features.add(it.subpathCount(executionState)) // sgs_i + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt new file mode 100644 index 0000000000..c78aa36d14 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt @@ -0,0 +1,153 @@ +package org.utbot.features + +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.FeatureProcessor +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.framework.UtSettings +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Paths +import kotlin.math.pow +import soot.jimple.Stmt + +/** + * Implementation of feature processor, in which we dump each test, so there will be several copies of each state. + * Extract features for state when this state will be marked visited in graph. + * Add test case when last state of test case will be traversed. + */ +class FeatureProcessorWithStatesRepetition( + graph: InterProceduralUnitGraph, + private val saveDir: String = UtSettings.featurePath +) : FeatureProcessor(graph) { + init { + File(saveDir).mkdirs() + } + companion object { + private val featureKeys = arrayOf( + "stack", + "successor", + "testCase", + "coverageByBranch", + "coverageByPath", + "depth", + "cpicnt", + "icnt", + "covNew", + "subpath1", + "subpath2", + "subpath4", + "subpath8" + ) + } + + private var generatedTestCases = 0 + private val featureExtractor = EngineAnalyticsContext.featureExtractorFactory(graph) + private val rewardEstimator = RewardEstimator() + + private val dumpedStates = mutableMapOf>() + private val visitedStmts = mutableSetOf() + private val testCases = mutableListOf() + + private fun extractFeatures(executionState: ExecutionState) { + featureExtractor.extractFeatures(executionState, generatedTestCases) + } + + private fun addTestCase(executionState: ExecutionState) { + val states = mutableListOf>() + var newCoverage = 0 + var currentState: ExecutionState? = executionState + + do { + val stateHashCode = currentState?.hashCode() ?: 0 + if (currentState?.features?.isEmpty() == true) { + extractFeatures(currentState) + } + currentState?.executingTime?.let { states += (stateHashCode to it) } + currentState?.features?.let { dumpedStates[stateHashCode] = it } + + currentState?.stmt?.let { + if (it !in visitedStmts && currentState?.isInNestedMethod() == false) { + visitedStmts.add(it) + newCoverage++ + } + } + + currentState = currentState?.parent + } while (currentState != null) + + generatedTestCases++ + testCases += TestCase(states, newCoverage, generatedTestCases) + } + + override fun dumpFeatures() { + val rewards = rewardEstimator.calculateRewards(testCases) + + testCases.forEach { ts -> + FileOutputStream( + Paths.get(saveDir, "${UtSettings.testCounter++}.csv").toFile(), + true + ).bufferedWriter().use { out -> + out.appendLine("newCov,reward,${featureKeys.joinToString(separator = ",")}") + val reversedStates = ts.states.asReversed() + + reversedStates.forEach { (state, _) -> + out.appendLine( + "${ts.newCoverage != 0},${rewards[state]}," + + "${dumpedStates[state]?.joinToString(separator = ",")}" + ) + } + + out.flush() + } + } + } + + override fun onTraversed(executionState: ExecutionState) { + addTestCase(executionState) + } + + override fun onVisit(executionState: ExecutionState) { + extractFeatures(executionState) + } +} + +internal class RewardEstimator { + + fun calculateRewards(testCases: List): Map { + val rewards = mutableMapOf() + val coverages = mutableMapOf() + val times = mutableMapOf() + + testCases.forEach { ts -> + var allTime = 0L + ts.states.forEach { (state, time) -> + coverages.compute(state) { _, v -> + ts.newCoverage + (v ?: 0) + } + val updateTime = !times.contains(state) + times.compute(state) { _, v -> + allTime + (v ?: time) + } + if (updateTime) { + allTime += time + } + } + } + + coverages.forEach { (state, coverage) -> + rewards[state] = reward(coverage.toDouble(), times[state]!!.toDouble()) + } + + return rewards + } + + companion object { + private const val minTime = 1.0 + private const val rewardDegree = 0.5 + + fun reward(coverage: Double, time: Double): Double = (coverage / maxOf(time, minTime)).pow(rewardDegree) + } +} + +data class TestCase(val states: List>, val newCoverage: Int, val testIndex: Int) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt new file mode 100644 index 0000000000..e8b479a7ae --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt @@ -0,0 +1,12 @@ +package org.utbot.features + +import org.utbot.analytics.FeatureProcessor +import org.utbot.analytics.FeatureProcessorFactory +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Implementation of feature processor factory, which creates FeatureProcessorWithStatesRepetition + */ +class FeatureProcessorWithStatesRepetitionFactory : FeatureProcessorFactory { + override fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor = FeatureProcessorWithStatesRepetition(graph) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt new file mode 100644 index 0000000000..76c3ee95b5 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt @@ -0,0 +1,586 @@ +package org.utbot.features + +import org.utbot.utils.BinaryUtil +import org.utbot.engine.pc.* +import java.util.* + + +class UtExpressionId: UtExpressionVisitor { + + fun getID(expr: UtExpression): Int = expr.accept(this) + + override fun visit(expr: UtArraySelectExpression): Int = 1 + override fun visit(expr: UtMkArrayExpression): Int = 2 + override fun visit(expr: UtArrayMultiStoreExpression): Int = 3 + override fun visit(expr: UtBvLiteral): Int = 4 + override fun visit(expr: UtBvConst): Int = 5 + override fun visit(expr: UtAddrExpression): Int = 6 + override fun visit(expr: UtFpLiteral): Int = 7 + override fun visit(expr: UtFpConst): Int = 8 + override fun visit(expr: UtOpExpression): Int = 9 + override fun visit(expr: UtTrue): Int = 10 + override fun visit(expr: UtFalse): Int = 11 + override fun visit(expr: UtEqExpression): Int = 12 + override fun visit(expr: UtBoolConst): Int = 13 + override fun visit(expr: NotBoolExpression): Int = 14 + override fun visit(expr: UtOrBoolExpression): Int = 15 + override fun visit(expr: UtAndBoolExpression): Int = 16 + override fun visit(expr: UtNegExpression): Int = 17 + override fun visit(expr: UtCastExpression): Int = 18 + override fun visit(expr: UtBoolOpExpression): Int = 19 + override fun visit(expr: UtIsExpression): Int = 20 + override fun visit(expr: UtIteExpression): Int = 21 + override fun visit(expr: UtMkTermArrayExpression): Int = 22 + override fun visit(expr: UtConcatExpression): Int = 23 + override fun visit(expr: UtConvertToString): Int = 24 + override fun visit(expr: UtStringLength): Int = 25 + override fun visit(expr: UtStringPositiveLength): Int = 26 + override fun visit(expr: UtStringCharAt): Int = 27 + override fun visit(expr: UtStringEq): Int = 28 + override fun visit(expr: UtSubstringExpression): Int = 29 + override fun visit(expr: UtReplaceExpression): Int = 30 + override fun visit(expr: UtStartsWithExpression): Int = 31 + override fun visit(expr: UtEndsWithExpression): Int = 32 + override fun visit(expr: UtIndexOfExpression): Int = 33 + override fun visit(expr: UtContainsExpression): Int = 34 + override fun visit(expr: UtToStringExpression): Int = 35 + override fun visit(expr: UtSeqLiteral): Int = 36 + override fun visit(expr: UtConstArrayExpression): Int = 37 + override fun visit(expr: UtGenericExpression): Int = 38 + override fun visit(expr: UtIsGenericTypeExpression): Int = 39 + override fun visit(expr: UtEqGenericTypeParametersExpression): Int = 40 + override fun visit(expr: UtInstanceOfExpression): Int = 41 + override fun visit(expr: UtStringConst): Int = 42 + override fun visit(expr: UtStringToInt): Int = 43 + override fun visit(expr: UtArrayToString): Int = 44 + override fun visit(expr: UtArrayInsert): Int = 45 + override fun visit(expr: UtArrayInsertRange): Int = 46 + override fun visit(expr: UtArrayRemove): Int = 47 + override fun visit(expr: UtArrayRemoveRange): Int = 48 + override fun visit(expr: UtArraySetRange): Int = 49 + override fun visit(expr: UtArrayShiftIndexes): Int = 50 + override fun visit(expr: UtArrayApplyForAll): Int = 51 + override fun visit(expr: UtStringToArray): Int = 52 + override fun visit(expr: UtAddNoOverflowExpression): Int = 53 + override fun visit(expr: UtSubNoOverflowExpression): Int = 54 +} + + +@Suppress("DuplicatedCode") +class UtExpressionGraphExtraction : UtExpressionVisitor { + + private val lineSeparator = System.lineSeparator() + val expressionIdsCache = IdentityHashMap() + private val expressionSubGraphCache = IdentityHashMap() + val literalValues = IdentityHashMap() + + fun getID(expr: UtExpression): String = expressionIdsCache.getOrPut(expr) { UUID.randomUUID().toString() } + + private fun extractSubgraph(expr: UtExpression): String { + return if (expressionSubGraphCache.containsKey(expr)) { + "" + } else { + val res = expr.accept(this) + expressionSubGraphCache[expr] = res + + res + } + } + + fun extractGraph(input: Iterable): String { + return input.fold("FROM,TO$lineSeparator") { acc, expr -> acc + extractSubgraph(expr) } + } + + override fun visit(expr: UtArraySelectExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + extractSubgraph(expr.index) + + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.index) + lineSeparator + + return res + } + + override fun visit(expr: UtMkArrayExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + return "" + } + + override fun visit(expr: UtArrayMultiStoreExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.initial) + expr.stores.forEach { res += extractSubgraph(it.value) + extractSubgraph(it.index) } + + res += id + "," + getID(expr.initial) + lineSeparator + expr.stores.forEach { res += id + "," + getID(it.value) + lineSeparator } + expr.stores.forEach { res += id + "," + getID(it.index) + lineSeparator } + + return res + } + + override fun visit(expr: UtBvLiteral): String { + literalValues[expr] = BinaryUtil.binaryValueString(expr.value.toInt()) + return "" + } + + override fun visit(expr: UtBvConst): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + return "" + } + + override fun visit(expr: UtAddrExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.internal) + res += id + "," + getID(expr.internal) + lineSeparator + + return res + } + + override fun visit(expr: UtFpLiteral): String { + literalValues[expr] = BinaryUtil.binaryValueString(expr.value.toInt()) + return "" + } + + override fun visit(expr: UtFpConst): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + return "" + } + + override fun visit(expr: UtOpExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.left.expr) + extractSubgraph(expr.right.expr) + res += id + "," + getID(expr.left.expr) + lineSeparator + res += id + "," + getID(expr.right.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtTrue): String { + literalValues[expr] = BinaryUtil.binaryValueString(1) + return "" + } + + override fun visit(expr: UtFalse): String { + literalValues[expr] = BinaryUtil.binaryValueString(0) + return "" + } + + override fun visit(expr: UtEqExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.left) + extractSubgraph(expr.right) + res += id + "," + getID(expr.left) + lineSeparator + res += id + "," + getID(expr.right) + lineSeparator + + return res + } + + override fun visit(expr: UtBoolConst): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + return "" + } + + override fun visit(expr: NotBoolExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.expr) + res += id + "," + getID(expr.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtOrBoolExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = "" + expr.exprs.forEach { res += extractSubgraph(it) } + expr.exprs.forEach { res += id + "," + getID(it) + lineSeparator } + + return res + } + + override fun visit(expr: UtAndBoolExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = "" + expr.exprs.forEach { res += extractSubgraph(it) } + expr.exprs.forEach { res += id + "," + getID(it) + lineSeparator } + + return res + } + + override fun visit(expr: UtNegExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.variable.expr) + res += id + "," + getID(expr.variable.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtCastExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.variable.expr) + res += id + "," + getID(expr.variable.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtBoolOpExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.left.expr) + extractSubgraph(expr.right.expr) + res += id + "," + getID(expr.left.expr) + lineSeparator + res += id + "," + getID(expr.right.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtIsExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.addr) + res += id + "," + getID(expr.addr) + lineSeparator + + return res + } + + override fun visit(expr: UtIteExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.condition) + extractSubgraph(expr.thenExpr) + extractSubgraph(expr.elseExpr) + res += id + "," + getID(expr.condition) + lineSeparator + res += id + "," + getID(expr.thenExpr) + lineSeparator + res += id + "," + getID(expr.elseExpr) + lineSeparator + + return res + } + + override fun visit(expr: UtConcatExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = "" + expr.parts.forEach { res += extractSubgraph(it) } + expr.parts.forEach { res += id + "," + getID(it) + lineSeparator } + + return res + } + + override fun visit(expr: UtConvertToString): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.expression) + res += id + "," + getID(expr.expression) + lineSeparator + + return res + } + + override fun visit(expr: UtStringLength): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + res += id + "," + getID(expr.string) + lineSeparator + + return res + } + + override fun visit(expr: UtStringPositiveLength): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + res += id + "," + getID(expr.string) + lineSeparator + + return res + } + + override fun visit(expr: UtStringCharAt): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + extractSubgraph(expr.index) + res += id + "," + getID(expr.string) + lineSeparator + res += id + "," + getID(expr.index) + lineSeparator + + return res + } + + override fun visit(expr: UtStringEq): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.left) + extractSubgraph(expr.right) + res += id + "," + getID(expr.left) + lineSeparator + res += id + "," + getID(expr.right) + lineSeparator + + return res + } + + override fun visit(expr: UtSubstringExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + extractSubgraph(expr.beginIndex) + extractSubgraph(expr.length) + res += id + "," + getID(expr.string) + lineSeparator + res += id + "," + getID(expr.beginIndex) + lineSeparator + res += id + "," + getID(expr.length) + lineSeparator + + return res + } + + override fun visit(expr: UtReplaceExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + extractSubgraph(expr.regex) + extractSubgraph(expr.replacement) + res += id + "," + getID(expr.string) + lineSeparator + res += id + "," + getID(expr.regex) + lineSeparator + res += id + "," + getID(expr.replacement) + lineSeparator + + return res + } + + override fun visit(expr: UtStartsWithExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + extractSubgraph(expr.prefix) + res += id + "," + getID(expr.string) + lineSeparator + res += id + "," + getID(expr.prefix) + lineSeparator + + return res + } + + override fun visit(expr: UtEndsWithExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + extractSubgraph(expr.suffix) + res += id + "," + getID(expr.string) + lineSeparator + res += id + "," + getID(expr.suffix) + lineSeparator + + return res + } + + override fun visit(expr: UtIndexOfExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + extractSubgraph(expr.substring) + res += id + "," + getID(expr.string) + lineSeparator + res += id + "," + getID(expr.substring) + lineSeparator + + return res + } + + override fun visit(expr: UtContainsExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.string) + extractSubgraph(expr.substring) + res += id + "," + getID(expr.string) + lineSeparator + res += id + "," + getID(expr.substring) + lineSeparator + + return res + } + + override fun visit(expr: UtToStringExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + val id = getID(expr) + var res = extractSubgraph(expr.notNullExpr) + extractSubgraph(expr.isNull) + res += id + "," + getID(expr.notNullExpr) + lineSeparator + res += id + "," + getID(expr.isNull) + lineSeparator + + return res + } + + override fun visit(expr: UtSeqLiteral): String { + literalValues[expr] = BinaryUtil.binaryValueString(expr.value.toInt()) + return "" + } + + override fun visit(expr: UtMkTermArrayExpression): String { + literalValues[expr] = BinaryUtil.binaryValueStringEmpty() + return "" + } + + override fun visit(expr: UtConstArrayExpression): String { + val id = getID(expr) + var res = extractSubgraph(expr.constValue) + res += id + "," + getID(expr.constValue) + lineSeparator + + return res + } + + override fun visit(expr: UtGenericExpression): String { + val id = getID(expr) + var res = extractSubgraph(expr.addr) + res += id + "," + getID(expr.addr) + lineSeparator + + return res + } + + override fun visit(expr: UtIsGenericTypeExpression): String { + val id = getID(expr) + var res = extractSubgraph(expr.addr) + res += extractSubgraph(expr.baseAddr) + res += id + "," + getID(expr.addr) + lineSeparator + res += id + "," + getID(expr.baseAddr) + lineSeparator + + return res + } + + override fun visit(expr: UtEqGenericTypeParametersExpression): String { + val id = getID(expr) + var res = extractSubgraph(expr.firstAddr) + res += extractSubgraph(expr.secondAddr) + res += id + "," + getID(expr.firstAddr) + lineSeparator + res += id + "," + getID(expr.secondAddr) + lineSeparator + + return res + } + + override fun visit(expr: UtInstanceOfExpression): String { + val id = getID(expr) + var res = extractSubgraph(expr.constraint) + res += id + "," + getID(expr.constraint) + lineSeparator + + return res + } + + override fun visit(expr: UtStringConst): String { + // TODO + return "" + } + + override fun visit(expr: UtStringToInt): String { + val id = getID(expr) + var res = extractSubgraph(expr.expression) + res += id + "," + getID(expr.expression) + lineSeparator + + return res + } + + override fun visit(expr: UtArrayToString): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += extractSubgraph(expr.offset.expr) + res += extractSubgraph(expr.length.expr) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.offset.expr) + lineSeparator + res += id + "," + getID(expr.length.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtArrayInsert): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += extractSubgraph(expr.element) + res += extractSubgraph(expr.index.expr) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.element) + lineSeparator + res += id + "," + getID(expr.index.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtArrayInsertRange): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += extractSubgraph(expr.elements) + res += extractSubgraph(expr.index.expr) + res += extractSubgraph(expr.from.expr) + res += extractSubgraph(expr.length.expr) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.elements) + lineSeparator + res += id + "," + getID(expr.index.expr) + lineSeparator + res += id + "," + getID(expr.from.expr) + lineSeparator + res += id + "," + getID(expr.length.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtArrayRemove): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += extractSubgraph(expr.index.expr) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.index.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtArrayRemoveRange): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += extractSubgraph(expr.index.expr) + res += extractSubgraph(expr.length.expr) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.index.expr) + lineSeparator + res += id + "," + getID(expr.length.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtArraySetRange): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += extractSubgraph(expr.elements) + res += extractSubgraph(expr.index.expr) + res += extractSubgraph(expr.from.expr) + res += extractSubgraph(expr.length.expr) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.elements) + lineSeparator + res += id + "," + getID(expr.index.expr) + lineSeparator + res += id + "," + getID(expr.from.expr) + lineSeparator + res += id + "," + getID(expr.length.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtArrayShiftIndexes): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += extractSubgraph(expr.offset.expr) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + res += id + "," + getID(expr.offset.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtArrayApplyForAll): String { + val id = getID(expr) + var res = extractSubgraph(expr.arrayExpression) + res += id + "," + getID(expr.arrayExpression) + lineSeparator + + return res + // TODO ?? + } + + override fun visit(expr: UtStringToArray): String { + val id = getID(expr) + var res = extractSubgraph(expr.stringExpression) + res += extractSubgraph(expr.offset.expr) + res += id + "," + getID(expr.stringExpression) + lineSeparator + res += id + "," + getID(expr.offset.expr) + lineSeparator + + return res + } + + override fun visit(expr: UtAddNoOverflowExpression): String { + val id = getID(expr) + var res = extractSubgraph(expr.left) + res += extractSubgraph(expr.right) + res += id + "," + getID(expr.left) + lineSeparator + res += id + "," + getID(expr.right) + lineSeparator + + return res + } + + override fun visit(expr: UtSubNoOverflowExpression): String { + val id = getID(expr) + var res = extractSubgraph(expr.left) + res += extractSubgraph(expr.right) + res += id + "," + getID(expr.left) + lineSeparator + res += id + "," + getID(expr.right) + lineSeparator + + return res + } +} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt new file mode 100644 index 0000000000..c75f22809d --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -0,0 +1,25 @@ +package org.utbot.predictors + +import org.utbot.analytics.UtBotAbstractPredictor +import org.utbot.framework.UtSettings +import java.io.File +import smile.math.matrix.Matrix + +/** + * Last weight is bias + */ +private fun loadWeights(path: String): Matrix { + return Matrix(File("${UtSettings.rewardModelPath}/${path}").readText().split(",").map(String::toDouble).toDoubleArray()) +} + +class LinearStateRewardPredictor : UtBotAbstractPredictor>, List> { + val weights = loadWeights("linear.txt") + + override fun predict(input: List>): List { + // add 1 to each feature vector + val X = Matrix(input.map { it.toMutableList().also { featureVector -> + featureVector.add(1.0) + }.toDoubleArray() }.toTypedArray()) + return X.mm(weights).col(0).toList() + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt new file mode 100644 index 0000000000..3b58d307c7 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt @@ -0,0 +1,5 @@ +package org.utbot.predictors + +import org.utbot.analytics.UtBotAbstractPredictor + +interface NNStateRewardPredictor : UtBotAbstractPredictor, Double> \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt new file mode 100644 index 0000000000..70159df951 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt @@ -0,0 +1,73 @@ +package org.utbot.predictors + +import com.google.gson.Gson +import org.utbot.framework.UtSettings +import java.io.FileReader +import java.nio.file.Paths +import kotlin.math.max +import smile.math.matrix.Matrix + +data class NNJson( + val linearLayers: Array> = arrayOf(), + val activationLayers: Array = arrayOf(), + val biases: Array = arrayOf() +) + +data class NN(val operations: List<(DoubleArray) -> DoubleArray>) + +private fun reLU(input: DoubleArray): DoubleArray { + return input.map { max(0.0, it) }.toDoubleArray() +} + +private fun loadNN(path: String): NN { + val nnJson: NNJson = Gson().fromJson(FileReader(Paths.get(UtSettings.rewardModelPath, path).toFile()), NNJson::class.java) ?: run { + System.err.println("Something went wrong while parsing NN model") + NNJson() + } + + val weights = nnJson.linearLayers.map { Matrix(it) } + val biases = nnJson.biases.map { Matrix(it) } + val operations = mutableListOf<(DoubleArray) -> DoubleArray>() + + (0 until nnJson.linearLayers.size).forEach { i -> + operations.add { + weights[i].mm(Matrix(it)).add(biases[i]).col(0) + } + if (i != nnJson.linearLayers.size - 1) { + operations.add { + when (nnJson.activationLayers[i]) { + "reLU" -> reLU(it) + else -> error("Unsupported activation") + } + } + } + } + + return NN(operations) +} + +data class StandardScaler(val mean: Matrix?, val variance: Matrix?) + +internal fun loadScaler(path: String): StandardScaler = + Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { + val mean = it.readLine()?.split(',')?.map(String::toDouble)?.toDoubleArray() + val variance = it.readLine()?.split(',')?.map(String::toDouble)?.toDoubleArray() + StandardScaler(Matrix(mean), Matrix(variance)) + } + +class NNStateRewardPredictorSmile(modelPath: String = "nn.json", scalerPath: String = "scaler.txt") : NNStateRewardPredictor { + val nn = loadNN(modelPath) + val scaler = loadScaler(scalerPath) + + override fun predict(input: List): Double { + var inputArray = input.toDoubleArray() + inputArray = Matrix(inputArray).sub(scaler.mean).div(scaler.variance).col(0) + + nn.operations.forEach { + inputArray = it(inputArray) + } + + check(inputArray.size == 1) + return inputArray[0] + } +} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt new file mode 100644 index 0000000000..3684913c5b --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt @@ -0,0 +1,37 @@ +package org.utbot.predictors + +import org.utbot.framework.UtSettings +import java.nio.file.Paths +import ai.djl.* +import ai.djl.inference.*; +import ai.djl.ndarray.*; +import ai.djl.translate.*; +import java.io.Closeable + +class NNStateRewardPredictorTorch : NNStateRewardPredictor, Closeable { + val model: ai.djl.Model = ai.djl.Model.newInstance("model") + init { + model.load(Paths.get(UtSettings.rewardModelPath, "model.pt1")) + } + val predictor: Predictor, Float> = model.newPredictor(object : Translator, Float> { + override fun processInput(ctx: TranslatorContext, input: List): NDList { + val manager: NDManager = ctx.getNDManager() + val array: NDArray = manager.create(input.toFloatArray()) + return NDList(array) + } + + override fun processOutput(ctx: TranslatorContext, list: NDList): Float { + val tmp: NDArray = list.get(0) + return tmp.getFloat() + } + }) + + override fun predict(input: List): Double { + val reward: Float = predictor.predict(input.map { it.toFloat() }.toList()) + return reward.toDouble() + } + + override fun close() { + predictor.close() + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt index d99f506016..d005dcdf71 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt @@ -1,15 +1,52 @@ package org.utbot.predictors import org.utbot.analytics.IUtBotSatPredictor +import org.utbot.analytics.NeuroSatData import org.utbot.analytics.UtBotAbstractPredictor import org.utbot.engine.pc.UtExpression import org.utbot.engine.pc.UtSolverStatusKind +import org.utbot.features.UtExpressionGraphExtraction +import org.utbot.features.UtExpressionId +import org.utbot.utils.BinaryUtil +import java.io.File +import java.io.FileOutputStream @Suppress("unused") -class UtBotSatPredictor : UtBotAbstractPredictor, UtSolverStatusKind>, +class UtBotSatPredictor : UtBotAbstractPredictor, NeuroSatData>, IUtBotSatPredictor> { - override fun provide(input: Iterable, expectedResult: UtSolverStatusKind, actualResult: UtSolverStatusKind) { + private var counter = 1 + private var folder = "logs/GRAPHS" + + init { + File("${folder}/graphs").mkdirs() + File("${folder}/SAT.txt").printWriter().use { out -> + out.println("ID,SAT,TIME") + } + } + + override fun provide(input: Iterable, expectedResult: NeuroSatData, actualResult: NeuroSatData) { + val extractor = UtExpressionGraphExtraction() + File("${folder}/graphs/$counter").mkdir() + File("${folder}/graphs/$counter/edges.txt").printWriter().use { out -> + out.print(extractor.extractGraph(input)) + } + File("${folder}/graphs/$counter/vertexesType.txt").printWriter().use { out -> + out.println("UUID,TYPE") + out.print(extractor.expressionIdsCache.keys.map { "${extractor.getID(it)},${ + BinaryUtil.binaryExpressionString(UtExpressionId().getID(it))}" }.joinToString("\n")) + } + File("${folder}/graphs/$counter/vertexesValues.txt").printWriter().use { out -> + out.println("UUID,VALUE") + out.print(extractor.literalValues.keys.map { "${extractor.getID(it)},${extractor.literalValues[it]}" }.joinToString("\n")) + } + + FileOutputStream("${folder}/SAT.txt", true).bufferedWriter().use { out -> + out.write("$counter,${actualResult.status},${actualResult.time}") + out.newLine() + } + + counter++ } override fun terminate() { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt new file mode 100644 index 0000000000..b3d058d652 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt @@ -0,0 +1,29 @@ +package org.utbot.utils + + +object BinaryUtil { + fun binaryExpression(value: Int): DoubleArray { + val binaryStr = binaryValueString(value, "%6s") + return binaryStr.chars().mapToDouble { c: Int -> (c - 48).toDouble() }.toArray() + } + + fun binaryExpressionString(value: Int): String { + return binaryValueString(value, "%6s") + } + + fun binaryValue(value: Int): DoubleArray { + val binaryStr = binaryValueString(value) + return binaryStr.chars().mapToDouble { c: Int -> (c - 48).toDouble() }.toArray() + } + + fun binaryValueString(value: Int): String = binaryValueString(value, "%32s") + "0" + + fun binaryValueEmpty(): DoubleArray = DoubleArray(33) { 0.0 }.apply { this[32] = 1.0 } + + fun binaryValueStringEmpty(): String = "0".repeat(32) + "1" + + private fun binaryValueString(value: Int, format: String): String { + val result = Integer.toBinaryString(value) + return String.format(format, result).replace(" ".toRegex(), "0") + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt new file mode 100644 index 0000000000..3951d983df --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt @@ -0,0 +1,129 @@ +package org.utbot.utils + +import java.io.File +import java.io.FileNotFoundException +import java.util.ArrayList +import java.util.Scanner + + +object MatrixUtil { + + + fun loadMatrix(file: String, swapAxis: Boolean): Array { + val scanner = Scanner(File(file)) + val shape = scanner.nextLine().split(",").toTypedArray() + val firstDim = shape[0].toInt() + val secondDim = shape[1].toInt() + val matrix: Array + matrix = if (swapAxis) { + Array(secondDim) { DoubleArray(firstDim) } + } else { + Array(firstDim) { DoubleArray(secondDim) } + } + for (i in 0 until firstDim) { + val row = scanner.nextLine().split(",").toTypedArray() + for (j in 0 until secondDim) { + if (swapAxis) { + matrix[j][i] = row[j].toDouble() + } else { + matrix[i][j] = row[j].toDouble() + } + } + } + return matrix + } + + /** + * Load vector from file + * @param file - file with vector + * @return vector + */ + @Throws(FileNotFoundException::class) + fun loadVector(file: String): DoubleArray { + val scanner = Scanner(File(file)) + val dim = scanner.nextLine().toInt() + val vector = DoubleArray(dim) + vector.forEachIndexed { i, _ -> vector[i] = scanner.nextLine().toDouble() } + + return vector + } + + /** + * Matrix-vector product - Ax + * [n, m] * [m,] = [n,] + * @param vector - [m,] + * @param matrix - [n, m] + * @return [n,] + */ + fun mmul(vector: DoubleArray, matrix: Array): DoubleArray { + val firstDim = matrix.size + val secondDim: Int = matrix[0].size + val res = DoubleArray(firstDim) + for (i in 0 until firstDim) { + for (j in 0 until secondDim) { + res[i] += vector[j] * matrix[i][j] + } + } + return res + } + + /** + * Matrix-vector product with bias - Ax + b + * [n, m] * [m,] = [n,] + * @param vector - [m,] + * @param matrix - [n, m] + * @param bias - [n,] + * @return [n,] + */ + fun mmulBias(vector: DoubleArray, matrix: Array, bias: DoubleArray): DoubleArray? { + val firstDim = matrix.size + val secondDim: Int = matrix[0].size + val res = DoubleArray(firstDim) + for (i in 0 until firstDim) { + for (j in 0 until secondDim) { + res[i] += vector[j] * matrix[i][j] + } + res[i] += bias[i] + } + return res + } + + /** + * Matrix-vector1.concat(vector2) product - Ax. + * @param vector1 - [m,] + * @param vector2 - [m,] + * @param matrix - [n, 2*m] + * @return [n,] + */ + fun mmul(vector1: DoubleArray, vector2: DoubleArray, matrix: Array): DoubleArray { + val dim = vector1.size + val res = DoubleArray(dim) + for (i in 0 until dim) { + for (j in 0 until dim) { + res[i] += vector1[j] * matrix[i][j] + vector2[j] * matrix[i][dim + j] + } + } + return res + } + + /** + * Sum of Matrix-vector product + * @param input - [k, m] + * @param matrix - [n, m] + * @return [n] + */ + fun mmulSum(input: ArrayList, matrix: Array): DoubleArray { + val firstDim = matrix.size + val secondDim: Int = matrix[0].size + val res = DoubleArray(firstDim) + for (vector in input) { + for (i in 0 until firstDim) { + for (j in 0 until secondDim) { + res[i] += vector[j] * matrix[i][j] + } + } + } + return res + } + +} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt new file mode 100644 index 0000000000..7e1c9ff44c --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt @@ -0,0 +1,27 @@ +package org.utbot.utils.layers + +import org.utbot.utils.MatrixUtil +import java.io.FileNotFoundException + +class MessageAggregation { + + lateinit var weight0: Array + lateinit var weight1: Array + + init { + try { + weight0 = MatrixUtil.loadMatrix("logs/model.encoder.x_update.0.weight", false) + weight1 = MatrixUtil.loadMatrix("logs/model.encoder.x_update.2.weight", false) + } catch (e: FileNotFoundException) { + e.printStackTrace() + } + } + + fun propagate(state: DoubleArray, message: DoubleArray): DoubleArray { + val x = MatrixUtil.mmul(message, state, weight0) + for (i in x.indices) { + x[i] = Math.max(x.get(i), 0.0) + } + return MatrixUtil.mmul(x, weight1) + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt new file mode 100644 index 0000000000..6da6b609dd --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt @@ -0,0 +1,27 @@ +package org.utbot.utils.layers + +import org.utbot.utils.MatrixUtil +import java.io.FileNotFoundException +import java.util.ArrayList + + +class MessagePassing { + + lateinit var weight: Array + + init { + try { + weight = MatrixUtil.loadMatrix("logs/model.encoder.weight.txt", true) + } catch (e: FileNotFoundException) { + e.printStackTrace() + } + } + + fun propagate(message: DoubleArray): DoubleArray { + return MatrixUtil.mmul(message, weight) + } + + fun propagate(messages: ArrayList): DoubleArray { + return MatrixUtil.mmulSum(messages, weight) + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt new file mode 100644 index 0000000000..f607f507cf --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt @@ -0,0 +1,53 @@ +package org.utbot.utils.layers + +import org.utbot.utils.MatrixUtil +import java.io.FileNotFoundException + + +class SimpleDecoder { + + lateinit var weight0: Array + lateinit var weight1: Array + lateinit var weight2: Array + lateinit var weight3: Array + + lateinit var bias0: DoubleArray + lateinit var bias1: DoubleArray + lateinit var bias2: DoubleArray + lateinit var bias3: DoubleArray + + init { + try { + weight0 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.hidden_layer_1.0.weight.txt", false) + weight1 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.hidden_layer_2.0.weight.txt", false) + weight2 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.hidden_layer_3.0.weight.txt", false) + weight3 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.output_layer.0.weight.txt", false) + bias0 = MatrixUtil.loadVector("logs/decoder/model.decoder.hidden_layer_1.0.bias.txt") + bias1 = MatrixUtil.loadVector("logs/decoder/model.decoder.hidden_layer_2.0.bias.txt") + bias2 = MatrixUtil.loadVector("logs/decoder/model.decoder.hidden_layer_3.0.bias.txt") + bias3 = MatrixUtil.loadVector("logs/decoder/model.decoder.output_layer.0.bias.txt") + } catch (e: FileNotFoundException) { + e.printStackTrace() + } + } + + fun propagate(state: DoubleArray): DoubleArray? { + var x = MatrixUtil.mmul(state, weight0) + for (i in x.indices) { + x[i] = Math.max(x[i] + bias0[i], 0.0) + } + x = MatrixUtil.mmul(x, weight1) + for (i in x.indices) { + x[i] = Math.max(x[i] + bias1[i], 0.0) + } + x = MatrixUtil.mmul(x, weight2) + for (i in x.indices) { + x[i] = Math.max(x[i] + bias2[i], 0.0) + } + x = MatrixUtil.mmul(x, weight3) + for (i in x.indices) { + x[i] += bias3[i] + } + return x + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt index 9f1673a57e..da505812b0 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt @@ -13,7 +13,7 @@ abstract class AbstractHtmlReport(bodyWidth: Int = 600) { "logs/Report_" + dateTimeFormatter.format(LocalDateTime.now()) + ".html" fun save(filename: String = nameWithDate()) { - builder.saveHtml(filename) + builder.saveHTML(filename) } } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt index 9e2784093b..ced6b5a0e8 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt @@ -2,50 +2,34 @@ package org.utbot.visual import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.Properties -import tech.tablesaw.plotly.components.Figure +import java.util.* -class ClassificationHtmlReport : AbstractHtmlReport() { - private var figuresNum = 0 +class ClassificationHTMLReport() { + private val builder = HtmlBuilder() fun addHeader(modelName: String, properties: Properties) { - val currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss")) - builder.addRawHTML("

Model : ${modelName}

") - builder.addRawHTML("

$currentDateTime

") - builder.addRawHTML("

Hyperparameters:

") - for (property in properties) { - builder.addText("${property.key} : ${property.value}") - } - } - - private fun addFigure(figure: Figure) { - builder.addRawHTML("
") - builder.addRawHTML(figure.asJavascript("plot$figuresNum")) - builder.addRawHTML("
") - figuresNum++ + builder.addHeader(modelName, properties) } fun addDataDistribution(y: DoubleArray, threshold: Double = 1000.0) { - val (data, notPresented) = y.partition { x -> x <= threshold } val rawHistogram = FigureBuilders.buildHistogram( - data.toDoubleArray(), + y.filter { x -> x <= threshold }.toDoubleArray(), xLabel = "ms", yLabel = "Number of samples", title = "Raw data distribution" ) - addFigure(rawHistogram) + builder.addFigure(rawHistogram) builder.addText("Number of samples: ${y.size}") - if (notPresented.isNotEmpty()) { - builder.addText( - "And ${notPresented.size} more samples longer than $threshold ms are not presented " + - "in the raw data distribution plot \n\n" - ) - } + builder.addText( + "And ${ + y.filter { x -> x > threshold }.size + } more samples longer than $threshold ms are not presented in the raw data distribution plot \n\n" + ) } fun addConfusionMatrix(confusionMatrix: Array) { - addFigure( + builder.addFigure( FigureBuilders.buildHeatmap( Array(confusionMatrix.size) { it }, Array(confusionMatrix.size) { it }, @@ -61,7 +45,7 @@ class ClassificationHtmlReport : AbstractHtmlReport() { var title = "Class distribution after resampling" if (before) title = "Class distribution before resampling" - addFigure( + builder.addFigure( FigureBuilders.buildBarPlot( Array(classSizes.size) { it }, classSizes, @@ -72,8 +56,8 @@ class ClassificationHtmlReport : AbstractHtmlReport() { ) } - fun addPCAPlot(variance: DoubleArray, cumulativeVariance: DoubleArray) { - addFigure( + fun addPCAPlot(variance: DoubleArray, cumulativeVariance: DoubleArray){ + builder.addFigure( FigureBuilders.buildTwoLinesPlot( variance, cumulativeVariance, @@ -84,19 +68,25 @@ class ClassificationHtmlReport : AbstractHtmlReport() { ) } - fun addMetrics(acc: Double, f1: Double, predTime: Double, precision: DoubleArray, recall: DoubleArray) { + fun addMetrics(acc: Double, f1: Double, predTime: Double, precision: DoubleArray, recall: DoubleArray){ builder.addText("Accuracy ${String.format("%.3f", acc)}") builder.addText("F1 macro ${String.format("%.3f", f1)}") builder.addText("AvgPredTime ${String.format("%.3f", predTime)}(ms)") - for ((i, value) in precision.withIndex()) { - builder.addText( - "For class $i precision ${String.format("%.3f", value)} recall ${ - String.format( - "%.3f", - recall[i] - ) - }" - ) + for ((i, value) in precision.withIndex()){ + builder.addText("For class $i precision ${String.format("%.3f", value)} recall ${String.format("%.3f", recall[i])}") } } + + fun save(filename: String = "default") { + if (filename == "default") { + val filename = "logs/Classification_Report_" + + LocalDateTime.now() + .format(DateTimeFormatter.ofPattern("dd-MM-yyyy_HH-mm-ss")) + + ".html" + builder.saveHTML(filename) + }else { + builder.saveHTML(filename) + } + } + } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt index 445934e513..206489b3a7 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt @@ -125,22 +125,38 @@ class FigureBuilders { return Figure(layout, trace1, trace2) } - fun buildLinePlotSmoothed( - x: DoubleArray, - y: DoubleArray, - smoothing: Double = 1.2, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Line plot" + fun buildSeveralLinesPlot( + x: List, + y: List, + colors: List, + names: List, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Line plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) - val trace: Trace = ScatterTrace.builder(x, y) + val traces = x.indices.map { + ScatterTrace.builder(x[it], y[it]) .mode(ScatterTrace.Mode.LINE_AND_MARKERS) - .line(Line.builder().shape(Line.Shape.SPLINE).smoothing(smoothing).build()) + .line(Line.builder().shape(Line.Shape.LINEAR).color(colors[it]).build()) + .name(names[it]) + .showLegend(true) .build() + } + return Figure(layout, *traces.toTypedArray()) + } + + fun buildBoxPlot(x: Array, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Box plot"): Figure { + val layout = getXYLayout(xLabel, yLabel, title) + val trace: BoxTrace = BoxTrace.builder(x, y).build() return Figure(layout, trace) } + } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt index 98037c2186..ad8baf8218 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt @@ -4,8 +4,11 @@ import java.io.File import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.* +import tech.tablesaw.plotly.components.Figure -class HtmlBuilder(bodyMaxWidth: Int = 600) { +class HtmlBuilder(bodyMaxWidth: Int = 600, + pathToStyle: List = listOf(), + pathToJs: List = listOf()) { private val pageTop = ("" + System.lineSeparator() + "" @@ -13,55 +16,75 @@ class HtmlBuilder(bodyMaxWidth: Int = 600) { + " Multi-plot test" + System.lineSeparator() + " " - + System.lineSeparator() - + "" + + pathToJs.joinToString("") { "" } + System.lineSeparator() + "" + System.lineSeparator() - + "" + + "" + System.lineSeparator()) private val pageBottom = "" + System.lineSeparator() + "" - private var pageBuilder = StringBuilder(pageTop).appendLine() + private var pageBuilder = StringBuilder(pageTop).append(System.lineSeparator()) + private var figres_num = 0 - fun saveHtml(fileName: String = "Report.html") { - File(fileName).writeText(pageBuilder.toString() + pageBottom) + fun addFigure(figure: Figure) { + figure.asJavascript("plot${this.figres_num}") + pageBuilder.append("
").append(System.lineSeparator()) + .append(figure.asJavascript("plot${this.figres_num}")).append(System.lineSeparator()).append("
") + this.figres_num += 1 } - fun addText(text: String) { - pageBuilder.append("
$text
") + fun saveHTML(fileName: String = "Report.html") { + File(fileName).writeText(pageBuilder.toString() + pageBottom) } - fun addRawHTML(HTMLCode: String) { - pageBuilder.append(HTMLCode) - } + fun addHeader(ModelName: String, properties: Properties) { + val currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss")) + pageBuilder.append("

Model : ${ModelName}

").append("

$currentDateTime

") + .append("

Hyperparameters:

") + for (property in properties) { + pageBuilder.append("
${property.key} : ${property.value}
").append(System.lineSeparator()) + } - fun addBreak() { - addText("
") } - fun addHeader1(header: String){ - addText("

$header

") + fun addHeader(header: String, h: String = "h1") { + pageBuilder.append("<$h>${header}") } - fun addHeader2(header: String){ - addText("

$header

") + fun addTable(statistics: List>, + selectorNames: List, + colors: List, + scope: String) { + pageBuilder.append("") + pageBuilder.append("\n") + + selectorNames.forEach { pageBuilder.append("\n") } + pageBuilder.append("") + + statistics.first().keys.forEach { key -> + pageBuilder.append("\n") + + for (i in statistics.indices) { + pageBuilder.append( + "\n" + ) + } + pageBuilder.append("") + } + pageBuilder.append("
$scope${it}
${key}\n" + + "\n" + + "

${String.format("%.0f", statistics[i][key])}

\n" + + "\n" + + "
") } - fun addHeader3(header: String){ - addText("

$header

") + fun addText(text: String) { + pageBuilder.append("
$text
") } } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt deleted file mode 100644 index e7f58a7cd4..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt +++ /dev/null @@ -1,167 +0,0 @@ -package org.utbot.visual - -import org.utbot.summary.tag.BasicTypeTag -import org.utbot.summary.tag.ExecutionTag -import org.utbot.summary.tag.UniquenessTag -import org.utbot.summary.tag.StatementTag -import org.utbot.summary.tag.TraceTag -import soot.jimple.JimpleBody - -class TracePathReport : AbstractHtmlReport(bodyWidth = 1200) { - private val tab = "    " - - private val freqSymbols = mapOf( - UniquenessTag.Common to " ", - UniquenessTag.Unique to "★ ", - UniquenessTag.Partly to "â–º " - ) - - private val basicTagColors = mapOf( - BasicTypeTag.Initialization to "#aab7b8", - BasicTypeTag.Condition to "black", - BasicTypeTag.Return to "#28b463", - BasicTypeTag.Assignment to "black", - BasicTypeTag.Basic to "black", - BasicTypeTag.ExceptionAssignment to "red", - BasicTypeTag.ExceptionThrow to "red", - BasicTypeTag.Invoke to "black", - BasicTypeTag.IterationStart to "blue", - BasicTypeTag.IterationEnd to "#2874a6" - ) - - private val executionPostfix = mapOf( - ExecutionTag.True to " : ✓", - ExecutionTag.False to " : ✗", - ExecutionTag.Executed to "" - ) - - fun addJimpleBody(jimpleBody: JimpleBody?) { - if (jimpleBody == null) - return - var result = "

Jimple Body

\n" - jimpleBody.units.forEach { - result += "${it.javaSourceStartLineNumber}:$tab $it
\n" - } - - builder.addRawHTML(result) - } - - private fun buildStructViewStatementTag(statementTag: StatementTag?): String { - if (statementTag == null) - return "" - var result = "
  • ${statementTag.step.stmt} \n" + - "
    Decision = ${markDecision(statementTag.executionTag)}\n" + - "
    Line = ${statementTag.line} \n" + - "
    Type = ${statementTag.basicTypeTag} \n" + - "
    Frequency = ${statementTag.uniquenessTag} \n" + - "
    Call times = ${statementTag.callOrderTag}
    \n" + - "
    • " - - if (statementTag.invoke != null) { - result += "\n
    Invocation:
    \n" + - "
      ${buildStructViewStatementTag(statementTag.invoke)}
    \n" - } - if (statementTag.iterations.size > 0) { - result += "Iterations:\n" - result += "
      " - for (i in 0 until statementTag.iterations.size) { - result += "
      Iteration $i
      ${buildStructViewStatementTag(statementTag.iterations[i])}\n" - } - result += "
    " - } - if (statementTag.next != null) { - result += "
    ${buildStructViewStatementTag(statementTag.next)}\n" - } - return result - } - - fun addStructViewTraces(tracesTags: Iterable) { - var table = "

    Struct View

    \n " + - "\n" - tracesTags.forEach { - table += "\n" - } - table += "\n" - builder.addRawHTML(table) - } - - fun addTracesTable(tagsToKeywords: List>, name: String) { - var table = "

    $name

    \n" + - "
      ${buildStructViewStatementTag(it.rootStatementTag)}
    " - - table += "" - table += "" - table += "" - table += " " - table += "" - - for ((clusterNum, clusterTraceTags) in tagsToKeywords.withIndex()) { - - for (traceTags in clusterTraceTags) { - table += "" - val traceTagsVisual = traceTags.rootStatementTag?.let { visualizeStatementTag(it) } - table += "" - - table += "" - - table += "" - - table += "" - table += "" - } - } - table += "
    â„– Jimple codeSource codeKeywordsComment
    $clusterNum
    "
-                table += traceTagsVisual ?: "None"
-                table += "
    "
-                table += "No source code yet"
-                table += "
    "
-                table += traceTags.summary
-                table += "
    " - builder.addRawHTML(table) - } - - private fun markDecision(executionTag: ExecutionTag): String { - var result = when (executionTag) { - ExecutionTag.True -> "" - ExecutionTag.False -> "" - else -> "" - } - result += "$executionTag " - return result - } - - fun addExecutionVisualisation(rootTag: StatementTag, name: String){ - var visualization = "

    $name

    \n" - visualization += "
    "
-        visualization += visualizeStatementTag(rootTag)
-        visualization += "
    " - builder.addRawHTML(visualization) - } - - private fun visualizeStatementTag(tag: StatementTag, tabPrefix: String = ""): String { - var localTabPrefix = tabPrefix - var visualization = "" - - visualization += "" - visualization += tag.line.toString().padEnd(3, ' ') + ':' - visualization += localTabPrefix - visualization += freqSymbols[tag.uniquenessTag] - if (tag.uniquenessTag == UniquenessTag.Unique) visualization += "" - visualization += tag.step.stmt.toString() - visualization += executionPostfix[tag.executionTag] - if (tag.uniquenessTag == UniquenessTag.Unique) visualization += "" - visualization += "" - visualization += "\n" - - if (tag.invoke != null) visualization += visualizeStatementTag(tag.invoke!!, localTabPrefix + "\t") - if (tag.iterations.size > 0) { - for (cycle in tag.iterations) visualization += visualizeStatementTag(cycle, localTabPrefix + "\t") - } - - if (tag.basicTypeTag == BasicTypeTag.IterationEnd) localTabPrefix = localTabPrefix.removePrefix("\t") - if (tag.next != null) visualization += visualizeStatementTag(tag.next!!, localTabPrefix) - - return visualization - } - -} \ No newline at end of file diff --git a/utbot-analytics/src/main/resources/config.properties b/utbot-analytics/src/main/resources/config.properties new file mode 100644 index 0000000000..e71418b7e6 --- /dev/null +++ b/utbot-analytics/src/main/resources/config.properties @@ -0,0 +1,3 @@ +project=antlr +selectors=random_120,cpi_120,fork_120,inheritors_120,random_120 +covStatistics=logs/covStatistics,logs/covStatistics \ No newline at end of file diff --git a/utbot-analytics/src/main/resources/css/coverage.css b/utbot-analytics/src/main/resources/css/coverage.css new file mode 100644 index 0000000000..b7b9e2a9b3 --- /dev/null +++ b/utbot-analytics/src/main/resources/css/coverage.css @@ -0,0 +1,168 @@ +/* + * Copyright 2000-2021 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +* { + margin: 0; + padding: 0; +} + +body { + background-color: #fff; + font-family: helvetica neue, tahoma, arial, sans-serif; + font-size: 82%; + color: #151515; +} + +h1 { + margin: 0.5em 0; + color: #010101; + font-weight: normal; + font-size: 18px; +} + +h2 { + margin: 0.5em 0; + color: #010101; + font-weight: normal; + font-size: 16px; +} + +a { + color: #1564C2; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +i { + background-color: #eee; +} + +span.separator { + color: #9BA9BA; + padding-left: 5px; + padding-right: 5px; +} + +div.content { + width: 99%; +} + +table.coverageStats { + width: 100%; + border-collapse: collapse; +} + +table.overallStats { + width: 20%; +} + +table.coverageStats td, table.coverageStats th { + padding: 4px 2px; + border-bottom: 1px solid #ccc; +} + +table.coverageStats th { + background-color: #959BA4; + border: none; + font-weight: bold; + text-align: left; + color: #FFF; +} + +table.coverageStats th.coverageStat { + width: 20%; +} + +table.coverageStats th a { + color: #FFF; +} + +table.coverageStats th a:hover { + text-decoration: none; +} + +table.coverageStats th.sortedDesc a { + background: url(../img/arrowDown.gif) no-repeat 100% 2px; + padding-right: 20px; +} + +table.coverageStats th.sortedAsc a { + background: url(../img/arrowUp.gif) no-repeat 100% 2px; + padding-right: 20px; +} + +div.footer { + margin: 2em .5em; + font-size: 85%; + text-align: left; + line-height: 140%; +} + +div.sourceCode { + width: 100%; + border: 1px solid #ccc; + font: normal 12px 'Menlo', 'Bitstream Vera Sans Mono', 'Courier New', 'Courier', monospace; + white-space: pre; +} + +div.sourceCode b { + font-weight: normal; +} + +div.sourceCode i { + display: block; + float: left; + width: 3em; + padding-right: 3px; + border-right: 1px solid #ccc; + font-style: normal; + text-align: right; +} + +div.sourceCode i.no-highlight span.number { + color: #151515; +} + +div.sourceCode .fc, div.sourceCode .fc i { + background-color: #cfc; +} + +div.sourceCode .pc, div.sourceCode .pc i { + background-color: #ffc; +} + +div.sourceCode .nc, div.sourceCode .nc i { + background-color: #fcc; +} + +.percent, .absValue { + font-size: 90%; +} + +.percent .green, .absValue .green { + color: #32cc32; +} + +.percent .red, .absValue .red { + color: #f00; +} + +.percent .totalDiff { + color: #3f3f3f; +} diff --git a/utbot-analytics/src/main/resources/css/highlight-idea.css b/utbot-analytics/src/main/resources/css/highlight-idea.css new file mode 100644 index 0000000000..1d1fa10150 --- /dev/null +++ b/utbot-analytics/src/main/resources/css/highlight-idea.css @@ -0,0 +1,153 @@ +/* + * Copyright 2000-2021 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + +Intellij Idea-like styling (c) Vasily Polovnyov + +*/ + +pre code { + display: block; padding: 0.5em; + color: #000; + background: #fff; +} + +pre .subst, +pre .title { + font-weight: normal; + color: #000; +} + +pre .comment, +pre .template_comment, +pre .javadoc, +pre .diff .header { + color: #808080; + font-style: italic; +} + +pre .annotation, +pre .decorator, +pre .preprocessor, +pre .doctype, +pre .pi, +pre .chunk, +pre .shebang, +pre .apache .cbracket, +pre .input_number, +pre .http .title { + color: #808000; +} + +pre .tag, +pre .pi { + background: #efefef; +} + +/* leonid.khachaturov: redefine background as it conflicts with change highlighting we apply on top of source highlighting */ +pre .changeAdded .tag, +pre .changeRemoved .tag, +pre .changeAdded .pi, +pre .changeRemoved .pi { + background: transparent; +} + +/* leonid.khachaturov: redefine .comment from main.css */ +pre .comment { + margin: 0; + padding: 0; + font-size: 100%; +} + +pre .tag .title, +pre .id, +pre .attr_selector, +pre .pseudo, +pre .literal, +pre .keyword, +pre .hexcolor, +pre .css .function, +pre .ini .title, +pre .css .class, +pre .list .title, +pre .nginx .title, +pre .tex .command, +pre .request, +pre .status { + font-weight: bold; + color: #000080; +} + +pre .attribute, +pre .rules .keyword, +pre .number, +pre .date, +pre .regexp, +pre .tex .special { + font-weight: bold; + color: #0000ff; +} + +pre .number, +pre .regexp { + font-weight: normal; +} + +pre .string, +pre .value, +pre .filter .argument, +pre .css .function .params, +pre .apache .tag { + color: #008000; + font-weight: bold; +} + +pre .symbol, +pre .ruby .symbol .string, +pre .ruby .symbol .keyword, +pre .ruby .symbol .keymethods, +pre .char, +pre .tex .formula { + color: #000; + background: #d0eded; + font-style: italic; +} + +pre .phpdoc, +pre .yardoctag, +pre .javadoctag { + text-decoration: underline; +} + +pre .variable, +pre .envvar, +pre .apache .sqbracket, +pre .nginx .built_in { + color: #660e7a; +} + +pre .addition { + background: #baeeba; +} + +pre .deletion { + background: #ffc8bd; +} + +pre .diff .change { + background: #bccff9; +} diff --git a/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt new file mode 100644 index 0000000000..390a9e27f6 --- /dev/null +++ b/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt @@ -0,0 +1,61 @@ +package org.utbot.features + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class FeatureProcessorWithRepetitionTest { + companion object { + fun reward(coverage: Double, time: Double) = RewardEstimator.reward(coverage, time) + } + + @Test + fun testDumpFeatures() { + val statesToInt = mapOf( + "a0" to 1, + "c0" to 2, + "f0" to 3, + "g0" to 4, + "c1" to 5, + "f1" to 6, + "g1" to 7, + "b0" to 8, + "d0" to 9 + ) + + val expectedRewards: Map = mapOf( + "a0" to reward(6.0, 15.0), + "c0" to reward(4.0, 10.0), + "f0" to reward(4.0, 8.0), + "g0" to reward(4.0, 2.0), + "c1" to reward(0.0, 4.0), + "f1" to reward(0.0, 3.0), + "g1" to reward(0.0, 2.0), + "b0" to reward(2.0, 4.0), + "d0" to reward(2.0, 2.0) + ) + + val rewardEstimator = RewardEstimator() + val testCases = listOf( + TestCase( + listOf(statesToInt["g0"]!! to 2L, statesToInt["f0"]!! to 2L, statesToInt["c0"]!! to 2L, statesToInt["a0"]!! to 1L), + newCoverage = 4, testIndex = 0 + ), + TestCase( + listOf(statesToInt["g1"]!! to 2L, statesToInt["f1"]!! to 1L,statesToInt["c1"]!! to 1L, statesToInt["f0"]!! to 2L, statesToInt["c0"]!! to 2L, statesToInt["a0"]!! to 1L), + newCoverage = 0, + testIndex = 1 + ), + TestCase( + listOf(statesToInt["d0"]!! to 2L, statesToInt["b0"]!! to 2L, statesToInt["a0"]!! to 1L), + newCoverage = 2, + testIndex = 2 + ) + ) + + val rewards = rewardEstimator.calculateRewards(testCases) + Assertions.assertEquals(9, rewards.size) + expectedRewards.forEach { + Assertions.assertEquals(it.value, rewards[statesToInt[it.key]]) + } + } +} diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt new file mode 100644 index 0000000000..4f52f07d7b --- /dev/null +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -0,0 +1,21 @@ +package org.utbot.predictors + +import org.utbot.examples.withRewardModelPath +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class LinearStateRewardPredictorTest { + @Test + fun simpleTest() { + withRewardModelPath("src/test/resources") { + val pred = LinearStateRewardPredictor() + + val features = listOf( + listOf(2.0, 3.0), + listOf(2.0, 3.0) + ) + + assertEquals(listOf(6.0, 6.0), pred.predict(features)) + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt new file mode 100644 index 0000000000..767ebd2301 --- /dev/null +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -0,0 +1,53 @@ +package org.utbot.predictors + +import org.utbot.examples.withRewardModelPath +import kotlin.system.measureNanoTime +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test + +class NNStateRewardPredictorTest { + @Test + fun simpleTest() { + withRewardModelPath("src/test/resources") { + val pred = NNStateRewardPredictorSmile() + + val features = listOf( + 0.0, 0.0 + ) + + assertEquals(5.0, pred.predict(features)) + } + } + + @Disabled("Just to see the performance of predictors") + @Test + fun performanceTest() { + val features = (1..13).map { 1.0 }.toList() + withRewardModelPath("models\\test\\0") { + val pred1 = NNStateRewardPredictorSmile() + + (1..100).map { + pred1.predict(features) + } + + println((1..100).map { + measureNanoTime { pred1.predict(features) } + }.sum().toDouble() / 100) + } + + + withRewardModelPath("models") { + val pred2 = NNStateRewardPredictorTorch() + + (1..100).map { + pred2.predict(features) + } + + println((1..100).map { + measureNanoTime { pred2.predict(features) } + }.sum().toDouble() / 100) + pred2.close() + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/linear.txt b/utbot-analytics/src/test/resources/linear.txt new file mode 100644 index 0000000000..8a6627b46a --- /dev/null +++ b/utbot-analytics/src/test/resources/linear.txt @@ -0,0 +1 @@ +1,1,1 \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/nn.json b/utbot-analytics/src/test/resources/nn.json new file mode 100644 index 0000000000..945b53aef8 --- /dev/null +++ b/utbot-analytics/src/test/resources/nn.json @@ -0,0 +1,24 @@ +{ + "linearLayers": [ + [ + [1, 0], + [0, 1] + ], + [ + [1, 0], + [0, 1] + ], + [ + [1, 1] + ] + ], + "activationLayers": [ + "reLU", + "reLU" + ], + "biases": [ + [1, 1], + [1, 1], + [1] + ] +} \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/scaler.txt b/utbot-analytics/src/test/resources/scaler.txt new file mode 100644 index 0000000000..b241d25f33 --- /dev/null +++ b/utbot-analytics/src/test/resources/scaler.txt @@ -0,0 +1,2 @@ +0,0 +1,1 diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 39da879aa6..46531da4e6 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -108,6 +108,11 @@ object UtSettings { */ var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.INHERITORS_SELECTOR) + /** + * Type of nnRewardGuidedSelector + */ + var nnRewardGuidedSelectorType: NNRewardGuidedSelectorType by getEnumProperty(NNRewardGuidedSelectorType.WITHOUT_RECALCULATION) + /** * Steps limit for path selector. */ @@ -286,6 +291,47 @@ object UtSettings { */ var enableUnsatCoreCalculationForHardConstraints by getBooleanProperty(false) + /** + * 2^{this} will be the length of observed subpath. + * See [SubpathGuidedSelector] + */ + var subpathGuidedSelectorIndex by getIntProperty(1) + var subpathGuidedSelectorIndexes = listOf(0, 1, 2, 3) + + /** + * Enable feature processing for executionStates + */ + var featureProcess by getBooleanProperty(false) + + /** + * Path to deserialized reward models + */ + var rewardModelPath by getStringProperty("models/cf") + + /** + * Number of model iterations that will be used during ContestEstimator + */ + var iterations by getIntProperty(4) + + /** + * Path for state features dir + */ + var featurePath by getStringProperty("eval/secondFeatures/antlr/INHERITORS_SELECTOR") + + /** + * Counter for tests during testGeneration for one project in ContestEstimator + */ + var testCounter by getIntProperty(0) + + var collectCoverage by getBooleanProperty(false) + + var coverageStatisticsDir by getStringProperty("logs/covStatistics") + + /** + * Flag for Subpath and NN selectors whether they are combined (Subpath use several indexes, NN use several models) + */ + var singleSelector by getBooleanProperty(true) + override fun toString(): String = properties .entries @@ -295,10 +341,22 @@ object UtSettings { enum class PathSelectorType { COVERED_NEW_SELECTOR, - INHERITORS_SELECTOR + INHERITORS_SELECTOR, + SUBPATH_GUIDED_SELECTOR, + CPI_SELECTOR, + FORK_DEPTH_SELECTOR, + LINEAR_REWARD_GUIDED_SELECTOR, + NN_REWARD_GUIDED_SELECTOR, + RANDOM_SELECTOR, + RANDOM_PATH_SELECTOR } enum class TestSelectionStrategyType { DO_NOT_MINIMIZE_STRATEGY, // Always adds new test COVERAGE_STRATEGY // Adds new test only if it increases coverage } + +enum class NNRewardGuidedSelectorType { + WITH_RECALCULATION, + WITHOUT_RECALCULATION +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt new file mode 100644 index 0000000000..cf8b79431d --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt @@ -0,0 +1,49 @@ +package org.utbot.analytics + +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.TraverseGraphStatistics +import org.utbot.engine.stmts +import org.utbot.framework.UtSettings +import java.io.File +import java.io.FileOutputStream +import mu.KotlinLogging + + +private val logger = KotlinLogging.logger {} + + +class CoverageStatistics( + private val method: String, + private val globalGraph: InterProceduralUnitGraph +): TraverseGraphStatistics(globalGraph) { + + private val outputFile: String = "${UtSettings.coverageStatisticsDir}/$method.txt" + + init { + File(outputFile).printWriter().use { out -> + out.println("TIME,COV_TARGET_STMT,TOTAL_TARGET_STMT,COV_ALL_STMT,TOTAL_ALL_STMT") + out.println("${System.nanoTime()}" + "," + getStatistics()) + } + } + + override fun onTraversed(executionState: ExecutionState) { + runCatching { + FileOutputStream(outputFile, true).bufferedWriter() + .use { out -> + out.write(System.nanoTime().toString() + "," + getStatistics()) + out.newLine() + } + }.onFailure { + logger.warn { "Failed to save statistics: ${it.message}" } + } + } + + fun getStatistics() = with(globalGraph) { + val allStmts = this.graphs.flatMap { it.stmts } + val graphStmts = this.graphs.first().stmts + + "${graphStmts.filter { this.isCovered(it) }.size},${graphStmts.size}," + + "${allStmts.filter { this.isCovered(it) }.size},${allStmts.size}" + } +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt new file mode 100644 index 0000000000..ddafedff5b --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -0,0 +1,32 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.NNRewardGuidedSelectorFactory +import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationFactory +import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationWeight +import org.utbot.engine.selectors.NNRewardGuidedSelectorWithoutRecalculationFactory +import org.utbot.framework.NNRewardGuidedSelectorType +import org.utbot.framework.UtSettings + +/** + * Class that stores all objects that need for work analytics module during symbolic execution + */ +object EngineAnalyticsContext { + var featureProcessorFactory: FeatureProcessorFactory = object : FeatureProcessorFactory { + override fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor { + error("Feature processor factory wasn't provided") + } + } + + var featureExtractorFactory: FeatureExtractorFactory = object : FeatureExtractorFactory { + override fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor { + error("Feature extractor factory wasn't provided") + } + } + + val nnRewardGuidedSelectorFactory: NNRewardGuidedSelectorFactory = when (UtSettings.nnRewardGuidedSelectorType) { + NNRewardGuidedSelectorType.WITHOUT_RECALCULATION -> NNRewardGuidedSelectorWithoutRecalculationFactory() + NNRewardGuidedSelectorType.WITH_RECALCULATION -> NNRewardGuidedSelectorWithRecalculationFactory() + else -> error("Unsupported nnRewardGuidedSelectorType") + } +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt new file mode 100644 index 0000000000..6dab6d8775 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt @@ -0,0 +1,10 @@ +package org.utbot.analytics + +import org.utbot.engine.ExecutionState + +/** + * Class that incapsulates work with FeatureExtractor during symbolic execution + */ +interface FeatureExtractor { + fun extractFeatures(executionState: ExecutionState, generatedTestCases: Int) +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt new file mode 100644 index 0000000000..011f587177 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt @@ -0,0 +1,10 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Class that can create FeatureExtractor. See [FeatureExtractor] + */ +interface FeatureExtractorFactory { + operator fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt new file mode 100644 index 0000000000..b13dc72ddf --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt @@ -0,0 +1,11 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.TraverseGraphStatistics + +/** + * Interface that incapsulates work with FeatureProcessor and can only dumpFeatures at the end of symbolic execution + */ +abstract class FeatureProcessor(graph: InterProceduralUnitGraph) : TraverseGraphStatistics(graph) { + abstract fun dumpFeatures() +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessorFactory.kt new file mode 100644 index 0000000000..d012aa7e89 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessorFactory.kt @@ -0,0 +1,10 @@ +package org.utbot.analytics + +import org.utbot.engine.InterProceduralUnitGraph + +/** + * Class that can create FeatureProcessor. See [FeatureProcessor] + */ +interface FeatureProcessorFactory { + operator fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt index d7876049cb..ecae642d15 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt @@ -23,4 +23,14 @@ object Predictors { object : UtBotAbstractPredictor, String> { override fun predict(input: Iterable): String = "stubName" } + + var stateRewardPredictor: UtBotAbstractPredictor, Double> = + object : UtBotAbstractPredictor, Double> { + override fun predict(input: List): Double { + TODO("Not yet implemented") + } + } + + var sat: IUtBotSatPredictor> = object: IUtBotSatPredictor> {} + var stateRewardPredictors: MutableList, Double>> = mutableListOf() } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt index 5c021e204f..dc5198d368 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt @@ -3,6 +3,9 @@ package org.utbot.analytics import org.utbot.engine.pc.UtSolverStatusKind +data class NeuroSatData(val status: UtSolverStatusKind, val time: Long) + + interface UtBotAbstractPredictor { /** * Initialization signal from controller @@ -53,15 +56,15 @@ inline fun UtBotNanoTimePredictor.learnOn(input: TIn, block: () -> * Predicts sat/unsat state of some request with input [TIn] * @see Predictors.smt */ -interface IUtBotSatPredictor : UtBotAbstractPredictor { - override fun predict(input: TIn) = UtSolverStatusKind.UNSAT //Zero for default predictor +interface IUtBotSatPredictor : UtBotAbstractPredictor { + override fun predict(input: TIn) = NeuroSatData(status = UtSolverStatusKind.UNSAT, time = 1) //Zero for default predictor } /** * Embrace [block()] inside this method to ask prediction before execution and send actual result after execution */ -fun IUtBotSatPredictor.learnOn(input: TIn, result: UtSolverStatusKind) { +fun IUtBotSatPredictor.learnOn(input: TIn, result: NeuroSatData) { val expectedResult = predict(input) provide(input, expectedResult, result) diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt index 79ab953a89..3519bf1643 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt @@ -5,6 +5,7 @@ import org.utbot.engine.pc.UtSolver import org.utbot.engine.symbolic.SymbolicState import org.utbot.engine.symbolic.SymbolicStateUpdate import org.utbot.framework.plugin.api.Step +import java.util.Objects import kotlinx.collections.immutable.PersistentList import kotlinx.collections.immutable.PersistentMap import kotlinx.collections.immutable.PersistentSet @@ -40,6 +41,55 @@ data class ExecutionStackElement( this.copy(localVariableMemory = localVariableMemory.update(memoryUpdate), doesntThrow = doesntThrow) } +/** + * Class that store all information about execution state that needed only for analytics module + */ +data class StateAnalyticsProperties( + /** + * Number of forks already performed along state's path, where fork is statement with multiple successors + */ + val depth: Int = 0, + var visitedAfterLastFork: Int = 0, + var visitedBeforeLastFork: Int = 0, + var stmtsSinceLastCovered: Int = 0, + val parent: ExecutionState? = null, +) { + var executingTime: Long = 0 + var reward: Double? = null + val features: MutableList = mutableListOf() + + /** + * Flag that indicates whether this state is fork or not. Fork here means that we have more than one successor + */ + var isFork: Boolean = false + + fun updateIsFork() { + isFork = true + } + + var isVisitedNew: Boolean = false + + fun updateIsVisitedNew() { + isVisitedNew = true + stmtsSinceLastCovered = 0 + visitedAfterLastFork++ + } + + private val successorDepth: Int get() = depth + if (isFork) 1 else 0 + + private val successorVisitedAfterLastFork: Int get() = if (!isFork) visitedAfterLastFork else 0 + private val successorVisitedBeforeLastFork: Int get() = visitedBeforeLastFork + if (isFork) visitedAfterLastFork else 0 + private val successorStmtSinceLastCovered: Int get() = 1 + stmtsSinceLastCovered + + fun successorProperties(parent: ExecutionState) = StateAnalyticsProperties( + successorDepth, + successorVisitedAfterLastFork, + successorVisitedBeforeLastFork, + successorStmtSinceLastCovered, + parent + ) +} + /** * [visitedStatementsHashesToCountInPath] is a map representing how many times each instruction from the [path] * has occurred. It is required to calculate priority of the branches and decrease the priority for branches leading @@ -61,12 +111,14 @@ data class ExecutionState( val lastMethod: SootMethod? = null, val methodResult: MethodResult? = null, val exception: SymbolicFailure? = null, - private var outgoingEdges: Int = 0 + private var stateAnalyticsProperties: StateAnalyticsProperties = StateAnalyticsProperties() ) : AutoCloseable { val solver: UtSolver by symbolicState::solver val memory: Memory by symbolicState::memory + private var outgoingEdges = 0 + fun isInNestedMethod() = executionStack.size > 1 val localVariableMemory @@ -107,7 +159,8 @@ data class ExecutionState( pathLength = pathLength + 1, lastEdge = edge, lastMethod = executionStack.last().method, - exception = exception + exception = exception, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -130,7 +183,8 @@ data class ExecutionState( pathLength = pathLength + 1, lastEdge = edge, lastMethod = executionStack.last().method, - methodResult + methodResult, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -164,7 +218,8 @@ data class ExecutionState( stmts = stmts.putIfAbsent(this.stmt, pathLength), pathLength = pathLength + 1, lastEdge = edge, - lastMethod = stackElement.method + lastMethod = stackElement.method, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -202,7 +257,8 @@ data class ExecutionState( stmts = stmts.putIfAbsent(stmt, pathLength), pathLength = pathLength + 1, lastEdge = edge, - lastMethod = stackElement.method + lastMethod = stackElement.method, + stateAnalyticsProperties = stateAnalyticsProperties.successorProperties(this) ) } @@ -261,4 +317,49 @@ data class ExecutionState( } return " MD5(path)=${prettifiedPath.md5()}\n$prettifiedPath" } + + fun updateIsFork() { + stateAnalyticsProperties.updateIsFork() + } + + fun updateIsVisitedNew() { + stateAnalyticsProperties.updateIsVisitedNew() + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as ExecutionState + + if (stmt != other.stmt) return false + if (symbolicState != other.symbolicState) return false + if (executionStack != other.executionStack) return false + if (path != other.path) return false + if (visitedStatementsHashesToCountInPath != other.visitedStatementsHashesToCountInPath) return false + if (decisionPath != other.decisionPath) return false + if (edges != other.edges) return false + if (stmts != other.stmts) return false + if (pathLength != other.pathLength) return false + if (lastEdge != other.lastEdge) return false + if (lastMethod != other.lastMethod) return false + if (methodResult != other.methodResult) return false + if (exception != other.exception) return false + + return true + } + + override fun hashCode(): Int = Objects.hash( + stmt, symbolicState, executionStack, path, visitedStatementsHashesToCountInPath, decisionPath, + edges, stmts, pathLength, lastEdge, lastMethod, methodResult, exception + ) + + var reward by stateAnalyticsProperties::reward + val features by stateAnalyticsProperties::features + var executingTime by stateAnalyticsProperties::executingTime + val depth by stateAnalyticsProperties::depth + var visitedBeforeLastFork by stateAnalyticsProperties::visitedBeforeLastFork + var visitedAfterLastFork by stateAnalyticsProperties::visitedAfterLastFork + var stmtsSinceLastCovered by stateAnalyticsProperties::stmtsSinceLastCovered + val parent by stateAnalyticsProperties::parent } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 8d081900c6..1e00f3c29d 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -6,17 +6,12 @@ import kotlinx.collections.immutable.persistentSetOf import kotlinx.collections.immutable.toPersistentList import kotlinx.collections.immutable.toPersistentMap import kotlinx.collections.immutable.toPersistentSet -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Job import kotlinx.coroutines.currentCoroutineContext -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.FlowCollector import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.isActive import kotlinx.coroutines.yield -import mu.KotlinLogging import org.utbot.analytics.Predictors import org.utbot.common.WorkaroundReason.HACK import org.utbot.common.WorkaroundReason.REMOVE_ANONYMOUS_CLASSES @@ -79,11 +74,7 @@ import org.utbot.engine.pc.mkNot import org.utbot.engine.pc.mkOr import org.utbot.engine.pc.select import org.utbot.engine.pc.store -import org.utbot.engine.selectors.PathSelector -import org.utbot.engine.selectors.coveredNewSelector -import org.utbot.engine.selectors.inheritorsSelector import org.utbot.engine.selectors.nurs.NonUniformRandomSearch -import org.utbot.engine.selectors.pollUntilFastSAT import org.utbot.engine.selectors.strategies.GraphViz import org.utbot.engine.symbolic.HardConstraint import org.utbot.engine.symbolic.SoftConstraint @@ -143,6 +134,40 @@ import org.utbot.fuzzer.collectConstantsForFuzzer import org.utbot.fuzzer.defaultModelProviders import org.utbot.fuzzer.fuzz import org.utbot.instrumentation.ConcreteExecutor +import java.lang.reflect.ParameterizedType +import java.util.BitSet +import java.util.IdentityHashMap +import java.util.TreeSet +import kotlin.collections.plus +import kotlin.collections.plusAssign +import kotlin.math.max +import kotlin.math.min +import kotlin.reflect.KClass +import kotlin.reflect.full.instanceParameter +import kotlin.reflect.full.valueParameters +import kotlin.reflect.jvm.javaType +import kotlin.system.measureTimeMillis +import kotlinx.collections.immutable.persistentHashMapOf +import kotlinx.collections.immutable.persistentListOf +import kotlinx.collections.immutable.persistentSetOf +import kotlinx.collections.immutable.toPersistentList +import kotlinx.collections.immutable.toPersistentSet +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.FlowCollector +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.onCompletion +import kotlinx.coroutines.flow.onStart +import kotlinx.coroutines.isActive +import kotlinx.coroutines.yield +import mu.KotlinLogging +import org.utbot.analytics.CoverageStatistics +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.FeatureProcessor +import org.utbot.engine.selectors.* +import org.utbot.framework.UtSettings.featureProcess import soot.ArrayType import soot.BooleanType import soot.ByteType @@ -233,7 +258,6 @@ import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl import sun.reflect.generics.reflectiveObjects.TypeVariableImpl import sun.reflect.generics.reflectiveObjects.WildcardTypeImpl import java.lang.reflect.Method -import java.lang.reflect.ParameterizedType import kotlin.collections.plus import kotlin.collections.plusAssign import kotlin.math.max @@ -269,6 +293,24 @@ private fun pathSelector(graph: InterProceduralUnitGraph, typeRegistry: TypeRegi PathSelectorType.INHERITORS_SELECTOR -> inheritorsSelector(graph, typeRegistry) { withStepsLimit(pathSelectorStepsLimit) } + PathSelectorType.SUBPATH_GUIDED_SELECTOR -> subpathGuidedSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.CPI_SELECTOR -> cpInstSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.FORK_DEPTH_SELECTOR -> forkDepthSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.NN_REWARD_GUIDED_SELECTOR -> nnRewardGuidedSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.RANDOM_SELECTOR -> randomSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.RANDOM_PATH_SELECTOR -> randomPathSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } else -> error("Unknown type") } @@ -284,6 +326,7 @@ class UtBotSymbolicEngine( ) : UtContextInitializer() { private val methodUnderAnalysisStmts: Set = graph.stmts.toSet() + private val visitedStmts: MutableSet = mutableSetOf() private val globalGraph = InterProceduralUnitGraph(graph) internal val typeRegistry: TypeRegistry = TypeRegistry() private val pathSelector: PathSelector = pathSelector(globalGraph, typeRegistry) @@ -333,6 +376,9 @@ class UtBotSymbolicEngine( private var queuedSymbolicStateUpdates = SymbolicStateUpdate() + private val featureProcessor: FeatureProcessor? = + if (featureProcess) EngineAnalyticsContext.featureProcessorFactory(globalGraph) else null + private val insideStaticInitializer get() = environment.state.executionStack.any { it.method.isStaticInitializer } @@ -355,6 +401,7 @@ class UtBotSymbolicEngine( logger.error(e) { "Closing resource failed" } } trackableResources.clear() + featureProcessor?.dumpFeatures() } private suspend fun preTraverse() { @@ -364,13 +411,16 @@ class UtBotSymbolicEngine( stateSelectedCount = 0 } - fun traverse(): Flow = traverseImpl().onStart { preTraverse() }.onCompletion { postTraverse() } + fun traverse(): Flow = traverseImpl() + .onStart { preTraverse() } + .onCompletion { postTraverse() } private fun traverseImpl(): Flow = flow { require(trackableResources.isEmpty()) if (useDebugVisualization) GraphViz(globalGraph, pathSelector) + if (UtSettings.collectCoverage) CoverageStatistics(methodUnderTest.toString(), globalGraph) val initStmt = graph.head val initState = ExecutionState( @@ -457,6 +507,7 @@ class UtBotSymbolicEngine( } else { val state = pathSelector.poll() + // state is null in case states queue is empty // or path selector exceed some limits (steps limit, for example) if (state == null) { @@ -476,32 +527,39 @@ class UtBotSymbolicEngine( } } - environment.state = state + state.executingTime += measureTimeMillis { + environment.state = state - val currentStmt = environment.state.stmt + val currentStmt = environment.state.stmt - environment.method = globalGraph.method(currentStmt) + if (currentStmt !in visitedStmts) { + environment.state.updateIsVisitedNew() + visitedStmts += currentStmt + } - environment.state.lastEdge?.let { - globalGraph.visitEdge(it) - } + environment.method = globalGraph.method(currentStmt) - try { - val exception = environment.state.exception - if (exception != null) { - traverseException(currentStmt, exception) - } else { - traverseStmt(currentStmt) + environment.state.lastEdge?.let { + globalGraph.visitEdge(it) } - } catch (ex: Throwable) { - environment.state.close() - if (ex !is CancellationException) { - logger.error(ex) { "Test generation failed on stmt $currentStmt, symbolic stack trace:\n$symbolicStackTrace" } - // TODO: enrich with nice description for known issues - emit(UtError(ex.description, ex)) - } else { - logger.debug(ex) { "Cancellation happened" } + try { + val exception = environment.state.exception + if (exception != null) { + traverseException(currentStmt, exception) + } else { + traverseStmt(currentStmt) + } + } catch (ex: Throwable) { + environment.state.close() + + if (ex !is CancellationException) { + logger.error(ex) { "Test generation failed on stmt $currentStmt, symbolic stack trace:\n$symbolicStackTrace" } + // TODO: enrich with nice description for known issues + emit(UtError(ex.description, ex)) + } else { + logger.debug(ex) { "Cancellation happened" } + } } } } @@ -1323,6 +1381,10 @@ class UtBotSymbolicEngine( val (positiveCaseSoftConstraint, negativeCaseSoftConstraint) = resolvedCondition.softConstraints val negativeCasePathConstraint = mkNot(positiveCasePathConstraint) + if (positiveCaseEdge != null) { + environment.state.updateIsFork() + } + positiveCaseEdge?.let { edge -> environment.state.expectUndefined() val positiveCaseState = environment.state.updateQueued( @@ -1393,6 +1455,11 @@ class UtBotSymbolicEngine( if (successors.size > 1) { environment.state.expectUndefined() } + + if (successors.size > 1) { + environment.state.updateIsFork() + } + successors.forEach { (target, expr) -> pathSelector.offer( environment.state.updateQueued( @@ -2600,6 +2667,10 @@ class UtBotSymbolicEngine( // If so, return the result of the override if (artificialMethodOverride.success) { + if (artificialMethodOverride.results.size > 1) { + environment.state.updateIsFork() + } + return mutableListOf().apply { for (result in artificialMethodOverride.results) { when (result) { @@ -2627,6 +2698,10 @@ class UtBotSymbolicEngine( // that will override the invocation. val overrideResults = targets.map { it to overrideInvocation(invocation, it) } + if (overrideResults.sumOf { (_, overriddenResult) -> overriddenResult.results.size } > 1) { + environment.state.updateIsFork() + } + // Separate targets for which invocation should be overridden // from the targets that should be processed regularly. val (overridden, original) = overrideResults.partition { it.second.success } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt index 2d3110908a..8c8137fd0c 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt @@ -59,7 +59,8 @@ abstract class BasePathSelector( /** * @return true if [utSolver] constraints are satisfiable */ - private fun checkUnsat(utSolver: UtSolver): Boolean = utSolver.assertions.isNotEmpty() && utSolver.check(respectSoft = false).statusKind != SAT + private fun checkUnsat(utSolver: UtSolver): Boolean = + utSolver.assertions.isNotEmpty() && utSolver.check(respectSoft = false).statusKind != SAT /** * check fast unsat on forks diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt new file mode 100644 index 0000000000..fb0325be4f --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt @@ -0,0 +1,70 @@ +package org.utbot.engine.selectors + +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.Predictors +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.nurs.GreedySearch +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf + * + * Calculates reward using neural network, when state is offered, and then peeks state with maximum reward + * + * @see [GreedySearch] + */ +abstract class NNRewardGuidedSelector( + protected val generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph +) : GreedySearch(choosingStrategy, stoppingStrategy, seed) { + protected val featureExtractor = EngineAnalyticsContext.featureExtractorFactory(graph) + + override val name: String + get() = "NNRewardGuidedSelector" +} + +/** + * Calculate weight of execution state only when it is offered. It has advantage, because it works faster, + * but disadvantage is that some features of execution state can change. + */ +class NNRewardGuidedSelectorWithoutRecalculationWeight( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph +) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { + override fun offerImpl(state: ExecutionState) { + super.offerImpl(state) + featureExtractor.extractFeatures(state, generatedTestCountingStatistics.generatedTestsCount) + } + + override val ExecutionState.weight: Double + get() { + reward = reward ?: Predictors.stateRewardPredictor.predict(features) + return reward as Double + } +} + +/** + * Calculate weight of execution state every time when it needed. It works slower, but features are always relevant + */ +class NNRewardGuidedSelectorWithRecalculationWeight( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph +) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { + override val ExecutionState.weight: Double + get() { + featureExtractor.extractFeatures(this, generatedTestCountingStatistics.generatedTestsCount) + return Predictors.stateRewardPredictor.predict(features) + } +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt new file mode 100644 index 0000000000..c426064ade --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt @@ -0,0 +1,49 @@ +package org.utbot.engine.selectors + +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * Creates [NNRewardGuidedSelector] + */ +interface NNRewardGuidedSelectorFactory { + operator fun invoke( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42, + graph: InterProceduralUnitGraph + ): NNRewardGuidedSelector +} + +/** + * Creates [NNRewardGuidedSelectorWithRecalculationWeight] + */ +class NNRewardGuidedSelectorWithRecalculationFactory : NNRewardGuidedSelectorFactory { + override fun invoke( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int, + graph: InterProceduralUnitGraph + ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithRecalculationWeight( + generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph + ) +} + +/** + * Creates [NNRewardGuidedSelectorWithoutRecalculationWeight] + */ +class NNRewardGuidedSelectorWithoutRecalculationFactory : NNRewardGuidedSelectorFactory { + override fun invoke( + generatedTestCountingStatistics: GeneratedTestCountingStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int, + graph: InterProceduralUnitGraph + ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithoutRecalculationWeight( + generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph + ) +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt index 8647995743..38b8c06ee2 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt @@ -2,22 +2,13 @@ package org.utbot.engine.selectors +import org.utbot.analytics.EngineAnalyticsContext import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.TypeRegistry import org.utbot.engine.selectors.StrategyOption.DISTANCE import org.utbot.engine.selectors.StrategyOption.VISIT_COUNTING -import org.utbot.engine.selectors.nurs.CoveredNewSelector -import org.utbot.engine.selectors.nurs.DepthSelector -import org.utbot.engine.selectors.nurs.MinimalDistanceToUncovered -import org.utbot.engine.selectors.nurs.NeuroSatSelector -import org.utbot.engine.selectors.nurs.InheritorsSelector -import org.utbot.engine.selectors.nurs.RPSelector -import org.utbot.engine.selectors.nurs.VisitCountingSelector -import org.utbot.engine.selectors.strategies.ChoosingStrategy -import org.utbot.engine.selectors.strategies.DistanceStatistics -import org.utbot.engine.selectors.strategies.EdgeVisitCountingStatistics -import org.utbot.engine.selectors.strategies.StepsLimitStoppingStrategy -import org.utbot.engine.selectors.strategies.StoppingStrategy +import org.utbot.engine.selectors.nurs.* +import org.utbot.engine.selectors.strategies.* import org.utbot.framework.UtSettings.seedInPathSelector /** @@ -57,6 +48,34 @@ fun inheritorsSelector( builder: InheritorsSelectorBuilder.() -> (Unit) ) = InheritorsSelectorBuilder(graph, typeRegistry).apply(builder).build() + +/** + * build [SubpathGuidedSelector] using [SubpathGuidedSelectorBuilder] + */ +fun subpathGuidedSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: SubpathGuidedSelectorBuilder.() -> (Unit) +) = SubpathGuidedSelectorBuilder(graph, strategy).apply(builder).build() + +/** + * build [CPInstSelector] using [CPInstSelectorBuilder] + */ +fun cpInstSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: CPInstSelectorBuilder.() -> (Unit) +) = CPInstSelectorBuilder(graph, strategy).apply(builder).build() + +/** + * build [ForkDepthSelector] using [ForkDepthSelectorBuilder] + */ +fun forkDepthSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: ForkDepthSelectorBuilder.() -> (Unit) +) = ForkDepthSelectorBuilder(graph, strategy).apply(builder).build() + /** * build [DepthSelector] using [DepthSelectorBuilder] */ @@ -133,10 +152,22 @@ fun visitCountingSelector( fun interleavedSelector(graph: InterProceduralUnitGraph, builder: InterleavedSelectorBuilder.() -> (Unit)) = InterleavedSelectorBuilder(graph).apply(builder).build() +/** + * build [NNRewardGuidedSelector] using [NNRewardGuidedSelectorBuilder] + */ +fun nnRewardGuidedSelector( + graph: InterProceduralUnitGraph, + strategy: StrategyOption, + builder: NNRewardGuidedSelectorBuilder.() -> Unit +) = NNRewardGuidedSelectorBuilder(graph, strategy).apply(builder).build() + data class PathSelectorContext( val graph: InterProceduralUnitGraph, var distanceStatistics: DistanceStatistics? = null, + var subpathStatistics: SubpathStatistics? = null, + var statementsStatistics: StatementsStatistics? = null, var countingStatistics: EdgeVisitCountingStatistics? = null, + var generatedTestCountingStatistics: GeneratedTestCountingStatistics? = null, var stoppingStrategy: StoppingStrategy? = null, ) @@ -192,6 +223,59 @@ class InheritorsSelectorBuilder internal constructor( ) } +/** + * Builder for [SubpathGuidedSelector]. Used in [subpathGuidedSelector] + */ +class SubpathGuidedSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + private val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph) +) : PathSelectorBuilder(graph, context) { + val seed: Int? = seedInPathSelector + + override fun build() = SubpathGuidedSelector( + withSubpathStatistics(), + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed!! + ) +} + +/** + * Builder for [CPInstSelectorBuilder]. Used in [cpInstSelector] + */ +class CPInstSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + private val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph) +) : PathSelectorBuilder(graph, context) { + val seed: Int = 42 + + override fun build() = CPInstSelector( + withStatementsStatistics(), + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed + ) +} + +/** + * Builder for [ForkDepthSelector]. Used in [forkDepthSelector] + */ +class ForkDepthSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph) +) : PathSelectorBuilder(graph, context) { + val seed: Int = 42 + + override fun build() = ForkDepthSelector( + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed + ) +} + /** * Builder for [DepthSelector]. Used in [depthSelector] * @@ -424,6 +508,24 @@ class InterleavedSelectorBuilder internal constructor( override fun build() = InterleavedSelector(builders.map { it.build() }) } +/** + * Builder for [NNRewardGuidedSelector]. Used in [] + */ +class NNRewardGuidedSelectorBuilder internal constructor( + graph: InterProceduralUnitGraph, + private val strategy: StrategyOption, + context: PathSelectorContext = PathSelectorContext(graph), +) : PathSelectorBuilder(graph, context) { + private val seed = seedInPathSelector + override fun build() = EngineAnalyticsContext.nnRewardGuidedSelectorFactory( + withGeneratedTestCountingStatistics(), + withChoosingStrategy(strategy), + requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, + seed!!, + graph + ) +} + /** * Base pathSelectorBuilder that maintains context to attach necessary statistics to graph */ @@ -460,6 +562,36 @@ sealed class PathSelectorBuilder( return context.distanceStatistics!! } + /** + * Create new [SubpathStatistics] and attach it to graph or use created one + */ + protected fun withSubpathStatistics(): SubpathStatistics { + if (context.subpathStatistics == null) { + context.subpathStatistics = SubpathStatistics(graph) + } + return context.subpathStatistics!! + } + + /** + * Create new [StatementsStatistics] and attach it to graph or use created one + */ + protected fun withStatementsStatistics(): StatementsStatistics { + if (context.statementsStatistics == null) { + context.statementsStatistics = StatementsStatistics(graph) + } + return context.statementsStatistics!! + } + + /** + * Create new [GeneratedTestCountingStatistics] and attach it to graph or use created one + */ + protected fun withGeneratedTestCountingStatistics(): GeneratedTestCountingStatistics { + if (context.generatedTestCountingStatistics == null) { + context.generatedTestCountingStatistics = GeneratedTestCountingStatistics(graph) + } + return context.generatedTestCountingStatistics!! + } + /** * Create new [EdgeVisitCountingStatistics] and attach ti to graph or use created one */ diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt new file mode 100644 index 0000000000..5eae91b111 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt @@ -0,0 +1,29 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StatementsStatistics +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * Execution states are weighted as 1.0 / StatementStatistics.statementsInMethodCount(exState), + * where StatementStatistics.statementsInMethodCount(exState) + * is number of visited instructions in the current function of execution state + * + * https://github.com/klee/klee/blob/085c54b980a2f62c7c475d32b5d0ce9c6f97904f/lib/Core/Searcher.cpp#L207 + * + * @see [NonUniformRandomSearch] + */ +class CPInstSelector( + private val statementStatistics: StatementsStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int? = 42 +) : NonUniformRandomSearch(choosingStrategy, stoppingStrategy, seed) { + + override val name: String + get() = "NURS:CPICnt" + + override val ExecutionState.cost: Double + get() = statementStatistics.statementInMethodCount(this).toDouble() +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/ForkDepthSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/ForkDepthSelector.kt new file mode 100644 index 0000000000..7c5e27aaf7 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/ForkDepthSelector.kt @@ -0,0 +1,24 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StoppingStrategy + +/** + * Execution states are weighted as 1.0 / (exState.depth + 1), + * where exState.depth is number of forks on the executionState's path + * + * @see [NonUniformRandomSearch] + */ +class ForkDepthSelector( + policy: ChoosingStrategy, + stoppingPolicy: StoppingStrategy, + seed: Int? = 42 +) : NonUniformRandomSearch(policy, stoppingPolicy, seed) { + + override val ExecutionState.cost: Double + get() = depth.toDouble() + + override val name: String + get() = "ForkDepth" +} diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt new file mode 100644 index 0000000000..1bc85c305c --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt @@ -0,0 +1,75 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.BasePathSelector +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StoppingStrategy +import org.utbot.engine.selectors.strategies.StrategyObserver +import kotlin.properties.Delegates +import kotlin.random.Random + +/** + * Selects ExecutionState with maximum weight. + * If there are several states with equal maximum weight, than selects random from them. + */ +abstract class GreedySearch( + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 0 +) : BasePathSelector(choosingStrategy, stoppingStrategy), StrategyObserver { + private val states = mutableSetOf() + + private val randomGen: Random = Random(seed) + + override fun update() { + // do nothing by default + } + + override fun offerImpl(state: ExecutionState) { + states.add(state) + } + + override fun pollImpl(): ExecutionState? = peekImpl()?.also { remove(it) } + + override fun peekImpl(): ExecutionState? { + if (isEmpty()) { + return null + } + + val candidates = mutableListOf() + var bestWeight by Delegates.notNull() + + states.forEach { + val weight = it.weight + + if (candidates.isEmpty()) { + bestWeight = weight + candidates.add(it) + } else { + if (bestWeight <= weight) { + if (bestWeight < weight) { + bestWeight = weight + candidates.clear() + } + candidates += it + } + } + } + + return candidates.random(randomGen) + } + + override fun removeImpl(state: ExecutionState): Boolean { + return states.remove(state) + } + + override fun isEmpty() = states.isEmpty() + + override fun close() { + states.forEach { + it.close() + } + } + + protected abstract val ExecutionState.weight: Double +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt new file mode 100644 index 0000000000..f871ad5702 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt @@ -0,0 +1,26 @@ +package org.utbot.engine.selectors.nurs + +import org.utbot.engine.ExecutionState +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.StoppingStrategy +import org.utbot.engine.selectors.strategies.SubpathStatistics + +/** + * Peeks state with minimum frequency of relative subpath. + * See https://github.com/eth-sri/learch/blob/master/klee/lib/Core/Searcher.cpp#L738 + * + * @see [GreedySearch] + */ +class SubpathGuidedSelector( + private val subpathStatistics: SubpathStatistics, + choosingStrategy: ChoosingStrategy, + stoppingStrategy: StoppingStrategy, + seed: Int = 42 +) : GreedySearch(choosingStrategy, stoppingStrategy, seed) { + + + override val name = "NURS:SubpathGuidedSearch" + + override val ExecutionState.weight: Double + get() = -subpathStatistics.subpathCount(this).toDouble() +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/GeneratedTestCountingStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/GeneratedTestCountingStatistics.kt new file mode 100644 index 0000000000..315673e042 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/GeneratedTestCountingStatistics.kt @@ -0,0 +1,15 @@ +package org.utbot.engine.selectors.strategies + +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph + +class GeneratedTestCountingStatistics( + graph: InterProceduralUnitGraph +) : TraverseGraphStatistics(graph) { + var generatedTestsCount = 0 + private set + + override fun onTraversed(executionState: ExecutionState) { + generatedTestsCount++ + } +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/StatementsStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/StatementsStatistics.kt new file mode 100644 index 0000000000..375462191c --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/StatementsStatistics.kt @@ -0,0 +1,37 @@ +package org.utbot.engine.selectors.strategies + +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import soot.SootMethod +import soot.jimple.Stmt + +/** + * Calculates + * - number of times for which state’s current instruction has been visited in [statementsCount] + * - number of instructions visited in state’s current method in [statementsInMethodCount] + */ +class StatementsStatistics( + graph: InterProceduralUnitGraph +) : TraverseGraphStatistics(graph) { + private val statementsCount = mutableMapOf() + private val statementsInMethodCount = mutableMapOf() + + override fun onVisit(executionState: ExecutionState) { + statementsCount.compute(executionState.stmt) { _, v -> + v?.plus(1) ?: 1 + } + + if (statementsCount[executionState.stmt] == 1) { + executionState.lastMethod?.let { + statementsInMethodCount.compute(it) { _, v -> + v?.plus(1) ?: 1 + } + } + } + } + + fun statementCount(executionState: ExecutionState) = statementsCount.getOrDefault(executionState.stmt, 1) + + fun statementInMethodCount(executionState: ExecutionState) = + executionState.lastMethod?.let { statementsInMethodCount.getOrDefault(it, 1) } ?: 0 +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt new file mode 100644 index 0000000000..52ec69bfb9 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt @@ -0,0 +1,47 @@ +package org.utbot.engine.selectors.strategies + +import org.utbot.engine.CALL_DECISION_NUM +import org.utbot.engine.Edge +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.framework.UtSettings +import kotlin.math.pow + +/** + * Calculates frequency of subpathes of statements with length equal to 2^{index}. + */ +class SubpathStatistics( + graph: InterProceduralUnitGraph, + index: Int = UtSettings.subpathGuidedSelectorIndex +) : TraverseGraphStatistics(graph) { + private val subpathCount = mutableMapOf, Int>() + private val length: Int = 2.0.pow(index).toInt() + + private fun ExecutionState.getSubpath(length: Int): List { + val subpath = mutableListOf() + val actualLength = if (pathLength >= length) length else pathLength + + var current = stmt + var exceptionNumber = 0 + (0 until actualLength).forEach { + val decision = decisionPath[decisionPath.size - it - 1] + if (decision < CALL_DECISION_NUM) { + exceptionNumber++ + } + val i = path.size - it - 1 + exceptionNumber + val prev = if (i == path.size) stmt else path[i] + subpath.add(Edge(prev, current, decision)) + current = prev + } + + return subpath + } + + override fun onVisit(executionState: ExecutionState) { + subpathCount.compute(executionState.getSubpath(length)) { _, v -> + v?.plus(1) ?: 1 + } + } + + fun subpathCount(executionState: ExecutionState): Int = subpathCount.getOrPut(executionState.getSubpath(length)) { 1 } +} \ No newline at end of file diff --git a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt index 1587b5661e..fa46b86250 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt @@ -2775,3 +2775,13 @@ inline fun withTreatingOverflowAsError(block: () -> T): T { UtSettings.treatOverflowAsError = prev } } + +inline fun withRewardModelPath(rewardModelPath: String, block: () -> T): T { + val prev = UtSettings.rewardModelPath + UtSettings.rewardModelPath = rewardModelPath + try { + return block() + } finally { + UtSettings.rewardModelPath = prev + } +} diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index cb8488684d..0a157bd0be 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -31,6 +31,9 @@ import kotlin.math.min import kotlin.system.exitProcess import mu.KotlinLogging import org.utbot.framework.JdkPathService +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.features.FeatureExtractorFactoryImpl +import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory private val logger = KotlinLogging.logger {} @@ -320,6 +323,8 @@ fun runEstimator( // Predictors.smt = UtBotTimePredictor() // Predictors.smtIncremental = UtBotTimePredictorIncremental() // Predictors.testName = StatementUniquenessPredictor() + EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() + EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() // fix for CTRL-ALT-SHIFT-C from IDEA, which copies in class#method form From a30845ef13206369aec12431acb82aa78765f009 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 27 May 2022 15:50:56 +0300 Subject: [PATCH 02/37] Fix style --- utbot-analytics/build.gradle | 7 +- .../kotlin/org/utbot/ClassifierTrainer.kt | 16 ++- .../main/kotlin/org/utbot/QualityAnalysis.kt | 61 +++++---- .../kotlin/org/utbot/QualityAnalysisV2.kt | 19 +-- .../kotlin/org/utbot/StmtCoverageReport.kt | 36 ++++-- .../FeatureProcessorWithStatesRepetition.kt | 3 +- .../features/UtExpressionGraphExtraction.kt | 60 ++++++++- .../predictors/LinearStateRewardPredictor.kt | 14 +- .../predictors/NNStateRewardPredictorSmile.kt | 14 +- .../predictors/NNStateRewardPredictorTorch.kt | 19 ++- .../org/utbot/predictors/UtBotSatPredictor.kt | 15 ++- .../org/utbot/utils/layers/MessagePassing.kt | 1 - .../utbot/visual/ClassificationHtmlReport.kt | 19 ++- .../kotlin/org/utbot/visual/FigureBuilders.kt | 120 +++++++++--------- .../kotlin/org/utbot/visual/HtmlBuilder.kt | 28 ++-- .../FeatureProcessorWithRepetitionTest.kt | 16 ++- .../LinearStateRewardPredictorTest.kt | 2 +- .../predictors/NNStateRewardPredictorTest.kt | 4 +- .../kotlin/org/utbot/framework/UtSettings.kt | 11 +- .../org/utbot/analytics/CoverageStatistics.kt | 4 +- .../utbot/analytics/EngineAnalyticsContext.kt | 1 - .../kotlin/org/utbot/analytics/Predictors.kt | 5 +- .../org/utbot/analytics/UtBotPredictor.kt | 3 +- .../kotlin/org/utbot/engine/ExecutionState.kt | 28 ++-- .../org/utbot/engine/UtBotSymbolicEngine.kt | 12 +- .../engine/selectors/BasePathSelector.kt | 2 +- .../engine/selectors/PathSelectorBuilder.kt | 20 ++- .../selectors/strategies/SubpathStatistics.kt | 3 +- .../examples/AbstractTestCaseGeneratorTest.kt | 14 +- .../org/utbot/contest/ContestEstimator.kt | 8 +- 30 files changed, 371 insertions(+), 194 deletions(-) diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index b41d264b9d..635a5fb7c5 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -50,8 +50,8 @@ dependencies { test { - useJUnitPlatform{ - excludeTags 'Summary' + useJUnitPlatform { + excludeTags 'Summary' } } @@ -64,7 +64,8 @@ processResources { } } -jar { dependsOn classes +jar { + dependsOn classes manifest { attributes 'Main-Class': 'org.utbot.QualityAnalysisKt' } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt index 3da7a15ab6..b99d3e9838 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt @@ -20,7 +20,7 @@ private const val dataPath = "logs/stats.txt" private const val logDir = "logs" class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = ClassifierModel.GBM) : - AbstractTrainer(data, savePcaVariance = true) { + AbstractTrainer(data, savePcaVariance = true) { private lateinit var metrics: ClassificationMetrics lateinit var model: Classifier val properties = Properties() @@ -62,7 +62,13 @@ class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = val xFrame = Formula.lhs(targetColumn).x(validationData) val x = xFrame.toArray(false, CategoricalEncoder.LEVEL) - metrics = ClassificationMetrics(classifierModel.name, model, Compose(transforms), actualLabel.map { it.toInt() }.toIntArray(), x) + metrics = ClassificationMetrics( + classifierModel.name, + model, + Compose(transforms), + actualLabel.map { it.toInt() }.toIntArray(), + x + ) } override fun visualize() { @@ -73,8 +79,10 @@ class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = addClassDistribution(classSizesBeforeResampling) addClassDistribution(classSizesAfterResampling, before = false) addPCAPlot(pcaVarianceProportion, pcaCumulativeVarianceProportion) - addMetrics(metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(), - metrics.recall.toDoubleArray()) + addMetrics( + metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(), + metrics.recall.toDoubleArray() + ) addConfusionMatrix(metrics.getNormalizedConfusionMatrix()) save() } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt index deaafb5b81..c55db51e19 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt @@ -1,16 +1,21 @@ package org.utbot -import org.utbot.QualityAnalysisConfig -import org.utbot.visual.FigureBuilders -import org.utbot.visual.HtmlBuilder import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.select.Elements +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder import java.io.File import java.nio.file.Paths -data class Coverage(val misInstructions: Int, val covInstruction: Int, val misBranches: Int, val covBranches: Int, val time: Double = 0.0) { +data class Coverage( + val misInstructions: Int, + val covInstruction: Int, + val misBranches: Int, + val covBranches: Int, + val time: Double = 0.0 +) { fun getInstructionCoverage(): Double = when { (this.misInstructions == 0 && this.covInstruction == 0) -> 0.0 (this.misInstructions == 0) -> 1.0 @@ -71,17 +76,20 @@ fun parseJacocoReport(path: String, classes: Set): Pair "Instructions (${it.first} model)" } }.flatten().toTypedArray(), - instructionMetrics.map { it.second }.flatten().toDoubleArray(), - title = "Coverage", - xLabel = "Instructions", - yLabel = "Count" - ) + FigureBuilders.buildBoxPlot( + instructionMetrics.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten() + .toTypedArray(), + instructionMetrics.map { it.second }.flatten().toDoubleArray(), + title = "Coverage", + xLabel = "Instructions", + yLabel = "Count" + ) ) // Instruction coverage report (sum covered instructions / sum instructions) @@ -128,7 +137,11 @@ fun main() { jacoco.first to jacoco.second.map { it.value.getInstructions() } }.toMap() covInstruction.forEach { - htmlBuilder.addText("Mean(Instruction (${it.first} model)) =${it.second.sum().toDouble() / (instructions[it.first]?.sum()?.toDouble() ?: 0.0)}") + htmlBuilder.addText( + "Mean(Instruction (${it.first} model)) =${ + it.second.sum().toDouble() / (instructions[it.first]?.sum()?.toDouble() ?: 0.0) + }" + ) } htmlBuilder.addFigure( FigureBuilders.buildBoxPlot( @@ -159,9 +172,11 @@ fun main() { ) // Save report - htmlBuilder.saveHTML(Paths.get( - QualityAnalysisConfig.outputDir, - QualityAnalysisConfig.project, - "test.html" - ).toFile().absolutePath) + htmlBuilder.saveHTML( + Paths.get( + QualityAnalysisConfig.outputDir, + QualityAnalysisConfig.project, + "test.html" + ).toFile().absolutePath + ) } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt index abb75b9c19..4b53d78afa 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt @@ -1,10 +1,10 @@ package org.utbot -import org.utbot.visual.FigureBuilders -import org.utbot.visual.HtmlBuilder import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.select.Elements +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder import java.io.File import java.nio.file.Paths @@ -19,7 +19,8 @@ fun parseReport(path: String, classes: Set): Map { val clsRows: Elements = packageDocument.select("table")[1].select("tr") for (j in 1 until clsRows.size) { - val clsName = clsRows[j].select("td")[0].select("a").attr("href").replace(".classes/", "").replace(".html", "") + val clsName = + clsRows[j].select("td")[0].select("a").attr("href").replace(".classes/", "").replace(".html", "") val fullName = packageHref.replace("/index.html", ".$clsName") if (classes.contains(fullName)) { @@ -59,9 +60,11 @@ fun main() { ) // Save report - htmlBuilder.saveHTML(Paths.get( - QualityAnalysisConfig.outputDir, - QualityAnalysisConfig.project, - QualityAnalysisConfig.selectors.joinToString("_") - ).toFile().absolutePath) + htmlBuilder.saveHTML( + Paths.get( + QualityAnalysisConfig.outputDir, + QualityAnalysisConfig.project, + QualityAnalysisConfig.selectors.joinToString("_") + ).toFile().absolutePath + ) } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt index a43bf2604a..b983c30e9b 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt @@ -1,18 +1,18 @@ package org.utbot +import org.apache.commons.io.FileUtils import org.utbot.visual.FigureBuilders import org.utbot.visual.HtmlBuilder +import smile.read import java.io.File import java.nio.file.Paths import kotlin.random.Random -import org.apache.commons.io.FileUtils -import smile.read data class Statistics( val perMethod: Map>, - val perClass: Map>, - val perProject: List + val perClass: Map>, + val perProject: List ) @@ -21,9 +21,9 @@ fun getStatistics(path: String, classes: Set): Statistics { var projectMinStartTime = Double.MAX_VALUE val rawStatistics = File(path).listFiles()?.filter { it.extension == "txt" && it.readLines().size > 1 }?.map { - val data = read.csv(it.absolutePath) - it.nameWithoutExtension to data.toArray() - }?.toMap() ?: emptyMap() + val data = read.csv(it.absolutePath) + it.nameWithoutExtension to data.toArray() + }?.toMap() ?: emptyMap() val statisticsPerMethod = rawStatistics.map { methodData -> val startTime = methodData.value[0][0] @@ -80,7 +80,9 @@ fun main() { File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } // Prepare folder - val projectDataPath = "${QualityAnalysisConfig.outputDir}/report/${QualityAnalysisConfig.project}/${QualityAnalysisConfig.selectors.joinToString("_")}" + val projectDataPath = "${QualityAnalysisConfig.outputDir}/report/${QualityAnalysisConfig.project}/${ + QualityAnalysisConfig.selectors.joinToString("_") + }" File(projectDataPath).deleteRecursively() classes.forEach { File(projectDataPath, it).mkdirs() } File(projectDataPath, "css").mkdirs() @@ -99,8 +101,10 @@ fun main() { "rgb(${rnd.nextInt(256)}, ${rnd.nextInt(256)}, ${rnd.nextInt(256)})" } - val htmlBuilderForProject = HtmlBuilder(pathToStyle = listOf("css/coverage.css", "css/highlight-idea.css"), - pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js")) + val htmlBuilderForProject = HtmlBuilder( + pathToStyle = listOf("css/coverage.css", "css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") + ) htmlBuilderForProject.addTable( statistics.map { it.perClass.map { it.key to (it.value.lastOrNull()?.lastOrNull() ?: 0.0) * 100 }.toMap() }, selectors, @@ -123,8 +127,10 @@ fun main() { classes.forEach { cls -> val outputDir = File(projectDataPath, cls) - val htmlBuilderForCls = HtmlBuilder(pathToStyle = listOf("../css/coverage.css", "../css/highlight-idea.css"), - pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js")) + val htmlBuilderForCls = HtmlBuilder( + pathToStyle = listOf("../css/coverage.css", "../css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") + ) val filteredStatistics = statistics.map { it.perMethod.filter { it.key.contains(cls) } } @@ -151,8 +157,10 @@ fun main() { htmlBuilderForCls.saveHTML(File(outputDir, "index.html").toString()) filteredStatistics.first().keys.forEach { method -> - val htmlBuilderForMethod = HtmlBuilder(pathToStyle = listOf("../../css/coverage.css", "../../css/highlight-idea.css"), - pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js")) + val htmlBuilderForMethod = HtmlBuilder( + pathToStyle = listOf("../../css/coverage.css", "../../css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") + ) htmlBuilderForMethod.addFigure( FigureBuilders.buildSeveralLinesPlot( filteredStatistics.map { it[method]?.map { it[0] / 1e9 }?.toDoubleArray() ?: doubleArrayOf() }, diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt index c78aa36d14..7c3ad0a2dc 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt @@ -5,11 +5,11 @@ import org.utbot.analytics.FeatureProcessor import org.utbot.engine.ExecutionState import org.utbot.engine.InterProceduralUnitGraph import org.utbot.framework.UtSettings +import soot.jimple.Stmt import java.io.File import java.io.FileOutputStream import java.nio.file.Paths import kotlin.math.pow -import soot.jimple.Stmt /** * Implementation of feature processor, in which we dump each test, so there will be several copies of each state. @@ -23,6 +23,7 @@ class FeatureProcessorWithStatesRepetition( init { File(saveDir).mkdirs() } + companion object { private val featureKeys = arrayOf( "stack", diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt index 76c3ee95b5..a0225ad807 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt @@ -1,8 +1,64 @@ package org.utbot.features +import org.utbot.engine.pc.NotBoolExpression +import org.utbot.engine.pc.UtAddNoOverflowExpression +import org.utbot.engine.pc.UtAddrExpression +import org.utbot.engine.pc.UtAndBoolExpression +import org.utbot.engine.pc.UtArrayApplyForAll +import org.utbot.engine.pc.UtArrayInsert +import org.utbot.engine.pc.UtArrayInsertRange +import org.utbot.engine.pc.UtArrayMultiStoreExpression +import org.utbot.engine.pc.UtArrayRemove +import org.utbot.engine.pc.UtArrayRemoveRange +import org.utbot.engine.pc.UtArraySelectExpression +import org.utbot.engine.pc.UtArraySetRange +import org.utbot.engine.pc.UtArrayShiftIndexes +import org.utbot.engine.pc.UtArrayToString +import org.utbot.engine.pc.UtBoolConst +import org.utbot.engine.pc.UtBoolOpExpression +import org.utbot.engine.pc.UtBvConst +import org.utbot.engine.pc.UtBvLiteral +import org.utbot.engine.pc.UtCastExpression +import org.utbot.engine.pc.UtConcatExpression +import org.utbot.engine.pc.UtConstArrayExpression +import org.utbot.engine.pc.UtContainsExpression +import org.utbot.engine.pc.UtConvertToString +import org.utbot.engine.pc.UtEndsWithExpression +import org.utbot.engine.pc.UtEqExpression +import org.utbot.engine.pc.UtEqGenericTypeParametersExpression +import org.utbot.engine.pc.UtExpression +import org.utbot.engine.pc.UtExpressionVisitor +import org.utbot.engine.pc.UtFalse +import org.utbot.engine.pc.UtFpConst +import org.utbot.engine.pc.UtFpLiteral +import org.utbot.engine.pc.UtGenericExpression +import org.utbot.engine.pc.UtIndexOfExpression +import org.utbot.engine.pc.UtInstanceOfExpression +import org.utbot.engine.pc.UtIsExpression +import org.utbot.engine.pc.UtIsGenericTypeExpression +import org.utbot.engine.pc.UtIteExpression +import org.utbot.engine.pc.UtMkArrayExpression +import org.utbot.engine.pc.UtMkTermArrayExpression +import org.utbot.engine.pc.UtNegExpression +import org.utbot.engine.pc.UtOpExpression +import org.utbot.engine.pc.UtOrBoolExpression +import org.utbot.engine.pc.UtReplaceExpression +import org.utbot.engine.pc.UtSeqLiteral +import org.utbot.engine.pc.UtStartsWithExpression +import org.utbot.engine.pc.UtStringCharAt +import org.utbot.engine.pc.UtStringConst +import org.utbot.engine.pc.UtStringEq +import org.utbot.engine.pc.UtStringLength +import org.utbot.engine.pc.UtStringPositiveLength +import org.utbot.engine.pc.UtStringToArray +import org.utbot.engine.pc.UtStringToInt +import org.utbot.engine.pc.UtSubNoOverflowExpression +import org.utbot.engine.pc.UtSubstringExpression +import org.utbot.engine.pc.UtToStringExpression +import org.utbot.engine.pc.UtTrue import org.utbot.utils.BinaryUtil -import org.utbot.engine.pc.* -import java.util.* +import java.util.IdentityHashMap +import java.util.UUID class UtExpressionId: UtExpressionVisitor { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index c75f22809d..8f7c079a56 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -2,14 +2,16 @@ package org.utbot.predictors import org.utbot.analytics.UtBotAbstractPredictor import org.utbot.framework.UtSettings -import java.io.File import smile.math.matrix.Matrix +import java.io.File /** * Last weight is bias */ private fun loadWeights(path: String): Matrix { - return Matrix(File("${UtSettings.rewardModelPath}/${path}").readText().split(",").map(String::toDouble).toDoubleArray()) + return Matrix( + File("${UtSettings.rewardModelPath}/${path}").readText().split(",").map(String::toDouble).toDoubleArray() + ) } class LinearStateRewardPredictor : UtBotAbstractPredictor>, List> { @@ -17,9 +19,11 @@ class LinearStateRewardPredictor : UtBotAbstractPredictor>, Li override fun predict(input: List>): List { // add 1 to each feature vector - val X = Matrix(input.map { it.toMutableList().also { featureVector -> - featureVector.add(1.0) - }.toDoubleArray() }.toTypedArray()) + val X = Matrix(input.map { + it.toMutableList().also { featureVector -> + featureVector.add(1.0) + }.toDoubleArray() + }.toTypedArray()) return X.mm(weights).col(0).toList() } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt index 70159df951..4b5f086d42 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt @@ -2,10 +2,10 @@ package org.utbot.predictors import com.google.gson.Gson import org.utbot.framework.UtSettings +import smile.math.matrix.Matrix import java.io.FileReader import java.nio.file.Paths import kotlin.math.max -import smile.math.matrix.Matrix data class NNJson( val linearLayers: Array> = arrayOf(), @@ -20,10 +20,11 @@ private fun reLU(input: DoubleArray): DoubleArray { } private fun loadNN(path: String): NN { - val nnJson: NNJson = Gson().fromJson(FileReader(Paths.get(UtSettings.rewardModelPath, path).toFile()), NNJson::class.java) ?: run { - System.err.println("Something went wrong while parsing NN model") - NNJson() - } + val nnJson: NNJson = + Gson().fromJson(FileReader(Paths.get(UtSettings.rewardModelPath, path).toFile()), NNJson::class.java) ?: run { + System.err.println("Something went wrong while parsing NN model") + NNJson() + } val weights = nnJson.linearLayers.map { Matrix(it) } val biases = nnJson.biases.map { Matrix(it) } @@ -55,7 +56,8 @@ internal fun loadScaler(path: String): StandardScaler = StandardScaler(Matrix(mean), Matrix(variance)) } -class NNStateRewardPredictorSmile(modelPath: String = "nn.json", scalerPath: String = "scaler.txt") : NNStateRewardPredictor { +class NNStateRewardPredictorSmile(modelPath: String = "nn.json", scalerPath: String = "scaler.txt") : + NNStateRewardPredictor { val nn = loadNN(modelPath) val scaler = loadScaler(scalerPath) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt index 3684913c5b..d53eb5a935 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt @@ -1,21 +1,26 @@ package org.utbot.predictors +import ai.djl.Model +import ai.djl.inference.Predictor +import ai.djl.ndarray.NDArray +import ai.djl.ndarray.NDList +import ai.djl.ndarray.NDManager +import ai.djl.translate.Translator +import ai.djl.translate.TranslatorContext import org.utbot.framework.UtSettings -import java.nio.file.Paths -import ai.djl.* -import ai.djl.inference.*; -import ai.djl.ndarray.*; -import ai.djl.translate.*; import java.io.Closeable +import java.nio.file.Paths class NNStateRewardPredictorTorch : NNStateRewardPredictor, Closeable { - val model: ai.djl.Model = ai.djl.Model.newInstance("model") + val model: Model = Model.newInstance("model") + init { model.load(Paths.get(UtSettings.rewardModelPath, "model.pt1")) } + val predictor: Predictor, Float> = model.newPredictor(object : Translator, Float> { override fun processInput(ctx: TranslatorContext, input: List): NDList { - val manager: NDManager = ctx.getNDManager() + val manager: NDManager = ctx.ndManager val array: NDArray = manager.create(input.toFloatArray()) return NDList(array) } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt index d005dcdf71..798a782bf3 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt @@ -4,7 +4,6 @@ import org.utbot.analytics.IUtBotSatPredictor import org.utbot.analytics.NeuroSatData import org.utbot.analytics.UtBotAbstractPredictor import org.utbot.engine.pc.UtExpression -import org.utbot.engine.pc.UtSolverStatusKind import org.utbot.features.UtExpressionGraphExtraction import org.utbot.features.UtExpressionId import org.utbot.utils.BinaryUtil @@ -13,7 +12,7 @@ import java.io.FileOutputStream @Suppress("unused") class UtBotSatPredictor : UtBotAbstractPredictor, NeuroSatData>, - IUtBotSatPredictor> { + IUtBotSatPredictor> { private var counter = 1 private var folder = "logs/GRAPHS" @@ -33,12 +32,16 @@ class UtBotSatPredictor : UtBotAbstractPredictor, NeuroSa } File("${folder}/graphs/$counter/vertexesType.txt").printWriter().use { out -> out.println("UUID,TYPE") - out.print(extractor.expressionIdsCache.keys.map { "${extractor.getID(it)},${ - BinaryUtil.binaryExpressionString(UtExpressionId().getID(it))}" }.joinToString("\n")) - } + out.print(extractor.expressionIdsCache.keys.map { + "${extractor.getID(it)},${ + BinaryUtil.binaryExpressionString(UtExpressionId().getID(it)) + }" + }.joinToString("\n")) + } File("${folder}/graphs/$counter/vertexesValues.txt").printWriter().use { out -> out.println("UUID,VALUE") - out.print(extractor.literalValues.keys.map { "${extractor.getID(it)},${extractor.literalValues[it]}" }.joinToString("\n")) + out.print(extractor.literalValues.keys.map { "${extractor.getID(it)},${extractor.literalValues[it]}" } + .joinToString("\n")) } FileOutputStream("${folder}/SAT.txt", true).bufferedWriter().use { out -> diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt index 6da6b609dd..6f1ce65336 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt @@ -2,7 +2,6 @@ package org.utbot.utils.layers import org.utbot.utils.MatrixUtil import java.io.FileNotFoundException -import java.util.ArrayList class MessagePassing { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt index ced6b5a0e8..9676fbc543 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt @@ -5,7 +5,7 @@ import java.time.format.DateTimeFormatter import java.util.* -class ClassificationHTMLReport() { +class ClassificationHTMLReport { private val builder = HtmlBuilder() fun addHeader(modelName: String, properties: Properties) { @@ -56,7 +56,7 @@ class ClassificationHTMLReport() { ) } - fun addPCAPlot(variance: DoubleArray, cumulativeVariance: DoubleArray){ + fun addPCAPlot(variance: DoubleArray, cumulativeVariance: DoubleArray) { builder.addFigure( FigureBuilders.buildTwoLinesPlot( variance, @@ -68,12 +68,19 @@ class ClassificationHTMLReport() { ) } - fun addMetrics(acc: Double, f1: Double, predTime: Double, precision: DoubleArray, recall: DoubleArray){ + fun addMetrics(acc: Double, f1: Double, predTime: Double, precision: DoubleArray, recall: DoubleArray) { builder.addText("Accuracy ${String.format("%.3f", acc)}") builder.addText("F1 macro ${String.format("%.3f", f1)}") builder.addText("AvgPredTime ${String.format("%.3f", predTime)}(ms)") - for ((i, value) in precision.withIndex()){ - builder.addText("For class $i precision ${String.format("%.3f", value)} recall ${String.format("%.3f", recall[i])}") + for ((i, value) in precision.withIndex()) { + builder.addText( + "For class $i precision ${String.format("%.3f", value)} recall ${ + String.format( + "%.3f", + recall[i] + ) + }" + ) } } @@ -84,7 +91,7 @@ class ClassificationHTMLReport() { .format(DateTimeFormatter.ofPattern("dd-MM-yyyy_HH-mm-ss")) + ".html" builder.saveHTML(filename) - }else { + } else { builder.saveHTML(filename) } } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt index 206489b3a7..8cfc0ddd15 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt @@ -10,37 +10,37 @@ import tech.tablesaw.plotly.traces.* class FigureBuilders { companion object { private fun getXYLayout( - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Plot" + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Plot" ): Layout { return Layout.builder() - .title(title) - .xAxis(Axis.builder().title(xLabel).build()) - .yAxis(Axis.builder().title(yLabel).build()) - .build() + .title(title) + .xAxis(Axis.builder().title(xLabel).build()) + .yAxis(Axis.builder().title(yLabel).build()) + .build() } private fun getXYZLayout( - xLabel: String = "X", - yLabel: String = "Y", - zLabel: String = "Y", - title: String = "Plot" + xLabel: String = "X", + yLabel: String = "Y", + zLabel: String = "Y", + title: String = "Plot" ): Layout { return Layout.builder() - .title(title) - .xAxis(Axis.builder().title(xLabel).build()) - .yAxis(Axis.builder().title(yLabel).build()) - .zAxis(Axis.builder().title(zLabel).build()) - .build() + .title(title) + .xAxis(Axis.builder().title(xLabel).build()) + .yAxis(Axis.builder().title(yLabel).build()) + .zAxis(Axis.builder().title(zLabel).build()) + .build() } fun buildScatterPlot( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Scatter plot" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Scatter plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = ScatterTrace.builder(x, y).build() @@ -49,10 +49,10 @@ class FigureBuilders { } fun buildHistogram( - data: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Histogram" + data: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Histogram" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = HistogramTrace.builder(data).build() @@ -61,11 +61,11 @@ class FigureBuilders { } fun build2DHistogram( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Histogram 2D" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Histogram 2D" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = Histogram2DTrace.builder(x, y).build() @@ -74,11 +74,11 @@ class FigureBuilders { } fun buildBarPlot( - x: Array, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "BarPlot" + x: Array, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "BarPlot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = BarTrace.builder(x, y).build() @@ -86,12 +86,12 @@ class FigureBuilders { } fun buildHeatmap( - x: Array, - y: Array, - z: Array, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Heatmap" + x: Array, + y: Array, + z: Array, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Heatmap" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = HeatmapTrace.builder(x, y, z).build() @@ -99,11 +99,11 @@ class FigureBuilders { } fun buildLinePlot( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Line plot" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Line plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = ScatterTrace.builder(x, y).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() @@ -112,15 +112,17 @@ class FigureBuilders { } fun buildTwoLinesPlot( - y1: DoubleArray, - y2: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Two lines plot" + y1: DoubleArray, + y2: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Two lines plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) - val trace1: Trace = ScatterTrace.builder(DoubleArray(y1.size) { it.toDouble() }, y1).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() - val trace2: Trace = ScatterTrace.builder(DoubleArray(y2.size) { it.toDouble() }, y2).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() + val trace1: Trace = ScatterTrace.builder(DoubleArray(y1.size) { it.toDouble() }, y1) + .mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() + val trace2: Trace = ScatterTrace.builder(DoubleArray(y2.size) { it.toDouble() }, y2) + .mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() return Figure(layout, trace1, trace2) } @@ -147,11 +149,13 @@ class FigureBuilders { return Figure(layout, *traces.toTypedArray()) } - fun buildBoxPlot(x: Array, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Box plot"): Figure { + fun buildBoxPlot( + x: Array, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Box plot" + ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: BoxTrace = BoxTrace.builder(x, y).build() return Figure(layout, trace) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt index ad8baf8218..91f7e4137f 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt @@ -1,14 +1,16 @@ package org.utbot.visual +import tech.tablesaw.plotly.components.Figure import java.io.File import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.* -import tech.tablesaw.plotly.components.Figure +import java.util.Properties -class HtmlBuilder(bodyMaxWidth: Int = 600, - pathToStyle: List = listOf(), - pathToJs: List = listOf()) { +class HtmlBuilder( + bodyMaxWidth: Int = 600, + pathToStyle: List = listOf(), + pathToJs: List = listOf() +) { private val pageTop = ("" + System.lineSeparator() + "" @@ -56,10 +58,12 @@ class HtmlBuilder(bodyMaxWidth: Int = 600, pageBuilder.append("<$h>${header}") } - fun addTable(statistics: List>, - selectorNames: List, - colors: List, - scope: String) { + fun addTable( + statistics: List>, + selectorNames: List, + colors: List, + scope: String + ) { pageBuilder.append("") pageBuilder.append("\n") @@ -72,10 +76,10 @@ class HtmlBuilder(bodyMaxWidth: Int = 600, for (i in statistics.indices) { pageBuilder.append( "\n" + "\n" + + "\n" ) } pageBuilder.append("") diff --git a/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt index 390a9e27f6..e847d7a057 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/features/FeatureProcessorWithRepetitionTest.kt @@ -37,11 +37,23 @@ class FeatureProcessorWithRepetitionTest { val rewardEstimator = RewardEstimator() val testCases = listOf( TestCase( - listOf(statesToInt["g0"]!! to 2L, statesToInt["f0"]!! to 2L, statesToInt["c0"]!! to 2L, statesToInt["a0"]!! to 1L), + listOf( + statesToInt["g0"]!! to 2L, + statesToInt["f0"]!! to 2L, + statesToInt["c0"]!! to 2L, + statesToInt["a0"]!! to 1L + ), newCoverage = 4, testIndex = 0 ), TestCase( - listOf(statesToInt["g1"]!! to 2L, statesToInt["f1"]!! to 1L,statesToInt["c1"]!! to 1L, statesToInt["f0"]!! to 2L, statesToInt["c0"]!! to 2L, statesToInt["a0"]!! to 1L), + listOf( + statesToInt["g1"]!! to 2L, + statesToInt["f1"]!! to 1L, + statesToInt["c1"]!! to 1L, + statesToInt["f0"]!! to 2L, + statesToInt["c0"]!! to 2L, + statesToInt["a0"]!! to 1L + ), newCoverage = 0, testIndex = 1 ), diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt index 4f52f07d7b..360290a2fc 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -1,8 +1,8 @@ package org.utbot.predictors -import org.utbot.examples.withRewardModelPath import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +import org.utbot.examples.withRewardModelPath class LinearStateRewardPredictorTest { @Test diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index 767ebd2301..8c0008a4e7 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -1,10 +1,10 @@ package org.utbot.predictors -import org.utbot.examples.withRewardModelPath -import kotlin.system.measureNanoTime import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test +import org.utbot.examples.withRewardModelPath +import kotlin.system.measureNanoTime class NNStateRewardPredictorTest { @Test diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 46531da4e6..418b293e9a 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -1,12 +1,12 @@ package org.utbot.framework +import mu.KotlinLogging import org.utbot.common.PathUtil.toPath import java.io.FileInputStream import java.io.IOException import java.util.Properties import kotlin.properties.PropertyDelegateProvider import kotlin.reflect.KProperty -import mu.KotlinLogging private val logger = KotlinLogging.logger {} @@ -68,7 +68,8 @@ object UtSettings { private fun getIntProperty(defaultValue: Int) = getProperty(defaultValue, String::toInt) private fun getLongProperty(defaultValue: Long) = getProperty(defaultValue, String::toLong) private fun getStringProperty(defaultValue: String) = getProperty(defaultValue) { it } - private inline fun > getEnumProperty(defaultValue: T) = getProperty(defaultValue) { enumValueOf(it) } + private inline fun > getEnumProperty(defaultValue: T) = + getProperty(defaultValue) { enumValueOf(it) } /** @@ -157,7 +158,7 @@ object UtSettings { /* * Activate or deactivate tests on comments && names/displayNames * */ - var testSummary by getBooleanProperty( true) + var testSummary by getBooleanProperty(true) var testName by getBooleanProperty(true) var testDisplayName by getBooleanProperty(true) @@ -269,7 +270,9 @@ object UtSettings { /** * Timeout for specific concrete execution (in milliseconds). */ - var concreteExecutionTimeoutInChildProcess: Long by getLongProperty(DEFAULT_CONCRETE_EXECUTION_TIMEOUT_IN_CHILD_PROCESS_MS) + var concreteExecutionTimeoutInChildProcess: Long by getLongProperty( + DEFAULT_CONCRETE_EXECUTION_TIMEOUT_IN_CHILD_PROCESS_MS + ) /** * Number of branch instructions using for clustering executions in the test minimization phase. diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt index cf8b79431d..5ddb2ea95c 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt @@ -1,5 +1,6 @@ package org.utbot.analytics +import mu.KotlinLogging import org.utbot.engine.ExecutionState import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.selectors.strategies.TraverseGraphStatistics @@ -7,7 +8,6 @@ import org.utbot.engine.stmts import org.utbot.framework.UtSettings import java.io.File import java.io.FileOutputStream -import mu.KotlinLogging private val logger = KotlinLogging.logger {} @@ -16,7 +16,7 @@ private val logger = KotlinLogging.logger {} class CoverageStatistics( private val method: String, private val globalGraph: InterProceduralUnitGraph -): TraverseGraphStatistics(globalGraph) { +) : TraverseGraphStatistics(globalGraph) { private val outputFile: String = "${UtSettings.coverageStatisticsDir}/$method.txt" diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt index ddafedff5b..d2fb78c405 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -3,7 +3,6 @@ package org.utbot.analytics import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.selectors.NNRewardGuidedSelectorFactory import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationFactory -import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationWeight import org.utbot.engine.selectors.NNRewardGuidedSelectorWithoutRecalculationFactory import org.utbot.framework.NNRewardGuidedSelectorType import org.utbot.framework.UtSettings diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt index ecae642d15..c58c3df2a4 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt @@ -1,7 +1,6 @@ package org.utbot.analytics import org.utbot.engine.pc.UtExpression -import kotlinx.collections.immutable.PersistentList import soot.jimple.Stmt /** @@ -18,7 +17,7 @@ object Predictors { * Predict z3's [Solver.check()] execution time in nanoseconds */ var smt: UtBotNanoTimePredictor> = object : UtBotNanoTimePredictor> {} - var smtIncremental: UtBotNanoTimePredictor = object : UtBotNanoTimePredictor {} + var smtIncremental: UtBotNanoTimePredictor = object : UtBotNanoTimePredictor {} var testName: UtBotAbstractPredictor, String> = object : UtBotAbstractPredictor, String> { override fun predict(input: Iterable): String = "stubName" @@ -31,6 +30,6 @@ object Predictors { } } - var sat: IUtBotSatPredictor> = object: IUtBotSatPredictor> {} + var sat: IUtBotSatPredictor> = object : IUtBotSatPredictor> {} var stateRewardPredictors: MutableList, Double>> = mutableListOf() } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt index dc5198d368..a6398096c7 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt @@ -57,7 +57,8 @@ inline fun UtBotNanoTimePredictor.learnOn(input: TIn, block: () -> * @see Predictors.smt */ interface IUtBotSatPredictor : UtBotAbstractPredictor { - override fun predict(input: TIn) = NeuroSatData(status = UtSolverStatusKind.UNSAT, time = 1) //Zero for default predictor + override fun predict(input: TIn) = + NeuroSatData(status = UtSolverStatusKind.UNSAT, time = 1) //Zero for default predictor } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt index 3519bf1643..3a27dab1cc 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt @@ -2,6 +2,7 @@ package org.utbot.engine import org.utbot.common.md5 import org.utbot.engine.pc.UtSolver +import org.utbot.engine.pc.UtSolverStatusUNDEFINED import org.utbot.engine.symbolic.SymbolicState import org.utbot.engine.symbolic.SymbolicStateUpdate import org.utbot.framework.plugin.api.Step @@ -13,7 +14,6 @@ import kotlinx.collections.immutable.persistentHashMapOf import kotlinx.collections.immutable.persistentHashSetOf import kotlinx.collections.immutable.persistentListOf import kotlinx.collections.immutable.plus -import org.utbot.engine.pc.UtSolverStatusUNDEFINED import soot.SootMethod import soot.jimple.Stmt @@ -176,7 +176,10 @@ data class ExecutionState( symbolicState = symbolicState, executionStack = executionStack.removeAt(executionStack.lastIndex), path = path + stmt, - visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put(stmtHashcode, stmtCountInPath), + visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put( + stmtHashcode, + stmtCountInPath + ), decisionPath = decisionPath + edge.decisionNum, edges = edges + edge, stmts = stmts.putIfAbsent(stmt, pathLength), @@ -212,7 +215,10 @@ data class ExecutionState( symbolicState = symbolicState.stateForNestedMethod() + update, executionStack = executionStack + stackElement, path = path + this.stmt, - visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put(stmtHashCode, stmtCountInPath), + visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put( + stmtHashCode, + stmtCountInPath + ), decisionPath = decisionPath + edge.decisionNum, edges = edges + edge, stmts = stmts.putIfAbsent(this.stmt, pathLength), @@ -228,7 +234,10 @@ data class ExecutionState( ): ExecutionState { val last = executionStack.last() val stackElement = last.update(stateUpdate.localMemoryUpdates) - return copy(symbolicState = symbolicState + stateUpdate, executionStack = executionStack.set(executionStack.lastIndex, stackElement)) + return copy( + symbolicState = symbolicState + stateUpdate, + executionStack = executionStack.set(executionStack.lastIndex, stackElement) + ) } fun update( @@ -251,7 +260,10 @@ data class ExecutionState( symbolicState = symbolicState + symbolicStateUpdate, executionStack = executionStack.set(executionStack.lastIndex, stackElement), path = path + stmt, - visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put(stmtHashCode, stmtCountInPath), + visitedStatementsHashesToCountInPath = visitedStatementsHashesToCountInPath.put( + stmtHashCode, + stmtCountInPath + ), decisionPath = decisionPath + edge.decisionNum, edges = edges + edge, stmts = stmts.putIfAbsent(stmt, pathLength), @@ -309,9 +321,9 @@ data class ExecutionState( val path = fullPath() val prettifiedPath = path.joinToString(separator = "\n") { (stmt, depth, decision) -> val prefix = when (decision) { - CALL_DECISION_NUM -> "call[${depth}] - " + "".padEnd(2*depth, ' ') - RETURN_DECISION_NUM -> " ret[${depth - 1}] - " + "".padEnd(2*depth, ' ') - else -> " "+"".padEnd(2*depth, ' ') + CALL_DECISION_NUM -> "call[${depth}] - " + "".padEnd(2 * depth, ' ') + RETURN_DECISION_NUM -> " ret[${depth - 1}] - " + "".padEnd(2 * depth, ' ') + else -> " " + "".padEnd(2 * depth, ' ') } "$prefix$stmt" } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 1e00f3c29d..2714a7322e 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -166,7 +166,17 @@ import mu.KotlinLogging import org.utbot.analytics.CoverageStatistics import org.utbot.analytics.EngineAnalyticsContext import org.utbot.analytics.FeatureProcessor -import org.utbot.engine.selectors.* +import org.utbot.engine.selectors.PathSelector +import org.utbot.engine.selectors.StrategyOption +import org.utbot.engine.selectors.coveredNewSelector +import org.utbot.engine.selectors.cpInstSelector +import org.utbot.engine.selectors.forkDepthSelector +import org.utbot.engine.selectors.inheritorsSelector +import org.utbot.engine.selectors.nnRewardGuidedSelector +import org.utbot.engine.selectors.pollUntilFastSAT +import org.utbot.engine.selectors.randomPathSelector +import org.utbot.engine.selectors.randomSelector +import org.utbot.engine.selectors.subpathGuidedSelector import org.utbot.framework.UtSettings.featureProcess import soot.ArrayType import soot.BooleanType diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt index 8c8137fd0c..571034332e 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/BasePathSelector.kt @@ -2,9 +2,9 @@ package org.utbot.engine.selectors import org.utbot.engine.ExecutionState import org.utbot.engine.pathLogger -import org.utbot.engine.pc.UtSolverStatusUNSAT import org.utbot.engine.pc.UtSolver import org.utbot.engine.pc.UtSolverStatusKind.SAT +import org.utbot.engine.pc.UtSolverStatusUNSAT import org.utbot.engine.selectors.strategies.ChoosingStrategy import org.utbot.engine.selectors.strategies.StoppingStrategy import org.utbot.framework.UtSettings diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt index 38b8c06ee2..1d5fb681d8 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt @@ -7,8 +7,24 @@ import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.TypeRegistry import org.utbot.engine.selectors.StrategyOption.DISTANCE import org.utbot.engine.selectors.StrategyOption.VISIT_COUNTING -import org.utbot.engine.selectors.nurs.* -import org.utbot.engine.selectors.strategies.* +import org.utbot.engine.selectors.nurs.CPInstSelector +import org.utbot.engine.selectors.nurs.CoveredNewSelector +import org.utbot.engine.selectors.nurs.DepthSelector +import org.utbot.engine.selectors.nurs.ForkDepthSelector +import org.utbot.engine.selectors.nurs.InheritorsSelector +import org.utbot.engine.selectors.nurs.MinimalDistanceToUncovered +import org.utbot.engine.selectors.nurs.NeuroSatSelector +import org.utbot.engine.selectors.nurs.RPSelector +import org.utbot.engine.selectors.nurs.SubpathGuidedSelector +import org.utbot.engine.selectors.nurs.VisitCountingSelector +import org.utbot.engine.selectors.strategies.ChoosingStrategy +import org.utbot.engine.selectors.strategies.DistanceStatistics +import org.utbot.engine.selectors.strategies.EdgeVisitCountingStatistics +import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics +import org.utbot.engine.selectors.strategies.StatementsStatistics +import org.utbot.engine.selectors.strategies.StepsLimitStoppingStrategy +import org.utbot.engine.selectors.strategies.StoppingStrategy +import org.utbot.engine.selectors.strategies.SubpathStatistics import org.utbot.framework.UtSettings.seedInPathSelector /** diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt index 52ec69bfb9..50f601097a 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt @@ -43,5 +43,6 @@ class SubpathStatistics( } } - fun subpathCount(executionState: ExecutionState): Int = subpathCount.getOrPut(executionState.getSubpath(length)) { 1 } + fun subpathCount(executionState: ExecutionState): Int = + subpathCount.getOrPut(executionState.getSubpath(length)) { 1 } } \ No newline at end of file diff --git a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt index fa46b86250..1ba649a847 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt @@ -1607,7 +1607,7 @@ abstract class AbstractTestCaseGeneratorTest( summaryNameChecks: List<(String?) -> Boolean> = listOf(), summaryDisplayNameChecks: List<(String?) -> Boolean> = listOf() ) = internalCheck( - method, mockStrategy, branches, matchers, coverage, T1::class, T2::class, T3::class, T4::class , + method, mockStrategy, branches, matchers, coverage, T1::class, T2::class, T3::class, T4::class, arguments = ::withParamsMutationsAndResult, additionalDependencies = additionalDependencies, summaryTextChecks = summaryTextChecks, @@ -2625,7 +2625,9 @@ fun withStaticsBefore(ex: UtValueExecution<*>) = ex.paramsBefore + ex.staticsBef fun withStaticsAfter(ex: UtValueExecution<*>) = ex.paramsBefore + ex.staticsAfter + ex.evaluatedResult fun withThisAndStaticsAfter(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.staticsAfter + ex.evaluatedResult fun withThisAndResult(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.evaluatedResult -fun withThisStaticsBeforeAndResult(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.staticsBefore + ex.evaluatedResult +fun withThisStaticsBeforeAndResult(ex: UtValueExecution<*>) = + listOf(ex.callerBefore) + ex.paramsBefore + ex.staticsBefore + ex.evaluatedResult + fun withThisAndException(ex: UtValueExecution<*>) = listOf(ex.callerBefore) + ex.paramsBefore + ex.returnValue fun withMocks(ex: UtValueExecution<*>) = ex.paramsBefore + listOf(ex.mocks) + ex.evaluatedResult fun withMocksAndInstrumentation(ex: UtValueExecution<*>) = @@ -2735,7 +2737,10 @@ inline fun withoutMinimization(block: () -> T): T { } } -inline fun withSettingsFromTestFrameworkConfiguration(config: TestFrameworkConfiguration, block: () -> T): T { +inline fun withSettingsFromTestFrameworkConfiguration( + config: TestFrameworkConfiguration, + block: () -> T +): T { val substituteStaticsWithSymbolicVariable = UtSettings.substituteStaticsWithSymbolicVariable UtSettings.substituteStaticsWithSymbolicVariable = config.resetNonFinalFieldsAfterClinit try { @@ -2750,8 +2755,7 @@ inline fun withoutSubstituteStaticsWithSymbolicVariable(block: () -> T) { UtSettings.substituteStaticsWithSymbolicVariable = false try { block() - } - finally { + } finally { UtSettings.substituteStaticsWithSymbolicVariable = substituteStaticsWithSymbolicVariable } } diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index 0a157bd0be..54bf868fc7 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -1,5 +1,7 @@ package org.utbot.contest +import mu.KotlinLogging +import org.utbot.analytics.EngineAnalyticsContext import org.utbot.common.FileUtil import org.utbot.common.bracket import org.utbot.common.info @@ -13,6 +15,8 @@ import org.utbot.contest.Paths.evosuiteReportFile import org.utbot.contest.Paths.jarsDir import org.utbot.contest.Paths.moduleTestDir import org.utbot.contest.Paths.outputDir +import org.utbot.features.FeatureExtractorFactoryImpl +import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.withUtContext @@ -29,11 +33,7 @@ import java.util.zip.ZipInputStream import kotlin.concurrent.thread import kotlin.math.min import kotlin.system.exitProcess -import mu.KotlinLogging import org.utbot.framework.JdkPathService -import org.utbot.analytics.EngineAnalyticsContext -import org.utbot.features.FeatureExtractorFactoryImpl -import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory private val logger = KotlinLogging.logger {} From 498dc30d6aebd2049bf0ddfde265bd6855012035 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 27 May 2022 17:07:01 +0300 Subject: [PATCH 03/37] Fix some issues --- .../kotlin/org/utbot/QualityAnalysisConfig.kt | 5 +- .../utbot/features/FeatureExtractorImpl.kt | 31 +++-- .../FeatureProcessorWithStatesRepetition.kt | 62 ++++----- .../predictors/LinearStateRewardPredictor.kt | 24 ++-- .../predictors/NNStateRewardPredictorSmile.kt | 55 ++++++-- .../predictors/NNStateRewardPredictorTorch.kt | 11 +- .../main/kotlin/org/utbot/predictors/util.kt | 4 + .../predictors/NNStateRewardPredictorTest.kt | 39 +++--- .../utbot/analytics/EngineAnalyticsContext.kt | 1 - .../kotlin/org/utbot/analytics/Predictors.kt | 3 +- .../kotlin/org/utbot/engine/ExecutionState.kt | 30 +++-- .../org/utbot/engine/UtBotSymbolicEngine.kt | 121 ++++++++++++------ .../engine/selectors/nurs/GreedySearch.kt | 4 +- .../selectors/strategies/SubpathStatistics.kt | 2 +- 14 files changed, 239 insertions(+), 153 deletions(-) create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt index 9d3fae8e09..46dc6c13dc 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt @@ -4,7 +4,8 @@ import java.io.FileInputStream import java.util.Properties object QualityAnalysisConfig { - const val configPath = "utbot-analytics/src/main/resources/config.properties" + private const val configPath = "utbot-analytics/src/main/resources/config.properties" + private val properties = Properties().also { props -> FileInputStream(configPath).use { inputStream -> props.load(inputStream) @@ -15,6 +16,6 @@ object QualityAnalysisConfig { val selectors: List = properties.getProperty("selectors").split(",") val covStatistics: List = properties.getProperty("covStatistics").split(",") - val outputDir: String = "eval/res" + const val outputDir: String = "eval/res" val classesList: String = "utbot-junit-contest/src/main/resources/classes/${project}/list" } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt index cd9362aa6f..5ea2c3038d 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt @@ -22,21 +22,24 @@ class FeatureExtractorImpl(private val graph: InterProceduralUnitGraph) : Featur private val statementStatistics = StatementsStatistics(graph) override fun extractFeatures(executionState: ExecutionState, generatedTestCases: Int) { - if (executionState.features.isNotEmpty()) { - executionState.features.clear() - } + with(executionState.features) { + if (isNotEmpty()) { + clear() + } + + add(executionState.executionStack) // stack + add(graph.succs(executionState.stmt)) // successor + add(generatedTestCases) // testCase + add(executionState.visitedAfterLastFork) // coverage by branch + add(executionState.visitedBeforeLastFork + executionState.visitedAfterLastFork) // coverage by path + add(executionState.depth) // depth + add(statementStatistics.statementInMethodCount(executionState)) // cpicnt + add(statementStatistics.statementCount(executionState)) // icnt + add(executionState.stmtsSinceLastCovered) // covNew - executionState.features.add(executionState.executionStack) // stack - executionState.features.add(graph.succs(executionState.stmt)) // successor - executionState.features.add(generatedTestCases) // testCase - executionState.features.add(executionState.visitedAfterLastFork) // coverage by branch - executionState.features.add(executionState.visitedBeforeLastFork + executionState.visitedAfterLastFork) // coverage by path - executionState.features.add(executionState.depth) // depth - executionState.features.add(statementStatistics.statementInMethodCount(executionState)) // cpicnt - executionState.features.add(statementStatistics.statementCount(executionState)) // icnt - executionState.features.add(executionState.stmtsSinceLastCovered) // covNew - subpathStatistics.forEach { - executionState.features.add(it.subpathCount(executionState)) // sgs_i + subpathStatistics.forEach { + add(it.subpathCount(executionState)) // sgs_i + } } } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt index 7c3ad0a2dc..398e398047 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt @@ -14,7 +14,7 @@ import kotlin.math.pow /** * Implementation of feature processor, in which we dump each test, so there will be several copies of each state. * Extract features for state when this state will be marked visited in graph. - * Add test case when last state of test case will be traversed. + * Add test case, when last state of it will be traversed. */ class FeatureProcessorWithStatesRepetition( graph: InterProceduralUnitGraph, @@ -57,25 +57,26 @@ class FeatureProcessorWithStatesRepetition( private fun addTestCase(executionState: ExecutionState) { val states = mutableListOf>() var newCoverage = 0 - var currentState: ExecutionState? = executionState - do { - val stateHashCode = currentState?.hashCode() ?: 0 - if (currentState?.features?.isEmpty() == true) { + generateSequence(executionState) { currentState -> + val stateHashCode = currentState.hashCode() + + if (currentState.features.isEmpty()) { extractFeatures(currentState) } - currentState?.executingTime?.let { states += (stateHashCode to it) } - currentState?.features?.let { dumpedStates[stateHashCode] = it } - currentState?.stmt?.let { - if (it !in visitedStmts && currentState?.isInNestedMethod() == false) { - visitedStmts.add(it) + states += stateHashCode to currentState.executingTime + dumpedStates[stateHashCode] = currentState.features + + currentState.stmt.let { + if (it !in visitedStmts && !currentState.isInNestedMethod()) { + visitedStmts += it newCoverage++ } } - currentState = currentState?.parent - } while (currentState != null) + currentState.parent + } generatedTestCases++ testCases += TestCase(states, newCoverage, generatedTestCases) @@ -85,22 +86,23 @@ class FeatureProcessorWithStatesRepetition( val rewards = rewardEstimator.calculateRewards(testCases) testCases.forEach { ts -> - FileOutputStream( - Paths.get(saveDir, "${UtSettings.testCounter++}.csv").toFile(), - true - ).bufferedWriter().use { out -> - out.appendLine("newCov,reward,${featureKeys.joinToString(separator = ",")}") - val reversedStates = ts.states.asReversed() - - reversedStates.forEach { (state, _) -> - out.appendLine( - "${ts.newCoverage != 0},${rewards[state]}," + - "${dumpedStates[state]?.joinToString(separator = ",")}" - ) + val outputFile = Paths.get(saveDir, "${UtSettings.testCounter++}.csv").toFile() + FileOutputStream(outputFile, true) + .bufferedWriter() + .use { out -> + out.appendLine("newCov,reward,${featureKeys.joinToString(separator = ",")}") + val reversedStates = ts.states.asReversed() + + reversedStates.forEach { (state, _) -> + val isCoveredNew = ts.newCoverage != 0 + val reward = rewards[state] + val features = dumpedStates[state]?.joinToString(separator = ",") + + out.appendLine("$isCoveredNew,$reward,$features") + } + + out.flush() } - - out.flush() - } } } @@ -126,18 +128,18 @@ internal class RewardEstimator { coverages.compute(state) { _, v -> ts.newCoverage + (v ?: 0) } - val updateTime = !times.contains(state) + val isNewState = state !in times times.compute(state) { _, v -> allTime + (v ?: time) } - if (updateTime) { + if (isNewState) { allTime += time } } } coverages.forEach { (state, coverage) -> - rewards[state] = reward(coverage.toDouble(), times[state]!!.toDouble()) + rewards[state] = reward(coverage.toDouble(), times.getValue(state).toDouble()) } return rewards diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index 8f7c079a56..1da24fd267 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -5,25 +5,29 @@ import org.utbot.framework.UtSettings import smile.math.matrix.Matrix import java.io.File +private const val DEFAULT_WEIGHT_PATH = "linear.txt" + /** * Last weight is bias */ private fun loadWeights(path: String): Matrix { - return Matrix( - File("${UtSettings.rewardModelPath}/${path}").readText().split(",").map(String::toDouble).toDoubleArray() - ) + val weightsFile = File("${UtSettings.rewardModelPath}/${path}") + val weightsArray = weightsFile.readText().splitByCommaIntoDoubleArray() + return Matrix(weightsArray) } -class LinearStateRewardPredictor : UtBotAbstractPredictor>, List> { - val weights = loadWeights("linear.txt") +class LinearStateRewardPredictor(weightsPath: String = DEFAULT_WEIGHT_PATH) : + UtBotAbstractPredictor>, List> { + private val weights = loadWeights(weightsPath) override fun predict(input: List>): List { // add 1 to each feature vector - val X = Matrix(input.map { - it.toMutableList().also { featureVector -> - featureVector.add(1.0) - }.toDoubleArray() - }.toTypedArray()) + val matrixValues = input + .map { (it + 1.0).toDoubleArray() } + .toTypedArray() + + val X = Matrix(matrixValues) + return X.mm(weights).col(0).toList() } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt index 4b5f086d42..660aa190eb 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt @@ -1,17 +1,43 @@ package org.utbot.predictors import com.google.gson.Gson +import mu.KotlinLogging import org.utbot.framework.UtSettings import smile.math.matrix.Matrix import java.io.FileReader import java.nio.file.Paths import kotlin.math.max +private const val DEFAULT_MODEL_PATH = "nn.json" +private const val DEFAULT_SCALER_PATH = "scaler.txt" + +private val logger = KotlinLogging.logger {} + data class NNJson( - val linearLayers: Array> = arrayOf(), - val activationLayers: Array = arrayOf(), - val biases: Array = arrayOf() -) + val linearLayers: Array> = emptyArray(), + val activationLayers: Array = emptyArray(), + val biases: Array = emptyArray() +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as NNJson + + if (!linearLayers.contentDeepEquals(other.linearLayers)) return false + if (!activationLayers.contentEquals(other.activationLayers)) return false + if (!biases.contentDeepEquals(other.biases)) return false + + return true + } + + override fun hashCode(): Int { + var result = linearLayers.contentDeepHashCode() + result = 31 * result + activationLayers.contentHashCode() + result = 31 * result + biases.contentDeepHashCode() + return result + } +} data class NN(val operations: List<(DoubleArray) -> DoubleArray>) @@ -20,9 +46,10 @@ private fun reLU(input: DoubleArray): DoubleArray { } private fun loadNN(path: String): NN { + val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() val nnJson: NNJson = - Gson().fromJson(FileReader(Paths.get(UtSettings.rewardModelPath, path).toFile()), NNJson::class.java) ?: run { - System.err.println("Something went wrong while parsing NN model") + Gson().fromJson(FileReader(modelFile), NNJson::class.java) ?: run { + logger.debug { "Something went wrong while parsing NN model" } NNJson() } @@ -30,11 +57,11 @@ private fun loadNN(path: String): NN { val biases = nnJson.biases.map { Matrix(it) } val operations = mutableListOf<(DoubleArray) -> DoubleArray>() - (0 until nnJson.linearLayers.size).forEach { i -> + nnJson.linearLayers.indices.forEach { i -> operations.add { weights[i].mm(Matrix(it)).add(biases[i]).col(0) } - if (i != nnJson.linearLayers.size - 1) { + if (i != nnJson.linearLayers.lastIndex) { operations.add { when (nnJson.activationLayers[i]) { "reLU" -> reLU(it) @@ -51,15 +78,15 @@ data class StandardScaler(val mean: Matrix?, val variance: Matrix?) internal fun loadScaler(path: String): StandardScaler = Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { - val mean = it.readLine()?.split(',')?.map(String::toDouble)?.toDoubleArray() - val variance = it.readLine()?.split(',')?.map(String::toDouble)?.toDoubleArray() + val mean = it.readLine()?.splitByCommaIntoDoubleArray() + val variance = it.readLine()?.splitByCommaIntoDoubleArray() StandardScaler(Matrix(mean), Matrix(variance)) } -class NNStateRewardPredictorSmile(modelPath: String = "nn.json", scalerPath: String = "scaler.txt") : +class NNStateRewardPredictorSmile(modelPath: String = DEFAULT_MODEL_PATH, scalerPath: String = DEFAULT_SCALER_PATH) : NNStateRewardPredictor { - val nn = loadNN(modelPath) - val scaler = loadScaler(scalerPath) + private val nn = loadNN(modelPath) + private val scaler = loadScaler(scalerPath) override fun predict(input: List): Double { var inputArray = input.toDoubleArray() @@ -69,7 +96,7 @@ class NNStateRewardPredictorSmile(modelPath: String = "nn.json", scalerPath: Str inputArray = it(inputArray) } - check(inputArray.size == 1) + check(inputArray.size == 1) { "Neural network have several outputs" } return inputArray[0] } } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt index d53eb5a935..b92eb856c6 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorTorch.kt @@ -4,7 +4,6 @@ import ai.djl.Model import ai.djl.inference.Predictor import ai.djl.ndarray.NDArray import ai.djl.ndarray.NDList -import ai.djl.ndarray.NDManager import ai.djl.translate.Translator import ai.djl.translate.TranslatorContext import org.utbot.framework.UtSettings @@ -18,17 +17,13 @@ class NNStateRewardPredictorTorch : NNStateRewardPredictor, Closeable { model.load(Paths.get(UtSettings.rewardModelPath, "model.pt1")) } - val predictor: Predictor, Float> = model.newPredictor(object : Translator, Float> { + private val predictor: Predictor, Float> = model.newPredictor(object : Translator, Float> { override fun processInput(ctx: TranslatorContext, input: List): NDList { - val manager: NDManager = ctx.ndManager - val array: NDArray = manager.create(input.toFloatArray()) + val array: NDArray = ctx.ndManager.create(input.toFloatArray()) return NDList(array) } - override fun processOutput(ctx: TranslatorContext, list: NDList): Float { - val tmp: NDArray = list.get(0) - return tmp.getFloat() - } + override fun processOutput(ctx: TranslatorContext, list: NDList): Float = list[0].getFloat() }) override fun predict(input: List): Double { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt new file mode 100644 index 0000000000..039f6b5ddb --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt @@ -0,0 +1,4 @@ +package org.utbot.predictors + +fun String.splitByCommaIntoDoubleArray() = + split(',').map(String::toDouble).toDoubleArray() \ No newline at end of file diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index 8c0008a4e7..abac473fec 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -12,9 +12,7 @@ class NNStateRewardPredictorTest { withRewardModelPath("src/test/resources") { val pred = NNStateRewardPredictorSmile() - val features = listOf( - 0.0, 0.0 - ) + val features = listOf(0.0, 0.0) assertEquals(5.0, pred.predict(features)) } @@ -25,29 +23,30 @@ class NNStateRewardPredictorTest { fun performanceTest() { val features = (1..13).map { 1.0 }.toList() withRewardModelPath("models\\test\\0") { - val pred1 = NNStateRewardPredictorSmile() - - (1..100).map { - pred1.predict(features) - } - - println((1..100).map { - measureNanoTime { pred1.predict(features) } - }.sum().toDouble() / 100) + val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorSmile, 100, features) + println(averageTime) } withRewardModelPath("models") { - val pred2 = NNStateRewardPredictorTorch() + val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorTorch, 100, features) + println(averageTime) + } + } - (1..100).map { - pred2.predict(features) - } + private fun calcAverageTimeForModelPredict( + model: () -> NNStateRewardPredictor, + iterations: Int, + features: List + ): Double { + val pred = model() - println((1..100).map { - measureNanoTime { pred2.predict(features) } - }.sum().toDouble() / 100) - pred2.close() + (1..iterations).map { + pred.predict(features) } + + return (1..iterations) + .map { measureNanoTime { pred.predict(features) } } + .average() } } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt index d2fb78c405..bbce3162cc 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -26,6 +26,5 @@ object EngineAnalyticsContext { val nnRewardGuidedSelectorFactory: NNRewardGuidedSelectorFactory = when (UtSettings.nnRewardGuidedSelectorType) { NNRewardGuidedSelectorType.WITHOUT_RECALCULATION -> NNRewardGuidedSelectorWithoutRecalculationFactory() NNRewardGuidedSelectorType.WITH_RECALCULATION -> NNRewardGuidedSelectorWithRecalculationFactory() - else -> error("Unsupported nnRewardGuidedSelectorType") } } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt index c58c3df2a4..e0c7dc3632 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt @@ -26,10 +26,9 @@ object Predictors { var stateRewardPredictor: UtBotAbstractPredictor, Double> = object : UtBotAbstractPredictor, Double> { override fun predict(input: List): Double { - TODO("Not yet implemented") + error("stateRewardPredictor wasn't provided") } } var sat: IUtBotSatPredictor> = object : IUtBotSatPredictor> {} - var stateRewardPredictors: MutableList, Double>> = mutableListOf() } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt index 3a27dab1cc..e733606f62 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt @@ -1,12 +1,5 @@ package org.utbot.engine -import org.utbot.common.md5 -import org.utbot.engine.pc.UtSolver -import org.utbot.engine.pc.UtSolverStatusUNDEFINED -import org.utbot.engine.symbolic.SymbolicState -import org.utbot.engine.symbolic.SymbolicStateUpdate -import org.utbot.framework.plugin.api.Step -import java.util.Objects import kotlinx.collections.immutable.PersistentList import kotlinx.collections.immutable.PersistentMap import kotlinx.collections.immutable.PersistentSet @@ -14,8 +7,15 @@ import kotlinx.collections.immutable.persistentHashMapOf import kotlinx.collections.immutable.persistentHashSetOf import kotlinx.collections.immutable.persistentListOf import kotlinx.collections.immutable.plus +import org.utbot.common.md5 +import org.utbot.engine.pc.UtSolver +import org.utbot.engine.pc.UtSolverStatusUNDEFINED +import org.utbot.engine.symbolic.SymbolicState +import org.utbot.engine.symbolic.SymbolicStateUpdate +import org.utbot.framework.plugin.api.Step import soot.SootMethod import soot.jimple.Stmt +import java.util.Objects const val RETURN_DECISION_NUM = -1 const val CALL_DECISION_NUM = -2 @@ -61,13 +61,13 @@ data class StateAnalyticsProperties( /** * Flag that indicates whether this state is fork or not. Fork here means that we have more than one successor */ - var isFork: Boolean = false + private var isFork: Boolean = false fun updateIsFork() { isFork = true } - var isVisitedNew: Boolean = false + private var isVisitedNew: Boolean = false fun updateIsVisitedNew() { isVisitedNew = true @@ -361,10 +361,14 @@ data class ExecutionState( return true } - override fun hashCode(): Int = Objects.hash( - stmt, symbolicState, executionStack, path, visitedStatementsHashesToCountInPath, decisionPath, - edges, stmts, pathLength, lastEdge, lastMethod, methodResult, exception - ) + private val hashCode by lazy { + Objects.hash( + stmt, symbolicState, executionStack, path, visitedStatementsHashesToCountInPath, decisionPath, + edges, stmts, pathLength, lastEdge, lastMethod, methodResult, exception + ) + } + + override fun hashCode(): Int = hashCode var reward by stateAnalyticsProperties::reward val features by stateAnalyticsProperties::features diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 2714a7322e..fefe1cbf55 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -7,11 +7,20 @@ import kotlinx.collections.immutable.toPersistentList import kotlinx.collections.immutable.toPersistentMap import kotlinx.collections.immutable.toPersistentSet import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.FlowCollector import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.isActive import kotlinx.coroutines.yield +import mu.KotlinLogging +import org.utbot.analytics.CoverageStatistics +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.FeatureProcessor import org.utbot.analytics.Predictors import org.utbot.common.WorkaroundReason.HACK import org.utbot.common.WorkaroundReason.REMOVE_ANONYMOUS_CLASSES @@ -74,8 +83,19 @@ import org.utbot.engine.pc.mkNot import org.utbot.engine.pc.mkOr import org.utbot.engine.pc.select import org.utbot.engine.pc.store +import org.utbot.engine.selectors.PathSelector +import org.utbot.engine.selectors.StrategyOption +import org.utbot.engine.selectors.coveredNewSelector +import org.utbot.engine.selectors.cpInstSelector +import org.utbot.engine.selectors.forkDepthSelector +import org.utbot.engine.selectors.inheritorsSelector +import org.utbot.engine.selectors.nnRewardGuidedSelector import org.utbot.engine.selectors.nurs.NonUniformRandomSearch +import org.utbot.engine.selectors.pollUntilFastSAT +import org.utbot.engine.selectors.randomPathSelector +import org.utbot.engine.selectors.randomSelector import org.utbot.engine.selectors.strategies.GraphViz +import org.utbot.engine.selectors.subpathGuidedSelector import org.utbot.engine.symbolic.HardConstraint import org.utbot.engine.symbolic.SoftConstraint import org.utbot.engine.symbolic.SymbolicState @@ -92,6 +112,7 @@ import org.utbot.engine.util.statics.concrete.makeSymbolicValuesFromEnumConcrete import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import org.utbot.framework.UtSettings.checkSolverTimeoutMillis +import org.utbot.framework.UtSettings.featureProcess import org.utbot.framework.UtSettings.pathSelectorStepsLimit import org.utbot.framework.UtSettings.pathSelectorType import org.utbot.framework.UtSettings.preferredCexOption @@ -142,7 +163,6 @@ import kotlin.collections.plus import kotlin.collections.plusAssign import kotlin.math.max import kotlin.math.min -import kotlin.reflect.KClass import kotlin.reflect.full.instanceParameter import kotlin.reflect.full.valueParameters import kotlin.reflect.jvm.javaType @@ -152,22 +172,12 @@ import kotlinx.collections.immutable.persistentListOf import kotlinx.collections.immutable.persistentSetOf import kotlinx.collections.immutable.toPersistentList import kotlinx.collections.immutable.toPersistentSet -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Job import kotlinx.coroutines.currentCoroutineContext -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.FlowCollector import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.isActive import kotlinx.coroutines.yield -import mu.KotlinLogging -import org.utbot.analytics.CoverageStatistics -import org.utbot.analytics.EngineAnalyticsContext -import org.utbot.analytics.FeatureProcessor -import org.utbot.engine.selectors.PathSelector -import org.utbot.engine.selectors.StrategyOption import org.utbot.engine.selectors.coveredNewSelector import org.utbot.engine.selectors.cpInstSelector import org.utbot.engine.selectors.forkDepthSelector @@ -177,7 +187,6 @@ import org.utbot.engine.selectors.pollUntilFastSAT import org.utbot.engine.selectors.randomPathSelector import org.utbot.engine.selectors.randomSelector import org.utbot.engine.selectors.subpathGuidedSelector -import org.utbot.framework.UtSettings.featureProcess import soot.ArrayType import soot.BooleanType import soot.ByteType @@ -275,6 +284,10 @@ import kotlin.math.min import kotlin.reflect.full.instanceParameter import kotlin.reflect.full.valueParameters import kotlin.reflect.jvm.javaType +import kotlin.reflect.full.instanceParameter +import kotlin.reflect.full.valueParameters +import kotlin.reflect.jvm.javaType +import kotlin.system.measureTimeMillis private val logger = KotlinLogging.logger {} val pathLogger = KotlinLogging.logger(logger.name + ".path") @@ -461,8 +474,10 @@ class UtBotSymbolicEngine( } stateSelectedCount++ - pathLogger.trace { "traverse<$methodUnderTest>: choosing next state($stateSelectedCount), " + - "queue size=${(pathSelector as? NonUniformRandomSearch)?.size ?: -1}" } + pathLogger.trace { + "traverse<$methodUnderTest>: choosing next state($stateSelectedCount), " + + "queue size=${(pathSelector as? NonUniformRandomSearch)?.size ?: -1}" + } if (controller.executeConcretely || statesForConcreteExecution.isNotEmpty()) { val state = pathSelector.pollUntilFastSAT() ?: statesForConcreteExecution.pollUntilSat() ?: break @@ -590,7 +605,7 @@ class UtBotSymbolicEngine( val isFuzzable = executableId.parameters.all { classId -> classId != Method::class.java.id && // causes the child process crash at invocation - classId != Class::class.java.id // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method) + classId != Class::class.java.id // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method) } if (!isFuzzable) { return@flow @@ -1289,7 +1304,12 @@ class UtBotSymbolicEngine( ?: error("Exception wasn't caught, stmt: $current, line: ${current.lines}") val nextState = environment.state.updateQueued( globalGraph.succ(current), - SymbolicStateUpdate(localMemoryUpdates = localMemoryUpdate(localVariable to value, CAUGHT_EXCEPTION to null)) + SymbolicStateUpdate( + localMemoryUpdates = localMemoryUpdate( + localVariable to value, + CAUGHT_EXCEPTION to null + ) + ) ) pathSelector.offer(nextState) } @@ -1377,7 +1397,8 @@ class UtBotSymbolicEngine( } } - queuedSymbolicStateUpdates += typeRegistry.genericTypeParameterConstraint(value.addr, typeStorages).asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.genericTypeParameterConstraint(value.addr, typeStorages) + .asHardConstraint() parameterAddrToGenericType += value.addr to type } } @@ -1464,9 +1485,6 @@ class UtBotSymbolicEngine( } if (successors.size > 1) { environment.state.expectUndefined() - } - - if (successors.size > 1) { environment.state.updateIsFork() } @@ -1505,7 +1523,8 @@ class UtBotSymbolicEngine( queuedSymbolicStateUpdates += MemoryUpdate(mockInfos = persistentListOf(MockInfoEnriched(mockInfo))) // add typeConstraint for mocked object. It's a declared type of the object. - queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all().asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all() + .asHardConstraint() queuedSymbolicStateUpdates += mkEq(typeRegistry.isMock(mockedObject.addr), UtTrue).asHardConstraint() return mockedObject @@ -1543,7 +1562,8 @@ class UtBotSymbolicEngine( queuedSymbolicStateUpdates += MemoryUpdate(mockInfos = persistentListOf(MockInfoEnriched(mockInfo))) // add typeConstraint for mocked object. It's a declared type of the object. - queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all().asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all() + .asHardConstraint() queuedSymbolicStateUpdates += mkEq(typeRegistry.isMock(mockedObject.addr), UtTrue).asHardConstraint() return mockedObject @@ -1578,7 +1598,8 @@ class UtBotSymbolicEngine( createObject(addr, refType, useConcreteType = true) } } else { - queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, TypeStorage(refType)).all().asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, TypeStorage(refType)).all() + .asHardConstraint() objectValue(refType, addr, StringWrapper()).also { initStringLiteral(it, this.value) @@ -1697,7 +1718,11 @@ class UtBotSymbolicEngine( val arrayType = type.arrayType val arrayValue = createNewArray(value.length.toPrimitiveValue(), arrayType, type).also { val defaultValue = arrayType.defaultSymValue - queuedSymbolicStateUpdates += arrayUpdateWithValue(it.addr, arrayType, defaultValue as UtArrayExpressionBase) + queuedSymbolicStateUpdates += arrayUpdateWithValue( + it.addr, + arrayType, + defaultValue as UtArrayExpressionBase + ) } queuedSymbolicStateUpdates += objectUpdate( stringWrapper.copy(typeStorage = TypeStorage(utStringClass.type)), @@ -1861,7 +1886,8 @@ class UtBotSymbolicEngine( return objectValue.copy(addr = nullObjectAddr) } - val typeConstraint = typeRegistry.typeConstraint(castedObject.addr, castedObject.typeStorage).isOrNullConstraint() + val typeConstraint = + typeRegistry.typeConstraint(castedObject.addr, castedObject.typeStorage).isOrNullConstraint() // When we do downCast, we should add possible equality to null // to avoid situation like this: @@ -2017,7 +2043,8 @@ class UtBotSymbolicEngine( ) if (objectValue.type.isJavaLangObject()) { - queuedSymbolicStateUpdates += typeRegistry.zeroDimensionConstraint(objectValue.addr).asSoftConstraint() + queuedSymbolicStateUpdates += typeRegistry.zeroDimensionConstraint(objectValue.addr) + .asSoftConstraint() } objectValue @@ -2889,7 +2916,11 @@ class UtBotSymbolicEngine( return listOf( MethodResult( newArray, - memoryUpdates = arrayUpdateWithValue(newArray.addr, arrayType, selectArrayExpressionFromMemory(src)) + memoryUpdates = arrayUpdateWithValue( + newArray.addr, + arrayType, + selectArrayExpressionFromMemory(src) + ) ) ) } @@ -3374,12 +3405,22 @@ class UtBotSymbolicEngine( mkNot(UtSubNoOverflowExpression(left.expr, right.expr)) } is JMulExpr -> when (sort.type) { - is ByteType -> lowerIntMulOverflowCheck(left, right, Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()) - is ShortType -> lowerIntMulOverflowCheck(left, right, Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) - is IntType -> higherIntMulOverflowCheck(left, right, Int.SIZE_BITS, Int.MIN_VALUE.toLong()) { it: UtExpression -> it.toIntValue() } - is LongType -> higherIntMulOverflowCheck(left, right, Long.SIZE_BITS, Long.MIN_VALUE) { it: UtExpression -> it.toLongValue() } - else -> null - } + is ByteType -> lowerIntMulOverflowCheck(left, right, Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()) + is ShortType -> lowerIntMulOverflowCheck(left, right, Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) + is IntType -> higherIntMulOverflowCheck( + left, + right, + Int.SIZE_BITS, + Int.MIN_VALUE.toLong() + ) { it: UtExpression -> it.toIntValue() } + is LongType -> higherIntMulOverflowCheck( + left, + right, + Long.SIZE_BITS, + Long.MIN_VALUE + ) { it: UtExpression -> it.toLongValue() } + else -> null + } else -> null } @@ -3593,11 +3634,19 @@ class UtBotSymbolicEngine( //choose types that have biggest priority resolvedParameters .filterIsInstance() - .forEach { queuedSymbolicStateUpdates += constructConstraintForType(it, it.possibleConcreteTypes).asSoftConstraint() } + .forEach { + queuedSymbolicStateUpdates += constructConstraintForType( + it, + it.possibleConcreteTypes + ).asSoftConstraint() + } val returnValue = (symbolicResult as? SymbolicSuccess)?.value as? ObjectValue if (returnValue != null) { - queuedSymbolicStateUpdates += constructConstraintForType(returnValue, returnValue.possibleConcreteTypes).asSoftConstraint() + queuedSymbolicStateUpdates += constructConstraintForType( + returnValue, + returnValue.possibleConcreteTypes + ).asSoftConstraint() workaround(REMOVE_ANONYMOUS_CLASSES) { val sootClass = returnValue.type.sootClass @@ -3747,7 +3796,7 @@ class UtBotSymbolicEngine( queuedSymbolicStateUpdates ) } finally { - queuedSymbolicStateUpdates = prevSymbolicStateUpdate + queuedSymbolicStateUpdates = prevSymbolicStateUpdate } } } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt index 1bc85c305c..b2a587217d 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt @@ -26,7 +26,7 @@ abstract class GreedySearch( } override fun offerImpl(state: ExecutionState) { - states.add(state) + states += state } override fun pollImpl(): ExecutionState? = peekImpl()?.also { remove(it) } @@ -44,7 +44,7 @@ abstract class GreedySearch( if (candidates.isEmpty()) { bestWeight = weight - candidates.add(it) + candidates += it } else { if (bestWeight <= weight) { if (bestWeight < weight) { diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt index 50f601097a..1422c2e784 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt @@ -30,7 +30,7 @@ class SubpathStatistics( } val i = path.size - it - 1 + exceptionNumber val prev = if (i == path.size) stmt else path[i] - subpath.add(Edge(prev, current, decision)) + subpath += Edge(prev, current, decision) current = prev } From b92813a1cb9c9e6b30cb975dd8ac922b8e31b544 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 27 May 2022 17:08:10 +0300 Subject: [PATCH 04/37] Add jvmArgs for framework tests --- utbot-framework/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utbot-framework/build.gradle b/utbot-framework/build.gradle index 2e3c4a7190..ce70c24f32 100644 --- a/utbot-framework/build.gradle +++ b/utbot-framework/build.gradle @@ -74,4 +74,6 @@ test { if (System.getProperty('DEBUG', 'false') == 'true') { jvmArgs '-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=9009' } + + jvmArgs "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap" } \ No newline at end of file From f584c83e88575fdfda6733273dfefc00ba588652 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 27 May 2022 17:40:20 +0300 Subject: [PATCH 05/37] Try to fix CI --- utbot-framework/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-framework/build.gradle b/utbot-framework/build.gradle index ce70c24f32..f55314bf13 100644 --- a/utbot-framework/build.gradle +++ b/utbot-framework/build.gradle @@ -75,5 +75,5 @@ test { jvmArgs '-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=9009' } - jvmArgs "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap" + maxHeapSize = "512m" } \ No newline at end of file From f4811e9998b4aa751791e57ecfb1621b5db6a1b0 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Tue, 31 May 2022 20:32:23 +0300 Subject: [PATCH 06/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 3 +++ .github/workflows/build-and-run-tests.yml | 3 +++ build.gradle | 4 +++- utbot-framework/build.gradle | 2 -- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index e6e8798bac..1899f0157b 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -2,6 +2,9 @@ name: "[M] UTBot Java: build and run tests" on: workflow_dispatch + +env: + GRADLE_OPTS: -Xmx2048m -Dorg.gradle.daemon=false jobs: build-and-run-tests: diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index 30af5f45ad..251e5969a6 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [main] +env: + GRADLE_OPTS: -Xmx2048m -Dorg.gradle.daemon=false + jobs: build_and_run_tests: runs-on: ubuntu-20.04 diff --git a/build.gradle b/build.gradle index c4da1c528d..f7d26fac95 100644 --- a/build.gradle +++ b/build.gradle @@ -41,4 +41,6 @@ subprojects { mavenCentral() maven { url 'https://jitpack.io' } } -} \ No newline at end of file +} + +test { maxHeapSize = "512m" } \ No newline at end of file diff --git a/utbot-framework/build.gradle b/utbot-framework/build.gradle index f55314bf13..2e3c4a7190 100644 --- a/utbot-framework/build.gradle +++ b/utbot-framework/build.gradle @@ -74,6 +74,4 @@ test { if (System.getProperty('DEBUG', 'false') == 'true') { jvmArgs '-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=9009' } - - maxHeapSize = "512m" } \ No newline at end of file From 6d007e7d83998e5a4b5871b772bee132661f398f Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Tue, 31 May 2022 22:04:21 +0300 Subject: [PATCH 07/37] Try to fix CI --- build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f7d26fac95..448035418b 100644 --- a/build.gradle +++ b/build.gradle @@ -43,4 +43,8 @@ subprojects { } } -test { maxHeapSize = "512m" } \ No newline at end of file +test { + maxHeapSize = "512m" + forkEvery 100 + jvmArgs '-Xmx512m', '-Xms512m' +} \ No newline at end of file From 90b580efbd62a65b28dc6f87f2549414424e215f Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Tue, 31 May 2022 22:42:17 +0300 Subject: [PATCH 08/37] Try to fix CI --- build.gradle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 448035418b..7b48ed2338 100644 --- a/build.gradle +++ b/build.gradle @@ -44,7 +44,6 @@ subprojects { } test { - maxHeapSize = "512m" - forkEvery 100 + maxHeapSize '512m' jvmArgs '-Xmx512m', '-Xms512m' } \ No newline at end of file From 2c971f586711208ae03c8f57ba790ade262d2d55 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Tue, 31 May 2022 23:48:34 +0300 Subject: [PATCH 09/37] Try to fix CI --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7b48ed2338..8545c57fe5 100644 --- a/build.gradle +++ b/build.gradle @@ -45,5 +45,5 @@ subprojects { test { maxHeapSize '512m' - jvmArgs '-Xmx512m', '-Xms512m' + jvmArgs '-Xmx2048m', '-Xms512m' } \ No newline at end of file From daa16b50d45aa01ccd28ed00f09f2892ef575af3 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 00:03:20 +0300 Subject: [PATCH 10/37] Try to fix CI --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 8545c57fe5..2452a72230 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,6 @@ subprojects { } test { - maxHeapSize '512m' - jvmArgs '-Xmx2048m', '-Xms512m' + maxHeapSize '256m' + jvmArgs '-Xmx256m', '-Xms256m' } \ No newline at end of file From 93824790c4101b357a3c23d37027a4a76974a4b0 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 00:24:25 +0300 Subject: [PATCH 11/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 5 +---- .github/workflows/build-and-run-tests.yml | 5 +---- build.gradle | 3 +-- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index 1899f0157b..3fbc2a85a9 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -2,9 +2,6 @@ name: "[M] UTBot Java: build and run tests" on: workflow_dispatch - -env: - GRADLE_OPTS: -Xmx2048m -Dorg.gradle.daemon=false jobs: build-and-run-tests: @@ -24,7 +21,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build --no-daemon + gradle clean build -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index 251e5969a6..28ef6ce4f3 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -6,9 +6,6 @@ on: pull_request: branches: [main] -env: - GRADLE_OPTS: -Xmx2048m -Dorg.gradle.daemon=false - jobs: build_and_run_tests: runs-on: ubuntu-20.04 @@ -26,7 +23,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build --no-daemon + gradle clean build -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/build.gradle b/build.gradle index 2452a72230..15d02d7987 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,5 @@ subprojects { } test { - maxHeapSize '256m' - jvmArgs '-Xmx256m', '-Xms256m' + jvmArgs "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap" } \ No newline at end of file From 1429636bb149e9138423af082f68c33d80424a4a Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 00:30:19 +0300 Subject: [PATCH 12/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 5 ++++- .github/workflows/build-and-run-tests.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index 3fbc2a85a9..4e6d7306b3 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -2,6 +2,9 @@ name: "[M] UTBot Java: build and run tests" on: workflow_dispatch + +env: + GRADLE_OPTS: -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap jobs: build-and-run-tests: @@ -21,7 +24,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + gradle clean build - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index 28ef6ce4f3..1652f3b7d6 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [main] +env: + GRADLE_OPTS: -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + jobs: build_and_run_tests: runs-on: ubuntu-20.04 @@ -23,7 +26,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + gradle clean build - name: Upload utbot-framework logs if: ${{ always() }} From 7e9947023ca23681c47f5c893347f32e1175fad1 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 00:32:39 +0300 Subject: [PATCH 13/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 2 +- .github/workflows/build-and-run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index 4e6d7306b3..72e80c87ab 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -4,7 +4,7 @@ on: workflow_dispatch env: - GRADLE_OPTS: -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + GRADLE_OPTS: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap jobs: build-and-run-tests: diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index 1652f3b7d6..053e63aa32 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -7,7 +7,7 @@ on: branches: [main] env: - GRADLE_OPTS: -Dorg.gradle.jvmargs=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + GRADLE_OPTS: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap jobs: build_and_run_tests: From 3fc97d31426500ade5cdef07198373a09e83f113 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 01:01:06 +0300 Subject: [PATCH 14/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 4 ++-- .github/workflows/build-and-run-tests.yml | 4 ++-- build.gradle | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index 72e80c87ab..5a0d1ca2fe 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -4,7 +4,7 @@ on: workflow_dispatch env: - GRADLE_OPTS: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + GRADLE_OPTS: -Xmx10248m -XX:MaxPermSize=256m jobs: build-and-run-tests: @@ -24,7 +24,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build + gradle clean build --max-workers 4 - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index 053e63aa32..56844ba68d 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -7,7 +7,7 @@ on: branches: [main] env: - GRADLE_OPTS: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + GRADLE_OPTS: -Xmx10248m -XX:MaxPermSize=256m jobs: build_and_run_tests: @@ -26,7 +26,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build + gradle clean build --max-workers 4 - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/build.gradle b/build.gradle index 15d02d7987..4b6bc3c486 100644 --- a/build.gradle +++ b/build.gradle @@ -44,5 +44,5 @@ subprojects { } test { - jvmArgs "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap" + maxHeapSize = "1024m" } \ No newline at end of file From 550ddcde5e66765159b430fb40be99339c0915b3 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 11:18:16 +0300 Subject: [PATCH 15/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 5 +---- .github/workflows/build-and-run-tests.yml | 5 +---- build.gradle | 7 ++++--- gradle.properties | 1 + 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index 5a0d1ca2fe..cde2e370ad 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -2,9 +2,6 @@ name: "[M] UTBot Java: build and run tests" on: workflow_dispatch - -env: - GRADLE_OPTS: -Xmx10248m -XX:MaxPermSize=256m jobs: build-and-run-tests: @@ -24,7 +21,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build --max-workers 4 + gradle clean build - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index 56844ba68d..b9ff177b9d 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -6,9 +6,6 @@ on: pull_request: branches: [main] -env: - GRADLE_OPTS: -Xmx10248m -XX:MaxPermSize=256m - jobs: build_and_run_tests: runs-on: ubuntu-20.04 @@ -26,7 +23,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build --max-workers 4 + gradle clean build - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/build.gradle b/build.gradle index 4b6bc3c486..6e73f8d687 100644 --- a/build.gradle +++ b/build.gradle @@ -41,8 +41,9 @@ subprojects { mavenCentral() maven { url 'https://jitpack.io' } } -} -test { - maxHeapSize = "1024m" + tasks.withType(Test) { + minHeapSize = '128m' + maxHeapSize = '512m' + } } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 8c7c2ceb21..bcdd369ca8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,4 +35,5 @@ eclipse_aether_version=1.1.0 maven_wagon_version=3.5.1 maven_plugin_api_version=3.8.5 maven_plugin_tools_version=3.6.4 +org.gradle.jvmargs=-Xmx1g -Xms256m # soot also depends on asm, so there could be two different versions \ No newline at end of file From 89c9d24c0186c4b150b914ff96e2e51454ef6ca0 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 11:34:46 +0300 Subject: [PATCH 16/37] Revert non-path-selector changes --- .../build-and-run-tests-from-branch.yml | 2 +- .github/workflows/build-and-run-tests.yml | 2 +- .../kotlin/org/utbot/ClassifierTrainer.kt | 20 +- .../main/kotlin/org/utbot/QualityAnalysis.kt | 182 ----- .../kotlin/org/utbot/QualityAnalysisConfig.kt | 21 - .../kotlin/org/utbot/QualityAnalysisV2.kt | 70 -- .../kotlin/org/utbot/StmtCoverageReport.kt | 180 ----- .../features/UtExpressionGraphExtraction.kt | 642 ------------------ .../org/utbot/predictors/UtBotSatPredictor.kt | 48 +- .../main/kotlin/org/utbot/utils/BinaryUtil.kt | 29 - .../main/kotlin/org/utbot/utils/MatrixUtil.kt | 129 ---- .../utbot/utils/layers/MessageAggregation.kt | 27 - .../org/utbot/utils/layers/MessagePassing.kt | 26 - .../org/utbot/utils/layers/SimpleDecoder.kt | 53 -- .../org/utbot/visual/AbstractHtmlReport.kt | 2 +- .../utbot/visual/ClassificationHtmlReport.kt | 57 +- .../kotlin/org/utbot/visual/FigureBuilders.kt | 144 ++-- .../kotlin/org/utbot/visual/HtmlBuilder.kt | 89 +-- .../src/main/resources/config.properties | 3 - .../src/main/resources/css/coverage.css | 168 ----- .../src/main/resources/css/highlight-idea.css | 153 ----- .../org/utbot/analytics/CoverageStatistics.kt | 49 -- .../org/utbot/analytics/UtBotPredictor.kt | 10 +- 23 files changed, 139 insertions(+), 1967 deletions(-) delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt delete mode 100644 utbot-analytics/src/main/resources/config.properties delete mode 100644 utbot-analytics/src/main/resources/css/coverage.css delete mode 100644 utbot-analytics/src/main/resources/css/highlight-idea.css delete mode 100644 utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index cde2e370ad..e6e8798bac 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -21,7 +21,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build + gradle clean build --no-daemon - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index b9ff177b9d..30af5f45ad 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -23,7 +23,7 @@ jobs: - name: Build and run tests in UTBot Java run: | export KOTLIN_HOME="/usr" - gradle clean build + gradle clean build --no-daemon - name: Upload utbot-framework logs if: ${{ always() }} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt index b99d3e9838..c3bba8ff70 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt @@ -6,7 +6,7 @@ import org.utbot.metrics.ClassificationMetrics import org.utbot.models.ClassifierModel import org.utbot.models.loadModelFromJson import org.utbot.models.save -import org.utbot.visual.ClassificationHTMLReport +import org.utbot.visual.ClassificationHtmlReport import smile.classification.Classifier import smile.data.CategoricalEncoder import smile.data.DataFrame @@ -20,7 +20,7 @@ private const val dataPath = "logs/stats.txt" private const val logDir = "logs" class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = ClassifierModel.GBM) : - AbstractTrainer(data, savePcaVariance = true) { + AbstractTrainer(data, savePcaVariance = true) { private lateinit var metrics: ClassificationMetrics lateinit var model: Classifier val properties = Properties() @@ -62,27 +62,19 @@ class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = val xFrame = Formula.lhs(targetColumn).x(validationData) val x = xFrame.toArray(false, CategoricalEncoder.LEVEL) - metrics = ClassificationMetrics( - classifierModel.name, - model, - Compose(transforms), - actualLabel.map { it.toInt() }.toIntArray(), - x - ) + metrics = ClassificationMetrics(classifierModel.name, model, Compose(transforms), actualLabel.map { it.toInt() }.toIntArray(), x) } override fun visualize() { - val report = ClassificationHTMLReport() + val report = ClassificationHtmlReport() report.run { addHeader(classifierModel.name, properties) addDataDistribution(formula.y(data).toDoubleArray()) addClassDistribution(classSizesBeforeResampling) addClassDistribution(classSizesAfterResampling, before = false) addPCAPlot(pcaVarianceProportion, pcaCumulativeVarianceProportion) - addMetrics( - metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(), - metrics.recall.toDoubleArray() - ) + addMetrics(metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(), + metrics.recall.toDoubleArray()) addConfusionMatrix(metrics.getNormalizedConfusionMatrix()) save() } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt deleted file mode 100644 index c55db51e19..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt +++ /dev/null @@ -1,182 +0,0 @@ -package org.utbot - -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import org.jsoup.select.Elements -import org.utbot.visual.FigureBuilders -import org.utbot.visual.HtmlBuilder -import java.io.File -import java.nio.file.Paths - - -data class Coverage( - val misInstructions: Int, - val covInstruction: Int, - val misBranches: Int, - val covBranches: Int, - val time: Double = 0.0 -) { - fun getInstructionCoverage(): Double = when { - (this.misInstructions == 0 && this.covInstruction == 0) -> 0.0 - (this.misInstructions == 0) -> 1.0 - else -> this.covInstruction.toDouble() / (this.covInstruction + this.misInstructions) - } - - fun getBranchesCoverage(): Double = when { - (this.misBranches == 0 && this.covBranches == 0) -> 0.0 - (this.misBranches == 0) -> 1.0 - else -> this.covBranches.toDouble() / (this.covBranches + this.misBranches) - } - - fun getInstructions() = misInstructions + covInstruction -} - - -fun parseJacocoReport(path: String, classes: Set): Pair, Map> { - val perClassResult = mutableMapOf() - val perMethodResult = mutableMapOf() - val contestDocument: Document = Jsoup.parse(File("$path/index.html"), null) - val pkgRows: Elements = contestDocument.select("table")[0].select("tr") - - for (i in 2 until pkgRows.size) { - val pkgHref = pkgRows[i].select("td")[0].select("a").attr("href") - val packageDocument: Document = Jsoup.parse(File("$path/$pkgHref"), null) - val classRows: Elements = packageDocument.select("table")[0].select("tr") - - for (j in 2 until classRows.size) { - val classHref = classRows[j].select("td")[0].select("a").attr("href") - val className = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0] - if (!classes.contains(className)) { - continue - } - - val classDocument: Document = Jsoup.parse(File("$path/${pkgHref.replace("index.html", classHref)}"), null) - val methodRows: Elements = classDocument.select("table")[0].select("tr") - - val coverageInfos = methodRows[1].select("td") - val (misInstructions, covInstructions) = coverageInfos[1].text()?.replace(",", "")?.split(" of ")?.let { - val misInstructions = it[0].toInt() - val allInstructions = it[1].toInt() - misInstructions to (allInstructions - misInstructions) - } ?: (0 to 0) - - val (misBranches, covBranches) = coverageInfos[3].text()?.replace(",", "")?.split(" of ")?.let { - val misBranches = it[0].toInt() - val allBranches = it[1].toInt() - misBranches to (allBranches - misBranches) - } ?: (0 to 0) - - val name = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0] - perClassResult[name] = Coverage(misInstructions, covInstructions, misBranches, covBranches) - - for (k in 2 until methodRows.size) { - - val cols = methodRows[k].select("td") - val methodHref = methodRows[k].select("td")[0].select("a").attr("href") - val methodName = classHref.replace("/index.html", "." + methodHref.replace("html", cols[0].text())) - - val instructions = cols[1].select("img") - val methodMisInstructions = - instructions.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt() - ?: 0 - val methodCovInstructions = - instructions.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt() - ?: 0 - - val branches = cols[3].select("img") - val methodMisBranches = branches.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0 - val methodCovBranches = branches.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0 - - if (methodMisInstructions == 0 && methodCovBranches == 0 && methodCovInstructions == 0 && methodMisBranches == 0) continue - perMethodResult[methodName] = - Coverage(methodMisInstructions, methodCovInstructions, methodMisBranches, methodCovBranches) - } - } - } - - return perClassResult to perMethodResult -} - - -fun main() { - val htmlBuilder = HtmlBuilder() - - val classes = mutableSetOf() - File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } - - // Parse data - val jacocoCoverage = QualityAnalysisConfig.selectors.map { - it to parseJacocoReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}", classes).first - } - - // Instruction coverage report (sum coverages percentages / classNum) - val instructionMetrics = jacocoCoverage.map { jacoco -> - jacoco.first to jacoco.second.map { it.value.getInstructionCoverage() } - } - htmlBuilder.addHeader("Instructions coverage (sum coverages percentages / classNum)") - instructionMetrics.forEach { - htmlBuilder.addText("Mean(Instruction (${it.first} model)) =${it.second.sum() / it.second.size}") - } - htmlBuilder.addFigure( - FigureBuilders.buildBoxPlot( - instructionMetrics.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten() - .toTypedArray(), - instructionMetrics.map { it.second }.flatten().toDoubleArray(), - title = "Coverage", - xLabel = "Instructions", - yLabel = "Count" - ) - ) - - // Instruction coverage report (sum covered instructions / sum instructions) - htmlBuilder.addHeader("Instructions coverage(sum covered instructions / sum instructions)") - val covInstruction = jacocoCoverage.map { jacoco -> - jacoco.first to jacoco.second.map { it.value.covInstruction } - } - val instructions = jacocoCoverage.map { jacoco -> - jacoco.first to jacoco.second.map { it.value.getInstructions() } - }.toMap() - covInstruction.forEach { - htmlBuilder.addText( - "Mean(Instruction (${it.first} model)) =${ - it.second.sum().toDouble() / (instructions[it.first]?.sum()?.toDouble() ?: 0.0) - }" - ) - } - htmlBuilder.addFigure( - FigureBuilders.buildBoxPlot( - covInstruction.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten().toTypedArray(), - covInstruction.map { it.second }.flatten().map { it.toDouble() }.toDoubleArray(), - title = "Coverage", - xLabel = "Instructions", - yLabel = "Count" - ) - ) - - // Branches coverage report - val branchesMetrics = jacocoCoverage.map { jacoco -> - jacoco.first to jacoco.second.map { it.value.getBranchesCoverage() } - } - htmlBuilder.addHeader("Branches coverage") - branchesMetrics.forEach { - htmlBuilder.addText("Mean(Branches (${it.first} model)) =${it.second.sum() / it.second.size}") - } - htmlBuilder.addFigure( - FigureBuilders.buildBoxPlot( - branchesMetrics.map { it.second.map { it2 -> "Branches (${it.first} model)" } }.flatten().toTypedArray(), - branchesMetrics.map { it.second }.flatten().toDoubleArray(), - title = "Coverage", - xLabel = "Branches", - yLabel = "Count" - ) - ) - - // Save report - htmlBuilder.saveHTML( - Paths.get( - QualityAnalysisConfig.outputDir, - QualityAnalysisConfig.project, - "test.html" - ).toFile().absolutePath - ) -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt deleted file mode 100644 index 46dc6c13dc..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt +++ /dev/null @@ -1,21 +0,0 @@ -package org.utbot - -import java.io.FileInputStream -import java.util.Properties - -object QualityAnalysisConfig { - private const val configPath = "utbot-analytics/src/main/resources/config.properties" - - private val properties = Properties().also { props -> - FileInputStream(configPath).use { inputStream -> - props.load(inputStream) - } - } - - val project: String = properties.getProperty("project") - val selectors: List = properties.getProperty("selectors").split(",") - val covStatistics: List = properties.getProperty("covStatistics").split(",") - - const val outputDir: String = "eval/res" - val classesList: String = "utbot-junit-contest/src/main/resources/classes/${project}/list" -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt deleted file mode 100644 index 4b53d78afa..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt +++ /dev/null @@ -1,70 +0,0 @@ -package org.utbot - -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import org.jsoup.select.Elements -import org.utbot.visual.FigureBuilders -import org.utbot.visual.HtmlBuilder -import java.io.File -import java.nio.file.Paths - - -fun parseReport(path: String, classes: Set): Map { - val result = mutableMapOf() - val contestDocument: Document = Jsoup.parse(File("$path\\index.html"), null) - val packageRows: Elements = contestDocument.select("table")[1].select("tr") - for (i in 1 until packageRows.size) { - val packageHref = packageRows[i].select("td")[0].select("a").attr("href") - val packageDocument: Document = Jsoup.parse(File("$path\\$packageHref"), null) - val clsRows: Elements = packageDocument.select("table")[1].select("tr") - - for (j in 1 until clsRows.size) { - val clsName = - clsRows[j].select("td")[0].select("a").attr("href").replace(".classes/", "").replace(".html", "") - val fullName = packageHref.replace("/index.html", ".$clsName") - - if (classes.contains(fullName)) { - result.put(fullName, clsRows[j].select("td")[3].select("span")[0].text().replace("%", "").toDouble()) - } - } - } - - return result -} - - -fun main() { - val htmlBuilder = HtmlBuilder() - - val classes = mutableSetOf() - File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } - - // Parse data - val jacocoCoverage = QualityAnalysisConfig.selectors.map { - it to parseReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}", classes) - } - - // Coverage report - htmlBuilder.addHeader("Line coverage") - jacocoCoverage.forEach { - htmlBuilder.addText("Mean(Line (${it.first} model)) =" + it.second.map { it.value }.sum() / it.second.size) - } - htmlBuilder.addFigure( - FigureBuilders.buildBoxPlot( - jacocoCoverage.map { it.second.map { it2 -> "Line (${it.first} model)" } }.flatten().toTypedArray(), - jacocoCoverage.map { it.second.map { it.value } }.flatten().toDoubleArray(), - title = "Coverage", - xLabel = "Instructions", - yLabel = "Count" - ) - ) - - // Save report - htmlBuilder.saveHTML( - Paths.get( - QualityAnalysisConfig.outputDir, - QualityAnalysisConfig.project, - QualityAnalysisConfig.selectors.joinToString("_") - ).toFile().absolutePath - ) -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt deleted file mode 100644 index b983c30e9b..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt +++ /dev/null @@ -1,180 +0,0 @@ -package org.utbot - -import org.apache.commons.io.FileUtils -import org.utbot.visual.FigureBuilders -import org.utbot.visual.HtmlBuilder -import smile.read -import java.io.File -import java.nio.file.Paths -import kotlin.random.Random - - -data class Statistics( - val perMethod: Map>, - val perClass: Map>, - val perProject: List -) - - -fun getStatistics(path: String, classes: Set): Statistics { - var projectTotalStmts = 0.0 - var projectMinStartTime = Double.MAX_VALUE - - val rawStatistics = File(path).listFiles()?.filter { it.extension == "txt" && it.readLines().size > 1 }?.map { - val data = read.csv(it.absolutePath) - it.nameWithoutExtension to data.toArray() - }?.toMap() ?: emptyMap() - - val statisticsPerMethod = rawStatistics.map { methodData -> - val startTime = methodData.value[0][0] - val value = methodData.value.map { - doubleArrayOf(it[0] - startTime, it[1] / it[2]) // time, percent - }.sortedBy { it[0] } - - methodData.key to value - }.toMap() - - val statisticsPerClass = classes.map { cls -> - var minStartTime = Double.MAX_VALUE - var totalStmts = 0.0 - val filteredStatistics = rawStatistics.filter { it.key.contains(cls) } - - filteredStatistics.forEach { - minStartTime = minOf(minStartTime, it.value[0][0]) - totalStmts += it.value.getOrNull(1)?.get(2) ?: 0.0 - } - projectTotalStmts += totalStmts - projectMinStartTime = minOf(minStartTime, projectMinStartTime) - - var prevCov = 0.0 - cls to filteredStatistics.toList().sortedBy { it.second[0][0] }.flatMap { methodData -> - val value = methodData.second.map { - doubleArrayOf(it[0] - minStartTime, (it[1] + prevCov) / totalStmts) - } - - prevCov += methodData.second.last()[1] - value - }.sortedBy { it[0] } - }.toMap() - - var prevCov = 0.0 - val statisticsPerProject = rawStatistics - .filter { classes.any { it2 -> it.key.contains(it2) } } - .toList() - .sortedBy { it.second[0][0] } - .flatMap { methodData -> - val value = methodData.second.map { - doubleArrayOf(it[0] - projectMinStartTime, (it[1] + prevCov) / projectTotalStmts) // time, percent - } - - prevCov += methodData.second.last()[1] - value - }.sortedBy { it[0] } - - return Statistics(statisticsPerMethod, statisticsPerClass, statisticsPerProject) -} - -fun main() { - // Processed classes - val classes = mutableSetOf() - File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } - - // Prepare folder - val projectDataPath = "${QualityAnalysisConfig.outputDir}/report/${QualityAnalysisConfig.project}/${ - QualityAnalysisConfig.selectors.joinToString("_") - }" - File(projectDataPath).deleteRecursively() - classes.forEach { File(projectDataPath, it).mkdirs() } - File(projectDataPath, "css").mkdirs() - listOf("css/coverage.css", "css/highlight-idea.css").forEach { - FileUtils.copyInputStreamToFile( - Statistics::class.java.classLoader.getResourceAsStream(it), - Paths.get(projectDataPath, it).toFile() - ) - } - - // Parse statistics - val selectors = QualityAnalysisConfig.selectors - val statistics = QualityAnalysisConfig.covStatistics.map { getStatistics(it, classes) } - val colors = QualityAnalysisConfig.selectors.map { - val rnd = Random.Default - "rgb(${rnd.nextInt(256)}, ${rnd.nextInt(256)}, ${rnd.nextInt(256)})" - } - - val htmlBuilderForProject = HtmlBuilder( - pathToStyle = listOf("css/coverage.css", "css/highlight-idea.css"), - pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") - ) - htmlBuilderForProject.addTable( - statistics.map { it.perClass.map { it.key to (it.value.lastOrNull()?.lastOrNull() ?: 0.0) * 100 }.toMap() }, - selectors, - colors, - "class" - ) - htmlBuilderForProject.addFigure( - FigureBuilders.buildSeveralLinesPlot( - statistics.map { it.perProject.map { it[0] / 1e9 }.toDoubleArray() }, - statistics.map { it.perProject.map { it[1] * 100 }.toDoubleArray() }, - colors, - selectors, - xLabel = "Time (sec)", - yLabel = "Coverage (%)", - title = QualityAnalysisConfig.project - ) - ) - htmlBuilderForProject.saveHTML(File(projectDataPath, "index.html").toString()) - - - classes.forEach { cls -> - val outputDir = File(projectDataPath, cls) - val htmlBuilderForCls = HtmlBuilder( - pathToStyle = listOf("../css/coverage.css", "../css/highlight-idea.css"), - pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") - ) - val filteredStatistics = statistics.map { - it.perMethod.filter { it.key.contains(cls) } - } - - htmlBuilderForCls.addTable( - filteredStatistics.map { it.map { it.key to it.value.last().last() * 100 }.toMap() }, - selectors, - colors, - "method" - ) - htmlBuilderForCls.addFigure( - FigureBuilders.buildSeveralLinesPlot( - statistics.map { it.perClass[cls]?.map { it[0] / 1e9 }?.toDoubleArray() ?: doubleArrayOf() }, - statistics.map { it.perClass[cls]?.map { it[1] * 100 }?.toDoubleArray() ?: doubleArrayOf() }, - colors, - selectors, - xLabel = "Time (sec)", - yLabel = "Coverage (%)", - title = cls - ) - ) - - File(outputDir.toString()).mkdir() - htmlBuilderForCls.saveHTML(File(outputDir, "index.html").toString()) - - filteredStatistics.first().keys.forEach { method -> - val htmlBuilderForMethod = HtmlBuilder( - pathToStyle = listOf("../../css/coverage.css", "../../css/highlight-idea.css"), - pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") - ) - htmlBuilderForMethod.addFigure( - FigureBuilders.buildSeveralLinesPlot( - filteredStatistics.map { it[method]?.map { it[0] / 1e9 }?.toDoubleArray() ?: doubleArrayOf() }, - filteredStatistics.map { it[method]?.map { it[1] * 100 }?.toDoubleArray() ?: doubleArrayOf() }, - colors, - selectors, - xLabel = "Time (sec)", - yLabel = "Coverage (%)", - title = method - ) - ) - - File(outputDir, method).mkdirs() - htmlBuilderForMethod.saveHTML(File(outputDir, "${method}/index.html").toString()) - } - } -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt deleted file mode 100644 index a0225ad807..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/UtExpressionGraphExtraction.kt +++ /dev/null @@ -1,642 +0,0 @@ -package org.utbot.features - -import org.utbot.engine.pc.NotBoolExpression -import org.utbot.engine.pc.UtAddNoOverflowExpression -import org.utbot.engine.pc.UtAddrExpression -import org.utbot.engine.pc.UtAndBoolExpression -import org.utbot.engine.pc.UtArrayApplyForAll -import org.utbot.engine.pc.UtArrayInsert -import org.utbot.engine.pc.UtArrayInsertRange -import org.utbot.engine.pc.UtArrayMultiStoreExpression -import org.utbot.engine.pc.UtArrayRemove -import org.utbot.engine.pc.UtArrayRemoveRange -import org.utbot.engine.pc.UtArraySelectExpression -import org.utbot.engine.pc.UtArraySetRange -import org.utbot.engine.pc.UtArrayShiftIndexes -import org.utbot.engine.pc.UtArrayToString -import org.utbot.engine.pc.UtBoolConst -import org.utbot.engine.pc.UtBoolOpExpression -import org.utbot.engine.pc.UtBvConst -import org.utbot.engine.pc.UtBvLiteral -import org.utbot.engine.pc.UtCastExpression -import org.utbot.engine.pc.UtConcatExpression -import org.utbot.engine.pc.UtConstArrayExpression -import org.utbot.engine.pc.UtContainsExpression -import org.utbot.engine.pc.UtConvertToString -import org.utbot.engine.pc.UtEndsWithExpression -import org.utbot.engine.pc.UtEqExpression -import org.utbot.engine.pc.UtEqGenericTypeParametersExpression -import org.utbot.engine.pc.UtExpression -import org.utbot.engine.pc.UtExpressionVisitor -import org.utbot.engine.pc.UtFalse -import org.utbot.engine.pc.UtFpConst -import org.utbot.engine.pc.UtFpLiteral -import org.utbot.engine.pc.UtGenericExpression -import org.utbot.engine.pc.UtIndexOfExpression -import org.utbot.engine.pc.UtInstanceOfExpression -import org.utbot.engine.pc.UtIsExpression -import org.utbot.engine.pc.UtIsGenericTypeExpression -import org.utbot.engine.pc.UtIteExpression -import org.utbot.engine.pc.UtMkArrayExpression -import org.utbot.engine.pc.UtMkTermArrayExpression -import org.utbot.engine.pc.UtNegExpression -import org.utbot.engine.pc.UtOpExpression -import org.utbot.engine.pc.UtOrBoolExpression -import org.utbot.engine.pc.UtReplaceExpression -import org.utbot.engine.pc.UtSeqLiteral -import org.utbot.engine.pc.UtStartsWithExpression -import org.utbot.engine.pc.UtStringCharAt -import org.utbot.engine.pc.UtStringConst -import org.utbot.engine.pc.UtStringEq -import org.utbot.engine.pc.UtStringLength -import org.utbot.engine.pc.UtStringPositiveLength -import org.utbot.engine.pc.UtStringToArray -import org.utbot.engine.pc.UtStringToInt -import org.utbot.engine.pc.UtSubNoOverflowExpression -import org.utbot.engine.pc.UtSubstringExpression -import org.utbot.engine.pc.UtToStringExpression -import org.utbot.engine.pc.UtTrue -import org.utbot.utils.BinaryUtil -import java.util.IdentityHashMap -import java.util.UUID - - -class UtExpressionId: UtExpressionVisitor { - - fun getID(expr: UtExpression): Int = expr.accept(this) - - override fun visit(expr: UtArraySelectExpression): Int = 1 - override fun visit(expr: UtMkArrayExpression): Int = 2 - override fun visit(expr: UtArrayMultiStoreExpression): Int = 3 - override fun visit(expr: UtBvLiteral): Int = 4 - override fun visit(expr: UtBvConst): Int = 5 - override fun visit(expr: UtAddrExpression): Int = 6 - override fun visit(expr: UtFpLiteral): Int = 7 - override fun visit(expr: UtFpConst): Int = 8 - override fun visit(expr: UtOpExpression): Int = 9 - override fun visit(expr: UtTrue): Int = 10 - override fun visit(expr: UtFalse): Int = 11 - override fun visit(expr: UtEqExpression): Int = 12 - override fun visit(expr: UtBoolConst): Int = 13 - override fun visit(expr: NotBoolExpression): Int = 14 - override fun visit(expr: UtOrBoolExpression): Int = 15 - override fun visit(expr: UtAndBoolExpression): Int = 16 - override fun visit(expr: UtNegExpression): Int = 17 - override fun visit(expr: UtCastExpression): Int = 18 - override fun visit(expr: UtBoolOpExpression): Int = 19 - override fun visit(expr: UtIsExpression): Int = 20 - override fun visit(expr: UtIteExpression): Int = 21 - override fun visit(expr: UtMkTermArrayExpression): Int = 22 - override fun visit(expr: UtConcatExpression): Int = 23 - override fun visit(expr: UtConvertToString): Int = 24 - override fun visit(expr: UtStringLength): Int = 25 - override fun visit(expr: UtStringPositiveLength): Int = 26 - override fun visit(expr: UtStringCharAt): Int = 27 - override fun visit(expr: UtStringEq): Int = 28 - override fun visit(expr: UtSubstringExpression): Int = 29 - override fun visit(expr: UtReplaceExpression): Int = 30 - override fun visit(expr: UtStartsWithExpression): Int = 31 - override fun visit(expr: UtEndsWithExpression): Int = 32 - override fun visit(expr: UtIndexOfExpression): Int = 33 - override fun visit(expr: UtContainsExpression): Int = 34 - override fun visit(expr: UtToStringExpression): Int = 35 - override fun visit(expr: UtSeqLiteral): Int = 36 - override fun visit(expr: UtConstArrayExpression): Int = 37 - override fun visit(expr: UtGenericExpression): Int = 38 - override fun visit(expr: UtIsGenericTypeExpression): Int = 39 - override fun visit(expr: UtEqGenericTypeParametersExpression): Int = 40 - override fun visit(expr: UtInstanceOfExpression): Int = 41 - override fun visit(expr: UtStringConst): Int = 42 - override fun visit(expr: UtStringToInt): Int = 43 - override fun visit(expr: UtArrayToString): Int = 44 - override fun visit(expr: UtArrayInsert): Int = 45 - override fun visit(expr: UtArrayInsertRange): Int = 46 - override fun visit(expr: UtArrayRemove): Int = 47 - override fun visit(expr: UtArrayRemoveRange): Int = 48 - override fun visit(expr: UtArraySetRange): Int = 49 - override fun visit(expr: UtArrayShiftIndexes): Int = 50 - override fun visit(expr: UtArrayApplyForAll): Int = 51 - override fun visit(expr: UtStringToArray): Int = 52 - override fun visit(expr: UtAddNoOverflowExpression): Int = 53 - override fun visit(expr: UtSubNoOverflowExpression): Int = 54 -} - - -@Suppress("DuplicatedCode") -class UtExpressionGraphExtraction : UtExpressionVisitor { - - private val lineSeparator = System.lineSeparator() - val expressionIdsCache = IdentityHashMap() - private val expressionSubGraphCache = IdentityHashMap() - val literalValues = IdentityHashMap() - - fun getID(expr: UtExpression): String = expressionIdsCache.getOrPut(expr) { UUID.randomUUID().toString() } - - private fun extractSubgraph(expr: UtExpression): String { - return if (expressionSubGraphCache.containsKey(expr)) { - "" - } else { - val res = expr.accept(this) - expressionSubGraphCache[expr] = res - - res - } - } - - fun extractGraph(input: Iterable): String { - return input.fold("FROM,TO$lineSeparator") { acc, expr -> acc + extractSubgraph(expr) } - } - - override fun visit(expr: UtArraySelectExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) + extractSubgraph(expr.index) - - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.index) + lineSeparator - - return res - } - - override fun visit(expr: UtMkArrayExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - return "" - } - - override fun visit(expr: UtArrayMultiStoreExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.initial) - expr.stores.forEach { res += extractSubgraph(it.value) + extractSubgraph(it.index) } - - res += id + "," + getID(expr.initial) + lineSeparator - expr.stores.forEach { res += id + "," + getID(it.value) + lineSeparator } - expr.stores.forEach { res += id + "," + getID(it.index) + lineSeparator } - - return res - } - - override fun visit(expr: UtBvLiteral): String { - literalValues[expr] = BinaryUtil.binaryValueString(expr.value.toInt()) - return "" - } - - override fun visit(expr: UtBvConst): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - return "" - } - - override fun visit(expr: UtAddrExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.internal) - res += id + "," + getID(expr.internal) + lineSeparator - - return res - } - - override fun visit(expr: UtFpLiteral): String { - literalValues[expr] = BinaryUtil.binaryValueString(expr.value.toInt()) - return "" - } - - override fun visit(expr: UtFpConst): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - return "" - } - - override fun visit(expr: UtOpExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.left.expr) + extractSubgraph(expr.right.expr) - res += id + "," + getID(expr.left.expr) + lineSeparator - res += id + "," + getID(expr.right.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtTrue): String { - literalValues[expr] = BinaryUtil.binaryValueString(1) - return "" - } - - override fun visit(expr: UtFalse): String { - literalValues[expr] = BinaryUtil.binaryValueString(0) - return "" - } - - override fun visit(expr: UtEqExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.left) + extractSubgraph(expr.right) - res += id + "," + getID(expr.left) + lineSeparator - res += id + "," + getID(expr.right) + lineSeparator - - return res - } - - override fun visit(expr: UtBoolConst): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - return "" - } - - override fun visit(expr: NotBoolExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.expr) - res += id + "," + getID(expr.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtOrBoolExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = "" - expr.exprs.forEach { res += extractSubgraph(it) } - expr.exprs.forEach { res += id + "," + getID(it) + lineSeparator } - - return res - } - - override fun visit(expr: UtAndBoolExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = "" - expr.exprs.forEach { res += extractSubgraph(it) } - expr.exprs.forEach { res += id + "," + getID(it) + lineSeparator } - - return res - } - - override fun visit(expr: UtNegExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.variable.expr) - res += id + "," + getID(expr.variable.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtCastExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.variable.expr) - res += id + "," + getID(expr.variable.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtBoolOpExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.left.expr) + extractSubgraph(expr.right.expr) - res += id + "," + getID(expr.left.expr) + lineSeparator - res += id + "," + getID(expr.right.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtIsExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.addr) - res += id + "," + getID(expr.addr) + lineSeparator - - return res - } - - override fun visit(expr: UtIteExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.condition) + extractSubgraph(expr.thenExpr) + extractSubgraph(expr.elseExpr) - res += id + "," + getID(expr.condition) + lineSeparator - res += id + "," + getID(expr.thenExpr) + lineSeparator - res += id + "," + getID(expr.elseExpr) + lineSeparator - - return res - } - - override fun visit(expr: UtConcatExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = "" - expr.parts.forEach { res += extractSubgraph(it) } - expr.parts.forEach { res += id + "," + getID(it) + lineSeparator } - - return res - } - - override fun visit(expr: UtConvertToString): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.expression) - res += id + "," + getID(expr.expression) + lineSeparator - - return res - } - - override fun visit(expr: UtStringLength): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) - res += id + "," + getID(expr.string) + lineSeparator - - return res - } - - override fun visit(expr: UtStringPositiveLength): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) - res += id + "," + getID(expr.string) + lineSeparator - - return res - } - - override fun visit(expr: UtStringCharAt): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) + extractSubgraph(expr.index) - res += id + "," + getID(expr.string) + lineSeparator - res += id + "," + getID(expr.index) + lineSeparator - - return res - } - - override fun visit(expr: UtStringEq): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.left) + extractSubgraph(expr.right) - res += id + "," + getID(expr.left) + lineSeparator - res += id + "," + getID(expr.right) + lineSeparator - - return res - } - - override fun visit(expr: UtSubstringExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) + extractSubgraph(expr.beginIndex) + extractSubgraph(expr.length) - res += id + "," + getID(expr.string) + lineSeparator - res += id + "," + getID(expr.beginIndex) + lineSeparator - res += id + "," + getID(expr.length) + lineSeparator - - return res - } - - override fun visit(expr: UtReplaceExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) + extractSubgraph(expr.regex) + extractSubgraph(expr.replacement) - res += id + "," + getID(expr.string) + lineSeparator - res += id + "," + getID(expr.regex) + lineSeparator - res += id + "," + getID(expr.replacement) + lineSeparator - - return res - } - - override fun visit(expr: UtStartsWithExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) + extractSubgraph(expr.prefix) - res += id + "," + getID(expr.string) + lineSeparator - res += id + "," + getID(expr.prefix) + lineSeparator - - return res - } - - override fun visit(expr: UtEndsWithExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) + extractSubgraph(expr.suffix) - res += id + "," + getID(expr.string) + lineSeparator - res += id + "," + getID(expr.suffix) + lineSeparator - - return res - } - - override fun visit(expr: UtIndexOfExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) + extractSubgraph(expr.substring) - res += id + "," + getID(expr.string) + lineSeparator - res += id + "," + getID(expr.substring) + lineSeparator - - return res - } - - override fun visit(expr: UtContainsExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.string) + extractSubgraph(expr.substring) - res += id + "," + getID(expr.string) + lineSeparator - res += id + "," + getID(expr.substring) + lineSeparator - - return res - } - - override fun visit(expr: UtToStringExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - val id = getID(expr) - var res = extractSubgraph(expr.notNullExpr) + extractSubgraph(expr.isNull) - res += id + "," + getID(expr.notNullExpr) + lineSeparator - res += id + "," + getID(expr.isNull) + lineSeparator - - return res - } - - override fun visit(expr: UtSeqLiteral): String { - literalValues[expr] = BinaryUtil.binaryValueString(expr.value.toInt()) - return "" - } - - override fun visit(expr: UtMkTermArrayExpression): String { - literalValues[expr] = BinaryUtil.binaryValueStringEmpty() - return "" - } - - override fun visit(expr: UtConstArrayExpression): String { - val id = getID(expr) - var res = extractSubgraph(expr.constValue) - res += id + "," + getID(expr.constValue) + lineSeparator - - return res - } - - override fun visit(expr: UtGenericExpression): String { - val id = getID(expr) - var res = extractSubgraph(expr.addr) - res += id + "," + getID(expr.addr) + lineSeparator - - return res - } - - override fun visit(expr: UtIsGenericTypeExpression): String { - val id = getID(expr) - var res = extractSubgraph(expr.addr) - res += extractSubgraph(expr.baseAddr) - res += id + "," + getID(expr.addr) + lineSeparator - res += id + "," + getID(expr.baseAddr) + lineSeparator - - return res - } - - override fun visit(expr: UtEqGenericTypeParametersExpression): String { - val id = getID(expr) - var res = extractSubgraph(expr.firstAddr) - res += extractSubgraph(expr.secondAddr) - res += id + "," + getID(expr.firstAddr) + lineSeparator - res += id + "," + getID(expr.secondAddr) + lineSeparator - - return res - } - - override fun visit(expr: UtInstanceOfExpression): String { - val id = getID(expr) - var res = extractSubgraph(expr.constraint) - res += id + "," + getID(expr.constraint) + lineSeparator - - return res - } - - override fun visit(expr: UtStringConst): String { - // TODO - return "" - } - - override fun visit(expr: UtStringToInt): String { - val id = getID(expr) - var res = extractSubgraph(expr.expression) - res += id + "," + getID(expr.expression) + lineSeparator - - return res - } - - override fun visit(expr: UtArrayToString): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += extractSubgraph(expr.offset.expr) - res += extractSubgraph(expr.length.expr) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.offset.expr) + lineSeparator - res += id + "," + getID(expr.length.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtArrayInsert): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += extractSubgraph(expr.element) - res += extractSubgraph(expr.index.expr) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.element) + lineSeparator - res += id + "," + getID(expr.index.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtArrayInsertRange): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += extractSubgraph(expr.elements) - res += extractSubgraph(expr.index.expr) - res += extractSubgraph(expr.from.expr) - res += extractSubgraph(expr.length.expr) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.elements) + lineSeparator - res += id + "," + getID(expr.index.expr) + lineSeparator - res += id + "," + getID(expr.from.expr) + lineSeparator - res += id + "," + getID(expr.length.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtArrayRemove): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += extractSubgraph(expr.index.expr) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.index.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtArrayRemoveRange): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += extractSubgraph(expr.index.expr) - res += extractSubgraph(expr.length.expr) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.index.expr) + lineSeparator - res += id + "," + getID(expr.length.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtArraySetRange): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += extractSubgraph(expr.elements) - res += extractSubgraph(expr.index.expr) - res += extractSubgraph(expr.from.expr) - res += extractSubgraph(expr.length.expr) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.elements) + lineSeparator - res += id + "," + getID(expr.index.expr) + lineSeparator - res += id + "," + getID(expr.from.expr) + lineSeparator - res += id + "," + getID(expr.length.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtArrayShiftIndexes): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += extractSubgraph(expr.offset.expr) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - res += id + "," + getID(expr.offset.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtArrayApplyForAll): String { - val id = getID(expr) - var res = extractSubgraph(expr.arrayExpression) - res += id + "," + getID(expr.arrayExpression) + lineSeparator - - return res - // TODO ?? - } - - override fun visit(expr: UtStringToArray): String { - val id = getID(expr) - var res = extractSubgraph(expr.stringExpression) - res += extractSubgraph(expr.offset.expr) - res += id + "," + getID(expr.stringExpression) + lineSeparator - res += id + "," + getID(expr.offset.expr) + lineSeparator - - return res - } - - override fun visit(expr: UtAddNoOverflowExpression): String { - val id = getID(expr) - var res = extractSubgraph(expr.left) - res += extractSubgraph(expr.right) - res += id + "," + getID(expr.left) + lineSeparator - res += id + "," + getID(expr.right) + lineSeparator - - return res - } - - override fun visit(expr: UtSubNoOverflowExpression): String { - val id = getID(expr) - var res = extractSubgraph(expr.left) - res += extractSubgraph(expr.right) - res += id + "," + getID(expr.left) + lineSeparator - res += id + "," + getID(expr.right) + lineSeparator - - return res - } -} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt index 798a782bf3..d99f506016 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/UtBotSatPredictor.kt @@ -1,55 +1,15 @@ package org.utbot.predictors import org.utbot.analytics.IUtBotSatPredictor -import org.utbot.analytics.NeuroSatData import org.utbot.analytics.UtBotAbstractPredictor import org.utbot.engine.pc.UtExpression -import org.utbot.features.UtExpressionGraphExtraction -import org.utbot.features.UtExpressionId -import org.utbot.utils.BinaryUtil -import java.io.File -import java.io.FileOutputStream +import org.utbot.engine.pc.UtSolverStatusKind @Suppress("unused") -class UtBotSatPredictor : UtBotAbstractPredictor, NeuroSatData>, - IUtBotSatPredictor> { +class UtBotSatPredictor : UtBotAbstractPredictor, UtSolverStatusKind>, + IUtBotSatPredictor> { - private var counter = 1 - private var folder = "logs/GRAPHS" - - init { - File("${folder}/graphs").mkdirs() - File("${folder}/SAT.txt").printWriter().use { out -> - out.println("ID,SAT,TIME") - } - } - - override fun provide(input: Iterable, expectedResult: NeuroSatData, actualResult: NeuroSatData) { - val extractor = UtExpressionGraphExtraction() - File("${folder}/graphs/$counter").mkdir() - File("${folder}/graphs/$counter/edges.txt").printWriter().use { out -> - out.print(extractor.extractGraph(input)) - } - File("${folder}/graphs/$counter/vertexesType.txt").printWriter().use { out -> - out.println("UUID,TYPE") - out.print(extractor.expressionIdsCache.keys.map { - "${extractor.getID(it)},${ - BinaryUtil.binaryExpressionString(UtExpressionId().getID(it)) - }" - }.joinToString("\n")) - } - File("${folder}/graphs/$counter/vertexesValues.txt").printWriter().use { out -> - out.println("UUID,VALUE") - out.print(extractor.literalValues.keys.map { "${extractor.getID(it)},${extractor.literalValues[it]}" } - .joinToString("\n")) - } - - FileOutputStream("${folder}/SAT.txt", true).bufferedWriter().use { out -> - out.write("$counter,${actualResult.status},${actualResult.time}") - out.newLine() - } - - counter++ + override fun provide(input: Iterable, expectedResult: UtSolverStatusKind, actualResult: UtSolverStatusKind) { } override fun terminate() { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt deleted file mode 100644 index b3d058d652..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/utils/BinaryUtil.kt +++ /dev/null @@ -1,29 +0,0 @@ -package org.utbot.utils - - -object BinaryUtil { - fun binaryExpression(value: Int): DoubleArray { - val binaryStr = binaryValueString(value, "%6s") - return binaryStr.chars().mapToDouble { c: Int -> (c - 48).toDouble() }.toArray() - } - - fun binaryExpressionString(value: Int): String { - return binaryValueString(value, "%6s") - } - - fun binaryValue(value: Int): DoubleArray { - val binaryStr = binaryValueString(value) - return binaryStr.chars().mapToDouble { c: Int -> (c - 48).toDouble() }.toArray() - } - - fun binaryValueString(value: Int): String = binaryValueString(value, "%32s") + "0" - - fun binaryValueEmpty(): DoubleArray = DoubleArray(33) { 0.0 }.apply { this[32] = 1.0 } - - fun binaryValueStringEmpty(): String = "0".repeat(32) + "1" - - private fun binaryValueString(value: Int, format: String): String { - val result = Integer.toBinaryString(value) - return String.format(format, result).replace(" ".toRegex(), "0") - } -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt deleted file mode 100644 index 3951d983df..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/utils/MatrixUtil.kt +++ /dev/null @@ -1,129 +0,0 @@ -package org.utbot.utils - -import java.io.File -import java.io.FileNotFoundException -import java.util.ArrayList -import java.util.Scanner - - -object MatrixUtil { - - - fun loadMatrix(file: String, swapAxis: Boolean): Array { - val scanner = Scanner(File(file)) - val shape = scanner.nextLine().split(",").toTypedArray() - val firstDim = shape[0].toInt() - val secondDim = shape[1].toInt() - val matrix: Array - matrix = if (swapAxis) { - Array(secondDim) { DoubleArray(firstDim) } - } else { - Array(firstDim) { DoubleArray(secondDim) } - } - for (i in 0 until firstDim) { - val row = scanner.nextLine().split(",").toTypedArray() - for (j in 0 until secondDim) { - if (swapAxis) { - matrix[j][i] = row[j].toDouble() - } else { - matrix[i][j] = row[j].toDouble() - } - } - } - return matrix - } - - /** - * Load vector from file - * @param file - file with vector - * @return vector - */ - @Throws(FileNotFoundException::class) - fun loadVector(file: String): DoubleArray { - val scanner = Scanner(File(file)) - val dim = scanner.nextLine().toInt() - val vector = DoubleArray(dim) - vector.forEachIndexed { i, _ -> vector[i] = scanner.nextLine().toDouble() } - - return vector - } - - /** - * Matrix-vector product - Ax - * [n, m] * [m,] = [n,] - * @param vector - [m,] - * @param matrix - [n, m] - * @return [n,] - */ - fun mmul(vector: DoubleArray, matrix: Array): DoubleArray { - val firstDim = matrix.size - val secondDim: Int = matrix[0].size - val res = DoubleArray(firstDim) - for (i in 0 until firstDim) { - for (j in 0 until secondDim) { - res[i] += vector[j] * matrix[i][j] - } - } - return res - } - - /** - * Matrix-vector product with bias - Ax + b - * [n, m] * [m,] = [n,] - * @param vector - [m,] - * @param matrix - [n, m] - * @param bias - [n,] - * @return [n,] - */ - fun mmulBias(vector: DoubleArray, matrix: Array, bias: DoubleArray): DoubleArray? { - val firstDim = matrix.size - val secondDim: Int = matrix[0].size - val res = DoubleArray(firstDim) - for (i in 0 until firstDim) { - for (j in 0 until secondDim) { - res[i] += vector[j] * matrix[i][j] - } - res[i] += bias[i] - } - return res - } - - /** - * Matrix-vector1.concat(vector2) product - Ax. - * @param vector1 - [m,] - * @param vector2 - [m,] - * @param matrix - [n, 2*m] - * @return [n,] - */ - fun mmul(vector1: DoubleArray, vector2: DoubleArray, matrix: Array): DoubleArray { - val dim = vector1.size - val res = DoubleArray(dim) - for (i in 0 until dim) { - for (j in 0 until dim) { - res[i] += vector1[j] * matrix[i][j] + vector2[j] * matrix[i][dim + j] - } - } - return res - } - - /** - * Sum of Matrix-vector product - * @param input - [k, m] - * @param matrix - [n, m] - * @return [n] - */ - fun mmulSum(input: ArrayList, matrix: Array): DoubleArray { - val firstDim = matrix.size - val secondDim: Int = matrix[0].size - val res = DoubleArray(firstDim) - for (vector in input) { - for (i in 0 until firstDim) { - for (j in 0 until secondDim) { - res[i] += vector[j] * matrix[i][j] - } - } - } - return res - } - -} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt deleted file mode 100644 index 7e1c9ff44c..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessageAggregation.kt +++ /dev/null @@ -1,27 +0,0 @@ -package org.utbot.utils.layers - -import org.utbot.utils.MatrixUtil -import java.io.FileNotFoundException - -class MessageAggregation { - - lateinit var weight0: Array - lateinit var weight1: Array - - init { - try { - weight0 = MatrixUtil.loadMatrix("logs/model.encoder.x_update.0.weight", false) - weight1 = MatrixUtil.loadMatrix("logs/model.encoder.x_update.2.weight", false) - } catch (e: FileNotFoundException) { - e.printStackTrace() - } - } - - fun propagate(state: DoubleArray, message: DoubleArray): DoubleArray { - val x = MatrixUtil.mmul(message, state, weight0) - for (i in x.indices) { - x[i] = Math.max(x.get(i), 0.0) - } - return MatrixUtil.mmul(x, weight1) - } -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt deleted file mode 100644 index 6f1ce65336..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/MessagePassing.kt +++ /dev/null @@ -1,26 +0,0 @@ -package org.utbot.utils.layers - -import org.utbot.utils.MatrixUtil -import java.io.FileNotFoundException - - -class MessagePassing { - - lateinit var weight: Array - - init { - try { - weight = MatrixUtil.loadMatrix("logs/model.encoder.weight.txt", true) - } catch (e: FileNotFoundException) { - e.printStackTrace() - } - } - - fun propagate(message: DoubleArray): DoubleArray { - return MatrixUtil.mmul(message, weight) - } - - fun propagate(messages: ArrayList): DoubleArray { - return MatrixUtil.mmulSum(messages, weight) - } -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt b/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt deleted file mode 100644 index f607f507cf..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/utils/layers/SimpleDecoder.kt +++ /dev/null @@ -1,53 +0,0 @@ -package org.utbot.utils.layers - -import org.utbot.utils.MatrixUtil -import java.io.FileNotFoundException - - -class SimpleDecoder { - - lateinit var weight0: Array - lateinit var weight1: Array - lateinit var weight2: Array - lateinit var weight3: Array - - lateinit var bias0: DoubleArray - lateinit var bias1: DoubleArray - lateinit var bias2: DoubleArray - lateinit var bias3: DoubleArray - - init { - try { - weight0 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.hidden_layer_1.0.weight.txt", false) - weight1 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.hidden_layer_2.0.weight.txt", false) - weight2 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.hidden_layer_3.0.weight.txt", false) - weight3 = MatrixUtil.loadMatrix("logs/decoder/model.decoder.output_layer.0.weight.txt", false) - bias0 = MatrixUtil.loadVector("logs/decoder/model.decoder.hidden_layer_1.0.bias.txt") - bias1 = MatrixUtil.loadVector("logs/decoder/model.decoder.hidden_layer_2.0.bias.txt") - bias2 = MatrixUtil.loadVector("logs/decoder/model.decoder.hidden_layer_3.0.bias.txt") - bias3 = MatrixUtil.loadVector("logs/decoder/model.decoder.output_layer.0.bias.txt") - } catch (e: FileNotFoundException) { - e.printStackTrace() - } - } - - fun propagate(state: DoubleArray): DoubleArray? { - var x = MatrixUtil.mmul(state, weight0) - for (i in x.indices) { - x[i] = Math.max(x[i] + bias0[i], 0.0) - } - x = MatrixUtil.mmul(x, weight1) - for (i in x.indices) { - x[i] = Math.max(x[i] + bias1[i], 0.0) - } - x = MatrixUtil.mmul(x, weight2) - for (i in x.indices) { - x[i] = Math.max(x[i] + bias2[i], 0.0) - } - x = MatrixUtil.mmul(x, weight3) - for (i in x.indices) { - x[i] += bias3[i] - } - return x - } -} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt index da505812b0..9f1673a57e 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt @@ -13,7 +13,7 @@ abstract class AbstractHtmlReport(bodyWidth: Int = 600) { "logs/Report_" + dateTimeFormatter.format(LocalDateTime.now()) + ".html" fun save(filename: String = nameWithDate()) { - builder.saveHTML(filename) + builder.saveHtml(filename) } } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt index 9676fbc543..9e2784093b 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt @@ -2,34 +2,50 @@ package org.utbot.visual import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.* +import java.util.Properties +import tech.tablesaw.plotly.components.Figure +class ClassificationHtmlReport : AbstractHtmlReport() { -class ClassificationHTMLReport { - private val builder = HtmlBuilder() + private var figuresNum = 0 fun addHeader(modelName: String, properties: Properties) { - builder.addHeader(modelName, properties) + val currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss")) + builder.addRawHTML("

    Model : ${modelName}

    ") + builder.addRawHTML("

    $currentDateTime

    ") + builder.addRawHTML("

    Hyperparameters:

    ") + for (property in properties) { + builder.addText("${property.key} : ${property.value}") + } + } + + private fun addFigure(figure: Figure) { + builder.addRawHTML("
    ") + builder.addRawHTML(figure.asJavascript("plot$figuresNum")) + builder.addRawHTML("
    ") + figuresNum++ } fun addDataDistribution(y: DoubleArray, threshold: Double = 1000.0) { + val (data, notPresented) = y.partition { x -> x <= threshold } val rawHistogram = FigureBuilders.buildHistogram( - y.filter { x -> x <= threshold }.toDoubleArray(), + data.toDoubleArray(), xLabel = "ms", yLabel = "Number of samples", title = "Raw data distribution" ) - builder.addFigure(rawHistogram) + addFigure(rawHistogram) builder.addText("Number of samples: ${y.size}") - builder.addText( - "And ${ - y.filter { x -> x > threshold }.size - } more samples longer than $threshold ms are not presented in the raw data distribution plot \n\n" - ) + if (notPresented.isNotEmpty()) { + builder.addText( + "And ${notPresented.size} more samples longer than $threshold ms are not presented " + + "in the raw data distribution plot \n\n" + ) + } } fun addConfusionMatrix(confusionMatrix: Array) { - builder.addFigure( + addFigure( FigureBuilders.buildHeatmap( Array(confusionMatrix.size) { it }, Array(confusionMatrix.size) { it }, @@ -45,7 +61,7 @@ class ClassificationHTMLReport { var title = "Class distribution after resampling" if (before) title = "Class distribution before resampling" - builder.addFigure( + addFigure( FigureBuilders.buildBarPlot( Array(classSizes.size) { it }, classSizes, @@ -57,7 +73,7 @@ class ClassificationHTMLReport { } fun addPCAPlot(variance: DoubleArray, cumulativeVariance: DoubleArray) { - builder.addFigure( + addFigure( FigureBuilders.buildTwoLinesPlot( variance, cumulativeVariance, @@ -83,17 +99,4 @@ class ClassificationHTMLReport { ) } } - - fun save(filename: String = "default") { - if (filename == "default") { - val filename = "logs/Classification_Report_" + - LocalDateTime.now() - .format(DateTimeFormatter.ofPattern("dd-MM-yyyy_HH-mm-ss")) + - ".html" - builder.saveHTML(filename) - } else { - builder.saveHTML(filename) - } - } - } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt index 8cfc0ddd15..445934e513 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt @@ -10,37 +10,37 @@ import tech.tablesaw.plotly.traces.* class FigureBuilders { companion object { private fun getXYLayout( - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Plot" + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Plot" ): Layout { return Layout.builder() - .title(title) - .xAxis(Axis.builder().title(xLabel).build()) - .yAxis(Axis.builder().title(yLabel).build()) - .build() + .title(title) + .xAxis(Axis.builder().title(xLabel).build()) + .yAxis(Axis.builder().title(yLabel).build()) + .build() } private fun getXYZLayout( - xLabel: String = "X", - yLabel: String = "Y", - zLabel: String = "Y", - title: String = "Plot" + xLabel: String = "X", + yLabel: String = "Y", + zLabel: String = "Y", + title: String = "Plot" ): Layout { return Layout.builder() - .title(title) - .xAxis(Axis.builder().title(xLabel).build()) - .yAxis(Axis.builder().title(yLabel).build()) - .zAxis(Axis.builder().title(zLabel).build()) - .build() + .title(title) + .xAxis(Axis.builder().title(xLabel).build()) + .yAxis(Axis.builder().title(yLabel).build()) + .zAxis(Axis.builder().title(zLabel).build()) + .build() } fun buildScatterPlot( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Scatter plot" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Scatter plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = ScatterTrace.builder(x, y).build() @@ -49,10 +49,10 @@ class FigureBuilders { } fun buildHistogram( - data: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Histogram" + data: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Histogram" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = HistogramTrace.builder(data).build() @@ -61,11 +61,11 @@ class FigureBuilders { } fun build2DHistogram( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Histogram 2D" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Histogram 2D" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = Histogram2DTrace.builder(x, y).build() @@ -74,11 +74,11 @@ class FigureBuilders { } fun buildBarPlot( - x: Array, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "BarPlot" + x: Array, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "BarPlot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = BarTrace.builder(x, y).build() @@ -86,12 +86,12 @@ class FigureBuilders { } fun buildHeatmap( - x: Array, - y: Array, - z: Array, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Heatmap" + x: Array, + y: Array, + z: Array, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Heatmap" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = HeatmapTrace.builder(x, y, z).build() @@ -99,11 +99,11 @@ class FigureBuilders { } fun buildLinePlot( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Line plot" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Line plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = ScatterTrace.builder(x, y).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() @@ -112,55 +112,35 @@ class FigureBuilders { } fun buildTwoLinesPlot( - y1: DoubleArray, - y2: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Two lines plot" + y1: DoubleArray, + y2: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Two lines plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) - val trace1: Trace = ScatterTrace.builder(DoubleArray(y1.size) { it.toDouble() }, y1) - .mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() - val trace2: Trace = ScatterTrace.builder(DoubleArray(y2.size) { it.toDouble() }, y2) - .mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() + val trace1: Trace = ScatterTrace.builder(DoubleArray(y1.size) { it.toDouble() }, y1).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() + val trace2: Trace = ScatterTrace.builder(DoubleArray(y2.size) { it.toDouble() }, y2).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() return Figure(layout, trace1, trace2) } - fun buildSeveralLinesPlot( - x: List, - y: List, - colors: List, - names: List, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Line plot" + fun buildLinePlotSmoothed( + x: DoubleArray, + y: DoubleArray, + smoothing: Double = 1.2, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Line plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) - val traces = x.indices.map { - ScatterTrace.builder(x[it], y[it]) + val trace: Trace = ScatterTrace.builder(x, y) .mode(ScatterTrace.Mode.LINE_AND_MARKERS) - .line(Line.builder().shape(Line.Shape.LINEAR).color(colors[it]).build()) - .name(names[it]) - .showLegend(true) + .line(Line.builder().shape(Line.Shape.SPLINE).smoothing(smoothing).build()) .build() - } - - return Figure(layout, *traces.toTypedArray()) - } - fun buildBoxPlot( - x: Array, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Box plot" - ): Figure { - val layout = getXYLayout(xLabel, yLabel, title) - val trace: BoxTrace = BoxTrace.builder(x, y).build() return Figure(layout, trace) } - } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt index 91f7e4137f..98037c2186 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt @@ -1,16 +1,11 @@ package org.utbot.visual -import tech.tablesaw.plotly.components.Figure import java.io.File import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.Properties +import java.util.* -class HtmlBuilder( - bodyMaxWidth: Int = 600, - pathToStyle: List = listOf(), - pathToJs: List = listOf() -) { +class HtmlBuilder(bodyMaxWidth: Int = 600) { private val pageTop = ("" + System.lineSeparator() + "" @@ -18,77 +13,55 @@ class HtmlBuilder( + " Multi-plot test" + System.lineSeparator() + " " - + "" - + pathToJs.joinToString("") { "" } + System.lineSeparator() + "" + System.lineSeparator() - + "" + + "" + System.lineSeparator()) private val pageBottom = "" + System.lineSeparator() + "" - private var pageBuilder = StringBuilder(pageTop).append(System.lineSeparator()) - private var figres_num = 0 + private var pageBuilder = StringBuilder(pageTop).appendLine() - fun addFigure(figure: Figure) { - figure.asJavascript("plot${this.figres_num}") - pageBuilder.append("
    ").append(System.lineSeparator()) - .append(figure.asJavascript("plot${this.figres_num}")).append(System.lineSeparator()).append("
    ") - this.figres_num += 1 - } - - fun saveHTML(fileName: String = "Report.html") { + fun saveHtml(fileName: String = "Report.html") { File(fileName).writeText(pageBuilder.toString() + pageBottom) } - fun addHeader(ModelName: String, properties: Properties) { - val currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss")) - pageBuilder.append("

    Model : ${ModelName}

    ").append("

    $currentDateTime

    ") - .append("

    Hyperparameters:

    ") - for (property in properties) { - pageBuilder.append("
    ${property.key} : ${property.value}
    ").append(System.lineSeparator()) - } - + fun addText(text: String) { + pageBuilder.append("
    $text
    ") } - fun addHeader(header: String, h: String = "h1") { - pageBuilder.append("<$h>${header}") + fun addRawHTML(HTMLCode: String) { + pageBuilder.append(HTMLCode) } - fun addTable( - statistics: List>, - selectorNames: List, - colors: List, - scope: String - ) { - pageBuilder.append("
    $scope\n" + - "\n" + + "\n" + "

    ${String.format("%.0f", statistics[i][key])}

    \n" + - "    \n" + - "
    ") - pageBuilder.append("\n") - - selectorNames.forEach { pageBuilder.append("\n") } - pageBuilder.append("") + fun addBreak() { + addText("
    ") + } - statistics.first().keys.forEach { key -> - pageBuilder.append("\n") + fun addHeader1(header: String){ + addText("

    $header

    ") + } - for (i in statistics.indices) { - pageBuilder.append( - "\n" - ) - } - pageBuilder.append("") - } - pageBuilder.append("
    $scope${it}
    ${key}\n" + - "\n" + - "

    ${String.format("%.0f", statistics[i][key])}

    \n" + - "    \n" + - "
    ") + fun addHeader2(header: String){ + addText("

    $header

    ") } - fun addText(text: String) { - pageBuilder.append("
    $text
    ") + fun addHeader3(header: String){ + addText("

    $header

    ") } } diff --git a/utbot-analytics/src/main/resources/config.properties b/utbot-analytics/src/main/resources/config.properties deleted file mode 100644 index e71418b7e6..0000000000 --- a/utbot-analytics/src/main/resources/config.properties +++ /dev/null @@ -1,3 +0,0 @@ -project=antlr -selectors=random_120,cpi_120,fork_120,inheritors_120,random_120 -covStatistics=logs/covStatistics,logs/covStatistics \ No newline at end of file diff --git a/utbot-analytics/src/main/resources/css/coverage.css b/utbot-analytics/src/main/resources/css/coverage.css deleted file mode 100644 index b7b9e2a9b3..0000000000 --- a/utbot-analytics/src/main/resources/css/coverage.css +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2000-2021 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -* { - margin: 0; - padding: 0; -} - -body { - background-color: #fff; - font-family: helvetica neue, tahoma, arial, sans-serif; - font-size: 82%; - color: #151515; -} - -h1 { - margin: 0.5em 0; - color: #010101; - font-weight: normal; - font-size: 18px; -} - -h2 { - margin: 0.5em 0; - color: #010101; - font-weight: normal; - font-size: 16px; -} - -a { - color: #1564C2; - text-decoration: none; -} - -a:hover { - text-decoration: underline; -} - -i { - background-color: #eee; -} - -span.separator { - color: #9BA9BA; - padding-left: 5px; - padding-right: 5px; -} - -div.content { - width: 99%; -} - -table.coverageStats { - width: 100%; - border-collapse: collapse; -} - -table.overallStats { - width: 20%; -} - -table.coverageStats td, table.coverageStats th { - padding: 4px 2px; - border-bottom: 1px solid #ccc; -} - -table.coverageStats th { - background-color: #959BA4; - border: none; - font-weight: bold; - text-align: left; - color: #FFF; -} - -table.coverageStats th.coverageStat { - width: 20%; -} - -table.coverageStats th a { - color: #FFF; -} - -table.coverageStats th a:hover { - text-decoration: none; -} - -table.coverageStats th.sortedDesc a { - background: url(../img/arrowDown.gif) no-repeat 100% 2px; - padding-right: 20px; -} - -table.coverageStats th.sortedAsc a { - background: url(../img/arrowUp.gif) no-repeat 100% 2px; - padding-right: 20px; -} - -div.footer { - margin: 2em .5em; - font-size: 85%; - text-align: left; - line-height: 140%; -} - -div.sourceCode { - width: 100%; - border: 1px solid #ccc; - font: normal 12px 'Menlo', 'Bitstream Vera Sans Mono', 'Courier New', 'Courier', monospace; - white-space: pre; -} - -div.sourceCode b { - font-weight: normal; -} - -div.sourceCode i { - display: block; - float: left; - width: 3em; - padding-right: 3px; - border-right: 1px solid #ccc; - font-style: normal; - text-align: right; -} - -div.sourceCode i.no-highlight span.number { - color: #151515; -} - -div.sourceCode .fc, div.sourceCode .fc i { - background-color: #cfc; -} - -div.sourceCode .pc, div.sourceCode .pc i { - background-color: #ffc; -} - -div.sourceCode .nc, div.sourceCode .nc i { - background-color: #fcc; -} - -.percent, .absValue { - font-size: 90%; -} - -.percent .green, .absValue .green { - color: #32cc32; -} - -.percent .red, .absValue .red { - color: #f00; -} - -.percent .totalDiff { - color: #3f3f3f; -} diff --git a/utbot-analytics/src/main/resources/css/highlight-idea.css b/utbot-analytics/src/main/resources/css/highlight-idea.css deleted file mode 100644 index 1d1fa10150..0000000000 --- a/utbot-analytics/src/main/resources/css/highlight-idea.css +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2000-2021 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - -Intellij Idea-like styling (c) Vasily Polovnyov - -*/ - -pre code { - display: block; padding: 0.5em; - color: #000; - background: #fff; -} - -pre .subst, -pre .title { - font-weight: normal; - color: #000; -} - -pre .comment, -pre .template_comment, -pre .javadoc, -pre .diff .header { - color: #808080; - font-style: italic; -} - -pre .annotation, -pre .decorator, -pre .preprocessor, -pre .doctype, -pre .pi, -pre .chunk, -pre .shebang, -pre .apache .cbracket, -pre .input_number, -pre .http .title { - color: #808000; -} - -pre .tag, -pre .pi { - background: #efefef; -} - -/* leonid.khachaturov: redefine background as it conflicts with change highlighting we apply on top of source highlighting */ -pre .changeAdded .tag, -pre .changeRemoved .tag, -pre .changeAdded .pi, -pre .changeRemoved .pi { - background: transparent; -} - -/* leonid.khachaturov: redefine .comment from main.css */ -pre .comment { - margin: 0; - padding: 0; - font-size: 100%; -} - -pre .tag .title, -pre .id, -pre .attr_selector, -pre .pseudo, -pre .literal, -pre .keyword, -pre .hexcolor, -pre .css .function, -pre .ini .title, -pre .css .class, -pre .list .title, -pre .nginx .title, -pre .tex .command, -pre .request, -pre .status { - font-weight: bold; - color: #000080; -} - -pre .attribute, -pre .rules .keyword, -pre .number, -pre .date, -pre .regexp, -pre .tex .special { - font-weight: bold; - color: #0000ff; -} - -pre .number, -pre .regexp { - font-weight: normal; -} - -pre .string, -pre .value, -pre .filter .argument, -pre .css .function .params, -pre .apache .tag { - color: #008000; - font-weight: bold; -} - -pre .symbol, -pre .ruby .symbol .string, -pre .ruby .symbol .keyword, -pre .ruby .symbol .keymethods, -pre .char, -pre .tex .formula { - color: #000; - background: #d0eded; - font-style: italic; -} - -pre .phpdoc, -pre .yardoctag, -pre .javadoctag { - text-decoration: underline; -} - -pre .variable, -pre .envvar, -pre .apache .sqbracket, -pre .nginx .built_in { - color: #660e7a; -} - -pre .addition { - background: #baeeba; -} - -pre .deletion { - background: #ffc8bd; -} - -pre .diff .change { - background: #bccff9; -} diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt deleted file mode 100644 index 5ddb2ea95c..0000000000 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt +++ /dev/null @@ -1,49 +0,0 @@ -package org.utbot.analytics - -import mu.KotlinLogging -import org.utbot.engine.ExecutionState -import org.utbot.engine.InterProceduralUnitGraph -import org.utbot.engine.selectors.strategies.TraverseGraphStatistics -import org.utbot.engine.stmts -import org.utbot.framework.UtSettings -import java.io.File -import java.io.FileOutputStream - - -private val logger = KotlinLogging.logger {} - - -class CoverageStatistics( - private val method: String, - private val globalGraph: InterProceduralUnitGraph -) : TraverseGraphStatistics(globalGraph) { - - private val outputFile: String = "${UtSettings.coverageStatisticsDir}/$method.txt" - - init { - File(outputFile).printWriter().use { out -> - out.println("TIME,COV_TARGET_STMT,TOTAL_TARGET_STMT,COV_ALL_STMT,TOTAL_ALL_STMT") - out.println("${System.nanoTime()}" + "," + getStatistics()) - } - } - - override fun onTraversed(executionState: ExecutionState) { - runCatching { - FileOutputStream(outputFile, true).bufferedWriter() - .use { out -> - out.write(System.nanoTime().toString() + "," + getStatistics()) - out.newLine() - } - }.onFailure { - logger.warn { "Failed to save statistics: ${it.message}" } - } - } - - fun getStatistics() = with(globalGraph) { - val allStmts = this.graphs.flatMap { it.stmts } - val graphStmts = this.graphs.first().stmts - - "${graphStmts.filter { this.isCovered(it) }.size},${graphStmts.size}," + - "${allStmts.filter { this.isCovered(it) }.size},${allStmts.size}" - } -} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt index a6398096c7..5c021e204f 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/UtBotPredictor.kt @@ -3,9 +3,6 @@ package org.utbot.analytics import org.utbot.engine.pc.UtSolverStatusKind -data class NeuroSatData(val status: UtSolverStatusKind, val time: Long) - - interface UtBotAbstractPredictor { /** * Initialization signal from controller @@ -56,16 +53,15 @@ inline fun UtBotNanoTimePredictor.learnOn(input: TIn, block: () -> * Predicts sat/unsat state of some request with input [TIn] * @see Predictors.smt */ -interface IUtBotSatPredictor : UtBotAbstractPredictor { - override fun predict(input: TIn) = - NeuroSatData(status = UtSolverStatusKind.UNSAT, time = 1) //Zero for default predictor +interface IUtBotSatPredictor : UtBotAbstractPredictor { + override fun predict(input: TIn) = UtSolverStatusKind.UNSAT //Zero for default predictor } /** * Embrace [block()] inside this method to ask prediction before execution and send actual result after execution */ -fun IUtBotSatPredictor.learnOn(input: TIn, result: NeuroSatData) { +fun IUtBotSatPredictor.learnOn(input: TIn, result: UtSolverStatusKind) { val expectedResult = predict(input) provide(input, expectedResult, result) From a2932399f50dd615233f9c10b6f50739ca54adf3 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 11:37:23 +0300 Subject: [PATCH 17/37] Revert non-path-selector changes --- build.gradle | 5 - gradle.properties | 1 - .../org/utbot/visual/TracePathReport.kt | 167 ++++++++++++++++++ .../org/utbot/engine/UtBotSymbolicEngine.kt | 2 - 4 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt diff --git a/build.gradle b/build.gradle index 6e73f8d687..c4da1c528d 100644 --- a/build.gradle +++ b/build.gradle @@ -41,9 +41,4 @@ subprojects { mavenCentral() maven { url 'https://jitpack.io' } } - - tasks.withType(Test) { - minHeapSize = '128m' - maxHeapSize = '512m' - } } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index bcdd369ca8..8c7c2ceb21 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,5 +35,4 @@ eclipse_aether_version=1.1.0 maven_wagon_version=3.5.1 maven_plugin_api_version=3.8.5 maven_plugin_tools_version=3.6.4 -org.gradle.jvmargs=-Xmx1g -Xms256m # soot also depends on asm, so there could be two different versions \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt new file mode 100644 index 0000000000..e7f58a7cd4 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/TracePathReport.kt @@ -0,0 +1,167 @@ +package org.utbot.visual + +import org.utbot.summary.tag.BasicTypeTag +import org.utbot.summary.tag.ExecutionTag +import org.utbot.summary.tag.UniquenessTag +import org.utbot.summary.tag.StatementTag +import org.utbot.summary.tag.TraceTag +import soot.jimple.JimpleBody + +class TracePathReport : AbstractHtmlReport(bodyWidth = 1200) { + private val tab = "    " + + private val freqSymbols = mapOf( + UniquenessTag.Common to " ", + UniquenessTag.Unique to "★ ", + UniquenessTag.Partly to "â–º " + ) + + private val basicTagColors = mapOf( + BasicTypeTag.Initialization to "#aab7b8", + BasicTypeTag.Condition to "black", + BasicTypeTag.Return to "#28b463", + BasicTypeTag.Assignment to "black", + BasicTypeTag.Basic to "black", + BasicTypeTag.ExceptionAssignment to "red", + BasicTypeTag.ExceptionThrow to "red", + BasicTypeTag.Invoke to "black", + BasicTypeTag.IterationStart to "blue", + BasicTypeTag.IterationEnd to "#2874a6" + ) + + private val executionPostfix = mapOf( + ExecutionTag.True to " : ✓", + ExecutionTag.False to " : ✗", + ExecutionTag.Executed to "" + ) + + fun addJimpleBody(jimpleBody: JimpleBody?) { + if (jimpleBody == null) + return + var result = "

    Jimple Body

    \n" + jimpleBody.units.forEach { + result += "${it.javaSourceStartLineNumber}:$tab $it
    \n" + } + + builder.addRawHTML(result) + } + + private fun buildStructViewStatementTag(statementTag: StatementTag?): String { + if (statementTag == null) + return "" + var result = "
  • ${statementTag.step.stmt} \n" + + "
    Decision = ${markDecision(statementTag.executionTag)}\n" + + "
    Line = ${statementTag.line} \n" + + "
    Type = ${statementTag.basicTypeTag} \n" + + "
    Frequency = ${statementTag.uniquenessTag} \n" + + "
    Call times = ${statementTag.callOrderTag}
    \n" + + "
    • " + + if (statementTag.invoke != null) { + result += "\n
    Invocation:
    \n" + + "
      ${buildStructViewStatementTag(statementTag.invoke)}
    \n" + } + if (statementTag.iterations.size > 0) { + result += "Iterations:\n" + result += "
      " + for (i in 0 until statementTag.iterations.size) { + result += "
      Iteration $i
      ${buildStructViewStatementTag(statementTag.iterations[i])}\n" + } + result += "
    " + } + if (statementTag.next != null) { + result += "
    ${buildStructViewStatementTag(statementTag.next)}\n" + } + return result + } + + fun addStructViewTraces(tracesTags: Iterable) { + var table = "

    Struct View

    \n " + + "\n" + tracesTags.forEach { + table += "\n" + } + table += "\n" + builder.addRawHTML(table) + } + + fun addTracesTable(tagsToKeywords: List>, name: String) { + var table = "

    $name

    \n" + + "
      ${buildStructViewStatementTag(it.rootStatementTag)}
    " + + table += "" + table += "" + table += "" + table += " " + table += "" + + for ((clusterNum, clusterTraceTags) in tagsToKeywords.withIndex()) { + + for (traceTags in clusterTraceTags) { + table += "" + val traceTagsVisual = traceTags.rootStatementTag?.let { visualizeStatementTag(it) } + table += "" + + table += "" + + table += "" + + table += "" + table += "" + } + } + table += "
    â„– Jimple codeSource codeKeywordsComment
    $clusterNum
    "
+                table += traceTagsVisual ?: "None"
+                table += "
    "
+                table += "No source code yet"
+                table += "
    "
+                table += traceTags.summary
+                table += "
    " + builder.addRawHTML(table) + } + + private fun markDecision(executionTag: ExecutionTag): String { + var result = when (executionTag) { + ExecutionTag.True -> "" + ExecutionTag.False -> "" + else -> "" + } + result += "$executionTag " + return result + } + + fun addExecutionVisualisation(rootTag: StatementTag, name: String){ + var visualization = "

    $name

    \n" + visualization += "
    "
+        visualization += visualizeStatementTag(rootTag)
+        visualization += "
    " + builder.addRawHTML(visualization) + } + + private fun visualizeStatementTag(tag: StatementTag, tabPrefix: String = ""): String { + var localTabPrefix = tabPrefix + var visualization = "" + + visualization += "" + visualization += tag.line.toString().padEnd(3, ' ') + ':' + visualization += localTabPrefix + visualization += freqSymbols[tag.uniquenessTag] + if (tag.uniquenessTag == UniquenessTag.Unique) visualization += "" + visualization += tag.step.stmt.toString() + visualization += executionPostfix[tag.executionTag] + if (tag.uniquenessTag == UniquenessTag.Unique) visualization += "" + visualization += "" + visualization += "\n" + + if (tag.invoke != null) visualization += visualizeStatementTag(tag.invoke!!, localTabPrefix + "\t") + if (tag.iterations.size > 0) { + for (cycle in tag.iterations) visualization += visualizeStatementTag(cycle, localTabPrefix + "\t") + } + + if (tag.basicTypeTag == BasicTypeTag.IterationEnd) localTabPrefix = localTabPrefix.removePrefix("\t") + if (tag.next != null) visualization += visualizeStatementTag(tag.next!!, localTabPrefix) + + return visualization + } + +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index fefe1cbf55..81b8d52804 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -18,7 +18,6 @@ import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.isActive import kotlinx.coroutines.yield import mu.KotlinLogging -import org.utbot.analytics.CoverageStatistics import org.utbot.analytics.EngineAnalyticsContext import org.utbot.analytics.FeatureProcessor import org.utbot.analytics.Predictors @@ -443,7 +442,6 @@ class UtBotSymbolicEngine( require(trackableResources.isEmpty()) if (useDebugVisualization) GraphViz(globalGraph, pathSelector) - if (UtSettings.collectCoverage) CoverageStatistics(methodUnderTest.toString(), globalGraph) val initStmt = graph.head val initState = ExecutionState( From d26c13ce8c73ac267fdb9d32c3afc79550380399 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 13:11:10 +0300 Subject: [PATCH 18/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 2 +- .github/workflows/build-and-run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index e6e8798bac..d3e59aaf71 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -5,7 +5,7 @@ on: jobs: build-and-run-tests: - runs-on: ubuntu-20.04 + runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index 30af5f45ad..aa3354876a 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -8,7 +8,7 @@ on: jobs: build_and_run_tests: - runs-on: ubuntu-20.04 + runs-on: windows-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 From c908853bd6a2c0955685f2a63dc9c7c555911943 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 13:39:52 +0300 Subject: [PATCH 19/37] Try to fix CI --- .github/workflows/build-and-run-tests-from-branch.yml | 2 +- .github/workflows/build-and-run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index d3e59aaf71..318a269afd 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -20,7 +20,7 @@ jobs: - name: Build and run tests in UTBot Java run: | - export KOTLIN_HOME="/usr" + SET KOTLIN_HOME="%windir%" gradle clean build --no-daemon - name: Upload utbot-framework logs diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index aa3354876a..f17c39adae 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -22,7 +22,7 @@ jobs: - name: Build and run tests in UTBot Java run: | - export KOTLIN_HOME="/usr" + SET KOTLIN_HOME="%windir%" gradle clean build --no-daemon - name: Upload utbot-framework logs From 79360aa6cab7b926d566c168f5e9df73492f60c4 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 15:18:36 +0300 Subject: [PATCH 20/37] Revert "Try to fix CI" This reverts commit 7a98a7ce87b5c62a71b52eb83694af99493421d7. --- .github/workflows/build-and-run-tests-from-branch.yml | 2 +- .github/workflows/build-and-run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index 318a269afd..d3e59aaf71 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -20,7 +20,7 @@ jobs: - name: Build and run tests in UTBot Java run: | - SET KOTLIN_HOME="%windir%" + export KOTLIN_HOME="/usr" gradle clean build --no-daemon - name: Upload utbot-framework logs diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index f17c39adae..aa3354876a 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -22,7 +22,7 @@ jobs: - name: Build and run tests in UTBot Java run: | - SET KOTLIN_HOME="%windir%" + export KOTLIN_HOME="/usr" gradle clean build --no-daemon - name: Upload utbot-framework logs From 2e415b655eb45e70c24562b0c8585da1b089bdb2 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 15:18:37 +0300 Subject: [PATCH 21/37] Revert "Try to fix CI" This reverts commit ff5c242bee02da3cff045cb17e12e036968236be. --- .github/workflows/build-and-run-tests-from-branch.yml | 2 +- .github/workflows/build-and-run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index d3e59aaf71..e6e8798bac 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -5,7 +5,7 @@ on: jobs: build-and-run-tests: - runs-on: windows-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 diff --git a/.github/workflows/build-and-run-tests.yml b/.github/workflows/build-and-run-tests.yml index aa3354876a..30af5f45ad 100644 --- a/.github/workflows/build-and-run-tests.yml +++ b/.github/workflows/build-and-run-tests.yml @@ -8,7 +8,7 @@ on: jobs: build_and_run_tests: - runs-on: windows-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v2 From 9e063d72ab8a0323a387839942d79d69f312566f Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 1 Jun 2022 15:25:16 +0300 Subject: [PATCH 22/37] Make parent optional --- .../src/main/kotlin/org/utbot/engine/ExecutionState.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt index e733606f62..f413ff0456 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt @@ -12,6 +12,7 @@ import org.utbot.engine.pc.UtSolver import org.utbot.engine.pc.UtSolverStatusUNDEFINED import org.utbot.engine.symbolic.SymbolicState import org.utbot.engine.symbolic.SymbolicStateUpdate +import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.Step import soot.SootMethod import soot.jimple.Stmt @@ -86,7 +87,7 @@ data class StateAnalyticsProperties( successorVisitedAfterLastFork, successorVisitedBeforeLastFork, successorStmtSinceLastCovered, - parent + if (UtSettings.featureProcess) parent else null ) } From 6c158578919367648c2578e720da77966d666adc Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 3 Jun 2022 17:08:03 +0300 Subject: [PATCH 23/37] Add docs --- docs/jlearch/execution-state-changes.md | 41 +++++ docs/jlearch/jlearch-architecture.md | 152 ++++++++++++++++++ .../jlearch/new-heuristical-path-selectors.md | 93 +++++++++++ .../org/utbot/analytics/FeatureProcessor.kt | 2 +- 4 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 docs/jlearch/execution-state-changes.md create mode 100644 docs/jlearch/jlearch-architecture.md create mode 100644 docs/jlearch/new-heuristical-path-selectors.md diff --git a/docs/jlearch/execution-state-changes.md b/docs/jlearch/execution-state-changes.md new file mode 100644 index 0000000000..50673df979 --- /dev/null +++ b/docs/jlearch/execution-state-changes.md @@ -0,0 +1,41 @@ +# Changes in Execution State + +```mermaid +classDiagram + class StateAnalyticsProperties{ + +int depth + +int visitedAfterLastFork + +int visitedBeforeLastFork + +int stmtsSinceLastCovered + +ExecutionState? parent + +long executingTime + +double reward + +List~Double~ features + -boolean isFork + -boolean isVisitedNew + -int successorDepth + -int successorVisitedAfterLastFork + -int successorVisitedBeforeLastFork + -int successorStmtSinceLastCovered + + +updateIsVisitedNew() + +updateIsFork() + } + ExecutionState o-- StateAnalyticsProperties +``` + +`StateAnalyticsProperties` maintains properties of `ExecutionState`, which don't need for symbolic execution, but need for `JLearch`. + +* `depth: Int` - number of forks on state's path excluded current state, if it is fork. In this case, fork is a state with more than one successors excluded implicit `NPE` branches. +* `visitedAfterLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path after last fork in first time. +* `visitedBeforeLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path before last fork in first time. +* `stmtsSinceLastCovered: Int` - number of `states` on this state's path after last state that visited any `stmt` in first time. +* `parent: ExecutionState?` - parent of current `state`. If `UtSettings.featureProcess == false`, then it is always null, because we don't need this field in this case. If it is not null, then we can't delete `state` until all successors of this state will be deleted, which may cause memory issue. +* `executingTime: Long` - amount of time, during which this state was traversed. +* `reward: Double?` - calculated reward of this state +* `features: List` - list of extracted features for this state + +Field with `successor` prefix is used for constructor of successor properties. + +* `updateIsFork()` - set `isFork` on true. This method called when traversing of `stmt` produce more than one explicit state. Now it may be during traversing of `IfStmt`, `SwitchStmt`, `AssignStmt` or `InvokeStmt`. +* `updateIsCoveredNew()` - set `isVisitedNew` on true, set `stmtsSinceLastCovered` on zero and increase `visitedAfterLastFork` on 1. This method is called in `UtBotSymbolicEngine` after new state `s` is polled and `s.stmt` was not visited yet. diff --git a/docs/jlearch/jlearch-architecture.md b/docs/jlearch/jlearch-architecture.md new file mode 100644 index 0000000000..434ea9a530 --- /dev/null +++ b/docs/jlearch/jlearch-architecture.md @@ -0,0 +1,152 @@ +# JLearch architecture + +# Global Class Diagram + +```mermaid +classDiagram + class FeatureProcessor{ + dumpFeatures() + } + <> FeatureProcessor + class TraverseGraphStatistics{ + onVisit(ExecutionState) + onTraversed(ExecutionState) + } + class InterproceduralUnitGraph + class FeatureExtractorFactory + <> FeatureExtractorFactory + class FeatureProcessorFactory + <> FeatureProcessorFactory + class EngineAnalyticsContext + class UtBotSymbolicEngine + class NNRewardGuidedSelectorFactory + <> NNRewardGuidedSelectorFactory + class FeatureExtractor{ + extractFeatures(ExecutionState) + } + <> FeatureExtractor + + UtBotSymbolicEngine ..> EngineAnalyticsContext + EngineAnalyticsContext o-- FeatureProcessorFactory + EngineAnalyticsContext o-- FeatureExtractorFactory + EngineAnalyticsContext o-- NNRewardGuidedSelectorFactory + + FeatureProcessor --|> TraverseGraphStatistics + InterproceduralUnitGraph o-- TraverseGraphStatistics + UtBotSymbolicEngine *-- FeatureProcessor + UtBotSymbolicEngine *-- InterproceduralUnitGraph + + class Predictors + class NNStateRewardPredictor + class NNRewardGuidedSelector + + + class GreedySearch + + class BasePathSelector + + GreedySearch --|> BasePathSelector + NNRewardGuidedSelector --|> GreedySearch + + UtBotSymbolicEngine *-- BasePathSelector + + Predictors o-- NNStateRewardPredictor + NNRewardGuidedSelector ..> Predictors + NNRewardGuidedSelector *-- FeatureExtractor + + NNStateRewardPredictorSmile --|> NNStateRewardPredictor + NNStateRewardPredictorTorch --|> NNStateRewardPredictor + + NNStateRewardGuidedSelectorWithRecalculationWeight --|> NNRewardGuidedSelector + NNStateRewardGuidedSelectorWithoutRecalculationWeight --|> NNRewardGuidedSelector +``` + +This diagram doesn't illustrate some details, so read them below. + +# FeatureProcessor + +It is interface in framework-module, that allow to use implementation from analytics module. + +* `dumpFeatures(state: ExecutionState)` - dump features and rewards in some format on disk. Called at the end of traverse in `UtBotSymbolicEngine` + +## Implementation class diagram + +```mermaid +classDiagram + class FeatureProcessorWithStatesRepetition{ + -Map~Int, FeatureList~ dumpedStates + -Set~Stmt~ visitedStmts + -List~TestCase~ testCases + -int generatedTestCases + dumpFeatures() + } + + class FeatureExtractor{ + extractFeatures(ExecutionState) + } + + class TraverseGraphStatistics{ + onVisit(ExecutionState) + onTraversed(ExecutionState) + } + + class RewardEstimator{ + calculateRewards(List~TestCase~) + } + + class TestCase{ + +List states + +int newCoverage + +int testIndex + } + + FeatureProcessorWithStatesRepetition --|> TraverseGraphStatistics + FeatureProcessorWithStatesRepetition o-- FeatureExtractor + FeatureProcessorWithStatesRepetition o-- RewardEstimator + FeatureProcessorWithStatesRepetition ..> EngineAnalyticsContext + +``` + +`State = Pair` + +`FeatureList = List): Map` - calculates `coverage` for each state and `time` for each state. `Coverage` - sum of `newCoverage` by `TestCase` that contain its state. `Time` - sum of `state.executingTime` by all states, that has this state on its path. Then calculates `reward(coverage, time)`. + +## FeatureProcessorWithStatesRepetition + +* `onVisit(state: ExecutionState)` - extractFeatures for state +* `onTraversed(state: ExecutionState)` - create `TestCase`, so we go from `state` to `state.parent` while it is not root, for each `state` on path add its features to `dumpedStates`, calculate coverage of its `TestCase`, increment `generatedTestCases` on 1 and add new `TestCase` in `testCases`. +* `dumpFeatures()` - call `RewardEstimator.calculateRewards()` and write `csv` file for each `TestCase` in format: `newCov,features,reward` for each `state` in it. `newCov` - flag that indicates whether this `TestCase` cover something new or not. So in this approach, each `state` will be written as many times as number of `TestCase` that has it. +For creating `FeatureExtractor` it uses `FeatureExtractorFactory` from `EngineAnalyticsContext`. + +# FeatureExtractor + +It is interface in framework-module, that allow to use implementation from analytics module. +* `extractFeatures(state: ExecutionState)` - create features list for state and store it in `state.features`. Now we extract all features, which was described in [paper](https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf). In feature, we can extend feature list by other features, for example, NeuroSMT. + +# NNStateRewardPredictor + +Interface for reward predictors. Now it has two implementations in `analytics` module: + +* `NNStateRewardPredictorSmile`: it uses our own format to store feedforward neural network and it uses `Smile` library to do multiplication of matrix. +* `NNStateRewardPredictorTorch`: it assumed that model is any type of model in `pt` format. It uses `Deep Java library` to use such models. + +It should be created at the beginning of work and stored at `Predictors` class to be used in `NNRewardGuidedSelector` from `framework` module. + + +# NNStateRewardGuidedSelector + +It uses an `EngineAnalyticsContext` to create `FeatureExtractor`. +We override `ExecutionState.weight` as `NNStateRewardPredictor.predict(this.features)`. +We have two different implementantions: +* `NNStateRewardGuidedSelectorWithRecalculation`: we recalculate reward every time, so in `ExecutionState.weight` we extract features and call predict. +* `NNStateRewardGuidedSlectorWithoutRecalculation`: we extract features in `offerImpl`, calculate `reward` and store it in `ExecutionState.reward` without recalculation it every time. + +# EngineAnalyticsContext + +It is object that should be filled by factories in the beginning of work to allow objects from `framework` module using objects from `analytics` module. diff --git a/docs/jlearch/new-heuristical-path-selectors.md b/docs/jlearch/new-heuristical-path-selectors.md new file mode 100644 index 0000000000..948dae6159 --- /dev/null +++ b/docs/jlearch/new-heuristical-path-selectors.md @@ -0,0 +1,93 @@ +# GreedySearch + +```mermaid +classDiagram + class GreedySearch{ + -Set~ExecutionState~ states + +ExecutionState.weight + } + GreedySearch --|> BasePathSelector +``` +Base methods such as `offer` or `remove` is implemented pretty simple and just a delegation to `states`. + +In `peekImpl` we find the set of `states` with maximum `weight` and peek random among them, so to use this class in implementation of some `pathSelector`, you just need to override an `ExecutionState.weight`. + +# SubpathStatistics + +```mermaid +classDiagram + class SubpathStatistics{ + +int index + -Map~Subpath, Int~ subpathCount + subpathCount(ExecutionState) + } + class TraverseGraphStatistics{ + onVisit(ExecutionState) + } + + SubpathStatistics --|> TraverseGraphStatistics + TraverseGraphStatistics o-- InterProceduralUnitGraph +``` +`Subpath` = `List` + +This class maintains frequency of each subpath with length `2^index`, which is presented as `List`, on certain instance of `InterproceduralUnitGraph` + +* `onVisit(state: ExecutionState)` - we calculate subpath of this state and increment its frequency on `1` +* `subpathCount(state: ExecutionState)` - we calculate subpath of this state and return its frequency + +# SubpathGuidedSelector + +```mermaid +classDiagram + SubpathGuidedSelector o-- SubpathStatistics + SubpathGuidedSelector --|> GreedySearch +``` + +Inspired by [paper](http://pxzhang.cn/paper/concolic_testing/oopsla13-pgse.pdf). + +We override `ExecutionState.weight` as `-StatementStatistics.subpathCount(this)`, so we pick `state`, which `subpath` is less traveled. + +# StatementStatistics + +```mermaid +classDiagram + class StatementStatistics{ + -Map~Stmt, Int~ statementsCount + -Map~SootMethod, Int~ statementsInMethodCount + +statementCount(ExecutionState) + +statementsInMethodCount(ExecutionState) + } + + class TraverseGraphStatistics{ + onVisit(ExecutionState) + } + + StatementStatistics --|> TraverseGraphStatistics + TraverseGraphStatistics o-- InterProceduralUnitGraph +``` + +This class maintains frequency of each `Stmt` and number of `Stmt`, that was visited in some `SootMethod`, on certain instance of `InterproceduralUnitGraph`. + +* `onVisit(state: ExecutionState)` - increment frequency of state's `stmt` on 1. If we visit this `stmt` for the first time, then increment number of `Stmt`, that we visit in current state's `method`, on 1. +* `statementCount(state: ExecutionState)` - get frequency of state's `stmt` +* `statementsInMethodCount(state: ExecutionState)` - get number of `stmt`, that was visited in current state's `method`. + +# CPInstSelector + +```mermaid +classDiagram + CPInstSelector o-- StatementStatistics + CPInstSelector --|> NonUniformRandomSearch +``` + +Override `ExecutionState.cost` as `StatementStatistics.statementInMethodCount(this)`, so we are more likely to explore the least explored `method`. + +# ForkDepthSelector + +```mermaid +classDiagram + ForkDepthSelector --|> NonUniformRandomSearch +``` + +Override `ExecutionState.cost` as `ExecutionState.depth`, so we are more likely to explore the least deep `state` in terms of number of forks on its path. + diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt index b13dc72ddf..4a3874f8b4 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt @@ -4,7 +4,7 @@ import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.selectors.strategies.TraverseGraphStatistics /** - * Interface that incapsulates work with FeatureProcessor and can only dumpFeatures at the end of symbolic execution + * Interface that encapsulates work with FeatureProcessor and can only dumpFeatures at the end of symbolic execution */ abstract class FeatureProcessor(graph: InterProceduralUnitGraph) : TraverseGraphStatistics(graph) { abstract fun dumpFeatures() From 58087e71ac03e2a2a8878d8bb40aca44cfcb7ff3 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 3 Jun 2022 17:16:12 +0300 Subject: [PATCH 24/37] Add versions in properties --- gradle.properties | 4 ++++ utbot-analytics/build.gradle | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/gradle.properties b/gradle.properties index 8c7c2ceb21..72477d3a33 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,4 +35,8 @@ eclipse_aether_version=1.1.0 maven_wagon_version=3.5.1 maven_plugin_api_version=3.8.5 maven_plugin_tools_version=3.6.4 +javacpp_version=1.5.3 +jsoup_version=1.7.2 +djl_api_version=0.16.0 +pytorch_native_version=1.9.1 # soot also depends on asm, so there could be two different versions \ No newline at end of file diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index 635a5fb7c5..26c56fdcee 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -30,7 +30,7 @@ dependencies { implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" - implementation group: 'org.bytedeco', name: 'javacpp', version: '1.5.3', classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" implementation group: 'tech.tablesaw', name: 'tablesaw-core', version: '0.38.2' implementation group: 'tech.tablesaw', name: 'tablesaw-jsplot', version: '0.38.2' @@ -39,11 +39,11 @@ dependencies { implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.22.1' - implementation group: 'org.jsoup', name: 'jsoup', version: '1.7.2' + implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version - implementation "ai.djl:api:0.16.0" - implementation "ai.djl.pytorch:pytorch-engine:0.16.0" - implementation "ai.djl.pytorch:pytorch-native-auto:1.9.1" + implementation "ai.djl:api:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" testCompile project(':utbot-framework').sourceSets.test.output } From 36b8a6f48f97532099e4d94b31341da63ea138c5 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Tue, 7 Jun 2022 14:51:14 +0300 Subject: [PATCH 25/37] Fix issues --- .../features/FeatureProcessorWithStatesRepetition.kt | 12 ++++++------ .../kotlin/org/utbot/engine/UtBotSymbolicEngine.kt | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt index 398e398047..8c76325a4d 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt @@ -120,16 +120,16 @@ internal class RewardEstimator { fun calculateRewards(testCases: List): Map { val rewards = mutableMapOf() val coverages = mutableMapOf() - val times = mutableMapOf() + val stateToExecutingTime = mutableMapOf() testCases.forEach { ts -> var allTime = 0L - ts.states.forEach { (state, time) -> - coverages.compute(state) { _, v -> + ts.states.forEach { (stateHash, time) -> + coverages.compute(stateHash) { _, v -> ts.newCoverage + (v ?: 0) } - val isNewState = state !in times - times.compute(state) { _, v -> + val isNewState = stateHash !in stateToExecutingTime + stateToExecutingTime.compute(stateHash) { _, v -> allTime + (v ?: time) } if (isNewState) { @@ -139,7 +139,7 @@ internal class RewardEstimator { } coverages.forEach { (state, coverage) -> - rewards[state] = reward(coverage.toDouble(), times.getValue(state).toDouble()) + rewards[state] = reward(coverage.toDouble(), stateToExecutingTime.getValue(state).toDouble()) } return rewards diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 81b8d52804..955a810af6 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -1395,8 +1395,7 @@ class UtBotSymbolicEngine( } } - queuedSymbolicStateUpdates += typeRegistry.genericTypeParameterConstraint(value.addr, typeStorages) - .asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.genericTypeParameterConstraint(value.addr, typeStorages).asHardConstraint() parameterAddrToGenericType += value.addr to type } } From 01ea45567ba6e76b6f76c15555b292a455f75485 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Tue, 7 Jun 2022 15:11:56 +0300 Subject: [PATCH 26/37] Revert engine style --- .../org/utbot/engine/UtBotSymbolicEngine.kt | 67 +++++-------------- 1 file changed, 15 insertions(+), 52 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 955a810af6..7716005e3f 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -472,8 +472,7 @@ class UtBotSymbolicEngine( } stateSelectedCount++ - pathLogger.trace { - "traverse<$methodUnderTest>: choosing next state($stateSelectedCount), " + + pathLogger.trace { "traverse<$methodUnderTest>: choosing next state($stateSelectedCount), " + "queue size=${(pathSelector as? NonUniformRandomSearch)?.size ?: -1}" } @@ -603,7 +602,7 @@ class UtBotSymbolicEngine( val isFuzzable = executableId.parameters.all { classId -> classId != Method::class.java.id && // causes the child process crash at invocation - classId != Class::class.java.id // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method) + classId != Class::class.java.id // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method) } if (!isFuzzable) { return@flow @@ -1302,12 +1301,7 @@ class UtBotSymbolicEngine( ?: error("Exception wasn't caught, stmt: $current, line: ${current.lines}") val nextState = environment.state.updateQueued( globalGraph.succ(current), - SymbolicStateUpdate( - localMemoryUpdates = localMemoryUpdate( - localVariable to value, - CAUGHT_EXCEPTION to null - ) - ) + SymbolicStateUpdate(localMemoryUpdates = localMemoryUpdate(localVariable to value, CAUGHT_EXCEPTION to null)) ) pathSelector.offer(nextState) } @@ -1520,8 +1514,7 @@ class UtBotSymbolicEngine( queuedSymbolicStateUpdates += MemoryUpdate(mockInfos = persistentListOf(MockInfoEnriched(mockInfo))) // add typeConstraint for mocked object. It's a declared type of the object. - queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all() - .asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all().asHardConstraint() queuedSymbolicStateUpdates += mkEq(typeRegistry.isMock(mockedObject.addr), UtTrue).asHardConstraint() return mockedObject @@ -1559,8 +1552,7 @@ class UtBotSymbolicEngine( queuedSymbolicStateUpdates += MemoryUpdate(mockInfos = persistentListOf(MockInfoEnriched(mockInfo))) // add typeConstraint for mocked object. It's a declared type of the object. - queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all() - .asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, mockedObject.typeStorage).all().asHardConstraint() queuedSymbolicStateUpdates += mkEq(typeRegistry.isMock(mockedObject.addr), UtTrue).asHardConstraint() return mockedObject @@ -1595,8 +1587,7 @@ class UtBotSymbolicEngine( createObject(addr, refType, useConcreteType = true) } } else { - queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, TypeStorage(refType)).all() - .asHardConstraint() + queuedSymbolicStateUpdates += typeRegistry.typeConstraint(addr, TypeStorage(refType)).all().asHardConstraint() objectValue(refType, addr, StringWrapper()).also { initStringLiteral(it, this.value) @@ -1715,11 +1706,7 @@ class UtBotSymbolicEngine( val arrayType = type.arrayType val arrayValue = createNewArray(value.length.toPrimitiveValue(), arrayType, type).also { val defaultValue = arrayType.defaultSymValue - queuedSymbolicStateUpdates += arrayUpdateWithValue( - it.addr, - arrayType, - defaultValue as UtArrayExpressionBase - ) + queuedSymbolicStateUpdates += arrayUpdateWithValue(it.addr, arrayType, defaultValue as UtArrayExpressionBase) } queuedSymbolicStateUpdates += objectUpdate( stringWrapper.copy(typeStorage = TypeStorage(utStringClass.type)), @@ -1883,8 +1870,7 @@ class UtBotSymbolicEngine( return objectValue.copy(addr = nullObjectAddr) } - val typeConstraint = - typeRegistry.typeConstraint(castedObject.addr, castedObject.typeStorage).isOrNullConstraint() + val typeConstraint = typeRegistry.typeConstraint(castedObject.addr, castedObject.typeStorage).isOrNullConstraint() // When we do downCast, we should add possible equality to null // to avoid situation like this: @@ -2040,8 +2026,7 @@ class UtBotSymbolicEngine( ) if (objectValue.type.isJavaLangObject()) { - queuedSymbolicStateUpdates += typeRegistry.zeroDimensionConstraint(objectValue.addr) - .asSoftConstraint() + queuedSymbolicStateUpdates += typeRegistry.zeroDimensionConstraint(objectValue.addr).asSoftConstraint() } objectValue @@ -2913,11 +2898,7 @@ class UtBotSymbolicEngine( return listOf( MethodResult( newArray, - memoryUpdates = arrayUpdateWithValue( - newArray.addr, - arrayType, - selectArrayExpressionFromMemory(src) - ) + memoryUpdates = arrayUpdateWithValue(newArray.addr, arrayType, selectArrayExpressionFromMemory(src)) ) ) } @@ -3404,18 +3385,8 @@ class UtBotSymbolicEngine( is JMulExpr -> when (sort.type) { is ByteType -> lowerIntMulOverflowCheck(left, right, Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()) is ShortType -> lowerIntMulOverflowCheck(left, right, Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) - is IntType -> higherIntMulOverflowCheck( - left, - right, - Int.SIZE_BITS, - Int.MIN_VALUE.toLong() - ) { it: UtExpression -> it.toIntValue() } - is LongType -> higherIntMulOverflowCheck( - left, - right, - Long.SIZE_BITS, - Long.MIN_VALUE - ) { it: UtExpression -> it.toLongValue() } + is IntType -> higherIntMulOverflowCheck(left, right, Int.SIZE_BITS, Int.MIN_VALUE.toLong()) { it: UtExpression -> it.toIntValue() } + is LongType -> higherIntMulOverflowCheck(left, right, Long.SIZE_BITS, Long.MIN_VALUE) { it: UtExpression -> it.toLongValue() } else -> null } else -> null @@ -3631,19 +3602,11 @@ class UtBotSymbolicEngine( //choose types that have biggest priority resolvedParameters .filterIsInstance() - .forEach { - queuedSymbolicStateUpdates += constructConstraintForType( - it, - it.possibleConcreteTypes - ).asSoftConstraint() - } + .forEach { queuedSymbolicStateUpdates += constructConstraintForType(it, it.possibleConcreteTypes).asSoftConstraint() } val returnValue = (symbolicResult as? SymbolicSuccess)?.value as? ObjectValue if (returnValue != null) { - queuedSymbolicStateUpdates += constructConstraintForType( - returnValue, - returnValue.possibleConcreteTypes - ).asSoftConstraint() + queuedSymbolicStateUpdates += constructConstraintForType(returnValue, returnValue.possibleConcreteTypes).asSoftConstraint() workaround(REMOVE_ANONYMOUS_CLASSES) { val sootClass = returnValue.type.sootClass @@ -3793,7 +3756,7 @@ class UtBotSymbolicEngine( queuedSymbolicStateUpdates ) } finally { - queuedSymbolicStateUpdates = prevSymbolicStateUpdate + queuedSymbolicStateUpdates = prevSymbolicStateUpdate } } } From 05c22aec6e09b5c064ba3d4a4eaa006a9333801e Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Tue, 7 Jun 2022 16:15:43 +0300 Subject: [PATCH 27/37] Fix spelling --- docs/jlearch/execution-state-changes.md | 12 +++++------ docs/jlearch/jlearch-architecture.md | 20 +++++++++---------- .../jlearch/new-heuristical-path-selectors.md | 14 ++++++------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/jlearch/execution-state-changes.md b/docs/jlearch/execution-state-changes.md index 50673df979..b3a88d3984 100644 --- a/docs/jlearch/execution-state-changes.md +++ b/docs/jlearch/execution-state-changes.md @@ -26,16 +26,16 @@ classDiagram `StateAnalyticsProperties` maintains properties of `ExecutionState`, which don't need for symbolic execution, but need for `JLearch`. -* `depth: Int` - number of forks on state's path excluded current state, if it is fork. In this case, fork is a state with more than one successors excluded implicit `NPE` branches. -* `visitedAfterLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path after last fork in first time. -* `visitedBeforeLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path before last fork in first time. -* `stmtsSinceLastCovered: Int` - number of `states` on this state's path after last state that visited any `stmt` in first time. +* `depth: Int` - number of forks on the state's path excluded current state, if it is fork. In this case, fork is a state with more than one successor excluded implicit `NPE` branches. +* `visitedAfterLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path after the last fork in first time. +* `visitedBeforeLastFork: Int` - number of `stmt`, that was visited by `states` on this state's path before the last fork in first time. +* `stmtsSinceLastCovered: Int` - number of `states` on this state's path after the last state that visited any `stmt` in first time. * `parent: ExecutionState?` - parent of current `state`. If `UtSettings.featureProcess == false`, then it is always null, because we don't need this field in this case. If it is not null, then we can't delete `state` until all successors of this state will be deleted, which may cause memory issue. * `executingTime: Long` - amount of time, during which this state was traversed. * `reward: Double?` - calculated reward of this state * `features: List` - list of extracted features for this state -Field with `successor` prefix is used for constructor of successor properties. +Field with `successor` prefix is used for a constructor of successor properties. -* `updateIsFork()` - set `isFork` on true. This method called when traversing of `stmt` produce more than one explicit state. Now it may be during traversing of `IfStmt`, `SwitchStmt`, `AssignStmt` or `InvokeStmt`. +* `updateIsFork()` - set `isFork` on true. This method is called when traversing of `stmt` produces more than one explicit state. Now it may be during the traversing of `IfStmt`, `SwitchStmt`, `AssignStmt` or `InvokeStmt`. * `updateIsCoveredNew()` - set `isVisitedNew` on true, set `stmtsSinceLastCovered` on zero and increase `visitedAfterLastFork` on 1. This method is called in `UtBotSymbolicEngine` after new state `s` is polled and `s.stmt` was not visited yet. diff --git a/docs/jlearch/jlearch-architecture.md b/docs/jlearch/jlearch-architecture.md index 434ea9a530..2263c4013d 100644 --- a/docs/jlearch/jlearch-architecture.md +++ b/docs/jlearch/jlearch-architecture.md @@ -65,7 +65,7 @@ This diagram doesn't illustrate some details, so read them below. # FeatureProcessor -It is interface in framework-module, that allow to use implementation from analytics module. +It is interface in framework-module, that allows to use implementation from analytics module. * `dumpFeatures(state: ExecutionState)` - dump features and rewards in some format on disk. Called at the end of traverse in `UtBotSymbolicEngine` @@ -115,28 +115,28 @@ classDiagram Maintains calculation of reward. -* `calculateRewards(List): Map` - calculates `coverage` for each state and `time` for each state. `Coverage` - sum of `newCoverage` by `TestCase` that contain its state. `Time` - sum of `state.executingTime` by all states, that has this state on its path. Then calculates `reward(coverage, time)`. +* `calculateRewards(List): Map` - calculates `coverage` for each state and `time` for each state. `Coverage` - sum of `newCoverage` by `TestCase` that contains its state. `Time` - sum of `state.executingTime` by all states, that has this state on its path. Then calculates `reward(coverage, time)`. ## FeatureProcessorWithStatesRepetition * `onVisit(state: ExecutionState)` - extractFeatures for state * `onTraversed(state: ExecutionState)` - create `TestCase`, so we go from `state` to `state.parent` while it is not root, for each `state` on path add its features to `dumpedStates`, calculate coverage of its `TestCase`, increment `generatedTestCases` on 1 and add new `TestCase` in `testCases`. -* `dumpFeatures()` - call `RewardEstimator.calculateRewards()` and write `csv` file for each `TestCase` in format: `newCov,features,reward` for each `state` in it. `newCov` - flag that indicates whether this `TestCase` cover something new or not. So in this approach, each `state` will be written as many times as number of `TestCase` that has it. -For creating `FeatureExtractor` it uses `FeatureExtractorFactory` from `EngineAnalyticsContext`. +* `dumpFeatures()` - call `RewardEstimator.calculateRewards()` and write `csv` file for each `TestCase` in format: `newCov,features,reward` for each `state` in it. `newCov` - flag that indicates whether this `TestCase` cover something new or not. So in this approach, each `state` will be written as many times as the number of `TestCase` that has it. +For creating `FeatureExtractor`, it uses `FeatureExtractorFactory` from `EngineAnalyticsContext`. # FeatureExtractor -It is interface in framework-module, that allow to use implementation from analytics module. -* `extractFeatures(state: ExecutionState)` - create features list for state and store it in `state.features`. Now we extract all features, which was described in [paper](https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf). In feature, we can extend feature list by other features, for example, NeuroSMT. +It is interface in framework-module, that allows to use implementation from analytics module. +* `extractFeatures(state: ExecutionState)` - create features list for state and store it in `state.features`. Now we extract all features, which were described in [paper](https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf). In feature, we can extend the feature list by other features, for example, NeuroSMT. # NNStateRewardPredictor Interface for reward predictors. Now it has two implementations in `analytics` module: -* `NNStateRewardPredictorSmile`: it uses our own format to store feedforward neural network and it uses `Smile` library to do multiplication of matrix. -* `NNStateRewardPredictorTorch`: it assumed that model is any type of model in `pt` format. It uses `Deep Java library` to use such models. +* `NNStateRewardPredictorSmile`: it uses our own format to store feedforward neural network, and it uses `Smile` library to do multiplication of matrix. +* `NNStateRewardPredictorTorch`: it assumed that a model is any type of model in `pt` format. It uses the `Deep Java library` to use such models. -It should be created at the beginning of work and stored at `Predictors` class to be used in `NNRewardGuidedSelector` from `framework` module. +It should be created at the beginning of work and stored at `Predictors` class to be used in `NNRewardGuidedSelector` from the `framework` module. # NNStateRewardGuidedSelector @@ -149,4 +149,4 @@ We have two different implementantions: # EngineAnalyticsContext -It is object that should be filled by factories in the beginning of work to allow objects from `framework` module using objects from `analytics` module. +It is an object that should be filled by factories in the beginning of work to allow objects from the `framework` module using objects from `analytics` module. diff --git a/docs/jlearch/new-heuristical-path-selectors.md b/docs/jlearch/new-heuristical-path-selectors.md index 948dae6159..756abb2c0e 100644 --- a/docs/jlearch/new-heuristical-path-selectors.md +++ b/docs/jlearch/new-heuristical-path-selectors.md @@ -30,7 +30,7 @@ classDiagram ``` `Subpath` = `List` -This class maintains frequency of each subpath with length `2^index`, which is presented as `List`, on certain instance of `InterproceduralUnitGraph` +This class maintains frequency of each subpath with length `2^index`, which is presented as `List`, in a certain instance of `InterproceduralUnitGraph` * `onVisit(state: ExecutionState)` - we calculate subpath of this state and increment its frequency on `1` * `subpathCount(state: ExecutionState)` - we calculate subpath of this state and return its frequency @@ -45,7 +45,7 @@ classDiagram Inspired by [paper](http://pxzhang.cn/paper/concolic_testing/oopsla13-pgse.pdf). -We override `ExecutionState.weight` as `-StatementStatistics.subpathCount(this)`, so we pick `state`, which `subpath` is less traveled. +We override `ExecutionState.weight` as `-StatementStatistics.subpathCount(this)`, so we pick the `state`, which `subpath` is less traveled. # StatementStatistics @@ -66,11 +66,11 @@ classDiagram TraverseGraphStatistics o-- InterProceduralUnitGraph ``` -This class maintains frequency of each `Stmt` and number of `Stmt`, that was visited in some `SootMethod`, on certain instance of `InterproceduralUnitGraph`. +This class maintains frequency of each `Stmt` and number of `Stmt`, that was visited in some `SootMethod`, on a certain instance of `InterproceduralUnitGraph`. -* `onVisit(state: ExecutionState)` - increment frequency of state's `stmt` on 1. If we visit this `stmt` for the first time, then increment number of `Stmt`, that we visit in current state's `method`, on 1. -* `statementCount(state: ExecutionState)` - get frequency of state's `stmt` -* `statementsInMethodCount(state: ExecutionState)` - get number of `stmt`, that was visited in current state's `method`. +* `onVisit(state: ExecutionState)` - increment frequency of state's `stmt` on 1. If we visit this `stmt` for the first time, then increment number of `Stmt`, that we visit in the current state's `method`, on 1. +* `statementCount(state: ExecutionState)` - get a frequency of state's `stmt` +* `statementsInMethodCount(state: ExecutionState)` - get number of `stmt`, that was visited in the current state's `method`. # CPInstSelector @@ -89,5 +89,5 @@ classDiagram ForkDepthSelector --|> NonUniformRandomSearch ``` -Override `ExecutionState.cost` as `ExecutionState.depth`, so we are more likely to explore the least deep `state` in terms of number of forks on its path. +Override `ExecutionState.cost` as `ExecutionState.depth`, so we are more likely to explore the least deep `state` in terms of the number of forks on its path. From 02b5c17ae4b8b08e18744c4a5e0764194073e8d6 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Thu, 9 Jun 2022 19:25:54 +0300 Subject: [PATCH 28/37] Add predictor --- .../src/main/kotlin/org/utbot/framework/UtSettings.kt | 4 ++-- .../src/main/kotlin/org/utbot/contest/ContestEstimator.kt | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 418b293e9a..1aec73cae4 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -309,12 +309,12 @@ object UtSettings { /** * Path to deserialized reward models */ - var rewardModelPath by getStringProperty("models/cf") + var rewardModelPath by getStringProperty("../models/0") /** * Number of model iterations that will be used during ContestEstimator */ - var iterations by getIntProperty(4) + var iterations by getIntProperty(1) /** * Path for state features dir diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index 54bf868fc7..532e33ea97 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -2,6 +2,7 @@ package org.utbot.contest import mu.KotlinLogging import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.analytics.Predictors import org.utbot.common.FileUtil import org.utbot.common.bracket import org.utbot.common.info @@ -21,6 +22,7 @@ import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.instrumentation.ConcreteExecutor +import org.utbot.predictors.NNStateRewardPredictorSmile import java.io.File import java.io.FileInputStream import java.net.URLClassLoader @@ -323,6 +325,7 @@ fun runEstimator( // Predictors.smt = UtBotTimePredictor() // Predictors.smtIncremental = UtBotTimePredictorIncremental() // Predictors.testName = StatementUniquenessPredictor() +// Predictors.stateRewardPredictor = NNStateRewardPredictorSmile() EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() From 97b0813153015341248ad257cf876abc44e4c73a Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 14:04:45 +0300 Subject: [PATCH 29/37] Fix issues --- docs/jlearch/features.md | 17 ++++++++++++++ gradle.properties | 2 +- .../utbot/features/FeatureExtractorImpl.kt | 5 ++++ .../FeatureProcessorWithStatesRepetition.kt | 23 +++++++------------ .../src/main/resources/jlearch_features | 13 +++++++++++ 5 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 docs/jlearch/features.md create mode 100644 utbot-analytics/src/main/resources/jlearch_features diff --git a/docs/jlearch/features.md b/docs/jlearch/features.md new file mode 100644 index 0000000000..1b3ca05a00 --- /dev/null +++ b/docs/jlearch/features.md @@ -0,0 +1,17 @@ +# Collecting features + +Now we collect 13 features, that will be described in original [paper](https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf), except constraint representation, but it can be extended. + +* `stack` - size of state’s current call stack. +* `successor` - number of successors of state’s current basic block. +* `testCase` - number of test cases generated so far +* `coverageByBranch` - number of instructions, which was covered first time on our last branch +* `coverageByPath` - number of instructions, which was covered first time on our path +* `depth` - number of forks already performed along state’s path. +* `cpicnt` - number of instructions visited in state's current function. +* `icnt` - number of times for which st ate’s current instruction has + been visited +* `covNew` - number of instructions executed by st ate since the last + time a new instruction is covered +* `subpath` - number of times for which st ate’s subpaths have been + visited. The length of the subpaths can be 1, 2, 4, or 8 respectively diff --git a/gradle.properties b/gradle.properties index 72477d3a33..749730f691 100644 --- a/gradle.properties +++ b/gradle.properties @@ -37,6 +37,6 @@ maven_plugin_api_version=3.8.5 maven_plugin_tools_version=3.6.4 javacpp_version=1.5.3 jsoup_version=1.7.2 -djl_api_version=0.16.0 +djl_api_version=0.17.0 pytorch_native_version=1.9.1 # soot also depends on asm, so there could be two different versions \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt index 5ea2c3038d..6786c9847a 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureExtractorImpl.kt @@ -9,9 +9,14 @@ import org.utbot.engine.selectors.strategies.SubpathStatistics /** * Implementation of feature extractor. * Extract features for state and stores it in features vector of this state. + * + * @param graph execution graph of current symbolic traverse */ class FeatureExtractorImpl(private val graph: InterProceduralUnitGraph) : FeatureExtractor { companion object { + /** + * Indexes for [SubpathStatistics], with which we want to collect our features + */ private val subpathGuidedSelectorIndexes = listOf(0, 1, 2, 3) } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt index 8c76325a4d..37fa404ef4 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt @@ -15,6 +15,9 @@ import kotlin.math.pow * Implementation of feature processor, in which we dump each test, so there will be several copies of each state. * Extract features for state when this state will be marked visited in graph. * Add test case, when last state of it will be traversed. + * + * @param graph execution graph of current symbolic traverse + * @param saveDir directory in which we will store features and rewards of [ExecutionState] */ class FeatureProcessorWithStatesRepetition( graph: InterProceduralUnitGraph, @@ -25,21 +28,11 @@ class FeatureProcessorWithStatesRepetition( } companion object { - private val featureKeys = arrayOf( - "stack", - "successor", - "testCase", - "coverageByBranch", - "coverageByPath", - "depth", - "cpicnt", - "icnt", - "covNew", - "subpath1", - "subpath2", - "subpath4", - "subpath8" - ) + private const val featureFile = "jlearch_features" + private val featureKeys = Companion::class.java.classLoader.getResourceAsStream(featureFile) + ?.bufferedReader().use { + it?.readText()?.split(System.lineSeparator()) ?: emptyList() + } } private var generatedTestCases = 0 diff --git a/utbot-analytics/src/main/resources/jlearch_features b/utbot-analytics/src/main/resources/jlearch_features new file mode 100644 index 0000000000..0a852928bf --- /dev/null +++ b/utbot-analytics/src/main/resources/jlearch_features @@ -0,0 +1,13 @@ +stack +successor +testCase +coverageByBranch +coverageByPath +depth +cpicnt +icnt +covNew +subpath1 +subpath2 +subpath4 +subpath8 \ No newline at end of file From d6d05933db02bba283f5ac095772719a93eb9f67 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 15:58:08 +0300 Subject: [PATCH 30/37] Add comments --- .../FeatureProcessorWithStatesRepetition.kt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt index 37fa404ef4..3515db7527 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetition.kt @@ -13,6 +13,7 @@ import kotlin.math.pow /** * Implementation of feature processor, in which we dump each test, so there will be several copies of each state. + * Goal is make weighted dataset, where more value for states, which generated more tests. * Extract features for state when this state will be marked visited in graph. * Add test case, when last state of it will be traversed. * @@ -139,11 +140,25 @@ internal class RewardEstimator { } companion object { + /** + * Threshold for time: executingTime less than that we don't distinct. We are not expiremented with changing it yet, + * now it is just minimal positive value distinct from 0. + */ private const val minTime = 1.0 + + /** + * Just degree of reward to make it smaller if it more than 1 and bigger if it less than 1. + */ private const val rewardDegree = 0.5 fun reward(coverage: Double, time: Double): Double = (coverage / maxOf(time, minTime)).pow(rewardDegree) } } +/** + * Class that represents test case. + * @param states pairs from stateHash and executingTime, created from each state of this test case + * @param newCoverage number of instructions, that was visited in first time by [states] + * @param testIndex number of test case, that was created before + */ data class TestCase(val states: List>, val newCoverage: Int, val testIndex: Int) From c042024bbc3c90109da9dc1cf006f609fed19e87 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 15:58:38 +0300 Subject: [PATCH 31/37] Handle exceptional situations --- .../org/utbot/predictors/LinearStateRewardPredictor.kt | 5 +++++ .../src/main/kotlin/org/utbot/predictors/util.kt | 6 +++++- .../utbot/predictors/LinearStateRewardPredictorTest.kt | 10 ++++++++++ .../src/test/resources/wrong_format_linear.txt | 1 + 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 utbot-analytics/src/test/resources/wrong_format_linear.txt diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index 1da24fd267..43ddfdb440 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -12,6 +12,11 @@ private const val DEFAULT_WEIGHT_PATH = "linear.txt" */ private fun loadWeights(path: String): Matrix { val weightsFile = File("${UtSettings.rewardModelPath}/${path}") + + if (!weightsFile.exists()) { + error("There is no file with weights with path: ${weightsFile.absolutePath}") + } + val weightsArray = weightsFile.readText().splitByCommaIntoDoubleArray() return Matrix(weightsArray) } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt index 039f6b5ddb..f0f0664642 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/util.kt @@ -1,4 +1,8 @@ package org.utbot.predictors fun String.splitByCommaIntoDoubleArray() = - split(',').map(String::toDouble).toDoubleArray() \ No newline at end of file + try { + split(',').map(String::toDouble).toDoubleArray() + } catch (e: NumberFormatException) { + error("Wrong format in $this, expect doubles separated by commas") + } \ No newline at end of file diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt index 360290a2fc..16e5b9e7ca 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -2,6 +2,7 @@ package org.utbot.predictors import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows import org.utbot.examples.withRewardModelPath class LinearStateRewardPredictorTest { @@ -18,4 +19,13 @@ class LinearStateRewardPredictorTest { assertEquals(listOf(6.0, 6.0), pred.predict(features)) } } + + @Test + fun wrongFormatTest() { + withRewardModelPath("src/test/resources") { + assertThrows { + LinearStateRewardPredictor("wrong_format_linear.txt") + } + } + } } \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/wrong_format_linear.txt b/utbot-analytics/src/test/resources/wrong_format_linear.txt new file mode 100644 index 0000000000..bf4d5a1b19 --- /dev/null +++ b/utbot-analytics/src/test/resources/wrong_format_linear.txt @@ -0,0 +1 @@ +1,asdasdsa,asdasd \ No newline at end of file From 088a5c93e2ca3fda99142bdab8789c67092ba778 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 16:13:30 +0300 Subject: [PATCH 32/37] Split methods and add comment --- .../org/utbot/predictors/NNStateRewardPredictor.kt | 3 +++ .../predictors/NNStateRewardPredictorSmile.kt | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt index 3b58d307c7..06a3ebe04d 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictor.kt @@ -2,4 +2,7 @@ package org.utbot.predictors import org.utbot.analytics.UtBotAbstractPredictor +/** + * Interface, which should predict reward for state by features list. + */ interface NNStateRewardPredictor : UtBotAbstractPredictor, Double> \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt index 660aa190eb..458453368a 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt @@ -39,13 +39,13 @@ data class NNJson( } } -data class NN(val operations: List<(DoubleArray) -> DoubleArray>) +data class NeuralNetwork(val operations: List<(DoubleArray) -> DoubleArray>) private fun reLU(input: DoubleArray): DoubleArray { return input.map { max(0.0, it) }.toDoubleArray() } -private fun loadNN(path: String): NN { +private fun loadModel(path: String): NNJson { val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() val nnJson: NNJson = Gson().fromJson(FileReader(modelFile), NNJson::class.java) ?: run { @@ -53,6 +53,10 @@ private fun loadNN(path: String): NN { NNJson() } + return nnJson +} + +private fun buildModel(nnJson: NNJson): NeuralNetwork { val weights = nnJson.linearLayers.map { Matrix(it) } val biases = nnJson.biases.map { Matrix(it) } val operations = mutableListOf<(DoubleArray) -> DoubleArray>() @@ -71,9 +75,11 @@ private fun loadNN(path: String): NN { } } - return NN(operations) + return NeuralNetwork(operations) } +private fun getModel(path: String) = buildModel(loadModel(path)) + data class StandardScaler(val mean: Matrix?, val variance: Matrix?) internal fun loadScaler(path: String): StandardScaler = @@ -85,7 +91,7 @@ internal fun loadScaler(path: String): StandardScaler = class NNStateRewardPredictorSmile(modelPath: String = DEFAULT_MODEL_PATH, scalerPath: String = DEFAULT_SCALER_PATH) : NNStateRewardPredictor { - private val nn = loadNN(modelPath) + private val nn = getModel(modelPath) private val scaler = loadScaler(scalerPath) override fun predict(input: List): Double { From 7258e33b8393e487ff3bf7379528f5d7b24c8a8f Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 16:40:25 +0300 Subject: [PATCH 33/37] Add parsing tests --- .../predictors/LinearStateRewardPredictor.kt | 1 + .../predictors/NNStateRewardPredictorSmile.kt | 14 ++++----- .../predictors/NNStateRewardPredictorTest.kt | 29 +++++++++++++++++++ .../src/test/resources/corrupted_nn.json | 25 ++++++++++++++++ .../src/test/resources/corrupted_scaler.txt | 1 + .../src/test/resources/empty_nn.json | 0 6 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 utbot-analytics/src/test/resources/corrupted_nn.json create mode 100644 utbot-analytics/src/test/resources/corrupted_scaler.txt create mode 100644 utbot-analytics/src/test/resources/empty_nn.json diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index 43ddfdb440..3e0df6a85d 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -18,6 +18,7 @@ private fun loadWeights(path: String): Matrix { } val weightsArray = weightsFile.readText().splitByCommaIntoDoubleArray() + return Matrix(weightsArray) } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt index 458453368a..29bb1c4ecf 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt @@ -1,7 +1,6 @@ package org.utbot.predictors import com.google.gson.Gson -import mu.KotlinLogging import org.utbot.framework.UtSettings import smile.math.matrix.Matrix import java.io.FileReader @@ -11,7 +10,9 @@ import kotlin.math.max private const val DEFAULT_MODEL_PATH = "nn.json" private const val DEFAULT_SCALER_PATH = "scaler.txt" -private val logger = KotlinLogging.logger {} +private object ActivationFunctions { + const val ReLU = "reLU" +} data class NNJson( val linearLayers: Array> = emptyArray(), @@ -49,8 +50,7 @@ private fun loadModel(path: String): NNJson { val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() val nnJson: NNJson = Gson().fromJson(FileReader(modelFile), NNJson::class.java) ?: run { - logger.debug { "Something went wrong while parsing NN model" } - NNJson() + error("Empty model") } return nnJson @@ -68,7 +68,7 @@ private fun buildModel(nnJson: NNJson): NeuralNetwork { if (i != nnJson.linearLayers.lastIndex) { operations.add { when (nnJson.activationLayers[i]) { - "reLU" -> reLU(it) + ActivationFunctions.ReLU -> reLU(it) else -> error("Unsupported activation") } } @@ -84,8 +84,8 @@ data class StandardScaler(val mean: Matrix?, val variance: Matrix?) internal fun loadScaler(path: String): StandardScaler = Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { - val mean = it.readLine()?.splitByCommaIntoDoubleArray() - val variance = it.readLine()?.splitByCommaIntoDoubleArray() + val mean = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not mean in $path") + val variance = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not variance in $path") StandardScaler(Matrix(mean), Matrix(variance)) } diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index abac473fec..0c93ecad83 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -1,8 +1,10 @@ package org.utbot.predictors +import com.google.gson.JsonSyntaxException import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows import org.utbot.examples.withRewardModelPath import kotlin.system.measureNanoTime @@ -49,4 +51,31 @@ class NNStateRewardPredictorTest { .map { measureNanoTime { pred.predict(features) } } .average() } + + @Test + fun corruptedModelFileTest() { + withRewardModelPath("src/test/resources") { + assertThrows { + NNStateRewardPredictorSmile(modelPath = "corrupted_nn.json") + } + } + } + + @Test + fun emptyModelFileTest() { + withRewardModelPath("src/test/resources") { + assertThrows { + NNStateRewardPredictorSmile(modelPath = "empty_nn.json") + } + } + } + + @Test + fun corruptedScalerTest() { + withRewardModelPath("src/test/resources") { + assertThrows { + NNStateRewardPredictorSmile(scalerPath = "corrupted_scaler.txt") + } + } + } } \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/corrupted_nn.json b/utbot-analytics/src/test/resources/corrupted_nn.json new file mode 100644 index 0000000000..ffc3ef7976 --- /dev/null +++ b/utbot-analytics/src/test/resources/corrupted_nn.json @@ -0,0 +1,25 @@ +{ + dsfds + "linearLayers": [ + [ + [1, 0], + [0, 1] + ], + [ + [1, 0], + [0, 1] + ], + [ + [1, 1] + ] + ], + "activationLayers": [ + "reLU", + "reLU" + ], + "biases": [ + [1, 1], + [1, 1], + [1] + ] +} \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/corrupted_scaler.txt b/utbot-analytics/src/test/resources/corrupted_scaler.txt new file mode 100644 index 0000000000..42ff95f905 --- /dev/null +++ b/utbot-analytics/src/test/resources/corrupted_scaler.txt @@ -0,0 +1 @@ +1,asada \ No newline at end of file diff --git a/utbot-analytics/src/test/resources/empty_nn.json b/utbot-analytics/src/test/resources/empty_nn.json new file mode 100644 index 0000000000..e69de29bb2 From 7d617928dde747ee77273592345202af81d5fe21 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 17:21:00 +0300 Subject: [PATCH 34/37] Fix issues --- .../kotlin/org/utbot/framework/UtSettings.kt | 19 ++++++++++++++----- .../utbot/analytics/EngineAnalyticsContext.kt | 4 ++-- .../org/utbot/analytics/FeatureExtractor.kt | 6 +++++- .../analytics/FeatureExtractorFactory.kt | 2 +- .../org/utbot/analytics/FeatureProcessor.kt | 3 +++ .../kotlin/org/utbot/analytics/Predictors.kt | 4 +--- .../org/utbot/engine/UtBotSymbolicEngine.kt | 1 - .../selectors/NNRewardGuidedSelector.kt | 11 +++++++---- .../engine/selectors/nurs/CPInstSelector.kt | 4 ++-- .../selectors/nurs/SubpathGuidedSelector.kt | 6 +++++- 10 files changed, 40 insertions(+), 20 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 1aec73cae4..439f76b60e 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -299,6 +299,10 @@ object UtSettings { * See [SubpathGuidedSelector] */ var subpathGuidedSelectorIndex by getIntProperty(1) + + /** + * Set of indexes, which will use [SubpathGuidedSelector] in not single mode + */ var subpathGuidedSelectorIndexes = listOf(0, 1, 2, 3) /** @@ -326,10 +330,6 @@ object UtSettings { */ var testCounter by getIntProperty(0) - var collectCoverage by getBooleanProperty(false) - - var coverageStatisticsDir by getStringProperty("logs/covStatistics") - /** * Flag for Subpath and NN selectors whether they are combined (Subpath use several indexes, NN use several models) */ @@ -348,7 +348,6 @@ enum class PathSelectorType { SUBPATH_GUIDED_SELECTOR, CPI_SELECTOR, FORK_DEPTH_SELECTOR, - LINEAR_REWARD_GUIDED_SELECTOR, NN_REWARD_GUIDED_SELECTOR, RANDOM_SELECTOR, RANDOM_PATH_SELECTOR @@ -359,7 +358,17 @@ enum class TestSelectionStrategyType { COVERAGE_STRATEGY // Adds new test only if it increases coverage } +/** + * Enum to specify [NNRewardGuidedSelector], see implementations for more details + */ enum class NNRewardGuidedSelectorType { + /** + * [NNRewardGuidedSelectorWithRecalculation] + */ WITH_RECALCULATION, + + /** + * [NNRewardGuidedSelectorWithoutRecalculation] + */ WITHOUT_RECALCULATION } diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt index bbce3162cc..0baf0c72cb 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -13,13 +13,13 @@ import org.utbot.framework.UtSettings object EngineAnalyticsContext { var featureProcessorFactory: FeatureProcessorFactory = object : FeatureProcessorFactory { override fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor { - error("Feature processor factory wasn't provided") + error("Feature processor factory is not provided.") } } var featureExtractorFactory: FeatureExtractorFactory = object : FeatureExtractorFactory { override fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor { - error("Feature extractor factory wasn't provided") + error("Feature extractor factory is not provided.") } } diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt index 6dab6d8775..c86cad1b54 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractor.kt @@ -3,8 +3,12 @@ package org.utbot.analytics import org.utbot.engine.ExecutionState /** - * Class that incapsulates work with FeatureExtractor during symbolic execution + * Class that encapsulates work with FeatureExtractor during symbolic execution. */ interface FeatureExtractor { + /** + * Extract features and store in it [ExecutionState.features] + * @param generatedTestCases number of generated tests so far + */ fun extractFeatures(executionState: ExecutionState, generatedTestCases: Int) } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt index 011f587177..6ad7a916a6 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureExtractorFactory.kt @@ -3,7 +3,7 @@ package org.utbot.analytics import org.utbot.engine.InterProceduralUnitGraph /** - * Class that can create FeatureExtractor. See [FeatureExtractor] + * Class that can create FeatureExtractor. See [FeatureExtractor]. */ interface FeatureExtractorFactory { operator fun invoke(graph: InterProceduralUnitGraph): FeatureExtractor diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt index 4a3874f8b4..e69e783a97 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/FeatureProcessor.kt @@ -7,5 +7,8 @@ import org.utbot.engine.selectors.strategies.TraverseGraphStatistics * Interface that encapsulates work with FeatureProcessor and can only dumpFeatures at the end of symbolic execution */ abstract class FeatureProcessor(graph: InterProceduralUnitGraph) : TraverseGraphStatistics(graph) { + /** + * Dump features and rewards of all states in collected test cases. + */ abstract fun dumpFeatures() } diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt index e0c7dc3632..562d6db7c3 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/Predictors.kt @@ -26,9 +26,7 @@ object Predictors { var stateRewardPredictor: UtBotAbstractPredictor, Double> = object : UtBotAbstractPredictor, Double> { override fun predict(input: List): Double { - error("stateRewardPredictor wasn't provided") + error("stateRewardPredictor is not provided.") } } - - var sat: IUtBotSatPredictor> = object : IUtBotSatPredictor> {} } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 7716005e3f..39f7bbc161 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -333,7 +333,6 @@ private fun pathSelector(graph: InterProceduralUnitGraph, typeRegistry: TypeRegi PathSelectorType.RANDOM_PATH_SELECTOR -> randomPathSelector(graph, StrategyOption.DISTANCE) { withStepsLimit(pathSelectorStepsLimit) } - else -> error("Unknown type") } class UtBotSymbolicEngine( diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt index fb0325be4f..010acceb47 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt @@ -10,11 +10,13 @@ import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics import org.utbot.engine.selectors.strategies.StoppingStrategy /** - * https://files.sri.inf.ethz.ch/website/papers/ccs21-learch.pdf + * @see Learch * * Calculates reward using neural network, when state is offered, and then peeks state with maximum reward * - * @see [GreedySearch] + * @see choosingStrategy [ChossingStrategy] for [GreedySearch] + * + * [GreedySearch] */ abstract class NNRewardGuidedSelector( protected val generatedTestCountingStatistics: GeneratedTestCountingStatistics, @@ -31,7 +33,7 @@ abstract class NNRewardGuidedSelector( /** * Calculate weight of execution state only when it is offered. It has advantage, because it works faster, - * but disadvantage is that some features of execution state can change. + * than with recalculation but disadvantage is that some features of execution state can change. */ class NNRewardGuidedSelectorWithoutRecalculationWeight( generatedTestCountingStatistics: GeneratedTestCountingStatistics, @@ -53,7 +55,8 @@ class NNRewardGuidedSelectorWithoutRecalculationWeight( } /** - * Calculate weight of execution state every time when it needed. It works slower, but features are always relevant + * Calculate weight of execution state every time when it needed. It works slower, + * than without recalculation but features are always relevant */ class NNRewardGuidedSelectorWithRecalculationWeight( generatedTestCountingStatistics: GeneratedTestCountingStatistics, diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt index 5eae91b111..261dfc2fc2 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/CPInstSelector.kt @@ -10,9 +10,9 @@ import org.utbot.engine.selectors.strategies.StoppingStrategy * where StatementStatistics.statementsInMethodCount(exState) * is number of visited instructions in the current function of execution state * - * https://github.com/klee/klee/blob/085c54b980a2f62c7c475d32b5d0ce9c6f97904f/lib/Core/Searcher.cpp#L207 + * @see @see Klee analog * - * @see [NonUniformRandomSearch] + * [NonUniformRandomSearch] */ class CPInstSelector( private val statementStatistics: StatementsStatistics, diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt index f871ad5702..a17e94a6a2 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/SubpathGuidedSelector.kt @@ -19,8 +19,12 @@ class SubpathGuidedSelector( ) : GreedySearch(choosingStrategy, stoppingStrategy, seed) { - override val name = "NURS:SubpathGuidedSearch" + override val name + get() = "NURS:SubpathGuidedSearch" + /** + * Use - to find minimum + */ override val ExecutionState.weight: Double get() = -subpathStatistics.subpathCount(this).toDouble() } \ No newline at end of file From dc03799edb113d918649a4f4be36b69e919b665a Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 19:20:17 +0300 Subject: [PATCH 35/37] Fix issues --- ...ureProcessorWithStatesRepetitionFactory.kt | 3 +- .../utbot/predictors/FeedForwardNetwork.kt | 36 ++++++ .../predictors/LinearStateRewardPredictor.kt | 17 ++- .../kotlin/org/utbot/predictors/NNJson.kt | 42 +++++++ .../predictors/NNStateRewardPredictorBase.kt | 43 +++++++ .../predictors/NNStateRewardPredictorSmile.kt | 108 ------------------ .../kotlin/org/utbot/predictors/scalers.kt | 14 +++ .../predictors/NNStateRewardPredictorTest.kt | 10 +- .../kotlin/org/utbot/framework/UtSettings.kt | 38 +++++- .../kotlin/org/utbot/engine/ExecutionState.kt | 8 +- .../org/utbot/engine/UtBotSymbolicEngine.kt | 47 +------- .../selectors/NNRewardGuidedSelector.kt | 4 +- .../NNRewardGuidedSelectorFactory.kt | 8 +- .../engine/selectors/nurs/GreedySearch.kt | 5 +- .../selectors/strategies/SubpathStatistics.kt | 3 + .../org/utbot/contest/ContestEstimator.kt | 2 - 16 files changed, 217 insertions(+), 171 deletions(-) create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/FeedForwardNetwork.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt delete mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt create mode 100644 utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt diff --git a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt index e8b479a7ae..51744e6918 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/features/FeatureProcessorWithStatesRepetitionFactory.kt @@ -5,7 +5,8 @@ import org.utbot.analytics.FeatureProcessorFactory import org.utbot.engine.InterProceduralUnitGraph /** - * Implementation of feature processor factory, which creates FeatureProcessorWithStatesRepetition + * Implementation of feature processor factory, which creates FeatureProcessorWithStatesRepetition. + * See [FeatureProcessorWithStatesRepetition]. */ class FeatureProcessorWithStatesRepetitionFactory : FeatureProcessorFactory { override fun invoke(graph: InterProceduralUnitGraph): FeatureProcessor = FeatureProcessorWithStatesRepetition(graph) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/FeedForwardNetwork.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/FeedForwardNetwork.kt new file mode 100644 index 0000000000..d392038bea --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/FeedForwardNetwork.kt @@ -0,0 +1,36 @@ +package org.utbot.predictors + +import smile.math.matrix.Matrix +import kotlin.math.max + +private object ActivationFunctions { + const val ReLU = "reLU" +} + +data class FeedForwardNetwork(val operations: List<(DoubleArray) -> DoubleArray>) + +private fun reLU(input: DoubleArray): DoubleArray { + return input.map { max(0.0, it) }.toDoubleArray() +} + +internal fun buildModel(nnJson: NNJson): FeedForwardNetwork { + val weights = nnJson.linearLayers.map { Matrix(it) } + val biases = nnJson.biases.map { Matrix(it) } + val operations = mutableListOf<(DoubleArray) -> DoubleArray>() + + nnJson.linearLayers.indices.forEach { i -> + operations.add { + weights[i].mm(Matrix(it)).add(biases[i]).col(0) + } + if (i != nnJson.linearLayers.lastIndex) { + operations.add { + when (nnJson.activationLayers[i]) { + ActivationFunctions.ReLU -> reLU(it) + else -> error("Unsupported activation") + } + } + } + } + + return FeedForwardNetwork(operations) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index 3e0df6a85d..e73bd8726f 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -1,12 +1,16 @@ package org.utbot.predictors +import mu.KotlinLogging import org.utbot.analytics.UtBotAbstractPredictor +import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import smile.math.matrix.Matrix import java.io.File private const val DEFAULT_WEIGHT_PATH = "linear.txt" +private val logger = KotlinLogging.logger {} + /** * Last weight is bias */ @@ -24,7 +28,18 @@ private fun loadWeights(path: String): Matrix { class LinearStateRewardPredictor(weightsPath: String = DEFAULT_WEIGHT_PATH) : UtBotAbstractPredictor>, List> { - private val weights = loadWeights(weightsPath) + private lateinit var weights: Matrix + + init { + try { + weights = loadWeights(weightsPath) + } catch (e: Exception) { + logger.error(e) { + "Error while initialization of LinearStateRewardPredictor. Changing pathSelectorType on INHERITORS_SELECTOR" + } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + } + } override fun predict(input: List>): List { // add 1 to each feature vector diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt new file mode 100644 index 0000000000..925bcd3751 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt @@ -0,0 +1,42 @@ +package org.utbot.predictors + +import com.google.gson.Gson +import org.utbot.framework.UtSettings +import java.io.FileReader +import java.nio.file.Paths + +data class NNJson( + val linearLayers: Array> = emptyArray(), + val activationLayers: Array = emptyArray(), + val biases: Array = emptyArray() +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as NNJson + + if (!linearLayers.contentDeepEquals(other.linearLayers)) return false + if (!activationLayers.contentEquals(other.activationLayers)) return false + if (!biases.contentDeepEquals(other.biases)) return false + + return true + } + + override fun hashCode(): Int { + var result = linearLayers.contentDeepHashCode() + result = 31 * result + activationLayers.contentHashCode() + result = 31 * result + biases.contentDeepHashCode() + return result + } +} + +internal fun loadModel(path: String): NNJson { + val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() + val nnJson: NNJson = + Gson().fromJson(FileReader(modelFile), NNJson::class.java) ?: run { + error("Empty model") + } + + return nnJson +} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt new file mode 100644 index 0000000000..017707c406 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt @@ -0,0 +1,43 @@ +package org.utbot.predictors + +import mu.KotlinLogging +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings +import smile.math.matrix.Matrix + +private const val DEFAULT_MODEL_PATH = "nn.json" +private const val DEFAULT_SCALER_PATH = "scaler.txt" + +private val logger = KotlinLogging.logger {} + +private fun getModel(path: String) = buildModel(loadModel(path)) + +class NNStateRewardPredictorBase(modelPath: String = DEFAULT_MODEL_PATH, scalerPath: String = DEFAULT_SCALER_PATH) : + NNStateRewardPredictor { + private lateinit var nn: FeedForwardNetwork + private lateinit var scaler: StandardScaler + + init { + try { + nn = getModel(modelPath) + scaler = loadScaler(scalerPath) + } catch (e: Exception) { + logger.error(e) { + "Error while initialization of NNStateRewardPredictorBase. Changing pathSelectorType on INHERITORS_SELECTOR" + } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + } + } + + override fun predict(input: List): Double { + var inputArray = input.toDoubleArray() + inputArray = Matrix(inputArray).sub(scaler.mean).div(scaler.variance).col(0) + + nn.operations.forEach { + inputArray = it(inputArray) + } + + check(inputArray.size == 1) { "Neural network have several outputs" } + return inputArray[0] + } +} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt deleted file mode 100644 index 29bb1c4ecf..0000000000 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorSmile.kt +++ /dev/null @@ -1,108 +0,0 @@ -package org.utbot.predictors - -import com.google.gson.Gson -import org.utbot.framework.UtSettings -import smile.math.matrix.Matrix -import java.io.FileReader -import java.nio.file.Paths -import kotlin.math.max - -private const val DEFAULT_MODEL_PATH = "nn.json" -private const val DEFAULT_SCALER_PATH = "scaler.txt" - -private object ActivationFunctions { - const val ReLU = "reLU" -} - -data class NNJson( - val linearLayers: Array> = emptyArray(), - val activationLayers: Array = emptyArray(), - val biases: Array = emptyArray() -) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false - - other as NNJson - - if (!linearLayers.contentDeepEquals(other.linearLayers)) return false - if (!activationLayers.contentEquals(other.activationLayers)) return false - if (!biases.contentDeepEquals(other.biases)) return false - - return true - } - - override fun hashCode(): Int { - var result = linearLayers.contentDeepHashCode() - result = 31 * result + activationLayers.contentHashCode() - result = 31 * result + biases.contentDeepHashCode() - return result - } -} - -data class NeuralNetwork(val operations: List<(DoubleArray) -> DoubleArray>) - -private fun reLU(input: DoubleArray): DoubleArray { - return input.map { max(0.0, it) }.toDoubleArray() -} - -private fun loadModel(path: String): NNJson { - val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() - val nnJson: NNJson = - Gson().fromJson(FileReader(modelFile), NNJson::class.java) ?: run { - error("Empty model") - } - - return nnJson -} - -private fun buildModel(nnJson: NNJson): NeuralNetwork { - val weights = nnJson.linearLayers.map { Matrix(it) } - val biases = nnJson.biases.map { Matrix(it) } - val operations = mutableListOf<(DoubleArray) -> DoubleArray>() - - nnJson.linearLayers.indices.forEach { i -> - operations.add { - weights[i].mm(Matrix(it)).add(biases[i]).col(0) - } - if (i != nnJson.linearLayers.lastIndex) { - operations.add { - when (nnJson.activationLayers[i]) { - ActivationFunctions.ReLU -> reLU(it) - else -> error("Unsupported activation") - } - } - } - } - - return NeuralNetwork(operations) -} - -private fun getModel(path: String) = buildModel(loadModel(path)) - -data class StandardScaler(val mean: Matrix?, val variance: Matrix?) - -internal fun loadScaler(path: String): StandardScaler = - Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { - val mean = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not mean in $path") - val variance = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not variance in $path") - StandardScaler(Matrix(mean), Matrix(variance)) - } - -class NNStateRewardPredictorSmile(modelPath: String = DEFAULT_MODEL_PATH, scalerPath: String = DEFAULT_SCALER_PATH) : - NNStateRewardPredictor { - private val nn = getModel(modelPath) - private val scaler = loadScaler(scalerPath) - - override fun predict(input: List): Double { - var inputArray = input.toDoubleArray() - inputArray = Matrix(inputArray).sub(scaler.mean).div(scaler.variance).col(0) - - nn.operations.forEach { - inputArray = it(inputArray) - } - - check(inputArray.size == 1) { "Neural network have several outputs" } - return inputArray[0] - } -} diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt new file mode 100644 index 0000000000..c50ffdc7c1 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt @@ -0,0 +1,14 @@ +package org.utbot.predictors + +import org.utbot.framework.UtSettings +import smile.math.matrix.Matrix +import java.nio.file.Paths + +data class StandardScaler(val mean: Matrix?, val variance: Matrix?) + +internal fun loadScaler(path: String): StandardScaler = + Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { + val mean = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not mean in $path") + val variance = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not variance in $path") + StandardScaler(Matrix(mean), Matrix(variance)) + } diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index 0c93ecad83..047143790a 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -12,7 +12,7 @@ class NNStateRewardPredictorTest { @Test fun simpleTest() { withRewardModelPath("src/test/resources") { - val pred = NNStateRewardPredictorSmile() + val pred = NNStateRewardPredictorBase() val features = listOf(0.0, 0.0) @@ -25,7 +25,7 @@ class NNStateRewardPredictorTest { fun performanceTest() { val features = (1..13).map { 1.0 }.toList() withRewardModelPath("models\\test\\0") { - val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorSmile, 100, features) + val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorBase, 100, features) println(averageTime) } @@ -56,7 +56,7 @@ class NNStateRewardPredictorTest { fun corruptedModelFileTest() { withRewardModelPath("src/test/resources") { assertThrows { - NNStateRewardPredictorSmile(modelPath = "corrupted_nn.json") + NNStateRewardPredictorBase(modelPath = "corrupted_nn.json") } } } @@ -65,7 +65,7 @@ class NNStateRewardPredictorTest { fun emptyModelFileTest() { withRewardModelPath("src/test/resources") { assertThrows { - NNStateRewardPredictorSmile(modelPath = "empty_nn.json") + NNStateRewardPredictorBase(modelPath = "empty_nn.json") } } } @@ -74,7 +74,7 @@ class NNStateRewardPredictorTest { fun corruptedScalerTest() { withRewardModelPath("src/test/resources") { assertThrows { - NNStateRewardPredictorSmile(scalerPath = "corrupted_scaler.txt") + NNStateRewardPredictorBase(scalerPath = "corrupted_scaler.txt") } } } diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 439f76b60e..76d1ed0c97 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -306,9 +306,9 @@ object UtSettings { var subpathGuidedSelectorIndexes = listOf(0, 1, 2, 3) /** - * Enable feature processing for executionStates + * Flag that indicates whether feature processing for execution states enabled or not */ - var featureProcess by getBooleanProperty(false) + var enableFeatureProcess by getBooleanProperty(false) /** * Path to deserialized reward models @@ -342,14 +342,48 @@ object UtSettings { .joinToString(separator = System.lineSeparator()) { "\t${it.key}=${it.value}" } } +/** + * Type of [BasePathSelector]. For each value see class in comment + */ enum class PathSelectorType { + /** + * [CoveredNewSelector] + */ COVERED_NEW_SELECTOR, + + /** + * [InheritorsSelector] + */ INHERITORS_SELECTOR, + + /** + * [SubpathGuidedSelector] + */ SUBPATH_GUIDED_SELECTOR, + + /** + * [CPInstSelector] + */ CPI_SELECTOR, + + /** + * [ForkDepthSelector] + */ FORK_DEPTH_SELECTOR, + + /** + * [NNRewardGuidedSelector] + */ NN_REWARD_GUIDED_SELECTOR, + + /** + * [RandomSelector] + */ RANDOM_SELECTOR, + + /** + * [RandomPathSelector] + */ RANDOM_PATH_SELECTOR } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt index f413ff0456..7e034050e7 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/ExecutionState.kt @@ -64,7 +64,7 @@ data class StateAnalyticsProperties( */ private var isFork: Boolean = false - fun updateIsFork() { + fun definitelyFork() { isFork = true } @@ -87,7 +87,7 @@ data class StateAnalyticsProperties( successorVisitedAfterLastFork, successorVisitedBeforeLastFork, successorStmtSinceLastCovered, - if (UtSettings.featureProcess) parent else null + if (UtSettings.enableFeatureProcess) parent else null ) } @@ -331,8 +331,8 @@ data class ExecutionState( return " MD5(path)=${prettifiedPath.md5()}\n$prettifiedPath" } - fun updateIsFork() { - stateAnalyticsProperties.updateIsFork() + fun definitelyFork() { + stateAnalyticsProperties.definitelyFork() } fun updateIsVisitedNew() { diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 39f7bbc161..9d203ca991 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -9,7 +9,6 @@ import kotlinx.collections.immutable.toPersistentSet import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Job -import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.FlowCollector import kotlinx.coroutines.flow.flow @@ -111,7 +110,7 @@ import org.utbot.engine.util.statics.concrete.makeSymbolicValuesFromEnumConcrete import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import org.utbot.framework.UtSettings.checkSolverTimeoutMillis -import org.utbot.framework.UtSettings.featureProcess +import org.utbot.framework.UtSettings.enableFeatureProcess import org.utbot.framework.UtSettings.pathSelectorStepsLimit import org.utbot.framework.UtSettings.pathSelectorType import org.utbot.framework.UtSettings.preferredCexOption @@ -155,9 +154,6 @@ import org.utbot.fuzzer.defaultModelProviders import org.utbot.fuzzer.fuzz import org.utbot.instrumentation.ConcreteExecutor import java.lang.reflect.ParameterizedType -import java.util.BitSet -import java.util.IdentityHashMap -import java.util.TreeSet import kotlin.collections.plus import kotlin.collections.plusAssign import kotlin.math.max @@ -166,26 +162,6 @@ import kotlin.reflect.full.instanceParameter import kotlin.reflect.full.valueParameters import kotlin.reflect.jvm.javaType import kotlin.system.measureTimeMillis -import kotlinx.collections.immutable.persistentHashMapOf -import kotlinx.collections.immutable.persistentListOf -import kotlinx.collections.immutable.persistentSetOf -import kotlinx.collections.immutable.toPersistentList -import kotlinx.collections.immutable.toPersistentSet -import kotlinx.coroutines.currentCoroutineContext -import kotlinx.coroutines.flow.flow -import kotlinx.coroutines.flow.onCompletion -import kotlinx.coroutines.flow.onStart -import kotlinx.coroutines.isActive -import kotlinx.coroutines.yield -import org.utbot.engine.selectors.coveredNewSelector -import org.utbot.engine.selectors.cpInstSelector -import org.utbot.engine.selectors.forkDepthSelector -import org.utbot.engine.selectors.inheritorsSelector -import org.utbot.engine.selectors.nnRewardGuidedSelector -import org.utbot.engine.selectors.pollUntilFastSAT -import org.utbot.engine.selectors.randomPathSelector -import org.utbot.engine.selectors.randomSelector -import org.utbot.engine.selectors.subpathGuidedSelector import soot.ArrayType import soot.BooleanType import soot.ByteType @@ -276,17 +252,6 @@ import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl import sun.reflect.generics.reflectiveObjects.TypeVariableImpl import sun.reflect.generics.reflectiveObjects.WildcardTypeImpl import java.lang.reflect.Method -import kotlin.collections.plus -import kotlin.collections.plusAssign -import kotlin.math.max -import kotlin.math.min -import kotlin.reflect.full.instanceParameter -import kotlin.reflect.full.valueParameters -import kotlin.reflect.jvm.javaType -import kotlin.reflect.full.instanceParameter -import kotlin.reflect.full.valueParameters -import kotlin.reflect.jvm.javaType -import kotlin.system.measureTimeMillis private val logger = KotlinLogging.logger {} val pathLogger = KotlinLogging.logger(logger.name + ".path") @@ -398,7 +363,7 @@ class UtBotSymbolicEngine( private var queuedSymbolicStateUpdates = SymbolicStateUpdate() private val featureProcessor: FeatureProcessor? = - if (featureProcess) EngineAnalyticsContext.featureProcessorFactory(globalGraph) else null + if (enableFeatureProcess) EngineAnalyticsContext.featureProcessorFactory(globalGraph) else null private val insideStaticInitializer get() = environment.state.executionStack.any { it.method.isStaticInitializer } @@ -1403,7 +1368,7 @@ class UtBotSymbolicEngine( val negativeCasePathConstraint = mkNot(positiveCasePathConstraint) if (positiveCaseEdge != null) { - environment.state.updateIsFork() + environment.state.definitelyFork() } positiveCaseEdge?.let { edge -> @@ -1475,7 +1440,7 @@ class UtBotSymbolicEngine( } if (successors.size > 1) { environment.state.expectUndefined() - environment.state.updateIsFork() + environment.state.definitelyFork() } successors.forEach { (target, expr) -> @@ -2686,7 +2651,7 @@ class UtBotSymbolicEngine( // If so, return the result of the override if (artificialMethodOverride.success) { if (artificialMethodOverride.results.size > 1) { - environment.state.updateIsFork() + environment.state.definitelyFork() } return mutableListOf().apply { @@ -2717,7 +2682,7 @@ class UtBotSymbolicEngine( val overrideResults = targets.map { it to overrideInvocation(invocation, it) } if (overrideResults.sumOf { (_, overriddenResult) -> overriddenResult.results.size } > 1) { - environment.state.updateIsFork() + environment.state.definitelyFork() } // Separate targets for which invocation should be overridden diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt index 010acceb47..c4f37c2924 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt @@ -35,7 +35,7 @@ abstract class NNRewardGuidedSelector( * Calculate weight of execution state only when it is offered. It has advantage, because it works faster, * than with recalculation but disadvantage is that some features of execution state can change. */ -class NNRewardGuidedSelectorWithoutRecalculationWeight( +class NNRewardGuidedSelectorWithoutWeightsRecalculation( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, @@ -58,7 +58,7 @@ class NNRewardGuidedSelectorWithoutRecalculationWeight( * Calculate weight of execution state every time when it needed. It works slower, * than without recalculation but features are always relevant */ -class NNRewardGuidedSelectorWithRecalculationWeight( +class NNRewardGuidedSelectorWithWeightsRecalculation( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt index c426064ade..cad9632381 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt @@ -19,7 +19,7 @@ interface NNRewardGuidedSelectorFactory { } /** - * Creates [NNRewardGuidedSelectorWithRecalculationWeight] + * Creates [NNRewardGuidedSelectorWithWeightsRecalculation] */ class NNRewardGuidedSelectorWithRecalculationFactory : NNRewardGuidedSelectorFactory { override fun invoke( @@ -28,13 +28,13 @@ class NNRewardGuidedSelectorWithRecalculationFactory : NNRewardGuidedSelectorFac stoppingStrategy: StoppingStrategy, seed: Int, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithRecalculationWeight( + ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithWeightsRecalculation( generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph ) } /** - * Creates [NNRewardGuidedSelectorWithoutRecalculationWeight] + * Creates [NNRewardGuidedSelectorWithoutWeightsRecalculation] */ class NNRewardGuidedSelectorWithoutRecalculationFactory : NNRewardGuidedSelectorFactory { override fun invoke( @@ -43,7 +43,7 @@ class NNRewardGuidedSelectorWithoutRecalculationFactory : NNRewardGuidedSelector stoppingStrategy: StoppingStrategy, seed: Int, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithoutRecalculationWeight( + ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithoutWeightsRecalculation( generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph ) } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt index b2a587217d..b00d67e871 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/nurs/GreedySearch.kt @@ -15,7 +15,7 @@ import kotlin.random.Random abstract class GreedySearch( choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, - seed: Int = 0 + seed: Int = 42 ) : BasePathSelector(choosingStrategy, stoppingStrategy), StrategyObserver { private val states = mutableSetOf() @@ -31,6 +31,9 @@ abstract class GreedySearch( override fun pollImpl(): ExecutionState? = peekImpl()?.also { remove(it) } + /** + * Creates set of states with maximum weight and then pick random + */ override fun peekImpl(): ExecutionState? { if (isEmpty()) { return null diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt index 1422c2e784..0110cf03c9 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/strategies/SubpathStatistics.kt @@ -17,6 +17,9 @@ class SubpathStatistics( private val subpathCount = mutableMapOf, Int>() private val length: Int = 2.0.pow(index).toInt() + /** + * Take length last edges from state's path and handle exception edges + */ private fun ExecutionState.getSubpath(length: Int): List { val subpath = mutableListOf() val actualLength = if (pathLength >= length) length else pathLength diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index 532e33ea97..354983f897 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -2,7 +2,6 @@ package org.utbot.contest import mu.KotlinLogging import org.utbot.analytics.EngineAnalyticsContext -import org.utbot.analytics.Predictors import org.utbot.common.FileUtil import org.utbot.common.bracket import org.utbot.common.info @@ -22,7 +21,6 @@ import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.instrumentation.ConcreteExecutor -import org.utbot.predictors.NNStateRewardPredictorSmile import java.io.File import java.io.FileInputStream import java.net.URLClassLoader From bec49b0e2dcaf5849ea571e6b23ec18cadfcda68 Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Fri, 10 Jun 2022 19:39:14 +0300 Subject: [PATCH 36/37] Change predictor in ContestEstimator --- .../src/main/kotlin/org/utbot/contest/ContestEstimator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index 354983f897..85ff761b17 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -323,7 +323,7 @@ fun runEstimator( // Predictors.smt = UtBotTimePredictor() // Predictors.smtIncremental = UtBotTimePredictorIncremental() // Predictors.testName = StatementUniquenessPredictor() -// Predictors.stateRewardPredictor = NNStateRewardPredictorSmile() +// Predictors.stateRewardPredictor = NNStateRewardPredictorBase() EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() From 93f93ec1137320aab6781e8ce0ceb8375e96e00b Mon Sep 17 00:00:00 2001 From: Victor Samoilov Date: Wed, 15 Jun 2022 11:34:11 +0300 Subject: [PATCH 37/37] Fix tests --- .../utbot/predictors/LinearStateRewardPredictor.kt | 2 +- .../utbot/predictors/NNStateRewardPredictorBase.kt | 2 +- .../predictors/LinearStateRewardPredictorTest.kt | 7 +++++-- .../utbot/predictors/NNStateRewardPredictorTest.kt | 14 +++++++++----- .../examples/AbstractTestCaseGeneratorTest.kt | 11 +++++++++++ 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index e73bd8726f..9898cadd63 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -34,7 +34,7 @@ class LinearStateRewardPredictor(weightsPath: String = DEFAULT_WEIGHT_PATH) : try { weights = loadWeights(weightsPath) } catch (e: Exception) { - logger.error(e) { + logger.info(e) { "Error while initialization of LinearStateRewardPredictor. Changing pathSelectorType on INHERITORS_SELECTOR" } UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt index 017707c406..1d9d5dac6d 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNStateRewardPredictorBase.kt @@ -22,7 +22,7 @@ class NNStateRewardPredictorBase(modelPath: String = DEFAULT_MODEL_PATH, scalerP nn = getModel(modelPath) scaler = loadScaler(scalerPath) } catch (e: Exception) { - logger.error(e) { + logger.info(e) { "Error while initialization of NNStateRewardPredictorBase. Changing pathSelectorType on INHERITORS_SELECTOR" } UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt index 16e5b9e7ca..1de5c0cb75 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -2,8 +2,10 @@ package org.utbot.predictors import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows +import org.utbot.examples.withPathSelectorType import org.utbot.examples.withRewardModelPath +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings class LinearStateRewardPredictorTest { @Test @@ -23,8 +25,9 @@ class LinearStateRewardPredictorTest { @Test fun wrongFormatTest() { withRewardModelPath("src/test/resources") { - assertThrows { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { LinearStateRewardPredictor("wrong_format_linear.txt") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } } } diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index 047143790a..510660d7ca 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -1,11 +1,12 @@ package org.utbot.predictors -import com.google.gson.JsonSyntaxException import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows +import org.utbot.examples.withPathSelectorType import org.utbot.examples.withRewardModelPath +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings import kotlin.system.measureNanoTime class NNStateRewardPredictorTest { @@ -55,8 +56,9 @@ class NNStateRewardPredictorTest { @Test fun corruptedModelFileTest() { withRewardModelPath("src/test/resources") { - assertThrows { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { NNStateRewardPredictorBase(modelPath = "corrupted_nn.json") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } } } @@ -64,8 +66,9 @@ class NNStateRewardPredictorTest { @Test fun emptyModelFileTest() { withRewardModelPath("src/test/resources") { - assertThrows { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { NNStateRewardPredictorBase(modelPath = "empty_nn.json") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } } } @@ -73,8 +76,9 @@ class NNStateRewardPredictorTest { @Test fun corruptedScalerTest() { withRewardModelPath("src/test/resources") { - assertThrows { + withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { NNStateRewardPredictorBase(scalerPath = "corrupted_scaler.txt") + assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } } } diff --git a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt index 1ba649a847..e2318ae026 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/examples/AbstractTestCaseGeneratorTest.kt @@ -67,6 +67,7 @@ import kotlin.reflect.KFunction4 import kotlin.reflect.KFunction5 import mu.KotlinLogging import org.junit.jupiter.api.Assertions.assertTrue +import org.utbot.framework.PathSelectorType val logger = KotlinLogging.logger {} @@ -2789,3 +2790,13 @@ inline fun withRewardModelPath(rewardModelPath: String, block: () -> UtSettings.rewardModelPath = prev } } + +inline fun withPathSelectorType(pathSelectorType: PathSelectorType, block: () -> T): T { + val prev = UtSettings.pathSelectorType + UtSettings.pathSelectorType = pathSelectorType + try { + return block() + } finally { + UtSettings.pathSelectorType = prev + } +}