Skip to content

Commit 2c25ef1

Browse files
author
Rustam Sadykov
committed
add coverage statistics by source
1 parent 0819e9c commit 2c25ef1

5 files changed

Lines changed: 113 additions & 79 deletions

File tree

‎docs/NightStatisticsMonitoring.md‎

Lines changed: 49 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -19,53 +19,43 @@ More about statistic: Statistics.kt.
1919

2020
Example input:
2121
```
22-
stats.json 3 20
22+
stats.json 2 20
2323
```
2424
Example output:
2525
```json
2626
[
27-
{
28-
"classes_for_generation": 20,
29-
"testcases_generated": 1204,
30-
"classes_without_problems": 12,
31-
"classes_canceled_by_timeout": 3,
32-
"total_methods_for_generation": 519,
33-
"methods_with_at_least_one_testcase_generated": 332,
34-
"methods_with_exceptions": 42,
35-
"suspicious_methods": 107,
36-
"test_classes_failed_to_compile": 0,
37-
"covered_instructions_count": 5282,
38-
"total_instructions_count": 10932,
39-
"avg_coverage": 57.43687433585721
40-
},
41-
{
42-
"classes_for_generation": 20,
43-
"testcases_generated": 1692,
44-
"classes_without_problems": 12,
45-
"classes_canceled_by_timeout": 2,
46-
"total_methods_for_generation": 519,
47-
"methods_with_at_least_one_testcase_generated": 426,
48-
"methods_with_exceptions": 49,
49-
"suspicious_methods": 29,
50-
"test_classes_failed_to_compile": 1,
51-
"covered_instructions_count": 6499,
52-
"total_instructions_count": 11023,
53-
"avg_coverage": 66.33821560285908
54-
},
55-
{
56-
"classes_for_generation": 20,
57-
"testcases_generated": 1406,
58-
"classes_without_problems": 12,
59-
"classes_canceled_by_timeout": 3,
60-
"total_methods_for_generation": 519,
61-
"methods_with_at_least_one_testcase_generated": 394,
62-
"methods_with_exceptions": 43,
63-
"suspicious_methods": 61,
64-
"test_classes_failed_to_compile": 0,
65-
"covered_instructions_count": 5851,
66-
"total_instructions_count": 11011,
67-
"avg_coverage": 60.71679400185094
68-
}
27+
{
28+
"classes_for_generation": 20,
29+
"testcases_generated": 958,
30+
"classes_without_problems": 12,
31+
"classes_canceled_by_timeout": 3,
32+
"total_methods_for_generation": 519,
33+
"methods_with_at_least_one_testcase_generated": 314,
34+
"methods_with_exceptions": 47,
35+
"suspicious_methods": 63,
36+
"test_classes_failed_to_compile": 1,
37+
"covered_instructions_count": 4388,
38+
"covered_instructions_count_by_fuzzing": 3651,
39+
"covered_instructions_count_by_concolic": 2178,
40+
"total_instructions_count": 9531,
41+
"avg_coverage": 60.10571074242921
42+
},
43+
{
44+
"classes_for_generation": 9,
45+
"testcases_generated": 557,
46+
"classes_without_problems": 5,
47+
"classes_canceled_by_timeout": 2,
48+
"total_methods_for_generation": 114,
49+
"methods_with_at_least_one_testcase_generated": 109,
50+
"methods_with_exceptions": 11,
51+
"suspicious_methods": 1,
52+
"test_classes_failed_to_compile": 0,
53+
"covered_instructions_count": 1675,
54+
"covered_instructions_count_by_fuzzing": 1276,
55+
"covered_instructions_count_by_concolic": 1332,
56+
"total_instructions_count": 2407,
57+
"avg_coverage": 70.9287503305422
58+
}
6959
]
7060
```
7161

@@ -87,20 +77,22 @@ monitoring/history.json stats.json monitoring/coverage_graph.png monitoring/quan
8777
Example output:
8878
```json
8979
[
90-
{
91-
"classes_for_generation": 20.0,
92-
"testcases_generated": 1434.0,
93-
"classes_without_problems": 12.0,
94-
"classes_canceled_by_timeout": 2.6666666666666665,
95-
"total_methods_for_generation": 519.0,
96-
"methods_with_at_least_one_testcase_generated": 384.0,
97-
"methods_with_exceptions": 44.666666666666664,
98-
"suspicious_methods": 65.66666666666667,
99-
"test_classes_failed_to_compile": 0.33333333333333337,
100-
"avg_coverage": 61.49729464685574,
101-
"total_coverage": 53.47106015409328,
102-
"timestamp": 1659715928504
103-
}
80+
{
81+
"classes_for_generation": 14.5,
82+
"testcases_generated": 757.5,
83+
"classes_without_problems": 8.5,
84+
"classes_canceled_by_timeout": 2.5,
85+
"total_methods_for_generation": 316.5,
86+
"methods_with_at_least_one_testcase_generated": 211.5,
87+
"methods_with_exceptions": 29.0,
88+
"suspicious_methods": 32.0,
89+
"test_classes_failed_to_compile": 0.5,
90+
"avg_coverage": 65.5172305364857,
91+
"total_coverage": 57.813969999804286,
92+
"total_coverage_by_fuzzing": 45.65931336298925,
93+
"total_coverage_by_concolic": 39.095171346713414,
94+
"timestamp": 1660132400587
95+
}
10496
]
10597
```
10698

‎monitoring/draw_stats_graphs.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,16 @@ def load(json_file):
2222

2323

2424
def transform_stats(stats):
25-
num = stats["covered_instructions_count"]
25+
common_prefix = "covered_instructions_count"
2626
denum = stats["total_instructions_count"]
27-
stats["total_coverage"] = 100 * num / denum if denum != 0 else 0
28-
del stats["covered_instructions_count"]
27+
28+
nums_keys = [(key, key.removeprefix(common_prefix)) for key in stats.keys() if key.startswith(common_prefix)]
29+
30+
for (key, by) in nums_keys:
31+
num = stats[key]
32+
stats["total_coverage" + by] = 100 * num / denum if denum != 0 else 0
33+
del stats[key]
34+
2935
del stats["total_instructions_count"]
3036

3137
return stats

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import org.utbot.framework.plugin.api.util.jClass
5050
import org.utbot.framework.plugin.api.util.utContext
5151
import org.utbot.framework.plugin.api.util.withUtContext
5252
import org.utbot.framework.util.isKnownSyntheticMethod
53+
import org.utbot.fuzzer.UtFuzzedExecution
5354
import org.utbot.instrumentation.ConcreteExecutor
5455
import org.utbot.instrumentation.ConcreteExecutorPool
5556
import org.utbot.instrumentation.Settings
@@ -326,7 +327,8 @@ fun runGeneration(
326327
result.coverage?.let {
327328
statsForClass.updateCoverage(
328329
newCoverage = it,
329-
isNewClass = !testedClasses.contains(className)
330+
isNewClass = !testedClasses.contains(className),
331+
fromFuzzing = result is UtFuzzedExecution
330332
)
331333
}
332334
testedClasses.add(className)
@@ -480,8 +482,23 @@ internal val Method.isVisibleFromGeneratedTest: Boolean
480482
get() = (this.modifiers and Modifier.ABSTRACT) == 0
481483
&& (this.modifiers and Modifier.NATIVE) == 0
482484

483-
private fun StatsForClass.updateCoverage(newCoverage: Coverage, isNewClass: Boolean) {
484-
coverage = coverage?.let { oldCoverage ->
485+
private fun StatsForClass.updateCoverage(newCoverage: Coverage, isNewClass: Boolean, fromFuzzing: Boolean) {
486+
coverage = coverage.update(newCoverage, isNewClass)
487+
// other coverage type updates by empty coverage to respect new class
488+
val emptyCoverage = newCoverage.copy(
489+
coveredInstructions = emptyList()
490+
)
491+
if (fromFuzzing) {
492+
fuzzedCoverage = fuzzedCoverage.update(newCoverage, isNewClass)
493+
concolicCoverage = concolicCoverage.update(emptyCoverage, isNewClass)
494+
} else {
495+
fuzzedCoverage = fuzzedCoverage.update(emptyCoverage, isNewClass)
496+
concolicCoverage = concolicCoverage.update(newCoverage, isNewClass)
497+
}
498+
}
499+
500+
private fun Coverage?.update(newCoverage: Coverage, isNewClass: Boolean) =
501+
this?.let { oldCoverage ->
485502
val instructionsCount = if (isNewClass && newCoverage.instructionsCount != null) {
486503
newCoverage.instructionsCount!! + (oldCoverage.instructionsCount ?: 0)
487504
} else {
@@ -492,4 +509,3 @@ private fun StatsForClass.updateCoverage(newCoverage: Coverage, isNewClass: Bool
492509
instructionsCount = instructionsCount
493510
)
494511
} ?: newCoverage
495-
}

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,22 @@ class GlobalStats {
4646
get() = statsForClasses.count { it.failedToCompile }
4747

4848
val coveredInstructionsCount: Int
49-
get() = statsForClasses.sumBy { it.getCoverageInfo().first }
49+
get() = statsForClasses.sumBy { it.coverage.getCoverageInfo(it.className).first }
50+
51+
val coveredInstructionsCountByFuzzing: Int
52+
get() = statsForClasses.sumBy { it.fuzzedCoverage.getCoverageInfo(it.className).first }
53+
54+
val coveredInstructionsCountByConcolic: Int
55+
get() = statsForClasses.sumBy { it.concolicCoverage.getCoverageInfo(it.className).first }
5056

5157
val totalInstructionsCount: Int
5258
get() = statsForClasses.sumBy { it.coverage?.instructionsCount?.toInt() ?: 0 }
5359

5460
val avgCoverage: Double
5561
get() = statsForClasses
5662
.filter { it.coverage?.instructionsCount?.let { cnt -> cnt != 0L } ?: false }
57-
.map { it.getCoverageInfo().run { if (second == 0) 0.0 else 100.0 * first / second } }.average()
63+
.map { it.coverage.getCoverageInfo(it.className).run { if (second == 0) 0.0 else 100.0 * first / second } }
64+
.average()
5865

5966
override fun toString(): String = "\n<Global statistics> :" +
6067
"\n\t#classes for generation = $classesForGeneration" +
@@ -78,9 +85,17 @@ class GlobalStats {
7885
.take(10)
7986
.printMultiline { (reason, names) -> " ${names.joinToString()}\n-->> In ${names.size} method(s) :: $reason" } +
8087
"\n----------------------------------------" +
81-
"\n\tTotal coverage: \n\t\t" +
82-
coveredInstructionsCount.let { num ->
83-
totalInstructionsCount.let { denum ->
88+
totalInstructionsCount.let { denum ->
89+
"\n\tTotal coverage: \n\t\t" +
90+
coveredInstructionsCount.let { num ->
91+
"$num/$denum (${(100.0 * num / denum).format(PRECISION)} %)"
92+
} +
93+
"\n\tTotal fuzzed coverage: \n\t\t" +
94+
coveredInstructionsCountByFuzzing.let { num ->
95+
"$num/$denum (${(100.0 * num / denum).format(PRECISION)} %)"
96+
} +
97+
"\n\tTotal concolic coverage: \n\t\t" +
98+
coveredInstructionsCountByConcolic.let { num ->
8499
"$num/$denum (${(100.0 * num / denum).format(PRECISION)} %)"
85100
}
86101
} +
@@ -100,15 +115,11 @@ class StatsForClass(val className: String) {
100115
val testcasesGenerated: Int get() = statsForMethods.sumBy { it.testsGeneratedCount }
101116

102117
var coverage: Coverage? = null
118+
var fuzzedCoverage: Coverage? = null
119+
var concolicCoverage: Coverage? = null
103120

104-
fun getCoverageInfo(): Pair<Int, Int> = coverage?.run {
105-
coveredInstructions.filter { instr ->
106-
instr.className.startsWith(className)
107-
}.toSet().size to (instructionsCount?.toInt() ?: 0)
108-
} ?: (0 to 0)
109-
110-
private fun prettyInfo(): String =
111-
getCoverageInfo().run { "$first/$second" }
121+
private fun Coverage?.prettyInfo(): String =
122+
getCoverageInfo(className).run { "$first/$second" }
112123

113124
override fun toString(): String = "\n<StatsForClass> :" +
114125
"\n\tcanceled by timeout = $canceledByTimeout" +
@@ -117,7 +128,9 @@ class StatsForClass(val className: String) {
117128
"\n\t#methods with at least one TC = ${statsForMethods.count { it.testsGeneratedCount > 0 }}" +
118129
"\n\t#methods with exceptions = $methodsWithAtLeastOneException" +
119130
"\n\t#generated TC = $testcasesGenerated" +
120-
"\n\t#coverage = ${prettyInfo()}"
131+
"\n\t#total coverage = ${coverage.prettyInfo()}" +
132+
"\n\t#fuzzed coverage = ${fuzzedCoverage.prettyInfo()}" +
133+
"\n\t#concolic coverage = ${concolicCoverage.prettyInfo()}"
121134
}
122135

123136

@@ -163,5 +176,10 @@ class FailReason(private val throwable: Throwable) {
163176
return stackTrace.contentHashCode()
164177
}
165178

179+
}
166180

167-
}
181+
private fun Coverage?.getCoverageInfo(className: String): Pair<Int, Int> = this?.run {
182+
coveredInstructions.filter { instr ->
183+
instr.className.startsWith(className)
184+
}.toSet().size to (instructionsCount?.toInt() ?: 0)
185+
} ?: (0 to 0)

‎utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ private fun GlobalStats.jsonString(baseTabs: Int = 0) =
100100
addValue("suspicious_methods", suspiciousMethods, tabs)
101101
addValue("test_classes_failed_to_compile", testClassesFailedToCompile, tabs)
102102
addValue("covered_instructions_count", coveredInstructionsCount, tabs)
103+
addValue("covered_instructions_count_by_fuzzing", coveredInstructionsCountByFuzzing, tabs)
104+
addValue("covered_instructions_count_by_concolic", coveredInstructionsCountByConcolic, tabs)
103105
addValue("total_instructions_count", totalInstructionsCount, tabs)
104106
addValue("avg_coverage", avgCoverage, tabs, needComma = false)
105107

0 commit comments

Comments
 (0)