Skip to content

Commit c2e7d83

Browse files
mmvpmmmvpm
committed
Add codeforces examples to contest
1 parent 22badb4 commit c2e7d83

6 files changed

Lines changed: 128 additions & 19 deletions

File tree

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,6 @@ import org.utbot.framework.codegen.domain.ForceStaticMocking
1414
import org.utbot.framework.codegen.domain.StaticsMocking
1515
import org.utbot.framework.codegen.domain.junitByVersion
1616
import org.utbot.framework.codegen.CodeGenerator
17-
import org.utbot.framework.plugin.api.CodegenLanguage
18-
import org.utbot.framework.plugin.api.Coverage
19-
import org.utbot.framework.plugin.api.ExecutableId
20-
import org.utbot.framework.plugin.api.MockStrategyApi
21-
import org.utbot.framework.plugin.api.TestCaseGenerator
22-
import org.utbot.framework.plugin.api.UtError
23-
import org.utbot.framework.plugin.api.UtExecution
24-
import org.utbot.framework.plugin.api.UtMethodTestSet
2517
import org.utbot.framework.plugin.api.util.UtContext
2618
import org.utbot.framework.plugin.api.util.executableId
2719
import org.utbot.framework.plugin.api.util.id
@@ -59,10 +51,10 @@ import kotlinx.coroutines.launch
5951
import kotlinx.coroutines.newSingleThreadContext
6052
import kotlinx.coroutines.runBlocking
6153
import kotlinx.coroutines.withTimeoutOrNull
62-
import kotlinx.coroutines.yield
6354
import org.utbot.framework.SummariesGenerationType
6455
import org.utbot.framework.codegen.services.language.CgLanguageAssistant
6556
import org.utbot.framework.minimization.minimizeExecutions
57+
import org.utbot.framework.plugin.api.*
6658
import org.utbot.framework.plugin.api.util.isSynthetic
6759
import org.utbot.framework.util.jimpleBody
6860
import org.utbot.summary.summarize
@@ -154,7 +146,8 @@ fun main(args: Array<String>) {
154146
fuzzingRatio = 0.1,
155147
classpathString,
156148
runFromEstimator = false,
157-
methodNameFilter = null
149+
methodNameFilter = null,
150+
expectedExceptions = ExpectedExceptionsForClass()
158151
)
159152
println("${ContestMessage.READY}")
160153
}
@@ -187,7 +180,8 @@ fun runGeneration(
187180
fuzzingRatio: Double,
188181
classpathString: String,
189182
runFromEstimator: Boolean,
190-
methodNameFilter: String? = null // For debug purposes you can specify method name
183+
methodNameFilter: String? = null, // For debug purposes you can specify method name
184+
expectedExceptions: ExpectedExceptionsForClass
191185
): StatsForClass = runBlocking {
192186
val testsByMethod: MutableMap<ExecutableId, MutableList<UtExecution>> = mutableMapOf()
193187
val currentContext = utContext
@@ -266,7 +260,10 @@ fun runGeneration(
266260
val methodJob = currentCoroutineContext().job
267261

268262
logger.debug { " ... " }
269-
val statsForMethod = StatsForMethod("${method.classId.simpleName}#${method.name}")
263+
val statsForMethod = StatsForMethod(
264+
"${method.classId.simpleName}#${method.name}",
265+
expectedExceptions.getForMethod(method.name).exceptionNames
266+
)
270267
statsForClass.statsForMethods.add(statsForMethod)
271268

272269

@@ -337,6 +334,9 @@ fun runGeneration(
337334
val className = Type.getInternalName(method.classId.jClass)
338335
logger.debug { "--new testCase collected, to generate: $testMethodName" }
339336
statsForMethod.testsGeneratedCount++
337+
result.result.exceptionOrNull()?.let { exception ->
338+
statsForMethod.detectedExceptionFqns += exception::class.java.name
339+
}
340340
result.coverage?.let {
341341
statsForClass.updateCoverage(
342342
newCoverage = it,

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt‎

Lines changed: 67 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ enum class Tool {
137137
methodNameFilter: String?,
138138
statsForProject: StatsForProject,
139139
compiledTestDir: File,
140-
classFqn: String
140+
classFqn: String,
141+
expectedExceptions: ExpectedExceptionsForClass
141142
) = withUtContext(ContextManager.createNewContext(project.classloader)) {
142143
val classStats: StatsForClass = try {
143144
runGeneration(
@@ -147,7 +148,8 @@ enum class Tool {
147148
fuzzingRatio,
148149
project.sootClasspathString,
149150
runFromEstimator = true,
150-
methodNameFilter
151+
methodNameFilter,
152+
expectedExceptions
151153
)
152154
} catch (e: CancellationException) {
153155
logger.info { "[$classFqn] finished with CancellationException" }
@@ -277,7 +279,8 @@ enum class Tool {
277279
methodNameFilter: String?,
278280
statsForProject: StatsForProject,
279281
compiledTestDir: File,
280-
classFqn: String
282+
classFqn: String,
283+
expectedExceptions: ExpectedExceptionsForClass
281284
)
282285

283286
abstract fun moveProducedFilesIfNeeded()
@@ -391,7 +394,13 @@ fun runEstimator(
391394
if (updatedMethodFilter != null)
392395
logger.info { "Filtering: class='$classFqnFilter', method ='$methodNameFilter'" }
393396

394-
val projectToClassFQNs = classesLists.listFiles()!!.associate { it.name to File(it, "list").readLines() }
397+
val projectDirs = classesLists.listFiles()!!
398+
val projectToClassFQNs = projectDirs.associate {
399+
it.name to File(it, "list").readLines()
400+
}
401+
val projectToExpectedExceptions = projectDirs.associate {
402+
it.name to parseExceptionsFile(File(it, "exceptions"))
403+
}
395404

396405
val projects = mutableListOf<ProjectToEstimate>()
397406

@@ -400,6 +409,7 @@ fun runEstimator(
400409
val project = ProjectToEstimate(
401410
name,
402411
classesFQN,
412+
projectToExpectedExceptions[name]!!,
403413
File(classpathDir, name).listFiles()!!.filter { it.toString().endsWith("jar") },
404414
testCandidatesDir,
405415
unzippedJars
@@ -463,7 +473,8 @@ fun runEstimator(
463473
methodNameFilter,
464474
statsForProject,
465475
compiledTestDir,
466-
classFqn
476+
classFqn,
477+
project.expectedExceptions.getForClass(classFqn)
467478
)
468479
}
469480
catch (e: Throwable) {
@@ -523,6 +534,57 @@ private fun classNamesByJar(jar: File): List<String> {
523534
.toList()
524535
}
525536

537+
class ExpectedExceptionsForMethod(
538+
val exceptionNames: List<String>
539+
)
540+
541+
class ExpectedExceptionsForClass {
542+
543+
private val byMethodName: MutableMap<String, ExpectedExceptionsForMethod> =
544+
mutableMapOf()
545+
546+
fun getForMethod(methodName: String): ExpectedExceptionsForMethod =
547+
byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf())
548+
549+
fun setForMethod(methodName: String, exceptionNames: List<String>) {
550+
byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames)
551+
}
552+
}
553+
554+
class ExpectedExceptionsForProject {
555+
556+
private val byClassName: MutableMap<String, ExpectedExceptionsForClass> =
557+
mutableMapOf()
558+
559+
fun getForClass(className: String): ExpectedExceptionsForClass =
560+
byClassName[className] ?: ExpectedExceptionsForClass()
561+
562+
fun setForClass(className: String, methodName: String, exceptionNames: List<String>) {
563+
val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() }
564+
forClass.setForMethod(methodName, exceptionNames)
565+
}
566+
}
567+
568+
fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject {
569+
if (!exceptionsDescriptionFile.exists()) {
570+
return ExpectedExceptionsForProject()
571+
}
572+
573+
val forProject = ExpectedExceptionsForProject()
574+
for (methodDescription in exceptionsDescriptionFile.readLines()) {
575+
val methodFqn = methodDescription.substringBefore(':').trim()
576+
val classFqn = methodFqn.substringBeforeLast('.')
577+
val methodName = methodFqn.substringAfterLast('.')
578+
val exceptionFqns = methodDescription.substringAfter(':')
579+
.split(' ')
580+
.map { it.trim() }
581+
.filter { it.isNotBlank() }
582+
forProject.setForClass(classFqn, methodName, exceptionFqns)
583+
}
584+
585+
return forProject
586+
}
587+
526588
class ProjectToEstimate(
527589
val name: String,
528590
val classFQNs: List<String>,

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,15 @@ class StatsForProject(val project: String) {
7575
else this
7676
}
7777

78+
val detectedExceptionsCount: Int
79+
get() = statsForClasses.sumOf { it.detectedExceptionsCount }
80+
val expectedExceptionsCount: Int
81+
get() = statsForClasses.sumOf { it.expectedExceptionsCount }
82+
7883
override fun toString(): String = "\n<StatsForProject($project)> :" +
7984
"\n\t#classes for generation = $classesForGeneration" +
8085
"\n\t#tc generated = $testCasesGenerated" +
86+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
8187
"\n\t#classes without problems = $classesWithoutProblems" +
8288
"\n\t#classes canceled by timeout = $classesCanceledByTimeout" +
8389
"\n----------------------------------------" +
@@ -128,6 +134,11 @@ class StatsForClass(val project: String, val className: String) {
128134
val methodsWithAtLeastOneException: Int get() = statsForMethods.count { it.failReasons.isNotEmpty() }
129135
val testcasesGenerated: Int get() = statsForMethods.sumOf { it.testsGeneratedCount }
130136

137+
val detectedExceptionsCount: Int
138+
get() = statsForMethods.sumOf { it.detectedExceptionsCount }
139+
val expectedExceptionsCount: Int
140+
get() = statsForMethods.sumOf { it.expectedExceptionsCount }
141+
131142
var coverage = CoverageInstructionsSet()
132143
var fuzzedCoverage = CoverageInstructionsSet()
133144
var concolicCoverage = CoverageInstructionsSet()
@@ -149,23 +160,37 @@ class StatsForClass(val project: String, val className: String) {
149160
"\n\t#methods with at least one TC = ${statsForMethods.count { it.testsGeneratedCount > 0 }}" +
150161
"\n\t#methods with exceptions = $methodsWithAtLeastOneException" +
151162
"\n\t#generated TC = $testcasesGenerated" +
163+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
152164
"\n\t#total coverage = ${coverage.prettyInfo()}" +
153165
"\n\t#fuzzed coverage = ${fuzzedCoverage.prettyInfo()}" +
154166
"\n\t#concolic coverage = ${concolicCoverage.prettyInfo()}"
155167
}
156168

157169

158-
class StatsForMethod(val methodName: String) {
170+
class StatsForMethod(
171+
val methodName: String,
172+
val expectedExceptionFqns: List<String>
173+
) {
159174
var testsGeneratedCount = 0
160175

161176
val failReasons: MutableMultiset<FailReason> = mutableMultisetOf()
162177

178+
val detectedExceptionFqns: MutableSet<String> = mutableSetOf()
179+
163180
//generated no TC, nor exception
164181
val isSuspicious: Boolean get() = failReasons.isEmpty() && testsGeneratedCount == 0
165182

183+
val detectedExceptionsCount: Int
184+
get() = expectedExceptionFqns.toSet().intersect(detectedExceptionFqns).size
185+
186+
val expectedExceptionsCount: Int =
187+
expectedExceptionFqns.size
188+
166189

167190
override fun toString(): String = "\n<StatsForMethod> :" + (if (isSuspicious) " SUSPICIOUS" else "") +
168-
"\n\t#generatedTC=$testsGeneratedCount\n\t" +
191+
"\n\t#generated TC = $testsGeneratedCount" +
192+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
193+
"\n\t" +
169194
(if (failReasons.isEmpty()) "WITH NO EXCEPTIONS"
170195
else "FAILED ${failReasons.sumOfMultiplicities} time(s) with ${failReasons.size} different exception(s)\"")
171196

‎utbot-junit-contest/src/main/resources/classes/codeforces/exceptions‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
org.utbot.examples.codeforces.error.stackof.Task_70A.solve: java.lang.StackOverflowError
2+
org.utbot.examples.codeforces.exception.aiobe.Task_70A.solve: java.lang.ArrayIndexOutOfBoundsException
3+
org.utbot.examples.codeforces.exception.aiobe.Task_71B.solve: java.lang.ArrayIndexOutOfBoundsException
4+
org.utbot.examples.codeforces.exception.aiobe.Task_1712C.solve: java.lang.ArrayIndexOutOfBoundsException
5+
org.utbot.examples.codeforces.exception.aiobe.Task_1749C.solve: java.lang.ArrayIndexOutOfBoundsException
6+
org.utbot.examples.codeforces.exception.arithmetic.Task_1715B.solve: java.lang.ArithmeticException
7+
org.utbot.examples.codeforces.exception.error.Task_1718A2.solve: java.lang.AssertionError
8+
org.utbot.examples.codeforces.exception.iobe.Task_1740C.solve: java.lang.IndexOutOfBoundsException
9+
org.utbot.examples.codeforces.exception.npe.Task_1703D.solve: java.lang.NullPointerException
10+
org.utbot.examples.codeforces.exception.siobe.Task_75A.solve: java.lang.StringIndexOutOfBoundsException
11+
org.utbot.examples.codeforces.exception.siobe.Task_1702B.solve: java.lang.StringIndexOutOfBoundsException

‎utbot-junit-contest/src/main/resources/classes/codeforces/list‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
org.utbot.examples.codeforces.error.stackof.Task_70A
2+
org.utbot.examples.codeforces.exception.aiobe.Task_70A
3+
org.utbot.examples.codeforces.exception.aiobe.Task_71B
4+
org.utbot.examples.codeforces.exception.aiobe.Task_1712C
5+
org.utbot.examples.codeforces.exception.aiobe.Task_1749C
6+
org.utbot.examples.codeforces.exception.arithmetic.Task_1715B
7+
org.utbot.examples.codeforces.exception.error.Task_1718A2
8+
org.utbot.examples.codeforces.exception.iobe.Task_1740C
9+
org.utbot.examples.codeforces.exception.npe.Task_1703D
10+
org.utbot.examples.codeforces.exception.siobe.Task_75A
11+
org.utbot.examples.codeforces.exception.siobe.Task_1702B

‎utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar‎

14.3 KB
Binary file not shown.

0 commit comments

Comments
 (0)