Skip to content

Commit 65fc08e

Browse files
author
Rustam Sadykov
committed
update format
1 parent 61d34d2 commit 65fc08e

6 files changed

Lines changed: 219 additions & 140 deletions

File tree

‎docs/NightStatisticsMonitoring.md‎

Lines changed: 112 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ To run statistics monitoring you have to specify the name of the JSON output fil
2121

2222
Input arguments: `<output json>`.
2323

24-
Output format: you get the JSON file, which contains an array of objects with statistics on each run.
24+
Output format: you get the JSON file, which contains an array of objects with statistics and input parameters on each run.
2525

2626
More about each statistic: `Statistics.kt`.
2727

@@ -38,64 +38,76 @@ Output example (the result of three runs during one night):
3838
```json
3939
[
4040
{
41-
"target": "guava",
42-
"class_timeout_sec": 20,
43-
"run_timeout_min": 20,
44-
"duration_ms": 604225,
45-
"classes_for_generation": 20,
46-
"testcases_generated": 1651,
47-
"classes_without_problems": 12,
48-
"classes_canceled_by_timeout": 2,
49-
"total_methods_for_generation": 519,
50-
"methods_with_at_least_one_testcase_generated": 365,
51-
"methods_with_exceptions": 46,
52-
"suspicious_methods": 85,
53-
"test_classes_failed_to_compile": 0,
54-
"covered_instructions": 5753,
55-
"covered_instructions_by_fuzzing": 4375,
56-
"covered_instructions_by_concolic": 4069,
57-
"total_instructions": 10182,
58-
"avg_coverage": 62.885408034613
41+
"parameters": {
42+
"target": "guava",
43+
"class_timeout_sec": 20,
44+
"run_timeout_min": 20
45+
},
46+
"metrics": {
47+
"duration_ms": 604225,
48+
"classes_for_generation": 20,
49+
"testcases_generated": 1651,
50+
"classes_without_problems": 12,
51+
"classes_canceled_by_timeout": 2,
52+
"total_methods_for_generation": 519,
53+
"methods_with_at_least_one_testcase_generated": 365,
54+
"methods_with_exceptions": 46,
55+
"suspicious_methods": 85,
56+
"test_classes_failed_to_compile": 0,
57+
"covered_instructions": 5753,
58+
"covered_instructions_by_fuzzing": 4375,
59+
"covered_instructions_by_concolic": 4069,
60+
"total_instructions": 10182,
61+
"avg_coverage": 62.885408034613
62+
}
5963
},
6064
{
61-
"target": "guava",
62-
"class_timeout_sec": 20,
63-
"run_timeout_min": 20,
64-
"duration_ms": 633713,
65-
"classes_for_generation": 20,
66-
"testcases_generated": 1872,
67-
"classes_without_problems": 12,
68-
"classes_canceled_by_timeout": 2,
69-
"total_methods_for_generation": 519,
70-
"methods_with_at_least_one_testcase_generated": 413,
71-
"methods_with_exceptions": 46,
72-
"suspicious_methods": 38,
73-
"test_classes_failed_to_compile": 0,
74-
"covered_instructions": 6291,
75-
"covered_instructions_by_fuzzing": 4470,
76-
"covered_instructions_by_concolic": 5232,
77-
"total_instructions": 11011,
78-
"avg_coverage": 62.966064315865275
65+
"parameters": {
66+
"target": "guava",
67+
"class_timeout_sec": 20,
68+
"run_timeout_min": 20
69+
},
70+
"metrics": {
71+
"duration_ms": 633713,
72+
"classes_for_generation": 20,
73+
"testcases_generated": 1872,
74+
"classes_without_problems": 12,
75+
"classes_canceled_by_timeout": 2,
76+
"total_methods_for_generation": 519,
77+
"methods_with_at_least_one_testcase_generated": 413,
78+
"methods_with_exceptions": 46,
79+
"suspicious_methods": 38,
80+
"test_classes_failed_to_compile": 0,
81+
"covered_instructions": 6291,
82+
"covered_instructions_by_fuzzing": 4470,
83+
"covered_instructions_by_concolic": 5232,
84+
"total_instructions": 11011,
85+
"avg_coverage": 62.966064315865275
86+
}
7987
},
8088
{
81-
"target": "guava",
82-
"class_timeout_sec": 20,
83-
"run_timeout_min": 20,
84-
"duration_ms": 660421,
85-
"classes_for_generation": 20,
86-
"testcases_generated": 1770,
87-
"classes_without_problems": 13,
88-
"classes_canceled_by_timeout": 2,
89-
"total_methods_for_generation": 519,
90-
"methods_with_at_least_one_testcase_generated": 405,
91-
"methods_with_exceptions": 44,
92-
"suspicious_methods": 43,
93-
"test_classes_failed_to_compile": 0,
94-
"covered_instructions": 6266,
95-
"covered_instructions_by_fuzzing": 4543,
96-
"covered_instructions_by_concolic": 5041,
97-
"total_instructions": 11011,
98-
"avg_coverage": 61.59069193429194
89+
"parameters": {
90+
"target": "guava",
91+
"class_timeout_sec": 20,
92+
"run_timeout_min": 20
93+
},
94+
"metrics": {
95+
"duration_ms": 660421,
96+
"classes_for_generation": 20,
97+
"testcases_generated": 1770,
98+
"classes_without_problems": 13,
99+
"classes_canceled_by_timeout": 2,
100+
"total_methods_for_generation": 519,
101+
"methods_with_at_least_one_testcase_generated": 405,
102+
"methods_with_exceptions": 44,
103+
"suspicious_methods": 43,
104+
"test_classes_failed_to_compile": 0,
105+
"covered_instructions": 6266,
106+
"covered_instructions_by_fuzzing": 4543,
107+
"covered_instructions_by_concolic": 5041,
108+
"total_instructions": 11011,
109+
"avg_coverage": 61.59069193429194
110+
}
99111
}
100112
]
101113
```
@@ -108,28 +120,40 @@ The `insert_metadata.py` script is responsible for doing this. To run it you hav
108120

109121
To get more information about input arguments call script with option `--help`.
110122

111-
Output format: you get the JSON file, containing statistics grouped by target project and metadata.
123+
Output format: you get the JSON file, containing statistics and parameters grouped by target project and metadata.
112124

113125
Input example:
114126
```
115127
--stats_file stats.json --output_file data/meta-stats.json
116128
--commit 66a1aeb6 --branch main
117-
--build 2022.8 --timestamp 1661174420
118-
--source_type github-action --source_id 2902082973
129+
--build 2022.8 --timestamp 1661330445
130+
--source_type github-action --source_id 2917672580
119131
```
120132

121-
Output example (an average for each statistic over the three runs followed by metadata):
133+
Output example (statistics followed by metadata):
122134
```json
123135
{
124136
"version": 1,
125137
"targets": [
126138
{
127139
"id": "guava",
128140
"version": "0",
129-
"metrics": [
141+
"parameters": [
130142
{
131143
"class_timeout_sec": 20,
132-
"run_timeout_min": 20,
144+
"run_timeout_min": 20
145+
},
146+
{
147+
"class_timeout_sec": 20,
148+
"run_timeout_min": 20
149+
},
150+
{
151+
"class_timeout_sec": 20,
152+
"run_timeout_min": 20
153+
}
154+
],
155+
"metrics": [
156+
{
133157
"duration_ms": 604225,
134158
"classes_for_generation": 20,
135159
"testcases_generated": 1651,
@@ -147,8 +171,6 @@ Output example (an average for each statistic over the three runs followed by me
147171
"avg_coverage": 62.885408034613
148172
},
149173
{
150-
"class_timeout_sec": 20,
151-
"run_timeout_min": 20,
152174
"duration_ms": 633713,
153175
"classes_for_generation": 20,
154176
"testcases_generated": 1872,
@@ -166,8 +188,6 @@ Output example (an average for each statistic over the three runs followed by me
166188
"avg_coverage": 62.966064315865275
167189
},
168190
{
169-
"class_timeout_sec": 20,
170-
"run_timeout_min": 20,
171191
"duration_ms": 660421,
172192
"classes_for_generation": 20,
173193
"testcases_generated": 1770,
@@ -190,21 +210,21 @@ Output example (an average for each statistic over the three runs followed by me
190210
"metadata": {
191211
"source": {
192212
"type": "github-action",
193-
"id": "2902082973"
213+
"id": "2917672580"
194214
},
195215
"commit_hash": "66a1aeb6",
196216
"branch": "main",
197217
"build_number": "2022.8",
198-
"timestamp": 1661174420,
199-
"date": "2022-08-22T13:20:20",
218+
"timestamp": 1661330445,
219+
"date": "2022-08-24T08:40:45",
200220
"environment": {
201-
"host": "fv-az377-887",
221+
"host": "fv-az183-700",
202222
"OS": "Linux version #20~20.04.1-Ubuntu SMP Fri Aug 5 12:16:53 UTC 2022",
203223
"java_version": "openjdk version \"11.0.16\" 2022-07-19 LTS\nOpenJDK Runtime Environment Zulu11.58+15-CA (build 11.0.16+8-LTS)\nOpenJDK 64-Bit Server VM Zulu11.58+15-CA (build 11.0.16+8-LTS, mixed mode)\n",
204224
"gradle_version": "Gradle 7.4.2",
205-
"JAVA_HOME": "/opt/hostedtoolcache/Java_Zulu_jdk+fx/8.0.345-1/x64",
225+
"JAVA_HOME": "/opt/hostedtoolcache/Java_Zulu_jdk+fx/11.0.16-8/x64",
206226
"KOTLIN_HOME": "/usr",
207-
"PATH": "/opt/hostedtoolcache/Python/3.9.13/x64/bin:/opt/hostedtoolcache/Python/3.9.13/x64:/home/runner/gradle-installations/installs/gradle-6.8/bin:/opt/hostedtoolcache/Java_Zulu_jdk+fx/8.0.345-1/x64/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:/home/runner/.local/bin:/opt/pipx_bin:/home/runner/.cargo/bin:/home/runner/.config/composer/vendor/bin:/usr/local/.ghcup/bin:/home/runner/.dotnet/tools:/snap/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin"
227+
"PATH": "/opt/hostedtoolcache/Python/3.9.13/x64/bin:/opt/hostedtoolcache/Python/3.9.13/x64:/home/runner/gradle-installations/installs/gradle-7.4.2/bin:/opt/hostedtoolcache/Java_Zulu_jdk+fx/11.0.16-8/x64/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:/home/runner/.local/bin:/opt/pipx_bin:/home/runner/.cargo/bin:/home/runner/.config/composer/vendor/bin:/usr/local/.ghcup/bin:/home/runner/.dotnet/tools:/snap/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin"
208228
}
209229
}
210230
}
@@ -218,13 +238,13 @@ To run aggregating you should provide the input.
218238

219239
To get more information about input arguments call script with option `--help`.
220240

221-
Output format: you get the JSON file, which contains arrays of summarised results for each of the nights during the specified period grouped by target.
241+
Output format: you get the JSON file, which contains arrays of grouped by target results for each of the nights during the specified period.
222242

223243
Input example:
224244

225245
```
226246
--input_data_dir ./data --output_file aggregated_data.json
227-
--timestamp_from 0 --timestamp_to 1661174420
247+
--timestamp_from 0 --timestamp_to 1661330445
228248
```
229249

230250
Output example (You'll get an array of several outputs without metadata):
@@ -233,10 +253,25 @@ Output example (You'll get an array of several outputs without metadata):
233253
{
234254
"id": "guava",
235255
"version": "0",
236-
"metrics": [
256+
"parameters": [
237257
{
238258
"class_timeout_sec": 20,
239259
"run_timeout_min": 20,
260+
"timestamp": 1661330445
261+
},
262+
{
263+
"class_timeout_sec": 20,
264+
"run_timeout_min": 20,
265+
"timestamp": 1661330445
266+
},
267+
{
268+
"class_timeout_sec": 20,
269+
"run_timeout_min": 20,
270+
"timestamp": 1661330445
271+
}
272+
],
273+
"metrics": [
274+
{
240275
"duration_ms": 604225,
241276
"classes_for_generation": 20,
242277
"testcases_generated": 1651,
@@ -250,12 +285,9 @@ Output example (You'll get an array of several outputs without metadata):
250285
"avg_coverage": 62.885408034613,
251286
"total_coverage": 56.50166961304262,
252287
"total_coverage_by_fuzzing": 42.967982714594385,
253-
"total_coverage_by_concolic": 39.96267923787075,
254-
"timestamp": 1661174420
288+
"total_coverage_by_concolic": 39.96267923787075
255289
},
256290
{
257-
"class_timeout_sec": 20,
258-
"run_timeout_min": 20,
259291
"duration_ms": 633713,
260292
"classes_for_generation": 20,
261293
"testcases_generated": 1872,
@@ -269,12 +301,9 @@ Output example (You'll get an array of several outputs without metadata):
269301
"avg_coverage": 62.966064315865275,
270302
"total_coverage": 57.133775315593496,
271303
"total_coverage_by_fuzzing": 40.595767868495145,
272-
"total_coverage_by_concolic": 47.51612024339297,
273-
"timestamp": 1661174420
304+
"total_coverage_by_concolic": 47.51612024339297
274305
},
275306
{
276-
"class_timeout_sec": 20,
277-
"run_timeout_min": 20,
278307
"duration_ms": 660421,
279308
"classes_for_generation": 20,
280309
"testcases_generated": 1770,
@@ -288,8 +317,7 @@ Output example (You'll get an array of several outputs without metadata):
288317
"avg_coverage": 61.59069193429194,
289318
"total_coverage": 56.90672963400236,
290319
"total_coverage_by_fuzzing": 41.25874125874126,
291-
"total_coverage_by_concolic": 45.78149123603669,
292-
"timestamp": 1661174420
320+
"total_coverage_by_concolic": 45.78149123603669
293321
}
294322
]
295323
}

‎monitoring/build_aggregated_data.py‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
import argparse
22
import json
3-
from collections import defaultdict
43
from os import listdir
54
from os.path import isfile, join
65
from time import time
76

87
from monitoring_settings import JSON_VERSION
9-
from utils import postprocess_targets
8+
from utils import *
109

1110

1211
def get_file_seq(input_data_dir):
@@ -34,7 +33,7 @@ def get_stats_seq(args):
3433
yield stats
3534

3635

37-
def transform_target_stats(stats, timestamp):
36+
def transform_target_stats(stats):
3837
common_prefix = "covered_instructions"
3938
denum = stats["total_instructions"]
4039

@@ -46,24 +45,24 @@ def transform_target_stats(stats, timestamp):
4645
del stats[key]
4746

4847
del stats["total_instructions"]
49-
stats["timestamp"] = timestamp
5048

5149
return stats
5250

5351

5452
def aggregate_stats(stats_seq):
55-
result = defaultdict(lambda: [])
53+
result = get_default_metrics_dict()
5654

5755
for stats in stats_seq:
5856
targets = stats["targets"]
5957
timestamp = stats["metadata"]["timestamp"]
6058
for target in targets:
6159
full_name = f'{target["id"]}-{target["version"]}'
62-
metrics = target["metrics"]
63-
for target_stats in metrics:
64-
result[full_name].append(
65-
transform_target_stats(target_stats, timestamp)
66-
)
60+
new_data = result[full_name]
61+
for target_stats in target["metrics"]:
62+
new_data["metrics"].append(transform_target_stats(target_stats))
63+
for target_params in target["parameters"]:
64+
target_params["timestamp"] = timestamp
65+
new_data["parameters"].append(target_params)
6766

6867
return postprocess_targets(result)
6968

0 commit comments

Comments
 (0)