add script to build aggregated data

Author: SBOne-Kenobi

.github/workflows/night-statistics-monitoring.yml

@@ -5,20 +5,33 @@ on:
      - cron: '0 0 * * *'
 
 env:
+  data_branch: monitoring-data
+  data_path: monitoring/data
+  aggregated_data_branch: monitoring-aggregated-data
+  aggregated_data_path: monitoring/aggregated_data
   monitoring_properties: monitoring/monitoring.properties
   output_stats: stats.json
-  history_file: monitoring/history.json
-  coverage_graph_file: monitoring/coverage_graph.png
-  quantitative_graph_file: monitoring/quantitative_graph.png
   KOTLIN_HOME: /usr
 
 jobs:
   build_and_run_monitoring:
     runs-on: ubuntu-20.04
     steps:
-      - name: Checkout repository
+      - name: Checkout main
         uses: actions/checkout@v3
 
+      - name: Checkout monitoring data
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.data_branch }}
+          path: ${{ env.data_path }}
+
+      - name: Checkout aggregated monitoring data
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.aggregated_data_branch }}
+          path: ${{ env.aggregated_data_path }}
+
       - uses: actions/setup-java@v3
         with:
           java-version: '8'
@@ -30,9 +43,6 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
-          cache: 'pip'
-      - name: Install matplotlib
-        run: pip install matplotlib
 
       - name: Build and run monitoring UTBot Java
         run: |
@@ -42,23 +52,56 @@ jobs:
             utbot-junit-contest/build/libs/monitoring.jar \
             $output_stats
 
-      - name: Update history and render graphs
-        run: |
-          python monitoring/draw_stats_graphs.py \
-            $history_file \
-            $output_stats \
-            $coverage_graph_file \
-            $quantitative_graph_file
-
       - name: Get current date
         id: date
-        run: echo "::set-output name=date::$(date +'%d-%m-%Y')"
+        run: |
+          echo "::set-output name=date::$(date +'%Y-%m-%d')"
+          echo "::set-output name=timestamp::$(date +%s)"
+          echo "::set-output name=last_month::$(date --date='last month' +%s)"
 
-      - name: Commit and push graphs and statistics
+      - name: Get commit hash
+        id: metadata
+        run: |
+          echo "::set-output name=commit::$(git rev-parse HEAD)"
+          echo "::set-output name=branch::$(git name-rev --name-only HEAD)"
+          echo "::set-output name=build::$(date +'%Y.%-m')"
+
+      - name: Insert metadata
+        run: |
+          python monitoring/insert_metadata.py \
+            $output_stats "$data_path/data-$branch-$date-$timestamp-$commit.json" \
+            "$commit" "$build"
+        env:
+          date: ${{ steps.date.outputs.date }}
+          timestamp: ${{ steps.date.outputs.timestamp }}
+          commit: ${{ steps.metadata.outputs.commit }}
+          branch: ${{ steps.metadata.outputs.branch }}
+          build: ${{ steps.metadata.outputs.build }}
+
+      - name: Build aggregated data (last month)
+        run: |
+          python monitoring/build_aggregated_data.py \
+            $data_path $aggregated_data_path/aggregated-data-$date.json \
+            $timestamp_from $timestamp
+        env:
+          date: ${{ steps.date.outputs.date }}
+          timestamp: ${{ steps.date.outputs.timestamp }}
+          timestamp_from: ${{ steps.date.outputs.last_month }}
+
+      - name: Commit and push statistics
         uses: actions-js/push@master
         with:
+          branch: ${{ env.data_branch }}
           message: 'night-monitoring-${{ steps.date.outputs.date }}'
-          directory: './monitoring'
+          directory: ${{ env.data_path }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Commit and push aggregated statistics
+        uses: actions-js/push@master
+        with:
+          branch: ${{ env.aggregated_data_branch }}
+          message: 'night-monitoring-${{ steps.date.outputs.date }}'
+          directory: ${{ env.aggregated_data_path }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload logs
@@ -67,10 +110,3 @@ jobs:
         with:
           name: logs
           path: logs/utbot.log
-
-      - name: Upload statistics
-        if: ${{ success() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: statistics
-          path: ${{ env.output_stats }}

monitoring/build_aggregated_data.py

@@ -0,0 +1,71 @@
+import json
+import re
+from os import listdir
+from os.path import isfile, join
+from sys import argv
+
+"""
+Input files data requires names format *-<timestamp>-<commit>.json
+"""
+FILE_FORMAT_REGEXP = re.compile(r'.+-(\d+)-[^-]+\.json')
+
+
+def get_file_timestamp(filename):
+    result = FILE_FORMAT_REGEXP.fullmatch(filename)
+    if result is None:
+        return None
+    return int(result.group(1))
+
+
+def check_timestamp(timestamp, timestamp_from, timestamp_to):
+    return timestamp is not None and timestamp_from <= timestamp <= timestamp_to
+
+
+def get_file_seq(input_data_dir, timestamp_from, timestamp_to):
+    for filename in listdir(input_data_dir):
+        path = join(input_data_dir, filename)
+        if isfile(path):
+            timestamp = get_file_timestamp(filename)
+            if check_timestamp(timestamp, timestamp_from, timestamp_to):
+                yield path, timestamp
+
+
+def get_stats_seq(input_data_dir, timestamp_from, timestamp_to):
+    for file, timestamp in get_file_seq(input_data_dir, timestamp_from, timestamp_to):
+        with open(file, "r") as f:
+            stats = json.load(f)
+        yield stats, timestamp
+
+
+def transform_stats(stats, timestamp):
+    del stats['metadata']
+    stats['timestamp'] = timestamp
+    return stats
+
+
+def aggregate_stats(stats_seq):
+    result = []
+    for stats, timestamp in stats_seq:
+        result.append(transform_stats(stats, timestamp))
+    return result
+
+
+def main():
+    args = argv[1:]
+    if len(args) != 4:
+        raise RuntimeError(
+            f"Expected <input data dir> <output file> "
+            f"<timestamp from> <timestamp to> "
+            f"but got {' '.join(args)}"
+        )
+    (input_data_dir, output_file, timestamp_from, timestamp_to) = args
+    timestamp_from = int(timestamp_from)
+    timestamp_to = int(timestamp_to)
+    stats_seq = get_stats_seq(input_data_dir, timestamp_from, timestamp_to)
+    result = aggregate_stats(stats_seq)
+    with open(output_file, "w") as f:
+        json.dump(result, f, indent=4)
+
+
+if __name__ == '__main__':
+    main()