plot fixes + use JOSN instead of pickle (#23)

* changed db type from bytes to string latency now displays #entries TTA looks more like TTA but would need to change epochs in x-axis for actual time * Fixed Memory representation of different timed runs * Changed datatype from Byte to JSON * Changed datatype from Byte to JSON * Added Timed TTA * Added JSON support * Added Timed TTA Tracker * Added timed TTA tracker --------- Co-authored-by: sjoze <[email protected]> Co-authored-by: sjoze <[email protected]>
hpides · Feb 27, 2024 · ac38c15 · ac38c15
1 parent 32ebfad
commit ac38c15
Show file tree

Hide file tree

Showing 9 changed files with 125 additions and 32 deletions.
diff --git a/pipelines/github_example/sample_db_file.db b/pipelines/github_example/sample_db_file.db
diff --git a/pipelines/sample_pipeline/main.py b/pipelines/sample_pipeline/main.py
@@ -15,19 +15,21 @@
                      PowerMetric,\
                      EnergyMetric,\
                      LossTracker,\
-                     CPUMetric
+                     CPUMetric, \
+                     TimedTTATracker
 
 
 bm = Benchmark('sample_db_file.db', description="le description")
 
+
 bloat_metrics = {
     "throughput": ThroughputMetric('bloat throughput'),
     "latency": LatencyMetric('bloat latency'),
     "time": TimeMetric('bloat time'),
-    "memory": MemoryMetric('bloat memory', interval=0.1),
     "power": PowerMetric('bloat power'),
+    "cpu": CPUMetric('bloat cpu', interval=0.1),
     "energy": EnergyMetric('bloat energy'),
-    "cpu": CPUMetric('bloat cpu', interval=0.1)
+    "memory": MemoryMetric('bloat memory', interval=0.1)
 }
 
 
@@ -36,7 +38,7 @@ def bloat():
     a = []
     for i in range(1, 2):
         a.append(np.random.randn(*([10] * i)))
-        time.sleep(5)
+        time.sleep(1)
     print(a)
     bloat_metrics["throughput"].track(420)
     bloat_metrics["latency"].track(69)
@@ -47,6 +49,12 @@ def main():
     labels = ['foo', 'bar', 'baz']
     ConfusionMatrixTracker(bm).track(conf_mat, labels, 'foobar')
 
+    tttaTracker = TimedTTATracker(bm, target_acc=90)
+
+    for i in range(5):
+        time.sleep(0.01)
+        tttaTracker.track(np.random.randint(0,100), description="Accuracies of timed TTA")
+
     with HyperparameterTracker(bm, "hyper params of sample pipeline", ['lr', 'num_epochs', 'num_layers'], 'loss') as ht:
         ht.track({'lr': 0.03, 'num_epochs': 10, 'num_layers': 4, 'loss': 42})
         ht.track({'lr': 0.08, 'num_epochs': 15, 'num_layers': 2, 'loss': 69})

diff --git a/pipelines/stock_market_pipeline/train.py b/pipelines/stock_market_pipeline/train.py
@@ -11,7 +11,6 @@
 
 bm = Benchmark('stock_market.db')
 
-
 lat = LatencyMetric('stock market latency')
 thr = ThroughputMetric('stock market throughput')
 tta = TTATracker(bm)

diff --git a/umlaut/umlaut/benchmark.py b/umlaut/umlaut/benchmark.py
@@ -106,6 +106,7 @@ def _database_thread_func(self):
                 while not self.queue.empty():
                     sleep(0)
                     measurement = self.queue.get()
+                    #measurement.measurement_data = str(measurement.measurement_data)
                     session.add(measurement)
                     log_staged = True
                 if log_staged:
@@ -149,7 +150,7 @@ def query_all_uuid_type_desc(self):
                            Measurement.measurement_type,
                            Measurement.measurement_description)
         col_names = [col_desc['name'] for col_desc in query.column_descriptions]
-        
+
         return pd.DataFrame(query.all(), columns=col_names).set_index('id')
 
     def join_visualization_queries(self, uuid_type_desc_df):
@@ -183,9 +184,10 @@ def join_visualization_queries(self, uuid_type_desc_df):
                                             Measurement.id.in_(uuid_type_desc_df.index))
         measure_col_names = [col_desc['name'] for col_desc in measurement_query.column_descriptions]
         measurement_df = pd.DataFrame(measurement_query.all(), columns=measure_col_names)
-        measurement_df['measurement_data'] = measurement_df['measurement_data'].map(pickle.loads)
+        #measurement_df['measurement_data'] = measurement_df['measurement_data'].map(pickle.loads)
+        measurement_df['measurement_data'] = measurement_df['measurement_data']
 
         joined_df = uuid_type_desc_df.reset_index().merge(meta_df, on='uuid')
         joined_df = joined_df.merge(measurement_df, on='id')
 
-        return joined_df.set_index('id')
+        return joined_df.set_index('id')
diff --git a/umlaut/umlaut/datamodel.py b/umlaut/umlaut/datamodel.py
@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, LargeBinary
+from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, LargeBinary, Float, JSON
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
@@ -12,12 +12,13 @@ class Measurement(Base):
     measurement_datetime = Column(DateTime)
     measurement_description = Column(String, nullable=False)
     measurement_type = Column(String)
-    measurement_data = Column(LargeBinary, nullable=False)
+    #measurement_data = Column(LargeBinary, nullable=False)
+    measurement_data = Column(JSON)
     measurement_unit = Column(String)
 
 
 class BenchmarkMetadata(Base):
     __tablename__ = 'benchmark_metadata'
     uuid = Column(String, primary_key=True)
     meta_description = Column(String)
-    meta_start_time = Column(DateTime)
+    meta_start_time = Column(DateTime)
diff --git a/umlaut/umlaut/metrics/__init__.py b/umlaut/umlaut/metrics/__init__.py
@@ -1,7 +1,8 @@
 from umlaut.metrics.valued_metrics import ConfusionMatrixTracker, \
                                             HyperparameterTracker,\
                                             TTATracker,\
-                                            LossTracker
+                                            LossTracker,\
+                                            TimedTTATracker
 from umlaut.metrics.supervised_metrics import BenchmarkSupervisor,\
                                                 TimeMetric,\
                                                 MemoryMetric,\

diff --git a/umlaut/umlaut/metrics/supervised_metrics.py b/umlaut/umlaut/metrics/supervised_metrics.py
@@ -11,6 +11,7 @@
 import numpy as np
 import psutil
 import pyRAPL
+import json
 
 
 class BenchmarkSupervisor:
@@ -90,7 +91,8 @@ def meanwhile(self, finish_event):
         pass
 
     def serialize(self):
-        return pickle.dumps(self.data)
+        return self.data
+        #return pickle.dumps(self.data)
 
     def log(self, benchmark):
         pass
@@ -155,7 +157,8 @@ def after(self):
         }
 
     def log(self, benchmark):
-        benchmark.log(self.description, self.measure_type, self.serialize(), unit="MiB")
+        json_data = json.dumps(self.serialize(), indent=4, default=str)
+        benchmark.log(self.description, self.measure_type, json_data, unit="MiB")
 
 
 class EnergyMetric(Metric):
@@ -190,7 +193,8 @@ def after(self):
 
     def log(self, benchmark):
         if self.successful:
-            benchmark.log(self.description, self.measure_type, self.serialize(), unit='µJ')
+            json_data = json.dumps(self.serialize(), indent=4, default=str)
+            benchmark.log(self.description, self.measure_type, json_data, unit='µJ')
 
 
 class PowerMetric(Metric):
@@ -247,7 +251,8 @@ def after(self):
 
     def log(self, benchmark):
         if self.successful:
-            benchmark.log(self.description, self.measure_type, self.serialize(), unit='Watt')
+            json_data = json.dumps(self.serialize(), indent=4, default=str)
+            benchmark.log(self.description, self.measure_type, json_data, unit='Watt')
 
 
 class LatencyMetric(Metric):
@@ -269,6 +274,7 @@ def before(self):
 
     def after(self):
         after_time = time.perf_counter()
+        #self.data = str([(after_time - self.before_time) / self.num_entries, self.num_entries])
         self.data = (after_time - self.before_time) / self.num_entries
 
 
@@ -283,7 +289,7 @@ def track(self, num_entries):
         self.num_entries = num_entries
 
     def log(self, benchmark):
-        benchmark.log(self.description, self.measure_type, self.serialize(), unit='Seconds/entry')
+        benchmark.log(str(self.description + "\n (#Entries = " + str(self.num_entries) + ")"), self.measure_type, self.serialize(), unit='Seconds/entry')
 
 
 class ThroughputMetric(Metric):
@@ -356,4 +362,5 @@ def after(self):
         }
 
     def log(self, benchmark):
-        benchmark.log(self.description, self.measure_type, self.serialize(), unit="%")
+        json_data = json.dumps(self.serialize(), indent=4, default=str)
+        benchmark.log(self.description, self.measure_type, json_data, unit="%")
diff --git a/umlaut/umlaut/metrics/valued_metrics.py b/umlaut/umlaut/metrics/valued_metrics.py
@@ -1,4 +1,6 @@
 import pickle
+import json
+import time
 
 import pandas as pd
 
@@ -44,7 +46,9 @@ def serialize(self, matrix, labels):
         pickle object
             Serialized data.
         """
-        return pickle.dumps({'matrix': matrix, 'labels': labels})
+        #return pickle.dumps({'matrix': matrix, 'labels': labels})
+        return json.dumps({'matrix': matrix, 'labels': labels}, indent=4, default=str)
+
 
 class HyperparameterTracker:
     MEASURE_TYPE = "hyperparameters"
@@ -103,12 +107,19 @@ def serialize(self):
         pickle object
             Serialized data.
         """
-        return pickle.dumps({
-            'hyperparameters' : self.hyperparameters,
-            'df' : self.df.to_dict(orient='list'),
-            'target' : self.target,
-            'low_means_good' : self.low_means_good
-        })
+        #return pickle.dumps({
+        #    'hyperparameters' : self.hyperparameters,
+        #    'df' : self.df.to_dict(orient='list'),
+        #    'target' : self.target,
+        #    'low_means_good' : self.low_means_good
+        #})
+        return {
+            'hyperparameters': self.hyperparameters,
+            'df': self.df.to_dict(orient='list'),
+            'target': self.target,
+            'low_means_good': self.low_means_good
+        }
+
 
 class TTATracker:
     MEASURE_TYPE = "tta"
@@ -132,6 +143,9 @@ def track(self, accuracies, description):
             Description of tracked TTA.
         """
         serialized = self.serialize(accuracies)
+        for i in range(len(serialized) - 1):
+            if serialized[i] > serialized[i+1]:
+                serialized[i+1] = serialized[i]
         self.benchmark.log(description, self.MEASURE_TYPE, serialized, unit='accuracy')
 
     def serialize(self, accuracies):
@@ -146,7 +160,10 @@ def serialize(self, accuracies):
         pickle object
             Serialized data.
         """
-        return pickle.dumps(accuracies)
+        #return pickle.dumps(accuracies)
+        accuracies = accuracies.tolist()
+        return accuracies
+
 
 class LossTracker:
     MEASURE_TYPE = "loss"
@@ -184,4 +201,51 @@ def serialize(self, loss_values):
         pickle object
             Serialized data.
         """
-        return pickle.dumps(loss_values)
+        #return pickle.dumps(loss_values)
+        loss_values = loss_values.tolist()
+        return loss_values
+
+
+class TimedTTATracker:
+    MEASURE_TYPE = "timed tta"
+
+    def __init__(self, benchmark, target_acc):
+        """
+        Parameters
+        ----------
+        benchmark : Benchmark
+            Benchmark object to store the data in / retrieve it from.
+        """
+        self.benchmark = benchmark
+        self.target_acc = target_acc
+        self.time = time.perf_counter()
+        self.logged = False
+
+    def track(self, accuracy, description):
+        """
+        Parameters
+        ----------
+        accuracy : int
+            Current accuracy of the run.
+        description : str
+            Description of tracked timed TTA.
+        """
+        if accuracy >= self.target_acc and not self.logged:
+            self.benchmark.log(description + " (Target: " + str(self.target_acc) + ")", self.MEASURE_TYPE, time.perf_counter() - self.time, unit='time for target accuracy')
+            self.logged = True
+
+    def serialize(self, accuracies):
+        """
+        Parameters
+        ----------
+        accuracies : list of ints
+            List of tracked accuracies of the run.
+
+        Returns
+        -------
+        pickle object
+            Serialized data.
+        """
+        #return pickle.dumps(accuracies)
+        accuracies = accuracies.tolist()
+        return accuracies
diff --git a/umlaut/umlaut/visualization.py b/umlaut/umlaut/visualization.py
@@ -2,6 +2,7 @@
 from math import floor
 import pickle
 import sys
+import ast
 
 import matplotlib.pyplot as plt
 import matplotlib.ticker as ticker
@@ -11,6 +12,7 @@
 import plotly.express as px
 import plotly.graph_objects as go
 import seaborn as sns
+import json
 
 class Visualizer:
     def __init__(self, df_from_cli, plotting_backend):
@@ -62,8 +64,9 @@ def plot_with_matplotlib(self):
         figs = []
         for _, row in self.df.iterrows():
             # every row needs to be visualized individually since each row corresponds to one confusion matrix
-            conf_mat_np = row['measurement_data']['matrix']
-            labels = row['measurement_data']['labels']
+            measurement_dict = json.loads(row['measurement_data'])
+            conf_mat_np = json.loads(measurement_dict['matrix'].replace(" ", ","))
+            labels = measurement_dict['labels']
             conf_mat_df = pd.DataFrame(conf_mat_np, 
                                        index=pd.Index(labels, name="predicted"),
                                        columns=pd.Index(labels, name="actual")
@@ -115,10 +118,11 @@ def __init__(self, df_from_cli, plotting_backend):
 
         for _, row in self.df.iterrows():
             measurement_dict = row['measurement_data']
+            measurement_dict = json.loads(measurement_dict)
             timestamps = pd.to_datetime(measurement_dict.pop('timestamps'))
             self.timedelta_lists.append(timestamps - timestamps[0])
-            if len(timestamps) > len(self.x_tick_vals):
-                x_tick_idx = np.floor(np.linspace(0, len(self.timedelta_lists[-1])-1, 5)).astype(int)
+            x_tick_idx = np.floor(np.linspace(0, len(self.timedelta_lists[-1])-1, 5)).astype(int)
+            if len(self.x_tick_vals) == 0 or self.timedelta_lists[-1][x_tick_idx][4] > self.x_tick_vals[4]:
                 self.x_tick_vals = self.timedelta_lists[-1][x_tick_idx]
                 self.x_tick_labels = self.x_tick_vals.map(self._strfdelta)
             self.measurements_lists.append(measurement_dict.pop('measurements'))
@@ -233,6 +237,7 @@ def plot_with_matplotlib(self):
         plt.rcParams.update({'font.size': 18})
         fig, ax = plt.subplots()
         plt.tight_layout()
+        self.df["measurement_data"] = float(self.df["measurement_data"])
         self.df.plot.barh(x='x_labels', y='measurement_data', stacked=False, legend=False, ax=ax)
 
         plt.title(self.title)
@@ -313,6 +318,11 @@ class CPUVisualizer(TimebasedMultiLineChartVisualizer):
     xaxis_label = "Time elapsed since start of pipeline run"
     yaxis_label = "CPU usage in %"
 
+class TimedTTAVisualizer(BarVisualizer):
+    title = "Metric: Time to target accuracy"
+    xaxis_label = "Time elapsed since until target accuracy reached"
+    yaxis_label = "Time taken in seconds"
+
 type_to_visualizer_class_mapper = {
     "throughput" : ThroughputVisualizer,
     "latency" : LatencyVisualizer,
@@ -324,5 +334,6 @@ class CPUVisualizer(TimebasedMultiLineChartVisualizer):
     "tta" : TTAVisualizer,
     "confusion-matrix" : ConfusionMatrixVisualizer,
     "hyperparameters" : HyperparemeterVisualizer,
-    "cpu": CPUVisualizer
+    "cpu": CPUVisualizer,
+    "timed tta": TimedTTAVisualizer
     }