-
Notifications
You must be signed in to change notification settings - Fork 0
/
evaluate_accuracy.py
69 lines (52 loc) · 2.54 KB
/
evaluate_accuracy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
from post_processing import load_pickle
from post_processing import measure_models_accuracy
class EvaluateAccuracy:
competence_measure: str = ''
models: list = []
lags: list = []
metric: str = ''
series: str = ''
training_level: str = ''
def __init__(self, competence_measure, lags, models, metric, series, training_level):
self.competence_measure = competence_measure
self.lags = lags
self.models = models
self.metric = metric
self.series = series
self.training_level = training_level
def generate_performance_accuracy(self):
from pandas import DataFrame
from os import makedirs
from os.path import exists
result = {}
for model in self.models:
result[model] = {}
for lag in self.lags:
pckl = load_pickle(self.series + '/' + self.metric + '/' + self.training_level + '/' + model + str(lag))
if pckl['preprocessor'].testing_y is not None:
pckl['preprocessor'].testing_set = pckl['preprocessor'].testing_y.reshape(-1, 1)
result[model][str(lag)] = measure_models_accuracy(pckl['preprocessor'].testing_set[:, -1],
pckl['pred_testing'], self.competence_measure)
df = DataFrame.from_dict(result, orient='index')
if not exists('results/' + pckl['folder']):
makedirs('results/' + pckl['folder'])
df.to_csv('results/' + pckl['folder'] + self.competence_measure + '.csv')
DataFrame(df.idxmin(axis=1)).transpose().to_csv('results/' + pckl['folder'] + 'lags.csv', index=False)
s = df.min(axis=1)
label = df.idxmin(axis=1)
print('Better ' + self.series + ' '+self.metric+' model: ' + s.idxmin(), label[s.idxmin()])
def generate_costs_accuracy(self):
from pandas import DataFrame
from os import makedirs
from os.path import exists
result = {}
for model in self.models:
result[model] = {}
pckl = load_pickle(self.series + '/' + self.metric + '/' + self.training_level + '/' + model)
result[model]['costs'] = pckl['time_to_fit'] + pckl['time_to_predict']
if not exists('results/' + pckl['folder']):
makedirs('results/' + pckl['folder'])
df = DataFrame.from_dict(result, orient='index').transpose()
df.to_csv('results/' + pckl['folder'] + 'costs.csv')
print('Costs '+self.metric+' '+self.series+':')
print(df.transpose())