-
Notifications
You must be signed in to change notification settings - Fork 1
/
push_tables_to_production.py
327 lines (271 loc) · 11.9 KB
/
push_tables_to_production.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
## Hi
## Author: Sean Dobbs ([email protected])
import os,sys
from ROOT import TFile,TTree
import rcdb
from optparse import OptionParser
from array import array
import pprint
import datetime
import ccdb
from ccdb import Directory, TypeTable, Assignment, ConstantSet
def LoadCCDB():
sqlite_connect_str = "mysql://[email protected]/ccdb"
#sqlite_connect_str = "sqlite:////scratch/gxproj3/ccdb.sqlite"
#sqlite_connect_str = "sqlite:////group/halld/www/halldweb/html/dist/ccdb.sqlite"
provider = ccdb.AlchemyProvider() # this class has all CCDB manipulation functions
provider.connect(sqlite_connect_str) # use usual connection string to connect to database
#provider.authentication.current_user_name = "gxproj3" # to have a name in logs
provider.authentication.current_user_name = "hdsys" # to have a name in logs
return provider
# assumes both are same table format
def check_out_of_tolerance(table1,table2,tolerance,verbose=0):
result = False
modified_channels = []
max_diff = 0
# I hope these are all non-strings...
for row in xrange(len(table1)):
for col in xrange(len(table1[row])):
#print float(table1[row][col]),float(table2[row][col])
diff = abs(float(table1[row][col])-float(table2[row][col]))
if diff > tolerance:
if diff > max_diff:
max_diff = diff
print "difference: (",str(row),",",str(col),") ",table1[row][col]," ",table2[row][col]
modified_channels.append( ( (row,col), table1[row][col], table2[row][col] ) )
result = True
# we're all good
if verbose>0 and result:
print "max delta = ",max_diff
return (result,modified_channels)
#--------------------------------------------------------
# MakeHEADER_HTML
#
# Return string with HTML for first part of log entry
#--------------------------------------------------------
def MakeHEADER_HTML(RUN,IS_DRY_RUN):
items =[];
items.append(['Time',datetime.datetime.now().strftime('%c')])
html = '<h2>Automated online calibration log entry for run '+ str(RUN) + '</h2>\n'
html += '<p><font size="-1"><i>This entry generated by the script: $Id$</i></font></p>\n'
if IS_DRY_RUN:
html += '<h3>Running in test mode, new calibrations are only stored in "calib" CCDB variation, not pushed to default</h3>'
html += '<center><hr width="80%"></center>\n'
#html += '<table border="0">\n'
#for (key,val) in items: html += ' <TR><TD align="right"><b>'+key+'</b></TD><TD>:</TD><TD>'+val+'</TD></TR>\n'
#html += '</table>\n'
return html
#--------------------------------------------------------
# MakeTOLERANCES_HTML
#--------------------------------------------------------
def MakeTOLERANCES_HTML(tolerance_table, color='black'):
# Open HTML table
html = '<p>\n'
html += '<b>' + "Tolerances" + '</b>\n'
html += '<table><tr><td><div style="border:2px solid '+color+'; background-color:#CCC">\n' # This just gives a border around outside of table
html += '<table border="0" bgcolor="#DDD" cellpadding="5" cellspacing="2">\n' # Actual table
# Add a header
html += '<TR> <TH>Table</TH><TH>Value</TH> </TR>'
for table_name in sorted(tolerance_table.keys()):
html += ' <TR>\n'
html += ' <TD align="right"><font color="black">' + table_name + '</font></TD>\n'
html += ' <TD align="center"><font color="black">' + str(tolerance_table[table_name]) + '</font></TD>\n'
html += ' </TR>\n'
# Close HTML table
html += '</table>\n' # actual table
html += '</div></td></tr></table>\n' # Border around table
html += '</p>\n'
html += '<center><hr width="80%"></center>\n'
return html
#--------------------------------------------------------
# MakeCCDBTABLE_HTML
#--------------------------------------------------------
def MakeCCDBTABLE_HTML(ccdb_table, modified_channels, color='black'):
# Open HTML table
html = '<p>\n'
html += '<b>' + ccdb_table + '</b>\n'
html += '<table><tr><td><div style="border:2px solid '+color+'; background-color:#CCC">\n' # This just gives a border around outside of table
html += '<table border="0" bgcolor="#DDD" cellpadding="5" cellspacing="2">\n' # Actual table
# Add a header
html += '<TR> <TH>Row,Column</TH><TH>Old Value</TH><TH>New Value</TH> </TR>\n'
for ( (row,col),old_val,new_val ) in modified_channels:
html += ' <TR>\n'
html += ' <TD align="center><font color="black">' + str(row) + ', ' + str(col) + '</font></TD>\n'
html += ' <TD align="center"><font color="black">' + str(old_val) + '</font></TD>\n'
html += ' <TD align="center"><font color="black">' + str(new_val) + '</font></TD>\n'
html += ' </TR>\n'
# Close HTML table
html += '</table>\n' # actual table
html += '</div></td></tr></table>\n' # Border around table
html += '</p>\n'
return html
#--------------------------------------------------------
# HTMLBox
#
# Wrap the given text in an HTML box such that the text is
# marked as the preformatted contents of the single cell of
# the table.
#--------------------------------------------------------
def HTMLBox(mylabel, mytext, bgcolor='#EEE'):
# Print label and open HTML table
html = ''
html += '<font size="-1"><i>' + mylabel + '</i></font>\n'
html += '<table><tr><td><div style="border:1px solid black; background-color:'+bgcolor+'">\n' # This just gives a border around outside of table
html += '<table border="0" bgcolor="'+bgcolor+'" cellpadding="5" cellspacing="2">\n' # Actual table
# Print content
html += '<TR><TD><pre>\n' + mytext + '</pre></TD></TR>\n';
# Close HTML table
html += '</table>\n' # actual table
html += '</div></td></tr></table>\n' # Border around table
return html
def main():
pp = pprint.PrettyPrinter(indent=4)
# Defaults
OUTPUT_FILENAME = "out.root"
#RCDB_QUERY = "@is_production and @status_approved"
RCDB_QUERY = "@is_2018production"
SRC_VARIATION = "calib"
DEST_VARIATION = "default"
BEGIN_RUN = 1
END_RUN = 10000000
VERBOSE = 2
DUMMY_RUN = False
# Define command line options
parser = OptionParser(usage = "dump_timeseries.py ccdb_tablename")
#parser.add_option("-T","--table_file", dest="table_file",
# help="File of CCDB tables to copy")
parser.add_option("-R","--run_number", dest="run_number",
help="Run number to process")
parser.add_option("-F","--run_file", dest="run_file",
help="File of runs to look at")
parser.add_option("-b","--begin-run", dest="begin_run",
help="Starting run for output")
parser.add_option("-e","--end-run", dest="end_run",
help="Ending run for output")
parser.add_option("-D","--dest_variation", dest="dest_variation",
help="Desitination CCDB variation to use")
parser.add_option("-S","--src_variation", dest="src_variation",
help="Source CCDB variation to use")
parser.add_option("-Y","--dummy-run", dest="dummy_run", action="store_true",
help="Do everything but actually committing changes to the CCDB")
parser.add_option("-L","--logentry", dest="logentry",
help="File to write elogbook entry.")
(options, args) = parser.parse_args(sys.argv)
if(len(args) < 2):
parser.print_help()
sys.exit(0)
CCDB_TABLES = []
CCDB_TOLERANCES = {}
with open(args[1]) as f:
for line in f:
if len(line.strip()) == 0:
continue
if line.strip()[0] == '#':
continue
try:
tokens = line.strip().split()
CCDB_TABLES.append(tokens[0])
CCDB_TOLERANCES[tokens[0]] = float(tokens[1])
except:
print "Ignoring line: "+line.strip()
if options.dest_variation:
DEST_VARIATION = options.dest_variation
if options.src_variation:
SRC_VARIATION = options.src_variation
if options.begin_run:
BEGIN_RUN = options.begin_run
if options.end_run:
END_RUN = options.end_run
if options.dummy_run:
DUMMY_RUN = True
LOGENTRY = None
if options.logentry:
LOGENTRY = options.logentry
if VERBOSE>0:
print "CCDB_TABLES:"
pp.pprint(CCDB_TABLES)
print "CCDB_TOLERANCES:"
pp.pprint(CCDB_TOLERANCES)
# Load CCDB
ccdb_conn = LoadCCDB()
# get run list
runs = None
if options.run_number:
runs = [ int(options.run_number) ]
elif options.run_file:
runs = []
with open(options.run_file) as f:
for line in f:
try:
runs.append(int(line.strip()))
except:
print "Ignoring line: "+line.strip()
else:
# Load RCDB
rcdb_conn = None
try:
rcdb_conn = rcdb.RCDBProvider("mysql://[email protected]/rcdb")
runs = [ r.number for r in rcdb_conn.select_runs(RCDB_QUERY,BEGIN_RUN,END_RUN) ]
except:
e = sys.exc_info()[0]
print "Could not connect to RCDB: " + str(e)
if runs is None:
print "no runs specified!"
return
if CCDB_TABLES is None or len(CCDB_TABLES)==0:
print "no tables specified!"
return
if LOGENTRY:
elogentry = open(LOGENTRY, "w")
for run in runs:
if LOGENTRY:
print>>elogentry, MakeHEADER_HTML(run,DUMMY_RUN)
print>>elogentry, MakeTOLERANCES_HTML(CCDB_TOLERANCES)
print>>elogentry, "<h2>Modified Tables</h2>\n"
print "===Checking run %d==="%run
for ccdb_table in CCDB_TABLES:
if VERBOSE>1:
print "==checking %s for run %d=="%(ccdb_table,run)
# get source data
assignment = ccdb_conn.get_assignment(ccdb_table, run, SRC_VARIATION)
reference_assignment = ccdb_conn.get_assignment(ccdb_table, run, DEST_VARIATION)
#pp.pprint(assignment.constant_set.data_table)
(success, modified_channels) = check_out_of_tolerance(assignment.constant_set.data_table, reference_assignment.constant_set.data_table, CCDB_TOLERANCES[ccdb_table], verbose=VERBOSE)
if not success:
continue
print "COPYING ",ccdb_table," ..."
if LOGENTRY:
print>>elogentry, MakeCCDBTABLE_HTML(ccdb_table, modified_channels)
if VERBOSE>1:
print "NEW"
pp.pprint(assignment.constant_set.data_table)
print "OLD"
pp.pprint(reference_assignment.constant_set.data_table)
if DUMMY_RUN:
continue
# add to destination
ccdb_conn.create_assignment(
data=assignment.constant_set.data_table,
path=ccdb_table,
variation_name=DEST_VARIATION,
min_run=run,
max_run=ccdb.INFINITE_RUN,
comment="Copied from variation \'%s\'"%ccdb_table)
# HANDLE SOME SPECIAL CASES
if ccdb_table == "/PHOTON_BEAM/RF/time_offset":
ccdb_table = "/PHOTON_BEAM/RF/time_offset_var"
assignment = ccdb_conn.get_assignment(ccdb_table, run, SRC_VARIATION)
ccdb_conn.create_assignment(
data=assignment.constant_set.data_table,
path=ccdb_table,
variation_name=DEST_VARIATION,
min_run=run,
max_run=ccdb.INFINITE_RUN,
comment="Copied from variation \'%s\'"%ccdb_table)
#print "===%d==="%run
#pp.pprint(assignment.constant_set.data_table)
if LOGENTRY:
elogentry.close()
## main function
if __name__ == "__main__":
main()