-
Notifications
You must be signed in to change notification settings - Fork 0
/
plot_median_scaled_data.py
148 lines (114 loc) · 6.4 KB
/
plot_median_scaled_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#%% Import require packages
import h5py
from argparse import ArgumentParser
import matplotlib.pyplot as plt
import os
import pandas as pd
import matplotlib.dates as mdates
import time
import sys
from dateutil.parser import parse
import re
parser = ArgumentParser("create dynspectrum plot from processed data");
parser.add_argument('-r','--rawdata',help='rawdata file',dest="rawfile",type=str,required=True)
parser.add_argument('-p','--processedata',help='processed data folder without trailing frontslash',type=str,dest="processedfolder",required=True)
parser.add_argument('-m','--processedmedian',help='processed median folder without trailing frontslash',type=str,dest="medianfolder",required=True)
parser.add_argument('-i','--vmin', help='vmin of plot',dest='vmin',type=float,default=0.5)
parser.add_argument('-a','--vmax', help='vmax of plot',dest='vmax',type=float,default=2)
parser.add_argument('-c','--cmap', help='matplotlib colormap',dest='cmap',default="viridis")
parser.add_argument('-n','--show_normalization', help='plot normalization',dest='show_normalization', action='store_true')
parser.add_argument('-w','--wait_time', help='wait time',dest='wait_time', default=5)
#https://stackoverflow.com/questions/5967500/how-to-correctly-sort-a-string-with-a-number-inside
def atof(text):
try:
retval = float(text)
except ValueError:
retval = text
return retval
#https://stackoverflow.com/questions/5967500/how-to-correctly-sort-a-string-with-a-number-inside
def natural_keys(text):
'''
alist.sort(key=natural_keys) sorts in human order
http://nedbatchelder.com/blog/200712/human_sorting.html
(See Toothy's implementation in the comments)
float regex comes from https://stackoverflow.com/a/12643073/190597
'''
return [ atof(c) for c in re.split(r'[+-]?([0-9]+(?:[.][0-9]*)?|[.][0-9]+)', text) ]
def get_metadata_from_h5(h5file):
#metadata=h5file.attrs[u'NOF_SUB_ARRAY_POINTINGS']
metadata=dict(h5file[h5file.visit(lambda x: x if 'STOKES' in x else None)].attrs)
metadata['freqs'] = h5file[h5file.visit(lambda x: x if 'COORDINATE_1' in x else None)].attrs[u'AXIS_VALUES_WORLD']
metadata=dict(metadata,**dict(h5file[h5file.visit(lambda x: x if 'BEAM' in x else None)].attrs))
return metadata
def plot_real_time(fig,axarr,processed_data,freqs,vmin,vmax,median_data,maxSamples=10000,skiptime=25,skipch=1,sampleSize=1./125.,cmap='Reds',show_norm=False):
ax=axarr
starttime_dt = parse(processed_data.iloc[0,-1])
endtime_dt = parse(processed_data.iloc[-1,-1])
fig.suptitle(starttime_dt.strftime("%m/%d/%Y"))
if show_norm:
ax=ax[0]
ax.cla()
myextent=[0,processed_data.shape[0],freqs[0]*1e-6,freqs[::skipch][-1]*1e-6]
myextent[0]=mdates.date2num(starttime_dt)
myextent[1]=mdates.date2num(endtime_dt)
ax.imshow((processed_data.iloc[:,:-1]).T,origin='lower',interpolation='nearest',aspect='auto',vmin=vmin,vmax=vmax,extent=myextent,cmap=cmap)
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
ax.set_ylabel("freq (MHz)")
if show_norm:
ax=axarr[1]
ax.cla()
ax.plot(freqs[::skipch]*1e-6,median_data.iloc[-1,:],'k') #plot only last acquired median
ax.set_xlabel("freq (MHz)")
plt.pause(.3)
def main(argv):
args=parser.parse_args(argv)
metadata = get_metadata_from_h5(h5py.File(args.rawfile.replace('.raw','.h5')))
processed_folder = args.processedfolder
median_folder = args.medianfolder
#Initalize list of filenames
scaled_data_filenames = []
median_data_filenames = []
all_processed_data = None
all_median_data = None
fig,axarr=plt.subplots(1+args.show_normalization,1)
while(True):
all_scaled_data_filenames = os.listdir(processed_folder)
all_median_data_filenames = os.listdir(median_folder)
new_scaled_data_filenames = sorted(list(set(all_scaled_data_filenames) - set(scaled_data_filenames)), key=natural_keys)
new_median_data_filenames = sorted(list(set(all_median_data_filenames) - set(median_data_filenames)), key=natural_keys)
scaled_data_filenames = all_scaled_data_filenames
median_data_filenames = all_median_data_filenames
#if new processed data is present
if(len(new_scaled_data_filenames) > 0):
try:
processed_filepaths = [processed_folder+"/"+filename for filename in new_scaled_data_filenames]
new_processed_data = pd.concat(map(pd.read_csv, processed_filepaths), axis=0)
except pd.errors.EmptyDataError:
pass
else:
if(all_processed_data is not None):
processed_data_list = [all_processed_data, new_processed_data]
else:
processed_data_list = [new_processed_data]
all_processed_data = pd.concat(processed_data_list, axis=0)
scaled_data_filenames = all_scaled_data_filenames
#if new median data is present
#Procedure is not needed for real-time plotting, as only the latest median is of interest. Still implemented for possible interactive plotting functionality later on.
if(len(new_median_data_filenames) > 0):
try:
median_filepaths = [median_folder+"/"+filename for filename in new_median_data_filenames]
new_median_data = pd.concat(map(pd.read_csv, median_filepaths), axis=0)
except pd.errors.EmptyDataError:
pass
else:
if(all_median_data is not None):
median_data_list = [all_median_data, new_median_data]
else:
median_data_list = [new_median_data]
all_median_data = pd.concat(median_data_list, axis=0)
median_data_filenames = all_median_data_filenames
if((len(new_median_data_filenames) > 0) | (len(new_scaled_data_filenames) > 0)):
plot_real_time(fig,axarr,all_processed_data, metadata['freqs'],args.vmin,args.vmax,median_data=all_median_data,sampleSize=metadata[u'SAMPLING_TIME'],cmap=args.cmap,show_norm=args.show_normalization)
time.sleep(args.wait_time)
if __name__ == '__main__':
main(sys.argv[1:])