'.format(data_uri1),
- video_tag1 = '
'.format(video_encoded1.decode()),
+ video_tag1 = ''.format(video_encoded1.decode()),
img_tag3 = ''.format(data_uri3),
#video_tag1 = ''.format(data_uri1),
- video_tag1 = ''.format(data_uri2),
+ video_tag1 = ''.format(data_uri2),
img_tag3 = ''.format(data_uri3),
))
@@ -411,9 +411,10 @@ def write_html_timereport(cfg_par):
exptime = np.round(cfg_par['rfi']['exptime']*60.,2),
polnum = cfg_par['rfi']['polnum'],
noise = np.round(cfg_par['rfi']['theo_rms'][0]*1e3,5),
- video_tag1 = ''.format(video_encoded1.decode()),
+ video_tag1 = ''.format(video_encoded1.decode()),
video_tag2 = ''.format(video_encoded2.decode()),
- video_tag3 = ''.format(video_encoded3.decode())
+ video_tag3 = ''.format(video_encoded3.decode()),
+ #video_tag4 = ''.format(video_encoded3.decode())
))
elif cfg_par['plots']['movies']['movies_in_report'] == False:
@@ -426,6 +427,73 @@ def write_html_timereport(cfg_par):
return 0
+
+def write_html_summaryreport(cfg_par):
+
+ # Configure Jinja and ready the template
+ env = Environment(
+ loader=FileSystemLoader(cfg_par['general']['template_folder'])
+ )
+ template = env.get_template('summary_template.html')
+
+ #base_template = env.get_template('report.html')
+ # Content to be published
+ title = 'RFI summary report: {0:s}'.format(cfg_par['general']['msname'])
+
+ #imagename1 = '/Users/maccagni/Projects/RFI/rfinder_test/rfi/plots/altaz/AltAZ_rfi1297-1317MHz.png'
+ #data_uri1 = open(imagename1, 'rb').read().encode('base64').replace('\n', '')
+ image_name1 = cfg_par['general']['plotdir'] + 'ant-summary.png'
+ if os.path.exists(image_name1):
+ #video_encoded1 = open(video_name1, "rb").read().encode("base64")
+ image_encoded1 = base64.b64encode(open(image_name1, "rb").read())
+ else:
+ image_encoded1 = None
+
+ image_name2 = cfg_par['general']['plotdir'] + 'scan-summary.png'
+ if os.path.exists(image_name2):
+ image_encoded2 = base64.b64encode(open(image_name2, "rb").read())
+ else:
+ image_encoded2 = None
+
+ image_name3 = cfg_par['general']['plotdir'] + 'corr-summary.png'
+ if os.path.exists(image_name3):
+ image_encoded3 = base64.b64encode(open(image_name3, "rb").read())
+ else:
+ image_encoded3 = None
+
+ if cfg_par['plots']['plot_summary']['report'] == True:
+
+ with open(cfg_par['general']['rfidir']+'summary_report.html', "w") as f:
+# lenghts = np.array([cfg_par['rfi']['baseline_lenghts']])+0.
+ f.write(template.render(
+ title=title,
+ fieldname=cfg_par['general']['fieldname'],
+ field=cfg_par['general']['field'],
+ totchans = int(cfg_par['rfi']['total_channels']),
+ chan_widths=round(cfg_par['rfi']['chan_widths']/1e3,4),
+ lowfreq=round(cfg_par['rfi']['lowfreq']/1e6,3),
+ highfreq=round(cfg_par['rfi']['highfreq']/1e6,3),
+ startdate = ('{0:%y}{0:%b}{0:%d} {0:%X}'.format(cfg_par['rfi']['startdate'].datetime)),
+ enddate = ('{0:%y}{0:%b}{0:%d} {0:%X}'.format(cfg_par['rfi']['enddate'].datetime)),
+ nant = cfg_par['rfi']['nant'],
+ ant_names = cfg_par['rfi']['ant_names'],
+ # maxbase = str(np.round(lenghts[0][-1],0)),
+ # minbase = str(np.round(lenghts[0][0],0)),
+ # totbase = cfg_par['rfi']['number_baseline'],
+ exptime = np.round(cfg_par['rfi']['exptime']*60.,2),
+ polnum = cfg_par['rfi']['polnum'],
+ noise = np.round(cfg_par['rfi']['theo_rms'][0]*1e3,5),
+ video_tag1 = ''.format(image_encoded1.decode()),
+ video_tag2 = ''.format(image_encoded2.decode()),
+ video_tag3 = ''.format(image_encoded3.decode()),
+ #video_tag4 = ''.format(image_encoded3.decode())
+ ))
+
+
+ logger.info('---- Html summary report done ----\n')
+
+ return 0
+
def find_altaz_plots(cfg_par):
tmp_arr=[]
diff --git a/rfinder/rfinder_plots.py b/rfinder/rfinder_plots.py
index 5e1142b..cb8f821 100644
--- a/rfinder/rfinder_plots.py
+++ b/rfinder/rfinder_plots.py
@@ -136,7 +136,7 @@ def plot_rfi_imshow(self,cfg_par,time_step=-1):
idx = (np.abs(freqs_plot_bin[i]-freqs)).argmin()
freqs_plot_idx[i]=idx
- tele= cfg_par['general']['telescope']
+ tele= cfg_par['general']['telescope']['name']
if tele == 'meerkat' or tele == 'MeerKAT' or tele == 'meerKAT' or tele == 'meer':
input_baselines = np.zeros(6)
for i in range (0,len(input_baselines)):
@@ -201,7 +201,7 @@ def plot_rfi_imshow(self,cfg_par,time_step=-1):
#box_y = [left_h, bottom, 0.12, height]
#box_cbar = [left_h+0.13, bottom, 0.05, height]
- fig = plt.figure(1, figsize=(12,8))
+ fig = plt.figure(1, figsize=(25,10))
ax = plt.axes(box_centre)
@@ -313,13 +313,13 @@ def plot_noise_frequency(self,cfg_par,time_step=-1):
#open file
if os.path.exists(rfi_table) == False:
+ print(rfi_table)
self.logger.error('### Table of RFI results does not exist ###')
else:
t = fits.open(rfi_table)
data_vec = t[1].data
cols = t[1].columns
-
freqs = np.array(data_vec['frequency'],dtype=float)
flags = np.array(data_vec['percentage_flags'],dtype=float)
noise_factor = np.array(data_vec['noise_factor'],dtype=float)
@@ -390,7 +390,7 @@ def plot_noise_frequency(self,cfg_par,time_step=-1):
box_centre = [left, bottom, width, height]
# initialize figure
- fig = plt.figure(1, figsize=(12,8))
+ fig = plt.figure(1, figsize=(18,8))
plt.rc('xtick', labelsize=20)
# Initialize subplots
@@ -520,6 +520,7 @@ def plot_altaz(self,cfg_par,number_chunks):
self.logger.info("\t ... Plotting Alt/Az for binned dataset ... \n")
if os.path.exists(cfg_par['general']['timetabledir']) == False:
+ print(cfg_par['general']['timetabledir'])
self.logger.error("\t Folder with time subsets missing")
table_tmp = str.split(cfg_par['general']['msname'][0],'.MS')
@@ -564,6 +565,7 @@ def plot_altaz(self,cfg_par,number_chunks):
#open file
if os.path.exists(rfi_table) == False:
+ print(rfi_table)
self.logger.error('### Table of RFI results does not exist ###')
continue
else:
@@ -879,7 +881,60 @@ def plot_altaz_short(self,cfg_par):
self.logger.info(('\t ... ALT/AZ for {0:s} ...\n').format(cfg_par['general']['fieldname']))
return 0
-
+
+ def plot_summary_stats(self, flag_stats, cfg_par, key='corr'):
+
+ summaryplot = cfg_par['general']['plotdir'] + f'{key}-summary.png'
+
+ # initialize plotting parameters
+ params = {'font.family' :' serif',
+ 'font.style' : 'normal',
+ 'font.weight' : 'book',
+ 'font.size' : 20.0,
+ 'axes.linewidth' : 1,
+ 'lines.linewidth' : 1,
+ 'xtick.labelsize' : 16,
+ 'ytick.labelsize' : 16,
+ 'xtick.direction' :'in',
+ 'ytick.direction' :'in',
+ 'xtick.major.size' : 4,
+ 'xtick.major.width' : 1,
+ 'xtick.minor.size' : 2,
+ 'xtick.minor.width' : 1,
+ 'ytick.major.size' : 4,
+ 'ytick.major.width' : 1,
+ 'ytick.minor.size' : 2,
+ 'ytick.minor.width' : 1,
+ 'text.usetex' : False,
+ }
+ plt.rcParams.update(params)
+ fig, ax = plt.subplots(figsize=(16,9))
+
+ plt.ylim(0, 100.0)
+ antenna_plot = ax.bar(flag_stats.keys(), flag_stats.values(), color="orange", ec="red", align='center')
+
+ if key in ["antenna", "ant"]:
+ plt.title("Antenna flags")
+ plt.xlabel("Antenna")
+ plt.ylabel("% flagged visibilities")
+ plt.xticks(rotation=90)
+ plt.savefig(summaryplot)
+ elif key in ["scan"]:
+ plt.title("Scan flags")
+ plt.xlabel("Scan Number")
+ plt.ylabel("% flagged visibilities")
+ plt.savefig(summaryplot)
+ elif key in ["correlation", "corr"]:
+ plt.title("Correlation flags")
+ plt.xlabel("Correlation")
+ plt.ylabel("Flagged percentage (%)")
+ plt.ylabel("% flagged visibilities")
+ plt.savefig(summaryplot)
+ self.logger.info(f" ------ Saving: {summaryplot} ------\n")
+ plt.close()
+
+ return 0
+
def gif_me_up(self,cfg_par,filenames,outmovie):
@@ -892,7 +947,7 @@ def gif_me_up(self,cfg_par,filenames,outmovie):
for filename in filenames:
images.append(io.imread(filename))
- io.mimsave(outmovie,images,duration=1.5)
+ io.mimsave(outmovie,images,duration=1000,loop=1000)
self.logger.info(('\t ... Movie written to file ...\n'))
diff --git a/rfinder/rfinder_stats.py b/rfinder/rfinder_stats.py
index 456ba69..ee87ef3 100644
--- a/rfinder/rfinder_stats.py
+++ b/rfinder/rfinder_stats.py
@@ -8,6 +8,7 @@
import glob
import casacore.tables as tables
import logging
+import multiprocessing
from astropy.time import Time
@@ -151,7 +152,7 @@ def predict_noise(self,cfg_par,channelWidths,interval,flag):
polnum = 2
cfg_par['rfi']['polnum'] = polnum
- tele= cfg_par['general']['telescope']
+ tele= cfg_par['general']['telescope']['name']
if tele == 'meerkat' or tele == 'MeerKAT' or tele == 'meerKAT' or tele == 'meer':
tsyseff = 30. # Tsys/eff(K)
@@ -160,9 +161,9 @@ def predict_noise(self,cfg_par,channelWidths,interval,flag):
tsyseff = 93.
diam = 25.
else:
- self.logger.error('\t Telescope not known or not specified, will use WSRT instead')
- tsyseff = 93.
- diam = 25.
+ self.logger.warning(f'\t Telescope {tele} not known or not specified, will use provided telescope info')
+ tsyseff = cfg_par['general']['telescope']['tsyseff']
+ diam = cfg_par['general']['telescope']['diameter']
Aant=np.pi*(diam/2)**2 # collecting area of 1 antenna (m^2)
SEFD=2*kB*tsyseff/Aant # frequency independent system equivalent flux density (Jy)
@@ -200,7 +201,7 @@ def alt_az(self,cfg_par,time):
self.logger.info("\t ... Altitude/Azimuth info ... \n")
#open miriad observation
- tele= cfg_par['general']['telescope']
+ tele= cfg_par['general']['telescope']['name']
coord = cfg_par['rfi']['coords']
if tele == 'meerkat' or tele == 'MeerKAT' or tele == 'meerKAT' or tele == 'meer':
@@ -209,7 +210,11 @@ def alt_az(self,cfg_par,time):
# Set position of Westerbork (source: http://www.sonel.org/spip.php?page=gps&idStation=884)
telescope = EarthLocation(lat= 52.91460037*u.deg, lon=6.60449982*u.deg, height=82.2786*u.m)
else:
- self.logger.error('\t Observing location unknown, impossible to estimate Alt/Az')
+ self.logger.warning('\t Observing location unknown, will use provided info to estimate Alt/Az')
+ lon = cfg_par['general']['telescope']['lat']*u.deg
+ lat = cfg_par['general']['telescope']['long']*u.deg
+ height = cfg_par['general']['telescope']['height']*u.m
+ telescope = EarthLocation(lat=lat, lon=lon, height=height)
time = Time(time/3600./24.,format='mjd',scale='utc')
frame = AltAz(obstime=time, location=telescope)
@@ -223,3 +228,99 @@ def alt_az(self,cfg_par,time):
##self.logger.info("\t ... Alt/Az done ... \n")
return obs_altaz
+
+
+ def get_flags_summary_stats(self, cfg_par, axis):
+ """Get total flagged visibilities per given axis value"""
+
+ def data_query(t, taql, name, axis='ant'):
+ """Query table for flagged data """
+ flagtab = t.query(query=taql, columns='DATA_DESC_ID,FLAG')
+ cell_shape = flagtab.getcell('FLAG', 0).shape
+ flag_col = np.empty((flagtab.nrows(), cell_shape[0], cell_shape[1]), dtype=bool)
+ flagtab.getcolnp('FLAG', flag_col)
+ ddid_col = flagtab.getcol('DATA_DESC_ID')
+ flagtab.done()
+
+ if axis in ['corr']:
+ # For this we need to index that data appropriately
+ vals,counts = np.unique(flag_col[:,:,name],return_counts=True)
+ name = cfg_par['rfi']['corrs'][name]
+ else:
+ vals,counts = np.unique(flag_col,return_counts=True)
+ name = str(name)
+
+ if len(vals) == 1:
+ flag_percent = 100.0 if vals[0] else 0.0
+ else:
+ flag_percent = round(100.0*float(counts[1])/float(np.sum(counts)),2)
+
+ self.flag_percents[name] = flag_percent
+
+ def flag_bars(flag_stats, key):
+ """Displays output directly to the screen console or logfile
+ https://github.com/IanHeywood/ms_info/blob/master/ms_flags.py
+ """
+ self.logger.info('')
+ self.logger.info(f'Flagged percentages per {key}:')
+ self.logger.info('')
+ self.logger.info(' 0% 20% 40% 60% 80% 100%')
+ self.logger.info(' | | | | | |')
+ for fs in flag_stats.items():
+ name = fs[0]
+ average_pc = fs[1]
+ length = int(average_pc / 2.0)
+ self.logger.info(' %-9s %-7s %s'% (name,str(round(average_pc,1))+'%','∎' * length))
+ self.logger.info('')
+
+ ncpu = cfg_par['general']['ncpu']
+ self.logger.info("\t ... Observing time Info ... \n")
+ self.msfile = cfg_par['general']['msfullpath']
+ t=tables.table(self.msfile)
+ t = t.query(query=f"FIELD_ID=={cfg_par['general']['field']}")
+ self.flag_percents = multiprocessing.Manager().dict()
+ processes = []
+ if axis in ['ant', 'antenna']:
+ for index, ant_name in enumerate(cfg_par['rfi']['ant_names']):
+ taql = f"ANTENNA1=={index} || ANTENNA2=={index}"
+ p = multiprocessing.Process(target=data_query, args=(t, taql, ant_name))
+ p.start()
+ processes.append(p)
+ if len(processes) == ncpu:
+ for p in processes:
+ p.join()
+ processes = []
+ if len(processes) > 1:
+ for p in processes:
+ p.join()
+ if axis in ['scan']:
+ scan_ids = list(set(t.getcol('SCAN_NUMBER')))
+ for index, scan_id in enumerate(scan_ids):
+ taql = f'SCAN_NUMBER=={str(scan_id)}'
+ p = multiprocessing.Process(target=data_query, args=(t, taql, scan_id))
+ p.start()
+ processes.append(p)
+ if len(processes) == ncpu:
+ for p in processes:
+ p.join()
+ processes = []
+ if len(processes) > 1:
+ for p in processes:
+ p.join()
+ if axis in ['corr']:
+ for index, corr_type in enumerate(cfg_par['rfi']['corrs']):
+ taql = f''
+ p = multiprocessing.Process(target=data_query, args=(t, taql, index, axis))
+ p.start()
+ processes.append(p)
+ if len(processes) == ncpu:
+ for p in processes:
+ p.join()
+ processes = []
+ if len(processes) > 1:
+ for p in processes:
+ p.join()
+ t.close()
+ flag_bars(self.flag_percents, axis)
+
+ return self.flag_percents
diff --git a/rfinder/templates/full_template.html b/rfinder/templates/full_template.html
index 7d2c446..6aa19c3 100644
--- a/rfinder/templates/full_template.html
+++ b/rfinder/templates/full_template.html
@@ -10,7 +10,7 @@
img.a{
position:absolute;
top: 75px;
- left: 580px;
+ left: 520px;
height:400px;
width: 600px;
font-family: Veranda;
@@ -26,7 +26,7 @@
img.c{
position:absolute;
top: 525px;
- left: 580px;
+ left: 520px;
height:372px;
width: 585px;
font-family: Veranda;
@@ -55,10 +55,31 @@
width: 580px;
font-family: Veranda;
}
+ img.g{
+ position:absolute;
+ top: 525px;
+ left: 1120px;
+ height:372px;
+ width: 585px;
+ font-family: Veranda;
+ }
+ img.h{
+ position:absolute;
+ top: 75px;
+ left: 1120px;
+ height:400px;
+ width: 600px;
+ font-family: Veranda;
+ }
+ p{
+ font-family: monospace;
+ width: 500px;
+ font-size: 14px;
+ }
Field: {{fieldname}} ( ID = {{field}})
Start Date =
End Date =
diff --git a/rfinder/templates/fullshort_template.html b/rfinder/templates/fullshort_template.html
index b80a706..08684f1 100644
--- a/rfinder/templates/fullshort_template.html
+++ b/rfinder/templates/fullshort_template.html
@@ -10,7 +10,7 @@
img.a{
position:absolute;
top: 75px;
- left: 580px;
+ left: 520px;
height:400px;
width: 600px;
font-family: Veranda;
@@ -26,7 +26,7 @@
img.c{
position:absolute;
top: 525px;
- left: 580px;
+ left: 520px;
height:372px;
width: 585px;
font-family: Veranda;
@@ -55,10 +55,31 @@
width: 580px;
font-family: Veranda;
}
+ img.g{
+ position:absolute;
+ top: 525px;
+ left: 1120px;
+ height:372px;
+ width: 585px;
+ font-family: Veranda;
+ }
+ img.h{
+ position:absolute;
+ top: 75px;
+ left: 1120px;
+ height:400px;
+ width: 600px;
+ font-family: Veranda;
+ }
+ p{
+ font-family: monospace;
+ width: 500px;
+ font-size: 14px;
+ }
Field: {{fieldname}} ( ID = {{field}})
Start Date =
End Date =
diff --git a/rfinder/templates/summary_template.html b/rfinder/templates/summary_template.html
new file mode 100644
index 0000000..4689119
--- /dev/null
+++ b/rfinder/templates/summary_template.html
@@ -0,0 +1,87 @@
+
+
+
Field: {{fieldname}} ( ID = {{field}})
+Start Date =
+End Date =
+ Spectral window
+Number of channels =
+Channel Width [kHz]=
+Start Frequency [GHz]=
+End Frequency [GHz]=
+ Antennas
+Number of Antennas =
+Antenna Names =
+ Natural noise predictions
+Time scan on source = {{exptime}} minutes for {{polnum}} polarizations
+Stokes I natural r.m.s. [mJy/b]= {{noise}}
+{{video_tag1}} {{video_tag3}} {{video_tag4}}
+
+RFInder automatically generated this report.
+
Field: {{fieldname}} ( ID = {{field}})
Start Date =
End Date =
@@ -53,9 +82,9 @@