Skip to content

Commit

Permalink
Merge pull request #38 from caracal-pipeline/updates
Browse files Browse the repository at this point in the history 
Updates
  • Loading branch information
@Athanaseus
Athanaseus authored Nov 14, 2023
2 parents 3248650 + fcf8ea3 commit 9378677
Show file tree
Hide file tree
Showing 15 changed files with 519 additions and 85 deletions.
23 changes: 22 additions & 1 deletion rfinder/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,7 @@ def go(self,cfg_par):
1: 2d plot of RFI flagged by frequency and baseline lenght (plot_rfi_im)
2: 1d plot of RFI flagged by frequency channel (baselines_from_ms)
3: 1d plot of noise increase by frequency channel (for long and short baselines) (priors_flag)
4: 1d plot of overall RFI flagged by scan, antenna and correlation.
If cfg_par['rfi']['chunks']['time_chunks'] is enabled
1: executes 'rfi' and 'plots' procedure dividing the observation in time-steps given by cfg_par['rfi']['chunks']['time_step']
2: collects the info about the % of RFI for each time step in Alt/Az plots
Expand Down Expand Up @@ -427,8 +428,28 @@ def go(self,cfg_par):
self.logger.warning("------ End of RFI analysis ------\n")

task = 'plots'

if self.cfg_par[task]['plot_summary']['enable']==True:
summary_results = {}
if not self.cfg_par['rfi']['rfi_enable']:
rfi.load_from_ms(self.cfg_par,0,0)

for axis in self.cfg_par[task]['plot_summary']['axis']:
flag_stats = rfiST.get_flags_summary_stats(self.cfg_par, axis)
summary_results[axis] = dict(flag_stats)
self.logger.warning(f" ------ Plotting {axis} summary plots ------\n")
rfiPL.plot_summary_stats(flag_stats, self.cfg_par, axis)
self.logger.info("------ Summary plot done ------\n")

if summary_results:
json_file = cfg_par['general']['rfidir'] + f'summary.json'
with open(json_file, 'w') as f:
json.dump(summary_results, f)

rfiFL.write_html_summaryreport(self.cfg_par)

if self.cfg_par[task]['plot_enable']==True:

if self.cfg_par['rfi']['chunks']['time_enable']==True:

times, start, end = rfiST.time_chunk(self.cfg_par)
Expand Down
28 changes: 23 additions & 5 deletions rfinder/rfi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@ def load_from_ms(self,cfg_par,times=0,counter=0):
Chan_width, Chan_freq
'''
STOKES_TYPES = {
0: "Undefined", 1: "I", 2: "Q", 3: "U", 4: "V",
5: "RR", 6: "RL", 7: "LR", 8: "LL",
9: "XX", 10: "XY", 11: "YX", 12: "YY",
13: "RX", 14: "RY", 15: "LX", 16: "LY",
17: "XR", 18: "XL", 19: "YR", 20: "YL",
21: "PP", 22: "PQ", 23: "QP", 24: "QQ"
}

if counter == 0 :
self.logger.warning("\t ... Field, Antenna & Bandwidth Info ...\n")
Expand All @@ -82,9 +90,11 @@ def load_from_ms(self,cfg_par,times=0,counter=0):

antennas = tables.table(self.msfile +'/ANTENNA')
self.ant_pos = np.array(antennas.getcol('POSITION'))
self.ant_wsrtnames = np.array(antennas.getcol('NAME'))
self.ant_names = np.array(antennas.getcol('NAME'))

#self.ant_wsrtnames = np.array(antennas.getcol('NAME'))

self.ant_names = np.arange(0,self.ant_wsrtnames.shape[0],1)
#self.ant_names = np.arange(0,self.ant_wsrtnames.shape[0],1)
self.nant = len(self.ant_names)

#logging
Expand All @@ -96,11 +106,14 @@ def load_from_ms(self,cfg_par,times=0,counter=0):
self.logger.warning("\tAntenna names:\t\t"+str(self.ant_names))

antennas.close()
self.ant_names = np.arange(0,self.ant_names.shape[0],1)


spw=tables.table(self.msfile+'/SPECTRAL_WINDOW')
self.channelWidths=spw.getcol('CHAN_WIDTH')

self.channelFreqs=spw.getcol('CHAN_FREQ')
cfg_par['rfi']['freqs'] = self.channelFreqs[0]
cfg_par['rfi']['chan_widths'] = self.channelWidths[0][0]
cfg_par['rfi']['lowfreq'] = float(self.channelFreqs[0][0])
cfg_par['rfi']['highfreq'] = float(self.channelFreqs[-1][-1])
Expand All @@ -119,6 +132,12 @@ def load_from_ms(self,cfg_par,times=0,counter=0):

t=tables.table(self.msfile)

if counter == 0 :
cfg_par['rfi']['scans'] = list(set(t.getcol('SCAN_NUMBER')))
corr = tables.table(self.msfile +'/POLARIZATION')
corr_types = corr.getcol('CORR_TYPE')[0]
cfg_par['rfi']['corrs'] = [STOKES_TYPES[corr_type] for corr_type in corr_types]

if counter !=0:
value_end = times[1]
value_start = times[0]
Expand Down Expand Up @@ -169,9 +188,6 @@ def load_from_ms(self,cfg_par,times=0,counter=0):
if cfg_par['rfi']['RFInder_mode'] == 'rms_clip':
self.logger.warning('\t Correct noise_measure_edges in rfi of parameter file ###')
empty_table=1


t.close()

if not self.aperfi_badant:
nrbadant =len(int(self.aperfi_badant))
Expand All @@ -188,11 +204,13 @@ def load_from_ms(self,cfg_par,times=0,counter=0):
rfiST.predict_noise(cfg_par,self.channelWidths,self.interval,self.flag)
cfg_par['rfi']['vis_alltimes_baseline'] = self.flag.shape[0]/nrBaseline

t.close()

self.logger.info("\t ... info from MS file loaded \n\n")

return empty_table


def baselines_from_ms(self,cfg_par):
'''
Reads which baselines were used in the observations
Expand Down
49 changes: 31 additions & 18 deletions rfinder/rfinder_default.yml
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,47 @@
general:
__helpstr: General INPUT directories and filenames
workdir: /home/ramaila/working/PicA/msdir/ #full path to working directory
outdir: results/ #full path to working directory
outlabel: results_3 #label of output directory: rfi_stokes_label
msname: PicA-calibs.ms #name of input ms file
field: 2 #field ID of target of file
telescope: meerkat #telescope of the observations (needed for antenna diameter)
cleanup_enable: true
workdir: /home/username/working/Project1/data/ #full path to working directory
outdir: results/ #full path to working output directory
outlabel: out_0 #label of output directory: rfi_stokes_label
msname: example.ms #name of input ms file
field: 0 #field ID of target of file
cleanup_enable: true #remove intermediate results
ncpu: 4 #number of cpu to get the total flagged data
telescope: #telescope of the observations (needed for antenna diameter)
__helpstr: Telescope of the observations, needed for determining sefd and alt/az
name: meerkat #name of the telescope for auto-completion of the following info. Available choices [meerkat, wsrt, apertif]
diameter: #diameter of the telescope in meters
tsyseff: #effective system temperature in Kelvin
long: #longitude of the telescope
lat: #latitude of the telescope
height: #height/altitude of location

rfi:
__helpstr: Find RFI in msfile over threshold
rfi_enable: true #if True this module is enabled
polarization: 'q' #xx, yy, xy, yx, q (also in CAPS)
bad_antenna: '[]' #list of bad antennas: for no antennas try '[]'
RFInder_mode: 'use_flags' #uses RFI clip method or simply reads the FLAG column 'use_flags' / 'rms_clip'
rms_clip: 5 #Threshold for RFI identification
noise_measure_edges: [1.400e9, 1.420e9] #min frequency and maximum frequency where to measure average std of visibilities
baseline_cut: 2000 #Cutoff baseline lenght: < = Short, > = Long (615m midpoint of baselines of WSRT)
rfi_enable: true #if True this module is enabled
polarization: 'q' #xx, yy, xy, yx, q (also in CAPS)
bad_antenna: '[]' #list of bad antennas: for no antennas try '[]'
RFInder_mode: 'use_flags' #uses RFI clip method or simply reads the FLAG column 'use_flags' / 'rms_clip'
rms_clip: 5 #Threshold for RFI identification
noise_measure_edges: [1.400e9, 1.420e9] #min frequency and maximum frequency where to measure average std of visibilities
baseline_cut: 200 #Cutoff baseline lenght: < = Short, > = Long (615m midpoint of baselines of WSRT)
chunks:
__helpstr: separate visibilities in small time intervals and/or larger spectral windows
time_enable: true
time_step: 60 #time chunks in minutes
time_step: 60 #time chunks in minutes
spw_enable: true
spw_width: 200 #channel width of rebinned output table [MHz]
spw_width: 200 #channel width of rebinned output table [MHz]

plots:
__helpstr: plot rfi over different axes
plot_enable: true
plot_noise: 'rfi' #choose to plot % of RFI, 'rfi', or noise, 'noise', or factor of noise increase, 'noise_factor' in 1D plots
plot_long_short: true #plot all baselines, or only the short and the long
plot_noise: 'rfi' #choose to plot % of RFI, 'rfi', or noise, 'noise', or factor of noise increase, 'noise_factor' in 1D plots
plot_long_short: true #plot all baselines, or only the short and the long
plot_summary:
__helpstr: plot total % flagged visibilities
enable: true
axis: [ant, scan, corr] # Avalable choices [ant, corr, scan]
report: true
movies:
__helpstr: make movies of output plots
2d_gif: true
Expand Down
76 changes: 72 additions & 4 deletions rfinder/rfinder_files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,7 @@ def write_html_fullreport(cfg_par):
polnum = cfg_par['rfi']['polnum'],
noise = np.round(cfg_par['rfi']['theo_rms'][0]*1e3,5),
img_tag1 = '<img class="a" src="data:image/png;base64,{0}">'.format(data_uri1),
video_tag1 = '<img class="b" src="data:image/gif;base64,{0}">'.format(video_encoded1.decode()),
video_tag1 = '<img class="g" src="data:image/gif;base64,{0}">'.format(video_encoded1.decode()),
img_tag3 = '<img class="c" src="data:image/png;base64,{0}">'.format(data_uri3),
#video_tag1 = '<img class="d" src="data:video/gif;base64,{0}">'.format(video_encoded1),
#video_tag1 = '<img class="d" src="data:video/gif;base64,{0}">'.format(video_encoded1),
Expand Down Expand Up @@ -348,7 +348,7 @@ def write_html_fullreport(cfg_par):
polnum = cfg_par['rfi']['polnum'],
noise = np.round(cfg_par['rfi']['theo_rms'][0]*1e3,5),
img_tag1 = '<img class="a" src="data:image/png;base64,{0}">'.format(data_uri1),
video_tag1 = '<img class="b" src="data:image/gif;base64,{0}">'.format(data_uri2),
video_tag1 = '<img class="g" src="data:image/gif;base64,{0}">'.format(data_uri2),
img_tag3 = '<img class="c" src="data:image/png;base64,{0}">'.format(data_uri3),
))

Expand Down Expand Up @@ -411,9 +411,10 @@ def write_html_timereport(cfg_par):
exptime = np.round(cfg_par['rfi']['exptime']*60.,2),
polnum = cfg_par['rfi']['polnum'],
noise = np.round(cfg_par['rfi']['theo_rms'][0]*1e3,5),
video_tag1 = '<img class="b" src="data:image/gif;base64,{0}">'.format(video_encoded1.decode()),
video_tag1 = '<img class="h" src="data:image/gif;base64,{0}">'.format(video_encoded1.decode()),
video_tag2 = '<img class="a" src="data:image/gif;base64,{0}">'.format(video_encoded2.decode()),
video_tag3 = '<img class="c" src="data:image/gif;base64,{0}">'.format(video_encoded3.decode())
video_tag3 = '<img class="c" src="data:image/gif;base64,{0}">'.format(video_encoded3.decode()),
#video_tag4 = '<img class="g" src="data:image/gif;base64,{0}">'.format(video_encoded3.decode())
))

elif cfg_par['plots']['movies']['movies_in_report'] == False:
Expand All @@ -426,6 +427,73 @@ def write_html_timereport(cfg_par):

return 0


def write_html_summaryreport(cfg_par):

# Configure Jinja and ready the template
env = Environment(
loader=FileSystemLoader(cfg_par['general']['template_folder'])
)
template = env.get_template('summary_template.html')

#base_template = env.get_template('report.html')
# Content to be published
title = 'RFI summary report: {0:s}'.format(cfg_par['general']['msname'])

#imagename1 = '/Users/maccagni/Projects/RFI/rfinder_test/rfi/plots/altaz/AltAZ_rfi1297-1317MHz.png'
#data_uri1 = open(imagename1, 'rb').read().encode('base64').replace('\n', '')
image_name1 = cfg_par['general']['plotdir'] + 'ant-summary.png'
if os.path.exists(image_name1):
#video_encoded1 = open(video_name1, "rb").read().encode("base64")
image_encoded1 = base64.b64encode(open(image_name1, "rb").read())
else:
image_encoded1 = None

image_name2 = cfg_par['general']['plotdir'] + 'scan-summary.png'
if os.path.exists(image_name2):
image_encoded2 = base64.b64encode(open(image_name2, "rb").read())
else:
image_encoded2 = None

image_name3 = cfg_par['general']['plotdir'] + 'corr-summary.png'
if os.path.exists(image_name3):
image_encoded3 = base64.b64encode(open(image_name3, "rb").read())
else:
image_encoded3 = None

if cfg_par['plots']['plot_summary']['report'] == True:

with open(cfg_par['general']['rfidir']+'summary_report.html', "w") as f:
# lenghts = np.array([cfg_par['rfi']['baseline_lenghts']])+0.
f.write(template.render(
title=title,
fieldname=cfg_par['general']['fieldname'],
field=cfg_par['general']['field'],
totchans = int(cfg_par['rfi']['total_channels']),
chan_widths=round(cfg_par['rfi']['chan_widths']/1e3,4),
lowfreq=round(cfg_par['rfi']['lowfreq']/1e6,3),
highfreq=round(cfg_par['rfi']['highfreq']/1e6,3),
startdate = ('{0:%y}{0:%b}{0:%d} {0:%X}'.format(cfg_par['rfi']['startdate'].datetime)),
enddate = ('{0:%y}{0:%b}{0:%d} {0:%X}'.format(cfg_par['rfi']['enddate'].datetime)),
nant = cfg_par['rfi']['nant'],
ant_names = cfg_par['rfi']['ant_names'],
# maxbase = str(np.round(lenghts[0][-1],0)),
# minbase = str(np.round(lenghts[0][0],0)),
# totbase = cfg_par['rfi']['number_baseline'],
exptime = np.round(cfg_par['rfi']['exptime']*60.,2),
polnum = cfg_par['rfi']['polnum'],
noise = np.round(cfg_par['rfi']['theo_rms'][0]*1e3,5),
video_tag1 = '<img class="h" src="data:image/png;base64,{0}">'.format(image_encoded1.decode()),
video_tag2 = '<img class="a" src="data:image/png;base64,{0}">'.format(image_encoded2.decode()),
video_tag3 = '<img class="c" src="data:image/png;base64,{0}">'.format(image_encoded3.decode()),
#video_tag4 = '<img class="g" src="data:image/png;base64,{0}">'.format(image_encoded3.decode())
))


logger.info('---- Html summary report done ----\n')

return 0

def find_altaz_plots(cfg_par):

tmp_arr=[]
Expand Down
Loading

0 comments on commit 9378677

Please sign in to comment.