Summary for a single antenna, and some report clean ups

rfinder/main.py

@@ -444,7 +444,7 @@ def go(self,cfg_par):
                 self.logger.info("------ Summary plot done ------\n")
 
             if summary_results:
-                self.logger.warning(f'------ Total % Flagged: {sum(summary_results[axis].values())/len(summary_results[axis].values())}% ------')
+                self.logger.warning(f'------ Total % Flagged: {round(sum(summary_results[axis].values())/len(summary_results[axis].values()),2)} ------')
                 json_file = cfg_par['general']['rfidir'] + f'summary.json'
                 with open(json_file, 'w') as f:
                     json.dump(summary_results, f)

rfinder/rfinder_default.yml

@@ -49,5 +49,5 @@ plots:
         __helpstr: plot total % flagged visibilities
         enable: true
         axis: [ant, scan, corr, freq]                  #avalable choices [ant, corr, scan, freq]
-        antenna: m001                                  #select an antenna to get summary
-        report: false
+        antenna:                                       #select an antenna to get summary
+        report: true                                   #generate html report with the results

rfinder/rfinder_files.py

@@ -461,6 +461,12 @@ def write_html_summaryreport(cfg_par):
     else:
         image_encoded3 = None
 
+    image_name4 = cfg_par['general']['plotdir'] + 'freq-summary.png'
+    if os.path.exists(image_name4):
+        image_encoded4 = base64.b64encode(open(image_name4, "rb").read())
+    else:
+        image_encoded4 = None
+
     if cfg_par['plots']['plot_summary']['report'] == True:
 
         with open(cfg_par['general']['rfidir']+'summary_report.html', "w") as f:
@@ -486,7 +492,7 @@ def write_html_summaryreport(cfg_par):
                 video_tag1 = '<img class="h" src="data:image/png;base64,{0}">'.format(image_encoded1.decode()),
                 video_tag2 = '<img class="a" src="data:image/png;base64,{0}">'.format(image_encoded2.decode()),
                 video_tag3 = '<img class="c" src="data:image/png;base64,{0}">'.format(image_encoded3.decode()),
-                #video_tag4 = '<img class="g" src="data:image/png;base64,{0}">'.format(image_encoded3.decode())
+                video_tag4 = '<img class="g" src="data:image/png;base64,{0}">'.format(image_encoded4.decode())
             ))
 
 

rfinder/rfinder_plots.py

@@ -329,21 +329,21 @@ def plot_noise_frequency(self,cfg_par,time_step=-1):
             flags_short = np.array(data_vec['percentage_flags_short'],dtype=float)
 
 
-            if cfg_par['plots']['plot_noise'] == 'noise':
+            if cfg_par['plots']['plot_detail']['plot_noise'] == 'noise':
                 rms = np.array(cfg_par['rfi']['theo_rms']*1e3,dtype=float)
                 noise_all = noise_factor*rms
                 noise_short = noise_factor_short*rms
                 noise_long = noise_factor_long*rms
                 out_plot = plotdir+'noise_'+time_name
 
-            if cfg_par['plots']['plot_noise'] == 'noise_factor':
+            if cfg_par['plots']['plot_detail']['plot_noise'] == 'noise_factor':
                 self.logger.info("\t ... Plotting factor of noise increas per frequency channel ...")
                 noise_all = noise_factor
                 noise_short = noise_factor_short
                 noise_long = noise_factor_long    
                 out_plot = plotdir+'noisefactor_'+time_name
 
-            if cfg_par['plots']['plot_noise'] == 'rfi':
+            if cfg_par['plots']['plot_detail']['plot_noise'] == 'rfi':
                 self.logger.info("\t ... Plotting percentage of flagged RFI per frequency channel ...")
                 noise_all = flags
                 noise_long = flags_long
@@ -423,7 +423,7 @@ def plot_noise_frequency(self,cfg_par,time_step=-1):
             ax1.set_xlim([(cfg_par['rfi']['lowfreq']-5.*cfg_par['rfi']['chan_widths'])/1e6,
                 (cfg_par['rfi']['highfreq']+5.*cfg_par['rfi']['chan_widths'])/1e6])
 
-            if cfg_par['plots']['plot_noise'] != 'rfi':
+            if cfg_par['plots']['plot_detail']['plot_noise'] != 'rfi':
                 ax1.set_yscale('log', base=10)
                 if np.isnan(np.sum(noise_all)):
                     noise_all=np.zeros([len(freqs)])+100.
@@ -439,7 +439,7 @@ def plot_noise_frequency(self,cfg_par,time_step=-1):
             label_long = r'Baselines $>$ '+str(cfg_par['rfi']['baseline_cut'])+' m'
             label_short = r'Baselines $<$ '+str(cfg_par['rfi']['baseline_cut'])+' m' 
 
-            if cfg_par['plots']['plot_long_short'] == True:
+            if cfg_par['plots']['plot_detail']['plot_long_short'] == True:
                 self.logger.info("\t ... Plotting RFI in long and short baselines ...")
                 ax1.step(freqs,noise_short, where= 'pre', color='red', linestyle='-',label=label_short)
                 ax1.step(freqs,noise_long, where= 'pre', color='blue', linestyle='-',label=label_long)
@@ -457,18 +457,18 @@ def plot_noise_frequency(self,cfg_par,time_step=-1):
             #xticks_num = np.linspace(cfg_par['rfi']['lowfreq'],cfg_par['rfi']['highfreq'],10,dtype=int)
             #ax1.set_xticks(xticks_num)
 
-            if cfg_par['plots']['plot_noise']  == 'noise_factor':
+            if cfg_par['plots']['plot_detail']['plot_noise']  == 'noise_factor':
                 ax1.set_yticks([1,round(np.sqrt(2),2),2,3,5,10,50]) 
                 ax1.set_yscale('linear')     
                 ax1.set_ylabel(r'Factor of noise increase')
                 ax1.get_yaxis().set_major_formatter(ticker.ScalarFormatter())
 
-            if cfg_par['plots']['plot_noise'] == 'noise':
+            if cfg_par['plots']['plot_detail']['plot_noise'] == 'noise':
                 ax1.set_yscale('linear')     
                 ax1.set_ylabel(r'Predicted noise [mJy beam$^{-1}$]')     
                 ax1.get_yaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())
 
-            if cfg_par['plots']['plot_noise'] == 'rfi':
+            if cfg_par['plots']['plot_detail']['plot_noise'] == 'rfi':
                 if cfg_par['rfi']['RFInder_mode'] == 'rms_clip':
                     ax1.set_ylabel(r'$\% > 5 \times$ r.m.s.')
                 if cfg_par['rfi']['RFInder_mode'] == 'use_flags':
@@ -917,7 +917,8 @@ def plot_summary_stats(self, flag_stats, cfg_par, key='corr'):
             plt.title("Antenna flags")
             plt.xlabel("Antenna")
             plt.ylabel("% flagged visibilities")
-            plt.xticks(rotation=90)
+            if len(flag_stats.keys()) > 30:
+                plt.xticks(rotation=90)
             plt.savefig(summaryplot)
         elif key in ["scan"]:
             plt.title("Scan flags")
@@ -936,7 +937,7 @@ def plot_summary_stats(self, flag_stats, cfg_par, key='corr'):
             plt.ylabel("% flagged visibilities")
             ax2 = ax.twiny()
             ax2.set_xlabel("Channel Number")
-            ax2.bar(list(range(len(flag_stats.keys()))), flag_stats.values(), color="orange", ec="red", align='center')
+            ax2.bar(list(range(len(flag_stats.keys()))), flag_stats.values(), color="orange", ec="orange", align='center')
             plt.savefig(summaryplot)
         self.logger.info(f" ------ Saving: {summaryplot} ------\n")
         plt.close()

rfinder/rfinder_stats.py

@@ -336,7 +336,7 @@ def flag_bars(flag_stats, key):
                 for p in processes:
                     p.join()
         if axis in ['freq']:
-            for index, freq_id in enumerate(cfg_par['rfi']['freqs']):
+            for index, freq in enumerate(cfg_par['rfi']['freqs']):
                 taql = f''
                 if cfg_par['plots']['plot_summary']['antenna']:
                     ant_id = list(cfg_par['rfi']['ant_names']).index(cfg_par['plots']['plot_summary']['antenna'])