Skip to content

Commit

Permalink
Merge pull request #39 from caracal-pipeline/pre-release110
Browse files Browse the repository at this point in the history 
Pre release110
  • Loading branch information
@Athanaseus
Athanaseus authored Dec 7, 2023
2 parents 9378677 + 826ed43 commit 4f0d3e7
Show file tree
Hide file tree
Showing 8 changed files with 304 additions and 119 deletions.
69 changes: 69 additions & 0 deletions .github/workflows/publish_package.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Upload Python Package

on:
push:
tags:
- '**'

env:
POETRY_VERSION: 1.4.0

jobs:
deploy:
runs-on: ubuntu-latest
# Run on a push to a tag or master
if: >
github.event_name == 'push' &&
(startsWith(github.event.ref, 'refs/tags') ||
github.event.ref == 'refs/heads/master')
steps:
- name: Set up Python 3
uses: actions/setup-python@v3
with:
python-version: '3.x'

- name: Install latest setuptools, wheel, pip
run: python3 -m pip install -U pip setuptools wheel

- name: Cache Installations
id: cache-installs
uses: actions/cache@v3
with:
path: ~/.local
key: install-${{ env.INSTALL_CACHE_HASH }}-3

- name: Install Poetry
if: steps.cache-installs.outputs.cache-hit != 'true'
run: |
curl -sSL https://install.python-poetry.org | python3 - --version ${{ env.POETRY_VERSION }}
- name: Test poetry
run: poetry --version

- name: Checkout source
uses: actions/checkout@v3
with:
fetch-depth: 1

- name: Install rfinder
run: poetry install

- name: Build distribution
run: poetry build

- name: Publish distribution 📦 to Test PyPI
uses: pypa/gh-action-pypi-publish@master
with:
user: __token__
password: ${{ secrets.TEST_PYPI_API_TOKEN }}
repository_url: https://test.pypi.org/legacy/
continue-on-error: true

- name: Publish distribution 📦 to PyPI
uses: pypa/gh-action-pypi-publish@master
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
Empty file added rfinder/__init__.py
View file
Empty file.
120 changes: 83 additions & 37 deletions rfinder/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@

sys.path.append(os.path.join(RFINDER_PATH, 'rfinder'))

import rfi
import rfinder_stats as rfi_stats
import rfinder_plots as rfi_plots
import rfinder_files as rfiFL
from rfinder import rfi
from rfinder import rfinder_stats as rfi_stats
from rfinder import rfinder_plots as rfi_plots
from rfinder import rfinder_files as rfiFL

rfi = rfi.rfi()
rfiST = rfi_stats.rfi_stats()
Expand Down Expand Up @@ -229,7 +229,30 @@ def readArgs(self,argv):
action='store_true',
help='enable cleanup of intermediate products')

add('-pltDet','--plot_details',
action='store_true',
help="plot percentage of RFI, 'rfi', or noise, 'noise', or factor of noise increase")

add('-pltSum','--plot_summary',
action='store_true',
help='plot percentage of RFI per ant,scan,freq,corr')

add('-summary','--summary_options',
choices=['ant', 'corr', 'scan', 'freq'],
default=['corr'],
nargs='+',
type=str,
help='enable cleanup of intermediate products')

add('-fbin', '--freq_bin',
type=int,
default=False,
help='Number of frequencies to bin into a single channel')

add('-ncpu', '--ncpu',
type=int,
default=False,
help='Number of cpu to use when generating summary stats')

args = self.parser.parse_args(argv)

Expand All @@ -246,8 +269,10 @@ def setArgs(self,args):
self.cfg_par['general']['msname'] = args.input
if args.field != None:
self.cfg_par['general']['field'] = args.field
if args.ncpu:
self.cfg_par['general']['ncpu'] = args.ncpu
if args.telescope:
self.cfg_par['general']['telescope'] = args.telescope
self.cfg_par['general']['telescope']['name'] = args.telescope
if args.polarization:
self.cfg_par['rfi']['polarization'] = args.polarization
if args.baseline_cut:
Expand All @@ -271,7 +296,7 @@ def setArgs(self,args):
self.cfg_par['rfi']['chunks']['spw_enable'] = True

if args.no_movies == True :
self.cfg_par['plots']['movies']['movies_in_report'] = False
self.cfg_par['plots']['plot_details']['movies']['movies_in_report'] = False

if args.no_cleanup==True:
self.cfg_par['general']['cleanup_enable'] = False
Expand All @@ -284,12 +309,24 @@ def setArgs(self,args):
else:
self.cfg_par['general']['outlabel'] = '_'+self.cfg_par['general']['outlabel']

if (args.rfimode == 'rms_clip' or args.rfimode == 'rms') :
self.cfg_par['rfi']['RFInder_mode'] = args.rfimode
if args.sigma_clip:
self.cfg_par['rfi']['rms_clip'] = args.sigma_clip
if args.frequency_interval:
self.cfg_par['rfi']['noise_measure_edges'] = args.frequency_interval
if (args.rfimode == 'rms_clip' or args.rfimode == 'use_flag'):
self.cfg_par['rfi']['RFInder_mode'] = args.rfimode
if args.sigma_clip:
self.cfg_par['rfi']['rms_clip'] = args.sigma_clip
if args.frequency_interval:
self.cfg_par['rfi']['noise_measure_edges'] = args.frequency_interval
else:
self.cfg_par['rfi']['rfi_enable'] = False

if args.plot_details:
self.cfg_par['plots']['plot_details']['enable'] = True

if args.plot_summary:
self.cfg_par['plots']['plot_summary']['enable'] = True
if args.summary_options:
self.cfg_par['plots']['plot_summary']['axis'] = args.summary_options
if args.freq_bin:
self.cfg_par['plots']['plot_summary']['freq_bin'] = args.freq_bin

return self

Expand Down Expand Up @@ -426,13 +463,15 @@ def go(self,cfg_par):
rfiFL.rfi_frequency(self.cfg_par,-1)
self.logger.warning("------ RFI saved to table ------\n")
self.logger.warning("------ End of RFI analysis ------\n")

else:
rfi.load_from_ms(self.cfg_par,0,0)
self.logger.warning("------ MSfile Loaded -----\n")

task = 'plots'

if self.cfg_par[task]['plot_summary']['enable']==True:
summary_results = {}
if not self.cfg_par['rfi']['rfi_enable']:
rfi.load_from_ms(self.cfg_par,0,0)

for axis in self.cfg_par[task]['plot_summary']['axis']:
flag_stats = rfiST.get_flags_summary_stats(self.cfg_par, axis)
Expand All @@ -442,13 +481,14 @@ def go(self,cfg_par):
self.logger.info("------ Summary plot done ------\n")

if summary_results:
self.logger.warning(f'------ Total % Flagged: {round(sum(summary_results[axis].values())/len(summary_results[axis].values()),2)} ------')
json_file = cfg_par['general']['rfidir'] + f'summary.json'
with open(json_file, 'w') as f:
json.dump(summary_results, f)

rfiFL.write_html_summaryreport(self.cfg_par)

if self.cfg_par[task]['plot_enable']==True:
if self.cfg_par[task]['plot_details']['enable']==True:

if self.cfg_par['rfi']['chunks']['time_enable']==True:

Expand Down Expand Up @@ -483,39 +523,40 @@ def go(self,cfg_par):

rfiPL.plot_rfi_imshow(self.cfg_par,i)
self.logger.info("------ RFI in 2D plotted ------\n")
self.cfg_par['plots']['plot_noise'] = 'rfi'
self.cfg_par['plots']['long_short'] = False
self.cfg_par['plots']['plot_details']['plot_noise'] = 'rfi'
self.cfg_par['plots']['plot_details']['long_short'] = False
rfiPL.plot_noise_frequency(self.cfg_par,i)
self.cfg_par['plots']['long_short'] = True
self.cfg_par['plots']['plot_details']['long_short'] = True
rfiPL.plot_noise_frequency(self.cfg_par,i)
self.cfg_par['plots']['plot_noise'] = 'noise_factor'
self.cfg_par['plots']['plot_details']['plot_noise'] = 'noise_factor'
rfiPL.plot_noise_frequency(self.cfg_par,i)
self.cfg_par['plots']['plot_noise'] = 'noise'
self.cfg_par['plots']['plot_details']['plot_noise'] = 'noise'
rfiPL.plot_noise_frequency(self.cfg_par,i)
self.logger.warning("------ RFI in 1D plotted ------\n")

rfiPL.plot_altaz(self.cfg_par,68)
self.logger.warning("------ RFI in ALT/AZ plotted ------\n")

if (self.cfg_par['plots']['movies']['altaz_gif']==True or self.cfg_par['plots']['movies']['2d_gif']==True or
self.cfg_par['plots']['movies']['1d_gif']==True):
if (self.cfg_par['plots']['plot_details']['movies']['altaz_gif']==True or
self.cfg_par['plots']['plot_details']['movies']['2d_gif']==True or
self.cfg_par['plots']['plot_details']['movies']['1d_gif']==True):
self.logger.warning("------ Making movies ------\n")

if self.cfg_par['plots']['movies']['altaz_gif']==True:
if self.cfg_par['plots']['plot_details']['movies']['altaz_gif']==True:

out_animation = self.cfg_par['general']['moviedir']+'AltAz_movie.gif'
filenames = rfiFL.find_altaz_plots(self.cfg_par)
rfiPL.gif_me_up(self.cfg_par,filenames,out_animation)
self.logger.info("------ AltAz movie done ------\n")

if self.cfg_par['plots']['movies']['2d_gif']==True:
if self.cfg_par['plots']['plot_details']['movies']['2d_gif']==True:
out_animation = self.cfg_par['general']['moviedir']+'Time_2Dplot_movie.gif'
filenames = rfiFL.find_2d_plots(self.cfg_par)
rfiPL.gif_me_up(self.cfg_par,filenames,out_animation)

self.logger.info("------ 2D movie done ------\n")

if self.cfg_par['plots']['movies']['1d_gif']==True:
if self.cfg_par['plots']['plot_details']['movies']['1d_gif']==True:
out_animation = self.cfg_par['general']['moviedir']+'TimeChunks_1D_flags.gif'
root_name = 'flags'
filenames = rfiFL.find_1d_plots(self.cfg_par,root_name)
Expand All @@ -532,8 +573,9 @@ def go(self,cfg_par):
rfiPL.gif_me_up(self.cfg_par,filenames,out_animation)
self.logger.info("------ 1D movies done ------\n")

if (self.cfg_par['plots']['movies']['altaz_gif']==True or self.cfg_par['plots']['movies']['2d_gif']==True or
self.cfg_par['plots']['movies']['1d_gif']==True):
if (self.cfg_par['plots']['plot_details']['movies']['altaz_gif']==True or
self.cfg_par['plots']['plot_details']['movies']['2d_gif']==True or
self.cfg_par['plots']['plot_details']['movies']['1d_gif']==True):
self.logger.warning("------ Movies done ------\n")

rfiFL.write_html_timereport(self.cfg_par)
Expand All @@ -554,14 +596,14 @@ def go(self,cfg_par):
self.logger.warning("------ Alt/Az plotted ------\n")
rfiPL.plot_rfi_imshow(self.cfg_par,-1)
self.logger.warning("------ RFI in 2D plotted ------\n")
self.cfg_par['plots']['plot_noise'] = 'rfi'
self.cfg_par['plots']['long_short'] = False
self.cfg_par['plots']['plot_details']['plot_noise'] = 'rfi'
self.cfg_par['plots']['plot_details']['long_short'] = False
rfiPL.plot_noise_frequency(self.cfg_par,-1)
self.cfg_par['plots']['long_short'] = True
self.cfg_par['plots']['plot_details']['long_short'] = True
rfiPL.plot_noise_frequency(self.cfg_par,-1)
self.cfg_par['plots']['plot_noise'] = 'noise_factor'
self.cfg_par['plots']['plot_details']['plot_noise'] = 'noise_factor'
rfiPL.plot_noise_frequency(self.cfg_par,-1)
self.cfg_par['plots']['plot_noise'] = 'noise'
self.cfg_par['plots']['plot_details']['plot_noise'] = 'noise'
rfiPL.plot_noise_frequency(self.cfg_par,-1)
self.logger.warning("------ RFI in 1D plotted ------\n")

Expand All @@ -588,8 +630,9 @@ def main (self,argv):
if args.help: #rfinder -h
self.parser.print_help()

print("""\nRun a command. This can be:\n \nrfinder \nrfinder -c path_to_config_file.yml
rfinder -i <ngc1399.ms> -fl <num> -tel <meerkat/apertif/wsrt>\n""")
print("\nRun a command. This can be:\n \nrfinder \nrfinder -c path_to_config_file.yml" +
"\nrfinder -i <ngc1399.ms> -fl <num> -tel <meerkat/apertif/wsrt>" +
"\nrfinder -i <ngc1399.ms> -fl <num> -tel <meerkat/apertif/wsrt> -mode rms_clip -plotSum")

sys.exit(0)

Expand Down Expand Up @@ -647,19 +690,22 @@ def main (self,argv):

return self


def driver():
logger = logging.getLogger('log-rfinder.log')
logger.setLevel(logging.INFO)
logger.propagate = False
logger.debug('info')
logger.info('info')

fh = logging.FileHandler('log-rfinder.log')
fh.setLevel(logging.INFO)

ch = logging.StreamHandler()
ch.setLevel(logging.WARNING)
ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(levelname)s - %(filename)s - %(message)s')
formatter_ch = logging.Formatter('%(message)s')
formatter = logging.Formatter('%(asctime)s; %(levelname)s - %(filename)s - %(message)s')
formatter_ch = logging.Formatter('%(asctime)s; %(message)s')

fh.setFormatter(formatter)
ch.setFormatter(formatter_ch)
Expand Down
7 changes: 3 additions & 4 deletions rfinder/rfi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@
from astropy.io import fits, ascii
from astropy import units as u
from astropy.table import Table, Column, MaskedColumn
import rfinder

import rfinder_stats as rfi_stats
import rfinder_files as rfi_files
from rfinder import rfinder_stats as rfi_stats
from rfinder import rfinder_files as rfi_files


rfiST = rfi_stats.rfi_stats()
Expand All @@ -32,7 +31,7 @@ class rfi:
def __init__(self):

self.logger = logging.getLogger('log-rfinder.log')
#self.logger.setLevel(logging.INFO)
self.logger.setLevel(logging.INFO)

#fh = logging.FileHandler('log-rfinder.log')
#fh.setLevel(logging.INFO)
Expand Down
33 changes: 19 additions & 14 deletions rfinder/rfinder_default.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ general:

rfi:
__helpstr: Find RFI in msfile over threshold
rfi_enable: true #if True this module is enabled
rfi_enable: false #if True this module is enabled else only obs info will logged
polarization: 'q' #xx, yy, xy, yx, q (also in CAPS)
bad_antenna: '[]' #list of bad antennas: for no antennas try '[]'
RFInder_mode: 'use_flags' #uses RFI clip method or simply reads the FLAG column 'use_flags' / 'rms_clip'
RFInder_mode: 'rms_clip' #uses RFI clip method or simply reads the FLAG column 'use_flags' / 'rms_clip'
rms_clip: 5 #Threshold for RFI identification
noise_measure_edges: [1.400e9, 1.420e9] #min frequency and maximum frequency where to measure average std of visibilities
baseline_cut: 200 #Cutoff baseline lenght: < = Short, > = Long (615m midpoint of baselines of WSRT)
Expand All @@ -34,17 +34,22 @@ rfi:

plots:
__helpstr: plot rfi over different axes
plot_enable: true
plot_noise: 'rfi' #choose to plot % of RFI, 'rfi', or noise, 'noise', or factor of noise increase, 'noise_factor' in 1D plots
plot_long_short: true #plot all baselines, or only the short and the long
plot_details:
__helpstr: plot formats
enable: false
plot_noise: 'rfi' #choose to plot % of RFI, 'rfi', or noise, 'noise', or factor of noise increase, 'noise_factor' in 1D plots
plot_long_short: true #plot all baselines, or only the short and the long
plot_eps: false #also generate eps plots. If false only pngs/movies
movies:
__helpstr: make movies of output plots
2d_gif: true
1d_gif: true
altaz_gif: true
movies_in_report: true
plot_summary:
__helpstr: plot total % flagged visibilities
enable: true
axis: [ant, scan, corr] # Avalable choices [ant, corr, scan]
report: true
movies:
__helpstr: make movies of output plots
2d_gif: true
1d_gif: true
altaz_gif: true
movies_in_report: true
enable: false
axis: [ant, scan] #avalable choices [ant, corr, scan, freq]
antenna: #select an antenna to get summary
freq_bin: 4 #bin frequency channels
report: true #generate html report with the results
Loading

0 comments on commit 4f0d3e7

Please sign in to comment.