Skip to content

Commit

Permalink
JP-3533 Update emicorr to run on FAST mode MIRI data (#8270)
Browse files Browse the repository at this point in the history
Co-authored-by: Howard Bushouse <[email protected]>
  • Loading branch information
penaguerrero and hbushouse authored Mar 19, 2024
1 parent 01a807b commit a3057ff
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 156 deletions.
4 changes: 4 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ emicorr
- Set skip=True by default in the code, to be turned on later by a parameter
reference file. [#8171]

- Step is skipped when no reference file is found and to add a parameter to
allow the user to run the step for given frequencies with an on-the-fly
generated reference file. [#8270]

exp_to_source
-------------

Expand Down
12 changes: 7 additions & 5 deletions docs/jwst/emicorr/arguments.rst
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ The ``emicorr`` step has the following step-specific arguments.

``--save_intermediate_results`` (string, default=False)
This is a boolean flag to specify whether to write a step output
file with the EMI correction. Additionaly, if the flag
``user_supplied_reffile`` is None and no CRDS reference file was
found, all the on-the-fly frequencies phase amplitudes will be
saved to an ASDF output with the same format as an EMI reference
file.
file with the EMI correction, and a reference file with all the
on-the-fly frequencies' phase amplitudes. The file will have an
ASDF output with the same format as an EMI reference file.

``--onthefly_corr_freq`` (list, default=None)
This is to tell the code to do correction for the frequencies in
the list with a reference file created on-the-fly instead of CRDS.

189 changes: 67 additions & 122 deletions jwst/emicorr/emicorr.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,105 +11,51 @@
log.setLevel(logging.DEBUG)


default_emi_freqs = {
"Hz390": 390.625,
"Hz390_sub128": 390.625,
"Hz10": 10.039216,
"Hz10_slow_MIRIMAGE": 10.039216,
"Hz10_slow_MIRIFULONG": 10.039216,
"Hz10_slow_MIRIFUSHORT": 10.039216
}

default_subarray_cases = {

# 390Hz out-of-phase - these all need 390hz correction
subarray_clocks = {

"SLITLESSPRISM": {
"rowclocks": 28,
"frameclocks": 15904,
"freqs": {"FAST": ["Hz390", "Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 15904},

"MASKLYOT": {
"rowclocks": 90,
"frameclocks": 32400,
"freqs": {"FAST": ["Hz390", "Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 32400},

"SUB64": {
"rowclocks": 28,
"frameclocks": 8512,
"freqs": {"FAST": ["Hz390", "Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 8512},

"SUB128": {
"rowclocks": 44,
"frameclocks": 11904,
"freqs": {"FAST": ["Hz390_sub128", "Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 11904},

"MASK1140": {
"rowclocks": 82,
"frameclocks": 23968,
"freqs": {"FAST": ["Hz390", "Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 23968},

"MASK1550": {
"rowclocks": 82,
"frameclocks": 23968,
"freqs": {"FAST": ["Hz390", "Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 23968},

"MASK1065": {
"rowclocks": 82,
"frameclocks": 23968,
"freqs": {"FAST": ["Hz390", "Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}},

# 390Hz already in-phase for these, but may need corr for other
# frequencies (e.g. 10Hz heater noise)
"frameclocks": 23968},

"FULL_FAST": {
"rowclocks": 271,
"frameclocks": 277504,
"freqs": {"FAST" : ["Hz10"]}},
"frameclocks": 277504},

"FULL_SLOW": {
"rowclocks": 2333,
"frameclocks": 2388992,
"freqs": {"SLOW": {"MIRIMAGE" : ["Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 2388992},

"BRIGHTSKY": {
"rowclocks": 162,
"frameclocks": 86528,
"freqs": {"FAST" : ["Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 86528},

"SUB256": {
"rowclocks": 96,
"frameclocks": 29952,
"freqs": {"FAST" : ["Hz10"],
"SLOW": {"MIRIMAGE" : ["Hz10_slow_MIRIMAGE"],
"MIRIFULONG" : ["Hz10_slow_MIRIFULONG"],
"MIRIFUSHORT" : ["Hz10_slow_MIRIFUSHORT"]}}},
"frameclocks": 29952},
}


Expand Down Expand Up @@ -146,8 +92,10 @@ def do_correction(input_model, emicorr_model, save_onthefly_reffile, **pars):
nints_to_phase = pars['nints_to_phase']
nbins = pars['nbins']
scale_reference = pars['scale_reference']
onthefly_corr_freq = pars['onthefly_corr_freq']

output_model = apply_emicorr(input_model, emicorr_model, save_onthefly_reffile,
output_model = apply_emicorr(input_model, emicorr_model,
onthefly_corr_freq, save_onthefly_reffile,
save_intermediate_results=save_intermediate_results,
user_supplied_reffile=user_supplied_reffile,
nints_to_phase=nints_to_phase,
Expand All @@ -158,7 +106,8 @@ def do_correction(input_model, emicorr_model, save_onthefly_reffile, **pars):
return output_model


def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile,
def apply_emicorr(input_model, emicorr_model,
onthefly_corr_freq, save_onthefly_reffile,
save_intermediate_results=False, user_supplied_reffile=None,
nints_to_phase=None, nbins_all=None, scale_reference=True):
"""
Expand Down Expand Up @@ -195,6 +144,9 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile,
emicorr_model : `~jwst.datamodels.EmiModel`
ImageModel of emi
onthefly_corr_freq : float
Frequency number to do correction with on-the-fly reference file
save_onthefly_reffile : str or None
Full path and root name of on-the-fly reference file
Expand Down Expand Up @@ -238,12 +190,20 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile,
freqs_numbers.append(freq)
reference_wave_list.append(ref_wave)
else:
log.info('Using default subarray case corrections.')
subname, rowclocks, frameclocks, freqs2correct = get_subarcase(default_subarray_cases, subarray, readpatt, detector)
if freqs2correct is not None:
for fnme in freqs2correct:
freq = get_frequency_info(default_emi_freqs, fnme)
freqs_numbers.append(freq)
# if we got here, the user requested to do correction with on-the-fly reference file
subname = subarray
if subname == 'FULL':
if 'FAST' in readpatt.upper():
subname += '_FAST'
elif 'SLOW' in readpatt.upper():
subname += '_SLOW'
if subname in subarray_clocks:
rowclocks = subarray_clocks[subname]['rowclocks']
frameclocks = subarray_clocks[subname]['frameclocks']
freqs_numbers = onthefly_corr_freq
freqs2correct = [repr(freq) for freq in onthefly_corr_freq]
else:
subname, rowclocks, frameclocks, freqs2correct = None, None, None, None

log.info('With configuration: Subarray={}, Read_pattern={}, Detector={}'.format(subarray, readpatt, detector))
if rowclocks is None or len(freqs_numbers) == 0:
Expand Down Expand Up @@ -379,7 +339,8 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile,
phaseall[ninti, ...] = phase_this_int - phase_this_int.astype('ulonglong')

# add a frame time to account for the extra frame reset between MIRI integrations
start_time += frameclocks
if readpatt.upper() == 'FASTR1' or readpatt.upper() == 'SLOWR1':
start_time += frameclocks

# use phaseall vs dd_all

Expand Down Expand Up @@ -433,7 +394,7 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile,
# These two methods give the same integer-reference_wave-element resolution results.
if emicorr_model is not None:
log.info('Using reference file to measure phase shift')
reference_wave = reference_wave_list[fi]
reference_wave = np.array(reference_wave_list[fi])
reference_wave_size = np.size(reference_wave)
rebinned_pa = rebin(pa, [reference_wave_size])
cc = np.zeros(reference_wave_size)
Expand Down Expand Up @@ -480,7 +441,7 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile,
output_model.data = corr_data

if save_intermediate_results and save_onthefly_reffile is not None:
if readpatt == 'FAST':
if 'FAST' in readpatt:
freqs_dict = {readpatt: freqs2correct}
else:
freqs_dict = {readpatt: {detector: freqs2correct} }
Expand Down Expand Up @@ -606,58 +567,42 @@ def get_subarcase(subarray_cases, subarray, readpatt, detector):
frequencies : list
List of frequencies to correct according to subarray name
freqs_names : list
List of frequency names to use
"""
subname, rowclocks, frameclocks, frequencies = None, None, None, None

# make sure the readpattern is defined as expected
readpatt = readpatt.upper()
if "SLOW" in readpatt:
readpatt = "SLOW"
elif "FAST" in readpatt:
readpatt = "FAST"
else:
# make sure the readpattern is defined as expected to read data from reference file
readpatt, no_slow_match_readpatt, no_fast_match_readpatt = readpatt.upper(), False, False
if "SLOW" not in readpatt:
no_slow_match_readpatt = True
if "FAST" not in readpatt:
no_fast_match_readpatt = True
if no_slow_match_readpatt and no_fast_match_readpatt:
log.info('Read pattern {} does not include expected string FAST or SLOW'.format(readpatt))
return subname, rowclocks, frameclocks, frequencies

# search and return the specific values for the configuration
if isinstance(subarray_cases, dict):
for subname in subarray_cases:
if subarray == 'FULL':
subarray = subarray + '_' + readpatt
if subarray != subname:
continue
rowclocks = subarray_cases[subname]["rowclocks"]
frameclocks = subarray_cases[subname]["frameclocks"]
if readpatt == "SLOW":
frequencies = subarray_cases[subname]["freqs"]["SLOW"][detector]
else:
frequencies = subarray_cases[subname]["freqs"]["FAST"]
frequencies = []
mdl_dict = subarray_cases.to_flat_dict()
for subname in subarray_cases.subarray_cases:
subname = subname.split(sep='.')[0]
if subarray not in subname:
continue
log.debug('Found subarray case {}!'.format(subname))
for item, val in mdl_dict.items():
if subname in item:
if 'FULL' in item and readpatt not in item:
continue
if "rowclocks" in item:
rowclocks = val
elif "frameclocks" in item:
frameclocks = val
else:
if "SLOW" in readpatt and "SLOW" in item and detector in item:
frequencies.append(val)
elif "FAST" in readpatt and "FAST" in item:
frequencies.append(val)
if subname is not None and rowclocks is not None and frameclocks is not None and frequencies is not None:
break
else:
frequencies = []
mdl_dict = subarray_cases.to_flat_dict()
for subname in subarray_cases.subarray_cases:
subname = subname.split(sep='.')[0]
if subarray not in subname:
continue
log.debug('Found subarray case {}!'.format(subname))
for item, val in mdl_dict.items():
if subname in item:
if 'FULL' in item and readpatt not in item:
continue
if "rowclocks" in item:
rowclocks = val
elif "frameclocks" in item:
frameclocks = val
else:
if readpatt == "SLOW" and "SLOW" in item and detector in item:
frequencies.append(val)
elif readpatt == "FAST" and "FAST" in item:
frequencies.append(val)
if subname is not None and rowclocks is not None and frameclocks is not None and frequencies is not None:
break
return subname, rowclocks, frameclocks, frequencies


Expand Down
42 changes: 27 additions & 15 deletions jwst/emicorr/emicorr_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,44 +22,56 @@ class EmiCorrStep(Step):
nints_to_phase = integer(default=None) # Number of integrations to phase
nbins = integer(default=None) # Number of bins in one phased wave
scale_reference = boolean(default=True) # If True, the reference wavelength will be scaled to the data's phase amplitude
skip = boolean(default=True)
skip = boolean(default=True) # Skip the step
onthefly_corr_freq = float_list(default=None) # Frequencies to use for correction
"""

reference_file_types = ['emicorr']

def process(self, input):
with datamodels.open(input) as input_model:

# Catch the cases to skip, i.e. all instruments other than MIRI
# Catch the cases to skip
instrument = input_model.meta.instrument.name
if instrument != 'MIRI':
self.log.warning('EMI correction not implemented for instrument: {}'.format(instrument))
input_model.meta.cal_step.emicorr = 'SKIPPED'
return input_model

readpatt = input_model.meta.exposure.readpatt
allowed_readpatts = ['FAST', 'FASTR1', 'SLOW', 'SLOWR1']
if readpatt.upper() not in allowed_readpatts:
self.log.warning('EMI correction not implemented for read pattern: {}'.format(readpatt))
input_model.meta.cal_step.emicorr = 'SKIPPED'
return input_model

# Setup parameters
pars = {
'save_intermediate_results': self.save_intermediate_results,
'user_supplied_reffile': self.user_supplied_reffile,
'nints_to_phase': self.nints_to_phase,
'nbins': self.nbins,
'scale_reference': self.scale_reference
'scale_reference': self.scale_reference,
'onthefly_corr_freq': self.onthefly_corr_freq
}

# Get the reference file
save_onthefly_reffile, emicorr_ref_filename, emicorr_model = None, None, None
if self.user_supplied_reffile is None:
try:
emicorr_ref_filename = self.get_reference_file(input_model, 'emicorr')
# Create the reference file only if told to save outputs, else correct on-the-fly
if emicorr_ref_filename == 'N/A':
emicorr_ref_filename = None
else:
self.log.info('Using CRDS reference file: {}'.format(emicorr_ref_filename))
emicorr_model = datamodels.EmiModel(emicorr_ref_filename)
except Exception:
# No reference file in CRDS
self.log.info('No CRDS emicorr reference file found. Creating on-the-fly reference file.')
if self.onthefly_corr_freq is not None:
emicorr_ref_filename = None
self.log.info('Correcting with reference file created on-the-fly.')

elif self.user_supplied_reffile is None:
emicorr_ref_filename = self.get_reference_file(input_model, 'emicorr')
# Skip the spep if no reference file is found
if emicorr_ref_filename == 'N/A':
self.log.warning('No reference file found.')
self.log.warning('EMICORR step will be skipped')
input_model.meta.cal_step.emicorr = 'SKIPPED'
return input_model
else:
self.log.info('Using CRDS reference file: {}'.format(emicorr_ref_filename))
emicorr_model = datamodels.EmiModel(emicorr_ref_filename)

else:
self.log.info('Using user-supplied reference file: {}'.format(self.user_supplied_reffile))
Expand Down
Loading

0 comments on commit a3057ff

Please sign in to comment.