diff --git a/CHANGES.rst b/CHANGES.rst index 4d1b965805f..45851f69b50 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -66,12 +66,19 @@ documentation - Correct the names of parameter options ``usigma`` and ``lsigma`` for ``sky_match``. [#8356] +- Updated ``outlier_detection`` for IFU data to explain the method more clearly. [#8360] + + emicorr ------- - Set skip=True by default in the code, to be turned on later by a parameter reference file. [#8171] +- Step is skipped when no reference file is found and to add a parameter to + allow the user to run the step for given frequencies with an on-the-fly + generated reference file. [#8270] + exp_to_source ------------- @@ -233,6 +240,9 @@ resample ``world_to_pixel_values``) for reproject, which also fixed a bug, and removed support for astropy model [#8172] +- Added sleep + check of output files that are median combined to fix intermittent + corruption of these files in operations [#8305] + - Replace use of ``check_memory_allocation``. [#8324] - Removed any reference to the "tophat" kernel for resample step. [#8364] diff --git a/docs/jwst/emicorr/arguments.rst b/docs/jwst/emicorr/arguments.rst index 9dedea0ec72..c84a6c06177 100644 --- a/docs/jwst/emicorr/arguments.rst +++ b/docs/jwst/emicorr/arguments.rst @@ -17,9 +17,11 @@ The ``emicorr`` step has the following step-specific arguments. ``--save_intermediate_results`` (string, default=False) This is a boolean flag to specify whether to write a step output - file with the EMI correction. Additionaly, if the flag - ``user_supplied_reffile`` is None and no CRDS reference file was - found, all the on-the-fly frequencies phase amplitudes will be - saved to an ASDF output with the same format as an EMI reference - file. + file with the EMI correction, and a reference file with all the + on-the-fly frequencies' phase amplitudes. The file will have an + ASDF output with the same format as an EMI reference file. + +``--onthefly_corr_freq`` (list, default=None) + This is to tell the code to do correction for the frequencies in + the list with a reference file created on-the-fly instead of CRDS. diff --git a/docs/jwst/outlier_detection/outlier_detection_ifu.rst b/docs/jwst/outlier_detection/outlier_detection_ifu.rst index a2a592846a8..ba16718d51b 100644 --- a/docs/jwst/outlier_detection/outlier_detection_ifu.rst +++ b/docs/jwst/outlier_detection/outlier_detection_ifu.rst @@ -14,11 +14,16 @@ in the :ref:`calwebb_detector1 ` pipeline. At this time it is believed that NIRSpec IFU data also have bad pixels that vary with time, though the time variation is still under study. -An algorithm was developed to flag pixels that are outliers when compared to their neighbors for a set of -input files contained in an association. The neighbor pixel differences are the neighbors in spatial direction. For MIRI data ,the neighbor differences are found to the left and right of every -science pixel. While for NIRSpec data neighbor differences are -found between the pixels above and below every science pixel. The pixel differences for each input model -in the association is determined and is stored in a stack of pixel differences. +The basis of the outlier detection flagging for IFU data is to look for pixels on the detector +that are regularly discrepant from their neighbors, with a sharper division than could be explained +by the detector PSF. The algorithm flags pixels that are outliers when compared to their neighbors for a set of +input files contained in an association. The neighbor pixel differences are the neighbors in the spatial direction. +For MIRI data neighbor differences are found to the left and right of every +science pixel, while for NIRSpec data the neighbor differences are +determined from pixels above and below every science pixel. The difference between the MIRI MRS and NIRPSpec algorithm for finding the +spatial pixel differences is due to the opposite dispersion directions between the two instruments. +The pixel differences for each input model +in the association are determined and stored in a stack of pixel differences. For each pixel the minimum difference through this stack is determined and normalized. The normalization uses a local median of the difference array (set by the kernel size). A pixel is flagged as an outlier if this normalized minimum difference diff --git a/jwst/emicorr/emicorr.py b/jwst/emicorr/emicorr.py index 896e8675063..e89bfca4af2 100644 --- a/jwst/emicorr/emicorr.py +++ b/jwst/emicorr/emicorr.py @@ -11,105 +11,51 @@ log.setLevel(logging.DEBUG) -default_emi_freqs = { - "Hz390": 390.625, - "Hz390_sub128": 390.625, - "Hz10": 10.039216, - "Hz10_slow_MIRIMAGE": 10.039216, - "Hz10_slow_MIRIFULONG": 10.039216, - "Hz10_slow_MIRIFUSHORT": 10.039216 -} - -default_subarray_cases = { - - # 390Hz out-of-phase - these all need 390hz correction +subarray_clocks = { "SLITLESSPRISM": { "rowclocks": 28, - "frameclocks": 15904, - "freqs": {"FAST": ["Hz390", "Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 15904}, "MASKLYOT": { "rowclocks": 90, - "frameclocks": 32400, - "freqs": {"FAST": ["Hz390", "Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 32400}, "SUB64": { "rowclocks": 28, - "frameclocks": 8512, - "freqs": {"FAST": ["Hz390", "Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 8512}, "SUB128": { "rowclocks": 44, - "frameclocks": 11904, - "freqs": {"FAST": ["Hz390_sub128", "Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 11904}, "MASK1140": { "rowclocks": 82, - "frameclocks": 23968, - "freqs": {"FAST": ["Hz390", "Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 23968}, "MASK1550": { "rowclocks": 82, - "frameclocks": 23968, - "freqs": {"FAST": ["Hz390", "Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 23968}, "MASK1065": { "rowclocks": 82, - "frameclocks": 23968, - "freqs": {"FAST": ["Hz390", "Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz390", "Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz390", "Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz390", "Hz10_slow_MIRIFUSHORT"]}}}, - - # 390Hz already in-phase for these, but may need corr for other - # frequencies (e.g. 10Hz heater noise) + "frameclocks": 23968}, "FULL_FAST": { "rowclocks": 271, - "frameclocks": 277504, - "freqs": {"FAST" : ["Hz10"]}}, + "frameclocks": 277504}, "FULL_SLOW": { "rowclocks": 2333, - "frameclocks": 2388992, - "freqs": {"SLOW": {"MIRIMAGE" : ["Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 2388992}, "BRIGHTSKY": { "rowclocks": 162, - "frameclocks": 86528, - "freqs": {"FAST" : ["Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 86528}, "SUB256": { "rowclocks": 96, - "frameclocks": 29952, - "freqs": {"FAST" : ["Hz10"], - "SLOW": {"MIRIMAGE" : ["Hz10_slow_MIRIMAGE"], - "MIRIFULONG" : ["Hz10_slow_MIRIFULONG"], - "MIRIFUSHORT" : ["Hz10_slow_MIRIFUSHORT"]}}}, + "frameclocks": 29952}, } @@ -146,8 +92,10 @@ def do_correction(input_model, emicorr_model, save_onthefly_reffile, **pars): nints_to_phase = pars['nints_to_phase'] nbins = pars['nbins'] scale_reference = pars['scale_reference'] + onthefly_corr_freq = pars['onthefly_corr_freq'] - output_model = apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, + output_model = apply_emicorr(input_model, emicorr_model, + onthefly_corr_freq, save_onthefly_reffile, save_intermediate_results=save_intermediate_results, user_supplied_reffile=user_supplied_reffile, nints_to_phase=nints_to_phase, @@ -158,7 +106,8 @@ def do_correction(input_model, emicorr_model, save_onthefly_reffile, **pars): return output_model -def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, +def apply_emicorr(input_model, emicorr_model, + onthefly_corr_freq, save_onthefly_reffile, save_intermediate_results=False, user_supplied_reffile=None, nints_to_phase=None, nbins_all=None, scale_reference=True): """ @@ -195,6 +144,9 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, emicorr_model : `~jwst.datamodels.EmiModel` ImageModel of emi + onthefly_corr_freq : float + Frequency number to do correction with on-the-fly reference file + save_onthefly_reffile : str or None Full path and root name of on-the-fly reference file @@ -238,12 +190,20 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, freqs_numbers.append(freq) reference_wave_list.append(ref_wave) else: - log.info('Using default subarray case corrections.') - subname, rowclocks, frameclocks, freqs2correct = get_subarcase(default_subarray_cases, subarray, readpatt, detector) - if freqs2correct is not None: - for fnme in freqs2correct: - freq = get_frequency_info(default_emi_freqs, fnme) - freqs_numbers.append(freq) + # if we got here, the user requested to do correction with on-the-fly reference file + subname = subarray + if subname == 'FULL': + if 'FAST' in readpatt.upper(): + subname += '_FAST' + elif 'SLOW' in readpatt.upper(): + subname += '_SLOW' + if subname in subarray_clocks: + rowclocks = subarray_clocks[subname]['rowclocks'] + frameclocks = subarray_clocks[subname]['frameclocks'] + freqs_numbers = onthefly_corr_freq + freqs2correct = [repr(freq) for freq in onthefly_corr_freq] + else: + subname, rowclocks, frameclocks, freqs2correct = None, None, None, None log.info('With configuration: Subarray={}, Read_pattern={}, Detector={}'.format(subarray, readpatt, detector)) if rowclocks is None or len(freqs_numbers) == 0: @@ -379,7 +339,8 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, phaseall[ninti, ...] = phase_this_int - phase_this_int.astype('ulonglong') # add a frame time to account for the extra frame reset between MIRI integrations - start_time += frameclocks + if readpatt.upper() == 'FASTR1' or readpatt.upper() == 'SLOWR1': + start_time += frameclocks # use phaseall vs dd_all @@ -433,7 +394,7 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, # These two methods give the same integer-reference_wave-element resolution results. if emicorr_model is not None: log.info('Using reference file to measure phase shift') - reference_wave = reference_wave_list[fi] + reference_wave = np.array(reference_wave_list[fi]) reference_wave_size = np.size(reference_wave) rebinned_pa = rebin(pa, [reference_wave_size]) cc = np.zeros(reference_wave_size) @@ -480,7 +441,7 @@ def apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, output_model.data = corr_data if save_intermediate_results and save_onthefly_reffile is not None: - if readpatt == 'FAST': + if 'FAST' in readpatt: freqs_dict = {readpatt: freqs2correct} else: freqs_dict = {readpatt: {detector: freqs2correct} } @@ -606,58 +567,42 @@ def get_subarcase(subarray_cases, subarray, readpatt, detector): frequencies : list List of frequencies to correct according to subarray name - - freqs_names : list - List of frequency names to use """ subname, rowclocks, frameclocks, frequencies = None, None, None, None - # make sure the readpattern is defined as expected - readpatt = readpatt.upper() - if "SLOW" in readpatt: - readpatt = "SLOW" - elif "FAST" in readpatt: - readpatt = "FAST" - else: + # make sure the readpattern is defined as expected to read data from reference file + readpatt, no_slow_match_readpatt, no_fast_match_readpatt = readpatt.upper(), False, False + if "SLOW" not in readpatt: + no_slow_match_readpatt = True + if "FAST" not in readpatt: + no_fast_match_readpatt = True + if no_slow_match_readpatt and no_fast_match_readpatt: + log.info('Read pattern {} does not include expected string FAST or SLOW'.format(readpatt)) return subname, rowclocks, frameclocks, frequencies # search and return the specific values for the configuration - if isinstance(subarray_cases, dict): - for subname in subarray_cases: - if subarray == 'FULL': - subarray = subarray + '_' + readpatt - if subarray != subname: - continue - rowclocks = subarray_cases[subname]["rowclocks"] - frameclocks = subarray_cases[subname]["frameclocks"] - if readpatt == "SLOW": - frequencies = subarray_cases[subname]["freqs"]["SLOW"][detector] - else: - frequencies = subarray_cases[subname]["freqs"]["FAST"] + frequencies = [] + mdl_dict = subarray_cases.to_flat_dict() + for subname in subarray_cases.subarray_cases: + subname = subname.split(sep='.')[0] + if subarray not in subname: + continue + log.debug('Found subarray case {}!'.format(subname)) + for item, val in mdl_dict.items(): + if subname in item: + if 'FULL' in item and readpatt not in item: + continue + if "rowclocks" in item: + rowclocks = val + elif "frameclocks" in item: + frameclocks = val + else: + if "SLOW" in readpatt and "SLOW" in item and detector in item: + frequencies.append(val) + elif "FAST" in readpatt and "FAST" in item: + frequencies.append(val) + if subname is not None and rowclocks is not None and frameclocks is not None and frequencies is not None: break - else: - frequencies = [] - mdl_dict = subarray_cases.to_flat_dict() - for subname in subarray_cases.subarray_cases: - subname = subname.split(sep='.')[0] - if subarray not in subname: - continue - log.debug('Found subarray case {}!'.format(subname)) - for item, val in mdl_dict.items(): - if subname in item: - if 'FULL' in item and readpatt not in item: - continue - if "rowclocks" in item: - rowclocks = val - elif "frameclocks" in item: - frameclocks = val - else: - if readpatt == "SLOW" and "SLOW" in item and detector in item: - frequencies.append(val) - elif readpatt == "FAST" and "FAST" in item: - frequencies.append(val) - if subname is not None and rowclocks is not None and frameclocks is not None and frequencies is not None: - break return subname, rowclocks, frameclocks, frequencies diff --git a/jwst/emicorr/emicorr_step.py b/jwst/emicorr/emicorr_step.py index a44b69099a7..06106200de0 100755 --- a/jwst/emicorr/emicorr_step.py +++ b/jwst/emicorr/emicorr_step.py @@ -22,7 +22,8 @@ class EmiCorrStep(Step): nints_to_phase = integer(default=None) # Number of integrations to phase nbins = integer(default=None) # Number of bins in one phased wave scale_reference = boolean(default=True) # If True, the reference wavelength will be scaled to the data's phase amplitude - skip = boolean(default=True) + skip = boolean(default=True) # Skip the step + onthefly_corr_freq = float_list(default=None) # Frequencies to use for correction """ reference_file_types = ['emicorr'] @@ -30,36 +31,47 @@ class EmiCorrStep(Step): def process(self, input): with datamodels.open(input) as input_model: - # Catch the cases to skip, i.e. all instruments other than MIRI + # Catch the cases to skip instrument = input_model.meta.instrument.name if instrument != 'MIRI': self.log.warning('EMI correction not implemented for instrument: {}'.format(instrument)) input_model.meta.cal_step.emicorr = 'SKIPPED' return input_model + readpatt = input_model.meta.exposure.readpatt + allowed_readpatts = ['FAST', 'FASTR1', 'SLOW', 'SLOWR1'] + if readpatt.upper() not in allowed_readpatts: + self.log.warning('EMI correction not implemented for read pattern: {}'.format(readpatt)) + input_model.meta.cal_step.emicorr = 'SKIPPED' + return input_model + # Setup parameters pars = { 'save_intermediate_results': self.save_intermediate_results, 'user_supplied_reffile': self.user_supplied_reffile, 'nints_to_phase': self.nints_to_phase, 'nbins': self.nbins, - 'scale_reference': self.scale_reference + 'scale_reference': self.scale_reference, + 'onthefly_corr_freq': self.onthefly_corr_freq } # Get the reference file save_onthefly_reffile, emicorr_ref_filename, emicorr_model = None, None, None - if self.user_supplied_reffile is None: - try: - emicorr_ref_filename = self.get_reference_file(input_model, 'emicorr') - # Create the reference file only if told to save outputs, else correct on-the-fly - if emicorr_ref_filename == 'N/A': - emicorr_ref_filename = None - else: - self.log.info('Using CRDS reference file: {}'.format(emicorr_ref_filename)) - emicorr_model = datamodels.EmiModel(emicorr_ref_filename) - except Exception: - # No reference file in CRDS - self.log.info('No CRDS emicorr reference file found. Creating on-the-fly reference file.') + if self.onthefly_corr_freq is not None: + emicorr_ref_filename = None + self.log.info('Correcting with reference file created on-the-fly.') + + elif self.user_supplied_reffile is None: + emicorr_ref_filename = self.get_reference_file(input_model, 'emicorr') + # Skip the spep if no reference file is found + if emicorr_ref_filename == 'N/A': + self.log.warning('No reference file found.') + self.log.warning('EMICORR step will be skipped') + input_model.meta.cal_step.emicorr = 'SKIPPED' + return input_model + else: + self.log.info('Using CRDS reference file: {}'.format(emicorr_ref_filename)) + emicorr_model = datamodels.EmiModel(emicorr_ref_filename) else: self.log.info('Using user-supplied reference file: {}'.format(self.user_supplied_reffile)) diff --git a/jwst/emicorr/tests/test_emicorr.py b/jwst/emicorr/tests/test_emicorr.py index f64824b91af..5a64f228077 100644 --- a/jwst/emicorr/tests/test_emicorr.py +++ b/jwst/emicorr/tests/test_emicorr.py @@ -9,7 +9,27 @@ from jwst.emicorr import emicorr, emicorr_step from stdatamodels.jwst.datamodels import Level1bModel, EmiModel -default_subarray_cases = emicorr.default_subarray_cases + +subarray_example_case = { + 'MASK1550': {'frameclocks': 23968, + 'freqs': { + 'FAST': ["Hz390", "Hz10"], + 'SLOW': { + 'MIRIFULONG': ['Hz390', 'Hz10_slow_MIRIFULONG'], + 'MIRIFUSHORT': ['Hz390', 'Hz10_slow_MIRIFUSHORT'], + 'MIRIMAGE': ['Hz390', 'Hz10_slow_MIRIMAGE']}}, + 'rowclocks': 82} +} + +emimdl = {'frequencies': { + "Hz390": {'frequency': 390.625, + 'phase_amplitudes': np.ones(20)+0.1}, + "Hz10": { 'frequency': 10.039216, + 'phase_amplitudes': np.ones(20)+0.5}}, + 'subarray_cases': subarray_example_case + } +emicorr_model = EmiModel(emimdl) + def mk_data_mdl(data, subarray, readpatt, detector): # create input_model @@ -48,17 +68,10 @@ def test_do_correction(): 'user_supplied_reffile': None, 'nints_to_phase': None, 'nbins': None, - 'scale_reference': True - } - emimdl = {'frequencies': { - "Hz390": {'frequency': 390.625, - 'phase_amplitudes': np.ones(20)+0.1}, - "Hz10": { 'frequency': 10.039216, - 'phase_amplitudes': np.ones(20)+0.5}}, - 'subarray_cases': default_subarray_cases + 'scale_reference': True, + 'onthefly_corr_freq': None } save_onthefly_reffile = None - emicorr_model = EmiModel(emimdl) outmdl = emicorr.do_correction(input_model, emicorr_model, save_onthefly_reffile, **pars) assert outmdl is not None @@ -67,8 +80,9 @@ def test_do_correction(): def test_apply_emicorr(): data = np.ones((1, 5, 20, 20)) input_model = mk_data_mdl(data, 'MASK1550', 'FAST', 'MIRIMAGE') - emicorr_model, save_onthefly_reffile = None, None - outmdl = emicorr.apply_emicorr(input_model, emicorr_model, save_onthefly_reffile, + emicorr_model, onthefly_corr_freq, save_onthefly_reffile = None, [218.3], None + outmdl = emicorr.apply_emicorr(input_model, emicorr_model, + onthefly_corr_freq, save_onthefly_reffile, save_intermediate_results=False, user_supplied_reffile=None, nints_to_phase=None, nbins_all=None, scale_reference=True) @@ -77,11 +91,11 @@ def test_apply_emicorr(): def test_get_subarcase(): subarray, readpatt, detector = 'MASK1550', 'FAST', 'MIRIMAGE' - subarray_info_r = emicorr.get_subarcase(default_subarray_cases, subarray, readpatt, detector) + subarray_info_r = emicorr.get_subarcase(emicorr_model, subarray, readpatt, detector) subname_r, rowclocks_r, frameclocks_r, freqs2correct_r = subarray_info_r # set up a fake configuration - subarray_info_f = emicorr.get_subarcase(default_subarray_cases, 'FULL', 'kkkkk', detector) + subarray_info_f = emicorr.get_subarcase(emicorr_model, 'FULL', 'kkkkk', detector) subname_f, rowclocks_f, frameclocks_f, freqs2correct_f = subarray_info_f # test if we get the right configuration diff --git a/jwst/outlier_detection/outlier_detection_ifu.py b/jwst/outlier_detection/outlier_detection_ifu.py index 1c9c16b67ff..48f1f536aab 100644 --- a/jwst/outlier_detection/outlier_detection_ifu.py +++ b/jwst/outlier_detection/outlier_detection_ifu.py @@ -41,8 +41,10 @@ class OutlierDetectionIFU(OutlierDetection): a. read in science data b. Store computed neighbor differences for all the pixels. The neighbor pixel differences are defined by the dispersion axis. - For MIRI (disp axis = 1) the neighbors to find differences are to the left and right of pixel - For NIRSpec (disp axis = 0) the neighbors to find the differences are above and below the pixel + For MIRI, with the dispersion axis along the y axis, the neighbors that are used to + to find the differences are to the left and right of each pixel being examined. + For NIRSpec, with the dispersion along the x axis, the neighbors that are used to + find the differences are above and below the pixel being examined. 3. For each input file store the minimum of the pixel neighbor differences 4. Comparing all the differences from all the input data find the minimum neighbor difference 5. Normalize minimum difference to local median of difference array @@ -206,8 +208,8 @@ def flag_outliers(self, idet, uq_det, ndet_files, # set all science data that have DO_NOT_USE to NAN sci[bad] = np.nan - # Compute left and right differences (MIRI dispersion axis = 1) - # For NIRSpec dispersion axis = 0, these differences are top, bottom + # Compute left and right differences (MIRI dispersion axis = 1 along y axis) + # For NIRSpec dispersion axis = 0 (along the x axis), these differences are top, bottom # prepend = 0 has the effect of keeping the same shape as sci and # for MIRI data (disp axis = 1) the first column = sci data # OR diff --git a/jwst/resample/resample.py b/jwst/resample/resample.py index b168151e8ce..7a8f7b67c48 100644 --- a/jwst/resample/resample.py +++ b/jwst/resample/resample.py @@ -1,5 +1,6 @@ import logging import os +import time import numpy as np from drizzle import util @@ -289,9 +290,18 @@ def resample_many_to_many(self): if not self.in_memory: # Write out model to disk, then return filename + # This sometimes causes problems in operations because + # the output file isn't always fully written before the + # execution continues output_name = output_model.meta.filename - output_model.save(output_name) - log.info(f"Saved model in {output_name}") + sleep_min = 1.0 + sleep_max = 40.0 + status = self.write_with_dimension_check(output_model, output_name, + min_wait_time=sleep_min, + max_wait_time=sleep_max) + if status: + log.warning(f"Wait time of {sleep_max}s exceeded.") + log.warning("Continuing with possibly corrupted i2d file") self.output_models.append(output_name) else: self.output_models.append(output_model.copy()) @@ -300,6 +310,60 @@ def resample_many_to_many(self): return self.output_models + def write_with_dimension_check(self, output_model, output_name, min_wait_time=1.0, max_wait_time=40.0): + """Write a datamodel to disk after waiting for a short interval, then read back the datamodel + and check if the dimensions match those of the model that was written. If not, delete the + output file, write the datamodel to disk, double the wait time, and check the dimensions + again. Keep iterating like this until either the datamodel data dimensions match, or the wait + wait time exceeds ``max_wait_time`` seconds. + + Parameters + ---------- + + output_model : JWST datamodel + The datamodel that is being written to disk + + output_name : string + The name of the file to be written to + + min_wait_time : float + Initial wait time between writing the file to disk and then reading it back + compare, in seconds + + max_wait_time : float + If the wait time exceeds this, in seconds, exit with a warning + + Returns + ------- + + status : int + Integer status value, 0 = good, 1 = max_wait_time exceeded, so file may be corrupted + + """ + status = 0 + filesavedOK = False + wait_time = min_wait_time + while not filesavedOK: + output_model.save(output_name) + if wait_time > max_wait_time: + log.info(f"Saved model in {output_name}") + status = 1 + break + log.info(f"Sleeping for {wait_time}s and reading back saved data") + time.sleep(wait_time) + readback_model = datamodels.open(output_name) + if output_model.data.shape == readback_model.data.shape: + log.info("Shape of read back model data matches that of input model") + filesavedOK = True + log.info(f"Saved model in {output_name}") + else: + log.info("Shapes of data in datamodel before and after saving don't match") + log.info("Removing output file and re-writing") + os.remove(output_name) + wait_time = wait_time * 2.0 + return status + + def resample_many_to_one(self): """Resample and coadd many inputs to a single output. diff --git a/jwst/stpipe/tests/test_crds.py b/jwst/stpipe/tests/test_crds.py index fa6449bcc9d..b4b9dc5957b 100644 --- a/jwst/stpipe/tests/test_crds.py +++ b/jwst/stpipe/tests/test_crds.py @@ -1,6 +1,4 @@ from os.path import join, dirname, basename -import shutil -import tempfile import pytest @@ -9,19 +7,6 @@ from jwst.stpipe import Step import crds -TMP_DIR = None -TMP_FITS = None - - -def setup(): - global TMP_DIR, TMP_FITS - TMP_DIR = tempfile.mkdtemp() - TMP_FITS = join(TMP_DIR, 'tmp.fits') - - -def teardown(): - shutil.rmtree(TMP_DIR) - class CrdsStep(Step): reference_file_types = ['flat'] @@ -53,7 +38,7 @@ def _run_flat_fetch_on_dataset(dataset_path): assert basename(step.ref_filename) == "jwst_nircam_flat_0296.fits" -def test_crds_step_override(): +def test_crds_step_override(tmp_path): """Run CRDS step with override parameter bypassing CRDS lookup.""" from stdatamodels.jwst import datamodels @@ -63,9 +48,10 @@ def test_crds_step_override(): assert step.ref_filename.endswith('data/flat.fits') assert result.meta.ref_file.flat.name.endswith('flat.fits') - result.to_fits(TMP_FITS) + fn = tmp_path / 'tmp.fits' + result.to_fits(fn) - with fits.open(TMP_FITS) as hdulist: + with fits.open(fn) as hdulist: assert hdulist[0].header['R_FLAT'].endswith('flat.fits') diff --git a/pyproject.toml b/pyproject.toml index 188c0c9d760..9ce066b06b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,7 +133,7 @@ test = [ "colorama>=0.4.1", "readchar>=3.0", "ruff", - "pytest>=6.0.0,<8.1.0", + "pytest>=6.0.0", "pytest-cov>=2.9.0", "pytest-doctestplus>=0.10.0", "requests_mock>=1.0",