From 0f4467440ba1a510a5305e5bed8b1bd4fd6957ca Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Thu, 5 Sep 2024 14:54:35 -0400 Subject: [PATCH 1/6] reenabled saving of blot models --- jwst/outlier_detection/imaging.py | 2 +- .../tests/test_outlier_detection.py | 24 +++++++++++++++++++ jwst/outlier_detection/utils.py | 23 +++++++++++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/jwst/outlier_detection/imaging.py b/jwst/outlier_detection/imaging.py index 559db8cec0..3b1c4b861f 100644 --- a/jwst/outlier_detection/imaging.py +++ b/jwst/outlier_detection/imaging.py @@ -127,7 +127,7 @@ def detect_outliers( with input_models: for image in input_models: if resample_data: - flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg) + flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg, save_blot=save_intermediate_results, make_output_path=make_output_path) else: flag_model_crs(image, median_data, snr1) input_models.shelve(image, modify=True) diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index b3b8c9b803..2355c2c1f0 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -405,6 +405,30 @@ def test_outlier_step_on_disk(three_sci_as_asn, tmp_cwd): assert zeroth.dq[12, 12] == OUTLIER_DO_NOT_USE result.shelve(zeroth, modify=False) + # Verify intermediate results were written to disk + dirname = tmp_cwd + all_files = glob(os.path.join(dirname, '*.fits')) + input_files = glob(os.path.join(dirname, '*_cal.fits')) + result_files = glob(os.path.join(dirname, '*outlierdetectionstep.fits')) + i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) + s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) + median_files = glob(os.path.join(dirname, '*median.fits')) + blot_files = glob(os.path.join(dirname, '*blot.fits')) + + assert len(result_files) == len(container) + + # i2d, median, blot files are written to the output directory + assert len(i2d_files) == len(container) + assert len(blot_files) == len(container) + assert len(median_files) == 1 + + # s2d files not written + assert len(s2d_files) == 0 + + # nothing else was written + assert len(all_files) == len(input_files) + len(i2d_files) + len(median_files) + len(result_files) + len(blot_files) + + def test_outlier_step_square_source_no_outliers(we_three_sci, tmp_cwd): """Test whole step with square source with sharp edges, no outliers""" diff --git a/jwst/outlier_detection/utils.py b/jwst/outlier_detection/utils.py index 6491415715..fc0c23442f 100644 --- a/jwst/outlier_detection/utils.py +++ b/jwst/outlier_detection/utils.py @@ -206,6 +206,8 @@ def flag_resampled_model_crs( scale1, scale2, backg, + save_blot=False, + make_output_path=None, ): if 'SPECTRAL' not in input_model.meta.wcs.output_frame.axes_type: input_pixflux_area = input_model.meta.photometry.pixelarea_steradians @@ -217,6 +219,15 @@ def flag_resampled_model_crs( pix_ratio = 1.0 blot = gwcs_blot(median_data, median_wcs, input_model.data.shape, input_model.meta.wcs, pix_ratio) + if save_blot: + if make_output_path is None: + raise ValueError("make_output_path must be provided if save_blot is True") + model_path = make_output_path(input_model.meta.filename, suffix='blot') + blot_model = _make_blot_model(input_model, blot) + blot_model.meta.filename = model_path + blot_model.save(model_path) + log.info(f"Saved model in {model_path}") + del blot_model # dq flags will be updated in-place _flag_resampled_model_crs(input_model, blot, snr1, snr2, scale1, scale2, backg) @@ -260,9 +271,11 @@ def flag_crs_in_models_with_resampling( scale1, scale2, backg, + save_blot=False, + make_output_path=None, ): for image in input_models: - flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg) + flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg, save_blot=save_blot, make_output_path=make_output_path) def flag_model_crs(image, blot, snr): @@ -276,3 +289,11 @@ def flag_model_crs(image, blot, snr): match_nans_and_flags(image) log.info(f"{np.count_nonzero(cr_mask)} pixels marked as outliers") + + +def _make_blot_model(input_model, blot): + blot_model = input_model.copy() + blot_model.data = blot + blot_model.dq *= 0 + blot_model.err *= 0.0 + return blot_model \ No newline at end of file From 4a35cfde7ea1ba2368d9fac71ca79a39c5fcc96b Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Thu, 5 Sep 2024 15:09:16 -0400 Subject: [PATCH 2/6] added changelog, restored mentions of saving blot models in docs --- CHANGES.rst | 2 ++ docs/jwst/outlier_detection/outlier_detection_imaging.rst | 3 ++- docs/jwst/outlier_detection/outlier_detection_spec.rst | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index eb25ae7a58..101a221acb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -496,6 +496,8 @@ outlier_detection - Fix errors in documentation describing arguments. [#8603] +- Re-enabled saving of blot models when `save_intermediate_results` is True. [#8758] + pathloss -------- diff --git a/docs/jwst/outlier_detection/outlier_detection_imaging.rst b/docs/jwst/outlier_detection/outlier_detection_imaging.rst index 15df877939..c2cd6318b9 100644 --- a/docs/jwst/outlier_detection/outlier_detection_imaging.rst +++ b/docs/jwst/outlier_detection/outlier_detection_imaging.rst @@ -75,7 +75,8 @@ Specifically, this routine performs the following operations: * The median image is written out to disk as `__median.fits` by default. #. By default, the median image is blotted back (inverse of resampling) to - match each original input image. + match each original input image. Resampled/blotted images are written out to disk if + the ``save_intermediate_results`` parameter is set to `True`. * **If resampling is turned off**, the median image is compared directly to each input image. diff --git a/docs/jwst/outlier_detection/outlier_detection_spec.rst b/docs/jwst/outlier_detection/outlier_detection_spec.rst index f66df77854..6d5b432b2f 100644 --- a/docs/jwst/outlier_detection/outlier_detection_spec.rst +++ b/docs/jwst/outlier_detection/outlier_detection_spec.rst @@ -39,6 +39,8 @@ Specifically, this routine performs the following operations (modified from the if the ``save_intermediate_results`` parameter is set to `True` #. Blot median image to match each original input image + - Resampled/blotted images are written out to disk if the ``save_intermediate_results`` + parameter is set to `True` - **If resampling is turned off**, the median image is used for comparison with the original input models for detecting outliers #. Perform statistical comparison between blotted image and original image to identify outliers From fcb47cb059d032ac5951f15ae97f37d77cc587fc Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Fri, 6 Sep 2024 16:00:49 -0400 Subject: [PATCH 3/6] changes from review, incl carrying save option into spec modes --- jwst/outlier_detection/imaging.py | 11 ++++++++++- jwst/outlier_detection/spec.py | 6 +++++- jwst/outlier_detection/utils.py | 11 ++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/jwst/outlier_detection/imaging.py b/jwst/outlier_detection/imaging.py index 3b1c4b861f..9f1aabf597 100644 --- a/jwst/outlier_detection/imaging.py +++ b/jwst/outlier_detection/imaging.py @@ -127,7 +127,16 @@ def detect_outliers( with input_models: for image in input_models: if resample_data: - flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg, save_blot=save_intermediate_results, make_output_path=make_output_path) + flag_resampled_model_crs(image, + median_data, + median_wcs, + snr1, + snr2, + scale1, + scale2, + backg, + save_blot=save_intermediate_results, + make_output_path=make_output_path) else: flag_model_crs(image, median_data, snr1) input_models.shelve(image, modify=True) diff --git a/jwst/outlier_detection/spec.py b/jwst/outlier_detection/spec.py index 74a21965a4..c482aad8b9 100644 --- a/jwst/outlier_detection/spec.py +++ b/jwst/outlier_detection/spec.py @@ -130,7 +130,11 @@ def detect_outliers( scale1, scale2, backg, + save_blot=save_intermediate_results, + make_output_path=make_output_path ) else: - flag_crs_in_models(input_models, median_data, snr1) + flag_crs_in_models(input_models, + median_data, + snr1) return input_models diff --git a/jwst/outlier_detection/utils.py b/jwst/outlier_detection/utils.py index fc0c23442f..2a18ee4b56 100644 --- a/jwst/outlier_detection/utils.py +++ b/jwst/outlier_detection/utils.py @@ -275,7 +275,16 @@ def flag_crs_in_models_with_resampling( make_output_path=None, ): for image in input_models: - flag_resampled_model_crs(image, median_data, median_wcs, snr1, snr2, scale1, scale2, backg, save_blot=save_blot, make_output_path=make_output_path) + flag_resampled_model_crs(image, + median_data, + median_wcs, + snr1, + snr2, + scale1, + scale2, + backg, + save_blot=save_blot, + make_output_path=make_output_path) def flag_model_crs(image, blot, snr): From 0ce4c604a84d6873675e945b13c6dcc897a5da64 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 9 Sep 2024 09:08:15 -0400 Subject: [PATCH 4/6] updated spec unit test to account for blot files --- .../tests/test_outlier_detection.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index 2355c2c1f0..ebd732437b 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -288,11 +288,13 @@ def test_outlier_step_spec(tmp_cwd, tmp_path): i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) median_files = glob(os.path.join(dirname, '*median.fits')) + blot_files = glob(os.path.join(dirname, '*blot.fits')) # intermediate files are removed assert len(i2d_files) == 0 assert len(s2d_files) == 0 assert len(median_files) == 0 + assert len(blot_files) == 0 # result files are written to the output directory if dirname == output_dir: @@ -320,25 +322,31 @@ def test_outlier_step_spec(tmp_cwd, tmp_path): i2d_files = glob(os.path.join(dirname, '*i2d*.fits')) s2d_files = glob(os.path.join(dirname, '*outlier_s2d.fits')) median_files = glob(os.path.join(dirname, '*median.fits')) + blot_files = glob(os.path.join(dirname, '*blot.fits')) if dirname == output_dir: # result files are written to the output directory assert len(result_files) == len(container) - # s2d and median files are written to the output directory + # s2d, median, and blot files are written to the output directory assert len(s2d_files) == len(container) + assert len(blot_files) == len(container) assert len(median_files) == 1 # i2d files not written assert len(i2d_files) == 0 # nothing else was written - assert len(all_files) == len(s2d_files) + len(median_files) + len(result_files) + assert len(all_files) == len(s2d_files) + \ + len(median_files) + \ + len(result_files) + \ + len(blot_files) else: # nothing should be written to the current directory assert len(result_files) == 0 assert len(s2d_files) == 0 assert len(median_files) == 0 assert len(i2d_files) == 0 + assert len(blot_files) == 0 assert len(all_files) == 0 miri_rate.close() From 42f33abfba2dd621c694a28982697b949e1eec14 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 9 Sep 2024 12:28:12 -0400 Subject: [PATCH 5/6] limit blotmodel to just sci array --- jwst/outlier_detection/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jwst/outlier_detection/utils.py b/jwst/outlier_detection/utils.py index 2a18ee4b56..9adccbba25 100644 --- a/jwst/outlier_detection/utils.py +++ b/jwst/outlier_detection/utils.py @@ -301,8 +301,6 @@ def flag_model_crs(image, blot, snr): def _make_blot_model(input_model, blot): - blot_model = input_model.copy() + blot_model = type(input_model)() blot_model.data = blot - blot_model.dq *= 0 - blot_model.err *= 0.0 return blot_model \ No newline at end of file From 8350208b53b706628825ca58b131d19705f0704b Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 9 Sep 2024 12:57:50 -0400 Subject: [PATCH 6/6] transfer metadata from input model to blot model --- jwst/outlier_detection/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwst/outlier_detection/utils.py b/jwst/outlier_detection/utils.py index 9adccbba25..4abdfbad1b 100644 --- a/jwst/outlier_detection/utils.py +++ b/jwst/outlier_detection/utils.py @@ -303,4 +303,5 @@ def flag_model_crs(image, blot, snr): def _make_blot_model(input_model, blot): blot_model = type(input_model)() blot_model.data = blot + blot_model.update(input_model) return blot_model \ No newline at end of file