From f9842372475d8fc4eb1634f2c0995a2489b0f327 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 15:06:10 -0400 Subject: [PATCH 01/24] Update test data for master background with dedicated background observation --- jwst/regtest/test_miri_lrs_dedicated_mbkg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py index 50ef78cb5c..1949f5646c 100644 --- a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py +++ b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py @@ -10,9 +10,9 @@ def run_pipeline(rtdata_module): rtdata = rtdata_module - rtdata.get_asn("miri/lrs/miri_lrs_mbkg_dedicated_spec3_asn.json") + rtdata.get_asn("miri/lrs/jw01529-o003_spec3_with_bg_00001_asn.json") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background') + MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub') return rtdata @@ -22,12 +22,12 @@ def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs): """Run a test for MIRI LRS data with dedicated background exposures.""" rtdata = run_pipeline - rtdata.output = "miri_lrs_seq2_exp2_master_background.fits" + rtdata.output = "jw01529003001_03103_00011_mirimage_mbsub.fits" # Get the truth file rtdata.get_truth(os.path.join( "truth/test_miri_lrs_dedicated_mbkg", - "miri_lrs_seq2_exp2_master_background.fits")) + "jw01529003001_03103_00011_mirimage_mbsub.fits")) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) From c1898e4533e301badca971527ac55327e0c66083 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 17:05:28 -0400 Subject: [PATCH 02/24] Copy wavelengths for first spectrum so they are not sorted in place --- jwst/combine_1d/combine1d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/combine_1d/combine1d.py b/jwst/combine_1d/combine1d.py index c4a6021e9d..9d69109ceb 100644 --- a/jwst/combine_1d/combine1d.py +++ b/jwst/combine_1d/combine1d.py @@ -355,7 +355,7 @@ def count_input(input_spectra): # only include spectra that have more than 1 data point if len(input_wl) > 1: if wl is None: - wl = input_wl + wl = input_wl.copy() else: wl = np.hstack((input_wl, wl)) wl.sort() From 359a7b90c6f2195c29b27bee6a2cdbc199f01384 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 17:06:38 -0400 Subject: [PATCH 03/24] Small fixes for new ModelContainer --- jwst/master_background/master_background_step.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jwst/master_background/master_background_step.py b/jwst/master_background/master_background_step.py index 727b351c47..ce9c21b560 100755 --- a/jwst/master_background/master_background_step.py +++ b/jwst/master_background/master_background_step.py @@ -90,7 +90,6 @@ def process(self, input): del _ result = ModelContainer() background_2d_collection = ModelContainer() - background_2d_collection.update(input_data) for model in input_data: background_2d = expand_to_2d(model, self.user_background) result.append(subtract_2d_background(model, background_2d)) @@ -102,7 +101,7 @@ def process(self, input): else: asn_id = None background_2d = expand_to_2d(input_data, self.user_background) - background_2d_collection = background_2d + background_2d_collection = ModelContainer([background_2d]) result = subtract_2d_background(input_data, background_2d) # Record name of user-supplied master background spectrum result.meta.background.master_background_file = basename(self.user_background) From 86b767ce78a392abc5cfeaef7d1985b5f11794f3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 17:19:53 -0400 Subject: [PATCH 04/24] Update data and test background products --- jwst/regtest/test_miri_lrs_dedicated_mbkg.py | 14 ++++++++------ jwst/regtest/test_miri_lrs_masterbg_user.py | 16 +++++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py index 1949f5646c..a2bcf42eb3 100644 --- a/jwst/regtest/test_miri_lrs_dedicated_mbkg.py +++ b/jwst/regtest/test_miri_lrs_dedicated_mbkg.py @@ -12,22 +12,24 @@ def run_pipeline(rtdata_module): rtdata.get_asn("miri/lrs/jw01529-o003_spec3_with_bg_00001_asn.json") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub') + MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub', + save_background=True) return rtdata @pytest.mark.bigdata -def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs): +@pytest.mark.parametrize('output', ['jw01529-o004_t002_miri_p750l_o003_masterbg1d.fits', + 'jw01529003001_03103_00011_mirimage_o003_masterbg2d.fits', + 'jw01529003001_03103_00011_mirimage_mbsub.fits']) +def test_miri_lrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs, output): """Run a test for MIRI LRS data with dedicated background exposures.""" rtdata = run_pipeline - rtdata.output = "jw01529003001_03103_00011_mirimage_mbsub.fits" + rtdata.output = output # Get the truth file - rtdata.get_truth(os.path.join( - "truth/test_miri_lrs_dedicated_mbkg", - "jw01529003001_03103_00011_mirimage_mbsub.fits")) + rtdata.get_truth(os.path.join("truth/test_miri_lrs_dedicated_mbkg", output)) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) diff --git a/jwst/regtest/test_miri_lrs_masterbg_user.py b/jwst/regtest/test_miri_lrs_masterbg_user.py index e57570bdbc..6e49984924 100644 --- a/jwst/regtest/test_miri_lrs_masterbg_user.py +++ b/jwst/regtest/test_miri_lrs_masterbg_user.py @@ -11,28 +11,30 @@ def run_pipeline(rtdata_module): rtdata = rtdata_module # This is the user-supplied background file. - rtdata.get_data("miri/lrs/miri_lrs_bkg_x1d.fits") + rtdata.get_data("miri/lrs/jw01529-o004_t002_miri_p750l_x1d.fits") user_bkg = rtdata.input # This is the input file for the master_background step. - rtdata.get_data("miri/lrs/miri_lrs_sci+bkg_cal.fits") + rtdata.get_data("miri/lrs/jw01529003001_03103_00011_mirimage_cal.fits") MasterBackgroundStep.call(rtdata.input, user_background=user_bkg, - save_results=True, suffix='master_background') + save_results=True, suffix='user_mbsub', + save_background=True) return rtdata @pytest.mark.bigdata -def test_miri_lrs_masterbg_user(run_pipeline, fitsdiff_default_kwargs): +@pytest.mark.parametrize('output', ['jw01529003001_03103_00011_mirimage_masterbg2d.fits', + 'jw01529003001_03103_00011_mirimage_user_mbsub.fits']) +def test_miri_lrs_masterbg_user(run_pipeline, fitsdiff_default_kwargs, output): """Run a test for MIRI LRS data with a user-supplied background file.""" rtdata = run_pipeline - rtdata.output = "miri_lrs_sci+bkg_master_background.fits" + rtdata.output = output # Get the truth file - rtdata.get_truth(os.path.join("truth/test_miri_lrs_masterbg_user", - "miri_lrs_sci+bkg_master_background.fits")) + rtdata.get_truth(os.path.join("truth/test_miri_lrs_masterbg_user", output)) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) From c889fa393997b1a1202e54ecde127512b7616bed Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 15:30:01 -0400 Subject: [PATCH 05/24] Add mbsub to the list of known suffixes --- jwst/lib/suffix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/lib/suffix.py b/jwst/lib/suffix.py index a7af86beb7..b72306dce4 100644 --- a/jwst/lib/suffix.py +++ b/jwst/lib/suffix.py @@ -41,7 +41,7 @@ 'c1d', 'cal', 'calints', 'cat', 'crf', 'crfints', 'dark', 'i2d', - 'median', + 'mbsub', 'median', 'phot', 'psf-amiavg', 'psfalign', 'psfstack', 'psfsub', 'ramp', 'rate', 'rateints', 'residual_fringe', 's2d', 's3d', 'snr', From c2cd58f690deb92e187c6359b99dd1a85094b943 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 15:34:26 -0400 Subject: [PATCH 06/24] Add a test/demo for background extraction for nodded LRS data --- jwst/regtest/test_miri_lrs_nod_masterbg.py | 83 ++++++++++++++++++---- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/jwst/regtest/test_miri_lrs_nod_masterbg.py b/jwst/regtest/test_miri_lrs_nod_masterbg.py index 0a3b2ba203..d17b65fad2 100644 --- a/jwst/regtest/test_miri_lrs_nod_masterbg.py +++ b/jwst/regtest/test_miri_lrs_nod_masterbg.py @@ -2,33 +2,92 @@ import pytest from astropy.io.fits.diff import FITSDiff -from jwst.master_background import MasterBackgroundStep +from jwst.stpipe import Step @pytest.fixture(scope="module") -def run_pipeline(rtdata_module): +def run_pipeline_with_master_bg(rtdata_module): rtdata = rtdata_module - rtdata.get_asn("miri/lrs/miri_lrs_mbkg_nodded_spec3_asn.json") + # Get rate files and custom extraction parameters + rate_files = ['jw01530005001_03103_00001_mirimage_rate.fits', + 'jw01530005001_03103_00002_mirimage_rate.fits'] + ref_files = ['jwst_miri_extract1d_nod1_bg.json', + 'jwst_miri_extract1d_nod2_bg.json'] + for rate, ref in zip(rate_files, ref_files): + rtdata.get_data(f"miri/lrs/{ref}") + rtdata.get_data(f"miri/lrs/{rate}") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background') + Step.from_cmdline([ + "calwebb_spec2", + rate, + f"--steps.extract_1d.override_extract1d={ref}", + "--steps.extract_1d.use_source_pos=False", + ]) + + # Get the ASN, but not the data - use the spec2 input products + rtdata.get_data("miri/lrs/jw01530-o005_20221202t204827_spec3_00001_asn.json") + Step.from_cmdline([ + "calwebb_spec3", + rtdata.input, + "--steps.master_background.skip=False", + "--steps.master_background.save_background=True", + "--steps.master_background.save_results=True", + ]) return rtdata @pytest.mark.bigdata -@pytest.mark.parametrize("nod_seq", ["seq1", "seq2"]) -def test_miri_lrs_nod_masterbg(run_pipeline, fitsdiff_default_kwargs, nod_seq): - """Run a regression test for nodded MIRI LRS data.""" +@pytest.mark.parametrize("nod", [1, 2]) +@pytest.mark.parametrize("suffix", ["cal", "x1d", "s2d", "mbsub", "o005_crf", + "o005_masterbg2d"]) +def test_miri_lrs_nod_bg(run_pipeline_with_master_bg, fitsdiff_default_kwargs, + nod, suffix): + """Run a regression test for nodded MIRI LRS data with background extraction.""" + rtdata = run_pipeline_with_master_bg + + # Check output products for nod 1 or 2 + output_file = f"jw01530005001_03103_0000{nod}_mirimage_{suffix}.fits" + rtdata.output = output_file + + # Get the truth file + rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file)) + + # Compare the results + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + + +def test_miri_lrs_nod_masterbg1d(run_pipeline_with_master_bg, fitsdiff_default_kwargs): + rtdata = run_pipeline_with_master_bg + + # Check 1D masterbg output product: created with root name from nod 1 + output_file = "jw01530005001_03103_00001_mirimage_o005_masterbg1d.fits" + rtdata.output = output_file + + # Get the truth file + rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file)) + + # Compare the results + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + + +@pytest.mark.bigdata +@pytest.mark.parametrize("suffix", ["s2d", "x1d"]) +def test_miri_lrs_nod_bg_spec3(run_pipeline_with_master_bg, fitsdiff_default_kwargs, + suffix): + """Run a regression test for nodded MIRI LRS with master background subtraction.""" + rtdata = run_pipeline_with_master_bg - rtdata = run_pipeline - rtdata.output = "miri_lrs_nod_" + nod_seq + "_exp1_master_background.fits" + # Check combined output products + output_file = f"jw01530-o005_t004_miri_p750l_{suffix}.fits" + rtdata.output = output_file # Get the truth file - rtdata.get_truth(os.path.join( - "truth/test_miri_lrs_nod_masterbg", - "miri_lrs_nod_" + nod_seq + "_exp1_master_background.fits")) + rtdata.get_truth(os.path.join("truth/test_miri_lrs_nod_masterbg", output_file)) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) From ce815ebd3c9d7bbcbd17fa5a5ea0de005ba051f8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 16:37:05 -0400 Subject: [PATCH 07/24] Add change notes --- changes/8927.combine_1d.rst | 1 + changes/8927.master_background.rst | 1 + changes/8927.pipeline.rst | 1 + 3 files changed, 3 insertions(+) create mode 100644 changes/8927.combine_1d.rst create mode 100644 changes/8927.master_background.rst create mode 100644 changes/8927.pipeline.rst diff --git a/changes/8927.combine_1d.rst b/changes/8927.combine_1d.rst new file mode 100644 index 0000000000..b05b3e95bc --- /dev/null +++ b/changes/8927.combine_1d.rst @@ -0,0 +1 @@ +Fix wavelength sort order for single input spectrum. diff --git a/changes/8927.master_background.rst b/changes/8927.master_background.rst new file mode 100644 index 0000000000..cf71da1e52 --- /dev/null +++ b/changes/8927.master_background.rst @@ -0,0 +1 @@ +Fix ModelContainer handling for user background input. diff --git a/changes/8927.pipeline.rst b/changes/8927.pipeline.rst new file mode 100644 index 0000000000..e03ac78254 --- /dev/null +++ b/changes/8927.pipeline.rst @@ -0,0 +1 @@ +Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``. From 952846af742cc1045ba4d50fc7907e909d80a66c Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 28 Oct 2024 16:56:26 -0400 Subject: [PATCH 08/24] Add missing 'bigdata' mark --- jwst/regtest/test_miri_lrs_nod_masterbg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwst/regtest/test_miri_lrs_nod_masterbg.py b/jwst/regtest/test_miri_lrs_nod_masterbg.py index d17b65fad2..69554b5a5a 100644 --- a/jwst/regtest/test_miri_lrs_nod_masterbg.py +++ b/jwst/regtest/test_miri_lrs_nod_masterbg.py @@ -60,6 +60,7 @@ def test_miri_lrs_nod_bg(run_pipeline_with_master_bg, fitsdiff_default_kwargs, assert diff.identical, diff.report() +@pytest.mark.bigdata def test_miri_lrs_nod_masterbg1d(run_pipeline_with_master_bg, fitsdiff_default_kwargs): rtdata = run_pipeline_with_master_bg From cdc98f9a8e11c904caf2e8f012fd37bf8babc30c Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 25 Oct 2024 13:18:45 -0400 Subject: [PATCH 09/24] Update dark data for MIRI dark pipeline tests --- jwst/regtest/test_miri_dark.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwst/regtest/test_miri_dark.py b/jwst/regtest/test_miri_dark.py index 61aeb1f9d0..78c3f5617b 100644 --- a/jwst/regtest/test_miri_dark.py +++ b/jwst/regtest/test_miri_dark.py @@ -7,7 +7,9 @@ @pytest.mark.bigdata @pytest.mark.parametrize( 'exposure', - ['jw00001001001_01101_00001_mirimage', 'jw02201001001_01101_00001_MIRIMAGE'] + ['jw04482014001_02102_00001_mirifulong', + 'jw04482014001_02102_00001_mirifushort', + 'jw04482014001_02102_00001_mirimage'] ) def test_miri_dark_pipeline(exposure, rtdata, fitsdiff_default_kwargs): """Test the DarkPipeline on MIRI dark exposures""" From c1890897ca0e09694b68c4bc54fabef4299e8f63 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 24 Oct 2024 11:16:53 -0400 Subject: [PATCH 10/24] Update NIRSpec BOTS regtest data --- jwst/regtest/test_nirspec_brightobj.py | 37 +++++++++++--------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/jwst/regtest/test_nirspec_brightobj.py b/jwst/regtest/test_nirspec_brightobj.py index cc5fedb94b..b70d074127 100644 --- a/jwst/regtest/test_nirspec_brightobj.py +++ b/jwst/regtest/test_nirspec_brightobj.py @@ -20,7 +20,7 @@ def run_tso_spec2_pipeline(rtdata_module, request): rtdata = rtdata_module # Get the input exposure - rtdata.get_data('nirspec/tso/jw02420001001_04101_00001-seg001_nrs1_rateints.fits') + rtdata.get_data('nirspec/tso/jw02420001001_04101_00001-first100_nrs1_rateints.fits') # Run the calwebb_spec2 pipeline; args = ["calwebb_spec2", rtdata.input, @@ -28,6 +28,7 @@ def run_tso_spec2_pipeline(rtdata_module, request): "--steps.extract_2d.save_results=True", "--steps.wavecorr.save_results=True", "--steps.flat_field.save_results=True", + "--steps.flat_field.save_interpolated_flat=True", "--steps.photom.save_results=True"] Step.from_cmdline(args) @@ -35,7 +36,9 @@ def run_tso_spec2_pipeline(rtdata_module, request): @pytest.mark.bigdata -@pytest.mark.parametrize("suffix", ['assign_wcs', 'extract_2d', 'wavecorr', 'flat_field', 'photom', 'calints', 'x1dints']) +@pytest.mark.parametrize("suffix", ['assign_wcs', 'extract_2d', 'wavecorr', + 'flat_field', 'photom', 'calints', 'x1dints', + 'interpolatedflat']) def test_nirspec_brightobj_spec2(run_tso_spec2_pipeline, fitsdiff_default_kwargs, suffix): """ Regression test of calwebb_spec2 pipeline performed on NIRSpec @@ -57,27 +60,18 @@ def test_nirspec_brightobj_spec2(run_tso_spec2_pipeline, fitsdiff_default_kwargs @pytest.mark.bigdata def test_flat_field_step_user_supplied_flat(rtdata, fitsdiff_default_kwargs): """Test providing a user-supplied flat field to the FlatFieldStep""" - data = rtdata.get_data('nirspec/tso/nrs2_wavecorr.fits') - user_supplied_flat = rtdata.get_data('nirspec/tso/nrs2_interpolatedflat.fits') + basename = 'jw02420001001_04101_00001-first100_nrs1' + output_file = f'{basename}_flat_from_user_file.fits' - data_flat_fielded = FlatFieldStep.call(data, user_supplied_flat=user_supplied_flat) - rtdata.output = 'flat_fielded_step_user_supplied.fits' - data_flat_fielded.write(rtdata.output) - - rtdata.get_truth('truth/test_nirspec_brightobj_spec2/flat_fielded_step_user_supplied.fits') - diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) - assert diff.identical, diff.report() + data = rtdata.get_data(f'nirspec/tso/{basename}_wavecorr.fits') + user_supplied_flat = rtdata.get_data(f'nirspec/tso/{basename}_user_flat.fits') + data_flat_fielded = FlatFieldStep.call(data, user_supplied_flat=user_supplied_flat, + save_results=False) + rtdata.output = output_file + data_flat_fielded.write(rtdata.output) -@pytest.mark.bigdata -def test_flat_field_bots_interp_flat(rtdata, fitsdiff_default_kwargs): - """Test the interpolated flat for a NRS BOTS exposure""" - data = rtdata.get_data('nirspec/tso/jw93056001001_short_nrs1_wavecorr.fits') - - FlatFieldStep.call(data, save_interpolated_flat=True) - rtdata.output = 'jw93056001001_short_nrs1_wavecorr_interpolatedflat.fits' - - rtdata.get_truth('truth/test_nirspec_brightobj_spec2/jw93056001001_short_nrs1_wavecorr_interpolatedflat.fits') + rtdata.get_truth(f'truth/test_nirspec_brightobj_spec2/{output_file}') diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report() @@ -85,7 +79,8 @@ def test_flat_field_bots_interp_flat(rtdata, fitsdiff_default_kwargs): @pytest.mark.bigdata def test_ff_inv(rtdata, fitsdiff_default_kwargs): """Test flat field inversion""" - with dm.open(rtdata.get_data('nirspec/tso/nrs2_wavecorr.fits')) as data: + basename = 'jw02420001001_04101_00001-first100_nrs1' + with dm.open(rtdata.get_data(f'nirspec/tso/{basename}_wavecorr.fits')) as data: flatted = FlatFieldStep.call(data) unflatted = FlatFieldStep.call(flatted, inverse=True) From 2b277b91466e24a1fea011b2e9493eb1da4e1b31 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 24 Oct 2024 13:43:55 -0400 Subject: [PATCH 11/24] Update inputs for NIRSpec FS steps --- jwst/regtest/test_nirspec_brightobj.py | 3 +++ jwst/regtest/test_nirspec_fs_spec2.py | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/jwst/regtest/test_nirspec_brightobj.py b/jwst/regtest/test_nirspec_brightobj.py index b70d074127..0a976050bc 100644 --- a/jwst/regtest/test_nirspec_brightobj.py +++ b/jwst/regtest/test_nirspec_brightobj.py @@ -20,6 +20,9 @@ def run_tso_spec2_pipeline(rtdata_module, request): rtdata = rtdata_module # Get the input exposure + # Input data is from jw02420001001_04101_00001-seg001_nrs1_rateints.fits, + # modified to truncate the data to the first 100 integrations, for + # faster processing. rtdata.get_data('nirspec/tso/jw02420001001_04101_00001-first100_nrs1_rateints.fits') # Run the calwebb_spec2 pipeline; diff --git a/jwst/regtest/test_nirspec_fs_spec2.py b/jwst/regtest/test_nirspec_fs_spec2.py index c2094dbcb8..e0bb412a1c 100644 --- a/jwst/regtest/test_nirspec_fs_spec2.py +++ b/jwst/regtest/test_nirspec_fs_spec2.py @@ -130,7 +130,8 @@ def test_nirspec_fs_spec2_pixel_replace(run_pipeline_pixel_replace, fitsdiff_def @pytest.mark.bigdata def test_pathloss_corrpars(rtdata): """Test PathLossStep using correction_pars""" - with dm.open(rtdata.get_data('nirspec/fs/nrs1_flat_field.fits')) as data: + basename = 'jw02072002001_05101_00001_nrs1_flatfieldstep' + with dm.open(rtdata.get_data(f'nirspec/fs/{basename}.fits')) as data: pls = PathLossStep() corrected = pls.run(data) @@ -148,7 +149,8 @@ def test_pathloss_corrpars(rtdata): @pytest.mark.bigdata def test_pathloss_inverse(rtdata): """Test PathLossStep using inversion""" - with dm.open(rtdata.get_data('nirspec/fs/nrs1_flat_field.fits')) as data: + basename = 'jw02072002001_05101_00001_nrs1_flatfieldstep' + with dm.open(rtdata.get_data(f'nirspec/fs/{basename}.fits')) as data: pls = PathLossStep() corrected = pls.run(data) @@ -168,7 +170,8 @@ def test_pathloss_inverse(rtdata): @pytest.mark.bigdata def test_pathloss_source_type(rtdata): """Test PathLossStep forcing source type""" - with dm.open(rtdata.get_data('nirspec/fs/nrs1_flat_field.fits')) as data: + basename = 'jw02072002001_05101_00001_nrs1_flatfieldstep' + with dm.open(rtdata.get_data(f'nirspec/fs/{basename}.fits')) as data: pls = PathLossStep() pls.source_type = 'extended' pls.run(data) @@ -189,7 +192,7 @@ def test_nirspec_fs_rateints_spec2(rtdata_module): """ rtdata = rtdata_module - rtdata.get_data("nirspec/fs/jw01128004001_03102_00001_nrs1_new_rateints.fits") + rtdata.get_data("nirspec/fs/jw01128004001_03102_00001_nrs1_rateints.fits") # Run the spec2 pipeline on a (3D) _rateints file args = ["calwebb_spec2", rtdata.input] From 5f26060868888d26635bd98d953ff147fe06d1e2 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 18 Oct 2024 13:15:35 -0400 Subject: [PATCH 12/24] Update MOS data for masterbackground tests --- jwst/regtest/test_nirspec_masterbackground.py | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/jwst/regtest/test_nirspec_masterbackground.py b/jwst/regtest/test_nirspec_masterbackground.py index 5c45d41265..39172222bc 100644 --- a/jwst/regtest/test_nirspec_masterbackground.py +++ b/jwst/regtest/test_nirspec_masterbackground.py @@ -18,8 +18,8 @@ def run_spec2_mbkg(rtdata_module): rtdata = rtdata_module # Get data - rtdata.get_data('nirspec/mos/jw01180025001_01_msa.fits') - rtdata.get_data('nirspec/mos/jw01180025001_05101_00001_nrs2_rate.fits') + rtdata.get_data('nirspec/mos/jw01448011001_01_msa.fits') + rtdata.get_data('nirspec/mos/jw01448011001_02101_00001_nrs2_rate.fits') # Run the pipeline step_params = { @@ -39,16 +39,18 @@ def run_spec2_mbkg_user(rtdata_module): rtdata = rtdata_module # Get data - rtdata.get_data('nirspec/mos/nrs_mos_3pointnod_1_msa.fits') - rtdata.get_data('nirspec/mos/jw00626030001_02103_00001_nrs1_masterbg1d.fits') - rtdata.get_data('nirspec/mos/jw00626030001_02103_00001_nrs1_rate.fits') + user_bg = 'jw01448011001_02101_00001_nrs2_user_bg.fits' + rtdata.get_data('nirspec/mos/jw01448011001_01_msa.fits') + rtdata.get_data(f'nirspec/mos/{user_bg}') + rtdata.get_data('nirspec/mos/jw01448011001_02101_00001_nrs2_rate.fits') # Run the pipeline step_params = { 'step': 'calwebb_spec2', 'args': [ '--steps.master_background_mos.skip=false', - '--steps.master_background_mos.user_background=jw00626030001_02103_00001_nrs1_masterbg1d.fits' + f'--steps.master_background_mos.user_background={user_bg}', + f'--output_file={user_bg}' ] } rtdata = rt.run_step_from_dict(rtdata, **step_params) @@ -57,7 +59,7 @@ def run_spec2_mbkg_user(rtdata_module): def test_masterbkg_rerun(rtdata): """Test to ensure sequential runs of the step are consistent""" - with dm.open(rtdata.get_data('nirspec/mos/nrs_mos_with_bkgslits_srctype.fits')) as data: + with dm.open(rtdata.get_data('nirspec/mos/jw01448011001_02101_00001_nrs2_srctype.fits')) as data: mbs = MasterBackgroundMosStep() corrected = mbs.run(data) corrected_again = mbs.run(data) @@ -72,7 +74,7 @@ def test_masterbkg_rerun(rtdata): def test_masterbkg_corrpars(rtdata): """Test for correction parameters""" - with dm.open(rtdata.get_data('nirspec/mos/nrs_mos_with_bkgslits_srctype.fits')) as data: + with dm.open(rtdata.get_data('nirspec/mos/jw01448011001_02101_00001_nrs2_srctype.fits')) as data: mbs = MasterBackgroundMosStep() corrected = mbs.run(data) @@ -94,6 +96,11 @@ def test_masterbkg_corrpars(rtdata): def test_nirspec_mos_mbkg(suffix, run_spec2_mbkg, fitsdiff_default_kwargs): """Run spec2 with master background""" rtdata = run_spec2_mbkg + + # Adjust tolerance for machine precision with float32 drizzle code + if suffix == "s2d": + fitsdiff_default_kwargs["rtol"] = 1e-2 + fitsdiff_default_kwargs["atol"] = 2e-4 rt.is_like_truth(rtdata, fitsdiff_default_kwargs, suffix, truth_path='truth/test_nirspec_mos_mbkg') @@ -104,7 +111,17 @@ def test_nirspec_mos_mbkg(suffix, run_spec2_mbkg, fitsdiff_default_kwargs): def test_nirspec_mos_mbkg_user(suffix, run_spec2_mbkg_user, fitsdiff_default_kwargs): """Run spec2 with master background and user-supplied mbkg""" rtdata = run_spec2_mbkg_user - rt.is_like_truth(rtdata, fitsdiff_default_kwargs, suffix, truth_path='truth/test_nirspec_mos_mbkg_user') + + output_filename = f"jw01448011001_02101_00001_nrs2_user_bg_{suffix}.fits" + rtdata.output = output_filename + rtdata.get_truth(f"truth/test_nirspec_mos_mbkg_user/{output_filename}") + + # Adjust tolerance for machine precision with float32 drizzle code + if suffix == "s2d": + fitsdiff_default_kwargs["rtol"] = 1e-2 + fitsdiff_default_kwargs["atol"] = 2e-4 + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() def test_nirspec_fs_mbkg_user(rtdata, fitsdiff_default_kwargs): From a72868d7844c549ce683b21af03c944acdd5b8c4 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 21 Oct 2024 17:35:14 -0400 Subject: [PATCH 13/24] Update FS master background check to use existing regtest data + new user bg --- jwst/regtest/test_nirspec_masterbackground.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwst/regtest/test_nirspec_masterbackground.py b/jwst/regtest/test_nirspec_masterbackground.py index 39172222bc..157f53f833 100644 --- a/jwst/regtest/test_nirspec_masterbackground.py +++ b/jwst/regtest/test_nirspec_masterbackground.py @@ -128,16 +128,16 @@ def test_nirspec_fs_mbkg_user(rtdata, fitsdiff_default_kwargs): """Run a test for NIRSpec FS data with a user-supplied background file.""" # Get user-supplied background - user_background = "v2_nrs_bkg_user_clean_x1d.fits" + user_background = "jw01309-o022_b000000021_nirspec_f290lp-g395h-s200a1-allslits_x1d.fits" rtdata.get_data(f"nirspec/fs/{user_background}") # Get input data - rtdata.get_data("nirspec/fs/nrs_sci+bkg_cal.fits") + rtdata.get_data("nirspec/fs/jw01309022001_04102_00001_nrs1_cal.fits") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background', + MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub', user_background=user_background) - output = "nrs_sci+bkg_master_background.fits" + output = "jw01309022001_04102_00001_nrs1_mbsub.fits" rtdata.output = output # Get the truth file From 73c8ea0243bceb2883d2a9927fe1262ccace43d1 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 22 Oct 2024 18:39:47 -0400 Subject: [PATCH 14/24] Update IFU data for master background test --- jwst/regtest/test_nirspec_masterbackground.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/jwst/regtest/test_nirspec_masterbackground.py b/jwst/regtest/test_nirspec_masterbackground.py index 157f53f833..f55d6ae783 100644 --- a/jwst/regtest/test_nirspec_masterbackground.py +++ b/jwst/regtest/test_nirspec_masterbackground.py @@ -151,16 +151,16 @@ def test_nirspec_fs_mbkg_user(rtdata, fitsdiff_default_kwargs): def test_nirspec_ifu_mbkg_user(rtdata, fitsdiff_default_kwargs): """Test NIRSpec IFU data with a user-supplied background file.""" # Get user-supplied background - user_background = "prism_bkg_x1d.fits" + user_background = "jw01252-o002_t005_nirspec_prism-clear_x1d.fits" rtdata.get_data(f"nirspec/ifu/{user_background}") # Get input data - rtdata.get_data("nirspec/ifu/prism_sci_bkg_cal.fits") + rtdata.get_data("nirspec/ifu/jw01252001001_03101_00001_nrs1_cal.fits") MasterBackgroundStep.call(rtdata.input, user_background=user_background, - save_results=True, suffix='master_background') + save_results=True, suffix='mbsub') - output = "prism_sci_bkg_master_background.fits" + output = "jw01252001001_03101_00001_nrs1_mbsub.fits" rtdata.output = output # Get the truth file @@ -173,18 +173,18 @@ def test_nirspec_ifu_mbkg_user(rtdata, fitsdiff_default_kwargs): @pytest.mark.parametrize( 'output_file', - ['ifu_prism_source_on_NRS1_master_background.fits', - 'ifu_prism_source_off_NRS1_o001_masterbg1d.fits', - 'ifu_prism_source_on_NRS1_o001_masterbg2d.fits'], + ['jw01252001001_03101_00001_nrs1_mbsub.fits', + 'jw01252-o002_t005_nirspec_prism-clear_o001_masterbg1d.fits', + 'jw01252001001_03101_00001_nrs1_o001_masterbg2d.fits'], ids=["on-source", "off-source", "on-source2d"] ) def test_nirspec_ifu_mbkg_nod(rtdata, fitsdiff_default_kwargs, output_file): """Test NIRSpec IFU prism nodded data.""" # Get input data - rtdata.get_asn("nirspec/ifu/nirspec_spec3_asn.json") + rtdata.get_asn("nirspec/ifu/jw01252-o001_spec3_00003_asn_with_bg.json") MasterBackgroundStep.call(rtdata.input, save_background=True, save_results=True, - suffix='master_background') + suffix='mbsub') rtdata.output = output_file From c508146a274ed6e4414b9f96b69e8c06f32ee6eb Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 4 Nov 2024 11:14:29 -0800 Subject: [PATCH 15/24] AL-837: Compute resample output WCS from known s_region instead of recalculating footprints (#8893) --- changes/8893.resample.rst | 1 + jwst/assign_mtwcs/moving_target_wcs.py | 4 +++ .../tests/test_outlier_detection.py | 2 ++ jwst/resample/resample_utils.py | 29 +++++++++++-------- 4 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 changes/8893.resample.rst diff --git a/changes/8893.resample.rst b/changes/8893.resample.rst new file mode 100644 index 0000000000..9e48ecea89 --- /dev/null +++ b/changes/8893.resample.rst @@ -0,0 +1 @@ +Use s_region list to calculate output footprint instead of re-computing via WCS transforms diff --git a/jwst/assign_mtwcs/moving_target_wcs.py b/jwst/assign_mtwcs/moving_target_wcs.py index be4a5a1b8b..0e3eb679ab 100644 --- a/jwst/assign_mtwcs/moving_target_wcs.py +++ b/jwst/assign_mtwcs/moving_target_wcs.py @@ -18,6 +18,8 @@ from jwst.datamodels import ModelLibrary from jwst.stpipe.utilities import record_step_status +from jwst.assign_wcs.util import update_s_region_imaging +from jwst.lib.exposure_types import IMAGING_TYPES log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -69,6 +71,8 @@ def assign_moving_target_wcs(input_models): model.meta.wcsinfo.mt_ra, model.meta.wcsinfo.mt_dec) del model.meta.wcs model.meta.wcs = new_wcs + if model.meta.exposure.type.lower() in IMAGING_TYPES: + update_s_region_imaging(model) record_step_status(model, "assign_mtwcs", True) input_models.shelve(model, i, modify=True) diff --git a/jwst/outlier_detection/tests/test_outlier_detection.py b/jwst/outlier_detection/tests/test_outlier_detection.py index b6dfe1c3eb..7d9849fc46 100644 --- a/jwst/outlier_detection/tests/test_outlier_detection.py +++ b/jwst/outlier_detection/tests/test_outlier_detection.py @@ -6,6 +6,7 @@ from gwcs.wcs import WCS from stdatamodels.jwst import datamodels +from stcal.alignment.util import compute_s_region_imaging from jwst.datamodels import ModelContainer, ModelLibrary from jwst.assign_wcs import AssignWcsStep @@ -151,6 +152,7 @@ def we_many_sci( # Replace the FITS-type WCS with an Identity WCS sci1.meta.wcs = create_fitswcs(sci1) + sci1.meta.wcsinfo.s_region = compute_s_region_imaging(sci1.meta.wcs, shape=shape, center=False) rng = np.random.default_rng(720) sci1.data = rng.normal(loc=background, size=shape, scale=sigma) sci1.err = np.zeros(shape) + sigma diff --git a/jwst/resample/resample_utils.py b/jwst/resample/resample_utils.py index c527b450f1..4d126ed28c 100644 --- a/jwst/resample/resample_utils.py +++ b/jwst/resample/resample_utils.py @@ -28,9 +28,15 @@ def make_output_wcs(input_models, ref_wcs=None, Parameters ---------- input_models : `~jwst.datamodel.ModelLibrary` - Each datamodel must have a ~gwcs.WCS object. + The datamodels to combine into a single output WCS. Each datamodel must + have a ``meta.wcs.s_region`` attribute. - pscale_ratio : float, optional + ref_wcs : gwcs.WCS, None, optional + Custom WCS to use as the output WCS. If not provided, + the reference WCS will be taken as the WCS of the first input model, with + its bounding box adjusted to encompass all input frames. + + pscale_ratio : float, None, optional Ratio of input to output pixel scale. Ignored when ``pscale`` is provided. @@ -67,27 +73,26 @@ def make_output_wcs(input_models, ref_wcs=None, WCS object, with defined domain, covering entire set of input frames """ if ref_wcs is None: + sregion_list = [] with input_models: - wcslist = [] for i, model in enumerate(input_models): - w = model.meta.wcs - if w.bounding_box is None: - w.bounding_box = wcs_bbox_from_shape(model.data.shape) - wcslist.append(w) + sregion_list.append(model.meta.wcsinfo.s_region) if i == 0: example_model = model + ref_wcs = example_model.meta.wcs + ref_wcsinfo = example_model.meta.wcsinfo.instance input_models.shelve(model) - naxes = wcslist[0].output_frame.naxes + naxes = ref_wcs.output_frame.naxes if naxes != 2: msg = ("Output WCS needs 2 spatial axes " f"but the supplied WCS has {naxes} axes.") raise RuntimeError(msg) - output_wcs = util.wcs_from_footprints( - wcslist, - ref_wcs=wcslist[0], - ref_wcsinfo=example_model.meta.wcsinfo.instance, + output_wcs = util.wcs_from_sregions( + sregion_list, + ref_wcs, + ref_wcsinfo, pscale_ratio=pscale_ratio, pscale=pscale, rotation=rotation, From 61032a14c59cf2f762bcb0035dcb1b52b9e605b4 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 29 Oct 2024 15:24:05 -0400 Subject: [PATCH 16/24] Update test data for MIRI MRS TSO --- jwst/regtest/test_miri_mrs_tso.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwst/regtest/test_miri_mrs_tso.py b/jwst/regtest/test_miri_mrs_tso.py index 434a3cbaf9..e602c64bb4 100644 --- a/jwst/regtest/test_miri_mrs_tso.py +++ b/jwst/regtest/test_miri_mrs_tso.py @@ -15,7 +15,7 @@ def run_spec2(rtdata_module): rtdata = rtdata_module # Setup the inputs - file_name = 'jw80600018001_02101_00003_mirifushort_rateints.fits' + file_name = 'jw01556001001_04102_00001-seg001_mirifushort_rateints.fits' rtdata.get_data(INPUT_PATH + '/' + file_name) # Run the pipeline @@ -37,7 +37,7 @@ def run_spec2(rtdata_module): def test_spec2(rtdata_module, run_spec2, fitsdiff_default_kwargs, suffix): """Test ensuring the calwebb_tso-spec2 is operating appropriately for MIRI MRS TSO data""" rtdata = rtdata_module - output = f"jw80600018001_02101_00003_mirifushort_{suffix}.fits" + output = f"jw01556001001_04102_00001-seg001_mirifushort_{suffix}.fits" rtdata.output = output rtdata.get_truth(f"{TRUTH_PATH}/{output}") From ea46c6023615fc420d923d05655baa73f8dede60 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 29 Oct 2024 17:41:37 -0400 Subject: [PATCH 17/24] Update MIRI MRS dedicated background data --- jwst/regtest/test_miri_mrs_dedicated_mbkg.py | 34 ++++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/jwst/regtest/test_miri_mrs_dedicated_mbkg.py b/jwst/regtest/test_miri_mrs_dedicated_mbkg.py index 93e5bcdba3..70edbee567 100644 --- a/jwst/regtest/test_miri_mrs_dedicated_mbkg.py +++ b/jwst/regtest/test_miri_mrs_dedicated_mbkg.py @@ -10,27 +10,41 @@ def run_pipeline(rtdata_module): rtdata = rtdata_module - rtdata.get_asn("miri/mrs/miri_mrs_mbkg_0304_spec3_asn.json") + rtdata.get_asn("miri/mrs/jw01031-c1004_20241028t205539_spec3_subset_asn.json") - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background') + MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub', + save_background=True) return rtdata @pytest.mark.bigdata -@pytest.mark.parametrize("exp_seq", ["exp1", "exp2", "exp3"]) -def test_miri_mrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs, - exp_seq): +@pytest.mark.parametrize("exposure", [1, 2, 3, 4]) +@pytest.mark.parametrize("suffix", ["mbsub", "c1004_masterbg2d"]) +def test_miri_mrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs, exposure, suffix): """Run a test for MIRI MRS data with dedicated background exposures.""" + rtdata = run_pipeline + output_file = f"jw01031001001_02101_0000{exposure}_mirifulong_{suffix}.fits" + rtdata.output = output_file + + # Get the truth file + rtdata.get_truth(os.path.join("truth/test_miri_mrs_dedicated_mbkg", output_file)) + + # Compare the results + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() + +@pytest.mark.bigdata +def test_miri_mrs_dedicated_masterbg1d(run_pipeline, fitsdiff_default_kwargs): rtdata = run_pipeline - rtdata.output = "miri_mrs_seq1_long_34_" + exp_seq + \ - "_master_background.fits" + + # Check 1D masterbg output product: created with root name from nod 1 + output_file = "jw01031006001_02101_00001_mirifulong_c1004_masterbg1d.fits" + rtdata.output = output_file # Get the truth file - rtdata.get_truth(os.path.join( - "truth/test_miri_mrs_dedicated_mbkg", - "miri_mrs_seq1_long_34_" + exp_seq + "_master_background.fits")) + rtdata.get_truth(os.path.join("truth/test_miri_mrs_dedicated_mbkg", output_file)) # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) From 3cb6359e0df4975cf9a33b0273a1df7a2e065325 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 5 Nov 2024 09:36:51 -0500 Subject: [PATCH 18/24] Remove unneeded test for MIRI MRS nodded background --- jwst/regtest/test_miri_mrs_nod_masterbg.py | 36 ---------------------- 1 file changed, 36 deletions(-) delete mode 100644 jwst/regtest/test_miri_mrs_nod_masterbg.py diff --git a/jwst/regtest/test_miri_mrs_nod_masterbg.py b/jwst/regtest/test_miri_mrs_nod_masterbg.py deleted file mode 100644 index d12a3bae20..0000000000 --- a/jwst/regtest/test_miri_mrs_nod_masterbg.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import pytest -from astropy.io.fits.diff import FITSDiff - -from jwst.master_background import MasterBackgroundStep - - -@pytest.fixture(scope="module") -def run_pipeline(rtdata_module): - - rtdata = rtdata_module - - rtdata.get_asn("miri/mrs/miri_mrs_mbkg_nodded_spec3_asn.json") - - MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background') - - return rtdata - - -@pytest.mark.bigdata -@pytest.mark.parametrize("nod_seq", ["seq1", "seq2"]) -def test_miri_mrs_nod_masterbg(run_pipeline, fitsdiff_default_kwargs, nod_seq): - """Run a test for MIRI MRS data with nodded background exposures.""" - - rtdata = run_pipeline - rtdata.output = "miri_mrs_nod_" + nod_seq + \ - "_short12_exp1_master_background.fits" - - # Get the truth file - rtdata.get_truth(os.path.join( - "truth/test_miri_mrs_nod_masterbg", - "miri_mrs_nod_" + nod_seq + "_short12_exp1_master_background.fits")) - - # Compare the results - diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) - assert diff.identical, diff.report() From 6fd952c7983dfac63d46a8b9b163dd58c2342fdd Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 6 Nov 2024 17:14:51 -0500 Subject: [PATCH 19/24] replace pipeline Step.__call__ uses with run --- jwst/pipeline/calwebb_ami3.py | 6 +- jwst/pipeline/calwebb_coron3.py | 12 +-- jwst/pipeline/calwebb_dark.py | 34 ++++---- jwst/pipeline/calwebb_detector1.py | 58 +++++++------- jwst/pipeline/calwebb_guider.py | 6 +- jwst/pipeline/calwebb_image2.py | 10 +-- jwst/pipeline/calwebb_image3.py | 16 ++-- jwst/pipeline/calwebb_spec2.py | 122 ++++++++++++++--------------- jwst/pipeline/calwebb_spec3.py | 28 +++---- jwst/pipeline/calwebb_tso3.py | 12 +-- 10 files changed, 152 insertions(+), 152 deletions(-) diff --git a/jwst/pipeline/calwebb_ami3.py b/jwst/pipeline/calwebb_ami3.py index ee58f4ac79..cbc6cd2caf 100644 --- a/jwst/pipeline/calwebb_ami3.py +++ b/jwst/pipeline/calwebb_ami3.py @@ -74,7 +74,7 @@ def process(self, input): # Do the LG analysis for this image log.debug('Do LG processing for member %s', input_file) - result1, result2, result3 = self.ami_analyze(input_file) + result1, result2, result3 = self.ami_analyze.run(input_file) # Save the averaged LG analysis results to a file result1.meta.asn.pool_name = asn['asn_pool'] @@ -90,7 +90,7 @@ def process(self, input): # Do the LG analysis for this image log.debug('Do LG processing for member %s', input_file) - result1, result2, result3 = self.ami_analyze(input_file) + result1, result2, result3 = self.ami_analyze.run(input_file) # Save the LG analysis results to a file result1.meta.asn.pool_name = asn['asn_pool'] @@ -104,7 +104,7 @@ def process(self, input): # assuming one ref star exposure per targ exposure if (len(psf_files) > 0) & (len(targ_files) > 0): for (targ, psf) in zip(targ_lg,psf_lg): - result = self.ami_normalize(targ, psf) + result = self.ami_normalize.run(targ, psf) # Save the result result.meta.asn.pool_name = asn['asn_pool'] result.meta.asn.table_name = op.basename(asn.filename) diff --git a/jwst/pipeline/calwebb_coron3.py b/jwst/pipeline/calwebb_coron3.py index 26310a5e7a..54b2b093ec 100644 --- a/jwst/pipeline/calwebb_coron3.py +++ b/jwst/pipeline/calwebb_coron3.py @@ -151,7 +151,7 @@ def process(self, user_input): # Perform outlier detection on the PSFs. if not skip_outlier_detection: for model in psf_models: - self.outlier_detection(model) + self.outlier_detection.run(model) # step may have been skipped for this model; # turn back on for next model self.outlier_detection.skip = False @@ -159,7 +159,7 @@ def process(self, user_input): self.log.info('Outlier detection skipped for PSF\'s') # Stack all the PSF images into a single CubeModel - psf_stack = self.stack_refs(psf_models) + psf_stack = self.stack_refs.run(psf_models) psf_models.close() # Save the resulting PSF stack @@ -175,13 +175,13 @@ def process(self, user_input): # Remove outliers from the target if not skip_outlier_detection: - target = self.outlier_detection(target) + target = self.outlier_detection.run(target) # step may have been skipped for this model; # turn back on for next model self.outlier_detection.skip = False # Call align_refs - psf_aligned = self.align_refs(target, psf_stack) + psf_aligned = self.align_refs.run(target, psf_stack) # Save the alignment results self.save_model( @@ -190,7 +190,7 @@ def process(self, user_input): ) # Call KLIP - psf_sub = self.klip(target, psf_aligned) + psf_sub = self.klip.run(target, psf_aligned) psf_aligned.close() # Save the psf subtraction results @@ -210,7 +210,7 @@ def process(self, user_input): resample_library = ModelLibrary(resample_input, on_disk=False) # Output is a single datamodel - result = self.resample(resample_library) + result = self.resample.run(resample_library) # Blend the science headers try: diff --git a/jwst/pipeline/calwebb_dark.py b/jwst/pipeline/calwebb_dark.py index 9673ffd882..68bc7d0c90 100644 --- a/jwst/pipeline/calwebb_dark.py +++ b/jwst/pipeline/calwebb_dark.py @@ -67,29 +67,29 @@ def process(self, input): # the steps are in a different order than NIR log.debug('Processing a MIRI exposure') - input = self.group_scale(input) - input = self.dq_init(input) - input = self.emicorr(input) - input = self.saturation(input) - input = self.ipc(input) - input = self.firstframe(input) - input = self.lastframe(input) - input = self.reset(input) - input = self.linearity(input) - input = self.rscd(input) + input = self.group_scale.run(input) + input = self.dq_init.run(input) + input = self.emicorr.run(input) + input = self.saturation.run(input) + input = self.ipc.run(input) + input = self.firstframe.run(input) + input = self.lastframe.run(input) + input = self.reset.run(input) + input = self.linearity.run(input) + input = self.rscd.run(input) else: # process Near-IR exposures log.debug('Processing a Near-IR exposure') - input = self.group_scale(input) - input = self.dq_init(input) - input = self.saturation(input) - input = self.ipc(input) - input = self.superbias(input) - input = self.refpix(input) - input = self.linearity(input) + input = self.group_scale.run(input) + input = self.dq_init.run(input) + input = self.saturation.run(input) + input = self.ipc.run(input) + input = self.superbias.run(input) + input = self.refpix.run(input) + input = self.linearity.run(input) log.info('... ending calwebb_dark') diff --git a/jwst/pipeline/calwebb_detector1.py b/jwst/pipeline/calwebb_detector1.py index afc3bc5ab3..44d80450d5 100644 --- a/jwst/pipeline/calwebb_detector1.py +++ b/jwst/pipeline/calwebb_detector1.py @@ -89,18 +89,18 @@ def process(self, input): # the steps are in a different order than NIR log.debug('Processing a MIRI exposure') - input = self.group_scale(input) - input = self.dq_init(input) - input = self.emicorr(input) - input = self.saturation(input) - input = self.ipc(input) - input = self.firstframe(input) - input = self.lastframe(input) - input = self.reset(input) - input = self.linearity(input) - input = self.rscd(input) - input = self.dark_current(input) - input = self.refpix(input) + input = self.group_scale.run(input) + input = self.dq_init.run(input) + input = self.emicorr.run(input) + input = self.saturation.run(input) + input = self.ipc.run(input) + input = self.firstframe.run(input) + input = self.lastframe.run(input) + input = self.reset.run(input) + input = self.linearity.run(input) + input = self.rscd.run(input) + input = self.dark_current.run(input) + input = self.refpix.run(input) # skip until MIRI team has figured out an algorithm # input = self.persistence(input) @@ -110,28 +110,28 @@ def process(self, input): # process Near-IR exposures log.debug('Processing a Near-IR exposure') - input = self.group_scale(input) - input = self.dq_init(input) - input = self.saturation(input) - input = self.ipc(input) - input = self.superbias(input) - input = self.refpix(input) - input = self.linearity(input) + input = self.group_scale.run(input) + input = self.dq_init.run(input) + input = self.saturation.run(input) + input = self.ipc.run(input) + input = self.superbias.run(input) + input = self.refpix.run(input) + input = self.linearity.run(input) # skip persistence for NIRSpec if instrument != 'NIRSPEC': - input = self.persistence(input) + input = self.persistence.run(input) - input = self.dark_current(input) + input = self.dark_current.run(input) # apply the charge_migration step - input = self.charge_migration(input) + input = self.charge_migration.run(input) # apply the jump step - input = self.jump(input) + input = self.jump.run(input) # apply the clean_flicker_noise step - input = self.clean_flicker_noise(input) + input = self.clean_flicker_noise.run(input) # save the corrected ramp data, if requested if self.save_calibrated_ramp: @@ -143,15 +143,15 @@ def process(self, input): # objects, but when the step is skipped due to `skip = True`, # only the input is returned when the step is invoked. if self.ramp_fit.skip: - input = self.ramp_fit(input) + input = self.ramp_fit.run(input) ints_model = None else: - input, ints_model = self.ramp_fit(input) + input, ints_model = self.ramp_fit.run(input) # apply the gain_scale step to the exposure-level product if input is not None: self.gain_scale.suffix = 'gain_scale' - input = self.gain_scale(input) + input = self.gain_scale.run(input) else: log.info("NoneType returned from ramp_fit. Gain Scale step skipped.") @@ -159,7 +159,7 @@ def process(self, input): # if it exists, and then save it if ints_model is not None: self.gain_scale.suffix = 'gain_scaleints' - ints_model = self.gain_scale(ints_model) + ints_model = self.gain_scale.run(ints_model) self.save_model(ints_model, 'rateints') # setup output_file for saving @@ -176,4 +176,4 @@ def setup_output(self, input): if input.meta.cal_step.ramp_fit == 'COMPLETE': self.suffix = 'rate' else: - self.suffix = 'ramp' \ No newline at end of file + self.suffix = 'ramp' diff --git a/jwst/pipeline/calwebb_guider.py b/jwst/pipeline/calwebb_guider.py index 87bfc864b0..0f519b5954 100644 --- a/jwst/pipeline/calwebb_guider.py +++ b/jwst/pipeline/calwebb_guider.py @@ -52,9 +52,9 @@ def process(self, input): input = datamodels.GuiderRawModel(input) # Apply the steps - input = self.dq_init(input) - input = self.guider_cds(input) - input = self.flat_field(input) + input = self.dq_init.run(input) + input = self.guider_cds.run(input) + input = self.flat_field.run(input) log.info('... ending calwebb_guider') diff --git a/jwst/pipeline/calwebb_image2.py b/jwst/pipeline/calwebb_image2.py index 64aab36d80..82c18ea149 100644 --- a/jwst/pipeline/calwebb_image2.py +++ b/jwst/pipeline/calwebb_image2.py @@ -150,12 +150,12 @@ def process_exposure_product( self.bkg_subtract.save_results = True # Call the background subtraction step - input = self.bkg_subtract(input, members_by_type['background']) + input = self.bkg_subtract.run(input, members_by_type['background']) # work on slope images - input = self.assign_wcs(input) - input = self.flat_field(input) - input = self.photom(input) + input = self.assign_wcs.run(input) + input = self.flat_field.run(input) + input = self.photom.run(input) # Resample individual exposures, but only if it's one of the # regular 2D science image types @@ -163,7 +163,7 @@ def process_exposure_product( len(input.data.shape) == 2: self.resample.save_results = self.save_results self.resample.suffix = 'i2d' - self.resample(input) + self.resample.run(input) # That's all folks self.log.info( diff --git a/jwst/pipeline/calwebb_image3.py b/jwst/pipeline/calwebb_image3.py index 757622a627..36d1a85680 100644 --- a/jwst/pipeline/calwebb_image3.py +++ b/jwst/pipeline/calwebb_image3.py @@ -84,24 +84,24 @@ def process(self, input_data): is_moving = is_moving_target(model) input_models.shelve(model, 0, modify=False) if is_moving: - input_models = self.assign_mtwcs(input_models) + input_models = self.assign_mtwcs.run(input_models) else: - input_models = self.tweakreg(input_models) + input_models = self.tweakreg.run(input_models) - input_models = self.skymatch(input_models) - input_models = self.outlier_detection(input_models) + input_models = self.skymatch.run(input_models) + input_models = self.outlier_detection.run(input_models) elif self.skymatch.skymethod == 'match': self.log.warning("Turning 'skymatch' step off for a single " "input image when 'skymethod' is 'match'") else: - input_models = self.skymatch(input_models) + input_models = self.skymatch.run(input_models) - result = self.resample(input_models) + result = self.resample.run(input_models) del input_models if isinstance(result, datamodels.ImageModel) and result.meta.cal_step.resample == 'COMPLETE': - self.source_catalog(result) + self.source_catalog.run(result) def _load_input_as_library(self, input): @@ -128,4 +128,4 @@ def _load_input_as_library(self, input): elif isinstance(input, datamodels.JwstDataModel): return ModelLibrary([input], asn_exptypes=['science'], on_disk=not self.in_memory) else: - raise TypeError(f"Input type {type(input)} not supported.") \ No newline at end of file + raise TypeError(f"Input type {type(input)} not supported.") diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 36e2993113..1502338cbd 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -225,7 +225,7 @@ def process_exposure_product( # `assign_wcs` is the critical step. Without it, processing cannot proceed. assign_wcs_exception = None try: - calibrated = self.assign_wcs(science) + calibrated = self.assign_wcs.run(science) except Exception as exception: assign_wcs_exception = exception if assign_wcs_exception is not None or \ @@ -252,7 +252,7 @@ def process_exposure_product( # Self-calibrate to flag bad/warm pixels, and apply flags # to both background and science exposures. # skipped by default for all modes - result = self.badpix_selfcal( + result = self.badpix_selfcal.run( calibrated, members_by_type['selfcal'], members_by_type['background'], ) if isinstance(result, datamodels.JwstDataModel): @@ -264,10 +264,10 @@ def process_exposure_product( members_by_type['background'] = bkg_outlier_flagged # apply msa_flagging (flag stuck open shutters for NIRSpec IFU and MOS) - calibrated = self.msa_flagging(calibrated) + calibrated = self.msa_flagging.run(calibrated) # apply the "nsclean" 1/f correction to NIRSpec images - calibrated = self.nsclean(calibrated) + calibrated = self.nsclean.run(calibrated) # Apply nsclean to NIRSpec imprint and background members if not self.nsclean.skip: @@ -279,7 +279,7 @@ def process_exposure_product( self.nsclean.output_file = imprint_file.meta.filename else: self.nsclean.output_file = os.path.basename(imprint_file) - imprint_nsclean = self.nsclean(imprint_file) + imprint_nsclean = self.nsclean.run(imprint_file) members_by_type['imprint'][i] = imprint_nsclean for i, bkg_file in enumerate(members_by_type['background']): @@ -288,21 +288,21 @@ def process_exposure_product( self.nsclean.output_file = bkg_file.meta.filename else: self.nsclean.output_file = os.path.basename(bkg_file) - bkg_nsclean = self.nsclean(bkg_file) + bkg_nsclean = self.nsclean.run(bkg_file) members_by_type['background'][i] = bkg_nsclean # Leakcal subtraction (imprint) occurs before background subtraction on a per-exposure basis. # If there is only one `imprint` member, this imprint exposure is subtracted from all the # science and background exposures. Otherwise, there will be as many `imprint` members as # there are science plus background members. - calibrated = self.imprint_subtract(calibrated, members_by_type['imprint']) + calibrated = self.imprint_subtract.run(calibrated, members_by_type['imprint']) # for each background image subtract an associated leak cal for i, bkg_file in enumerate(members_by_type['background']): - bkg_imprint_sub = self.imprint_subtract(bkg_file, members_by_type['imprint']) + bkg_imprint_sub = self.imprint_subtract.run(bkg_file, members_by_type['imprint']) members_by_type['background'][i] = bkg_imprint_sub - calibrated = self.bkg_subtract(calibrated, members_by_type['background']) + calibrated = self.bkg_subtract.run(calibrated, members_by_type['background']) # The order of the next few steps is tricky, depending on mode: # WFSS/Grism data need flat_field before extract_2d, but other modes @@ -334,8 +334,8 @@ def process_exposure_product( resampled = calibrated.copy() # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. - resampled = self.pixel_replace(resampled) - resampled = self.resample_spec(resampled) + resampled = self.pixel_replace.run(resampled) + resampled = self.resample_spec.run(resampled) elif is_nrs_slit_linelamp(calibrated): @@ -343,8 +343,8 @@ def process_exposure_product( resampled = calibrated.copy() # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. - resampled = self.pixel_replace(resampled) - resampled = self.resample_spec(resampled) + resampled = self.pixel_replace.run(resampled) + resampled = self.resample_spec.run(resampled) elif (exp_type in ['MIR_MRS', 'NRS_IFU']) or is_nrs_ifu_linelamp(calibrated): @@ -355,21 +355,21 @@ def process_exposure_product( resampled = calibrated.copy() # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. - resampled = self.pixel_replace(resampled) - resampled = self.cube_build(resampled) + resampled = self.pixel_replace.run(resampled) + resampled = self.cube_build.run(resampled) if query_step_status(resampled, "cube_build") == 'COMPLETE': self.save_model(resampled[0], suffix='s3d') elif exp_type in ['MIR_LRS-SLITLESS']: resampled = calibrated.copy() # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. - resampled = self.pixel_replace(resampled) + resampled = self.pixel_replace.run(resampled) else: # will be run if set in parameter ref file or by user resampled = calibrated.copy() # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. - resampled = self.pixel_replace(resampled) + resampled = self.pixel_replace.run(resampled) # Extract a 1D spectrum from the 2D/3D data if exp_type in ['MIR_MRS', 'NRS_IFU'] and query_step_status(resampled, "cube_build") == 'SKIPPED': # Skip extract_1d for IFU modes where no cube was built @@ -384,7 +384,7 @@ def process_exposure_product( self.photom.suffix = 'x1d' self.extract_1d.save_results = False x1d = resampled.copy() - x1d = self.extract_1d(x1d) + x1d = self.extract_1d.run(x1d) # Possible that no fit was possible - if so, skip photom if (x1d is None) or (x1d.meta.cal_step.extract_1d == "SKIPPED"): @@ -401,7 +401,7 @@ def process_exposure_product( x1d = resampled.copy() else: x1d = resampled.copy() - x1d = self.extract_1d(x1d) + x1d = self.extract_1d.run(x1d) resampled.close() if x1d is not None: @@ -510,7 +510,7 @@ def _process_grism(self, data): """ # Apply flat-field correction - calibrated = self.flat_field(data) + calibrated = self.flat_field.run(data) # Create and save a WFSS e-/sec image, if requested if self.save_wfss_esec: @@ -547,14 +547,14 @@ def _process_grism(self, data): # Continue with remaining calibration steps, using the original # DN/sec image - calibrated = self.extract_2d(calibrated) - calibrated = self.srctype(calibrated) - calibrated = self.straylight(calibrated) - calibrated = self.fringe(calibrated) - calibrated = self.pathloss(calibrated) - calibrated = self.barshadow(calibrated) - calibrated = self.wfss_contam(calibrated) - calibrated = self.photom(calibrated) + calibrated = self.extract_2d.run(calibrated) + calibrated = self.srctype.run(calibrated) + calibrated = self.straylight.run(calibrated) + calibrated = self.fringe.run(calibrated) + calibrated = self.pathloss.run(calibrated) + calibrated = self.barshadow.run(calibrated) + calibrated = self.wfss_contam.run(calibrated) + calibrated = self.photom.run(calibrated) return calibrated def _process_nirspec_slits(self, data): @@ -568,14 +568,14 @@ def _process_nirspec_slits(self, data): Note that NIRSpec MOS and FS need srctype and wavecorr before flat_field. """ - calibrated = self.extract_2d(data) - calibrated = self.srctype(calibrated) - calibrated = self.master_background_mos(calibrated) - calibrated = self.wavecorr(calibrated) - calibrated = self.flat_field(calibrated) - calibrated = self.pathloss(calibrated) - calibrated = self.barshadow(calibrated) - calibrated = self.photom(calibrated) + calibrated = self.extract_2d.run(data) + calibrated = self.srctype.run(calibrated) + calibrated = self.master_background_mos.run(calibrated) + calibrated = self.wavecorr.run(calibrated) + calibrated = self.flat_field.run(calibrated) + calibrated = self.pathloss.run(calibrated) + calibrated = self.barshadow.run(calibrated) + calibrated = self.photom.run(calibrated) return calibrated @@ -592,8 +592,8 @@ def _process_nirspec_msa_slits(self, data): Note that NIRSpec MOS and FS need srctype and wavecorr before flat_field. Also have to deal with master background operations. """ - calibrated = self.extract_2d(data) - calibrated = self.srctype(calibrated) + calibrated = self.extract_2d.run(data) + calibrated = self.srctype.run(calibrated) # Split the datamodel into 2 pieces: one with MOS slits and # the other with FS slits @@ -609,12 +609,12 @@ def _process_nirspec_msa_slits(self, data): # First process MOS slits through all remaining steps calib_mos.update(calibrated) if len(calib_mos.slits) > 0: - calib_mos = self.master_background_mos(calib_mos) - calib_mos = self.wavecorr(calib_mos) - calib_mos = self.flat_field(calib_mos) - calib_mos = self.pathloss(calib_mos) - calib_mos = self.barshadow(calib_mos) - calib_mos = self.photom(calib_mos) + calib_mos = self.master_background_mos.run(calib_mos) + calib_mos = self.wavecorr.run(calib_mos) + calib_mos = self.flat_field.run(calib_mos) + calib_mos = self.pathloss.run(calib_mos) + calib_mos = self.barshadow.run(calib_mos) + calib_mos = self.photom.run(calib_mos) # Now repeat for FS slits if len(calib_fss.slits) > 0: @@ -631,7 +631,7 @@ def _process_nirspec_msa_slits(self, data): step.suffix = f'{current_suffix}_fs' # Run step - calib_fss = step(calib_fss) + calib_fss = step.run(calib_fss) # Reset suffix step.suffix = current_suffix @@ -655,25 +655,25 @@ def _process_niriss_soss(self, data): New SOSS extraction requires input to extract_1d step in units of DN/s, with photom step to be run afterwards. """ - calibrated = self.srctype(data) - calibrated = self.flat_field(calibrated) - calibrated = self.straylight(calibrated) - calibrated = self.fringe(calibrated) - calibrated = self.pathloss(calibrated) - calibrated = self.barshadow(calibrated) + calibrated = self.srctype.run(data) + calibrated = self.flat_field.run(calibrated) + calibrated = self.straylight.run(calibrated) + calibrated = self.fringe.run(calibrated) + calibrated = self.pathloss.run(calibrated) + calibrated = self.barshadow.run(calibrated) return calibrated def _process_common(self, data): """Common spectral processing""" - calibrated = self.srctype(data) - calibrated = self.straylight(calibrated) - calibrated = self.flat_field(calibrated) - calibrated = self.fringe(calibrated) - calibrated = self.pathloss(calibrated) - calibrated = self.barshadow(calibrated) - calibrated = self.photom(calibrated) - calibrated = self.residual_fringe(calibrated) # only run on MIRI_MRS data + calibrated = self.srctype.run(data) + calibrated = self.straylight.run(calibrated) + calibrated = self.flat_field.run(calibrated) + calibrated = self.fringe.run(calibrated) + calibrated = self.pathloss.run(calibrated) + calibrated = self.barshadow.run(calibrated) + calibrated = self.photom.run(calibrated) + calibrated = self.residual_fringe.run(calibrated) # only run on MIRI_MRS data return calibrated @@ -707,13 +707,13 @@ def _extract_nirspec_msa_slits(self, resampled): self.extract_1d.save_results = False if len(resamp_mos.slits) > 0: self.log.info(f'Extracting {len(resamp_mos.slits)} MSA slitlets') - x1d = self.extract_1d(resamp_mos) + x1d = self.extract_1d.run(resamp_mos) # Extract the FS slits if len(resamp_fss.slits) > 0: self.log.info(f'Extracting {len(resamp_fss.slits)} fixed slits') resamp_fss.meta.exposure.type = "NRS_FIXEDSLIT" - x1d_fss = self.extract_1d(resamp_fss) + x1d_fss = self.extract_1d.run(resamp_fss) if x1d is None: x1d = x1d_fss x1d.meta.exposure.type = "NRS_MSASPEC" diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index d62bcc926f..707a67f5c0 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -140,11 +140,11 @@ def process(self, input): if is_moving_target(input_models[0]): self.log.info("Assigning WCS to a Moving Target exposure.") # assign_mtwcs modifies input_models in-place - self.assign_mtwcs(input_models) + self.assign_mtwcs.run(input_models) # If background data are present, call the master background step if members_by_type['background']: - source_models = self.master_background(input_models) + source_models = self.master_background.run(input_models) source_models.asn_table = input_models.asn_table # If the step is skipped, do the container splitting that @@ -218,7 +218,7 @@ def process(self, input): # Call the skymatch step for MIRI MRS data if exptype in ['MIR_MRS']: - result = self.mrs_imatch(result) + result = self.mrs_imatch.run(result) # Call outlier detection and pixel replacement if exptype not in SLITLESS_TYPES: @@ -231,21 +231,21 @@ def process(self, input): self.outlier_detection.mode = 'ifu' else: self.outlier_detection.mode = 'spec' - result = self.outlier_detection(result) + result = self.outlier_detection.run(result) # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. - result = self.pixel_replace(result) + result = self.pixel_replace.run(result) # Resample time. Dependent on whether the data is IFU or not. resample_complete = None if exptype in IFU_EXPTYPES: - result = self.cube_build(result) + result = self.cube_build.run(result) try: resample_complete = result[0].meta.cal_step.cube_build except AttributeError: pass else: - result = self.resample_spec(result) + result = self.resample_spec.run(result) try: resample_complete = result.meta.cal_step.resample except AttributeError: @@ -256,7 +256,7 @@ def process(self, input): # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE - result = self.pixel_replace(result) + result = self.pixel_replace.run(result) # For slitless data, extract 1D spectra and then combine them if exptype in ['NIS_SOSS']: @@ -265,17 +265,17 @@ def process(self, input): # those instead. self.extract_1d.save_results = False - result = self.extract_1d(result) + result = self.extract_1d.run(result) # SOSS F277W may return None - don't bother with that. if result is not None: self.photom.save_results = self.save_results self.photom.suffix = 'x1d' - result = self.photom(result) + result = self.photom.run(result) else: - result = self.extract_1d(result) + result = self.extract_1d.run(result) - result = self.combine_1d(result) + result = self.combine_1d.run(result) elif resample_complete is not None and resample_complete.upper() == 'COMPLETE': @@ -290,10 +290,10 @@ def process(self, input): self.spectral_leak.search_output_file = False self.spectral_leak.save_results = self.save_results - result = self.extract_1d(result) + result = self.extract_1d.run(result) if exptype in ['MIR_MRS']: - result = self.spectral_leak(result) + result = self.spectral_leak.run(result) else: self.log.warning( 'Resampling was not completed. Skipping extract_1d.' diff --git a/jwst/pipeline/calwebb_tso3.py b/jwst/pipeline/calwebb_tso3.py index f5d974f48b..58c87d753b 100644 --- a/jwst/pipeline/calwebb_tso3.py +++ b/jwst/pipeline/calwebb_tso3.py @@ -91,7 +91,7 @@ def process(self, input): break self.log.info("Performing outlier detection on input images ...") - cube = self.outlier_detection(cube) + cube = self.outlier_detection.run(cube) # Save crfints products if cube.meta.cal_step.outlier_detection == 'COMPLETE': @@ -124,7 +124,7 @@ def process(self, input): for cube in input_models: # Extract Photometry from imaging data - phot_result_list.append(self.tso_photometry(cube)) + phot_result_list.append(self.tso_photometry.run(cube)) # Spectroscopy else: @@ -147,12 +147,12 @@ def process(self, input): for cube in input_models: # interpolate pixels that have a NaN value or are flagged # as DO_NOT_USE or NON_SCIENCE. - cube = self.pixel_replace(cube) + cube = self.pixel_replace.run(cube) state = cube.meta.cal_step.pixel_replace # Process spectroscopic TSO data # extract 1D self.log.info("Extracting 1-D spectra ...") - result = self.extract_1d(cube) + result = self.extract_1d.run(cube) for row in cube.int_times: # Subtract one to assign 1-indexed int_nums to int_times array locations x1d_result.int_times[row[0] - 1] = row @@ -164,13 +164,13 @@ def process(self, input): if input_exptype == 'NIS_SOSS': # SOSS data have yet to be photometrically calibrated # Calibrate 1D spectra here. - result = self.photom(result) + result = self.photom.run(result) x1d_result.spec.extend(result.spec) # perform white-light photometry on 1d extracted data self.log.info("Performing white-light photometry ...") - phot_result_list.append(self.white_light(result)) + phot_result_list.append(self.white_light.run(result)) # Update some metadata from the association x1d_result.meta.asn.pool_name = input_models.asn_table["asn_pool"] From 4029bf57a89fe22192d29136942347341491c065 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 6 Nov 2024 17:42:01 -0500 Subject: [PATCH 20/24] remove Step.__call__ use in tests --- jwst/stpipe/tests/steps/__init__.py | 16 ++++++++-------- jwst/stpipe/tests/test_pipeline.py | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/jwst/stpipe/tests/steps/__init__.py b/jwst/stpipe/tests/steps/__init__.py index eab156a0e5..dd1f2f6295 100644 --- a/jwst/stpipe/tests/steps/__init__.py +++ b/jwst/stpipe/tests/steps/__init__.py @@ -23,7 +23,7 @@ class PipeWithReference(Pipeline): step_defs = {'step_with_reference': StepWithReference} def process(self, data): - result = self.step_with_reference(data) + result = self.step_with_reference.run(data) return result @@ -236,13 +236,13 @@ def process(self, *args): model = ImageModel(args[0]) self.stepwithmodel.suffix = 'swm' - r = self.stepwithmodel(model) + r = self.stepwithmodel.run(model) self.another_stepwithmodel.suffix = 'aswm' - r = self.another_stepwithmodel(r) + r = self.another_stepwithmodel.run(r) self.stepwithcontainer.suffix = 'swc' - r = self.stepwithcontainer(r) + r = self.stepwithcontainer.run(r) self.withdefaultsstep.suffix = 'wds' - r = self.withdefaultsstep(r) + r = self.withdefaultsstep.run(r) return r @@ -261,8 +261,8 @@ class SavePipeline(Pipeline): def process(self, *args): model = ImageModel(args[0]) - r = self.stepwithmodel(model) - r = self.savestep(r) + r = self.stepwithmodel.run(model) + r = self.savestep.run(r) return r @@ -280,4 +280,4 @@ class MakeListPipeline(Pipeline): def process(self, *args): - return self.make_list(*args) + return self.make_list.run(*args) diff --git a/jwst/stpipe/tests/test_pipeline.py b/jwst/stpipe/tests/test_pipeline.py index 96b17a20f7..3075d74429 100644 --- a/jwst/stpipe/tests/test_pipeline.py +++ b/jwst/stpipe/tests/test_pipeline.py @@ -94,9 +94,9 @@ def process(self, *args): self.flat_filename = join(dirname(__file__), "data", "flat.fits") flat = datamodels.open(self.flat_filename) calibrated = [] - calibrated.append(self.flat_field(science, flat)) - combined = self.combine(calibrated) - self.display(combined) + calibrated.append(self.flat_field.run(science, flat)) + combined = self.combine.run(calibrated) + self.display.run(combined) dm = datamodels.ImageModel(combined) self.save_model(dm) science.close() From 13dbcc313d989d4d2a13d0d3299e6897cdddc5cf Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 7 Nov 2024 08:29:45 -0500 Subject: [PATCH 21/24] remove Step.__call__ use in master background --- .../master_background_mos_step.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py index 45c641ff5c..9b8baa8791 100644 --- a/jwst/master_background/master_background_mos_step.py +++ b/jwst/master_background/master_background_mos_step.py @@ -235,10 +235,10 @@ def _calc_master_background(self, data, user_background=None): self.barshadow.source_type = 'EXTENDED' self.photom.source_type = 'EXTENDED' - pre_calibrated = self.flat_field(data) - pre_calibrated = self.pathloss(pre_calibrated) - pre_calibrated = self.barshadow(pre_calibrated) - pre_calibrated = self.photom(pre_calibrated) + pre_calibrated = self.flat_field.run(data) + pre_calibrated = self.pathloss.run(pre_calibrated) + pre_calibrated = self.barshadow.run(pre_calibrated) + pre_calibrated = self.photom.run(pre_calibrated) # Create the 1D, fully calibrated master background. if user_background: @@ -269,9 +269,9 @@ def _calc_master_background(self, data, user_background=None): self.flat_field.use_correction_pars = True self.flat_field.inverse = True - mb_multislit = self.photom(mb_multislit) - mb_multislit = self.barshadow(mb_multislit) - mb_multislit = self.pathloss(mb_multislit) - mb_multislit = self.flat_field(mb_multislit) + mb_multislit = self.photom.run(mb_multislit) + mb_multislit = self.barshadow.run(mb_multislit) + mb_multislit = self.pathloss.run(mb_multislit) + mb_multislit = self.flat_field.run(mb_multislit) return master_background, mb_multislit From 50df638e78cfb20f4bd794c91eef2c14b764d751 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 7 Nov 2024 08:30:40 -0500 Subject: [PATCH 22/24] remove Step.__call__ use in calwebb_spec2 --- jwst/pipeline/calwebb_spec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 1502338cbd..98b26c586b 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -391,7 +391,7 @@ def process_exposure_product( self.log.warning("Extract_1d did not return a DataModel - skipping photom.") else: self.photom.save_results = self.save_results - x1d = self.photom(x1d) + x1d = self.photom.run(x1d) elif exp_type == 'NRS_MSASPEC': # Special handling for MSA spectra, to handle mixed-in # fixed slits separately From ba195b9a8346fea576ee4b628ba0b783c2d353a1 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 7 Nov 2024 12:30:14 -0500 Subject: [PATCH 23/24] add changelog --- changes/8945.stpipe.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/8945.stpipe.rst diff --git a/changes/8945.stpipe.rst b/changes/8945.stpipe.rst new file mode 100644 index 0000000000..73937c9f0f --- /dev/null +++ b/changes/8945.stpipe.rst @@ -0,0 +1 @@ +Remove all uses of Step.__call__ to allow it's deprecation. From e9a482dd0d08d8a56de42ee40a2c26a71cf542cd Mon Sep 17 00:00:00 2001 From: Tyler Pauly Date: Thu, 14 Nov 2024 10:43:45 -0500 Subject: [PATCH 24/24] Update changelog and README to reflect patch release (#8960) --- CHANGES.rst | 9 +++++++++ README.md | 13 +++++++------ jwst/resample/resample_spec.py | 5 ++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7500ad652f..1041652782 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +1.16.1 (2024-10-30) +=================== + +resample_spec +------------- + +- Update NIRSpec spectral resampling to add a missing correction factor in resampled + WCS tangent plane transformation. [#8908] + 1.16.0 (2024-09-20) =================== diff --git a/README.md b/README.md index b1bd23db8b..84d50d76c8 100644 --- a/README.md +++ b/README.md @@ -216,12 +216,13 @@ the specified context and less than the context for the next release. | jwst tag | DMS build | SDP_VER | CRDS_CONTEXT | Released | Ops Install | Notes | |---------------------|-----------|----------|--------------|------------|-------------|-----------------------------------------------| -| 1.16.0 | B11.1rc1 | TBD | 1281 | 2024-09-20 | TBD | First release candidate for B11.1 | -| 1.15.1 | B11.0 | 2024.2.2 | 1242 | 2024-07-08 | 2024-09-12 | Final release candidate for B11.0 | -| 1.15.0 | B11.0rc1 | | 1241 | 2024-06-26 | | First release candidate for B11.0 | -| 1.14.1 | | | 1238 | 2024-06-27 | | PyPI-only release for external users | -| 1.14.0 | B10.2.1 | 2024.1.1 | 1238 | 2024-03-29 | 2024-06-12 | Final release candidate for B10.2.1 | -| 1.13.4 | | | 1185 | 2024-01-25 | | PyPI-only release for external users | +| 1.16.1 | B11.1.1 | 2024.3.1 | 1298 | 2024-11-13 | TBD | Final release candidate for B11.1 | +| 1.16.0 | B11.1 | 2024.3.0 | 1298 | 2024-09-20 | TBD | First release candidate for B11.1 | +| 1.15.1 | B11.0 | 2024.2.2 | 1293 | 2024-07-08 | 2024-09-12 | Final release candidate for B11.0 | +| 1.15.0 | B11.0rc1 | | 1274 | 2024-06-26 | | First release candidate for B11.0 | +| 1.14.1 | | | 1240 | 2024-06-27 | | PyPI-only release for external users | +| 1.14.0 | B10.2.1 | 2024.1.1 | 1240 | 2024-03-29 | 2024-06-12 | Final release candidate for B10.2.1 | +| 1.13.4 | | | 1210 | 2024-01-25 | | PyPI-only release for external users | | 1.13.3 | B10.1 | 2023.4.0 | 1181 | 2024-01-05 | | Final release candidate for B10.1 | | 1.13.2 | B10.1rc3 | 2023.4.0 | 1181 | 2023-12-21 | | Third release candidate for B10.1 | | 1.13.1 | B10.1rc2 | 2023.4.0 | 1181 | 2023-12-19 | | Second release candidate for B10.1 | diff --git a/jwst/resample/resample_spec.py b/jwst/resample/resample_spec.py index 5160034ddb..7872513c1c 100644 --- a/jwst/resample/resample_spec.py +++ b/jwst/resample/resample_spec.py @@ -362,8 +362,8 @@ def build_nirspec_output_wcs(self, input_models, refmodel=None): # Correct the intercept for the new minimum value. # Also account for integer pixel size to make sure the # data is centered in the array. - offset_y = ny/2 * pix_to_tan_slope_y - diff_y/2 - offset_x = ny/2 * pix_to_tan_slope_x - diff_x/2 + offset_y = ny / 2 * pix_to_tan_slope_y - diff_y / 2 + offset_x = ny / 2 * pix_to_tan_slope_x - diff_x / 2 if slope_sign_y > 0: zero_value_y = min_tan_y @@ -375,7 +375,6 @@ def build_nirspec_output_wcs(self, input_models, refmodel=None): else: zero_value_x = max_tan_x - pix_to_ytan.intercept = zero_value_y - slope_sign_y * offset_y pix_to_xtan.intercept = zero_value_x - slope_sign_x * offset_x