Merge branch 'main' into jp-2928-nirspec-masterbackground

jwst/regtest/test_miri_dark.py

@@ -7,7 +7,9 @@
 @pytest.mark.bigdata
 @pytest.mark.parametrize(
     'exposure',
-    ['jw00001001001_01101_00001_mirimage', 'jw02201001001_01101_00001_MIRIMAGE']
+    ['jw04482014001_02102_00001_mirifulong',
+     'jw04482014001_02102_00001_mirifushort',
+     'jw04482014001_02102_00001_mirimage']
 )
 def test_miri_dark_pipeline(exposure, rtdata, fitsdiff_default_kwargs):
     """Test the DarkPipeline on MIRI dark exposures"""

jwst/regtest/test_miri_mrs_dedicated_mbkg.py

@@ -10,27 +10,41 @@ def run_pipeline(rtdata_module):
 
     rtdata = rtdata_module
 
-    rtdata.get_asn("miri/mrs/miri_mrs_mbkg_0304_spec3_asn.json")
+    rtdata.get_asn("miri/mrs/jw01031-c1004_20241028t205539_spec3_subset_asn.json")
 
-    MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background')
+    MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='mbsub',
+                              save_background=True)
 
     return rtdata
 
 
 @pytest.mark.bigdata
-@pytest.mark.parametrize("exp_seq", ["exp1", "exp2", "exp3"])
-def test_miri_mrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs,
-                                 exp_seq):
+@pytest.mark.parametrize("exposure", [1, 2, 3, 4])
+@pytest.mark.parametrize("suffix", ["mbsub", "c1004_masterbg2d"])
+def test_miri_mrs_dedicated_mbkg(run_pipeline, fitsdiff_default_kwargs, exposure, suffix):
     """Run a test for MIRI MRS data with dedicated background exposures."""
+    rtdata = run_pipeline
+    output_file = f"jw01031001001_02101_0000{exposure}_mirifulong_{suffix}.fits"
+    rtdata.output = output_file
+
+    # Get the truth file
+    rtdata.get_truth(os.path.join("truth/test_miri_mrs_dedicated_mbkg", output_file))
+
+    # Compare the results
+    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
+    assert diff.identical, diff.report()
+
 
+@pytest.mark.bigdata
+def test_miri_mrs_dedicated_masterbg1d(run_pipeline, fitsdiff_default_kwargs):
     rtdata = run_pipeline
-    rtdata.output = "miri_mrs_seq1_long_34_" + exp_seq + \
-                    "_master_background.fits"
+
+    # Check 1D masterbg output product: created with root name from nod 1
+    output_file = "jw01031006001_02101_00001_mirifulong_c1004_masterbg1d.fits"
+    rtdata.output = output_file
 
     # Get the truth file
-    rtdata.get_truth(os.path.join(
-        "truth/test_miri_mrs_dedicated_mbkg",
-        "miri_mrs_seq1_long_34_" + exp_seq + "_master_background.fits"))
+    rtdata.get_truth(os.path.join("truth/test_miri_mrs_dedicated_mbkg", output_file))
 
     # Compare the results
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)

jwst/regtest/test_miri_mrs_tso.py

@@ -15,7 +15,7 @@ def run_spec2(rtdata_module):
     rtdata = rtdata_module
 
     # Setup the inputs
-    file_name = 'jw80600018001_02101_00003_mirifushort_rateints.fits'
+    file_name = 'jw01556001001_04102_00001-seg001_mirifushort_rateints.fits'
     rtdata.get_data(INPUT_PATH + '/' + file_name)
 
     # Run the pipeline
@@ -37,7 +37,7 @@ def run_spec2(rtdata_module):
 def test_spec2(rtdata_module, run_spec2, fitsdiff_default_kwargs, suffix):
     """Test ensuring the calwebb_tso-spec2 is operating appropriately for MIRI MRS TSO data"""
     rtdata = rtdata_module
-    output = f"jw80600018001_02101_00003_mirifushort_{suffix}.fits"
+    output = f"jw01556001001_04102_00001-seg001_mirifushort_{suffix}.fits"
     rtdata.output = output
 
     rtdata.get_truth(f"{TRUTH_PATH}/{output}")

jwst/regtest/test_nirspec_brightobj.py

@@ -20,22 +20,28 @@ def run_tso_spec2_pipeline(rtdata_module, request):
     rtdata = rtdata_module
 
     # Get the input exposure
-    rtdata.get_data('nirspec/tso/jw02420001001_04101_00001-seg001_nrs1_rateints.fits')
+    # Input data is from jw02420001001_04101_00001-seg001_nrs1_rateints.fits,
+    # modified to truncate the data to the first 100 integrations, for
+    # faster processing.
+    rtdata.get_data('nirspec/tso/jw02420001001_04101_00001-first100_nrs1_rateints.fits')
 
     # Run the calwebb_spec2 pipeline;
     args = ["calwebb_spec2", rtdata.input,
             "--steps.assign_wcs.save_results=True",
             "--steps.extract_2d.save_results=True",
             "--steps.wavecorr.save_results=True",
             "--steps.flat_field.save_results=True",
+            "--steps.flat_field.save_interpolated_flat=True",
             "--steps.photom.save_results=True"]
     Step.from_cmdline(args)
 
     return rtdata
 
 
 @pytest.mark.bigdata
-@pytest.mark.parametrize("suffix", ['assign_wcs', 'extract_2d', 'wavecorr', 'flat_field', 'photom', 'calints', 'x1dints'])
+@pytest.mark.parametrize("suffix", ['assign_wcs', 'extract_2d', 'wavecorr',
+                                    'flat_field', 'photom', 'calints', 'x1dints',
+                                    'interpolatedflat'])
 def test_nirspec_brightobj_spec2(run_tso_spec2_pipeline, fitsdiff_default_kwargs, suffix):
     """
         Regression test of calwebb_spec2 pipeline performed on NIRSpec
@@ -57,35 +63,27 @@ def test_nirspec_brightobj_spec2(run_tso_spec2_pipeline, fitsdiff_default_kwargs
 @pytest.mark.bigdata
 def test_flat_field_step_user_supplied_flat(rtdata, fitsdiff_default_kwargs):
     """Test providing a user-supplied flat field to the FlatFieldStep"""
-    data = rtdata.get_data('nirspec/tso/nrs2_wavecorr.fits')
-    user_supplied_flat = rtdata.get_data('nirspec/tso/nrs2_interpolatedflat.fits')
+    basename = 'jw02420001001_04101_00001-first100_nrs1'
+    output_file = f'{basename}_flat_from_user_file.fits'
 
-    data_flat_fielded = FlatFieldStep.call(data, user_supplied_flat=user_supplied_flat)
-    rtdata.output = 'flat_fielded_step_user_supplied.fits'
-    data_flat_fielded.write(rtdata.output)
-
-    rtdata.get_truth('truth/test_nirspec_brightobj_spec2/flat_fielded_step_user_supplied.fits')
-    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
-    assert diff.identical, diff.report()
+    data = rtdata.get_data(f'nirspec/tso/{basename}_wavecorr.fits')
+    user_supplied_flat = rtdata.get_data(f'nirspec/tso/{basename}_user_flat.fits')
 
+    data_flat_fielded = FlatFieldStep.call(data, user_supplied_flat=user_supplied_flat,
+                                           save_results=False)
+    rtdata.output = output_file
+    data_flat_fielded.write(rtdata.output)
 
-@pytest.mark.bigdata
-def test_flat_field_bots_interp_flat(rtdata, fitsdiff_default_kwargs):
-    """Test the interpolated flat for a NRS BOTS exposure"""
-    data = rtdata.get_data('nirspec/tso/jw93056001001_short_nrs1_wavecorr.fits')
-
-    FlatFieldStep.call(data, save_interpolated_flat=True)
-    rtdata.output = 'jw93056001001_short_nrs1_wavecorr_interpolatedflat.fits'
-
-    rtdata.get_truth('truth/test_nirspec_brightobj_spec2/jw93056001001_short_nrs1_wavecorr_interpolatedflat.fits')
+    rtdata.get_truth(f'truth/test_nirspec_brightobj_spec2/{output_file}')
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
     assert diff.identical, diff.report()
 
 
 @pytest.mark.bigdata
 def test_ff_inv(rtdata, fitsdiff_default_kwargs):
     """Test flat field inversion"""
-    with dm.open(rtdata.get_data('nirspec/tso/nrs2_wavecorr.fits')) as data:
+    basename = 'jw02420001001_04101_00001-first100_nrs1'
+    with dm.open(rtdata.get_data(f'nirspec/tso/{basename}_wavecorr.fits')) as data:
         flatted = FlatFieldStep.call(data)
         unflatted = FlatFieldStep.call(flatted, inverse=True)
 

jwst/regtest/test_nirspec_fs_spec2.py

@@ -130,7 +130,8 @@ def test_nirspec_fs_spec2_pixel_replace(run_pipeline_pixel_replace, fitsdiff_def
 @pytest.mark.bigdata
 def test_pathloss_corrpars(rtdata):
     """Test PathLossStep using correction_pars"""
-    with dm.open(rtdata.get_data('nirspec/fs/nrs1_flat_field.fits')) as data:
+    basename = 'jw02072002001_05101_00001_nrs1_flatfieldstep'
+    with dm.open(rtdata.get_data(f'nirspec/fs/{basename}.fits')) as data:
         pls = PathLossStep()
         corrected = pls.run(data)
 
@@ -148,7 +149,8 @@ def test_pathloss_corrpars(rtdata):
 @pytest.mark.bigdata
 def test_pathloss_inverse(rtdata):
     """Test PathLossStep using inversion"""
-    with dm.open(rtdata.get_data('nirspec/fs/nrs1_flat_field.fits')) as data:
+    basename = 'jw02072002001_05101_00001_nrs1_flatfieldstep'
+    with dm.open(rtdata.get_data(f'nirspec/fs/{basename}.fits')) as data:
         pls = PathLossStep()
         corrected = pls.run(data)
 
@@ -168,7 +170,8 @@ def test_pathloss_inverse(rtdata):
 @pytest.mark.bigdata
 def test_pathloss_source_type(rtdata):
     """Test PathLossStep forcing source type"""
-    with dm.open(rtdata.get_data('nirspec/fs/nrs1_flat_field.fits')) as data:
+    basename = 'jw02072002001_05101_00001_nrs1_flatfieldstep'
+    with dm.open(rtdata.get_data(f'nirspec/fs/{basename}.fits')) as data:
         pls = PathLossStep()
         pls.source_type = 'extended'
         pls.run(data)
@@ -189,7 +192,7 @@ def test_nirspec_fs_rateints_spec2(rtdata_module):
     """
     rtdata = rtdata_module
 
-    rtdata.get_data("nirspec/fs/jw01128004001_03102_00001_nrs1_new_rateints.fits")
+    rtdata.get_data("nirspec/fs/jw01128004001_03102_00001_nrs1_rateints.fits")
 
     # Run the spec2 pipeline on a (3D) _rateints file
     args = ["calwebb_spec2", rtdata.input]