Merge branch 'main' into jp-3677

spacetelescope · Nov 14, 2024 · 2b1824a · 2b1824a
2 parents c6195a4 + e9a482d
commit 2b1824a
Show file tree

Hide file tree

Showing 209 changed files with 2,367 additions and 3,784 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -16,7 +16,7 @@ jobs:
     name: check that `requirements-sdp.txt` is populated
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
       - run: grep -v '^ *#' requirements-sdp.txt
   build:
     needs: [ check ]

diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -18,10 +18,10 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog-entry-needed') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3  # v5.2.0
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
         with:
           python-version: 3
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           fetch-depth: 0
       - run: pip install .
@@ -32,7 +32,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'allow-manual-changelog-edit') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           fetch-depth: 0
       - name: prevent direct changes to `CHANGES.rst`

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,8 +34,8 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871  # v4.2.1
-    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3  # v5.2.0
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+    - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
       with:
         python-version: '3.12'
     - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd  # v3.0.1

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,3 +1,12 @@
+1.16.1 (2024-10-30)
+===================
+
+resample_spec
+-------------
+
+- Update NIRSpec spectral resampling to add a missing correction factor in resampled
+  WCS tangent plane transformation. [#8908]
+
 1.16.0 (2024-09-20)
 ===================
 

diff --git a/README.md b/README.md
@@ -216,12 +216,13 @@ the specified context and less than the context for the next release.
 
 | jwst tag            | DMS build | SDP_VER  | CRDS_CONTEXT | Released   | Ops Install | Notes                                         |
 |---------------------|-----------|----------|--------------|------------|-------------|-----------------------------------------------|
-| 1.16.0              | B11.1rc1  | TBD      | 1281         | 2024-09-20 | TBD         | First release candidate for B11.1             |
-| 1.15.1              | B11.0     | 2024.2.2 | 1242         | 2024-07-08 | 2024-09-12  | Final release candidate for B11.0             |
-| 1.15.0              | B11.0rc1  |          | 1241         | 2024-06-26 |             | First release candidate for B11.0             |
-| 1.14.1              |           |          | 1238         | 2024-06-27 |             | PyPI-only release for external users          |
-| 1.14.0              | B10.2.1   | 2024.1.1 | 1238         | 2024-03-29 | 2024-06-12  | Final release candidate for B10.2.1           |
-| 1.13.4              |           |          | 1185         | 2024-01-25 |             | PyPI-only release for external users          |
+| 1.16.1              | B11.1.1   | 2024.3.1 | 1298         | 2024-11-13 | TBD         | Final release candidate for B11.1             |
+| 1.16.0              | B11.1     | 2024.3.0 | 1298         | 2024-09-20 | TBD         | First release candidate for B11.1             |
+| 1.15.1              | B11.0     | 2024.2.2 | 1293         | 2024-07-08 | 2024-09-12  | Final release candidate for B11.0             |
+| 1.15.0              | B11.0rc1  |          | 1274         | 2024-06-26 |             | First release candidate for B11.0             |
+| 1.14.1              |           |          | 1240         | 2024-06-27 |             | PyPI-only release for external users          |
+| 1.14.0              | B10.2.1   | 2024.1.1 | 1240         | 2024-03-29 | 2024-06-12  | Final release candidate for B10.2.1           |
+| 1.13.4              |           |          | 1210         | 2024-01-25 |             | PyPI-only release for external users          |
 | 1.13.3              | B10.1     | 2023.4.0 | 1181         | 2024-01-05 |             | Final release candidate for B10.1             |
 | 1.13.2              | B10.1rc3  | 2023.4.0 | 1181         | 2023-12-21 |             | Third release candidate for B10.1             |
 | 1.13.1              | B10.1rc2  | 2023.4.0 | 1181         | 2023-12-19 |             | Second release candidate for B10.1            |

diff --git a/changes/8828.outlier_detection.rst b/changes/8828.outlier_detection.rst
@@ -0,0 +1 @@
+For slit spectra, threshold outliers with a median error across exposures instead of input error from the exposure itself.
diff --git a/changes/8831.datamodels.rst b/changes/8831.datamodels.rst
@@ -0,0 +1 @@
+Remove memory-saving options from ModelContainer
diff --git a/changes/8866.resample.rst b/changes/8866.resample.rst
@@ -0,0 +1 @@
+Updated resample code to support the new ``drizzle`` API, see https://github.com/spacetelescope/drizzle/pull/134 for more details.
diff --git a/changes/8874.assign_wcs.rst b/changes/8874.assign_wcs.rst
@@ -0,0 +1 @@
+Catch NaN values in msa tables for source positions in slit and replace with slit center.
diff --git a/changes/8880.outlier_detection.rst b/changes/8880.outlier_detection.rst
@@ -0,0 +1 @@
+reorganize outlier detection documentation
diff --git a/changes/8885.general.rst b/changes/8885.general.rst
@@ -0,0 +1 @@
+Give regtest okify results unique subdirectories.
diff --git a/changes/8892.skymatch.rst b/changes/8892.skymatch.rst
@@ -0,0 +1 @@
+Resolve warnings emitted by NumPy 2 when running skymatch.
diff --git a/changes/8893.resample.rst b/changes/8893.resample.rst
@@ -0,0 +1 @@
+Use s_region list to calculate output footprint instead of re-computing via WCS transforms
diff --git a/changes/8897.assign_wcs.rst b/changes/8897.assign_wcs.rst
@@ -0,0 +1 @@
+Use pixel vertices to define s_region instead of pixel centers."
diff --git a/changes/8907.pipeline.rst b/changes/8907.pipeline.rst
@@ -0,0 +1 @@
+Fixed a bug leading to incorrect area extensions, and sometimes crashes, in the coron3 pipeline
diff --git a/changes/8908.resample_spec.rst b/changes/8908.resample_spec.rst
@@ -0,0 +1 @@
+Update NIRSpec spectral resampling to add a missing correction factor in resampled WCS tangent plane transformation.
diff --git a/changes/8909.scripts.rst b/changes/8909.scripts.rst
@@ -0,0 +1 @@
+Remove the outdated schema_editor script.
diff --git a/changes/8911.cube_build.rst b/changes/8911.cube_build.rst
@@ -0,0 +1 @@
+For moving-target IFU data, set RA, Dec header information of s3d products according to the mean of input models instead of the first input model.
diff --git a/changes/8913.cube_build.rst b/changes/8913.cube_build.rst
@@ -0,0 +1 @@
+Tweak the cube_build spaxel debugging option to provide filename info and match outputs to the stated ordering.
diff --git a/changes/8916.background.rst b/changes/8916.background.rst
@@ -0,0 +1 @@
+Apply bitwise operations in correct order when counting good pixels in the background mask during WFSS background subtraction.
diff --git a/changes/8918.datamodels.rst b/changes/8918.datamodels.rst
@@ -0,0 +1 @@
+Update ModelLibrary to use meta.asn.exptype instead of meta.exptype.
diff --git a/changes/8926.docs.rst b/changes/8926.docs.rst
@@ -0,0 +1 @@
+Mention possible need to provide package name to strun when using aliases.
diff --git a/changes/8927.combine_1d.rst b/changes/8927.combine_1d.rst
@@ -0,0 +1 @@
+Fix wavelength sort order for single input spectrum.
diff --git a/changes/8927.master_background.rst b/changes/8927.master_background.rst
@@ -0,0 +1 @@
+Fix ModelContainer handling for user background input.
diff --git a/changes/8927.pipeline.rst b/changes/8927.pipeline.rst
@@ -0,0 +1 @@
+Add 'mbsub' to the list of known suffixes, for ``master_background`` correction in ``calwebb_spec3``.
diff --git a/changes/8935.general.rst b/changes/8935.general.rst
@@ -0,0 +1 @@
+Include xml and db test data as package data for lib tests.
diff --git a/changes/8945.stpipe.rst b/changes/8945.stpipe.rst
@@ -0,0 +1 @@
+Remove all uses of Step.__call__ to allow it's deprecation.
diff --git a/docs/jwst/outlier_detection/arguments.rst b/docs/jwst/outlier_detection/arguments.rst
@@ -1,19 +1,53 @@
 .. _outlier_detection_step_args:
 
+Step Arguments
+==============
+
+The outlier detection step has the following optional arguments
+that control the behavior of the processing.
 For more details about step arguments (including datatypes, possible values
 and defaults) see :py:obj:`jwst.outlier_detection.OutlierDetectionStep.spec`.
 
-Step Arguments for Non-IFU data
-===============================
-The `outlier_detection` step for non-IFU data has the following optional arguments
-that control the behavior of the processing:
+
+General Step Arguments
+----------------------
+The following arguments apply to all modes unless otherwise specified:
+
+``--save_intermediate_results``
+  Specifies whether or not to save any intermediate products created
+  during step processing.
+
+``--good_bits``
+  The DQ bit values from the input image DQ arrays
+  that should be considered 'good'. Any pixel with a DQ value not included
+  in this value (or list of values) will be ignored when resampling and flagged
+  when building the weight mask. See DQ flag :ref:`dq_parameter_specification` for details.
+  Has no effect for IFU data.
+
+``--snr``
+  The signal-to-noise values to use for bad pixel identification.
+  Since cosmic rays often extend across several pixels, the user
+  must specify two cut-off values for determining whether a pixel should
+  be masked: the first for detecting the primary cosmic ray, and the
+  second (typically lower threshold) for masking lower-level bad pixels
+  adjacent to those found in the first pass.  Valid values are a pair of
+  floating-point values in a single string (for example "5.0 4.0").
+  Has no effect for IFU data.
+
+
+Step Arguments for Imaging and Slit-like Spectroscopic data
+-----------------------------------------------------------
 
 ``--weight_type``
-  The type of data weighting to use during resampling.
+  The type of data weighting to apply to the resampled data. Available options are ``ivm``
+  (default) to compute and use an inverse-variance map, and ``exptime`` to
+  weight by the exposure time.
 
 ``--pixfrac``
-  The pixel fraction used during resampling;
-  valid values go from 0.0 to 1.0.
+  The pixel fraction used during resampling; valid values go from 0.0 to 1.0.
+  Indicates the fraction by which input pixels are "shrunk" before being drizzled onto the
+  output image grid. This specifies the size of the footprint, or "dropsize", of a pixel
+  in units of the input pixel size.
 
 ``--kernel``
   The form of the kernel function used to distribute flux onto a
@@ -23,21 +57,12 @@ that control the behavior of the processing:
   The value to assign to resampled image pixels that have zero weight or
   do not receive any flux from any input pixels during drizzling.
   Any floating-point value, given as a string, is valid.
-  A value of 'INDEF' will use the last zero weight flux.
+  The default value of 'NAN' sets NaN values.
 
 ``--maskpt``
   The percent of maximum weight to use as lower-limit for valid data;
   valid values go from 0.0 to 1.0.
 
-``--snr``
-  The signal-to-noise values to use for bad pixel identification.
-  Since cosmic rays often extend across several pixels the user
-  must specify two cut-off values for determining whether a pixel should
-  be masked: the first for detecting the primary cosmic ray, and the
-  second (typically lower threshold) for masking lower-level bad pixels
-  adjacent to those found in the first pass.  Valid values are a pair of
-  floating-point values in a single string (for example "5.0 4.0").
-
 ``--scale``
   The scaling factor applied to derivative used to identify bad pixels.
   Since cosmic rays often extend across several pixels the user
@@ -50,59 +75,54 @@ that control the behavior of the processing:
 ``--backg``
   User-specified background value to apply to the median image.
 
-``--rolling_window_width``
-  Number of integrations over which to take the median when using rolling-window
-  median for TSO observations.
-
-``--save_intermediate_results``
-  Specifies whether or not to save any intermediate products created
-  during step processing.
-
 ``--resample_data``
   Specifies whether or not to resample the input images when
   performing outlier detection.
 
-``--good_bits``
-  The DQ bit values from the input image DQ arrays
-  that should be considered 'good' when building the weight mask. See
-  DQ flag :ref:`dq_parameter_specification` for details.
-
 ``--allowed_memory``
   Specifies the fractional amount of
   free memory to allow when creating the resampled image. If ``None``, the
   environment variable ``DMODEL_ALLOWED_MEMORY`` is used. If not defined, no
   check is made. If the resampled image would be larger than specified, an
   ``OutputTooLargeError`` exception will be generated.
-
   For example, if set to ``0.5``, only resampled images that use less than half
   the available memory can be created.
 
 ``--in_memory``
   Specifies whether or not to load and create all images that are used during
   processing into memory. If ``False``, input files are loaded from disk when
   needed and all intermediate files are stored on disk, rather than in memory.
-  This flag is superseded by the pipeline-level ``--in-memory`` flag, and thus
-  has no effect when running the full level 3 pipeline.
+  Has no effect for spectroscopic data. For imaging data this parameter is 
+  superseded by the pipeline-level ``in_memory`` parameter set by
+  ``calwebb_image3``.
+
 
 Step Arguments for IFU data
-===========================
-The `outlier_detection` step for IFU data has the following optional arguments
-that control the behavior of the processing:
+---------------------------
 
 ``--kernel_size``
   The size of the kernel to use to normalize the pixel differences. The kernel size
   must only contain odd values. Valid values are a pair of ints in a single string
-  (for example "7 7").
+  (for example "7 7", the default).
 
 ``--threshold_percent``
   The threshold (in percent) of the normalized minimum pixel difference used to identify bad pixels.
-  Pixels with   a normalized minimum pixel difference above this percentage are flagged as a outlier.
+  Pixels with a normalized minimum difference above this percentage are flagged as outliers.
 
-``--save_intermediate_results``
-  Specifies whether or not to save any intermediate products created
-  during step processing.
+``--ifu_second_check``
+  Perform a secondary check searching for outliers. This will set outliers
+  where ever the difference array of adjacent pixels is a Nan.
 
-``--in_memory``
-  Specifies whether or not to load and create all images that are used during
-  processing into memory. If ``False``, input files are loaded from disk when
-  needed and all intermediate files are stored on disk, rather than in memory.
+
+Step Arguments for TSO data
+---------------------------
+
+``--rolling_window_width``
+  Number of integrations over which to take the median when using rolling-window
+  median for TSO observations. The default is 25. If the number of integrations
+  is less than or equal to ``rolling_window_width``, a simple median is used instead.
+
+
+Step Arguments for Coronagraphic data
+-------------------------------------
+General step arguments apply to coronagraphic data. No additional arguments are used.
diff --git a/docs/jwst/outlier_detection/index.rst b/docs/jwst/outlier_detection/index.rst
@@ -8,12 +8,12 @@ Outlier Detection
    :maxdepth: 2
 
    main.rst
+   reference_files.rst
    arguments.rst
-   outlier_detection_step.rst
    outlier_detection_imaging.rst
+   outlier_detection_spec.rst
    outlier_detection_coron.rst
    outlier_detection_tso.rst
    outlier_detection_ifu.rst
-   outlier_detection_spec.rst
 
 .. automodapi:: jwst.outlier_detection