MIRI JWST Pipeline Notebook 1.6 Version Incompatibility Issue

[hcha9]

I am going through this notebook to rerun the demo before loading our data, but it appears I had no issues with previous versions of JWST, but new ver 1.6 returns an error when processing the data. I reinstalled the previous version which worked just fine.

"https://github.com/spacetelescope/jwst-pipeline-notebooks/blob/main/notebooks/MIRI/JWPipeNB-MIRI-MRS.ipynb"

This was line 19.

"2024-09-30 01:18:15,568 - stpipe.Detector1Pipeline.ramp_fit - INFO - MIRI dataset has all pixels in the final group flagged as DO_NOT_USE.
2024-09-30 01:18:15,568 - stpipe.Detector1Pipeline.ramp_fit - INFO - Number of processors used for multiprocessing: 6
Error - [C:2582] pr->orig_gdq is NULL.
Error - [C:2582] pr->orig_gdq is NULL.
Error - [C:2582] pr->orig_gdq is NULL.
Error - [C:2582] pr->orig_gdq is NULL.
Error - [C:2582] pr->orig_gdq is NULL.
Error - [C:2582] pr->orig_gdq is NULL.

RemoteTraceback Traceback (most recent call last)
RemoteTraceback:
"""
MemoryError: pr->orig_gdq is NULL.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/usr/local/Cellar/[email protected]/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/pool.py", line 125, in worker
result = (True, func(*args, **kwds))
^^^^^^^^^^^^^^^^^^^
File "/usr/local/Cellar/[email protected]/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/pool.py", line 51, in starmapstar
return list(itertools.starmap(args[0], args[1]))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/hcha/jwst-env/lib/python3.12/site-packages/stcal/ramp_fitting/ols_fit.py", line 676, in ols_ramp_fit_single
image_info, integ_info, opt_info = ols_slope_fitter(
^^^^^^^^^^^^^^^^^
SystemError: returned a result with an exception set
"""

The above exception was the direct cause of the following exception:

SystemError Traceback (most recent call last)
Cell In[20], line 5
3 if dodet1:
4 for file in uncal_files:
----> 5 Detector1Pipeline.call(file, steps=det1dict, save_results=True, output_dir=det1_dir)
6 else:
7 print('Skipping Detector1 processing for SCI data')

File ~/jwst-env/lib/python3.12/site-packages/stpipe/step.py:697, in Step.call(cls, *args, **kwargs)
694 name = config.get("name", None)
695 instance = cls.from_config_section(config, name=name, config_file=config_file)
--> 697 return instance.run(*args)

File ~/jwst-env/lib/python3.12/site-packages/jwst/stpipe/core.py:100, in JwstStep.run(self, *args, **kwargs)
98 @wraps(Step.run)
99 def run(self, *args, **kwargs):
--> 100 result = super().run(*args, **kwargs)
101 if not self.parent:
102 log.info(f"Results used jwst version: {version}")

File ~/jwst-env/lib/python3.12/site-packages/stpipe/step.py:524, in Step.run(self, *args)
522 self.prefetch(*args)
523 try:
--> 524 step_result = self.process(*args)
525 except TypeError as e:
526 if "process() takes exactly" in str(e):

File ~/jwst-env/lib/python3.12/site-packages/jwst/pipeline/calwebb_detector1.py:149, in Detector1Pipeline.process(self, input)
147 ints_model = None
148 else:
--> 149 input, ints_model = self.ramp_fit(input)
151 # apply the gain_scale step to the exposure-level product
152 if input is not None:

File ~/jwst-env/lib/python3.12/site-packages/stpipe/step.py:524, in Step.run(self, *args)
522 self.prefetch(*args)
523 try:
--> 524 step_result = self.process(*args)
525 except TypeError as e:
526 if "process() takes exactly" in str(e):

File ~/jwst-env/lib/python3.12/site-packages/jwst/ramp_fitting/ramp_fit_step.py:464, in RampFitStep.process(self, step_input)
459 input_model_W = result.copy()
461 # Run ramp_fit(), ignoring all DO_NOT_USE groups, and return the
462 # ramp fitting arrays for the ImageModel, the CubeModel, and the
463 # RampFitOutputModel.
--> 464 image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit(
465 result, buffsize, self.save_opt, readnoise_2d, gain_2d,
466 self.algorithm, self.weighting, max_cores, dqflags.pixel,
467 suppress_one_group=self.suppress_one_group)
469 # Create a gdq to modify if there are charge_migrated groups
470 if self.algorithm == "OLS":

File ~/jwst-env/lib/python3.12/site-packages/stcal/ramp_fitting/ramp_fit.py:195, in ramp_fit(model, buffsize, save_opt, readnoise_2d, gain_2d, algorithm, weighting, max_cores, dqflags, suppress_one_group)
190 # Create an instance of the internal ramp class, using only values needed
191 # for ramp fitting from the to remove further ramp fitting dependence on
192 # data models.
193 ramp_data = create_ramp_fit_class(model, algorithm, dqflags, suppress_one_group)
--> 195 return ramp_fit_data(
196 ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, algorithm, weighting, max_cores, dqflags
197 )

File ~/jwst-env/lib/python3.12/site-packages/stcal/ramp_fitting/ramp_fit.py:295, in ramp_fit_data(ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, algorithm, weighting, max_cores, dqflags)
292 suppress_one_good_group_ramps(ramp_data)
294 # Compute ramp fitting using ordinary least squares.
--> 295 image_info, integ_info, opt_info = ols_fit.ols_ramp_fit_multi(
296 ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, weighting, max_cores
297 )
298 gls_opt_info = None
300 return image_info, integ_info, opt_info, gls_opt_info

File ~/jwst-env/lib/python3.12/site-packages/stcal/ramp_fitting/ols_fit.py:111, in ols_ramp_fit_multi(ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, weighting, max_cores)
108 return image_info, integ_info, opt_info
110 # Call ramp fitting for multi-processor (multiple data slices) case
--> 111 image_info, integ_info, opt_info = ols_ramp_fit_multiprocessing(
112 ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, weighting, number_slices
113 )
115 return image_info, integ_info, opt_info

File ~/jwst-env/lib/python3.12/site-packages/stcal/ramp_fitting/ols_fit.py:169, in ols_ramp_fit_multiprocessing(ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, weighting, number_slices)
167 ctx = multiprocessing.get_context("forkserver")
168 pool = ctx.Pool(processes=number_slices)
--> 169 pool_results = pool.starmap(ols_ramp_fit_single, slices)
170 pool.close()
171 pool.join()

File /usr/local/Cellar/[email protected]/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/pool.py:375, in Pool.starmap(self, func, iterable, chunksize)
369 def starmap(self, func, iterable, chunksize=None):
370 '''
371 Like map() method but the elements of the iterable are expected to
372 be iterables as well and will be unpacked as arguments. Hence
373 func and (a, b) becomes func(a, b).
374 '''
--> 375 return self._map_async(func, iterable, starmapstar, chunksize).get()

File /usr/local/Cellar/[email protected]/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/multiprocessing/pool.py:774, in ApplyResult.get(self, timeout)
772 return self._value
773 else:
--> 774 raise self._value

SystemError: returned a result with an exception set"

[melanieclarke]

Thanks for reporting this! @kmacdonald-stsci - can you please take a look?

[hcha9]

Thanks for reporting this! @kmacdonald-stsci - can you please take a look?

Interesting. The page you provided has only up to 1.5.1, but this page (https://github.com/spacetelescope/jwst) says the latest version is 1.6 which was released about 10 days ago. hmm... I pip upgraded jwst this morning which automatically gives 1.6. The issue was resolved after downgrading to 1.4.1 version.

[melanieclarke]

Thanks for checking, but I was just tagging in one of our developers to take a look at your issue. :)

To be clear - you saw the issue with jwst 1.16.0, and did not see when you re-installed 1.14.1?

[hcha9]

Thanks for checking, but I was just tagging in one of our developers to take a look at your issue. :)

To be clear - you saw the issue with jwst 1.16.0, and did not see when you re-installed 1.14.1?

Oh, my mistake. Yes, it was 1.16.0 giving back issues, but 1.14.1 works just fine.
Thank you for the reply tho!

[melanieclarke]

I can reproduce the issue with one of the MIRI uncal files from the notebook, with multiprocessing on for ramp fitting:
strun calwebb_detector1 jw01523003001_03102_00001_mirifulong_uncal.fits --steps.ramp_fit.maximum_cores=half

@hcha9 - it looks like you should be able to work around it with v1.16.0 by turning off multiprocessing for ramp fitting. Try commenting out this line in cell 16: det1dict['ramp_fit']['maximum_cores'] = 'half'

[hcha9]

I can reproduce the issue with one of the MIRI uncal files from the notebook, with multiprocessing on for ramp fitting: strun calwebb_detector1 jw01523003001_03102_00001_mirifulong_uncal.fits --steps.ramp_fit.maximum_cores=half

@hcha9 - it looks like you should be able to work around it with v1.16.0 by turning off multiprocessing for ramp fitting. Try commenting out this line in cell 16: det1dict['ramp_fit']['maximum_cores'] = 'half'

There we go! Awesome! Thank you so much! I missed that part and works just fine with 1.16.0.

[melanieclarke]

One more follow up - it looks like this issue should be fixed when spacetelescope/stcal#289 is merged. Testing with stcal on that branch I do not see the initialization error with the above test case.

[kmacdonald-stsci]

This bug is caused by erroneously going into the CHARGELOSS computation here:

https://github.com/spacetelescope/stcal/blob/4ba7a620ad6545067695fe3808b405a7a50e4d53/src/stcal/ramp_fitting/src/slope_fitter.c#L2257

It is because this default True value:

https://github.com/spacetelescope/stcal/blob/4ba7a620ad6545067695fe3808b405a7a50e4d53/src/stcal/ramp_fitting/ramp_fit_class.py#L51

I found this potential bug while testing a currently open PR

spacetelescope/stcal#289

In this PR that value now defaults to False and is set to True only if orig_gdq is non-NULL:

https://github.com/kmacdonald-stsci/stcal/blob/a7f72d4012c504264780e7dea4d89b06abcd569e/src/stcal/ramp_fitting/ramp_fit.py#L67

[hcha9]

One more follow up - it looks like this issue should be fixed when spacetelescope/stcal#289 is merged. Testing with stcal on that branch I do not see the initialization error with the above test case.

This bug is caused by erroneously going into the CHARGELOSS computation here:

https://github.com/spacetelescope/stcal/blob/4ba7a620ad6545067695fe3808b405a7a50e4d53/src/stcal/ramp_fitting/src/slope_fitter.c#L2257

It is because this default True value:

https://github.com/spacetelescope/stcal/blob/4ba7a620ad6545067695fe3808b405a7a50e4d53/src/stcal/ramp_fitting/ramp_fit_class.py#L51

I found this potential bug while testing a currently open PR

spacetelescope/stcal#289

In this PR that value now defaults to False and is set to True only if orig_gdq is non-NULL:

https://github.com/kmacdonald-stsci/stcal/blob/a7f72d4012c504264780e7dea4d89b06abcd569e/src/stcal/ramp_fitting/ramp_fit.py#L67

Thank you so much for helping me to figure out bugs with the latest version!

I just ran into another bug :
" soss_tikfac: None
soss_width: 40.0
soss_bad_pix: masking
soss_modelname: None
2024-09-30 13:21:43,043 - stpipe.Spec2Pipeline - INFO - Prefetching reference files for dataset: 'jw01523004001_02101_00001_mirifulong_rate.fits' reftypes = ['apcorr', 'area', 'barshadow', 'camera', 'collimator', 'cubepar', 'dflat', 'disperser', 'distortion', 'extract1d', 'fflat', 'filteroffset', 'flat', 'fore', 'fpa', 'fringe', 'ifufore', 'ifupost', 'ifuslicer', 'mrsxartcorr', 'msa', 'msaoper', 'ote', 'pastasoss', 'pathloss', 'photom', 'regions', 'sflat', 'speckernel', 'specprofile', 'specwcs', 'wavecorr', 'wavelengthrange']
2024-09-30 13:21:43,047 - CRDS - ERROR - Error determining best reference for 'pastasoss' = Unknown reference type 'pastasoss'

CrdsLookupError Traceback (most recent call last)
Cell In[29], line 3
1 if dospec2bg:
2 for file in bgfiles:
----> 3 Spec2Pipeline.call(file, steps=spec2dict, save_results=True, output_dir=spec2_bgdir)
4 else:
5 print('Skipping Spec2 processing for BG data')

File ~/jwst-env/lib/python3.12/site-packages/stpipe/step.py:697, in Step.call(cls, *args, **kwargs)
694 name = config.get("name", None)
695 instance = cls.from_config_section(config, name=name, config_file=config_file)
--> 697 return instance.run(*args)

File ~/jwst-env/lib/python3.12/site-packages/jwst/stpipe/core.py:100, in JwstStep.run(self, *args, **kwargs)
98 @wraps(Step.run)
99 def run(self, *args, **kwargs):
--> 100 result = super().run(*args, **kwargs)
101 if not self.parent:
102 log.info(f"Results used jwst version: {version}")

File ~/jwst-env/lib/python3.12/site-packages/stpipe/step.py:522, in Step.run(self, *args)
520 else:
521 if self.prefetch_references:
--> 522 self.prefetch(*args)
523 try:
524 step_result = self.process(*args)

File ~/jwst-env/lib/python3.12/site-packages/stpipe/step.py:641, in Step.prefetch(self, *args)
639 # prefetch truly occurs at the Pipeline (or subclass) level.
640 if len(args) and len(self.reference_file_types) and not self.skip:
--> 641 self._precache_references(args[0])

File ~/jwst-env/lib/python3.12/site-packages/stpipe/pipeline.py:266, in Pipeline._precache_references(self, input_file)
262 try:
263 with self.open_model(
264 input_file, asn_n_members=1, asn_exptypes=["science"]
265 ) as model:
--> 266 self._precache_references_opened(model)
267 except (ValueError, TypeError, OSError):
268 self.log.info("First argument %s does not appear to be a model", input_file)

File ~/jwst-env/lib/python3.12/site-packages/stpipe/pipeline.py:292, in Pipeline._precache_references_opened(self, model_or_container)
289 model_or_container.shelve(model, i, modify=False)
290 else:
291 # precache a single model object
--> 292 self._precache_references_impl(model_or_container)

File ~/jwst-env/lib/python3.12/site-packages/stpipe/pipeline.py:319, in Pipeline._precache_references_impl(self, model)
312 fetch_types = sorted(set(self.reference_file_types) - set(ovr_refs.keys()))
314 self.log.info(
315 "Prefetching reference files for dataset: %r reftypes = %r",
316 model.meta.filename,
317 fetch_types,
318 )
--> 319 crds_refs = crds_client.get_multiple_reference_paths(
320 model.get_crds_parameters(), fetch_types, model.crds_observatory
321 )
323 ref_path_map = dict(list(crds_refs.items()) + list(ovr_refs.items()))
325 for reftype, refpath in sorted(ref_path_map.items()):

File ~/jwst-env/lib/python3.12/site-packages/stpipe/crds_client.py:53, in get_multiple_reference_paths(parameters, reference_file_types, observatory)
50 raise TypeError("First argument must be a dict of parameters")
52 log.set_log_time(True)
---> 53 return _get_refpaths(parameters, tuple(reference_file_types), observatory)

File ~/jwst-env/lib/python3.12/site-packages/stpipe/crds_client.py:65, in _get_refpaths(data_dict, reference_file_types, observatory)
63 return {}
64 with crds_cache_locking.get_cache_lock():
---> 65 bestrefs = crds.getreferences(
66 data_dict,
67 reftypes=reference_file_types,
68 observatory=observatory,
69 )
70 return {
71 filetype: filepath if "N/A" not in filepath.upper() else "N/A"
72 for (filetype, filepath) in bestrefs.items()
73 }

File ~/jwst-env/lib/python3.12/site-packages/crds/core/heavy_client.py:127, in getreferences(parameters, reftypes, context, ignore_cache, observatory, fast)
122 final_context, bestrefs = _initial_recommendations("getreferences",
123 parameters, reftypes, context, ignore_cache, observatory, fast)
125 # Attempt to cache the recommended references, which unlike dump_mappings
126 # should work without network access if files are already cached.
--> 127 best_refs_paths = api.cache_references(
128 final_context, bestrefs, ignore_cache=ignore_cache)
130 return best_refs_paths

File ~/jwst-env/lib/python3.12/site-packages/crds/client/api.py:848, in cache_references(pipeline_context, bestrefs, ignore_cache)
840 def cache_references(pipeline_context, bestrefs, ignore_cache=False):
841 """Given a CRDS context pipeline_context and bestrefs dictionary, download missing
842 reference files and cache them on the local file system.
843
(...)
846 Returns: { reference_keyword : reference_local_filepath ... }
847 """
--> 848 wanted = _get_cache_filelist_and_report_errors(bestrefs)
850 if config.S3_RETURN_URI:
851 localrefs = {name: get_flex_uri(name) for name in wanted}

File ~/jwst-env/lib/python3.12/site-packages/crds/client/api.py:891, in _get_cache_filelist_and_report_errors(bestrefs)
889 log.error(str(last_error))
890 if last_error is not None:
--> 891 raise last_error
892 return wanted

CrdsLookupError: Error determining best reference for 'pastasoss' = Unknown reference type 'pastasoss'"

[melanieclarke]

Sorry you're running into all these issues at once!

This one:

CrdsLookupError: Error determining best reference for 'pastasoss' = Unknown reference type 'pastasoss'"

is a known issue, relating to some release coordination between CRDS and the new pipeline version. The issue and fix are described here:
#8832 (comment)