From 4aadf2111b4a3b970984edea1272bf9d060f8f31 Mon Sep 17 00:00:00 2001 From: David Waterman Date: Fri, 11 Oct 2024 08:01:31 +0100 Subject: [PATCH] Load lookups when not checking format (#760) * Load lookups (when available) even if check_format=False Change the behaviour of ExperimentListDict._load_pickle_path so that the lookup files for mask, gain, pedestal and offset maps are loaded whenever the file is available. Previously these were only loaded when check_format=True. However, these files are produced by data processing so normally they would be available alongside experiments and reflections even if the raw images are not available. The existence of these files does not really have anything to do with the Format. This change ensures that dx, dy maps are loaded even when check_format=False, which fixes https://github.com/dials/dials/issues/2744 * Lack of dx, dy maps affects processing, so be strict about loading them. If they are expected but not loaded, then fail. Thanks to @ndevenish for the suggestion. --- newsfragments/760.bugfix | 3 +++ src/dxtbx/model/experiment_list.py | 35 +++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 10 deletions(-) create mode 100644 newsfragments/760.bugfix diff --git a/newsfragments/760.bugfix b/newsfragments/760.bugfix new file mode 100644 index 000000000..d1cacea56 --- /dev/null +++ b/newsfragments/760.bugfix @@ -0,0 +1,3 @@ +Ensure that data processing auxililary files (mask, gain, pedestal, and +dx and dy maps) are loaded whenever available. This fixes +https://github.com/dials/dials/issues/2744 diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index 9145bcf65..1386fd231 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -242,7 +242,7 @@ def _extract_models(self, name, from_dict): loaded from the named file in the target directory. If any experiments point to a file in this way, the imageset is - loaded and the experiment is rewritted with an integer pointing + loaded and the experiment is rewritten with an integer pointing to the new ImageSet in the returned list. Returns: @@ -284,7 +284,10 @@ def _load_pickle_path( self, imageset_data: dict, param: str ) -> tuple[str | None, Any]: """ - Read a filename from an imageset dict and load if required. + Read a filename from an imageset dict and load if available. This is + used to load mask, gain, pedestal and offset maps. In some situations + (such as tests) these files are not available, in which case the + filename is kept but the data is None. Args: imageset_data: The dictionary holding imageset information @@ -292,19 +295,24 @@ def _load_pickle_path( Returns: A tuple of (filename, data) where data has been loaded from - the pickle file. If there is no key entry then (None, None) - is returned. If the configuration parameter check_format is - False then (filename, None) will be returned. + the pickle file, or is None if the file is inaccessible. If there + is no key entry then ("", None) is returned. """ if param not in imageset_data: return "", None filename = resolve_path(imageset_data[param], directory=self._directory) - if self._check_format and filename: - with open(filename, "rb") as fh: - return filename, pickle.load(fh, encoding="bytes") + data = None + if filename: + try: + with open(filename, "rb") as fh: + data = pickle.load(fh, encoding="bytes") + except OSError: + pass + else: + filename = "" - return filename or "", None + return filename, data def _imageset_from_imageset_data(self, imageset_data, models): """Make an imageset from imageset_data - help with refactor decode.""" @@ -326,6 +334,14 @@ def _imageset_from_imageset_data(self, imageset_data, models): dx_filename, dx = self._load_pickle_path(imageset_data, "dx") dy_filename, dy = self._load_pickle_path(imageset_data, "dy") + # If dx, dy maps are expected then they must be loaded even when + # self._check_format == False, because they affect the operation of + # programs (dials.index, dials.refine) that do not need the image data. + if (dx_filename or dy_filename) and not all((dx, dx)): + raise RuntimeError( + f"dx ({dx_filename}) and dy ({dy_filename}) maps are expected" + ) + if imageset_data["__id__"] == "ImageSet": imageset = self._make_stills(imageset_data, format_kwargs=format_kwargs) elif imageset_data["__id__"] == "ImageGrid": @@ -400,7 +416,6 @@ def _imageset_from_imageset_data(self, imageset_data, models): imageset.set_detector(detector, i) imageset.set_goniometer(goniometer, i) imageset.set_scan(scan, i) - imageset.update_detector_px_mm_data() return imageset