Load lookups when not checking format (#760)

* Load lookups (when available) even if check_format=False

Change the behaviour of ExperimentListDict._load_pickle_path so that the
lookup files for mask, gain, pedestal and offset maps are loaded whenever
the file is available. Previously these were only loaded when
check_format=True. However, these files are produced by data processing
so normally they would be available alongside experiments and reflections
even if the raw images are not available. The existence of these
files does not really have anything to do with the Format.

This change ensures that dx, dy maps are loaded even when
check_format=False, which fixes dials/dials#2744

* Lack of dx, dy maps affects processing, so be strict about loading
them. If they are expected but not loaded, then fail.

Thanks to @ndevenish for the suggestion.

newsfragments/760.bugfix

@@ -0,0 +1,3 @@
+Ensure that data processing auxililary files (mask, gain, pedestal, and
+dx and dy maps) are loaded whenever available. This fixes
+https://github.com/dials/dials/issues/2744

src/dxtbx/model/experiment_list.py

@@ -242,7 +242,7 @@ def _extract_models(self, name, from_dict):
         loaded from the named file in the target directory.
 
         If any experiments point to a file in this way, the imageset is
-        loaded and the experiment is rewritted with an integer pointing
+        loaded and the experiment is rewritten with an integer pointing
         to the new ImageSet in the returned list.
 
         Returns:
@@ -284,27 +284,35 @@ def _load_pickle_path(
         self, imageset_data: dict, param: str
     ) -> tuple[str | None, Any]:
         """
-        Read a filename from an imageset dict and load if required.
+        Read a filename from an imageset dict and load if available. This is
+        used to load mask, gain, pedestal and offset maps. In some situations
+        (such as tests) these files are not available, in which case the
+        filename is kept but the data is None.
 
         Args:
             imageset_data: The dictionary holding imageset information
             param: The key name to lookup in the imageset dictionary
 
         Returns:
             A tuple of (filename, data) where data has been loaded from
-            the pickle file. If there is no key entry then (None, None)
-            is returned. If the configuration parameter check_format is
-            False then (filename, None) will be returned.
+            the pickle file, or is None if the file is inaccessible. If there
+            is no key entry then ("", None) is returned.
         """
         if param not in imageset_data:
             return "", None
 
         filename = resolve_path(imageset_data[param], directory=self._directory)
-        if self._check_format and filename:
-            with open(filename, "rb") as fh:
-                return filename, pickle.load(fh, encoding="bytes")
+        data = None
+        if filename:
+            try:
+                with open(filename, "rb") as fh:
+                    data = pickle.load(fh, encoding="bytes")
+            except OSError:
+                pass
+        else:
+            filename = ""
 
-        return filename or "", None
+        return filename, data
 
     def _imageset_from_imageset_data(self, imageset_data, models):
         """Make an imageset from imageset_data - help with refactor decode."""
@@ -326,6 +334,14 @@ def _imageset_from_imageset_data(self, imageset_data, models):
         dx_filename, dx = self._load_pickle_path(imageset_data, "dx")
         dy_filename, dy = self._load_pickle_path(imageset_data, "dy")
 
+        # If dx, dy maps are expected then they must be loaded even when
+        # self._check_format == False, because they affect the operation of
+        # programs (dials.index, dials.refine) that do not need the image data.
+        if (dx_filename or dy_filename) and not all((dx, dx)):
+            raise RuntimeError(
+                f"dx ({dx_filename}) and dy ({dy_filename}) maps are expected"
+            )
+
         if imageset_data["__id__"] == "ImageSet":
             imageset = self._make_stills(imageset_data, format_kwargs=format_kwargs)
         elif imageset_data["__id__"] == "ImageGrid":
@@ -400,7 +416,6 @@ def _imageset_from_imageset_data(self, imageset_data, models):
                     imageset.set_detector(detector, i)
                     imageset.set_goniometer(goniometer, i)
                     imageset.set_scan(scan, i)
-
             imageset.update_detector_px_mm_data()
 
         return imageset