Merge pull request #873 from lsst/tickets/DM-40297

DM-40297: Stop checking dataset existence when creating deferred dataset handle

pyproject.toml

@@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "lsst-daf-butler"
+requires-python = ">=3.10.0"
 description = "An abstraction layer for reading and writing astronomical data to datastores."
 license = {text = "GPLv3+ License"}
 readme = "README.md"

python/lsst/daf/butler/_butler.py

@@ -1279,9 +1279,9 @@ def getDirectDeferred(
         LookupError
             Raised if no matching dataset exists in the `Registry`.
         """
-        # Check that dataset actually exists.
-        if not self._datastore.exists(ref):
-            raise LookupError(f"Dataset reference {ref} does not exist.")
+        # Check that dataset is known to the datastore.
+        if not self._datastore.knows(ref):
+            raise LookupError(f"Dataset reference {ref} is not known to datastore.")
         return DeferredDatasetHandle(butler=self, ref=ref, parameters=parameters, storageClass=storageClass)
 
     def getDeferred(
@@ -1332,16 +1332,20 @@ def getDeferred(
         Raises
         ------
         LookupError
-            Raised if no matching dataset exists in the `Registry`.
+            Raised if no matching dataset exists in the `Registry` or
+            datastore.
         ValueError
             Raised if a resolved `DatasetRef` was passed as an input, but it
             differs from the one found in the registry.
         TypeError
             Raised if no collections were provided.
         """
-        if isinstance(datasetRefOrType, DatasetRef) and not self._datastore.exists(datasetRefOrType):
-            raise LookupError(f"Dataset reference {datasetRefOrType} does not exist.")
-        ref = self._findDatasetRef(datasetRefOrType, dataId, collections=collections, **kwargs)
+        if isinstance(datasetRefOrType, DatasetRef):
+            if not self._datastore.knows(datasetRefOrType):
+                raise LookupError(f"Dataset reference {datasetRefOrType} does not exist.")
+            ref = datasetRefOrType
+        else:
+            ref = self._findDatasetRef(datasetRefOrType, dataId, collections=collections, **kwargs)
         return DeferredDatasetHandle(butler=self, ref=ref, parameters=parameters, storageClass=storageClass)
 
     def get(

python/lsst/daf/butler/datastores/fileDatastore.py

@@ -1332,8 +1332,9 @@ def _read_artifact_into_memory(
                         formatter.name(),
                     )
                     try:
-                        with formatter._updateLocation(newLocation):
-                            with time_this(
+                        with (
+                            formatter._updateLocation(newLocation),
+                            time_this(
                                 log,
                                 msg="Reading%s from location %s %s with formatter %s",
                                 args=(
@@ -1342,8 +1343,9 @@ def _read_artifact_into_memory(
                                     msg,
                                     formatter.name(),
                                 ),
-                            ):
-                                result = formatter.read(component=getInfo.component if isComponent else None)
+                            ),
+                        ):
+                            result = formatter.read(component=getInfo.component if isComponent else None)
                     except Exception as e:
                         raise ValueError(
                             f"Failure from formatter '{formatter.name()}' for dataset {ref.id}"

tests/test_butler.py

@@ -387,6 +387,10 @@ def runPutGetTest(self, storageClass: StorageClass, datasetTypeName: str) -> But
         # Create DatasetRef for put using default run.
         refIn = DatasetRef(datasetType, dataId, id=uuid.UUID(int=1), run=butler.run)
 
+        # Check that getDeferred fails with standalone ref.
+        with self.assertRaises(LookupError):
+            butler.getDeferred(refIn)
+
         # Put the dataset again, since the last thing we did was remove it
         # and we want to use the default collection.
         ref = butler.put(metric, refIn)