From e14eae14ae273768d84a4150d267d50abf7bfc86 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Fri, 6 Sep 2024 10:10:52 -0700 Subject: [PATCH] Only use ephemeral URIs when we need them in get_many_uris --- .../lsst/daf/butler/datastores/chainedDatastore.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/lsst/daf/butler/datastores/chainedDatastore.py b/python/lsst/daf/butler/datastores/chainedDatastore.py index 406b7ddc05..09432a2c21 100644 --- a/python/lsst/daf/butler/datastores/chainedDatastore.py +++ b/python/lsst/daf/butler/datastores/chainedDatastore.py @@ -651,6 +651,8 @@ def getManyURIs( # Docstring inherited uris: dict[DatasetRef, DatasetRefURIs] = {} + ephemeral_uris: dict[DatasetRef, DatasetRefURIs] = {} + missing_refs = set(refs) # If predict is True we don't want to predict a dataset in the first @@ -666,11 +668,23 @@ def getManyURIs( except NotImplementedError: # some datastores may not implement generating URIs continue + if datastore.isEphemeral: + # Only use these as last resort so do not constrain + # subsequent queries. + ephemeral_uris.update(got_uris) + continue + missing_refs -= got_uris.keys() uris.update(got_uris) if not missing_refs: break + if missing_refs and ephemeral_uris: + ephemeral_refs = missing_refs.intersection(ephemeral_uris.keys()) + for ref in ephemeral_refs: + uris[ref] = ephemeral_uris[ref] + missing_refs.remove(ref) + if missing_refs and not allow_missing: raise FileNotFoundError(f"Dataset(s) {missing_refs} not in this datastore.")