diff --git a/python/lsst/daf/butler/remote_butler/__init__.py b/python/lsst/daf/butler/remote_butler/__init__.py new file mode 100644 index 0000000000..56a73ebef3 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/__init__.py @@ -0,0 +1,28 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from ._remote_butler import * diff --git a/python/lsst/daf/butler/remote_butler/_config.py b/python/lsst/daf/butler/remote_butler/_config.py new file mode 100644 index 0000000000..a825c69045 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/_config.py @@ -0,0 +1,38 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from pydantic import AnyHttpUrl + +from .._compat import _BaseModelCompat + + +class RemoteButlerOptionsModel(_BaseModelCompat): + url: AnyHttpUrl + + +class RemoteButlerConfigModel(_BaseModelCompat): + remote_butler: RemoteButlerOptionsModel diff --git a/python/lsst/daf/butler/remote_butler/_remote_butler.py b/python/lsst/daf/butler/remote_butler/_remote_butler.py new file mode 100644 index 0000000000..a9a0273618 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/_remote_butler.py @@ -0,0 +1,319 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__all__ = ("RemoteButler",) + +from collections.abc import Collection, Iterable, Sequence +from contextlib import AbstractContextManager +from typing import Any, TextIO + +import httpx +from lsst.daf.butler import __version__ +from lsst.resources import ResourcePath, ResourcePathExpression +from lsst.utils.introspection import get_full_type_name + +from .._butler import Butler +from .._butler_config import ButlerConfig +from .._config import Config +from .._dataset_existence import DatasetExistence +from .._dataset_ref import DatasetIdGenEnum, DatasetRef +from .._dataset_type import DatasetType +from .._deferredDatasetHandle import DeferredDatasetHandle +from .._file_dataset import FileDataset +from .._limited_butler import LimitedButler +from .._storage_class import StorageClass +from ..datastore import DatasetRefURIs +from ..dimensions import DataId, DimensionConfig, DimensionUniverse +from ..registry import Registry, RegistryDefaults +from ..transfers import RepoExportContext +from ._config import RemoteButlerConfigModel + + +class RemoteButler(Butler): + def __init__( + self, + # These parameters are inherited from the Butler() constructor + config: Config | ResourcePathExpression | None = None, + *, + collections: Any = None, + run: str | None = None, + searchPaths: Sequence[ResourcePathExpression] | None = None, + writeable: bool | None = None, + inferDefaults: bool = True, + # Parameters unique to RemoteButler + http_client: httpx.Client | None = None, + **kwargs: Any, + ): + butler_config = ButlerConfig(config, searchPaths, without_datastore=True) + self._config = RemoteButlerConfigModel.model_validate(butler_config) + self._dimensions: DimensionUniverse | None = None + # TODO: RegistryDefaults should have finish() called on it, but this + # requires getCollectionSummary() which is not yet implemented + self._registry_defaults = RegistryDefaults(collections, run, inferDefaults, **kwargs) + + if http_client is not None: + # We have injected a client explicitly in to the class. + # This is generally done for testing. + self._client = http_client + else: + headers = {"user-agent": f"{get_full_type_name(self)}/{__version__}"} + self._client = httpx.Client(headers=headers, base_url=str(self._config.remote_butler.url)) + + def isWriteable(self) -> bool: + # Docstring inherited. + return False + + @property + def dimensions(self) -> DimensionUniverse: + # Docstring inherited. + if self._dimensions is not None: + return self._dimensions + + response = self._client.get(self._get_url("universe")) + response.raise_for_status() + + config = DimensionConfig.fromString(response.text, format="json") + self._dimensions = DimensionUniverse(config) + return self._dimensions + + def getDatasetType(self, name: str) -> DatasetType: + # Docstring inherited. + raise NotImplementedError() + + def transaction(self) -> AbstractContextManager[None]: + """Will always raise NotImplementedError. + Transactions are not supported by RemoteButler. + """ + raise NotImplementedError() + + def put( + self, + obj: Any, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + run: str | None = None, + **kwargs: Any, + ) -> DatasetRef: + # Docstring inherited. + raise NotImplementedError() + + def getDeferred( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + parameters: dict | None = None, + collections: Any = None, + storageClass: str | StorageClass | None = None, + **kwargs: Any, + ) -> DeferredDatasetHandle: + # Docstring inherited. + raise NotImplementedError() + + def get( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + parameters: dict[str, Any] | None = None, + collections: Any = None, + storageClass: StorageClass | str | None = None, + **kwargs: Any, + ) -> Any: + # Docstring inherited. + raise NotImplementedError() + + def getURIs( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + predict: bool = False, + collections: Any = None, + run: str | None = None, + **kwargs: Any, + ) -> DatasetRefURIs: + # Docstring inherited. + raise NotImplementedError() + + def getURI( + self, + datasetRefOrType: DatasetRef | DatasetType | str, + /, + dataId: DataId | None = None, + *, + predict: bool = False, + collections: Any = None, + run: str | None = None, + **kwargs: Any, + ) -> ResourcePath: + # Docstring inherited. + raise NotImplementedError() + + def retrieveArtifacts( + self, + refs: Iterable[DatasetRef], + destination: ResourcePathExpression, + transfer: str = "auto", + preserve_path: bool = True, + overwrite: bool = False, + ) -> list[ResourcePath]: + # Docstring inherited. + raise NotImplementedError() + + def exists( + self, + dataset_ref_or_type: DatasetRef | DatasetType | str, + /, + data_id: DataId | None = None, + *, + full_check: bool = True, + collections: Any = None, + **kwargs: Any, + ) -> DatasetExistence: + # Docstring inherited. + raise NotImplementedError() + + def _exists_many( + self, + refs: Iterable[DatasetRef], + /, + *, + full_check: bool = True, + ) -> dict[DatasetRef, DatasetExistence]: + # Docstring inherited. + raise NotImplementedError() + + def removeRuns(self, names: Iterable[str], unstore: bool = True) -> None: + # Docstring inherited. + raise NotImplementedError() + + def ingest( + self, + *datasets: FileDataset, + transfer: str | None = "auto", + run: str | None = None, + idGenerationMode: DatasetIdGenEnum | None = None, + record_validation_info: bool = True, + ) -> None: + # Docstring inherited. + raise NotImplementedError() + + def export( + self, + *, + directory: str | None = None, + filename: str | None = None, + format: str | None = None, + transfer: str | None = None, + ) -> AbstractContextManager[RepoExportContext]: + # Docstring inherited. + raise NotImplementedError() + + def import_( + self, + *, + directory: ResourcePathExpression | None = None, + filename: ResourcePathExpression | TextIO | None = None, + format: str | None = None, + transfer: str | None = None, + skip_dimensions: set | None = None, + ) -> None: + # Docstring inherited. + raise NotImplementedError() + + def transfer_from( + self, + source_butler: LimitedButler, + source_refs: Iterable[DatasetRef], + transfer: str = "auto", + skip_missing: bool = True, + register_dataset_types: bool = False, + transfer_dimensions: bool = False, + ) -> Collection[DatasetRef]: + # Docstring inherited. + raise NotImplementedError() + + def validateConfiguration( + self, + logFailures: bool = False, + datasetTypeNames: Iterable[str] | None = None, + ignore: Iterable[str] | None = None, + ) -> None: + # Docstring inherited. + raise NotImplementedError() + + @property + def collections(self) -> Sequence[str]: + # Docstring inherited. + return self._registry_defaults.collections + + @property + def run(self) -> str | None: + # Docstring inherited. + return self._registry_defaults.run + + @property + def registry(self) -> Registry: + # Docstring inherited. + raise NotImplementedError() + + def pruneDatasets( + self, + refs: Iterable[DatasetRef], + *, + disassociate: bool = True, + unstore: bool = False, + tags: Iterable[str] = (), + purge: bool = False, + ) -> None: + # Docstring inherited. + raise NotImplementedError() + + def _get_url(self, path: str, version: str = "v1") -> str: + """Form the complete path to an endpoint on the server + + Parameters + ---------- + path : `str` + The relative path to the server endpoint. Should not include the + "/butler" prefix. + version : `str`, optional + Version string to prepend to path. Defaults to "v1". + + Returns + ------- + path : `str` + The full path to the endpoint + """ + prefix = "butler" + return f"{prefix}/{version}/{path}" diff --git a/python/lsst/daf/butler/remote_butler/server/__init__.py b/python/lsst/daf/butler/remote_butler/server/__init__.py new file mode 100644 index 0000000000..d63badaf11 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/server/__init__.py @@ -0,0 +1,29 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from ._factory import * +from ._server import * diff --git a/python/lsst/daf/butler/remote_butler/server/_factory.py b/python/lsst/daf/butler/remote_butler/server/_factory.py new file mode 100644 index 0000000000..7d57c9c246 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/server/_factory.py @@ -0,0 +1,38 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from lsst.daf.butler import Butler + +__all__ = ("Factory",) + + +class Factory: + def __init__(self, *, butler: Butler): + self._butler = butler + + def create_butler(self) -> Butler: + return self._butler diff --git a/python/lsst/daf/butler/remote_butler/server/_server.py b/python/lsst/daf/butler/remote_butler/server/_server.py new file mode 100644 index 0000000000..3be9348223 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/server/_server.py @@ -0,0 +1,63 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import annotations + +__all__ = ("app", "factory_dependency") + +import logging +from functools import cache +from typing import Any + +from fastapi import Depends, FastAPI +from fastapi.middleware.gzip import GZipMiddleware +from lsst.daf.butler import Butler + +from ._factory import Factory + +BUTLER_ROOT = "ci_hsc_gen3/DATA" + +log = logging.getLogger(__name__) + +app = FastAPI() +app.add_middleware(GZipMiddleware, minimum_size=1000) + + +@cache +def _make_global_butler() -> Butler: + return Butler.from_config(BUTLER_ROOT) + + +def factory_dependency() -> Factory: + return Factory(butler=_make_global_butler()) + + +@app.get("/butler/v1/universe", response_model=dict[str, Any]) +def get_dimension_universe(factory: Factory = Depends(factory_dependency)) -> dict[str, Any]: + """Allow remote client to get dimensions definition.""" + butler = factory.create_butler() + return butler.dimensions.dimensionConfig.toDict() diff --git a/python/lsst/daf/butler/remote_butler/server/_server_models.py b/python/lsst/daf/butler/remote_butler/server/_server_models.py new file mode 100644 index 0000000000..1c34747e33 --- /dev/null +++ b/python/lsst/daf/butler/remote_butler/server/_server_models.py @@ -0,0 +1,28 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Models used for client/server communication.""" diff --git a/python/lsst/daf/butler/server.py b/python/lsst/daf/butler/server.py deleted file mode 100644 index 1839838954..0000000000 --- a/python/lsst/daf/butler/server.py +++ /dev/null @@ -1,451 +0,0 @@ -# This file is part of daf_butler. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (http://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This software is dual licensed under the GNU General Public License and also -# under a 3-clause BSD license. Recipients may choose which of these licenses -# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, -# respectively. If you choose the GPL option then the following text applies -# (but note that there is still no warranty even if you opt for BSD instead): -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from __future__ import annotations - -__all__ = () - -import logging -from collections.abc import Mapping -from enum import Enum, auto -from typing import Any - -from fastapi import Depends, FastAPI, HTTPException, Query -from fastapi.middleware.gzip import GZipMiddleware -from lsst.daf.butler import ( - Butler, - Config, - DataCoordinate, - DatasetId, - DatasetRef, - SerializedDataCoordinate, - SerializedDatasetRef, - SerializedDatasetType, - SerializedDimensionRecord, -) -from lsst.daf.butler.registry import CollectionType -from lsst.daf.butler.server_models import ( - ExpressionQueryParameter, - QueryDataIdsModel, - QueryDatasetsModel, - QueryDimensionRecordsModel, -) - -BUTLER_ROOT = "ci_hsc_gen3/DATA" - -log = logging.getLogger("excalibur") - - -class CollectionTypeNames(str, Enum): - """Collection type names supported by the interface.""" - - def _generate_next_value_(name, start, count, last_values) -> str: # type: ignore # noqa: N805 - # Use the name directly as the value - return name - - RUN = auto() - CALIBRATION = auto() - CHAINED = auto() - TAGGED = auto() - - -app = FastAPI() -app.add_middleware(GZipMiddleware, minimum_size=1000) - - -GLOBAL_READWRITE_BUTLER: Butler | None = None -GLOBAL_READONLY_BUTLER: Butler | None = None - - -def _make_global_butler() -> None: - global GLOBAL_READONLY_BUTLER, GLOBAL_READWRITE_BUTLER - if GLOBAL_READONLY_BUTLER is None: - GLOBAL_READONLY_BUTLER = Butler.from_config(BUTLER_ROOT, writeable=False) - if GLOBAL_READWRITE_BUTLER is None: - GLOBAL_READWRITE_BUTLER = Butler.from_config(BUTLER_ROOT, writeable=True) - - -def butler_readonly_dependency() -> Butler: - """Return global read-only butler.""" - _make_global_butler() - return Butler.from_config(butler=GLOBAL_READONLY_BUTLER) - - -def butler_readwrite_dependency() -> Butler: - """Return read-write butler.""" - _make_global_butler() - return Butler.from_config(butler=GLOBAL_READWRITE_BUTLER) - - -def unpack_dataId(butler: Butler, data_id: SerializedDataCoordinate | None) -> DataCoordinate | None: - """Convert the serialized dataId back to full DataCoordinate. - - Parameters - ---------- - butler : `lsst.daf.butler.Butler` - The butler to use for registry and universe. - data_id : `SerializedDataCoordinate` or `None` - The serialized form. - - Returns - ------- - dataId : `DataCoordinate` or `None` - The DataId usable by registry. - """ - if data_id is None: - return None - return DataCoordinate.from_simple(data_id, registry=butler.registry) - - -@app.get("/butler/") -def read_root() -> str: - """Return message when accessing the root URL.""" - return "Welcome to Excalibur... aka your Butler Server" - - -@app.get("/butler/butler.json", response_model=dict[str, Any]) -def read_server_config() -> Mapping: - """Return the butler configuration that the client should use.""" - config_str = f""" -datastore: - root: {BUTLER_ROOT} -registry: - cls: lsst.daf.butler.registries.remote.RemoteRegistry - db: -""" - config = Config.fromString(config_str, format="yaml") - return config.toDict() - - -@app.get("/butler/v1/universe", response_model=dict[str, Any]) -def get_dimension_universe(butler: Butler = Depends(butler_readonly_dependency)) -> dict[str, Any]: - """Allow remote client to get dimensions definition.""" - return butler.dimensions.dimensionConfig.toDict() - - -@app.get("/butler/v1/uri/{id}", response_model=str) -def get_uri(id: DatasetId, butler: Butler = Depends(butler_readonly_dependency)) -> str: - """Return a single URI of non-disassembled dataset.""" - ref = butler.registry.getDataset(id) - if not ref: - raise HTTPException(status_code=404, detail=f"Dataset with id {id} does not exist.") - - uri = butler.getURI(ref) - - # In reality would have to convert this to a signed URL - return str(uri) - - -@app.put("/butler/v1/registry/refresh") -def refresh(butler: Butler = Depends(butler_readonly_dependency)) -> None: - """Refresh the registry cache.""" - # Unclear whether this should exist. Which butler is really being - # refreshed? How do we know the server we are refreshing is used later? - # For testing at the moment it is important if a test adds a dataset type - # directly in the server since the test client will not see it. - butler.registry.refresh() - - -@app.get( - "/butler/v1/registry/datasetType/{datasetTypeName}", - summary="Retrieve this dataset type definition.", - response_model=SerializedDatasetType, - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def get_dataset_type( - datasetTypeName: str, butler: Butler = Depends(butler_readonly_dependency) -) -> SerializedDatasetType: - """Return the dataset type.""" - datasetType = butler.registry.getDatasetType(datasetTypeName) - return datasetType.to_simple() - - -@app.get( - "/butler/v1/registry/datasetTypes", - summary="Retrieve all dataset type definitions.", - response_model=list[SerializedDatasetType], - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def query_all_dataset_types( - components: bool | None = Query(None), butler: Butler = Depends(butler_readonly_dependency) -) -> list[SerializedDatasetType]: - """Return all dataset types.""" - datasetTypes = butler.registry.queryDatasetTypes(..., components=components) - return [d.to_simple() for d in datasetTypes] - - -@app.get( - "/butler/v1/registry/datasetTypes/re", - summary="Retrieve dataset type definitions matching expressions", - response_model=list[SerializedDatasetType], - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def query_dataset_types_re( - regex: list[str] | None = Query(None), - glob: list[str] | None = Query(None), - components: bool | None = Query(None), - butler: Butler = Depends(butler_readonly_dependency), -) -> list[SerializedDatasetType]: - """Return all dataset types matching a regular expression.""" - expression_params = ExpressionQueryParameter(regex=regex, glob=glob) - - datasetTypes = butler.registry.queryDatasetTypes(expression_params.expression(), components=components) - return [d.to_simple() for d in datasetTypes] - - -@app.get("/butler/v1/registry/collection/chain/{parent:path}", response_model=list[str]) -def get_collection_chain(parent: str, butler: Butler = Depends(butler_readonly_dependency)) -> list[str]: - """Return the collection chain members.""" - chain = butler.registry.getCollectionChain(parent) - return list(chain) - - -@app.get("/butler/v1/registry/collections", response_model=list[str]) -def query_collections( - regex: list[str] | None = Query(None), - glob: list[str] | None = Query(None), - datasetType: str | None = Query(None), - flattenChains: bool = Query(False), - collectionType: list[CollectionTypeNames] | None = Query(None), - includeChains: bool | None = Query(None), - butler: Butler = Depends(butler_readonly_dependency), -) -> list[str]: - """Return collections matching query.""" - expression_params = ExpressionQueryParameter(regex=regex, glob=glob) - collectionTypes = CollectionType.from_names(collectionType) - dataset_type = butler.registry.getDatasetType(datasetType) if datasetType else None - - collections = butler.registry.queryCollections( - expression=expression_params.expression(), - datasetType=dataset_type, - collectionTypes=collectionTypes, - flattenChains=flattenChains, - includeChains=includeChains, - ) - return list(collections) - - -@app.get("/butler/v1/registry/collection/type/{name:path}", response_model=str) -def get_collection_type(name: str, butler: Butler = Depends(butler_readonly_dependency)) -> str: - """Return type for named collection.""" - collectionType = butler.registry.getCollectionType(name) - return collectionType.name - - -@app.put("/butler/v1/registry/collection/{name:path}/{type_}", response_model=str) -def register_collection( - name: str, - collectionTypeName: CollectionTypeNames, - doc: str | None = Query(None), - butler: Butler = Depends(butler_readwrite_dependency), -) -> str: - """Register a collection.""" - collectionType = CollectionType.from_name(collectionTypeName) - butler.registry.registerCollection(name, collectionType, doc) - - # Need to refresh the global read only butler otherwise other clients - # may not see this change. - if GLOBAL_READONLY_BUTLER is not None: # for mypy - GLOBAL_READONLY_BUTLER.registry.refresh() - - return name - - -@app.get( - "/butler/v1/registry/dataset/{id}", - summary="Retrieve this dataset definition.", - response_model=SerializedDatasetRef | None, - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def get_dataset( - id: DatasetId, butler: Butler = Depends(butler_readonly_dependency) -) -> SerializedDatasetRef | None: - """Return a single dataset reference.""" - ref = butler.registry.getDataset(id) - if ref is not None: - return ref.to_simple() - # This could raise a 404 since id is not found. The standard regsitry - # getDataset method returns without error so follow that example here. - return ref - - -@app.get("/butler/v1/registry/datasetLocations/{id}", response_model=list[str]) -def get_dataset_locations(id: DatasetId, butler: Butler = Depends(butler_readonly_dependency)) -> list[str]: - """Return locations of datasets.""" - # Takes an ID so need to convert to a real DatasetRef - fake_ref = SerializedDatasetRef(id=id) - - try: - # Converting this to a real DatasetRef takes time and is not - # needed internally since only the ID is used. - ref = DatasetRef.from_simple(fake_ref, registry=butler.registry) - except Exception: - # SQL getDatasetLocations looks at ID in datastore and does not - # check it is in registry. Follow that example and return without - # error. - return [] - - return list(butler.registry.getDatasetLocations(ref)) - - -# TimeSpan not yet a pydantic model -@app.post( - "/butler/v1/registry/findDataset/{datasetType}", - summary="Retrieve this dataset definition from collection, dataset type, and dataId", - response_model=SerializedDatasetRef, - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def find_dataset( - datasetType: str, - dataId: SerializedDataCoordinate | None = None, - collections: list[str] | None = Query(None), - butler: Butler = Depends(butler_readonly_dependency), -) -> SerializedDatasetRef | None: - """Return a single dataset reference matching query.""" - collection_query = collections if collections else None - - ref = butler.registry.findDataset( - datasetType, dataId=unpack_dataId(butler, dataId), collections=collection_query - ) - return ref.to_simple() if ref else None - - -# POST is used for the complex dict data structures -@app.post( - "/butler/v1/registry/datasets", - summary="Query all dataset holdings.", - response_model=list[SerializedDatasetRef], - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def query_datasets( - query: QueryDatasetsModel, butler: Butler = Depends(butler_readonly_dependency) -) -> list[SerializedDatasetRef]: - """Return datasets matching query.""" - # This method might return a lot of results - - if query.collections: - collections = query.collections.expression() - else: - collections = None - - datasets = butler.registry.queryDatasets( - query.datasetType.expression(), - collections=collections, - dimensions=query.dimensions, - dataId=unpack_dataId(butler, query.dataId), - where=query.where, - findFirst=query.findFirst, - components=query.components, - bind=query.bind, - check=query.check, - **query.kwargs(), - ) - return [ref.to_simple() for ref in datasets] - - -# POST is used for the complex dict data structures -@app.post( - "/butler/v1/registry/dataIds", - summary="Query all data IDs.", - response_model=list[SerializedDataCoordinate], - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def query_data_ids( - query: QueryDataIdsModel, butler: Butler = Depends(butler_readonly_dependency) -) -> list[SerializedDataCoordinate]: - """Return data IDs matching query.""" - if query.datasets: - datasets = query.datasets.expression() - else: - datasets = None - if query.collections: - collections = query.collections.expression() - else: - collections = None - - dataIds = butler.registry.queryDataIds( - query.dimensions, - collections=collections, - datasets=datasets, - dataId=unpack_dataId(butler, query.dataId), - where=query.where, - components=query.components, - bind=query.bind, - check=query.check, - **query.kwargs(), - ) - return [coord.to_simple() for coord in dataIds] - - -# Uses POST to handle the DataId -@app.post( - "/butler/v1/registry/dimensionRecords/{element}", - summary="Retrieve dimension records matching query", - response_model=list[SerializedDimensionRecord], - response_model_exclude_unset=True, - response_model_exclude_defaults=True, - response_model_exclude_none=True, -) -def query_dimension_records( - element: str, query: QueryDimensionRecordsModel, butler: Butler = Depends(butler_readonly_dependency) -) -> list[SerializedDimensionRecord]: - """Return dimension records matching query.""" - if query.datasets: - datasets = query.datasets.expression() - else: - datasets = None - if query.collections: - collections = query.collections.expression() - else: - collections = None - - records = butler.registry.queryDimensionRecords( - element, - dataId=unpack_dataId(butler, query.dataId), - collections=collections, - where=query.where, - datasets=datasets, - components=query.components, - bind=query.bind, - check=query.check, - **query.kwargs(), - ) - return [r.to_simple() for r in records] diff --git a/python/lsst/daf/butler/server_models.py b/python/lsst/daf/butler/server_models.py deleted file mode 100644 index 4cb4c5e929..0000000000 --- a/python/lsst/daf/butler/server_models.py +++ /dev/null @@ -1,312 +0,0 @@ -# This file is part of daf_butler. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (http://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This software is dual licensed under the GNU General Public License and also -# under a 3-clause BSD license. Recipients may choose which of these licenses -# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, -# respectively. If you choose the GPL option then the following text applies -# (but note that there is still no warranty even if you opt for BSD instead): -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""Models used for client/server communication.""" - -__all__ = ( - "QueryDatasetsModel", - "QueryDataIdsModel", - "QueryDimensionRecordsModel", - "ExpressionQueryParameter", - "DatasetsQueryParameter", -) - -import re -from collections.abc import Mapping -from typing import Any, ClassVar - -import pydantic -from lsst.utils.iteration import ensure_iterable -from pydantic import Field - -from ._compat import PYDANTIC_V2, _BaseModelCompat -from .dimensions import DataIdValue, SerializedDataCoordinate -from .utils import globToRegex - -# Simple scalar python types. -ScalarType = int | bool | float | str - -# Bind parameters can have any scalar type. -BindType = dict[str, ScalarType] - -# For serialization purposes a data ID key must be a str. -SimpleDataId = Mapping[str, DataIdValue] - - -# While supporting pydantic v1 and v2 keep this outside the model. -_expression_query_schema_extra = { - "examples": [ - { - "regex": ["^cal.*"], - "glob": ["cal*", "raw"], - } - ] -} - - -class ExpressionQueryParameter(_BaseModelCompat): - """Represents a specification for an expression query. - - Generally used for collection or dataset type expressions. This - implementation returns ``...`` by default. - """ - - _allow_ellipsis: ClassVar[bool] = True - """Control whether expression can match everything.""" - - regex: list[str] | None = Field( - None, - title="List of regular expression strings.", - examples=["^cal.*"], - ) - - glob: list[str] | None = Field( - None, - title="List of globs or explicit strings to use in expression.", - examples=["cal*"], - ) - - if PYDANTIC_V2: - model_config = { - "json_schema_extra": _expression_query_schema_extra, # type: ignore[typeddict-item] - } - else: - - class Config: - """Local configuration overrides for model.""" - - schema_extra = _expression_query_schema_extra - - def expression(self) -> Any: - """Combine regex and glob lists into single expression.""" - if self.glob is None and self.regex is None: - if self._allow_ellipsis: - return ... - # Rather than matching all, interpret this as no expression - # at all. - return None - - expression: list[str | re.Pattern] = [] - if self.regex is not None: - for r in self.regex: - expression.append(re.compile(r)) - if self.glob is not None: - regexes = globToRegex(self.glob) - if isinstance(regexes, list): - expression.extend(regexes) - else: - if self._allow_ellipsis: - return ... - raise ValueError("Expression matches everything but that is not allowed.") - return expression - - @classmethod - def from_expression(cls, expression: Any) -> "ExpressionQueryParameter": - """Convert a standard dataset type expression to wire form.""" - if expression is ...: - return cls() - - expressions = ensure_iterable(expression) - params: dict[str, list[str]] = {"glob": [], "regex": []} - for expression in expressions: - if expression is ...: - # This matches everything - return cls() - - if isinstance(expression, re.Pattern): - params["regex"].append(expression.pattern) - elif isinstance(expression, str): - params["glob"].append(expression) - elif hasattr(expression, "name"): - params["glob"].append(expression.name) - else: - raise ValueError(f"Unrecognized type given to expression: {expression!r}") - - # Clean out empty dicts. - for k in list(params): - if not params[k]: - del params[k] - - return cls(**params) - - -class DatasetsQueryParameter(ExpressionQueryParameter): - """Represents a specification for a dataset expression query. - - This differs from the standard expression query in that an empty - expression will return `None` rather than ``...``. - """ - - _allow_ellipsis: ClassVar[bool] = False - - -# Shared field definitions -Where = Field( - "", - title="String expression similar to a SQL WHERE clause.", - examples=["detector = 5 AND instrument = 'HSC'"], -) -Collections = Field( - None, - title="An expression that identifies the collections to search.", -) -Datasets = Field( - None, - title="An expression that identifies dataset types to search (must not match all datasets).", -) -OptionalDimensions = Field( - None, - title="Relevant dimensions to include.", - examples=["detector", "physical_filter"], -) -Dimensions = Field( - ..., - title="Relevant dimensions to include.", - examples=["detector", "physical_filter"], -) -DataId = Field( - None, - title="Data ID to constrain the query.", -) -FindFirst = Field( - False, - title="Control whether only first matching dataset ref or type is returned.", -) -Components = Field( - None, - title="Control how expressions apply to components.", -) -Bind = Field( - None, - title="Mapping to use to inject values into the WHERE parameter clause.", -) -Check = Field( - True, - title="Control whether to check the query for consistency.", -) -KeywordArgs = Field( - None, - title="Additional parameters to use when standardizing the supplied data ID.", -) - - -class QueryBaseModel(_BaseModelCompat): - """Base model for all query models.""" - - if PYDANTIC_V2: - - @pydantic.field_validator("keyword_args", check_fields=False) # type: ignore[attr-defined] - @classmethod - def _check_keyword_args(cls, v: SimpleDataId) -> SimpleDataId | None: - """Convert kwargs into None if empty. - - This retains the property at its default value and can therefore - remove it from serialization. - - The validator will be ignored if the subclass does not have this - property in its model. - """ - if not v: - return None - return v - - else: - - @pydantic.validator("keyword_args", check_fields=False) - def _check_keyword_args(cls, v, values) -> SimpleDataId | None: # type: ignore # noqa: N805 - """Convert kwargs into None if empty. - - This retains the property at its default value and can therefore - remove it from serialization. - - The validator will be ignored if the subclass does not have this - property in its model. - """ - if not v: - return None - return v - - def kwargs(self) -> SimpleDataId: - """Return keyword args, converting None to a `dict`. - - Returns - ------- - **kwargs - The keword arguments stored in the model. `None` is converted - to an empty dict. Returns empty dict if the ``keyword_args`` - property is not defined. - """ - try: - # mypy does not know about the except - kwargs = self.keyword_args # type: ignore - except AttributeError: - kwargs = {} - if kwargs is None: - return {} - return kwargs - - -class QueryDatasetsModel(QueryBaseModel): - """Information needed for a registry dataset query.""" - - datasetType: ExpressionQueryParameter = Field(..., title="Dataset types to query. Can match all.") - collections: ExpressionQueryParameter | None = Collections - dimensions: list[str] | None = OptionalDimensions - dataId: SerializedDataCoordinate | None = DataId - where: str = Where - findFirst: bool = FindFirst - components: bool | None = Components - bind: BindType | None = Bind - check: bool = Check - keyword_args: SimpleDataId | None = KeywordArgs # mypy refuses to allow kwargs in model - - -class QueryDataIdsModel(QueryBaseModel): - """Information needed to query data IDs.""" - - dimensions: list[str] = Dimensions - dataId: SerializedDataCoordinate | None = DataId - datasets: DatasetsQueryParameter | None = Datasets - collections: ExpressionQueryParameter | None = Collections - where: str = Where - components: bool | None = Components - bind: BindType | None = Bind - check: bool = Check - keyword_args: SimpleDataId | None = KeywordArgs # mypy refuses to allow kwargs in model - - -class QueryDimensionRecordsModel(QueryBaseModel): - """Information needed to query the dimension records.""" - - dataId: SerializedDataCoordinate | None = DataId - datasets: DatasetsQueryParameter | None = Datasets - collections: ExpressionQueryParameter | None = Collections - where: str = Where - components: bool | None = Components - bind: SimpleDataId | None = Bind - check: bool = Check - keyword_args: SimpleDataId | None = KeywordArgs # mypy refuses to allow kwargs in model diff --git a/tests/test_remote_butler.py b/tests/test_remote_butler.py new file mode 100644 index 0000000000..3a671311b9 --- /dev/null +++ b/tests/test_remote_butler.py @@ -0,0 +1,64 @@ +# This file is part of daf_butler. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This software is dual licensed under the GNU General Public License and also +# under a 3-clause BSD license. Recipients may choose which of these licenses +# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, +# respectively. If you choose the GPL option then the following text applies +# (but note that there is still no warranty even if you opt for BSD instead): +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest + +from lsst.daf.butler import Butler +from pydantic import ValidationError + +try: + from lsst.daf.butler.remote_butler import RemoteButler +except ImportError: + # httpx is not available in rubin-env yet, so skip these tests if it's not + # available + RemoteButler = None + + +@unittest.skipIf(RemoteButler is None, "httpx is not installed") +class RemoteButlerConfigTests(unittest.TestCase): + """Test construction of RemoteButler via Butler()""" + + def test_instantiate_via_butler(self): + butler = Butler( + { + "cls": "lsst.daf.butler.remote_butler.RemoteButler", + "remote_butler": {"url": "https://validurl.example"}, + }, + collections=["collection1", "collection2"], + run="collection2", + ) + assert isinstance(butler, RemoteButler) + self.assertEqual(butler.collections, ("collection1", "collection2")) + self.assertEqual(butler.run, "collection2") + + def test_bad_config(self): + with self.assertRaises(ValidationError): + Butler({"cls": "lsst.daf.butler.remote_butler.RemoteButler", "remote_butler": {"url": "!"}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_server.py b/tests/test_server.py index a52ddf1b66..401e0126dd 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -30,21 +30,32 @@ try: # Failing to import any of these should disable the tests. - import lsst.daf.butler.server from fastapi.testclient import TestClient - from lsst.daf.butler.server import app + from lsst.daf.butler.remote_butler import RemoteButler + from lsst.daf.butler.remote_butler.server import Factory, app, factory_dependency except ImportError: TestClient = None app = None -from lsst.daf.butler import Butler, CollectionType, Config, DataCoordinate, DatasetRef -from lsst.daf.butler.tests import addDatasetType +from lsst.daf.butler import Butler from lsst.daf.butler.tests.utils import MetricTestRepo, makeTestTempDir, removeTestTempDir TESTDIR = os.path.abspath(os.path.dirname(__file__)) -@unittest.skip("Test does not work after RemoteRegistry removal, to be fixed later.") +def _make_remote_butler(http_client): + return RemoteButler( + config={ + "remote_butler": { + # This URL is ignored because we override the HTTP client, but + # must be valid to satisfy the config validation + "url": "https://test.example" + } + }, + http_client=http_client, + ) + + @unittest.skipIf(TestClient is None or app is None, "FastAPI not installed.") class ButlerClientServerTestCase(unittest.TestCase): """Test for Butler client/server.""" @@ -54,112 +65,32 @@ def setUpClass(cls): # First create a butler and populate it. cls.root = makeTestTempDir(TESTDIR) cls.repo = MetricTestRepo(root=cls.root, configFile=os.path.join(TESTDIR, "config/basic/butler.yaml")) + # Override the server's Butler initialization to point at our test repo + server_butler = Butler.from_config(cls.root) - # Add a collection chain. - cls.repo.butler.registry.registerCollection("chain", CollectionType.CHAINED) - cls.repo.butler.registry.setCollectionChain("chain", ["ingest"]) - - # Globally change where the server thinks its butler repository - # is located. This will prevent any other server tests and is - # not a long term fix. - lsst.daf.butler.server.BUTLER_ROOT = cls.root - cls.client = TestClient(app) + def create_factory_dependency(): + return Factory(butler=server_butler) - # Create a client butler. We need to modify the contents of the - # server configuration to reflect the use of the test client. - response = cls.client.get("/butler/butler.json") - config = Config(response.json()) - config["registry", "db"] = cls.client + app.dependency_overrides[factory_dependency] = create_factory_dependency - # Since there is no client datastore we also need to specify - # the datastore root. - config["datastore", "root"] = cls.root - cls.butler = Butler(config) + # Set up the RemoteButler that will connect to the server + cls.client = TestClient(app) + cls.butler = _make_remote_butler(cls.client) @classmethod def tearDownClass(cls): + del app.dependency_overrides[factory_dependency] removeTestTempDir(cls.root) def test_simple(self): - response = self.client.get("/butler/") - self.assertEqual(response.status_code, 200) - self.assertIn("Butler Server", response.json()) - - response = self.client.get("/butler/butler.json") - self.assertEqual(response.status_code, 200) - self.assertIn("registry", response.json()) - response = self.client.get("/butler/v1/universe") self.assertEqual(response.status_code, 200) self.assertIn("namespace", response.json()) - def test_registry(self): + def test_remote_butler(self): universe = self.butler.dimensions self.assertEqual(universe.namespace, "daf_butler") - dataset_type = self.butler.registry.getDatasetType("test_metric_comp") - self.assertEqual(dataset_type.name, "test_metric_comp") - - dataset_types = list(self.butler.registry.queryDatasetTypes(...)) - self.assertIn("test_metric_comp", [ds.name for ds in dataset_types]) - dataset_types = list(self.butler.registry.queryDatasetTypes("test_*")) - self.assertEqual(len(dataset_types), 1) - - collections = self.butler.registry.queryCollections( - ..., collectionTypes={CollectionType.RUN, CollectionType.TAGGED} - ) - self.assertEqual(len(collections), 2, collections) - - collection_type = self.butler.registry.getCollectionType("ingest") - self.assertEqual(collection_type.name, "TAGGED") - - chain = self.butler.registry.getCollectionChain("chain") - self.assertEqual(list(chain), ["ingest"]) - - datasets = list(self.butler.registry.queryDatasets("test_metric_comp", collections=...)) - self.assertEqual(len(datasets), 2) - - ref = self.butler.registry.getDataset(datasets[0].id) - self.assertEqual(ref, datasets[0]) - - locations = self.butler.registry.getDatasetLocations(ref) - self.assertEqual(locations[0], "FileDatastore@") - - fake_ref = DatasetRef( - dataset_type, - dataId={"instrument": "DummyCamComp", "physical_filter": "d-r", "visit": 424}, - run="missing", - ) - locations = self.butler.registry.getDatasetLocations(fake_ref) - self.assertEqual(locations, []) - - dataIds = list(self.butler.registry.queryDataIds("visit", dataId={"instrument": "DummyCamComp"})) - self.assertEqual(len(dataIds), 2) - - # Create a DataCoordinate to test the alternate path for specifying - # a data ID. - data_id = DataCoordinate.standardize({"instrument": "DummyCamComp"}, universe=self.butler.dimensions) - records = list(self.butler.registry.queryDimensionRecords("physical_filter", dataId=data_id)) - self.assertEqual(len(records), 1) - - def test_experimental(self): - """Experimental interfaces.""" - # Got URI testing we can not yet support disassembly so must - # add a dataset with a different dataset type. - datasetType = addDatasetType( - self.repo.butler, "metric", {"instrument", "visit"}, "StructuredCompositeReadCompNoDisassembly" - ) - - self.repo.addDataset({"instrument": "DummyCamComp", "visit": 424}, datasetType=datasetType) - self.butler.registry.refresh() - - # Need a DatasetRef. - datasets = list(self.butler.registry.queryDatasets("metric", collections=...)) - - response = self.client.get(f"/butler/v1/uri/{datasets[0].id}") - self.assertEqual(response.status_code, 200) - self.assertIn("file://", response.json()) - if __name__ == "__main__": unittest.main()