diff --git a/python/lsst/daf/butler/_formatter.py b/python/lsst/daf/butler/_formatter.py index 3bb3736c2e..378bad0f50 100644 --- a/python/lsst/daf/butler/_formatter.py +++ b/python/lsst/daf/butler/_formatter.py @@ -349,7 +349,7 @@ def read( component: str | None = None, expected_size: int = -1, cache_manager: AbstractDatastoreCacheManager | None = None, - ) -> Any: + ) -> object: """Read a Dataset. Parameters @@ -386,7 +386,8 @@ def read( * `read_from_uri` (but with a local file) It is possible for `read_from_uri` to be skipped if the implementation - raises `FormatterNotImplementedError`. If `read_from_stream` is + raises `FormatterNotImplementedError`, for example if the + implementation would prefer to read. If `read_from_stream` is called `read_from_local_file` will never be called. If `read_from_uri` was skipped and `read_from_local_file` is not implemented, it will be called with a local file as a last resort. @@ -486,12 +487,12 @@ def read( def _read_from_possibly_cached_location_no_cache_write( self, - callback: Callable[[ResourcePath, str | None, int], Any], + callback: Callable[[ResourcePath, str | None, int], object], component: str | None = None, expected_size: int = -1, *, cache_manager: AbstractDatastoreCacheManager | None = None, - ) -> Any: + ) -> object: """Read from the cache and call payload without writing to cache.""" cache_manager = self._ensure_cache(cache_manager) @@ -533,7 +534,7 @@ def read_from_possibly_cached_stream( expected_size: int = -1, *, cache_manager: AbstractDatastoreCacheManager | None = None, - ) -> Any: + ) -> object: """Read from a stream, checking for possible presence in local cache. Parameters @@ -562,7 +563,7 @@ def read_from_possibly_cached_stream( a file to the local cache. """ - def _open_stream(uri: ResourcePath, comp: str | None, size: int = -1) -> Any: + def _open_stream(uri: ResourcePath, comp: str | None, size: int = -1) -> object: with uri.open("rb") as fd: return self.read_from_stream(fd, comp, expected_size=size) @@ -576,7 +577,7 @@ def read_directly_from_possibly_cached_uri( expected_size: int = -1, *, cache_manager: AbstractDatastoreCacheManager | None = None, - ) -> Any: + ) -> object: """Read from arbitrary URI, checking for possible presence in local cache. @@ -608,7 +609,7 @@ def read_directly_from_possibly_cached_uri( The URI will be read by calling `read_from_uri`. """ - def _open_uri(uri: ResourcePath, comp: str | None, size: int = -1) -> Any: + def _open_uri(uri: ResourcePath, comp: str | None, size: int = -1) -> object: return self.read_from_uri(uri, comp, expected_size=size) return self._read_from_possibly_cached_location_no_cache_write( @@ -621,7 +622,7 @@ def read_from_possibly_cached_local_file( expected_size: int = -1, *, cache_manager: AbstractDatastoreCacheManager | None = None, - ) -> Any: + ) -> object: """Read a dataset ensuring that a local file is used, checking the cache for it. @@ -730,7 +731,9 @@ def read_from_possibly_cached_local_file( return result - def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any: + def read_from_uri( + self, uri: ResourcePath, component: str | None = None, expected_size: int = -1 + ) -> object: """Read a dataset from a URI that can be local or remote. Parameters @@ -775,7 +778,7 @@ def read_from_uri(self, uri: ResourcePath, component: str | None = None, expecte def read_from_stream( self, stream: BinaryIO | ResourceHandleProtocol, component: str | None = None, expected_size: int = -1 - ) -> Any: + ) -> object: """Read from an open file descriptor. Parameters @@ -802,7 +805,7 @@ def read_from_stream( def read_from_local_file( self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1 - ) -> Any: + ) -> object: """Read a dataset from a URI guaranteed to refer to the local file system. @@ -1328,7 +1331,7 @@ def name(cls) -> str: return get_full_type_name(cls) @abstractmethod - def read(self, component: str | None = None) -> Any: + def read(self, component: str | None = None) -> object: """Read a Dataset. Parameters @@ -2020,7 +2023,7 @@ def validate_write_recipes( # type: ignore def read_from_local_file( self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1 - ) -> Any: + ) -> object: # Need to temporarily override the location since the V1 formatter # will not know anything about this local file. diff --git a/python/lsst/daf/butler/formatters/astropyTable.py b/python/lsst/daf/butler/formatters/astropyTable.py index a2b4f98a1f..cc7599d0b9 100644 --- a/python/lsst/daf/butler/formatters/astropyTable.py +++ b/python/lsst/daf/butler/formatters/astropyTable.py @@ -54,7 +54,7 @@ def get_write_extension(self) -> str: def read_from_local_file( self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1 - ) -> Any: + ) -> object: pytype = self.file_descriptor.storageClass.pytype if not issubclass(pytype, astropy.table.Table): raise TypeError(f"Python type {pytype} does not seem to be a astropy Table type") diff --git a/python/lsst/daf/butler/formatters/json.py b/python/lsst/daf/butler/formatters/json.py index 0e4a54a775..fe76887a9a 100644 --- a/python/lsst/daf/butler/formatters/json.py +++ b/python/lsst/daf/butler/formatters/json.py @@ -46,7 +46,9 @@ class JsonFormatter(TypelessFormatter): unsupported_parameters = None can_read_from_uri = True - def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any: + def read_from_uri( + self, uri: ResourcePath, component: str | None = None, expected_size: int = -1 + ) -> object: # json.load() reads the entire file content into memory # and is no different from json.loads(uri.read()). It does not attempt # to support incremental reading to minimize memory usage. diff --git a/python/lsst/daf/butler/formatters/logs.py b/python/lsst/daf/butler/formatters/logs.py index 54cce9b7f7..748a6ac84c 100644 --- a/python/lsst/daf/butler/formatters/logs.py +++ b/python/lsst/daf/butler/formatters/logs.py @@ -63,7 +63,7 @@ def _get_read_pytype(self) -> type[ButlerLogRecords]: def read_from_local_file( self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1 - ) -> Any: + ) -> object: # ResourcePath open() cannot do a per-line read so can not use # `read_from_stream` and `read_from_uri` does not give any advantage # over pre-downloading the whole file (which can be very large). diff --git a/python/lsst/daf/butler/formatters/packages.py b/python/lsst/daf/butler/formatters/packages.py index 22f96e76cd..a7507b2b7d 100644 --- a/python/lsst/daf/butler/formatters/packages.py +++ b/python/lsst/daf/butler/formatters/packages.py @@ -55,7 +55,9 @@ def get_write_extension(self) -> str: raise RuntimeError(f"Requested file format '{format}' is not supported for Packages") return ext - def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any: + def read_from_uri( + self, uri: ResourcePath, component: str | None = None, expected_size: int = -1 + ) -> object: # Read the full file using the class associated with the # storage class it was originally written with. # Read the bytes directly from resource. These are not going to be diff --git a/python/lsst/daf/butler/formatters/parquet.py b/python/lsst/daf/butler/formatters/parquet.py index ddefa14fa6..69dc5c987b 100644 --- a/python/lsst/daf/butler/formatters/parquet.py +++ b/python/lsst/daf/butler/formatters/parquet.py @@ -79,7 +79,7 @@ class ParquetFormatter(FormatterV2): def read_from_local_file( self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1 - ) -> Any: + ) -> object: # Docstring inherited from Formatter.read. schema = pq.read_schema(local_uri.ospath) diff --git a/python/lsst/daf/butler/formatters/pickle.py b/python/lsst/daf/butler/formatters/pickle.py index 1c66ec1851..92a151c3e1 100644 --- a/python/lsst/daf/butler/formatters/pickle.py +++ b/python/lsst/daf/butler/formatters/pickle.py @@ -48,7 +48,9 @@ class PickleFormatter(TypelessFormatter): unsupported_parameters = None can_read_from_uri = True - def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any: + def read_from_uri( + self, uri: ResourcePath, component: str | None = None, expected_size: int = -1 + ) -> object: # Read the pickle file directly from the resource into memory. try: data = pickle.loads(uri.read()) diff --git a/python/lsst/daf/butler/formatters/yaml.py b/python/lsst/daf/butler/formatters/yaml.py index 107cafac06..fdef282afc 100644 --- a/python/lsst/daf/butler/formatters/yaml.py +++ b/python/lsst/daf/butler/formatters/yaml.py @@ -47,7 +47,9 @@ class YamlFormatter(TypelessFormatter): supported_write_parameters = frozenset({"unsafe_dump"}) can_read_from_uri = True - def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any: + def read_from_uri( + self, uri: ResourcePath, component: str | None = None, expected_size: int = -1 + ) -> object: # Can not use ResourcePath.open() data = yaml.safe_load(uri.read()) return data diff --git a/python/lsst/daf/butler/tests/_datasetsHelper.py b/python/lsst/daf/butler/tests/_datasetsHelper.py index 6f1ee0bf7e..8b2a23b956 100644 --- a/python/lsst/daf/butler/tests/_datasetsHelper.py +++ b/python/lsst/daf/butler/tests/_datasetsHelper.py @@ -225,7 +225,9 @@ class MultiDetectorFormatter(YamlFormatter): can_read_from_uri = True - def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any: + def read_from_uri( + self, uri: ResourcePath, component: str | None = None, expected_size: int = -1 + ) -> object: if self.data_id is None: raise RuntimeError("This formatter requires a dataId") if "detector" not in self.data_id: @@ -234,6 +236,7 @@ def read_from_uri(self, uri: ResourcePath, component: str | None = None, expecte key = f"detector{self.data_id['detector']}" data = super().read_from_uri(uri, component) + assert isinstance(data, Mapping) # For mypy if key not in data: raise RuntimeError(f"Could not find '{key}' in data file.")