Skip to content

Commit

Permalink
Use object rather than Any for the formatter read annotation
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Jul 9, 2024
1 parent 54a472a commit 7285a7c
Show file tree
Hide file tree
Showing 9 changed files with 36 additions and 22 deletions.
31 changes: 17 additions & 14 deletions python/lsst/daf/butler/_formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ def read(
component: str | None = None,
expected_size: int = -1,
cache_manager: AbstractDatastoreCacheManager | None = None,
) -> Any:
) -> object:
"""Read a Dataset.
Parameters
Expand Down Expand Up @@ -386,7 +386,8 @@ def read(
* `read_from_uri` (but with a local file)
It is possible for `read_from_uri` to be skipped if the implementation
raises `FormatterNotImplementedError`. If `read_from_stream` is
raises `FormatterNotImplementedError`, for example if the
implementation would prefer to read. If `read_from_stream` is
called `read_from_local_file` will never be called. If `read_from_uri`
was skipped and `read_from_local_file` is not implemented, it will
be called with a local file as a last resort.
Expand Down Expand Up @@ -486,12 +487,12 @@ def read(

def _read_from_possibly_cached_location_no_cache_write(
self,
callback: Callable[[ResourcePath, str | None, int], Any],
callback: Callable[[ResourcePath, str | None, int], object],
component: str | None = None,
expected_size: int = -1,
*,
cache_manager: AbstractDatastoreCacheManager | None = None,
) -> Any:
) -> object:
"""Read from the cache and call payload without writing to cache."""
cache_manager = self._ensure_cache(cache_manager)

Expand Down Expand Up @@ -533,7 +534,7 @@ def read_from_possibly_cached_stream(
expected_size: int = -1,
*,
cache_manager: AbstractDatastoreCacheManager | None = None,
) -> Any:
) -> object:
"""Read from a stream, checking for possible presence in local cache.
Parameters
Expand Down Expand Up @@ -562,7 +563,7 @@ def read_from_possibly_cached_stream(
a file to the local cache.
"""

def _open_stream(uri: ResourcePath, comp: str | None, size: int = -1) -> Any:
def _open_stream(uri: ResourcePath, comp: str | None, size: int = -1) -> object:
with uri.open("rb") as fd:
return self.read_from_stream(fd, comp, expected_size=size)

Expand All @@ -576,7 +577,7 @@ def read_directly_from_possibly_cached_uri(
expected_size: int = -1,
*,
cache_manager: AbstractDatastoreCacheManager | None = None,
) -> Any:
) -> object:
"""Read from arbitrary URI, checking for possible presence in local
cache.
Expand Down Expand Up @@ -608,7 +609,7 @@ def read_directly_from_possibly_cached_uri(
The URI will be read by calling `read_from_uri`.
"""

def _open_uri(uri: ResourcePath, comp: str | None, size: int = -1) -> Any:
def _open_uri(uri: ResourcePath, comp: str | None, size: int = -1) -> object:
return self.read_from_uri(uri, comp, expected_size=size)

return self._read_from_possibly_cached_location_no_cache_write(
Expand All @@ -621,7 +622,7 @@ def read_from_possibly_cached_local_file(
expected_size: int = -1,
*,
cache_manager: AbstractDatastoreCacheManager | None = None,
) -> Any:
) -> object:
"""Read a dataset ensuring that a local file is used, checking the
cache for it.
Expand Down Expand Up @@ -730,7 +731,9 @@ def read_from_possibly_cached_local_file(

return result

def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
def read_from_uri(
self, uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> object:
"""Read a dataset from a URI that can be local or remote.
Parameters
Expand Down Expand Up @@ -775,7 +778,7 @@ def read_from_uri(self, uri: ResourcePath, component: str | None = None, expecte

def read_from_stream(
self, stream: BinaryIO | ResourceHandleProtocol, component: str | None = None, expected_size: int = -1
) -> Any:
) -> object:
"""Read from an open file descriptor.
Parameters
Expand All @@ -802,7 +805,7 @@ def read_from_stream(

def read_from_local_file(
self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> Any:
) -> object:
"""Read a dataset from a URI guaranteed to refer to the local file
system.
Expand Down Expand Up @@ -1328,7 +1331,7 @@ def name(cls) -> str:
return get_full_type_name(cls)

@abstractmethod
def read(self, component: str | None = None) -> Any:
def read(self, component: str | None = None) -> object:
"""Read a Dataset.
Parameters
Expand Down Expand Up @@ -2020,7 +2023,7 @@ def validate_write_recipes( # type: ignore

def read_from_local_file(
self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> Any:
) -> object:
# Need to temporarily override the location since the V1 formatter
# will not know anything about this local file.

Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/formatters/astropyTable.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def get_write_extension(self) -> str:

def read_from_local_file(
self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> Any:
) -> object:
pytype = self.file_descriptor.storageClass.pytype
if not issubclass(pytype, astropy.table.Table):
raise TypeError(f"Python type {pytype} does not seem to be a astropy Table type")

Check warning on line 60 in python/lsst/daf/butler/formatters/astropyTable.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/daf/butler/formatters/astropyTable.py#L60

Added line #L60 was not covered by tests
Expand Down
4 changes: 3 additions & 1 deletion python/lsst/daf/butler/formatters/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@ class JsonFormatter(TypelessFormatter):
unsupported_parameters = None
can_read_from_uri = True

def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
def read_from_uri(
self, uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> object:
# json.load() reads the entire file content into memory
# and is no different from json.loads(uri.read()). It does not attempt
# to support incremental reading to minimize memory usage.
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/formatters/logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def _get_read_pytype(self) -> type[ButlerLogRecords]:

def read_from_local_file(
self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> Any:
) -> object:
# ResourcePath open() cannot do a per-line read so can not use
# `read_from_stream` and `read_from_uri` does not give any advantage
# over pre-downloading the whole file (which can be very large).
Expand Down
4 changes: 3 additions & 1 deletion python/lsst/daf/butler/formatters/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ def get_write_extension(self) -> str:
raise RuntimeError(f"Requested file format '{format}' is not supported for Packages")
return ext

def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
def read_from_uri(
self, uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> object:
# Read the full file using the class associated with the
# storage class it was originally written with.
# Read the bytes directly from resource. These are not going to be
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/formatters/parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class ParquetFormatter(FormatterV2):

def read_from_local_file(
self, local_uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> Any:
) -> object:
# Docstring inherited from Formatter.read.
schema = pq.read_schema(local_uri.ospath)

Expand Down
4 changes: 3 additions & 1 deletion python/lsst/daf/butler/formatters/pickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ class PickleFormatter(TypelessFormatter):
unsupported_parameters = None
can_read_from_uri = True

def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
def read_from_uri(
self, uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> object:
# Read the pickle file directly from the resource into memory.
try:
data = pickle.loads(uri.read())
Expand Down
4 changes: 3 additions & 1 deletion python/lsst/daf/butler/formatters/yaml.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ class YamlFormatter(TypelessFormatter):
supported_write_parameters = frozenset({"unsafe_dump"})
can_read_from_uri = True

def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
def read_from_uri(
self, uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> object:
# Can not use ResourcePath.open()
data = yaml.safe_load(uri.read())
return data
Expand Down
5 changes: 4 additions & 1 deletion python/lsst/daf/butler/tests/_datasetsHelper.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,9 @@ class MultiDetectorFormatter(YamlFormatter):

can_read_from_uri = True

def read_from_uri(self, uri: ResourcePath, component: str | None = None, expected_size: int = -1) -> Any:
def read_from_uri(
self, uri: ResourcePath, component: str | None = None, expected_size: int = -1
) -> object:
if self.data_id is None:
raise RuntimeError("This formatter requires a dataId")
if "detector" not in self.data_id:
Expand All @@ -234,6 +236,7 @@ def read_from_uri(self, uri: ResourcePath, component: str | None = None, expecte
key = f"detector{self.data_id['detector']}"

data = super().read_from_uri(uri, component)
assert isinstance(data, Mapping) # For mypy
if key not in data:
raise RuntimeError(f"Could not find '{key}' in data file.")

Check warning on line 241 in python/lsst/daf/butler/tests/_datasetsHelper.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/daf/butler/tests/_datasetsHelper.py#L241

Added line #L241 was not covered by tests

Expand Down

0 comments on commit 7285a7c

Please sign in to comment.