Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Jun 8, 2024
1 parent 78a6035 commit 73a6edb
Showing 1 changed file with 26 additions and 12 deletions.
38 changes: 26 additions & 12 deletions python/lsst/daf/butler/_formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,9 @@ def read(
# In case someone set the flag incorrectly, if this is not
# implemented fall back to the other methods.
with contextlib.suppress(NotImplementedError):
return self.read_from_cached_uri(component, expected_size, cache_manager=cache_manager)
return self.read_from_possibly_cached_uri(
component, expected_size, cache_manager=cache_manager
)

# Should we check the size with file_uri.size() if expected size is -1?
# Do we give a choice here at all and allow a formatter to say one
Expand All @@ -351,11 +353,15 @@ def read(
# If this method is not implemented, fall back to using a file.
# V1 formatters in emulation won't set the read threshold.
with contextlib.suppress(NotImplementedError):
return self.read_from_cached_bytes(component, expected_size, cache_manager=cache_manager)
return self.read_from_possibly_cached_bytes(
component, expected_size, cache_manager=cache_manager
)

return self.read_from_cached_local_file(component, expected_size, cache_manager=cache_manager)
return self.read_from_possibly_cached_local_file(
component, expected_size, cache_manager=cache_manager
)

def read_from_cached_bytes(
def read_from_possibly_cached_bytes(
self,
component: str | None = None,
expected_size: int = -1,
Expand Down Expand Up @@ -425,7 +431,7 @@ def read_from_cached_bytes(

return self.read_from_bytes(serialized_dataset, component=component)

def read_from_cached_uri(
def read_from_possibly_cached_uri(
self,
component: str | None = None,
expected_size: int = -1,
Expand Down Expand Up @@ -501,11 +507,11 @@ def read_from_cached_uri(
self.name(),
),
):
result = self.read_cached_file(desired_uri, component=component)
result = self.read_direct_file(desired_uri, component=component)

return result

def read_from_cached_local_file(
def read_from_possibly_cached_local_file(
self,
component: str | None = None,
expected_size: int = -1,
Expand Down Expand Up @@ -612,8 +618,8 @@ def read_from_cached_local_file(

return result

def read_cached_file(self, uri: ResourcePath, component: str | None = None) -> Any:
"""Read a dataset from a, possibly cached, URI.
def read_direct_file(self, uri: ResourcePath, component: str | None = None) -> Any:
"""Read a dataset from a URI that can be local or remote.
Parameters
----------
Expand All @@ -632,6 +638,10 @@ def read_cached_file(self, uri: ResourcePath, component: str | None = None) -> A
-----
This method is only called if the class property
``allow_remote_file_read`` is set to `True`.
It is possible that a cached local file will be given to this method
even if it was originally a remote URI. This can happen if the write
resulted in the file being added to the local cache.
"""
raise NotImplementedError("This formatter does not know how to read a file.")

Expand Down Expand Up @@ -730,10 +740,14 @@ def write_direct(
Notes
-----
This method will call `to_bytes` to serialize the in memory dataset
and then will call the `~lsst.resources.ResourcePath.write` method
directly.
If the dataset should be cached or is local the file will not be
written. Local URIs should be written to temporary file name and then
renamed to allow atomic writes. That path is handled by
`write_locally_then_move`.
written and the method will return `False`. This is because local URIs
should be written to a temporary file name and then renamed to allow
atomic writes. That path is handled by `write_locally_then_move`.
"""
if cache_manager is None:
# Circular import avoidance.
Expand Down

0 comments on commit 73a6edb

Please sign in to comment.