Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Coverage to 90% #1198

Draft
wants to merge 6 commits into
base: dev
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 1 addition & 23 deletions src/hdmf/__init__.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,10 @@
from . import query
from .backends.hdf5.h5_utils import H5Dataset, H5RegionSlicer
from .backends.hdf5.h5_utils import H5Dataset
from .container import Container, Data, DataRegion, HERDManager
from .region import ListSlicer
from .utils import docval, getargs
from .term_set import TermSet, TermSetWrapper, TypeConfigurator


@docval(
{"name": "dataset", "type": None, "doc": "the HDF5 dataset to slice"},
{"name": "region", "type": None, "doc": "the region reference to use to slice"},
is_method=False,
)
def get_region_slicer(**kwargs):
import warnings # noqa: E402

warnings.warn(
"get_region_slicer is deprecated and will be removed in HDMF 3.0.",
DeprecationWarning,
)

dataset, region = getargs("dataset", "region", kwargs)
if isinstance(dataset, (list, tuple, Data)):
return ListSlicer(dataset, region)
elif isinstance(dataset, H5Dataset):
return H5RegionSlicer(dataset, region)
return None


try:
# see https://effigies.gitlab.io/posts/python-packaging-2023/
from ._version import __version__
Expand Down
2 changes: 1 addition & 1 deletion src/hdmf/backends/hdf5/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
from . import h5_utils, h5tools
from .h5_utils import H5RegionSlicer, H5DataIO
from .h5_utils import H5DataIO
from .h5tools import HDF5IO, H5SpecWriter, H5SpecReader
23 changes: 0 additions & 23 deletions src/hdmf/backends/hdf5/h5_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from ...array import Array
from ...data_utils import DataIO, AbstractDataChunkIterator, append_data
from ...query import HDMFDataset, ReferenceResolver, ContainerResolver, BuilderResolver
from ...region import RegionSlicer
from ...spec import SpecWriter, SpecReader
from ...utils import docval, getargs, popargs, get_docval, get_data_shape

Expand Down Expand Up @@ -420,28 +419,6 @@ def read_namespace(self, ns_path):
return ret


class H5RegionSlicer(RegionSlicer):

@docval({'name': 'dataset', 'type': (Dataset, H5Dataset), 'doc': 'the HDF5 dataset to slice'},
{'name': 'region', 'type': RegionReference, 'doc': 'the region reference to use to slice'})
def __init__(self, **kwargs):
self.__dataset = getargs('dataset', kwargs)
self.__regref = getargs('region', kwargs)
self.__len = self.__dataset.regionref.selection(self.__regref)[0]
self.__region = None

def __read_region(self):
if self.__region is None:
self.__region = self.__dataset[self.__regref]

def __getitem__(self, idx):
self.__read_region()
return self.__region[idx]

def __len__(self):
return self.__len


class H5DataIO(DataIO):
"""
Wrap data arrays for write via HDF5IO to customize I/O behavior, such as compression and chunking
Expand Down
106 changes: 21 additions & 85 deletions src/hdmf/backends/hdf5/h5tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -750,9 +750,7 @@ def __read_attrs(self, h5obj):
for k, v in h5obj.attrs.items():
if k == SPEC_LOC_ATTR: # ignore cached spec
continue
if isinstance(v, RegionReference):
raise ValueError("cannot read region reference attributes yet")
elif isinstance(v, Reference):
if isinstance(v, Reference):
ret[k] = self.__read_ref(h5obj.file[v])
else:
ret[k] = v
Expand Down Expand Up @@ -919,7 +917,6 @@ def get_type(cls, data):
"ref": H5_REF,
"reference": H5_REF,
"object": H5_REF,
"region": H5_REGREF,
"isodatetime": H5_TEXT,
"datetime": H5_TEXT,
}
Expand Down Expand Up @@ -1237,29 +1234,12 @@ def _filler():
dset = self.__scalar_fill__(parent, name, data, options)
else:
dset = self.__list_fill__(parent, name, data, options)
# Write a dataset containing references, i.e., a region or object reference.
# Write a dataset containing references, i.e., object reference.
# NOTE: we can ignore options['io_settings'] for scalar data
elif self.__is_ref(options['dtype']):
_dtype = self.__dtypes.get(options['dtype'])
# Write a scalar data region reference dataset
if isinstance(data, RegionBuilder):
dset = parent.require_dataset(name, shape=(), dtype=_dtype)
self.__set_written(builder)
self.logger.debug("Queueing reference resolution and set attribute on dataset '%s' containing a "
"region reference. attributes: %s"
% (name, list(attributes.keys())))

@self.__queue_ref
def _filler():
self.logger.debug("Resolving region reference and setting attribute on dataset '%s' "
"containing attributes: %s"
% (name, list(attributes.keys())))
ref = self.__get_ref(data.builder, data.region)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
# Write a scalar object reference dataset
elif isinstance(data, ReferenceBuilder):
if isinstance(data, ReferenceBuilder):
dset = parent.require_dataset(name, dtype=_dtype, shape=())
self.__set_written(builder)
self.logger.debug("Queueing reference resolution and set attribute on dataset '%s' containing an "
Expand All @@ -1277,44 +1257,24 @@ def _filler():
self.set_attributes(dset, attributes)
# Write an array dataset of references
else:
# Write a array of region references
if options['dtype'] == 'region':
dset = parent.require_dataset(name, dtype=_dtype, shape=(len(data),), **options['io_settings'])
self.__set_written(builder)
self.logger.debug("Queueing reference resolution and set attribute on dataset '%s' containing "
"region references. attributes: %s"
% (name, list(attributes.keys())))

@self.__queue_ref
def _filler():
self.logger.debug("Resolving region references and setting attribute on dataset '%s' "
"containing attributes: %s"
% (name, list(attributes.keys())))
refs = list()
for item in data:
refs.append(self.__get_ref(item.builder, item.region))
dset = parent[name]
dset[()] = refs
self.set_attributes(dset, attributes)
# Write array of object references
else:
dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype, **options['io_settings'])
self.__set_written(builder)
self.logger.debug("Queueing reference resolution and set attribute on dataset '%s' containing "
"object references. attributes: %s"
% (name, list(attributes.keys())))
dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype, **options['io_settings'])
self.__set_written(builder)
self.logger.debug("Queueing reference resolution and set attribute on dataset '%s' containing "
"object references. attributes: %s"
% (name, list(attributes.keys())))

@self.__queue_ref
def _filler():
self.logger.debug("Resolving object references and setting attribute on dataset '%s' "
"containing attributes: %s"
% (name, list(attributes.keys())))
refs = list()
for item in data:
refs.append(self.__get_ref(item))
dset = parent[name]
dset[()] = refs
self.set_attributes(dset, attributes)
@self.__queue_ref
def _filler():
self.logger.debug("Resolving object references and setting attribute on dataset '%s' "
"containing attributes: %s"
% (name, list(attributes.keys())))
refs = list()
for item in data:
refs.append(self.__get_ref(item))
dset = parent[name]
dset[()] = refs
self.set_attributes(dset, attributes)
return
# write a "regular" dataset
else:
Expand Down Expand Up @@ -1502,11 +1462,9 @@ def __list_fill__(cls, parent, name, data, options=None):

@docval({'name': 'container', 'type': (Builder, Container, ReferenceBuilder), 'doc': 'the object to reference',
'default': None},
{'name': 'region', 'type': (slice, list, tuple), 'doc': 'the region reference indexing object',
'default': None},
returns='the reference', rtype=Reference)
def __get_ref(self, **kwargs):
container, region = getargs('container', 'region', kwargs)
container = getargs('container', kwargs)
if container is None:
return None
if isinstance(container, Builder):
Expand All @@ -1523,21 +1481,10 @@ def __get_ref(self, **kwargs):
builder = self.manager.build(container)
path = self.__get_path(builder)

self.logger.debug("Getting reference at path '%s'" % path)
if isinstance(container, RegionBuilder):
region = container.region
if region is not None:
dset = self.__file[path]
if not isinstance(dset, Dataset):
raise ValueError('cannot create region reference without Dataset')
return self.__file[path].regionref[region]
else:
return self.__file[path].ref
return self.__file[path].ref

@docval({'name': 'container', 'type': (Builder, Container, ReferenceBuilder), 'doc': 'the object to reference',
'default': None},
{'name': 'region', 'type': (slice, list, tuple), 'doc': 'the region reference indexing object',
'default': None},
returns='the reference', rtype=Reference)
def _create_ref(self, **kwargs):
return self.__get_ref(**kwargs)
Expand Down Expand Up @@ -1569,17 +1516,6 @@ def __queue_ref(self, func):
# dependency
self.__ref_queue.append(func)

def __rec_get_ref(self, ref_list):
ret = list()
for elem in ref_list:
if isinstance(elem, (list, tuple)):
ret.append(self.__rec_get_ref(elem))
elif isinstance(elem, (Builder, Container)):
ret.append(self.__get_ref(elem))
else:
ret.append(elem)
return ret

@property
def mode(self):
"""
Expand Down
14 changes: 0 additions & 14 deletions src/hdmf/build/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -490,20 +490,6 @@ def load_namespaces(self, **kwargs):
self.register_container_type(new_ns, dt, container_cls)
return deps

@docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type"},
{"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"},
{"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True},
returns='the class for the given namespace and data_type', rtype=type)
def get_container_cls(self, **kwargs):
"""Get the container class from data type specification.
If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically
created and returned.
"""
# NOTE: this internally used function get_container_cls will be removed in favor of get_dt_container_cls
# Deprecated: Will be removed by HDMF 4.0
namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs)
return self.get_dt_container_cls(data_type, namespace, autogen)

@docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"},
{"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None},
{'name': 'post_init_method', 'type': Callable, 'default': None,
Expand Down
48 changes: 15 additions & 33 deletions src/hdmf/build/objectmapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -963,41 +963,23 @@ def _filler():
return _filler

def __get_ref_builder(self, builder, dtype, shape, container, build_manager):
bldr_data = None
if dtype.is_region():
if shape is None:
if not isinstance(container, DataRegion):
msg = "'container' must be of type DataRegion if spec represents region reference"
raise ValueError(msg)
self.logger.debug("Setting %s '%s' data to region reference builder"
% (builder.__class__.__name__, builder.name))
target_builder = self.__get_target_builder(container.data, build_manager, builder)
bldr_data = RegionBuilder(container.region, target_builder)
else:
self.logger.debug("Setting %s '%s' data to list of region reference builders"
% (builder.__class__.__name__, builder.name))
bldr_data = list()
for d in container.data:
target_builder = self.__get_target_builder(d.target, build_manager, builder)
bldr_data.append(RegionBuilder(d.slice, target_builder))
self.logger.debug("Setting object reference dataset on %s '%s' data"
% (builder.__class__.__name__, builder.name))
if isinstance(container, Data):
self.logger.debug("Setting %s '%s' data to list of reference builders"
% (builder.__class__.__name__, builder.name))
bldr_data = list()
for d in container.data:
target_builder = self.__get_target_builder(d, build_manager, builder)
bldr_data.append(ReferenceBuilder(target_builder))
if isinstance(container.data, H5DataIO):
# This is here to support appending a dataset of references.
bldr_data = H5DataIO(bldr_data, **container.data.get_io_params())
else:
self.logger.debug("Setting object reference dataset on %s '%s' data"
self.logger.debug("Setting %s '%s' data to reference builder"
% (builder.__class__.__name__, builder.name))
if isinstance(container, Data):
self.logger.debug("Setting %s '%s' data to list of reference builders"
% (builder.__class__.__name__, builder.name))
bldr_data = list()
for d in container.data:
target_builder = self.__get_target_builder(d, build_manager, builder)
bldr_data.append(ReferenceBuilder(target_builder))
if isinstance(container.data, H5DataIO):
# This is here to support appending a dataset of references.
bldr_data = H5DataIO(bldr_data, **container.data.get_io_params())
else:
self.logger.debug("Setting %s '%s' data to reference builder"
% (builder.__class__.__name__, builder.name))
target_builder = self.__get_target_builder(container, build_manager, builder)
bldr_data = ReferenceBuilder(target_builder)
target_builder = self.__get_target_builder(container, build_manager, builder)
bldr_data = ReferenceBuilder(target_builder)
return bldr_data

def __get_target_builder(self, container, build_manager, builder):
Expand Down
Loading
Loading