diff --git a/aeon/io/api.py b/aeon/io/api.py index f79724d1..502bf1a3 100644 --- a/aeon/io/api.py +++ b/aeon/io/api.py @@ -76,7 +76,7 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No fileset = { chunk_key(fname):fname for path in root - for fname in Path(path).glob(f"{epoch_pattern}/**/{reader.pattern}*.{reader.extension}")} + for fname in Path(path).glob(f"{epoch_pattern}/**/{reader.pattern}.{reader.extension}")} files = sorted(fileset.items()) if time is not None: @@ -117,27 +117,25 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No return pd.concat(dataframes) if start is not None or end is not None: - chunkfilter = chunk_range(start, end) - files = list(filter(lambda item: item[0][1] in chunkfilter, files)) - else: - chunkfilter = None + chunk_start = chunk(start) if start is not None else pd.Timestamp.min + chunk_end = chunk(end) if end is not None else pd.Timestamp.max + files = list(filter(lambda item: chunk_start <= chunk(item[0][1]) <= chunk_end, files)) if len(files) == 0: return _empty(reader.columns) data = pd.concat([reader.read(file) for _, file in files]) _set_index(data) - if chunkfilter is not None: + if start is not None or end is not None: try: return data.loc[start:end] except KeyError: import warnings - # if not data.index.has_duplicates: - # warnings.warn('data index for {0} contains out-of-order timestamps!'.format(reader.pattern)) - # data = data.sort_index() - # else: - # warnings.warn('data index for {0} contains duplicate keys!'.format(reader.pattern)) - # data = data[~data.index.duplicated(keep='first')] - # return data.loc[start:end] - return data + if not data.index.has_duplicates: + warnings.warn('data index for {0} contains out-of-order timestamps!'.format(reader.pattern)) + data = data.sort_index() + else: + warnings.warn('data index for {0} contains duplicate keys!'.format(reader.pattern)) + data = data[~data.index.duplicated(keep='first')] + return data.loc[start:end] return data diff --git a/aeon/io/reader.py b/aeon/io/reader.py index a2961166..96477fc6 100644 --- a/aeon/io/reader.py +++ b/aeon/io/reader.py @@ -21,10 +21,10 @@ } class Reader: - """Extracts data from raw chunk files in an Aeon dataset. + """Extracts data from raw files in an Aeon dataset. Attributes: - pattern (str): Pattern used to find raw chunk files, + pattern (str): Pattern used to find raw files, usually in the format `_`. columns (str or array-like): Column labels to use for the data. extension (str): Extension of data file pathnames. @@ -35,7 +35,7 @@ def __init__(self, pattern, columns, extension): self.extension = extension def read(self, _): - """Reads data from the specified chunk file.""" + """Reads data from the specified file.""" return pd.DataFrame(columns=self.columns, index=pd.DatetimeIndex([])) class Harp(Reader): diff --git a/aeon/schema/core.py b/aeon/schema/core.py index f399bad8..36fe2bec 100644 --- a/aeon/schema/core.py +++ b/aeon/schema/core.py @@ -3,15 +3,15 @@ def video(pattern): """Video frame metadata.""" - return { "Video": _reader.Video(pattern) } + return { "Video": _reader.Video(f"{pattern}_*") } def position(pattern): """Position tracking data for the specified camera.""" - return { "Position": _reader.Position(f"{pattern}_200") } + return { "Position": _reader.Position(f"{pattern}_200_*") } def encoder(pattern): """Wheel magnetic encoder data.""" - return { "Encoder": _reader.Encoder(f"{pattern}_90") } + return { "Encoder": _reader.Encoder(f"{pattern}_90_*") } def environment(pattern): """Metadata for environment mode and subjects.""" @@ -19,15 +19,15 @@ def environment(pattern): def environment_state(pattern): """Environment state log.""" - return { "EnvironmentState": _reader.Csv(f"{pattern}_EnvironmentState", ['state']) } + return { "EnvironmentState": _reader.Csv(f"{pattern}_EnvironmentState_*", ['state']) } def subject_state(pattern): """Subject state log.""" - return { "SubjectState": _reader.Subject(f"{pattern}_SubjectState") } + return { "SubjectState": _reader.Subject(f"{pattern}_SubjectState_*") } def messageLog(pattern): """Message log data.""" - return { "MessageLog": _reader.Log(f"{pattern}_MessageLog") } + return { "MessageLog": _reader.Log(f"{pattern}_MessageLog_*") } def metadata(pattern): """Metadata for acquisition epochs.""" diff --git a/aeon/schema/foraging.py b/aeon/schema/foraging.py index 89fc240e..5580bf80 100644 --- a/aeon/schema/foraging.py +++ b/aeon/schema/foraging.py @@ -48,11 +48,11 @@ def __init__(self, pattern): def region(pattern): """Region tracking data for the specified camera.""" - return { "Region": _RegionReader(f"{pattern}_201") } + return { "Region": _RegionReader(f"{pattern}_201_*") } def depletionFunction(pattern): """State of the linear depletion function for foraging patches.""" - return { "DepletionState": _PatchState(f"{pattern}_State") } + return { "DepletionState": _PatchState(f"{pattern}_State_*") } def feeder(pattern): """Feeder commands and events.""" @@ -60,11 +60,11 @@ def feeder(pattern): def beam_break(pattern): """Beam break events for pellet detection.""" - return { "BeamBreak": _reader.BitmaskEvent(f"{pattern}_32", 0x22, 'PelletDetected') } + return { "BeamBreak": _reader.BitmaskEvent(f"{pattern}_32_*", 0x22, 'PelletDetected') } def deliver_pellet(pattern): """Pellet delivery commands.""" - return { "DeliverPellet": _reader.BitmaskEvent(f"{pattern}_35", 0x80, 'TriggerPellet') } + return { "DeliverPellet": _reader.BitmaskEvent(f"{pattern}_35_*", 0x80, 'TriggerPellet') } def patch(pattern): """Data streams for a patch.""" @@ -76,16 +76,16 @@ def weight(pattern): def weight_raw(pattern): """Raw weight measurement for a specific nest.""" - return { "WeightRaw": _Weight(f"{pattern}_200") } + return { "WeightRaw": _Weight(f"{pattern}_200_*") } def weight_filtered(pattern): """Filtered weight measurement for a specific nest.""" - return { "WeightFiltered": _Weight(f"{pattern}_202") } + return { "WeightFiltered": _Weight(f"{pattern}_202_*") } def weight_subject(pattern): """Subject weight measurement for a specific nest.""" - return { "WeightSubject": _Weight(f"{pattern}_204") } + return { "WeightSubject": _Weight(f"{pattern}_204_*") } def session(pattern): """Session metadata for Experiment 0.1.""" - return { pattern: _reader.Csv(f"{pattern}_2", columns=['id','weight','event']) } + return { pattern: _reader.Csv(f"{pattern}_2*", columns=['id','weight','event']) } diff --git a/aeon/schema/octagon.py b/aeon/schema/octagon.py index fe8adc60..b283c905 100644 --- a/aeon/schema/octagon.py +++ b/aeon/schema/octagon.py @@ -3,24 +3,24 @@ import aeon.schema.core as _stream def photodiode(pattern): - return { "Photodiode": _reader.Harp(f"{pattern}_44", columns=['adc', 'encoder']) } + return { "Photodiode": _reader.Harp(f"{pattern}_44_*", columns=['adc', 'encoder']) } class OSC: @staticmethod def background_color(pattern): - return { "BackgroundColor": _reader.Csv(f"{pattern}_backgroundcolor", columns=['typetag', 'r', 'g', 'b', 'a']) } + return { "BackgroundColor": _reader.Csv(f"{pattern}_backgroundcolor_*", columns=['typetag', 'r', 'g', 'b', 'a']) } @staticmethod def change_subject_state(pattern): - return { "ChangeSubjectState": _reader.Csv(f"{pattern}_changesubjectstate", columns=['typetag', 'id', 'weight', 'event']) } + return { "ChangeSubjectState": _reader.Csv(f"{pattern}_changesubjectstate_*", columns=['typetag', 'id', 'weight', 'event']) } @staticmethod def end_trial(pattern): - return { "EndTrial": _reader.Csv(f"{pattern}_endtrial", columns=['typetag', 'value']) } + return { "EndTrial": _reader.Csv(f"{pattern}_endtrial_*", columns=['typetag', 'value']) } @staticmethod def slice(pattern): - return { "Slice": _reader.Csv(f"{pattern}_octagonslice", columns=[ + return { "Slice": _reader.Csv(f"{pattern}_octagonslice_*", columns=[ 'typetag', 'wall_id', 'r', 'g', 'b', 'a', @@ -28,7 +28,7 @@ def slice(pattern): @staticmethod def gratings_slice(pattern): - return { "GratingsSlice": _reader.Csv(f"{pattern}_octagongratingsslice", columns=[ + return { "GratingsSlice": _reader.Csv(f"{pattern}_octagongratingsslice_*", columns=[ 'typetag', 'wall_id', 'contrast', @@ -40,7 +40,7 @@ def gratings_slice(pattern): @staticmethod def poke(pattern): - return { "Poke": _reader.Csv(f"{pattern}_poke", columns=[ + return { "Poke": _reader.Csv(f"{pattern}_poke_*", columns=[ 'typetag', 'wall_id', 'poke_id', @@ -51,7 +51,7 @@ def poke(pattern): @staticmethod def response(pattern): - return { "Response": _reader.Csv(f"{pattern}_response", columns=[ + return { "Response": _reader.Csv(f"{pattern}_response_*", columns=[ 'typetag', 'wall_id', 'poke_id', @@ -59,7 +59,7 @@ def response(pattern): @staticmethod def run_pre_trial_no_poke(pattern): - return { "RunPreTrialNoPoke": _reader.Csv(f"{pattern}_run_pre_no_poke", columns=[ + return { "RunPreTrialNoPoke": _reader.Csv(f"{pattern}_run_pre_no_poke_*", columns=[ 'typetag', 'wait_for_poke', 'reward_iti', @@ -69,94 +69,94 @@ def run_pre_trial_no_poke(pattern): @staticmethod def start_new_session(pattern): - return { "StartNewSession": _reader.Csv(f"{pattern}_startnewsession", columns=['typetag', 'path' ]) } + return { "StartNewSession": _reader.Csv(f"{pattern}_startnewsession_*", columns=['typetag', 'path' ]) } class TaskLogic: @staticmethod def trial_initiation(pattern): - return { "TrialInitiation": _reader.Harp(f"{pattern}_1", columns=['trial_type']) } + return { "TrialInitiation": _reader.Harp(f"{pattern}_1_*", columns=['trial_type']) } @staticmethod def response(pattern): - return { "Response": _reader.Harp(f"{pattern}_2", columns=['wall_id', 'poke_id']) } + return { "Response": _reader.Harp(f"{pattern}_2_*", columns=['wall_id', 'poke_id']) } @staticmethod def pre_trial(pattern): - return { "PreTrialState": _reader.Harp(f"{pattern}_3", columns=['state']) } + return { "PreTrialState": _reader.Harp(f"{pattern}_3_*", columns=['state']) } @staticmethod def inter_trial_interval(pattern): - return { "InterTrialInterval": _reader.Harp(f"{pattern}_4", columns=['state']) } + return { "InterTrialInterval": _reader.Harp(f"{pattern}_4_*", columns=['state']) } @staticmethod def slice_onset(pattern): - return { "SliceOnset": _reader.Harp(f"{pattern}_10", columns=['wall_id']) } + return { "SliceOnset": _reader.Harp(f"{pattern}_10_*", columns=['wall_id']) } @staticmethod def draw_background(pattern): - return { "DrawBackground": _reader.Harp(f"{pattern}_11", columns=['state']) } + return { "DrawBackground": _reader.Harp(f"{pattern}_11_*", columns=['state']) } @staticmethod def gratings_slice_onset(pattern): - return { "GratingsSliceOnset": _reader.Harp(f"{pattern}_12", columns=['wall_id']) } + return { "GratingsSliceOnset": _reader.Harp(f"{pattern}_12_*", columns=['wall_id']) } class Wall: @staticmethod def beam_break0(pattern): - return { "BeamBreak0": _reader.DigitalBitmask(f"{pattern}_32", 0x1, columns=['state']) } + return { "BeamBreak0": _reader.DigitalBitmask(f"{pattern}_32_*", 0x1, columns=['state']) } @staticmethod def beam_break1(pattern): - return { "BeamBreak1": _reader.DigitalBitmask(f"{pattern}_32", 0x2, columns=['state']) } + return { "BeamBreak1": _reader.DigitalBitmask(f"{pattern}_32_*", 0x2, columns=['state']) } @staticmethod def beam_break2(pattern): - return { "BeamBreak2": _reader.DigitalBitmask(f"{pattern}_32", 0x4, columns=['state']) } + return { "BeamBreak2": _reader.DigitalBitmask(f"{pattern}_32_*", 0x4, columns=['state']) } @staticmethod def set_led0(pattern): - return { "SetLed0": _reader.BitmaskEvent(f"{pattern}_34", 0x1, 'Set') } + return { "SetLed0": _reader.BitmaskEvent(f"{pattern}_34_*", 0x1, 'Set') } @staticmethod def set_led1(pattern): - return { "SetLed1": _reader.BitmaskEvent(f"{pattern}_34", 0x2, 'Set') } + return { "SetLed1": _reader.BitmaskEvent(f"{pattern}_34_*", 0x2, 'Set') } @staticmethod def set_led2(pattern): - return { "SetLed2": _reader.BitmaskEvent(f"{pattern}_34", 0x4, 'Set') } + return { "SetLed2": _reader.BitmaskEvent(f"{pattern}_34_*", 0x4, 'Set') } @staticmethod def set_valve0(pattern): - return { "SetValve0": _reader.BitmaskEvent(f"{pattern}_34", 0x8, 'Set') } + return { "SetValve0": _reader.BitmaskEvent(f"{pattern}_34_*", 0x8, 'Set') } @staticmethod def set_valve1(pattern): - return { "SetValve1": _reader.BitmaskEvent(f"{pattern}_34", 0x10, 'Set') } + return { "SetValve1": _reader.BitmaskEvent(f"{pattern}_34_*", 0x10, 'Set') } @staticmethod def set_valve2(pattern): - return { "SetValve2": _reader.BitmaskEvent(f"{pattern}_34", 0x20, 'Set') } + return { "SetValve2": _reader.BitmaskEvent(f"{pattern}_34_*", 0x20, 'Set') } @staticmethod def clear_led0(pattern): - return { "ClearLed0": _reader.BitmaskEvent(f"{pattern}_35", 0x1, 'Clear') } + return { "ClearLed0": _reader.BitmaskEvent(f"{pattern}_35_*", 0x1, 'Clear') } @staticmethod def clear_led1(pattern): - return { "ClearLed1": _reader.BitmaskEvent(f"{pattern}_35", 0x2, 'Clear') } + return { "ClearLed1": _reader.BitmaskEvent(f"{pattern}_35_*", 0x2, 'Clear') } @staticmethod def clear_led2(pattern): - return { "ClearLed2": _reader.BitmaskEvent(f"{pattern}_35", 0x4, 'Clear') } + return { "ClearLed2": _reader.BitmaskEvent(f"{pattern}_35_*", 0x4, 'Clear') } @staticmethod def clear_valve0(pattern): - return { "ClearValve0": _reader.BitmaskEvent(f"{pattern}_35", 0x8, 'Clear') } + return { "ClearValve0": _reader.BitmaskEvent(f"{pattern}_35_*", 0x8, 'Clear') } @staticmethod def clear_valve1(pattern): - return { "ClearValve1": _reader.BitmaskEvent(f"{pattern}_35", 0x10, 'Clear') } + return { "ClearValve1": _reader.BitmaskEvent(f"{pattern}_35_*", 0x10, 'Clear') } @staticmethod def clear_valve2(pattern): - return { "ClearValve2": _reader.BitmaskEvent(f"{pattern}_35", 0x20, 'Clear') } \ No newline at end of file + return { "ClearValve2": _reader.BitmaskEvent(f"{pattern}_35_*", 0x20, 'Clear') } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index bce7b420..b045d6f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,3 +92,8 @@ exclude = ''' [tool.isort] profile = "black" color_output = false + +[tool.pytest.ini_options] +markers = [ + "api", +] diff --git a/tests/data/monotonic/2022-06-13T13_14_25/Patch2/Patch2_90_2022-06-13T12-00-00.bin b/tests/data/monotonic/2022-06-13T13_14_25/Patch2/Patch2_90_2022-06-13T12-00-00.bin new file mode 100644 index 00000000..f7ad9008 Binary files /dev/null and b/tests/data/monotonic/2022-06-13T13_14_25/Patch2/Patch2_90_2022-06-13T12-00-00.bin differ diff --git a/tests/data/nonmonotonic/2022-06-06T09-24-28/Metadata.yml b/tests/data/nonmonotonic/2022-06-06T09-24-28/Metadata.yml new file mode 100644 index 00000000..34eb6c12 --- /dev/null +++ b/tests/data/nonmonotonic/2022-06-06T09-24-28/Metadata.yml @@ -0,0 +1,11 @@ +{ + "Workflow": "Experiment0.2.bonsai", + "Commit": "249cdc654af63e6959e64f7ff2c21f219cc912ea", + "Devices": { + "VideoController": { + "PortName": "COM3", + "GlobalTriggerFrequency": "50", + "LocalTriggerFrequency": "125" + } + } +} \ No newline at end of file diff --git a/tests/data/nonmonotonic/2022-06-06T09-24-28/Patch2/Patch2_90_2022-06-06T13-00-00.bin b/tests/data/nonmonotonic/2022-06-06T09-24-28/Patch2/Patch2_90_2022-06-06T13-00-00.bin new file mode 100644 index 00000000..b7e58015 Binary files /dev/null and b/tests/data/nonmonotonic/2022-06-06T09-24-28/Patch2/Patch2_90_2022-06-06T13-00-00.bin differ diff --git a/tests/io/test_api.py b/tests/io/test_api.py new file mode 100644 index 00000000..304ead49 --- /dev/null +++ b/tests/io/test_api.py @@ -0,0 +1,42 @@ +import pytest +from pytest import mark +import aeon.io.api as aeon +from aeon.schema.dataset import exp02 +import pandas as pd + +@mark.api +def test_load_start_only(): + data = aeon.load( + './tests/data/nonmonotonic', + exp02.Patch2.Encoder, + start=pd.Timestamp('2022-06-06T13:00:49')) + assert len(data) > 0 + +@mark.api +def test_load_end_only(): + data = aeon.load( + './tests/data/nonmonotonic', + exp02.Patch2.Encoder, + end=pd.Timestamp('2022-06-06T13:00:49')) + assert len(data) > 0 + +@mark.api +def test_load_filter_nonchunked(): + data = aeon.load( + './tests/data/nonmonotonic', + exp02.Metadata, + start=pd.Timestamp('2022-06-06T09:00:00')) + assert len(data) > 0 + +@mark.api +def test_load_monotonic(): + data = aeon.load('./tests/data/monotonic', exp02.Patch2.Encoder) + assert data.index.is_monotonic_increasing + +@mark.api +def test_load_nonmonotonic(): + data = aeon.load('./tests/data/nonmonotonic', exp02.Patch2.Encoder) + assert not data.index.is_monotonic_increasing + +if __name__ == '__main__': + pytest.main() \ No newline at end of file