Merge pull request #217 from SainsburyWellcomeCentre/main

Merge from main

aeon/io/api.py

@@ -76,7 +76,7 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No
     fileset = {
         chunk_key(fname):fname
         for path in root
-        for fname in Path(path).glob(f"{epoch_pattern}/**/{reader.pattern}*.{reader.extension}")}
+        for fname in Path(path).glob(f"{epoch_pattern}/**/{reader.pattern}.{reader.extension}")}
     files = sorted(fileset.items())
 
     if time is not None:
@@ -117,27 +117,25 @@ def load(root, reader, start=None, end=None, time=None, tolerance=None, epoch=No
         return pd.concat(dataframes)
 
     if start is not None or end is not None:
-        chunkfilter = chunk_range(start, end)
-        files = list(filter(lambda item: item[0][1] in chunkfilter, files))
-    else:
-        chunkfilter = None
+        chunk_start = chunk(start) if start is not None else pd.Timestamp.min
+        chunk_end = chunk(end) if end is not None else pd.Timestamp.max
+        files = list(filter(lambda item: chunk_start <= chunk(item[0][1]) <= chunk_end, files))
 
     if len(files) == 0:
         return _empty(reader.columns)
 
     data = pd.concat([reader.read(file) for _, file in files])
     _set_index(data)
-    if chunkfilter is not None:
+    if start is not None or end is not None:
         try:
             return data.loc[start:end]
         except KeyError:
             import warnings
-            # if not data.index.has_duplicates:
-            #     warnings.warn('data index for {0} contains out-of-order timestamps!'.format(reader.pattern))
-            #     data = data.sort_index()
-            # else:
-            #     warnings.warn('data index for {0} contains duplicate keys!'.format(reader.pattern))
-            #     data = data[~data.index.duplicated(keep='first')]
-            # return data.loc[start:end]
-            return data
+            if not data.index.has_duplicates:
+                warnings.warn('data index for {0} contains out-of-order timestamps!'.format(reader.pattern))
+                data = data.sort_index()
+            else:
+                warnings.warn('data index for {0} contains duplicate keys!'.format(reader.pattern))
+                data = data[~data.index.duplicated(keep='first')]
+            return data.loc[start:end]
     return data

aeon/io/reader.py

@@ -21,10 +21,10 @@
 }
 
 class Reader:
-    """Extracts data from raw chunk files in an Aeon dataset.
+    """Extracts data from raw files in an Aeon dataset.
     
     Attributes:
-        pattern (str): Pattern used to find raw chunk files,
+        pattern (str): Pattern used to find raw files,
             usually in the format `<Device>_<DataStream>`.
         columns (str or array-like): Column labels to use for the data.
         extension (str): Extension of data file pathnames.
@@ -35,7 +35,7 @@ def __init__(self, pattern, columns, extension):
         self.extension = extension
 
     def read(self, _):
-        """Reads data from the specified chunk file."""
+        """Reads data from the specified file."""
         return pd.DataFrame(columns=self.columns, index=pd.DatetimeIndex([]))
 
 class Harp(Reader):

aeon/schema/core.py

@@ -3,31 +3,31 @@
 
 def video(pattern):
     """Video frame metadata."""
-    return { "Video": _reader.Video(pattern) }
+    return { "Video": _reader.Video(f"{pattern}_*") }
 
 def position(pattern):
     """Position tracking data for the specified camera."""
-    return { "Position": _reader.Position(f"{pattern}_200") }
+    return { "Position": _reader.Position(f"{pattern}_200_*") }
 
 def encoder(pattern):
     """Wheel magnetic encoder data."""
-    return { "Encoder": _reader.Encoder(f"{pattern}_90") }
+    return { "Encoder": _reader.Encoder(f"{pattern}_90_*") }
 
 def environment(pattern):
     """Metadata for environment mode and subjects."""
     return _device.compositeStream(pattern, environment_state, subject_state)
 
 def environment_state(pattern):
     """Environment state log."""
-    return { "EnvironmentState": _reader.Csv(f"{pattern}_EnvironmentState", ['state']) }
+    return { "EnvironmentState": _reader.Csv(f"{pattern}_EnvironmentState_*", ['state']) }
 
 def subject_state(pattern):
     """Subject state log."""
-    return { "SubjectState": _reader.Subject(f"{pattern}_SubjectState") }
+    return { "SubjectState": _reader.Subject(f"{pattern}_SubjectState_*") }
 
 def messageLog(pattern):
     """Message log data."""
-    return { "MessageLog": _reader.Log(f"{pattern}_MessageLog") }
+    return { "MessageLog": _reader.Log(f"{pattern}_MessageLog_*") }
 
 def metadata(pattern):
     """Metadata for acquisition epochs."""

aeon/schema/foraging.py

@@ -48,23 +48,23 @@ def __init__(self, pattern):
 
 def region(pattern):
     """Region tracking data for the specified camera."""
-    return { "Region": _RegionReader(f"{pattern}_201") }
+    return { "Region": _RegionReader(f"{pattern}_201_*") }
 
 def depletionFunction(pattern):
     """State of the linear depletion function for foraging patches."""
-    return { "DepletionState": _PatchState(f"{pattern}_State") }
+    return { "DepletionState": _PatchState(f"{pattern}_State_*") }
 
 def feeder(pattern):
     """Feeder commands and events."""
     return _device.compositeStream(pattern, beam_break, deliver_pellet)
 
 def beam_break(pattern):
     """Beam break events for pellet detection."""
-    return { "BeamBreak": _reader.BitmaskEvent(f"{pattern}_32", 0x22, 'PelletDetected') }
+    return { "BeamBreak": _reader.BitmaskEvent(f"{pattern}_32_*", 0x22, 'PelletDetected') }
 
 def deliver_pellet(pattern):
     """Pellet delivery commands."""
-    return { "DeliverPellet": _reader.BitmaskEvent(f"{pattern}_35", 0x80, 'TriggerPellet') }
+    return { "DeliverPellet": _reader.BitmaskEvent(f"{pattern}_35_*", 0x80, 'TriggerPellet') }
 
 def patch(pattern):
     """Data streams for a patch."""
@@ -76,16 +76,16 @@ def weight(pattern):
 
 def weight_raw(pattern):
     """Raw weight measurement for a specific nest."""
-    return { "WeightRaw": _Weight(f"{pattern}_200") }
+    return { "WeightRaw": _Weight(f"{pattern}_200_*") }
 
 def weight_filtered(pattern):
     """Filtered weight measurement for a specific nest."""
-    return { "WeightFiltered": _Weight(f"{pattern}_202") }
+    return { "WeightFiltered": _Weight(f"{pattern}_202_*") }
 
 def weight_subject(pattern):
     """Subject weight measurement for a specific nest."""
-    return { "WeightSubject": _Weight(f"{pattern}_204") }
+    return { "WeightSubject": _Weight(f"{pattern}_204_*") }
 
 def session(pattern):
     """Session metadata for Experiment 0.1."""
-    return { pattern: _reader.Csv(f"{pattern}_2", columns=['id','weight','event']) }
+    return { pattern: _reader.Csv(f"{pattern}_2*", columns=['id','weight','event']) }

aeon/schema/octagon.py

@@ -3,32 +3,32 @@
 import aeon.schema.core as _stream
 
 def photodiode(pattern):
-    return { "Photodiode": _reader.Harp(f"{pattern}_44", columns=['adc', 'encoder']) }
+    return { "Photodiode": _reader.Harp(f"{pattern}_44_*", columns=['adc', 'encoder']) }
 
 class OSC:
     @staticmethod
     def background_color(pattern):
-        return { "BackgroundColor": _reader.Csv(f"{pattern}_backgroundcolor", columns=['typetag', 'r', 'g', 'b', 'a']) }
+        return { "BackgroundColor": _reader.Csv(f"{pattern}_backgroundcolor_*", columns=['typetag', 'r', 'g', 'b', 'a']) }
 
     @staticmethod
     def change_subject_state(pattern):
-        return { "ChangeSubjectState": _reader.Csv(f"{pattern}_changesubjectstate", columns=['typetag', 'id', 'weight', 'event']) }
+        return { "ChangeSubjectState": _reader.Csv(f"{pattern}_changesubjectstate_*", columns=['typetag', 'id', 'weight', 'event']) }
 
     @staticmethod
     def end_trial(pattern):
-        return { "EndTrial": _reader.Csv(f"{pattern}_endtrial", columns=['typetag', 'value']) }
+        return { "EndTrial": _reader.Csv(f"{pattern}_endtrial_*", columns=['typetag', 'value']) }
 
     @staticmethod
     def slice(pattern):
-        return { "Slice": _reader.Csv(f"{pattern}_octagonslice", columns=[
+        return { "Slice": _reader.Csv(f"{pattern}_octagonslice_*", columns=[
             'typetag',
             'wall_id',
             'r', 'g', 'b', 'a',
             'delay']) }
 
     @staticmethod
     def gratings_slice(pattern):
-        return { "GratingsSlice": _reader.Csv(f"{pattern}_octagongratingsslice", columns=[
+        return { "GratingsSlice": _reader.Csv(f"{pattern}_octagongratingsslice_*", columns=[
             'typetag',
             'wall_id',
             'contrast',
@@ -40,7 +40,7 @@ def gratings_slice(pattern):
 
     @staticmethod
     def poke(pattern):
-        return { "Poke": _reader.Csv(f"{pattern}_poke", columns=[
+        return { "Poke": _reader.Csv(f"{pattern}_poke_*", columns=[
             'typetag',
             'wall_id',
             'poke_id',
@@ -51,15 +51,15 @@ def poke(pattern):
 
     @staticmethod
     def response(pattern):
-        return { "Response": _reader.Csv(f"{pattern}_response", columns=[
+        return { "Response": _reader.Csv(f"{pattern}_response_*", columns=[
             'typetag',
             'wall_id',
             'poke_id',
             'response_time' ]) }
 
     @staticmethod
     def run_pre_trial_no_poke(pattern):
-        return { "RunPreTrialNoPoke": _reader.Csv(f"{pattern}_run_pre_no_poke", columns=[
+        return { "RunPreTrialNoPoke": _reader.Csv(f"{pattern}_run_pre_no_poke_*", columns=[
             'typetag',
             'wait_for_poke',
             'reward_iti',
@@ -69,94 +69,94 @@ def run_pre_trial_no_poke(pattern):
 
     @staticmethod
     def start_new_session(pattern):
-        return { "StartNewSession": _reader.Csv(f"{pattern}_startnewsession", columns=['typetag', 'path' ]) }
+        return { "StartNewSession": _reader.Csv(f"{pattern}_startnewsession_*", columns=['typetag', 'path' ]) }
 
 class TaskLogic:
     @staticmethod
     def trial_initiation(pattern):
-        return { "TrialInitiation": _reader.Harp(f"{pattern}_1", columns=['trial_type']) }
+        return { "TrialInitiation": _reader.Harp(f"{pattern}_1_*", columns=['trial_type']) }
 
     @staticmethod
     def response(pattern):
-        return { "Response": _reader.Harp(f"{pattern}_2", columns=['wall_id', 'poke_id']) }
+        return { "Response": _reader.Harp(f"{pattern}_2_*", columns=['wall_id', 'poke_id']) }
 
     @staticmethod
     def pre_trial(pattern):
-        return { "PreTrialState": _reader.Harp(f"{pattern}_3", columns=['state']) }
+        return { "PreTrialState": _reader.Harp(f"{pattern}_3_*", columns=['state']) }
 
     @staticmethod
     def inter_trial_interval(pattern):
-        return { "InterTrialInterval": _reader.Harp(f"{pattern}_4", columns=['state']) }
+        return { "InterTrialInterval": _reader.Harp(f"{pattern}_4_*", columns=['state']) }
 
     @staticmethod
     def slice_onset(pattern):
-        return { "SliceOnset": _reader.Harp(f"{pattern}_10", columns=['wall_id']) }
+        return { "SliceOnset": _reader.Harp(f"{pattern}_10_*", columns=['wall_id']) }
 
     @staticmethod
     def draw_background(pattern):
-        return { "DrawBackground": _reader.Harp(f"{pattern}_11", columns=['state']) }
+        return { "DrawBackground": _reader.Harp(f"{pattern}_11_*", columns=['state']) }
 
     @staticmethod
     def gratings_slice_onset(pattern):
-        return { "GratingsSliceOnset": _reader.Harp(f"{pattern}_12", columns=['wall_id']) }
+        return { "GratingsSliceOnset": _reader.Harp(f"{pattern}_12_*", columns=['wall_id']) }
 
 class Wall:
     @staticmethod
     def beam_break0(pattern):
-        return { "BeamBreak0": _reader.DigitalBitmask(f"{pattern}_32", 0x1, columns=['state']) }
+        return { "BeamBreak0": _reader.DigitalBitmask(f"{pattern}_32_*", 0x1, columns=['state']) }
 
     @staticmethod
     def beam_break1(pattern):
-        return { "BeamBreak1": _reader.DigitalBitmask(f"{pattern}_32", 0x2, columns=['state']) }
+        return { "BeamBreak1": _reader.DigitalBitmask(f"{pattern}_32_*", 0x2, columns=['state']) }
 
     @staticmethod
     def beam_break2(pattern):
-        return { "BeamBreak2": _reader.DigitalBitmask(f"{pattern}_32", 0x4, columns=['state']) }
+        return { "BeamBreak2": _reader.DigitalBitmask(f"{pattern}_32_*", 0x4, columns=['state']) }
 
     @staticmethod
     def set_led0(pattern):
-        return { "SetLed0": _reader.BitmaskEvent(f"{pattern}_34", 0x1, 'Set') }
+        return { "SetLed0": _reader.BitmaskEvent(f"{pattern}_34_*", 0x1, 'Set') }
 
     @staticmethod
     def set_led1(pattern):
-        return { "SetLed1": _reader.BitmaskEvent(f"{pattern}_34", 0x2, 'Set') }
+        return { "SetLed1": _reader.BitmaskEvent(f"{pattern}_34_*", 0x2, 'Set') }
 
     @staticmethod
     def set_led2(pattern):
-        return { "SetLed2": _reader.BitmaskEvent(f"{pattern}_34", 0x4, 'Set') }
+        return { "SetLed2": _reader.BitmaskEvent(f"{pattern}_34_*", 0x4, 'Set') }
 
     @staticmethod
     def set_valve0(pattern):
-        return { "SetValve0": _reader.BitmaskEvent(f"{pattern}_34", 0x8, 'Set') }
+        return { "SetValve0": _reader.BitmaskEvent(f"{pattern}_34_*", 0x8, 'Set') }
 
     @staticmethod
     def set_valve1(pattern):
-        return { "SetValve1": _reader.BitmaskEvent(f"{pattern}_34", 0x10, 'Set') }
+        return { "SetValve1": _reader.BitmaskEvent(f"{pattern}_34_*", 0x10, 'Set') }
 
     @staticmethod
     def set_valve2(pattern):
-        return { "SetValve2": _reader.BitmaskEvent(f"{pattern}_34", 0x20, 'Set') }
+        return { "SetValve2": _reader.BitmaskEvent(f"{pattern}_34_*", 0x20, 'Set') }
 
     @staticmethod
     def clear_led0(pattern):
-        return { "ClearLed0": _reader.BitmaskEvent(f"{pattern}_35", 0x1, 'Clear') }
+        return { "ClearLed0": _reader.BitmaskEvent(f"{pattern}_35_*", 0x1, 'Clear') }
 
     @staticmethod
     def clear_led1(pattern):
-        return { "ClearLed1": _reader.BitmaskEvent(f"{pattern}_35", 0x2, 'Clear') }
+        return { "ClearLed1": _reader.BitmaskEvent(f"{pattern}_35_*", 0x2, 'Clear') }
 
     @staticmethod
     def clear_led2(pattern):
-        return { "ClearLed2": _reader.BitmaskEvent(f"{pattern}_35", 0x4, 'Clear') }
+        return { "ClearLed2": _reader.BitmaskEvent(f"{pattern}_35_*", 0x4, 'Clear') }
 
     @staticmethod
     def clear_valve0(pattern):
-        return { "ClearValve0": _reader.BitmaskEvent(f"{pattern}_35", 0x8, 'Clear') }
+        return { "ClearValve0": _reader.BitmaskEvent(f"{pattern}_35_*", 0x8, 'Clear') }
 
     @staticmethod
     def clear_valve1(pattern):
-        return { "ClearValve1": _reader.BitmaskEvent(f"{pattern}_35", 0x10, 'Clear') }
+        return { "ClearValve1": _reader.BitmaskEvent(f"{pattern}_35_*", 0x10, 'Clear') }
 
     @staticmethod
     def clear_valve2(pattern):
-        return { "ClearValve2": _reader.BitmaskEvent(f"{pattern}_35", 0x20, 'Clear') }
+        return { "ClearValve2": _reader.BitmaskEvent(f"{pattern}_35_*", 0x20, 'Clear') }

pyproject.toml

@@ -92,3 +92,8 @@ exclude = '''
 [tool.isort]
 profile = "black"
 color_output = false
+
+[tool.pytest.ini_options]
+markers = [
+    "api",
+]

tests/data/nonmonotonic/2022-06-06T09-24-28/Metadata.yml

@@ -0,0 +1,11 @@
+{
+  "Workflow": "Experiment0.2.bonsai",
+  "Commit": "249cdc654af63e6959e64f7ff2c21f219cc912ea",
+  "Devices": {
+    "VideoController": {
+      "PortName": "COM3",
+      "GlobalTriggerFrequency": "50",
+      "LocalTriggerFrequency": "125"
+    }
+  }
+}