From 7a02b236656125a4afb67125d7518d306766a83c Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 3 Apr 2023 05:24:00 +0000 Subject: [PATCH 01/54] use aeon api to read metadata --- aeon/dj_pipeline/utils/load_metadata.py | 36 ++++++++++++++++++------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 9b7e701f..fcd3685a 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -1,13 +1,13 @@ +import datetime import pathlib import re -import datetime +import numpy as np import pandas as pd import yaml -from aeon.dj_pipeline import acquisition, lab, subject -from aeon.dj_pipeline import dict_to_uuid - +from aeon.dj_pipeline import acquisition, dict_to_uuid, lab, subject +from aeon.io import api as io_api _weight_scale_rate = 100 _weight_scale_nest = 1 @@ -32,14 +32,26 @@ def extract_epoch_metadata(experiment_name, metadata_yml_filepath): epoch_start = datetime.datetime.strptime( metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" ) - with open(metadata_yml_filepath, "r") as f: - experiment_setup = yaml.safe_load(f) - commit = experiment_setup.get("Commit", experiment_setup.get("Revision")) + experiment_setup: dict = ( + io_api.load( + str(metadata_yml_filepath.parent), + acquisition._device_schema_mapping[experiment_name].Metadata, + ) + .reset_index() + .to_dict("records")[0] + ) + + commit = experiment_setup.get("commit") + if isinstance(commit, float) and np.isnan(commit): + commit = experiment_setup["metadata"]["Revision"] + else: + commit = None + assert commit, f'Neither "Commit" nor "Revision" found in {metadata_yml_filepath}' return { "experiment_name": experiment_name, "epoch_start": epoch_start, - "bonsai_workflow": experiment_setup["Workflow"], + "bonsai_workflow": experiment_setup["workflow"], "commit": commit, "metadata": experiment_setup, "metadata_file_path": metadata_yml_filepath, @@ -61,8 +73,12 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): epoch_start = datetime.datetime.strptime( metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" ) - with open(metadata_yml_filepath, "r") as f: - experiment_setup = yaml.safe_load(f) + + experiment_setup = io_api.load( + str(metadata_yml_filepath.parent), + acquisition._device_schema_mapping[experiment_name].Metadata, + ) + experiment_key = {"experiment_name": experiment_name} # Check if there has been any changes in the arena setup # by comparing the "Commit" against the most immediate preceding epoch From f5f310e55c2523aac8083e30a13b5f72936622e1 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 01:28:09 +0000 Subject: [PATCH 02/54] add get_device_info to dataset.py --- aeon/schema/dataset.py | 95 ++++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index ae787456..f11504ac 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -24,34 +24,67 @@ ] ) -exp01 = DotMap([ - Device("SessionData", foraging.session), - Device("FrameTop", stream.video, stream.position), - Device("FrameEast", stream.video), - Device("FrameGate", stream.video), - Device("FrameNorth", stream.video), - Device("FramePatch1", stream.video), - Device("FramePatch2", stream.video), - Device("FrameSouth", stream.video), - Device("FrameWest", stream.video), - Device("Patch1", foraging.depletionFunction, stream.encoder, foraging.feeder), - Device("Patch2", foraging.depletionFunction, stream.encoder, foraging.feeder) -]) - -octagon01 = DotMap([ - Device("Metadata", stream.metadata), - Device("CameraTop", stream.video, stream.position), - Device("CameraColorTop", stream.video), - Device("ExperimentalMetadata", stream.subject_state), - Device("Photodiode", octagon.photodiode), - Device("OSC", octagon.OSC), - Device("TaskLogic", octagon.TaskLogic), - Device("Wall1", octagon.Wall), - Device("Wall2", octagon.Wall), - Device("Wall3", octagon.Wall), - Device("Wall4", octagon.Wall), - Device("Wall5", octagon.Wall), - Device("Wall6", octagon.Wall), - Device("Wall7", octagon.Wall), - Device("Wall8", octagon.Wall) -]) +exp01 = DotMap( + [ + Device("SessionData", foraging.session), + Device("FrameTop", stream.video, stream.position), + Device("FrameEast", stream.video), + Device("FrameGate", stream.video), + Device("FrameNorth", stream.video), + Device("FramePatch1", stream.video), + Device("FramePatch2", stream.video), + Device("FrameSouth", stream.video), + Device("FrameWest", stream.video), + Device("Patch1", foraging.depletionFunction, stream.encoder, foraging.feeder), + Device("Patch2", foraging.depletionFunction, stream.encoder, foraging.feeder), + ] +) + +octagon01 = DotMap( + [ + Device("Metadata", stream.metadata), + Device("CameraTop", stream.video, stream.position), + Device("CameraColorTop", stream.video), + Device("ExperimentalMetadata", stream.subject_state), + Device("Photodiode", octagon.photodiode), + Device("OSC", octagon.OSC), + Device("TaskLogic", octagon.TaskLogic), + Device("Wall1", octagon.Wall), + Device("Wall2", octagon.Wall), + Device("Wall3", octagon.Wall), + Device("Wall4", octagon.Wall), + Device("Wall5", octagon.Wall), + Device("Wall6", octagon.Wall), + Device("Wall7", octagon.Wall), + Device("Wall8", octagon.Wall), + ] +) + + +def get_device_info(schema: DotMap) -> dict[dict]: + """ + Read from the above DotMap object and returns device dictionary {device_name: {stream_name: reader}} + """ + from collections import defaultdict + + device_info = {} + + for device_name in schema: + if not device_name.startswith("_"): + device_info[device_name] = defaultdict(list) + if isinstance(schema[device_name], DotMap): + for stream_type in schema[device_name].keys(): + if schema[device_name][stream_type].__class__.__module__ in [ + "aeon.io.reader", + "aeon.schema.foraging", + "aeon.schema.octagon", + ]: + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["reader"].append( + schema[device_name][stream_type].__class__ + ) + else: + stream_type = schema[device_name].__class__.__name__ + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["reader"].append(schema[device_name].__class__) + return device_info From a522d74f7a579aa313c68e5c6a34813180fdc90c Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 16:16:09 +0000 Subject: [PATCH 03/54] add pattern info to device_info --- aeon/schema/dataset.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index f11504ac..aac291e7 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -1,3 +1,5 @@ +from pathlib import Path + from dotmap import DotMap import aeon.schema.core as stream @@ -63,10 +65,28 @@ def get_device_info(schema: DotMap) -> dict[dict]: """ - Read from the above DotMap object and returns device dictionary {device_name: {stream_name: reader}} + Read from the above DotMap object and returns a device dictionary as the following. + + Args: + schema (DotMap): DotMap object (e.g., exp02) + + e.g. {'CameraTop': + {'stream_type': ['Video', 'Position', 'Region'], + 'reader': [ + aeon.io.reader.Video, + aeon.io.reader.Position, + aeon.schema.foraging._RegionReader + ], + 'pattern': ['{pattern}', '{pattern}_200', '{pattern}_201'] + } + } """ + import json from collections import defaultdict + schema_json = json.dumps(schema, default=lambda o: o.__dict__, indent=4) + schema_dict = json.loads(schema_json) + device_info = {} for device_name in schema: @@ -87,4 +107,17 @@ def get_device_info(schema: DotMap) -> dict[dict]: stream_type = schema[device_name].__class__.__name__ device_info[device_name]["stream_type"].append(stream_type) device_info[device_name]["reader"].append(schema[device_name].__class__) + + """Add a 'pattern' key with a value of e.g., ['{pattern}_State', '{pattern}_90', '{pattern}_32','{pattern}_35']""" + + for device_name in device_info: + if pattern := schema_dict[device_name].get("pattern"): + pattern = pattern.replace(device_name, "{pattern}") + device_info[device_name]["pattern"].append(pattern) + else: + for stream_type in device_info[device_name]["stream_type"]: + pattern = schema_dict[device_name][stream_type]["pattern"] + pattern = pattern.replace(device_name, "{pattern}") + device_info[device_name]["pattern"].append(pattern) + return device_info From bdde45bc999682e18b31e8cb97b3d9d9a5eb2555 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 16:17:04 +0000 Subject: [PATCH 04/54] add add_device_type to device_info --- aeon/schema/dataset.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index aac291e7..d04e4f6e 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -121,3 +121,40 @@ def get_device_info(schema: DotMap) -> dict[dict]: device_info[device_name]["pattern"].append(pattern) return device_info + + +def add_device_type(schema: DotMap, metadata_yml_filepath: Path): + """Update device_info with device_type based on metadata.yml. + + Args: + schema (DotMap): DotMap object (e.g., exp02) + metadata_yml_filepath (Path): Path to metadata.yml. + + Returns: + device_info (dict): Updated device_info. + """ + from aeon.io import api + + meta_data = ( + api.load( + str(metadata_yml_filepath.parent), + schema.Metadata, + ) + .reset_index() + .to_dict("records")[0]["metadata"] + ) + + # Get device_type_mapper based on metadata.yml {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} + device_type_mapper = {} + for item in meta_data.Devices: + device_type_mapper[item.Name] = item.Type + + device_info = { + device_name: { + "device_type": device_type_mapper.get(device_name, None), + **device_info[device_name], + } + for device_name in device_info + } + + return device_info From 4af7def81d2f1c5456afc0c329e262b06a33a3d2 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 16:20:20 +0000 Subject: [PATCH 05/54] add presocial dataset schema --- aeon/schema/dataset.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index d04e4f6e..e87c896b 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -5,6 +5,7 @@ import aeon.schema.core as stream import aeon.schema.foraging as foraging import aeon.schema.octagon as octagon +from aeon.io import reader from aeon.io.device import Device exp02 = DotMap( @@ -62,6 +63,20 @@ ] ) +presocial = exp02 +presocial.Patch1.BeamBreak = reader.BitmaskEvent( + pattern="Patch1_32", value=0x22, tag="BeamBroken" +) +presocial.Patch2.BeamBreak = reader.BitmaskEvent( + pattern="Patch2_32", value=0x22, tag="BeamBroken" +) +presocial.Patch1.DeliverPellet = reader.BitmaskEvent( + pattern="Patch1_35", value=0x1, tag="TriggeredPellet" +) +presocial.Patch2.DeliverPellet = reader.BitmaskEvent( + pattern="Patch2_35", value=0x1, tag="TriggeredPellet" +) + def get_device_info(schema: DotMap) -> dict[dict]: """ From c0528cfaad020da5e7439c8b386b85456d3731fd Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 17:04:26 +0000 Subject: [PATCH 06/54] add presocial device mapping --- aeon/dj_pipeline/acquisition.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index e180382c..31d286c0 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -25,6 +25,7 @@ "social0-r1": "FrameTop", "exp0.2-r0": "CameraTop", "oct1.0-r0": "CameraTop", + "presocial0.1-a2": "CameraTop", } _device_schema_mapping = { @@ -32,6 +33,7 @@ "social0-r1": aeon_schema.exp01, "exp0.2-r0": aeon_schema.exp02, "oct1.0-r0": aeon_schema.octagon01, + "presocial0.1-a2": aeon_schema.presocial, } From 99b6478b66b76a7aeac53d6a2deb12b16f85890e Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 17:04:52 +0000 Subject: [PATCH 07/54] add kwargs instead of pattern --- aeon/schema/dataset.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index e87c896b..04ae6f51 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -123,17 +123,28 @@ def get_device_info(schema: DotMap) -> dict[dict]: device_info[device_name]["stream_type"].append(stream_type) device_info[device_name]["reader"].append(schema[device_name].__class__) - """Add a 'pattern' key with a value of e.g., ['{pattern}_State', '{pattern}_90', '{pattern}_32','{pattern}_35']""" + """Add a kwargs such as pattern, columns, extension, dtype + e.g., {'pattern': '{pattern}_SubjectState', + 'columns': ['id', 'weight', 'event'], + 'extension': 'csv', + 'dtype': None}""" + # Add a kwargs that includes pattern for device_name in device_info: if pattern := schema_dict[device_name].get("pattern"): - pattern = pattern.replace(device_name, "{pattern}") - device_info[device_name]["pattern"].append(pattern) + schema_dict[device_name]["pattern"] = pattern.replace( + device_name, "{pattern}" + ) + device_info[device_name]["kwargs"].append(schema_dict[device_name]) else: for stream_type in device_info[device_name]["stream_type"]: pattern = schema_dict[device_name][stream_type]["pattern"] - pattern = pattern.replace(device_name, "{pattern}") - device_info[device_name]["pattern"].append(pattern) + schema_dict[device_name][stream_type]["pattern"] = pattern.replace( + device_name, "{pattern}" + ) + device_info[device_name]["kwargs"].append( + schema_dict[device_name][stream_type] + ) return device_info From c39c9e6d43bd5ae31fb7e2ce4fbad0ce79c9ce96 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 18:47:58 +0000 Subject: [PATCH 08/54] minor formatting --- aeon/dj_pipeline/lab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aeon/dj_pipeline/lab.py b/aeon/dj_pipeline/lab.py index 9c193483..07ea8b16 100644 --- a/aeon/dj_pipeline/lab.py +++ b/aeon/dj_pipeline/lab.py @@ -16,7 +16,7 @@ class Colony(dj.Lookup): --- reference_weight=null : float sex='U' : enum('M', 'F', 'U') - subject_birth_date=null : date # date of birth + subject_birth_date=null : date # date of birth note='' : varchar(1024) """ From fb9bc01dc76687fc83f5984cd8d2b24bb1fe60d9 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 19:18:31 +0000 Subject: [PATCH 09/54] add device stream hash --- aeon/schema/dataset.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index 04ae6f51..45a11753 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -99,6 +99,8 @@ def get_device_info(schema: DotMap) -> dict[dict]: import json from collections import defaultdict + from aeon.dj_pipeline import dict_to_uuid + schema_json = json.dumps(schema, default=lambda o: o.__dict__, indent=4) schema_dict = json.loads(schema_json) @@ -123,27 +125,40 @@ def get_device_info(schema: DotMap) -> dict[dict]: device_info[device_name]["stream_type"].append(stream_type) device_info[device_name]["reader"].append(schema[device_name].__class__) - """Add a kwargs such as pattern, columns, extension, dtype + """Add a kwargs such as pattern, columns, extension, dtype and hash e.g., {'pattern': '{pattern}_SubjectState', 'columns': ['id', 'weight', 'event'], 'extension': 'csv', 'dtype': None}""" - - # Add a kwargs that includes pattern for device_name in device_info: if pattern := schema_dict[device_name].get("pattern"): schema_dict[device_name]["pattern"] = pattern.replace( device_name, "{pattern}" ) - device_info[device_name]["kwargs"].append(schema_dict[device_name]) + + # Add stream_reader_kwargs + kwargs = schema_dict[device_name] + device_info[device_name]["stream_reader_kwargs"].append(kwargs) + stream_reader = device_info[device_name]["stream_reader"] + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid({**kwargs, "stream_reader": stream_reader}) + ) + else: for stream_type in device_info[device_name]["stream_type"]: pattern = schema_dict[device_name][stream_type]["pattern"] schema_dict[device_name][stream_type]["pattern"] = pattern.replace( device_name, "{pattern}" ) - device_info[device_name]["kwargs"].append( - schema_dict[device_name][stream_type] + # Add stream_reader_kwargs + kwargs = schema_dict[device_name][stream_type] + device_info[device_name]["stream_reader_kwargs"].append(kwargs) + stream_ind = device_info[device_name]["stream_type"].index(stream_type) + stream_reader = device_info[device_name]["stream_reader"][stream_ind] + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid({**kwargs, "stream_reader": stream_reader}) ) return device_info From 6cd64eeda5993429a75b9ddc516e1fd0cdd07a21 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 19:41:53 +0000 Subject: [PATCH 10/54] move functions to streams.py --- aeon/dj_pipeline/streams.py | 245 ++++++++++++++++++++++++++---------- aeon/schema/dataset.py | 125 ------------------ 2 files changed, 176 insertions(+), 194 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 4a2992a0..9cf00dc9 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -2,9 +2,11 @@ import re from collections import defaultdict, namedtuple from functools import cached_property +from pathlib import Path import datajoint as dj import pandas as pd +from dotmap import DotMap import aeon import aeon.schema.core as stream @@ -21,11 +23,11 @@ schema = dj.schema(schema_name) -# __all__ = [ -# "StreamType", -# "DeviceType", -# "Device", -# ] +__all__ = [ + "StreamType", + "DeviceType", + "Device", +] # Read from this list of device configurations @@ -85,63 +87,32 @@ class StreamType(dj.Lookup): """ definition = """ # Catalog of all stream types used across Project Aeon - stream_type: varchar(20) + stream_type : varchar(20) --- - stream_reader: varchar(256) # name of the reader class found in `aeon_mecha` package (e.g. aeon.io.reader.Video) - stream_reader_kwargs: longblob # keyword arguments to instantiate the reader class - stream_description='': varchar(256) - stream_hash: uuid # hash of dict(stream_reader_kwargs, stream_reader=stream_reader) + stream_reader : varchar(256) # name of the reader class found in `aeon_mecha` package (e.g. aeon.io.reader.Video) + stream_reader_kwargs : longblob # keyword arguments to instantiate the reader class + stream_description='': varchar(256) + stream_hash : uuid # hash of dict(stream_reader_kwargs, stream_reader=stream_reader) unique index (stream_hash) """ - @staticmethod - def get_stream_entries(device_streams: tuple, pattern="{pattern}") -> dict: - - composite = aeon.io.device.compositeStream(pattern, *device_streams) - stream_entries = [] - for stream_name, stream_reader in composite.items(): - if stream_name == pattern: - stream_name = stream_reader.__class__.__name__ - entry = { - "stream_type": stream_name, - "stream_reader": f"{stream_reader.__module__}.{stream_reader.__class__.__name__}", - "stream_reader_kwargs": { - k: v - for k, v in vars(stream_reader).items() - if k - in inspect.signature(stream_reader.__class__.__init__).parameters - }, - } - entry["stream_hash"] = dict_to_uuid( - { - **entry["stream_reader_kwargs"], - "stream_reader": entry["stream_reader"], - } - ) - stream_entries.append(entry) - - return stream_entries - @classmethod - def insert_streams(cls, device_configs: list[namedtuple] = []): + def insert_streams(cls, schema: DotMap): - if not device_configs: - device_configs = get_device_configs() + stream_entries = get_stream_entries(schema) - for device in device_configs: - stream_entries = cls.get_stream_entries(device.streams) - for entry in stream_entries: - q_param = cls & {"stream_hash": entry["stream_hash"]} - if q_param: # If the specified stream type already exists - pname = q_param.fetch1("stream_type") - if pname != entry["stream_type"]: - # If the existed stream type does not have the same name: - # human error, trying to add the same content with different name - raise dj.DataJointError( - f"The specified stream type already exists - name: {pname}" - ) + for entry in stream_entries: + q_param = cls & {"stream_hash": entry["stream_hash"]} + if q_param: # If the specified stream type already exists + pname = q_param.fetch1("stream_type") + if pname != entry["stream_type"]: + # If the existed stream type does not have the same name: + # human error, trying to add the same content with different name + raise dj.DataJointError( + f"The specified stream type already exists - name: {pname}" + ) - cls.insert(stream_entries, skip_duplicates=True) + cls.insert(stream_entries, skip_duplicates=True) @schema @@ -423,25 +394,161 @@ def create_device_stream_tables(self): self._schema.activate(schema_name) -# Main function -def main(): +def get_device_info(schema: DotMap) -> dict[dict]: + """ + Read from the above DotMap object and returns a device dictionary as the following. + + Args: + schema (DotMap): DotMap object (e.g., exp02, octagon01) + + Returns: + device_info (dict[dict]): A dictionary of device information + + e.g. {'CameraTop': + {'stream_type': ['Video', 'Position', 'Region'], + 'reader': [ + aeon.io.reader.Video, + aeon.io.reader.Position, + aeon.schema.foraging._RegionReader + ], + 'pattern': ['{pattern}', '{pattern}_200', '{pattern}_201'] + } + } + """ + import json + from collections import defaultdict + + from aeon.dj_pipeline import dict_to_uuid + + schema_json = json.dumps(schema, default=lambda o: o.__dict__, indent=4) + schema_dict = json.loads(schema_json) + + device_info = {} + + for device_name in schema: + if not device_name.startswith("_"): + device_info[device_name] = defaultdict(list) + if isinstance(schema[device_name], DotMap): + for stream_type in schema[device_name].keys(): + if schema[device_name][stream_type].__class__.__module__ in [ + "aeon.io.reader", + "aeon.schema.foraging", + "aeon.schema.octagon", + ]: + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["reader"].append( + schema[device_name][stream_type].__class__ + ) + else: + stream_type = schema[device_name].__class__.__name__ + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["reader"].append(schema[device_name].__class__) + + """Add a kwargs such as pattern, columns, extension, dtype and hash + e.g., {'pattern': '{pattern}_SubjectState', + 'columns': ['id', 'weight', 'event'], + 'extension': 'csv', + 'dtype': None}""" + for device_name in device_info: + if pattern := schema_dict[device_name].get("pattern"): + schema_dict[device_name]["pattern"] = pattern.replace( + device_name, "{pattern}" + ) - # Populate StreamType - StreamType.insert_streams() + # Add stream_reader_kwargs + kwargs = schema_dict[device_name] + device_info[device_name]["stream_reader_kwargs"].append(kwargs) + stream_reader = device_info[device_name]["stream_reader"] + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid({**kwargs, "stream_reader": stream_reader}) + ) - # Populate DeviceType - DeviceType.insert_devices() + else: + for stream_type in device_info[device_name]["stream_type"]: + pattern = schema_dict[device_name][stream_type]["pattern"] + schema_dict[device_name][stream_type]["pattern"] = pattern.replace( + device_name, "{pattern}" + ) + # Add stream_reader_kwargs + kwargs = schema_dict[device_name][stream_type] + device_info[device_name]["stream_reader_kwargs"].append(kwargs) + stream_ind = device_info[device_name]["stream_type"].index(stream_type) + stream_reader = device_info[device_name]["stream_reader"][stream_ind] + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid({**kwargs, "stream_reader": stream_reader}) + ) - # Populate device tables - tbmg = DeviceTableManager(context=inspect.currentframe().f_back.f_locals) + return device_info - # # List all tables - # tbmg.device_tables - # tbmg.device_stream_tables - # Create device & device stream tables - tbmg.create_device_tables() - tbmg.create_device_stream_tables() +def add_device_type(schema: DotMap, metadata_yml_filepath: Path): + """Update device_info with device_type based on metadata.yml. + Args: + schema (DotMap): DotMap object (e.g., exp02) + metadata_yml_filepath (Path): Path to metadata.yml. -# main() + Returns: + device_info (dict): Updated device_info. + """ + from aeon.io import api + + meta_data = ( + api.load( + str(metadata_yml_filepath.parent), + schema.Metadata, + ) + .reset_index() + .to_dict("records")[0]["metadata"] + ) + + # Get device_type_mapper based on metadata.yml {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} + device_type_mapper = {} + for item in meta_data.Devices: + device_type_mapper[item.Name] = item.Type + + device_info = { + device_name: { + "device_type": device_type_mapper.get(device_name, None), + **device_info[device_name], + } + for device_name in device_info + } + + return device_info + + +def get_stream_entries(schema: DotMap) -> list[dict]: + """Returns a list of dictionaries containing the stream entries for a given device, + + Args: + schema (DotMap): DotMap object (e.g., exp02, octagon01) + + Returns: + stream_info (list[dict]): list of dictionaries containing the stream entries for a given device, + + e.g. {'stream_type': 'EnvironmentState', + 'stream_reader': aeon.io.reader.Csv, + 'stream_reader_kwargs': {'pattern': '{pattern}_EnvironmentState', + 'columns': ['state'], + 'extension': 'csv', + 'dtype': None} + """ + device_info = get_device_info(schema) + return [ + { + "stream_type": stream_type, + "stream_reader": stream_reader, + "stream_reader_kwargs": stream_reader_kwargs, + "stream_hash": stream_hash, + } + for stream_info in device_info.values() + for stream_type, stream_reader, stream_reader_kwargs, stream_hash in zip( + stream_info["stream_type"], + stream_info["stream_reader"], + stream_info["stream_reader_kwargs"], + stream_info["stream_hash"], + ) + ] diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index 45a11753..5aae992f 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -1,5 +1,3 @@ -from pathlib import Path - from dotmap import DotMap import aeon.schema.core as stream @@ -76,126 +74,3 @@ presocial.Patch2.DeliverPellet = reader.BitmaskEvent( pattern="Patch2_35", value=0x1, tag="TriggeredPellet" ) - - -def get_device_info(schema: DotMap) -> dict[dict]: - """ - Read from the above DotMap object and returns a device dictionary as the following. - - Args: - schema (DotMap): DotMap object (e.g., exp02) - - e.g. {'CameraTop': - {'stream_type': ['Video', 'Position', 'Region'], - 'reader': [ - aeon.io.reader.Video, - aeon.io.reader.Position, - aeon.schema.foraging._RegionReader - ], - 'pattern': ['{pattern}', '{pattern}_200', '{pattern}_201'] - } - } - """ - import json - from collections import defaultdict - - from aeon.dj_pipeline import dict_to_uuid - - schema_json = json.dumps(schema, default=lambda o: o.__dict__, indent=4) - schema_dict = json.loads(schema_json) - - device_info = {} - - for device_name in schema: - if not device_name.startswith("_"): - device_info[device_name] = defaultdict(list) - if isinstance(schema[device_name], DotMap): - for stream_type in schema[device_name].keys(): - if schema[device_name][stream_type].__class__.__module__ in [ - "aeon.io.reader", - "aeon.schema.foraging", - "aeon.schema.octagon", - ]: - device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["reader"].append( - schema[device_name][stream_type].__class__ - ) - else: - stream_type = schema[device_name].__class__.__name__ - device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["reader"].append(schema[device_name].__class__) - - """Add a kwargs such as pattern, columns, extension, dtype and hash - e.g., {'pattern': '{pattern}_SubjectState', - 'columns': ['id', 'weight', 'event'], - 'extension': 'csv', - 'dtype': None}""" - for device_name in device_info: - if pattern := schema_dict[device_name].get("pattern"): - schema_dict[device_name]["pattern"] = pattern.replace( - device_name, "{pattern}" - ) - - # Add stream_reader_kwargs - kwargs = schema_dict[device_name] - device_info[device_name]["stream_reader_kwargs"].append(kwargs) - stream_reader = device_info[device_name]["stream_reader"] - # Add hash - device_info[device_name]["stream_hash"].append( - dict_to_uuid({**kwargs, "stream_reader": stream_reader}) - ) - - else: - for stream_type in device_info[device_name]["stream_type"]: - pattern = schema_dict[device_name][stream_type]["pattern"] - schema_dict[device_name][stream_type]["pattern"] = pattern.replace( - device_name, "{pattern}" - ) - # Add stream_reader_kwargs - kwargs = schema_dict[device_name][stream_type] - device_info[device_name]["stream_reader_kwargs"].append(kwargs) - stream_ind = device_info[device_name]["stream_type"].index(stream_type) - stream_reader = device_info[device_name]["stream_reader"][stream_ind] - # Add hash - device_info[device_name]["stream_hash"].append( - dict_to_uuid({**kwargs, "stream_reader": stream_reader}) - ) - - return device_info - - -def add_device_type(schema: DotMap, metadata_yml_filepath: Path): - """Update device_info with device_type based on metadata.yml. - - Args: - schema (DotMap): DotMap object (e.g., exp02) - metadata_yml_filepath (Path): Path to metadata.yml. - - Returns: - device_info (dict): Updated device_info. - """ - from aeon.io import api - - meta_data = ( - api.load( - str(metadata_yml_filepath.parent), - schema.Metadata, - ) - .reset_index() - .to_dict("records")[0]["metadata"] - ) - - # Get device_type_mapper based on metadata.yml {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} - device_type_mapper = {} - for item in meta_data.Devices: - device_type_mapper[item.Name] = item.Type - - device_info = { - device_name: { - "device_type": device_type_mapper.get(device_name, None), - **device_info[device_name], - } - for device_name in device_info - } - - return device_info From 2f81d6813409ea2e63273e4bfefc4167c5297d23 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 21:15:17 +0000 Subject: [PATCH 11/54] add insert_device_type classmethod --- aeon/dj_pipeline/streams.py | 58 ++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 9cf00dc9..10cecb4b 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -134,29 +134,39 @@ class Stream(dj.Part): """ @classmethod - def insert_devices(cls, device_configs: list[namedtuple] = []): - if not device_configs: - device_configs = get_device_configs() - for device in device_configs: - stream_entries = StreamType.get_stream_entries(device.streams) - with cls.connection.transaction: - cls.insert1( - { - "device_type": device.type, - "device_description": device.desc, - }, - skip_duplicates=True, - ) - cls.Stream.insert( - [ + def insert_device_types(cls, schema: DotMap, metadata_yml_filepath: Path): + """Use dataset.schema and metadata.yml to insert device types and streams. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml.""" + device_info = get_device_info(schema) + device_type_mapper = get_device_type(schema, metadata_yml_filepath) + + device_info = { + device_name: { + "device_type": device_type_mapper.get(device_name, None), + **device_info[device_name], + } + for device_name in device_info + } + + for device_name, info in device_info.items(): + if info["device_type"]: + with cls.connections.transaction: + cls.insert1( { - "device_type": device.type, - "stream_type": e["stream_type"], - } - for e in stream_entries - ], - skip_duplicates=True, - ) + "device_type": info["device_type"], + "device_description": "", + }, + skip_duplicates=True, + ) + cls.Stream.insert( + [ + { + "device_type": info["device_type"], + "stream_type": e, + } + for e in info["stream_type"] + ], + skip_duplicates=True, + ) @schema @@ -483,8 +493,8 @@ def get_device_info(schema: DotMap) -> dict[dict]: return device_info -def add_device_type(schema: DotMap, metadata_yml_filepath: Path): - """Update device_info with device_type based on metadata.yml. +def get_device_type(schema: DotMap, metadata_yml_filepath: Path): + """Returns a mapping dictionary between device name and device type based on the dataset schema and metadata.yml from the experiment. Args: schema (DotMap): DotMap object (e.g., exp02) From edcf89e50069f45ce7a244107353dba88a8af678 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 21:43:28 +0000 Subject: [PATCH 12/54] add get_device_mapper --- aeon/dj_pipeline/streams.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 10cecb4b..ccd2bac8 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -181,13 +181,6 @@ class Device(dj.Lookup): ## --------- Helper functions & classes --------- ## -def get_device_configs(device_configs=DEVICE_CONFIGS) -> list[namedtuple]: - """Returns a list of device configurations from DEVICE_CONFIGS""" - - device = namedtuple("device", "type desc streams") - return [device._make(c) for c in device_configs] - - def get_device_template(device_type): """Returns table class template for ExperimentDevice""" device_title = device_type @@ -493,7 +486,7 @@ def get_device_info(schema: DotMap) -> dict[dict]: return device_info -def get_device_type(schema: DotMap, metadata_yml_filepath: Path): +def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): """Returns a mapping dictionary between device name and device type based on the dataset schema and metadata.yml from the experiment. Args: @@ -501,7 +494,8 @@ def get_device_type(schema: DotMap, metadata_yml_filepath: Path): metadata_yml_filepath (Path): Path to metadata.yml. Returns: - device_info (dict): Updated device_info. + device_type_mapper (dict): {"device_name", "device_type"} + e.g. {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} """ from aeon.io import api @@ -514,20 +508,12 @@ def get_device_type(schema: DotMap, metadata_yml_filepath: Path): .to_dict("records")[0]["metadata"] ) - # Get device_type_mapper based on metadata.yml {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} + # Get device_type_mapper based on metadata.yml device_type_mapper = {} for item in meta_data.Devices: device_type_mapper[item.Name] = item.Type - device_info = { - device_name: { - "device_type": device_type_mapper.get(device_name, None), - **device_info[device_name], - } - for device_name in device_info - } - - return device_info + return device_type_mapper def get_stream_entries(schema: DotMap) -> list[dict]: From 49fa9d3b12cc1510c7053736ecf9520c8f058122 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Apr 2023 22:34:41 +0000 Subject: [PATCH 13/54] remove DEVICE_CONFIGS from streamsm.py --- aeon/dj_pipeline/streams.py | 62 +++---------------------------------- 1 file changed, 4 insertions(+), 58 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index ccd2bac8..de2cc803 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -1,6 +1,7 @@ import inspect +import json import re -from collections import defaultdict, namedtuple +from collections import defaultdict from functools import cached_property from pathlib import Path @@ -9,9 +10,6 @@ from dotmap import DotMap import aeon -import aeon.schema.core as stream -import aeon.schema.foraging as foraging -import aeon.schema.octagon as octagon from aeon.dj_pipeline import acquisition, dict_to_uuid, get_schema_name from aeon.io import api as io_api @@ -30,53 +28,6 @@ ] -# Read from this list of device configurations -# (device_type, description, streams) -DEVICE_CONFIGS = [ - ( - "Camera", - "Camera device", - (stream.video, stream.position, foraging.region), - ), - ("Metadata", "Metadata", (stream.metadata,)), - ( - "ExperimentalMetadata", - "ExperimentalMetadata", - (stream.environment, stream.messageLog), - ), - ( - "NestScale", - "Weight scale at nest", - (foraging.weight,), - ), - ( - "FoodPatch", - "Food patch", - (foraging.patch,), - ), - ( - "Photodiode", - "Photodiode", - (octagon.photodiode,), - ), - ( - "OSC", - "OSC", - (octagon.OSC,), - ), - ( - "TaskLogic", - "TaskLogic", - (octagon.TaskLogic,), - ), - ( - "Wall", - "Wall", - (octagon.Wall,), - ), -] - - @schema class StreamType(dj.Lookup): """ @@ -137,7 +88,7 @@ class Stream(dj.Part): def insert_device_types(cls, schema: DotMap, metadata_yml_filepath: Path): """Use dataset.schema and metadata.yml to insert device types and streams. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml.""" device_info = get_device_info(schema) - device_type_mapper = get_device_type(schema, metadata_yml_filepath) + device_type_mapper = get_device_mapper(schema, metadata_yml_filepath) device_info = { device_name: { @@ -418,12 +369,7 @@ def get_device_info(schema: DotMap) -> dict[dict]: } } """ - import json - from collections import defaultdict - - from aeon.dj_pipeline import dict_to_uuid - - schema_json = json.dumps(schema, default=lambda o: o.__dict__, indent=4) + schema_json = json.dumps(schema, default=lambda x: x.__dict__, indent=4) schema_dict = json.loads(schema_json) device_info = {} From 2e665eb794f5ff1fa1e1eaf9b50e32f25f9190ca Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 10 Apr 2023 17:56:42 +0000 Subject: [PATCH 14/54] create_presocial exp --- .../create_experiments/create_presocial_a2.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 aeon/dj_pipeline/create_experiments/create_presocial_a2.py diff --git a/aeon/dj_pipeline/create_experiments/create_presocial_a2.py b/aeon/dj_pipeline/create_experiments/create_presocial_a2.py new file mode 100644 index 00000000..74d09a5c --- /dev/null +++ b/aeon/dj_pipeline/create_experiments/create_presocial_a2.py @@ -0,0 +1,61 @@ +from aeon.dj_pipeline import acquisition, lab, subject + +experiment_type = "presocial" +experiment_name = "presocial0.1-a2" # AEON2 acquisition computer +location = "4th floor" + + +def create_new_experiment(): + + lab.Location.insert1({"lab": "SWC", "location": location}, skip_duplicates=True) + + acquisition.ExperimentType.insert1( + {"experiment_type": experiment_type}, skip_duplicates=True + ) + + acquisition.Experiment.insert1( + { + "experiment_name": experiment_name, + "experiment_start_time": "2023-02-25 00:00:00", + "experiment_description": "presocial experiment 0.1 in aeon2", + "arena_name": "circle-2m", + "lab": "SWC", + "location": location, + "experiment_type": experiment_type, + }, + skip_duplicates=True, + ) + + acquisition.Experiment.Subject.insert( + [ + {"experiment_name": experiment_name, "subject": s} + for s in subject.Subject.fetch("subject") + ], + skip_duplicates=True, + ) + + acquisition.Experiment.Directory.insert( + [ + { + "experiment_name": experiment_name, + "repository_name": "ceph_aeon", + "directory_type": "raw", + "directory_path": "aeon/data/raw/AEON2/presocial0.1", + }, + { + "experiment_name": experiment_name, + "repository_name": "ceph_aeon", + "directory_type": "quality-control", + "directory_path": "aeon/data/qc/AEON2/presocial0.1", + }, + ], + skip_duplicates=True, + ) + + +def main(): + create_new_experiment() + + +if __name__ == "__main__": + main() From 0e6dc009bda12db2d6435743d1685cfbceaf2612 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 10 Apr 2023 18:54:58 +0000 Subject: [PATCH 15/54] fix load_metadata to use aeon api --- aeon/dj_pipeline/acquisition.py | 4 +- aeon/dj_pipeline/utils/load_metadata.py | 467 ++++++++++++------------ 2 files changed, 229 insertions(+), 242 deletions(-) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index 31d286c0..f28c1ef4 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -12,7 +12,7 @@ from . import get_schema_name, lab, subject from .utils import paths -from .utils.load_metadata import extract_epoch_metadata, ingest_epoch_metadata +from .utils.load_metadata import extract_epoch_config, ingest_epoch_metadata logger = dj.logger schema = dj.schema(get_schema_name("acquisition")) @@ -303,7 +303,7 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): if experiment_name != "exp0.1-r0": metadata_yml_filepath = epoch_dir / "Metadata.yml" if metadata_yml_filepath.exists(): - epoch_config = extract_epoch_metadata( + epoch_config = extract_epoch_config( experiment_name, metadata_yml_filepath ) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index fcd3685a..30627167 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -1,10 +1,10 @@ import datetime +import json import pathlib import re import numpy as np import pandas as pd -import yaml from aeon.dj_pipeline import acquisition, dict_to_uuid, lab, subject from aeon.io import api as io_api @@ -27,12 +27,21 @@ def ingest_subject(colony_csv_path: pathlib.Path = _colony_csv_path) -> None: ) -def extract_epoch_metadata(experiment_name, metadata_yml_filepath): +def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> dict: + """Parse experiment metadata YAML file and extract epoch configuration. + + Args: + experiment_name (str) + metadata_yml_filepath (str) + + Returns: + dict: epoch_config [dict] + """ metadata_yml_filepath = pathlib.Path(metadata_yml_filepath) epoch_start = datetime.datetime.strptime( metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" ) - experiment_setup: dict = ( + epoch_config: dict = ( io_api.load( str(metadata_yml_filepath.parent), acquisition._device_schema_mapping[experiment_name].Metadata, @@ -41,19 +50,26 @@ def extract_epoch_metadata(experiment_name, metadata_yml_filepath): .to_dict("records")[0] ) - commit = experiment_setup.get("commit") + commit = epoch_config.get("commit") if isinstance(commit, float) and np.isnan(commit): - commit = experiment_setup["metadata"]["Revision"] - else: - commit = None + commit = epoch_config["metadata"]["Revision"] assert commit, f'Neither "Commit" nor "Revision" found in {metadata_yml_filepath}' + + devices: dict[str, dict] = json.loads( + json.dumps( + epoch_config["metadata"]["Devices"], default=lambda x: x.__dict__, indent=4 + ) + ) + + # devices: dict = {d.pop("Name"): d for d in devices} # {deivce_name: device_config} + return { "experiment_name": experiment_name, "epoch_start": epoch_start, - "bonsai_workflow": experiment_setup["workflow"], + "bonsai_workflow": epoch_config["workflow"], "commit": commit, - "metadata": experiment_setup, + "devices": devices, "metadata_file_path": metadata_yml_filepath, } @@ -65,145 +81,108 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): + camera/patch location + patch, weightscale serial number """ + if experiment_name.startswith("oct"): ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath) return + experiment_key = {"experiment_name": experiment_name} metadata_yml_filepath = pathlib.Path(metadata_yml_filepath) - epoch_start = datetime.datetime.strptime( - metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" - ) + epoch_config = extract_epoch_config(experiment_name, metadata_yml_filepath) - experiment_setup = io_api.load( - str(metadata_yml_filepath.parent), - acquisition._device_schema_mapping[experiment_name].Metadata, - ) - - experiment_key = {"experiment_name": experiment_name} - # Check if there has been any changes in the arena setup - # by comparing the "Commit" against the most immediate preceding epoch - commit = experiment_setup.get("Commit", experiment_setup.get("Revision")) - assert commit, f'Neither "Commit" nor "Revision" found in {metadata_yml_filepath}' previous_epoch = (acquisition.Experiment & experiment_key).aggr( - acquisition.Epoch & f'epoch_start < "{epoch_start}"', + acquisition.Epoch & f'epoch_start < "{epoch_config["epoch_start"]}"', epoch_start="MAX(epoch_start)", ) - if len(acquisition.Epoch.Config & previous_epoch) and commit == ( + if len(acquisition.Epoch.Config & previous_epoch) and epoch_config["commit"] == ( acquisition.Epoch.Config & previous_epoch ).fetch1("commit"): # if identical commit -> no changes return - if isinstance(experiment_setup["Devices"], list): - experiment_devices = experiment_setup.pop("Devices") - elif isinstance(experiment_setup["Devices"], dict): - experiment_devices = [] - for device_name, device_info in experiment_setup.pop("Devices").items(): - if device_name.startswith("VideoController"): - device_type = "VideoController" - elif all(v in device_info for v in ("TriggerFrequency", "FrameEvents")): - device_type = "VideoSource" - elif all(v in device_info for v in ("PelletDelivered", "PatchEvents")): - device_type = "Patch" - elif all(v in device_info for v in ("TareWeight", "WeightEvents")): - device_type = "WeightScale" - elif device_name.startswith("AudioAmbient"): - device_type = "AudioAmbient" - elif device_name.startswith("Wall"): - device_type = "Wall" - elif device_name.startswith("Photodiode"): - device_type = "Photodiode" - else: - raise ValueError(f"Unrecognized Device Type for {device_name}") - experiment_devices.append( - {"Name": device_name, "Type": device_type, **device_info} - ) - else: - raise ValueError( - f"Unexpected devices variable type: {type(experiment_setup['Devices'])}" - ) - # ---- Video Controller ---- - video_controller = [ - device for device in experiment_devices if device["Type"] == "VideoController" - ] - assert ( - len(video_controller) == 1 - ), "Unable to find one unique VideoController device" - video_controller = video_controller[0] + device_frequency_mapper = { name: float(value) - for name, value in video_controller.items() + for name, value in epoch_config["devices"]["VideoController"].items() if name.endswith("Frequency") } + # ---- Load cameras ---- - cameras = [ - device for device in experiment_devices if device["Type"] == "VideoSource" - ] camera_list, camera_installation_list, camera_removal_list, camera_position_list = ( [], [], [], [], ) - for camera in cameras: - # ---- Check if this is a new camera, add to lab.Camera if needed - camera_key = {"camera_serial_number": camera["SerialNumber"]} - camera_list.append(camera_key) - camera_installation = { - "experiment_name": experiment_name, - **camera_key, - "camera_install_time": epoch_start, - "camera_description": camera["Name"], - "camera_sampling_rate": device_frequency_mapper[camera["TriggerFrequency"]], - "camera_gain": float(camera["Gain"]), - "camera_bin": int(camera["Binning"]), - } - if "position" in camera: - camera_position = { + # Check if this is a new camera, add to lab.Camera if needed + for device_name, device_config in epoch_config["devices"].items(): + if device_config["Type"] == "VideoSource": + camera_key = {"camera_serial_number": device_config["SerialNumber"]} + camera_list.append(camera_key) + + camera_installation = { **camera_key, "experiment_name": experiment_name, - "camera_install_time": epoch_start, - "camera_position_x": camera["position"]["x"], - "camera_position_y": camera["position"]["y"], - "camera_position_z": camera["position"]["z"], + "camera_install_time": epoch_config["epoch_start"], + "camera_description": device_name, + "camera_sampling_rate": device_frequency_mapper[ + device_config["TriggerFrequency"] + ], + "camera_gain": float(device_config["Gain"]), + "camera_bin": int(device_config["Binning"]), } - else: - camera_position = { - "camera_position_x": None, - "camera_position_y": None, - "camera_position_z": None, - "camera_rotation_x": None, - "camera_rotation_y": None, - "camera_rotation_z": None, - } - # ---- Check if this camera is currently installed - # If the same camera serial number is currently installed - # check for any changes in configuration, if not, skip this - current_camera_query = ( - acquisition.ExperimentCamera - acquisition.ExperimentCamera.RemovalTime - & experiment_key - & camera_key - ) - if current_camera_query: - current_camera_config = current_camera_query.join( - acquisition.ExperimentCamera.Position, left=True - ).fetch1() - new_camera_config = {**camera_installation, **camera_position} - current_camera_config.pop("camera_install_time") - new_camera_config.pop("camera_install_time") - if dict_to_uuid(current_camera_config) == dict_to_uuid(new_camera_config): - continue - # ---- Remove old camera - camera_removal_list.append( - { - **current_camera_query.fetch1("KEY"), - "camera_remove_time": epoch_start, + + if "position" in device_config: + camera_position = { + **camera_key, + "experiment_name": experiment_name, + "camera_install_time": epoch_config["epoch_start"], + "camera_position_x": device_config["position"]["x"], + "camera_position_y": device_config["position"]["y"], + "camera_position_z": device_config["position"]["z"], } + else: + camera_position = { + "camera_position_x": None, + "camera_position_y": None, + "camera_position_z": None, + "camera_rotation_x": None, + "camera_rotation_y": None, + "camera_rotation_z": None, + } + + """Check if this camera is currently installed. If the same camera serial number is currently installed check for any changes in configuration. If not, skip this""" + current_camera_query = ( + acquisition.ExperimentCamera - acquisition.ExperimentCamera.RemovalTime + & experiment_key + & camera_key ) - # ---- Install new camera - camera_installation_list.append(camera_installation) - if "position" in camera: - camera_position_list.append(camera_position) - # remove the currently installed cameras that are absent in this config + + if current_camera_query: + current_camera_config = current_camera_query.join( + acquisition.ExperimentCamera.Position, left=True + ).fetch1() + + new_camera_config = {**camera_installation, **camera_position} + current_camera_config.pop("camera_install_time") + new_camera_config.pop("camera_install_time") + + if dict_to_uuid(current_camera_config) == dict_to_uuid( + new_camera_config + ): + continue + # Remove old camera + camera_removal_list.append( + { + **current_camera_query.fetch1("KEY"), + "camera_remove_time": epoch_config["epoch_start"], + } + ) + # Install new camera + camera_installation_list.append(camera_installation) + + if "position" in device_config: + camera_position_list.append(camera_position) + # Remove the currently installed cameras that are absent in this config camera_removal_list.extend( ( acquisition.ExperimentCamera @@ -212,146 +191,154 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): & experiment_key ).fetch("KEY") ) + # ---- Load food patches ---- - food_patches = [ - device for device in experiment_devices if device["Type"] == "Patch" - ] patch_list, patch_installation_list, patch_removal_list, patch_position_list = ( [], [], [], [], ) - for patch in food_patches: - # ---- Check if this is a new food patch, add to lab.FoodPatch if needed - patch_key = { - "food_patch_serial_number": patch.get("SerialNumber") or patch["PortName"] - } - patch_list.append(patch_key) - patch_installation = { - **patch_key, - "experiment_name": experiment_name, - "food_patch_install_time": epoch_start, - "food_patch_description": patch["Name"], - "wheel_sampling_rate": float( - re.search(r"\d+", patch["SampleRate"]).group() - ), - "wheel_radius": float(patch["Radius"]), - } - if "position" in patch: - patch_position = { + + # Check if this is a new food patch, add to lab.FoodPatch if needed + for device_name, device_config in epoch_config["devices"].items(): + if device_config["Type"] == "Patch": + + patch_key = { + "food_patch_serial_number": device_config.get( + "SerialNumber", device_config.get("PortName") + ) + } + patch_list.append(patch_key) + patch_installation = { **patch_key, "experiment_name": experiment_name, - "food_patch_install_time": epoch_start, - "food_patch_position_x": patch["position"]["x"], - "food_patch_position_y": patch["position"]["y"], - "food_patch_position_z": patch["position"]["z"], + "food_patch_install_time": epoch_config["epoch_start"], + "food_patch_description": device_config["Name"], + "wheel_sampling_rate": float( + re.search(r"\d+", device_config["SampleRate"]).group() + ), + "wheel_radius": float(device_config["Radius"]), } - else: - patch_position = { - "food_patch_position_x": None, - "food_patch_position_y": None, - "food_patch_position_z": None, - } - # ---- Check if this camera is currently installed - # If the same camera serial number is currently installed - # check for any changes in configuration, if not, skip this - current_patch_query = ( - acquisition.ExperimentFoodPatch - - acquisition.ExperimentFoodPatch.RemovalTime - & experiment_key - & patch_key - ) - if current_patch_query: - current_patch_config = current_patch_query.join( - acquisition.ExperimentFoodPatch.Position, left=True - ).fetch1() - new_patch_config = {**patch_installation, **patch_position} - current_patch_config.pop("food_patch_install_time") - new_patch_config.pop("food_patch_install_time") - if dict_to_uuid(current_patch_config) == dict_to_uuid(new_patch_config): - continue - # ---- Remove old food patch - patch_removal_list.append( - { - **current_patch_query.fetch1("KEY"), - "food_patch_remove_time": epoch_start, + if "position" in device_config: + patch_position = { + **patch_key, + "experiment_name": experiment_name, + "food_patch_install_time": epoch_config["epoch_start"], + "food_patch_position_x": device_config["position"]["x"], + "food_patch_position_y": device_config["position"]["y"], + "food_patch_position_z": device_config["position"]["z"], + } + else: + patch_position = { + "food_patch_position_x": None, + "food_patch_position_y": None, + "food_patch_position_z": None, } + + """Check if this camera is currently installed. If the same camera serial number is currently installed, check for any changes in configuration, if not, skip this""" + current_patch_query = ( + acquisition.ExperimentFoodPatch + - acquisition.ExperimentFoodPatch.RemovalTime + & experiment_key + & patch_key ) - # ---- Install new food patch - patch_installation_list.append(patch_installation) - if "position" in patch: - patch_position_list.append(patch_position) - # remove the currently installed patches that are absent in this config - patch_removal_list.extend( - ( - acquisition.ExperimentFoodPatch - - acquisition.ExperimentFoodPatch.RemovalTime - - patch_list - & experiment_key - ).fetch("KEY") - ) + if current_patch_query: + current_patch_config = current_patch_query.join( + acquisition.ExperimentFoodPatch.Position, left=True + ).fetch1() + new_patch_config = {**patch_installation, **patch_position} + current_patch_config.pop("food_patch_install_time") + new_patch_config.pop("food_patch_install_time") + if dict_to_uuid(current_patch_config) == dict_to_uuid(new_patch_config): + continue + # Remove old food patch + patch_removal_list.append( + { + **current_patch_query.fetch1("KEY"), + "food_patch_remove_time": epoch_config["epoch_start"], + } + ) + # Install new food patch + patch_installation_list.append(patch_installation) + if "position" in device_config: + patch_position_list.append(patch_position) + # Remove the currently installed patches that are absent in this config + patch_removal_list.extend( + ( + acquisition.ExperimentFoodPatch + - acquisition.ExperimentFoodPatch.RemovalTime + - patch_list + & experiment_key + ).fetch("KEY") + ) + # ---- Load weight scales ---- - weight_scales = [ - device for device in experiment_devices if device["Type"] == "WeightScale" - ] weight_scale_list, weight_scale_installation_list, weight_scale_removal_list = ( [], [], [], ) - for weight_scale in weight_scales: - # ---- Check if this is a new weight scale, add to lab.WeightScale if needed - weight_scale_key = { - "weight_scale_serial_number": weight_scale.get("SerialNumber") - or weight_scale["PortName"] - } - weight_scale_list.append(weight_scale_key) - arena_key = (lab.Arena & acquisition.Experiment & experiment_key).fetch1("KEY") - weight_scale_installation = { - "experiment_name": experiment_name, - **weight_scale_key, - "weight_scale_install_time": epoch_start, - **arena_key, - "nest": _weight_scale_nest, - "weight_scale_description": weight_scale["Name"], - "weight_scale_sampling_rate": float(_weight_scale_rate), - } - # ---- Check if this weight scale is currently installed - if so, remove it - current_weight_scale_query = ( - acquisition.ExperimentWeightScale - - acquisition.ExperimentWeightScale.RemovalTime - & experiment_key - & weight_scale_key - ) - if current_weight_scale_query: - current_weight_scale_config = current_weight_scale_query.fetch1() - new_weight_scale_config = weight_scale_installation.copy() - current_weight_scale_config.pop("weight_scale_install_time") - new_weight_scale_config.pop("weight_scale_install_time") - if dict_to_uuid(current_weight_scale_config) == dict_to_uuid( - new_weight_scale_config - ): - continue - # ---- Remove old weight scale - weight_scale_removal_list.append( - { - **current_weight_scale_query.fetch1("KEY"), - "weight_scale_remove_time": epoch_start, - } + + # Check if this is a new weight scale, add to lab.WeightScale if needed + for device_name, device_config in epoch_config["devices"].items(): + if device_config["Type"] == "WeightScale": + + weight_scale_key = { + "weight_scale_serial_number": device_config.get( + "SerialNumber", device_config.get("PortName") + ) + } + weight_scale_list.append(weight_scale_key) + arena_key = (lab.Arena & acquisition.Experiment & experiment_key).fetch1( + "KEY" ) - # ---- Install new weight scale - weight_scale_installation_list.append(weight_scale_installation) - # remove the currently installed weight scales that are absent in this config - weight_scale_removal_list.extend( - ( - acquisition.ExperimentWeightScale - - acquisition.ExperimentWeightScale.RemovalTime - - weight_scale_list - & experiment_key - ).fetch("KEY") - ) - # ---- insert ---- + weight_scale_installation = { + **weight_scale_key, + **arena_key, + "experiment_name": experiment_name, + "weight_scale_install_time": epoch_config["epoch_start"], + "nest": _weight_scale_nest, + "weight_scale_description": device_name, + "weight_scale_sampling_rate": float(_weight_scale_rate), + } + + # Check if this weight scale is currently installed - if so, remove it + current_weight_scale_query = ( + acquisition.ExperimentWeightScale + - acquisition.ExperimentWeightScale.RemovalTime + & experiment_key + & weight_scale_key + ) + if current_weight_scale_query: + current_weight_scale_config = current_weight_scale_query.fetch1() + new_weight_scale_config = weight_scale_installation.copy() + current_weight_scale_config.pop("weight_scale_install_time") + new_weight_scale_config.pop("weight_scale_install_time") + if dict_to_uuid(current_weight_scale_config) == dict_to_uuid( + new_weight_scale_config + ): + continue + # Remove old weight scale + weight_scale_removal_list.append( + { + **current_weight_scale_query.fetch1("KEY"), + "weight_scale_remove_time": epoch_config["epoch_start"], + } + ) + # Install new weight scale + weight_scale_installation_list.append(weight_scale_installation) + # Remove the currently installed weight scales that are absent in this config + weight_scale_removal_list.extend( + ( + acquisition.ExperimentWeightScale + - acquisition.ExperimentWeightScale.RemovalTime + - weight_scale_list + & experiment_key + ).fetch("KEY") + ) + + # Insert def insert(): lab.Camera.insert(camera_list, skip_duplicates=True) acquisition.ExperimentCamera.RemovalTime.insert(camera_removal_list) From 150bec765af6d285e54d83dd12e39d4a45a2ac71 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 10 Apr 2023 21:48:25 +0000 Subject: [PATCH 16/54] fix keyword error --- aeon/dj_pipeline/streams.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index de2cc803..ca023fe4 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -97,16 +97,19 @@ def insert_device_types(cls, schema: DotMap, metadata_yml_filepath: Path): } for device_name in device_info } - + #! return only a list of device types that have been inserted. for device_name, info in device_info.items(): if info["device_type"]: + + if cls & {"device_type": info["device_type"]}: + continue + with cls.connections.transaction: cls.insert1( { "device_type": info["device_type"], "device_description": "", }, - skip_duplicates=True, ) cls.Stream.insert( [ @@ -116,7 +119,6 @@ def insert_device_types(cls, schema: DotMap, metadata_yml_filepath: Path): } for e in info["stream_type"] ], - skip_duplicates=True, ) @@ -159,10 +161,10 @@ class RemovalTime(dj.Part): definition = f""" -> master --- - {device_type}_remove_time: datetime(6) # time of the camera being removed from this position + {device_type}_remove_time: datetime(6) # time of the {device_type} being removed """ - ExperimentDevice.__name__ = f"Experiment{device_title}" + ExperimentDevice.__name__ = f"{device_title}" return ExperimentDevice @@ -171,7 +173,7 @@ def get_device_stream_template(device_type, stream_type): """Returns table class template for DeviceDataStream""" ExperimentDevice = get_device_template(device_type) - exp_device_table_name = f"Experiment{device_type}" + exp_device_table_name = f"{device_type}" # DeviceDataStream table(s) stream_detail = ( @@ -188,7 +190,7 @@ def get_device_stream_template(device_type, stream_type): stream = reader(**stream_detail["stream_reader_kwargs"]) table_definition = f""" # Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) - -> Experiment{device_type} + -> {device_type} -> acquisition.Chunk --- sample_count: int # number of data points acquired from this stream for a given chunk @@ -262,6 +264,8 @@ def make(self, key): class DeviceTableManager: + + # TODO: Simplify this class. def __init__(self, context=None): if context is None: @@ -360,7 +364,7 @@ def get_device_info(schema: DotMap) -> dict[dict]: e.g. {'CameraTop': {'stream_type': ['Video', 'Position', 'Region'], - 'reader': [ + 'stream_reader': [ aeon.io.reader.Video, aeon.io.reader.Position, aeon.schema.foraging._RegionReader @@ -385,13 +389,15 @@ def get_device_info(schema: DotMap) -> dict[dict]: "aeon.schema.octagon", ]: device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["reader"].append( + device_info[device_name]["stream_reader"].append( schema[device_name][stream_type].__class__ ) else: stream_type = schema[device_name].__class__.__name__ device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["reader"].append(schema[device_name].__class__) + device_info[device_name]["stream_reader"].append( + schema[device_name].__class__ + ) """Add a kwargs such as pattern, columns, extension, dtype and hash e.g., {'pattern': '{pattern}_SubjectState', From 0423b71316ec59870832cf3ba44370bad5839f0d Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 11 Apr 2023 17:25:33 +0000 Subject: [PATCH 17/54] handle type missing attribut error --- aeon/dj_pipeline/streams.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index ca023fe4..388f7dba 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -439,7 +439,7 @@ def get_device_info(schema: DotMap) -> dict[dict]: def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): - """Returns a mapping dictionary between device name and device type based on the dataset schema and metadata.yml from the experiment. + """Returns a mapping dictionary between device name and device type based on the dataset schema and metadata.yml from the experiment. Store the mapper dictionary and read from it if the type info doesn't exist in Metadata.yml. Args: schema (DotMap): DotMap object (e.g., exp02) @@ -449,8 +449,11 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): device_type_mapper (dict): {"device_name", "device_type"} e.g. {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} """ + import os + from aeon.io import api + metadata_yml_filepath = Path(metadata_yml_filepath) meta_data = ( api.load( str(metadata_yml_filepath.parent), @@ -460,10 +463,27 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): .to_dict("records")[0]["metadata"] ) - # Get device_type_mapper based on metadata.yml + # Store the mapper dictionary here + repository_root = ( + os.popen("git rev-parse --show-toplevel").read().strip() + ) # repo root path + filename = Path( + repository_root + "/aeon/dj_pipeline/create_experiments/device_type_mapper.json" + ) + device_type_mapper = {} - for item in meta_data.Devices: - device_type_mapper[item.Name] = item.Type + + if filename.is_file(): + with filename.open("r") as f: + device_type_mapper = json.load(f) + + try: # if the device type is not in the mapper, add it + for item in meta_data.Devices: + device_type_mapper[item.Name] = item.Type + with filename.open("w") as f: + json.dump(device_type_mapper, f) + except AttributeError: + pass return device_type_mapper From 45111e8063bee810d3508339ccdf6950a33c44cf Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 11 Apr 2023 19:35:41 +0000 Subject: [PATCH 18/54] insert into streams.StreamType from schema dotmap --- aeon/dj_pipeline/populate/worker.py | 7 +++---- aeon/dj_pipeline/utils/load_metadata.py | 13 ++++++++++++- aeon/schema/dataset.py | 2 ++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/aeon/dj_pipeline/populate/worker.py b/aeon/dj_pipeline/populate/worker.py index 994ad60a..4c506347 100644 --- a/aeon/dj_pipeline/populate/worker.py +++ b/aeon/dj_pipeline/populate/worker.py @@ -1,8 +1,8 @@ import datajoint as dj from datajoint_utilities.dj_worker import ( DataJointWorker, - WorkerLog, ErrorLog, + WorkerLog, is_djtable, ) @@ -12,12 +12,11 @@ db_prefix, qc, report, - tracking, streams, + tracking, ) from aeon.dj_pipeline.utils import load_metadata - __all__ = [ "high_priority", "mid_priority", @@ -44,6 +43,7 @@ ) high_priority(load_metadata.ingest_subject) +high_priority(load_metadata.ingest_streams) high_priority(acquisition.Epoch.ingest_epochs, experiment_name=_current_experiment) high_priority(acquisition.Chunk.ingest_chunks, experiment_name=_current_experiment) high_priority(acquisition.ExperimentLog) @@ -101,4 +101,3 @@ for attr in vars(streams).values(): if is_djtable(attr) and hasattr(attr, "populate"): streams_worker(attr) - diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 30627167..12512630 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from aeon.dj_pipeline import acquisition, dict_to_uuid, lab, subject +from aeon.dj_pipeline import acquisition, dict_to_uuid, lab, streams, subject from aeon.io import api as io_api _weight_scale_rate = 100 @@ -27,6 +27,17 @@ def ingest_subject(colony_csv_path: pathlib.Path = _colony_csv_path) -> None: ) +def ingest_streams(): + """Insert into stream.streamType table all streams in the dataset schema.""" + from dotmap import DotMap + + from aeon.schema import dataset + + schemas = [v for v in dataset.__dict__.values() if isinstance(v, DotMap)] + for schema in schemas: + streams.StreamType.insert_streams(schema) + + def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> dict: """Parse experiment metadata YAML file and extract epoch configuration. diff --git a/aeon/schema/dataset.py b/aeon/schema/dataset.py index 5aae992f..f74bcce7 100644 --- a/aeon/schema/dataset.py +++ b/aeon/schema/dataset.py @@ -6,6 +6,8 @@ from aeon.io import reader from aeon.io.device import Device +__all__ = ["exp02", "exp01", "octagon01", "presocial"] + exp02 = DotMap( [ Device("Metadata", stream.metadata), From d20902511ea156a7192c9e30345a46520249b8d7 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 11 Apr 2023 19:56:17 +0000 Subject: [PATCH 19/54] move StreamType insertion functions to load_metadata --- aeon/dj_pipeline/streams.py | 191 +---------------------- aeon/dj_pipeline/utils/load_metadata.py | 195 +++++++++++++++++++++++- 2 files changed, 193 insertions(+), 193 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 388f7dba..3d6d9ca4 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -1,5 +1,4 @@ import inspect -import json import re from collections import defaultdict from functools import cached_property @@ -47,24 +46,6 @@ class StreamType(dj.Lookup): unique index (stream_hash) """ - @classmethod - def insert_streams(cls, schema: DotMap): - - stream_entries = get_stream_entries(schema) - - for entry in stream_entries: - q_param = cls & {"stream_hash": entry["stream_hash"]} - if q_param: # If the specified stream type already exists - pname = q_param.fetch1("stream_type") - if pname != entry["stream_type"]: - # If the existed stream type does not have the same name: - # human error, trying to add the same content with different name - raise dj.DataJointError( - f"The specified stream type already exists - name: {pname}" - ) - - cls.insert(stream_entries, skip_duplicates=True) - @schema class DeviceType(dj.Lookup): @@ -97,7 +78,7 @@ def insert_device_types(cls, schema: DotMap, metadata_yml_filepath: Path): } for device_name in device_info } - #! return only a list of device types that have been inserted. + # Return only a list of device types that have been inserted. for device_name, info in device_info.items(): if info["device_type"]: @@ -350,173 +331,3 @@ def create_device_stream_tables(self): self._schema(table_class, context=self.context) self._schema.activate(schema_name) - - -def get_device_info(schema: DotMap) -> dict[dict]: - """ - Read from the above DotMap object and returns a device dictionary as the following. - - Args: - schema (DotMap): DotMap object (e.g., exp02, octagon01) - - Returns: - device_info (dict[dict]): A dictionary of device information - - e.g. {'CameraTop': - {'stream_type': ['Video', 'Position', 'Region'], - 'stream_reader': [ - aeon.io.reader.Video, - aeon.io.reader.Position, - aeon.schema.foraging._RegionReader - ], - 'pattern': ['{pattern}', '{pattern}_200', '{pattern}_201'] - } - } - """ - schema_json = json.dumps(schema, default=lambda x: x.__dict__, indent=4) - schema_dict = json.loads(schema_json) - - device_info = {} - - for device_name in schema: - if not device_name.startswith("_"): - device_info[device_name] = defaultdict(list) - if isinstance(schema[device_name], DotMap): - for stream_type in schema[device_name].keys(): - if schema[device_name][stream_type].__class__.__module__ in [ - "aeon.io.reader", - "aeon.schema.foraging", - "aeon.schema.octagon", - ]: - device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["stream_reader"].append( - schema[device_name][stream_type].__class__ - ) - else: - stream_type = schema[device_name].__class__.__name__ - device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["stream_reader"].append( - schema[device_name].__class__ - ) - - """Add a kwargs such as pattern, columns, extension, dtype and hash - e.g., {'pattern': '{pattern}_SubjectState', - 'columns': ['id', 'weight', 'event'], - 'extension': 'csv', - 'dtype': None}""" - for device_name in device_info: - if pattern := schema_dict[device_name].get("pattern"): - schema_dict[device_name]["pattern"] = pattern.replace( - device_name, "{pattern}" - ) - - # Add stream_reader_kwargs - kwargs = schema_dict[device_name] - device_info[device_name]["stream_reader_kwargs"].append(kwargs) - stream_reader = device_info[device_name]["stream_reader"] - # Add hash - device_info[device_name]["stream_hash"].append( - dict_to_uuid({**kwargs, "stream_reader": stream_reader}) - ) - - else: - for stream_type in device_info[device_name]["stream_type"]: - pattern = schema_dict[device_name][stream_type]["pattern"] - schema_dict[device_name][stream_type]["pattern"] = pattern.replace( - device_name, "{pattern}" - ) - # Add stream_reader_kwargs - kwargs = schema_dict[device_name][stream_type] - device_info[device_name]["stream_reader_kwargs"].append(kwargs) - stream_ind = device_info[device_name]["stream_type"].index(stream_type) - stream_reader = device_info[device_name]["stream_reader"][stream_ind] - # Add hash - device_info[device_name]["stream_hash"].append( - dict_to_uuid({**kwargs, "stream_reader": stream_reader}) - ) - - return device_info - - -def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): - """Returns a mapping dictionary between device name and device type based on the dataset schema and metadata.yml from the experiment. Store the mapper dictionary and read from it if the type info doesn't exist in Metadata.yml. - - Args: - schema (DotMap): DotMap object (e.g., exp02) - metadata_yml_filepath (Path): Path to metadata.yml. - - Returns: - device_type_mapper (dict): {"device_name", "device_type"} - e.g. {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} - """ - import os - - from aeon.io import api - - metadata_yml_filepath = Path(metadata_yml_filepath) - meta_data = ( - api.load( - str(metadata_yml_filepath.parent), - schema.Metadata, - ) - .reset_index() - .to_dict("records")[0]["metadata"] - ) - - # Store the mapper dictionary here - repository_root = ( - os.popen("git rev-parse --show-toplevel").read().strip() - ) # repo root path - filename = Path( - repository_root + "/aeon/dj_pipeline/create_experiments/device_type_mapper.json" - ) - - device_type_mapper = {} - - if filename.is_file(): - with filename.open("r") as f: - device_type_mapper = json.load(f) - - try: # if the device type is not in the mapper, add it - for item in meta_data.Devices: - device_type_mapper[item.Name] = item.Type - with filename.open("w") as f: - json.dump(device_type_mapper, f) - except AttributeError: - pass - - return device_type_mapper - - -def get_stream_entries(schema: DotMap) -> list[dict]: - """Returns a list of dictionaries containing the stream entries for a given device, - - Args: - schema (DotMap): DotMap object (e.g., exp02, octagon01) - - Returns: - stream_info (list[dict]): list of dictionaries containing the stream entries for a given device, - - e.g. {'stream_type': 'EnvironmentState', - 'stream_reader': aeon.io.reader.Csv, - 'stream_reader_kwargs': {'pattern': '{pattern}_EnvironmentState', - 'columns': ['state'], - 'extension': 'csv', - 'dtype': None} - """ - device_info = get_device_info(schema) - return [ - { - "stream_type": stream_type, - "stream_reader": stream_reader, - "stream_reader_kwargs": stream_reader_kwargs, - "stream_hash": stream_hash, - } - for stream_info in device_info.values() - for stream_type, stream_reader, stream_reader_kwargs, stream_hash in zip( - stream_info["stream_type"], - stream_info["stream_reader"], - stream_info["stream_reader_kwargs"], - stream_info["stream_hash"], - ) - ] diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 12512630..b56f86eb 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -2,9 +2,12 @@ import json import pathlib import re +from collections import defaultdict +from pathlib import Path import numpy as np import pandas as pd +from dotmap import DotMap from aeon.dj_pipeline import acquisition, dict_to_uuid, lab, streams, subject from aeon.io import api as io_api @@ -29,13 +32,25 @@ def ingest_subject(colony_csv_path: pathlib.Path = _colony_csv_path) -> None: def ingest_streams(): """Insert into stream.streamType table all streams in the dataset schema.""" - from dotmap import DotMap - from aeon.schema import dataset schemas = [v for v in dataset.__dict__.values() if isinstance(v, DotMap)] for schema in schemas: - streams.StreamType.insert_streams(schema) + + stream_entries = get_stream_entries(schema) + + for entry in stream_entries: + q_param = streams.StreamType & {"stream_hash": entry["stream_hash"]} + if q_param: # If the specified stream type already exists + pname = q_param.fetch1("stream_type") + if pname != entry["stream_type"]: + # If the existed stream type does not have the same name: + # human error, trying to add the same content with different name + raise dj.DataJointError( + f"The specified stream type already exists - name: {pname}" + ) + + streams.StreamType.insert(stream_entries, skip_duplicates=True) def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> dict: @@ -412,3 +427,177 @@ def ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath): experiment_table.insert1( (experiment_name, device_sn, epoch_start, device_name) ) + + +# region Get stream & device information +def get_device_info(schema: DotMap) -> dict[dict]: + """ + Read from the above DotMap object and returns a device dictionary as the following. + + Args: + schema (DotMap): DotMap object (e.g., exp02, octagon01) + + Returns: + device_info (dict[dict]): A dictionary of device information + + e.g. {'CameraTop': + {'stream_type': ['Video', 'Position', 'Region'], + 'stream_reader': [ + aeon.io.reader.Video, + aeon.io.reader.Position, + aeon.schema.foraging._RegionReader + ], + 'pattern': ['{pattern}', '{pattern}_200', '{pattern}_201'] + } + } + """ + schema_json = json.dumps(schema, default=lambda x: x.__dict__, indent=4) + schema_dict = json.loads(schema_json) + + device_info = {} + + for device_name in schema: + if not device_name.startswith("_"): + device_info[device_name] = defaultdict(list) + if isinstance(schema[device_name], DotMap): + for stream_type in schema[device_name].keys(): + if schema[device_name][stream_type].__class__.__module__ in [ + "aeon.io.reader", + "aeon.schema.foraging", + "aeon.schema.octagon", + ]: + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["stream_reader"].append( + schema[device_name][stream_type].__class__ + ) + else: + stream_type = schema[device_name].__class__.__name__ + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["stream_reader"].append( + schema[device_name].__class__ + ) + + """Add a kwargs such as pattern, columns, extension, dtype and hash + e.g., {'pattern': '{pattern}_SubjectState', + 'columns': ['id', 'weight', 'event'], + 'extension': 'csv', + 'dtype': None}""" + for device_name in device_info: + if pattern := schema_dict[device_name].get("pattern"): + schema_dict[device_name]["pattern"] = pattern.replace( + device_name, "{pattern}" + ) + + # Add stream_reader_kwargs + kwargs = schema_dict[device_name] + device_info[device_name]["stream_reader_kwargs"].append(kwargs) + stream_reader = device_info[device_name]["stream_reader"] + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid({**kwargs, "stream_reader": stream_reader}) + ) + + else: + for stream_type in device_info[device_name]["stream_type"]: + pattern = schema_dict[device_name][stream_type]["pattern"] + schema_dict[device_name][stream_type]["pattern"] = pattern.replace( + device_name, "{pattern}" + ) + # Add stream_reader_kwargs + kwargs = schema_dict[device_name][stream_type] + device_info[device_name]["stream_reader_kwargs"].append(kwargs) + stream_ind = device_info[device_name]["stream_type"].index(stream_type) + stream_reader = device_info[device_name]["stream_reader"][stream_ind] + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid({**kwargs, "stream_reader": stream_reader}) + ) + + return device_info + + +def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): + """Returns a mapping dictionary between device name and device type based on the dataset schema and metadata.yml from the experiment. Store the mapper dictionary and read from it if the type info doesn't exist in Metadata.yml. + + Args: + schema (DotMap): DotMap object (e.g., exp02) + metadata_yml_filepath (Path): Path to metadata.yml. + + Returns: + device_type_mapper (dict): {"device_name", "device_type"} + e.g. {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} + """ + import os + + from aeon.io import api + + metadata_yml_filepath = Path(metadata_yml_filepath) + meta_data = ( + api.load( + str(metadata_yml_filepath.parent), + schema.Metadata, + ) + .reset_index() + .to_dict("records")[0]["metadata"] + ) + + # Store the mapper dictionary here + repository_root = ( + os.popen("git rev-parse --show-toplevel").read().strip() + ) # repo root path + filename = Path( + repository_root + "/aeon/dj_pipeline/create_experiments/device_type_mapper.json" + ) + + device_type_mapper = {} + + if filename.is_file(): + with filename.open("r") as f: + device_type_mapper = json.load(f) + + try: # if the device type is not in the mapper, add it + for item in meta_data.Devices: + device_type_mapper[item.Name] = item.Type + with filename.open("w") as f: + json.dump(device_type_mapper, f) + except AttributeError: + pass + + return device_type_mapper + + +def get_stream_entries(schema: DotMap) -> list[dict]: + """Returns a list of dictionaries containing the stream entries for a given device, + + Args: + schema (DotMap): DotMap object (e.g., exp02, octagon01) + + Returns: + stream_info (list[dict]): list of dictionaries containing the stream entries for a given device, + + e.g. {'stream_type': 'EnvironmentState', + 'stream_reader': aeon.io.reader.Csv, + 'stream_reader_kwargs': {'pattern': '{pattern}_EnvironmentState', + 'columns': ['state'], + 'extension': 'csv', + 'dtype': None} + """ + device_info = get_device_info(schema) + return [ + { + "stream_type": stream_type, + "stream_reader": stream_reader, + "stream_reader_kwargs": stream_reader_kwargs, + "stream_hash": stream_hash, + } + for stream_info in device_info.values() + for stream_type, stream_reader, stream_reader_kwargs, stream_hash in zip( + stream_info["stream_type"], + stream_info["stream_reader"], + stream_info["stream_reader_kwargs"], + stream_info["stream_hash"], + ) + ] + + +# endregion From c8c928a468e84e277e161567a289de5085202c46 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 11 Apr 2023 22:36:13 +0000 Subject: [PATCH 20/54] move insert_devices into load_metadata --- aeon/dj_pipeline/streams.py | 37 --------------- aeon/dj_pipeline/utils/load_metadata.py | 60 +++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 3d6d9ca4..1886b22f 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -65,43 +65,6 @@ class Stream(dj.Part): -> StreamType """ - @classmethod - def insert_device_types(cls, schema: DotMap, metadata_yml_filepath: Path): - """Use dataset.schema and metadata.yml to insert device types and streams. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml.""" - device_info = get_device_info(schema) - device_type_mapper = get_device_mapper(schema, metadata_yml_filepath) - - device_info = { - device_name: { - "device_type": device_type_mapper.get(device_name, None), - **device_info[device_name], - } - for device_name in device_info - } - # Return only a list of device types that have been inserted. - for device_name, info in device_info.items(): - if info["device_type"]: - - if cls & {"device_type": info["device_type"]}: - continue - - with cls.connections.transaction: - cls.insert1( - { - "device_type": info["device_type"], - "device_description": "", - }, - ) - cls.Stream.insert( - [ - { - "device_type": info["device_type"], - "stream_type": e, - } - for e in info["stream_type"] - ], - ) - @schema class Device(dj.Lookup): diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index b56f86eb..9a8f7cc7 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -31,7 +31,7 @@ def ingest_subject(colony_csv_path: pathlib.Path = _colony_csv_path) -> None: def ingest_streams(): - """Insert into stream.streamType table all streams in the dataset schema.""" + """Insert into streams.streamType table all streams in the dataset schema.""" from aeon.schema import dataset schemas = [v for v in dataset.__dict__.values() if isinstance(v, DotMap)] @@ -53,6 +53,54 @@ def ingest_streams(): streams.StreamType.insert(stream_entries, skip_duplicates=True) +def insert_devices(schema: DotMap, metadata_yml_filepath: Path): + """Use dataset.schema and metadata.yml to insert into streams.DeviceType and streams.Device. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml.""" + device_info: dict[dict] = get_device_info(schema) + device_type_mapper, device_sn = get_device_mapper(schema, metadata_yml_filepath) + + # Add device type to device_info. + device_info = { + device_name: { + "device_type": device_type_mapper.get(device_name, None), + **device_info[device_name], + } + for device_name in device_info + } + # Return only a list of device types that have been inserted. + for device_name, info in device_info.items(): + + if info["device_type"]: + + streams.DeviceType.insert1( + { + "device_type": info["device_type"], + "device_description": "", + }, + skip_duplicates=True, + ) + streams.DeviceType.Stream.insert( + [ + { + "device_type": info["device_type"], + "stream_type": e, + } + for e in info["stream_type"] + ], + skip_duplicates=True, + ) + + if device_sn[device_name]: + if streams.Device & {"device_serial_number": device_sn[device_name]}: + continue + streams.Device.insert1( + { + "device_serial_number": device_sn[device_name], + "device_type": info["device_type"], + }, + skip_duplicates=True, + ) + + def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> dict: """Parse experiment metadata YAML file and extract epoch configuration. @@ -526,6 +574,8 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): Returns: device_type_mapper (dict): {"device_name", "device_type"} e.g. {'CameraTop': 'VideoSource', 'Patch1': 'Patch'} + device_sn (dict): {"device_name", "serial_number"} + e.g. {'CameraTop': '21053810'} """ import os @@ -549,7 +599,8 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): repository_root + "/aeon/dj_pipeline/create_experiments/device_type_mapper.json" ) - device_type_mapper = {} + device_type_mapper = {} # {device_name: device_type} + device_sn = {} # device serial number if filename.is_file(): with filename.open("r") as f: @@ -558,12 +609,15 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): try: # if the device type is not in the mapper, add it for item in meta_data.Devices: device_type_mapper[item.Name] = item.Type + device_sn[item.Name] = ( + item.SerialNumber if not isinstance(item.SerialNumber, DotMap) else None + ) with filename.open("w") as f: json.dump(device_type_mapper, f) except AttributeError: pass - return device_type_mapper + return device_type_mapper, device_sn def get_stream_entries(schema: DotMap) -> list[dict]: From 1b073393f52f96931e290d2661cab795c5608819 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Wed, 12 Apr 2023 17:06:45 +0000 Subject: [PATCH 21/54] fix bugs in device_info and simplify --- aeon/dj_pipeline/utils/load_metadata.py | 197 ++++++++++++------------ 1 file changed, 99 insertions(+), 98 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 9a8f7cc7..92517c7d 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -1,10 +1,12 @@ import datetime +import inspect import json import pathlib import re from collections import defaultdict from pathlib import Path +import datajoint as dj import numpy as np import pandas as pd from dotmap import DotMap @@ -53,7 +55,7 @@ def ingest_streams(): streams.StreamType.insert(stream_entries, skip_duplicates=True) -def insert_devices(schema: DotMap, metadata_yml_filepath: Path): +def ingest_devices(schema: DotMap, metadata_yml_filepath: Path): """Use dataset.schema and metadata.yml to insert into streams.DeviceType and streams.Device. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml.""" device_info: dict[dict] = get_device_info(schema) device_type_mapper, device_sn = get_device_mapper(schema, metadata_yml_filepath) @@ -89,16 +91,17 @@ def insert_devices(schema: DotMap, metadata_yml_filepath: Path): skip_duplicates=True, ) - if device_sn[device_name]: - if streams.Device & {"device_serial_number": device_sn[device_name]}: - continue - streams.Device.insert1( - { - "device_serial_number": device_sn[device_name], - "device_type": info["device_type"], - }, - skip_duplicates=True, - ) + if streams.Device & {"device_serial_number": device_sn[device_name]}: + continue + + streams.Device.insert1( + { + "device_serial_number": device_sn[device_name] + or device_name, #! insert device name if not exists + "device_type": info["device_type"], + }, + skip_duplicates=True, + ) def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> dict: @@ -478,6 +481,40 @@ def ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath): # region Get stream & device information +def get_stream_entries(schema: DotMap) -> list[dict]: + """Returns a list of dictionaries containing the stream entries for a given device, + + Args: + schema (DotMap): DotMap object (e.g., exp02, octagon01) + + Returns: + stream_info (list[dict]): list of dictionaries containing the stream entries for a given device, + + e.g. {'stream_type': 'EnvironmentState', + 'stream_reader': aeon.io.reader.Csv, + 'stream_reader_kwargs': {'pattern': '{pattern}_EnvironmentState', + 'columns': ['state'], + 'extension': 'csv', + 'dtype': None} + """ + device_info = get_device_info(schema) + return [ + { + "stream_type": stream_type, + "stream_reader": stream_reader, + "stream_reader_kwargs": stream_reader_kwargs, + "stream_hash": stream_hash, + } + for stream_info in device_info.values() + for stream_type, stream_reader, stream_reader_kwargs, stream_hash in zip( + stream_info["stream_type"], + stream_info["stream_reader"], + stream_info["stream_reader_kwargs"], + stream_info["stream_hash"], + ) + ] + + def get_device_info(schema: DotMap) -> dict[dict]: """ Read from the above DotMap object and returns a device dictionary as the following. @@ -499,69 +536,67 @@ def get_device_info(schema: DotMap) -> dict[dict]: } } """ + + def _get_class_path(obj): + return f"{obj.__class__.__module__}.{obj.__class__.__name__}" + schema_json = json.dumps(schema, default=lambda x: x.__dict__, indent=4) schema_dict = json.loads(schema_json) - device_info = {} - for device_name in schema: - if not device_name.startswith("_"): - device_info[device_name] = defaultdict(list) - if isinstance(schema[device_name], DotMap): - for stream_type in schema[device_name].keys(): - if schema[device_name][stream_type].__class__.__module__ in [ - "aeon.io.reader", - "aeon.schema.foraging", - "aeon.schema.octagon", - ]: - device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["stream_reader"].append( - schema[device_name][stream_type].__class__ - ) - else: - stream_type = schema[device_name].__class__.__name__ - device_info[device_name]["stream_type"].append(stream_type) - device_info[device_name]["stream_reader"].append( - schema[device_name].__class__ - ) + for device_name, device in schema.items(): + if device_name.startswith("_"): + continue + + device_info[device_name] = defaultdict(list) + + if isinstance(device, DotMap): + for stream_type, stream_obj in device.items(): + if stream_obj.__class__.__module__ in [ + "aeon.io.reader", + "aeon.schema.foraging", + "aeon.schema.octagon", + ]: + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["stream_reader"].append( + _get_class_path(stream_obj) + ) + + required_args = [ + k + for k in inspect.signature(stream_obj.__init__).parameters + if k != "self" + ] + pattern = schema_dict[device_name][stream_type].get("pattern") + schema_dict[device_name][stream_type]["pattern"] = pattern.replace( + device_name, "{pattern}" + ) - """Add a kwargs such as pattern, columns, extension, dtype and hash - e.g., {'pattern': '{pattern}_SubjectState', - 'columns': ['id', 'weight', 'event'], - 'extension': 'csv', - 'dtype': None}""" - for device_name in device_info: - if pattern := schema_dict[device_name].get("pattern"): + kwargs = { + k: v + for k, v in schema_dict[device_name][stream_type].items() + if k in required_args + } + device_info[device_name]["stream_reader_kwargs"].append(kwargs) + else: + stream_type = device.__class__.__name__ + device_info[device_name]["stream_type"].append(stream_type) + device_info[device_name]["stream_reader"].append(_get_class_path(device)) + + required_args = { + k: None + for k in inspect.signature(device.__init__).parameters + if k != "self" + } + pattern = schema_dict[device_name].get("pattern") schema_dict[device_name]["pattern"] = pattern.replace( device_name, "{pattern}" ) - # Add stream_reader_kwargs - kwargs = schema_dict[device_name] + kwargs = { + k: v for k, v in schema_dict[device_name].items() if k in required_args + } device_info[device_name]["stream_reader_kwargs"].append(kwargs) - stream_reader = device_info[device_name]["stream_reader"] - # Add hash - device_info[device_name]["stream_hash"].append( - dict_to_uuid({**kwargs, "stream_reader": stream_reader}) - ) - - else: - for stream_type in device_info[device_name]["stream_type"]: - pattern = schema_dict[device_name][stream_type]["pattern"] - schema_dict[device_name][stream_type]["pattern"] = pattern.replace( - device_name, "{pattern}" - ) - # Add stream_reader_kwargs - kwargs = schema_dict[device_name][stream_type] - device_info[device_name]["stream_reader_kwargs"].append(kwargs) - stream_ind = device_info[device_name]["stream_type"].index(stream_type) - stream_reader = device_info[device_name]["stream_reader"][stream_ind] - # Add hash - device_info[device_name]["stream_hash"].append( - dict_to_uuid({**kwargs, "stream_reader": stream_reader}) - ) - - return device_info def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): @@ -620,38 +655,4 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): return device_type_mapper, device_sn -def get_stream_entries(schema: DotMap) -> list[dict]: - """Returns a list of dictionaries containing the stream entries for a given device, - - Args: - schema (DotMap): DotMap object (e.g., exp02, octagon01) - - Returns: - stream_info (list[dict]): list of dictionaries containing the stream entries for a given device, - - e.g. {'stream_type': 'EnvironmentState', - 'stream_reader': aeon.io.reader.Csv, - 'stream_reader_kwargs': {'pattern': '{pattern}_EnvironmentState', - 'columns': ['state'], - 'extension': 'csv', - 'dtype': None} - """ - device_info = get_device_info(schema) - return [ - { - "stream_type": stream_type, - "stream_reader": stream_reader, - "stream_reader_kwargs": stream_reader_kwargs, - "stream_hash": stream_hash, - } - for stream_info in device_info.values() - for stream_type, stream_reader, stream_reader_kwargs, stream_hash in zip( - stream_info["stream_type"], - stream_info["stream_reader"], - stream_info["stream_reader_kwargs"], - stream_info["stream_hash"], - ) - ] - - # endregion From b479e92850b67f341f75555628928d0e0ff522c0 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Wed, 12 Apr 2023 17:08:09 +0000 Subject: [PATCH 22/54] fix table generation --- aeon/dj_pipeline/streams.py | 47 ++++++++++++++----------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 1886b22f..809c84ce 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -15,18 +15,11 @@ logger = dj.logger -schema_name = f'u_{dj.config["database.user"]}_streams' # for testing -# schema_name = get_schema_name("streams") +# schema_name = f'u_{dj.config["database.user"]}_streams' # for testing +schema_name = get_schema_name("streams") schema = dj.schema(schema_name) -__all__ = [ - "StreamType", - "DeviceType", - "Device", -] - - @schema class StreamType(dj.Lookup): """ @@ -75,7 +68,7 @@ class Device(dj.Lookup): """ -## --------- Helper functions & classes --------- ## +# region Helper functions for creating device tables. def get_device_template(device_type): @@ -208,8 +201,8 @@ def make(self, key): class DeviceTableManager: + """Class for managing device tables""" - # TODO: Simplify this class. def __init__(self, context=None): if context is None: @@ -218,21 +211,13 @@ def __init__(self, context=None): self.context = context self._schema = dj.schema(context=self.context) - self._device_tables = [] self._device_stream_tables = [] - self._device_types = DeviceType.fetch("device_type") self._device_stream_map = defaultdict( list ) # dictionary for showing hierarchical relationship between device type and stream type - def _add_device_tables(self): - for device_type in self._device_types: - table_name = f"Experiment{device_type}" - if table_name not in self._device_tables: - self._device_tables.append(table_name) - def _add_device_stream_tables(self): - for device_type in self._device_types: + for device_type in self.device_tables: for stream_type in ( StreamType & (DeviceType.Stream & {"device_type": device_type}) ).fetch("stream_type"): @@ -243,18 +228,13 @@ def _add_device_stream_tables(self): self._device_stream_map[device_type].append(stream_type) - @property - def device_types(self): - return self._device_types - @cached_property def device_tables(self) -> list: """ Name of the device tables to be created """ - self._add_device_tables() - return self._device_tables + return list(DeviceType.fetch("device_type")) @cached_property def device_stream_tables(self) -> list: @@ -271,9 +251,7 @@ def device_stream_map(self) -> dict: def create_device_tables(self): - for device_table in self.device_tables: - - device_type = re.sub(r"\bExperiment", "", device_table) + for device_type in self.device_tables: table_class = get_device_template(device_type) @@ -294,3 +272,14 @@ def create_device_stream_tables(self): self._schema(table_class, context=self.context) self._schema.activate(schema_name) + + +# endregion + + +if __name__ == "__main__": + + # Create device & device stream tables + tbmg = DeviceTableManager(context=inspect.currentframe().f_back.f_locals) + tbmg.create_device_tables() + tbmg.create_device_stream_tables() From 1b980f279c1630ad7f187acbb6d12cd59e30cc03 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Thu, 13 Apr 2023 22:40:13 +0000 Subject: [PATCH 23/54] add back stream uuid --- aeon/dj_pipeline/utils/load_metadata.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 92517c7d..52dd4604 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -578,6 +578,12 @@ def _get_class_path(obj): if k in required_args } device_info[device_name]["stream_reader_kwargs"].append(kwargs) + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid( + {**kwargs, "stream_reader": _get_class_path(stream_obj)} + ) + ) else: stream_type = device.__class__.__name__ device_info[device_name]["stream_type"].append(stream_type) @@ -597,6 +603,11 @@ def _get_class_path(obj): k: v for k, v in schema_dict[device_name].items() if k in required_args } device_info[device_name]["stream_reader_kwargs"].append(kwargs) + # Add hash + device_info[device_name]["stream_hash"].append( + dict_to_uuid({**kwargs, "stream_reader": _get_class_path(device)}) + ) + return device_info def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): From edd27c8005047ce5bb35d90c6324e2415faeafb3 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 14 Apr 2023 01:06:57 +0000 Subject: [PATCH 24/54] remove DeviceTableManager from streams.py --- aeon/dj_pipeline/streams.py | 97 ++----------------------------------- 1 file changed, 4 insertions(+), 93 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 809c84ce..37f4596e 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -1,15 +1,8 @@ -import inspect -import re -from collections import defaultdict -from functools import cached_property -from pathlib import Path - import datajoint as dj import pandas as pd -from dotmap import DotMap import aeon -from aeon.dj_pipeline import acquisition, dict_to_uuid, get_schema_name +from aeon.dj_pipeline import acquisition, get_schema_name from aeon.io import api as io_api logger = dj.logger @@ -71,7 +64,7 @@ class Device(dj.Lookup): # region Helper functions for creating device tables. -def get_device_template(device_type): +def get_device_template(device_type: str): """Returns table class template for ExperimentDevice""" device_title = device_type device_type = dj.utils.from_camel_case(device_type) @@ -80,7 +73,7 @@ class ExperimentDevice(dj.Manual): definition = f""" # {device_title} placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-{aeon.__version__}) -> acquisition.Experiment - -> Device + -> streams.Device {device_type}_install_time: datetime(6) # time of the {device_type} placed and started operation at this position --- {device_type}_name: varchar(36) @@ -106,7 +99,7 @@ class RemovalTime(dj.Part): return ExperimentDevice -def get_device_stream_template(device_type, stream_type): +def get_device_stream_template(device_type: str, stream_type: str): """Returns table class template for DeviceDataStream""" ExperimentDevice = get_device_template(device_type) @@ -200,86 +193,4 @@ def make(self, key): return DeviceDataStream -class DeviceTableManager: - """Class for managing device tables""" - - def __init__(self, context=None): - - if context is None: - self.context = inspect.currentframe().f_back.f_locals - else: - self.context = context - - self._schema = dj.schema(context=self.context) - self._device_stream_tables = [] - self._device_stream_map = defaultdict( - list - ) # dictionary for showing hierarchical relationship between device type and stream type - - def _add_device_stream_tables(self): - for device_type in self.device_tables: - for stream_type in ( - StreamType & (DeviceType.Stream & {"device_type": device_type}) - ).fetch("stream_type"): - - table_name = f"{device_type}{stream_type}" - if table_name not in self._device_stream_tables: - self._device_stream_tables.append(table_name) - - self._device_stream_map[device_type].append(stream_type) - - @cached_property - def device_tables(self) -> list: - """ - Name of the device tables to be created - """ - - return list(DeviceType.fetch("device_type")) - - @cached_property - def device_stream_tables(self) -> list: - """ - Name of the device stream tables to be created - """ - self._add_device_stream_tables() - return self._device_stream_tables - - @cached_property - def device_stream_map(self) -> dict: - self._add_device_stream_tables() - return self._device_stream_map - - def create_device_tables(self): - - for device_type in self.device_tables: - - table_class = get_device_template(device_type) - - self.context[table_class.__name__] = table_class - self._schema(table_class, context=self.context) - - self._schema.activate(schema_name) - - def create_device_stream_tables(self): - - for device_type in self.device_stream_map: - - for stream_type in self.device_stream_map[device_type]: - - table_class = get_device_stream_template(device_type, stream_type) - - self.context[table_class.__name__] = table_class - self._schema(table_class, context=self.context) - - self._schema.activate(schema_name) - - # endregion - - -if __name__ == "__main__": - - # Create device & device stream tables - tbmg = DeviceTableManager(context=inspect.currentframe().f_back.f_locals) - tbmg.create_device_tables() - tbmg.create_device_stream_tables() From 67d6b0c964ead3e3c1ce23d162cdc5ee756c5d07 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 14 Apr 2023 01:25:36 +0000 Subject: [PATCH 25/54] create new device tables in ingest_devices --- aeon/dj_pipeline/utils/load_metadata.py | 113 +++++++++++++++++------- 1 file changed, 79 insertions(+), 34 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 52dd4604..bcf97270 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -11,7 +11,14 @@ import pandas as pd from dotmap import DotMap -from aeon.dj_pipeline import acquisition, dict_to_uuid, lab, streams, subject +from aeon.dj_pipeline import ( + acquisition, + dict_to_uuid, + get_schema_name, + lab, + streams, + subject, +) from aeon.io import api as io_api _weight_scale_rate = 100 @@ -56,52 +63,90 @@ def ingest_streams(): def ingest_devices(schema: DotMap, metadata_yml_filepath: Path): - """Use dataset.schema and metadata.yml to insert into streams.DeviceType and streams.Device. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml.""" + """Use dataset.schema and metadata.yml to insert into streams.DeviceType and streams.Device. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml. It then creates new device tables under streams schema.""" device_info: dict[dict] = get_device_info(schema) device_type_mapper, device_sn = get_device_mapper(schema, metadata_yml_filepath) - # Add device type to device_info. + # Add device type to device_info. Only add if device types that are defined in Metadata.yml device_info = { device_name: { - "device_type": device_type_mapper.get(device_name, None), + "device_type": device_type_mapper.get(device_name), **device_info[device_name], } for device_name in device_info + if device_type_mapper.get(device_name) } - # Return only a list of device types that have been inserted. - for device_name, info in device_info.items(): - if info["device_type"]: + # Create a map of device_type to stream_type. + device_stream_map: dict[list] = {} - streams.DeviceType.insert1( - { - "device_type": info["device_type"], - "device_description": "", - }, - skip_duplicates=True, - ) - streams.DeviceType.Stream.insert( - [ - { - "device_type": info["device_type"], - "stream_type": e, - } - for e in info["stream_type"] - ], - skip_duplicates=True, - ) + for device_config in device_info.values(): + device_type = device_config["device_type"] + stream_types = device_config["stream_type"] - if streams.Device & {"device_serial_number": device_sn[device_name]}: - continue + if device_type not in device_stream_map: + device_stream_map[device_type] = [] - streams.Device.insert1( - { - "device_serial_number": device_sn[device_name] - or device_name, #! insert device name if not exists - "device_type": info["device_type"], - }, - skip_duplicates=True, - ) + for stream_type in stream_types: + if stream_type not in device_stream_map[device_type]: + device_stream_map[device_type].append(stream_type) + + # List only new device & stream types that need to be inserted & created. + new_device_types = [ + {"device_type": device_type} + for device_type in device_stream_map.keys() + if not streams.DeviceType & {"device_type": device_type} + ] + + new_device_stream_types = [ + {"device_type": device_type, "stream_type": stream_type} + for device_type, stream_list in device_stream_map.items() + for stream_type in stream_list + if not streams.DeviceType.Stream + & {"device_type": device_type, "stream_type": stream_type} + ] + + new_devices = [ + { + "device_serial_number": device_sn[device_name], + "device_type": device_config["device_type"], + } + for device_name, device_config in device_info.items() + if device_sn[device_name] + and not streams.Device & {"device_serial_number": device_sn[device_name]} + ] + + # Insert new entries. + if new_device_types: + streams.DeviceType.insert(new_device_types) + + if new_device_stream_types: + streams.DeviceType.Stream.insert(new_device_stream_types) + + if new_devices: + streams.Device.insert(new_devices) + + # Create tables. + context = inspect.currentframe().f_back.f_locals + + for device_info in new_device_types: + table_class = streams.get_device_template(device_info["device_type"]) + context[table_class.__name__] = table_class + streams.schema(table_class, context=context) + + # Create device_type tables + for device_info in new_device_stream_types: + table_class = streams.get_device_stream_template( + device_info["device_type"], device_info["stream_type"] + ) + context[table_class.__name__] = table_class + streams.schema(table_class, context=context) + + streams.schema.activate(streams.schema_name, add_objects=context) + vm = dj.VirtualModule(streams.schema_name, streams.schema_name) + for k, v in vm.__dict__.items(): + if "Table" in str(v.__class__): + streams.__dict__[k] = v def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> dict: From b820b0fcaf087fa27840519f89626613709c91de Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 14 Apr 2023 19:23:38 +0000 Subject: [PATCH 26/54] add new device ingestion routine in load_metadata.py --- aeon/dj_pipeline/utils/load_metadata.py | 424 ++++++++---------------- 1 file changed, 141 insertions(+), 283 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index bcf97270..7197fa69 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -178,20 +178,22 @@ def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> di assert commit, f'Neither "Commit" nor "Revision" found in {metadata_yml_filepath}' - devices: dict[str, dict] = json.loads( + devices: list[dict] = json.loads( json.dumps( epoch_config["metadata"]["Devices"], default=lambda x: x.__dict__, indent=4 ) ) - # devices: dict = {d.pop("Name"): d for d in devices} # {deivce_name: device_config} + devices: dict = { + d.pop("Name"): d for d in devices + } # {deivce_name: device_config} #! may not work for presocial return { "experiment_name": experiment_name, "epoch_start": epoch_start, "bonsai_workflow": epoch_config["workflow"], "commit": commit, - "devices": devices, + "metadata": devices, #! this format might have changed since using aeon metadata reader "metadata_file_path": metadata_yml_filepath, } @@ -203,6 +205,7 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): + camera/patch location + patch, weightscale serial number """ + from aeon.dj_pipeline import streams if experiment_name.startswith("oct"): ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath) @@ -224,305 +227,116 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): device_frequency_mapper = { name: float(value) - for name, value in epoch_config["devices"]["VideoController"].items() + for name, value in epoch_config["metadata"]["VideoController"].items() if name.endswith("Frequency") - } + } # May not be needed? - # ---- Load cameras ---- - camera_list, camera_installation_list, camera_removal_list, camera_position_list = ( - [], - [], - [], - [], - ) - # Check if this is a new camera, add to lab.Camera if needed - for device_name, device_config in epoch_config["devices"].items(): - if device_config["Type"] == "VideoSource": - camera_key = {"camera_serial_number": device_config["SerialNumber"]} - camera_list.append(camera_key) - - camera_installation = { - **camera_key, - "experiment_name": experiment_name, - "camera_install_time": epoch_config["epoch_start"], - "camera_description": device_name, - "camera_sampling_rate": device_frequency_mapper[ - device_config["TriggerFrequency"] - ], - "camera_gain": float(device_config["Gain"]), - "camera_bin": int(device_config["Binning"]), - } + # Insert into each device table + for device_name, device_config in epoch_config["metadata"].items(): + if table := getattr(streams, device_config["Type"], None): + device_sn = device_config.get("SerialNumber", device_config.get("PortName")) + device_key = {"device_serial_number": device_sn} - if "position" in device_config: - camera_position = { - **camera_key, + if not ( + table + & { "experiment_name": experiment_name, - "camera_install_time": epoch_config["epoch_start"], - "camera_position_x": device_config["position"]["x"], - "camera_position_y": device_config["position"]["y"], - "camera_position_z": device_config["position"]["z"], + "device_serial_number": device_sn, } - else: - camera_position = { - "camera_position_x": None, - "camera_position_y": None, - "camera_position_z": None, - "camera_rotation_x": None, - "camera_rotation_y": None, - "camera_rotation_z": None, - } - - """Check if this camera is currently installed. If the same camera serial number is currently installed check for any changes in configuration. If not, skip this""" - current_camera_query = ( - acquisition.ExperimentCamera - acquisition.ExperimentCamera.RemovalTime - & experiment_key - & camera_key - ) - - if current_camera_query: - current_camera_config = current_camera_query.join( - acquisition.ExperimentCamera.Position, left=True - ).fetch1() - - new_camera_config = {**camera_installation, **camera_position} - current_camera_config.pop("camera_install_time") - new_camera_config.pop("camera_install_time") - - if dict_to_uuid(current_camera_config) == dict_to_uuid( - new_camera_config - ): - continue - # Remove old camera - camera_removal_list.append( - { - **current_camera_query.fetch1("KEY"), - "camera_remove_time": epoch_config["epoch_start"], - } - ) - # Install new camera - camera_installation_list.append(camera_installation) - - if "position" in device_config: - camera_position_list.append(camera_position) - # Remove the currently installed cameras that are absent in this config - camera_removal_list.extend( - ( - acquisition.ExperimentCamera - - acquisition.ExperimentCamera.RemovalTime - - camera_list - & experiment_key - ).fetch("KEY") - ) + ): - # ---- Load food patches ---- - patch_list, patch_installation_list, patch_removal_list, patch_position_list = ( - [], - [], - [], - [], - ) - - # Check if this is a new food patch, add to lab.FoodPatch if needed - for device_name, device_config in epoch_config["devices"].items(): - if device_config["Type"] == "Patch": - - patch_key = { - "food_patch_serial_number": device_config.get( - "SerialNumber", device_config.get("PortName") - ) - } - patch_list.append(patch_key) - patch_installation = { - **patch_key, - "experiment_name": experiment_name, - "food_patch_install_time": epoch_config["epoch_start"], - "food_patch_description": device_config["Name"], - "wheel_sampling_rate": float( - re.search(r"\d+", device_config["SampleRate"]).group() - ), - "wheel_radius": float(device_config["Radius"]), - } - if "position" in device_config: - patch_position = { - **patch_key, + table_entry = { "experiment_name": experiment_name, - "food_patch_install_time": epoch_config["epoch_start"], - "food_patch_position_x": device_config["position"]["x"], - "food_patch_position_y": device_config["position"]["y"], - "food_patch_position_z": device_config["position"]["z"], - } - else: - patch_position = { - "food_patch_position_x": None, - "food_patch_position_y": None, - "food_patch_position_z": None, + "device_serial_number": device_sn, + f"{dj.utils.from_camel_case(table.__name__)}_install_time": epoch_config[ + "epoch_start" + ], + f"{dj.utils.from_camel_case(table.__name__)}_name": device_name, } - """Check if this camera is currently installed. If the same camera serial number is currently installed, check for any changes in configuration, if not, skip this""" - current_patch_query = ( - acquisition.ExperimentFoodPatch - - acquisition.ExperimentFoodPatch.RemovalTime - & experiment_key - & patch_key - ) - if current_patch_query: - current_patch_config = current_patch_query.join( - acquisition.ExperimentFoodPatch.Position, left=True - ).fetch1() - new_patch_config = {**patch_installation, **patch_position} - current_patch_config.pop("food_patch_install_time") - new_patch_config.pop("food_patch_install_time") - if dict_to_uuid(current_patch_config) == dict_to_uuid(new_patch_config): - continue - # Remove old food patch - patch_removal_list.append( + table_attribute_entry = [ { - **current_patch_query.fetch1("KEY"), - "food_patch_remove_time": epoch_config["epoch_start"], + "experiment_name": experiment_name, + "device_serial_number": device_sn, + f"{dj.utils.from_camel_case(table.__name__)}_install_time": epoch_config[ + "epoch_start" + ], + f"{dj.utils.from_camel_case(table.__name__)}_name": device_name, + "attribute_name": attribute_name, + "attribute_value": attribute_value, } - ) - # Install new food patch - patch_installation_list.append(patch_installation) - if "position" in device_config: - patch_position_list.append(patch_position) - # Remove the currently installed patches that are absent in this config - patch_removal_list.extend( - ( - acquisition.ExperimentFoodPatch - - acquisition.ExperimentFoodPatch.RemovalTime - - patch_list - & experiment_key - ).fetch("KEY") - ) - - # ---- Load weight scales ---- - weight_scale_list, weight_scale_installation_list, weight_scale_removal_list = ( - [], - [], - [], - ) + for attribute_name, attribute_value in device_config.items() + ] - # Check if this is a new weight scale, add to lab.WeightScale if needed - for device_name, device_config in epoch_config["devices"].items(): - if device_config["Type"] == "WeightScale": - - weight_scale_key = { - "weight_scale_serial_number": device_config.get( - "SerialNumber", device_config.get("PortName") + """Check if this camera is currently installed. If the same camera serial number is currently installed check for any changes in configuration. If not, skip this""" + current_device_query = ( + table.Attribute - table.RemovalTime & experiment_key & device_key ) - } - weight_scale_list.append(weight_scale_key) - arena_key = (lab.Arena & acquisition.Experiment & experiment_key).fetch1( - "KEY" - ) - weight_scale_installation = { - **weight_scale_key, - **arena_key, - "experiment_name": experiment_name, - "weight_scale_install_time": epoch_config["epoch_start"], - "nest": _weight_scale_nest, - "weight_scale_description": device_name, - "weight_scale_sampling_rate": float(_weight_scale_rate), - } - - # Check if this weight scale is currently installed - if so, remove it - current_weight_scale_query = ( - acquisition.ExperimentWeightScale - - acquisition.ExperimentWeightScale.RemovalTime - & experiment_key - & weight_scale_key - ) - if current_weight_scale_query: - current_weight_scale_config = current_weight_scale_query.fetch1() - new_weight_scale_config = weight_scale_installation.copy() - current_weight_scale_config.pop("weight_scale_install_time") - new_weight_scale_config.pop("weight_scale_install_time") - if dict_to_uuid(current_weight_scale_config) == dict_to_uuid( - new_weight_scale_config - ): - continue - # Remove old weight scale - weight_scale_removal_list.append( - { - **current_weight_scale_query.fetch1("KEY"), - "weight_scale_remove_time": epoch_config["epoch_start"], - } - ) - # Install new weight scale - weight_scale_installation_list.append(weight_scale_installation) - # Remove the currently installed weight scales that are absent in this config - weight_scale_removal_list.extend( - ( - acquisition.ExperimentWeightScale - - acquisition.ExperimentWeightScale.RemovalTime - - weight_scale_list - & experiment_key - ).fetch("KEY") - ) - - # Insert - def insert(): - lab.Camera.insert(camera_list, skip_duplicates=True) - acquisition.ExperimentCamera.RemovalTime.insert(camera_removal_list) - acquisition.ExperimentCamera.insert(camera_installation_list) - acquisition.ExperimentCamera.Position.insert(camera_position_list) - lab.FoodPatch.insert(patch_list, skip_duplicates=True) - acquisition.ExperimentFoodPatch.RemovalTime.insert(patch_removal_list) - acquisition.ExperimentFoodPatch.insert(patch_installation_list) - acquisition.ExperimentFoodPatch.Position.insert(patch_position_list) - lab.WeightScale.insert(weight_scale_list, skip_duplicates=True) - acquisition.ExperimentWeightScale.RemovalTime.insert(weight_scale_removal_list) - acquisition.ExperimentWeightScale.insert(weight_scale_installation_list) - - if acquisition.Experiment.connection.in_transaction: - insert() - else: - with acquisition.Experiment.connection.transaction: - insert() + if current_device_query: + current_device_config: list[dict] = current_device_query.fetch( + "experiment_name", + "device_serial_number", + "attribute_name", + "attribute_value", + as_dict=True, + ) + new_device_config: list[dict] = [ + { + k: v + for k, v in entry.items() + if k + != f"{dj.utils.from_camel_case(table.__name__)}_install_time" + } + for entry in table_attribute_entry + ] -def ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath): - """ - Temporary ingestion routine to load devices' meta information for Octagon arena experiments - """ - from aeon.dj_pipeline import streams + if dict_to_uuid(current_device_config) == dict_to_uuid( + new_device_config + ): # Skip if none of the configuration has changed. + continue + + # Remove old device + table_removal_entry = [ + { + **entry, + f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ + "epoch_start" + ], + } + for entry in current_device_config + ] - oct01_devices = [ - ("Metadata", "Metadata"), - ("CameraTop", "Camera"), - ("CameraColorTop", "Camera"), - ("ExperimentalMetadata", "ExperimentalMetadata"), - ("Photodiode", "Photodiode"), - ("OSC", "OSC"), - ("TaskLogic", "TaskLogic"), - ("Wall1", "Wall"), - ("Wall2", "Wall"), - ("Wall3", "Wall"), - ("Wall4", "Wall"), - ("Wall5", "Wall"), - ("Wall6", "Wall"), - ("Wall7", "Wall"), - ("Wall8", "Wall"), - ] + # Insert into table. + with table.connection.in_transaction: + table.insert1(table_entry) + table.Attribute.insert(table_attribute_entry) + table.RemovalTime.insert(table_removal_entry) - epoch_start = datetime.datetime.strptime( - metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" + # Remove the currently installed devices that are absent in this config + device_removal_list.extend( + (table - table.RemovalTime - device_list & experiment_key).fetch("KEY") ) - for device_idx, (device_name, device_type) in enumerate(oct01_devices): - device_sn = f"oct01_{device_idx}" - streams.Device.insert1( - {"device_serial_number": device_sn, "device_type": device_type}, - skip_duplicates=True, - ) - experiment_table = getattr(streams, f"Experiment{device_type}") - if not ( - experiment_table - & {"experiment_name": experiment_name, "device_serial_number": device_sn} - ): - experiment_table.insert1( - (experiment_name, device_sn, epoch_start, device_name) - ) + # Insert + # def insert(): + # lab.Camera.insert(camera_list, skip_duplicates=True) + # acquisition.ExperimentCamera.RemovalTime.insert(camera_removal_list) + # acquisition.ExperimentCamera.insert(camera_installation_list) + # acquisition.ExperimentCamera.Position.insert(camera_position_list) + # lab.FoodPatch.insert(patch_list, skip_duplicates=True) + # acquisition.ExperimentFoodPatch.RemovalTime.insert(patch_removal_list) + # acquisition.ExperimentFoodPatch.insert(patch_installation_list) + # acquisition.ExperimentFoodPatch.Position.insert(patch_position_list) + # lab.WeightScale.insert(weight_scale_list, skip_duplicates=True) + # acquisition.ExperimentWeightScale.RemovalTime.insert(weight_scale_removal_list) + # acquisition.ExperimentWeightScale.insert(weight_scale_installation_list) + + # if acquisition.Experiment.connection.in_transaction: + # insert() + # else: + # with acquisition.Experiment.connection.transaction: + # insert() # region Get stream & device information @@ -711,4 +525,48 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): return device_type_mapper, device_sn +def ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath): + """ + Temporary ingestion routine to load devices' meta information for Octagon arena experiments + """ + from aeon.dj_pipeline import streams + + oct01_devices = [ + ("Metadata", "Metadata"), + ("CameraTop", "Camera"), + ("CameraColorTop", "Camera"), + ("ExperimentalMetadata", "ExperimentalMetadata"), + ("Photodiode", "Photodiode"), + ("OSC", "OSC"), + ("TaskLogic", "TaskLogic"), + ("Wall1", "Wall"), + ("Wall2", "Wall"), + ("Wall3", "Wall"), + ("Wall4", "Wall"), + ("Wall5", "Wall"), + ("Wall6", "Wall"), + ("Wall7", "Wall"), + ("Wall8", "Wall"), + ] + + epoch_start = datetime.datetime.strptime( + metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" + ) + + for device_idx, (device_name, device_type) in enumerate(oct01_devices): + device_sn = f"oct01_{device_idx}" + streams.Device.insert1( + {"device_serial_number": device_sn, "device_type": device_type}, + skip_duplicates=True, + ) + experiment_table = getattr(streams, f"Experiment{device_type}") + if not ( + experiment_table + & {"experiment_name": experiment_name, "device_serial_number": device_sn} + ): + experiment_table.insert1( + (experiment_name, device_sn, epoch_start, device_name) + ) + + # endregion From 62fe7ee7ae33787c8b73ac5e0f2a31dc70d9878a Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 14 Apr 2023 19:24:02 +0000 Subject: [PATCH 27/54] add device_type_mapper.json --- aeon/dj_pipeline/create_experiments/device_type_mapper.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 aeon/dj_pipeline/create_experiments/device_type_mapper.json diff --git a/aeon/dj_pipeline/create_experiments/device_type_mapper.json b/aeon/dj_pipeline/create_experiments/device_type_mapper.json new file mode 100644 index 00000000..a802f72a --- /dev/null +++ b/aeon/dj_pipeline/create_experiments/device_type_mapper.json @@ -0,0 +1 @@ +{"VideoController": "VideoController", "CameraTop": "VideoSource", "CameraWest": "VideoSource", "CameraEast": "VideoSource", "CameraNorth": "VideoSource", "CameraSouth": "VideoSource", "CameraPatch1": "VideoSource", "CameraPatch2": "VideoSource", "CameraNest": "VideoSource", "AudioAmbient": "AudioSource", "Patch1": "Patch", "Patch2": "Patch", "WeightNest": "WeightScale", "TrackingTop": "PositionTracking", "ActivityCenter": "ActivityTracking", "ActivityArena": "ActivityTracking", "ActivityNest": "ActivityTracking", "ActivityPatch1": "ActivityTracking", "ActivityPatch2": "ActivityTracking", "InNest": "RegionTracking", "InPatch1": "RegionTracking", "InPatch2": "RegionTracking", "ArenaCenter": "DistanceFromPoint", "InArena": "InRange", "InCorridor": "InRange"} \ No newline at end of file From b40edfdf98818a7529b21a656126e5d557a3cb46 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 14 Apr 2023 19:25:33 +0000 Subject: [PATCH 28/54] change attribute name to removal time --- aeon/dj_pipeline/streams.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 37f4596e..a9bedb31 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -74,24 +74,24 @@ class ExperimentDevice(dj.Manual): # {device_title} placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-{aeon.__version__}) -> acquisition.Experiment -> streams.Device - {device_type}_install_time: datetime(6) # time of the {device_type} placed and started operation at this position + {device_type}_install_time : datetime(6) # time of the {device_type} placed and started operation at this position --- - {device_type}_name: varchar(36) + {device_type}_name : varchar(36) """ class Attribute(dj.Part): definition = """ # metadata/attributes (e.g. FPS, config, calibration, etc.) associated with this experimental device -> master - attribute_name : varchar(32) + attribute_name : varchar(32) --- - attribute_value='': varchar(2000) + attribute_value=null : longblob """ class RemovalTime(dj.Part): definition = f""" -> master --- - {device_type}_remove_time: datetime(6) # time of the {device_type} being removed + {device_type}_removal_time: datetime(6) # time of the {device_type} being removed """ ExperimentDevice.__name__ = f"{device_title}" From fd1835683df2203c0531aeddf7e02175331f2ea2 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 14 Apr 2023 19:27:07 +0000 Subject: [PATCH 29/54] add ingest_device function into ingest_epochs --- aeon/dj_pipeline/acquisition.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index f28c1ef4..cdd10d0a 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -12,7 +12,11 @@ from . import get_schema_name, lab, subject from .utils import paths -from .utils.load_metadata import extract_epoch_config, ingest_epoch_metadata +from .utils.load_metadata import ( + extract_epoch_config, + ingest_devices, + ingest_epoch_metadata, +) logger = dj.logger schema = dj.schema(get_schema_name("acquisition")) @@ -348,6 +352,13 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): cls.insert1(epoch_key) if epoch_config: cls.Config.insert1(epoch_config) + + # Ingest streams.DeviceType, streams.Device and create device tables. + ingest_devices( + _device_schema_mapping[epoch_key["experiment_name"]], + metadata_yml_filepath, + ) + ingest_epoch_metadata(experiment_name, metadata_yml_filepath) epoch_list.append(epoch_key) # update previous epoch From 21988035b8ac3355db38986f11970be598e5acec Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 17 Apr 2023 15:23:22 +0000 Subject: [PATCH 30/54] use PortName if SerialNumber doesn't exist --- aeon/dj_pipeline/utils/load_metadata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 7197fa69..c8dedc18 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -515,8 +515,9 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): for item in meta_data.Devices: device_type_mapper[item.Name] = item.Type device_sn[item.Name] = ( - item.SerialNumber if not isinstance(item.SerialNumber, DotMap) else None - ) + item.SerialNumber or item.PortName or None + ) # assign either the serial number (if it exists) or port name. If neither exists, assign None + with filename.open("w") as f: json.dump(device_type_mapper, f) except AttributeError: From 420c83140b80b5682d92db9ef1a9e967ba5a2eba Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 17 Apr 2023 15:27:50 +0000 Subject: [PATCH 31/54] handle different data structure between exp02 and presocial --- aeon/dj_pipeline/utils/load_metadata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index c8dedc18..bf4dae9c 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -184,9 +184,10 @@ def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> di ) ) - devices: dict = { - d.pop("Name"): d for d in devices - } # {deivce_name: device_config} #! may not work for presocial + if isinstance(devices, list): # In exp02, it is a list of dict. In presocial. It's a dict of dict. + devices: dict = { + d.pop("Name"): d for d in devices + } # {deivce_name: device_config} return { "experiment_name": experiment_name, From 3d9ca95c75add5725bdb0545c0563e127e24ddc5 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 17 Apr 2023 15:28:37 +0000 Subject: [PATCH 32/54] run insert_stream_types when worker starts --- aeon/dj_pipeline/populate/worker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aeon/dj_pipeline/populate/worker.py b/aeon/dj_pipeline/populate/worker.py index 4c506347..7b1678dc 100644 --- a/aeon/dj_pipeline/populate/worker.py +++ b/aeon/dj_pipeline/populate/worker.py @@ -30,6 +30,7 @@ logger = dj.logger _current_experiment = "exp0.2-r0" worker_schema_name = db_prefix + "workerlog" +load_metadata.insert_stream_types() # ---- Define worker(s) ---- @@ -41,9 +42,7 @@ run_duration=-1, sleep_duration=600, ) - high_priority(load_metadata.ingest_subject) -high_priority(load_metadata.ingest_streams) high_priority(acquisition.Epoch.ingest_epochs, experiment_name=_current_experiment) high_priority(acquisition.Chunk.ingest_chunks, experiment_name=_current_experiment) high_priority(acquisition.ExperimentLog) From d03dd93f8b75a8ccbb13808b22ab09bb992cb8b1 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 17 Apr 2023 15:29:15 +0000 Subject: [PATCH 33/54] code review --- aeon/dj_pipeline/acquisition.py | 9 +-- aeon/dj_pipeline/streams.py | 26 ++++++++ aeon/dj_pipeline/utils/load_metadata.py | 84 +++++++------------------ 3 files changed, 54 insertions(+), 65 deletions(-) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index cdd10d0a..4a598184 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -10,17 +10,17 @@ from aeon.io import reader as io_reader from aeon.schema import dataset as aeon_schema -from . import get_schema_name, lab, subject +from . import get_schema_name, lab, streams, subject from .utils import paths from .utils.load_metadata import ( extract_epoch_config, - ingest_devices, ingest_epoch_metadata, + insert_device_types, ) logger = dj.logger schema = dj.schema(get_schema_name("acquisition")) - +# streams = dj.VirtualModule("streams", get_schema_name("streams")) # ------------------- Some Constants -------------------------- @@ -354,10 +354,11 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): cls.Config.insert1(epoch_config) # Ingest streams.DeviceType, streams.Device and create device tables. - ingest_devices( + insert_device_types( _device_schema_mapping[epoch_key["experiment_name"]], metadata_yml_filepath, ) + streams.main() ingest_epoch_metadata(experiment_name, metadata_yml_filepath) epoch_list.append(epoch_key) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index a9bedb31..b59a0c66 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -1,3 +1,5 @@ +import inspect + import datajoint as dj import pandas as pd @@ -12,6 +14,8 @@ schema_name = get_schema_name("streams") schema = dj.schema(schema_name) +schema.spawn_missing_classes() + @schema class StreamType(dj.Lookup): @@ -194,3 +198,25 @@ def make(self, key): # endregion + + +def main(): + + context = inspect.currentframe().f_back.f_locals + + # Create tables. + for device_info in (DeviceType).fetch(as_dict=True): + table_class = get_device_template(device_info["device_type"]) + context[table_class.__name__] = table_class + schema(table_class, context=context) + + # Create DeviceDataStream tables. + for device_info in (DeviceType.Stream).fetch(as_dict=True): + table_class = get_device_stream_template( + device_info["device_type"], device_info["stream_type"] + ) + context[table_class.__name__] = table_class + schema(table_class, context=context) + + +main() diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index bf4dae9c..3414c624 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -39,7 +39,7 @@ def ingest_subject(colony_csv_path: pathlib.Path = _colony_csv_path) -> None: ) -def ingest_streams(): +def insert_stream_types(): """Insert into streams.streamType table all streams in the dataset schema.""" from aeon.schema import dataset @@ -62,7 +62,7 @@ def ingest_streams(): streams.StreamType.insert(stream_entries, skip_duplicates=True) -def ingest_devices(schema: DotMap, metadata_yml_filepath: Path): +def insert_device_types(schema: DotMap, metadata_yml_filepath: Path): """Use dataset.schema and metadata.yml to insert into streams.DeviceType and streams.Device. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml. It then creates new device tables under streams schema.""" device_info: dict[dict] = get_device_info(schema) device_type_mapper, device_sn = get_device_mapper(schema, metadata_yml_filepath) @@ -126,28 +126,6 @@ def ingest_devices(schema: DotMap, metadata_yml_filepath: Path): if new_devices: streams.Device.insert(new_devices) - # Create tables. - context = inspect.currentframe().f_back.f_locals - - for device_info in new_device_types: - table_class = streams.get_device_template(device_info["device_type"]) - context[table_class.__name__] = table_class - streams.schema(table_class, context=context) - - # Create device_type tables - for device_info in new_device_stream_types: - table_class = streams.get_device_stream_template( - device_info["device_type"], device_info["stream_type"] - ) - context[table_class.__name__] = table_class - streams.schema(table_class, context=context) - - streams.schema.activate(streams.schema_name, add_objects=context) - vm = dj.VirtualModule(streams.schema_name, streams.schema_name) - for k, v in vm.__dict__.items(): - if "Table" in str(v.__class__): - streams.__dict__[k] = v - def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> dict: """Parse experiment metadata YAML file and extract epoch configuration. @@ -208,6 +186,8 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): """ from aeon.dj_pipeline import streams + streams = dj.VirtualModule("streams", get_schema_name("streams")) + if experiment_name.startswith("oct"): ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath) return @@ -233,11 +213,12 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): } # May not be needed? # Insert into each device table + device_list = [] for device_name, device_config in epoch_config["metadata"].items(): if table := getattr(streams, device_config["Type"], None): device_sn = device_config.get("SerialNumber", device_config.get("PortName")) device_key = {"device_serial_number": device_sn} - + device_list.append(device_key) if not ( table & { @@ -245,7 +226,6 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): "device_serial_number": device_sn, } ): - table_entry = { "experiment_name": experiment_name, "device_serial_number": device_sn, @@ -271,11 +251,13 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): """Check if this camera is currently installed. If the same camera serial number is currently installed check for any changes in configuration. If not, skip this""" current_device_query = ( - table.Attribute - table.RemovalTime & experiment_key & device_key + table - table.RemovalTime & experiment_key & device_key ) if current_device_query: - current_device_config: list[dict] = current_device_query.fetch( + current_device_config: list[dict] = ( + table.Attribute & current_device_query + ).fetch( "experiment_name", "device_serial_number", "attribute_name", @@ -298,46 +280,26 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): continue # Remove old device - table_removal_entry = [ - { - **entry, - f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ - "epoch_start" - ], - } - for entry in current_device_config - ] + table_removal_entry = { + **table_entry, + f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ + "epoch_start" + ], + } # Insert into table. with table.connection.in_transaction: table.insert1(table_entry) table.Attribute.insert(table_attribute_entry) - table.RemovalTime.insert(table_removal_entry) + table.RemovalTime.insert1( + table_removal_entry, ignore_extra_fields=True + ) # Remove the currently installed devices that are absent in this config - device_removal_list.extend( - (table - table.RemovalTime - device_list & experiment_key).fetch("KEY") - ) - - # Insert - # def insert(): - # lab.Camera.insert(camera_list, skip_duplicates=True) - # acquisition.ExperimentCamera.RemovalTime.insert(camera_removal_list) - # acquisition.ExperimentCamera.insert(camera_installation_list) - # acquisition.ExperimentCamera.Position.insert(camera_position_list) - # lab.FoodPatch.insert(patch_list, skip_duplicates=True) - # acquisition.ExperimentFoodPatch.RemovalTime.insert(patch_removal_list) - # acquisition.ExperimentFoodPatch.insert(patch_installation_list) - # acquisition.ExperimentFoodPatch.Position.insert(patch_position_list) - # lab.WeightScale.insert(weight_scale_list, skip_duplicates=True) - # acquisition.ExperimentWeightScale.RemovalTime.insert(weight_scale_removal_list) - # acquisition.ExperimentWeightScale.insert(weight_scale_installation_list) - - # if acquisition.Experiment.connection.in_transaction: - # insert() - # else: - # with acquisition.Experiment.connection.transaction: - # insert() + device_removal_list = ( + table - table.RemovalTime - device_list & experiment_key + ).fetch("KEY") + table.RemovalTime.insert1(device_removal_list, ignore_extra_fields=True) # region Get stream & device information From 4fc8aae1073672f4ab9747b6c636ccf2b1c0cf67 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 18 Apr 2023 21:13:58 +0000 Subject: [PATCH 34/54] thinh change --- aeon/dj_pipeline/acquisition.py | 14 +++++++++++--- aeon/dj_pipeline/utils/video.py | 20 ++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index 4a598184..6837893e 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -352,16 +352,24 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): cls.insert1(epoch_key) if epoch_config: cls.Config.insert1(epoch_config) - + if metadata_yml_filepath and metadata_yml_filepath.exists(): + try: # Ingest streams.DeviceType, streams.Device and create device tables. insert_device_types( _device_schema_mapping[epoch_key["experiment_name"]], metadata_yml_filepath, ) streams.main() + with cls.connection.transaction: + ingest_epoch_metadata( + experiment_name, metadata_yml_filepath + ) + epoch_list.append(epoch_key) + except Exception as e: + (cls.Config & epoch_key).delete_quick() + (cls & epoch_key).delete_quick() + raise e - ingest_epoch_metadata(experiment_name, metadata_yml_filepath) - epoch_list.append(epoch_key) # update previous epoch if ( previous_epoch_key diff --git a/aeon/dj_pipeline/utils/video.py b/aeon/dj_pipeline/utils/video.py index 15cdbfb7..60434419 100644 --- a/aeon/dj_pipeline/utils/video.py +++ b/aeon/dj_pipeline/utils/video.py @@ -1,18 +1,15 @@ -import numpy as np import base64 -import pandas as pd -import pathlib import datetime +import pathlib + import cv2 +import numpy as np +import pandas as pd +import aeon.io.reader as io_reader from aeon.io import api as io_api from aeon.io import video as io_video -import aeon.io.reader as io_reader - -camera_name = "CameraTop" -start_time = datetime.datetime(2022, 4, 3, 13, 0, 0) -end_time = datetime.datetime(2022, 4, 3, 15, 0, 0) raw_data_dir = pathlib.Path("/ceph/aeon/aeon/data/raw/AEON2/experiment0.2") @@ -39,11 +36,6 @@ def retrieve_video_frames( framedata = videodata[start_frame : start_frame + chunk_size] - # downsample - # actual_fps = 1 / np.median(np.diff(videodata.index) / np.timedelta64(1, "s")) - # final_fps = min(desired_fps, actual_fps) - # ds_factor = int(np.around(actual_fps / final_fps)) - # framedata = videodata[::ds_factor] final_fps = desired_fps # read frames @@ -64,4 +56,4 @@ def retrieve_video_frames( "finalChunk": bool(last_frame_time >= end_time), }, "frames": encoded_frames, - } + } \ No newline at end of file From 9daa25563be7204beacd8a8567012eec781110fe Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 18 Apr 2023 21:36:41 +0000 Subject: [PATCH 35/54] auto insertion of device entries --- aeon/dj_pipeline/acquisition.py | 3 +- aeon/dj_pipeline/streams.py | 3 - aeon/dj_pipeline/utils/load_metadata.py | 139 ++++++++++++------------ 3 files changed, 69 insertions(+), 76 deletions(-) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index 6837893e..9d7cad95 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -10,7 +10,7 @@ from aeon.io import reader as io_reader from aeon.schema import dataset as aeon_schema -from . import get_schema_name, lab, streams, subject +from . import get_schema_name, lab, subject from .utils import paths from .utils.load_metadata import ( extract_epoch_config, @@ -275,6 +275,7 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): - if not specified, ingest all epochs Note: "start" and "end" are datetime specified a string in the format: "%Y-%m-%d %H:%M:%S" """ + from aeon.dj_pipeline import acquisition, streams device_name = _ref_device_mapping.get(experiment_name, "CameraTop") all_chunks, raw_data_dirs = _get_all_chunks(experiment_name, device_name) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index b59a0c66..435bd441 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -217,6 +217,3 @@ def main(): ) context[table_class.__name__] = table_class schema(table_class, context=context) - - -main() diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 3414c624..8c8cca94 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -137,6 +137,7 @@ def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> di Returns: dict: epoch_config [dict] """ + metadata_yml_filepath = pathlib.Path(metadata_yml_filepath) epoch_start = datetime.datetime.strptime( metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" @@ -184,9 +185,8 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): + camera/patch location + patch, weightscale serial number """ - from aeon.dj_pipeline import streams - streams = dj.VirtualModule("streams", get_schema_name("streams")) + # streams = dj.VirtualModule("streams", get_schema_name("streams")) if experiment_name.startswith("oct"): ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath) @@ -214,92 +214,87 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): # Insert into each device table device_list = [] + device_removal_list = [] + for device_name, device_config in epoch_config["metadata"].items(): if table := getattr(streams, device_config["Type"], None): device_sn = device_config.get("SerialNumber", device_config.get("PortName")) device_key = {"device_serial_number": device_sn} + device_list.append(device_key) - if not ( - table - & { - "experiment_name": experiment_name, - "device_serial_number": device_sn, - } - ): - table_entry = { - "experiment_name": experiment_name, - "device_serial_number": device_sn, - f"{dj.utils.from_camel_case(table.__name__)}_install_time": epoch_config[ - "epoch_start" - ], - f"{dj.utils.from_camel_case(table.__name__)}_name": device_name, + table_entry = { + "experiment_name": experiment_name, + **device_key, + f"{dj.utils.from_camel_case(table.__name__)}_install_time": epoch_config[ + "epoch_start" + ], + f"{dj.utils.from_camel_case(table.__name__)}_name": device_name, + } + + table_attribute_entry = [ + { + **table_entry, + "attribute_name": attribute_name, + "attribute_value": attribute_value, } + for attribute_name, attribute_value in device_config.items() + ] - table_attribute_entry = [ + """Check if this camera is currently installed. If the same camera serial number is currently installed check for any changes in configuration. If not, skip this""" + current_device_query = ( + table - table.RemovalTime & experiment_key & device_key + ) + + if current_device_query: + current_device_config: list[dict] = ( + table.Attribute & current_device_query + ).fetch( + "experiment_name", + "device_serial_number", + "attribute_name", + "attribute_value", + as_dict=True, + ) + new_device_config: list[dict] = [ { - "experiment_name": experiment_name, - "device_serial_number": device_sn, - f"{dj.utils.from_camel_case(table.__name__)}_install_time": epoch_config[ - "epoch_start" - ], - f"{dj.utils.from_camel_case(table.__name__)}_name": device_name, - "attribute_name": attribute_name, - "attribute_value": attribute_value, + k: v + for k, v in entry.items() + if k + != f"{dj.utils.from_camel_case(table.__name__)}_install_time" } - for attribute_name, attribute_value in device_config.items() + for entry in table_attribute_entry ] - """Check if this camera is currently installed. If the same camera serial number is currently installed check for any changes in configuration. If not, skip this""" - current_device_query = ( - table - table.RemovalTime & experiment_key & device_key - ) - - if current_device_query: - current_device_config: list[dict] = ( - table.Attribute & current_device_query - ).fetch( - "experiment_name", - "device_serial_number", - "attribute_name", - "attribute_value", - as_dict=True, - ) - new_device_config: list[dict] = [ - { - k: v - for k, v in entry.items() - if k - != f"{dj.utils.from_camel_case(table.__name__)}_install_time" - } - for entry in table_attribute_entry - ] + if dict_to_uuid(current_device_config) == dict_to_uuid( + new_device_config + ): # Skip if none of the configuration has changed. + continue - if dict_to_uuid(current_device_config) == dict_to_uuid( - new_device_config - ): # Skip if none of the configuration has changed. - continue - - # Remove old device - table_removal_entry = { - **table_entry, - f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ - "epoch_start" - ], - } + # Remove old device + table_removal_entry = { + **table_entry, + f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ + "epoch_start" + ], + } - # Insert into table. - with table.connection.in_transaction: - table.insert1(table_entry) - table.Attribute.insert(table_attribute_entry) + # Insert into table. + with table.connection.in_transaction: + table.insert1(table_entry, skip_duplicates=True) + table.Attribute.insert(table_attribute_entry, ignore_extra_fields=True) + try: table.RemovalTime.insert1( table_removal_entry, ignore_extra_fields=True ) - - # Remove the currently installed devices that are absent in this config - device_removal_list = ( - table - table.RemovalTime - device_list & experiment_key - ).fetch("KEY") - table.RemovalTime.insert1(device_removal_list, ignore_extra_fields=True) + except NameError: + pass + + # Remove the currently installed devices that are absent in this config + device_removal_list = ( + table - table.RemovalTime - device_list & experiment_key + ).fetch("KEY") + if device_removal_list: + table.RemovalTime.insert(device_removal_list, ignore_extra_fields=True) # region Get stream & device information From 13b3301a734dbf8f445fbe5415f70dbd937e1ecc Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 24 Apr 2023 22:42:08 +0000 Subject: [PATCH 36/54] fix bugs --- aeon/dj_pipeline/acquisition.py | 2 +- aeon/dj_pipeline/utils/load_metadata.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index 9d7cad95..513c960f 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -360,7 +360,7 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): _device_schema_mapping[epoch_key["experiment_name"]], metadata_yml_filepath, ) - streams.main() + streams.main() # create device tables under streams schema with cls.connection.transaction: ingest_epoch_metadata( experiment_name, metadata_yml_filepath diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 8c8cca94..be85d5b0 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -74,7 +74,7 @@ def insert_device_types(schema: DotMap, metadata_yml_filepath: Path): **device_info[device_name], } for device_name in device_info - if device_type_mapper.get(device_name) + if device_type_mapper.get(device_name) and device_sn.get(device_name) } # Create a map of device_type to stream_type. @@ -180,10 +180,7 @@ def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> di def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): """ - work-in-progress - Missing: - + camera/patch location - + patch, weightscale serial number + Make entries into device tables """ # streams = dj.VirtualModule("streams", get_schema_name("streams")) @@ -212,12 +209,16 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): if name.endswith("Frequency") } # May not be needed? + schema = acquisition._device_schema_mapping[experiment_name] + device_type_mapper, _ = get_device_mapper(schema, metadata_yml_filepath) + # Insert into each device table device_list = [] device_removal_list = [] for device_name, device_config in epoch_config["metadata"].items(): - if table := getattr(streams, device_config["Type"], None): + if table := getattr(streams, device_type_mapper.get(device_name) or "", None): + device_sn = device_config.get("SerialNumber", device_config.get("PortName")) device_key = {"device_serial_number": device_sn} From 835af127c043f52bee5ec79664f380c1de55eb62 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 24 Apr 2023 22:44:09 +0000 Subject: [PATCH 37/54] add device removal time --- aeon/dj_pipeline/utils/load_metadata.py | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index be85d5b0..6683d029 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -260,29 +260,28 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): { k: v for k, v in entry.items() - if k - != f"{dj.utils.from_camel_case(table.__name__)}_install_time" + if dj.utils.from_camel_case(table.__name__) not in k } for entry in table_attribute_entry ] - if dict_to_uuid(current_device_config) == dict_to_uuid( - new_device_config + if dict_to_uuid({config["attribute_name"]: config["attribute_value"] for config in current_device_config}) == dict_to_uuid( + {config["attribute_name"]: config["attribute_value"] for config in new_device_config} ): # Skip if none of the configuration has changed. continue # Remove old device - table_removal_entry = { - **table_entry, + device_removal_list.append({ + **current_device_query.fetch1("KEY"), f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ "epoch_start" ], - } + }) # Insert into table. with table.connection.in_transaction: - table.insert1(table_entry, skip_duplicates=True) - table.Attribute.insert(table_attribute_entry, ignore_extra_fields=True) + table.insert1(table_entry, skip_duplicates=True) + table.Attribute.insert(table_attribute_entry, ignore_extra_fields=True) try: table.RemovalTime.insert1( table_removal_entry, ignore_extra_fields=True @@ -290,12 +289,13 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): except NameError: pass - # Remove the currently installed devices that are absent in this config - device_removal_list = ( - table - table.RemovalTime - device_list & experiment_key - ).fetch("KEY") - if device_removal_list: - table.RemovalTime.insert(device_removal_list, ignore_extra_fields=True) + # Remove the currently installed devices that are absent in this config + device_removal_list.extend( + (table - table.RemovalTime - device_list & experiment_key + ).fetch("KEY")) + + if device_removal_list: + table.RemovalTime.insert(device_removal_list, ignore_extra_fields=True) # region Get stream & device information From d1ca8e1344216ee64883b62845a26345c62cf941 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 24 Apr 2023 22:44:23 +0000 Subject: [PATCH 38/54] add mapping dictionary --- aeon/dj_pipeline/create_experiments/device_type_mapper.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aeon/dj_pipeline/create_experiments/device_type_mapper.json b/aeon/dj_pipeline/create_experiments/device_type_mapper.json index a802f72a..028ae594 100644 --- a/aeon/dj_pipeline/create_experiments/device_type_mapper.json +++ b/aeon/dj_pipeline/create_experiments/device_type_mapper.json @@ -1 +1 @@ -{"VideoController": "VideoController", "CameraTop": "VideoSource", "CameraWest": "VideoSource", "CameraEast": "VideoSource", "CameraNorth": "VideoSource", "CameraSouth": "VideoSource", "CameraPatch1": "VideoSource", "CameraPatch2": "VideoSource", "CameraNest": "VideoSource", "AudioAmbient": "AudioSource", "Patch1": "Patch", "Patch2": "Patch", "WeightNest": "WeightScale", "TrackingTop": "PositionTracking", "ActivityCenter": "ActivityTracking", "ActivityArena": "ActivityTracking", "ActivityNest": "ActivityTracking", "ActivityPatch1": "ActivityTracking", "ActivityPatch2": "ActivityTracking", "InNest": "RegionTracking", "InPatch1": "RegionTracking", "InPatch2": "RegionTracking", "ArenaCenter": "DistanceFromPoint", "InArena": "InRange", "InCorridor": "InRange"} \ No newline at end of file +{"VideoController": "VideoController", "CameraTop": "VideoSource", "CameraWest": "VideoSource", "CameraEast": "VideoSource", "CameraNorth": "VideoSource", "CameraSouth": "VideoSource", "CameraPatch1": "VideoSource", "CameraPatch2": "VideoSource", "CameraNest": "VideoSource", "AudioAmbient": "AudioSource", "Patch1": "Patch", "Patch2": "Patch", "WeightNest": "WeightScale", "TrackingTop": "PositionTracking", "ActivityCenter": "ActivityTracking", "ActivityArena": "ActivityTracking", "ActivityNest": "ActivityTracking", "ActivityPatch1": "ActivityTracking", "ActivityPatch2": "ActivityTracking", "InNest": "RegionTracking", "InPatch1": "RegionTracking", "InPatch2": "RegionTracking", "ArenaCenter": "DistanceFromPoint", "InArena": "InRange", "InCorridor": "InRange", "ClockSynchronizer": null} \ No newline at end of file From a14520f2c32a393b29cde4b1cad7ada210dad9fc Mon Sep 17 00:00:00 2001 From: JaerongA Date: Mon, 24 Apr 2023 22:47:43 +0000 Subject: [PATCH 39/54] support different metadata structure for presocial --- aeon/dj_pipeline/utils/load_metadata.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 6683d029..2b66bb83 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -183,8 +183,6 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): Make entries into device tables """ - # streams = dj.VirtualModule("streams", get_schema_name("streams")) - if experiment_name.startswith("oct"): ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath) return @@ -279,15 +277,8 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): }) # Insert into table. - with table.connection.in_transaction: table.insert1(table_entry, skip_duplicates=True) table.Attribute.insert(table_attribute_entry, ignore_extra_fields=True) - try: - table.RemovalTime.insert1( - table_removal_entry, ignore_extra_fields=True - ) - except NameError: - pass # Remove the currently installed devices that are absent in this config device_removal_list.extend( @@ -464,7 +455,7 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): ) device_type_mapper = {} # {device_name: device_type} - device_sn = {} # device serial number + device_sn = {} # {device_name: device_sn} if filename.is_file(): with filename.open("r") as f: @@ -472,10 +463,15 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): try: # if the device type is not in the mapper, add it for item in meta_data.Devices: - device_type_mapper[item.Name] = item.Type - device_sn[item.Name] = ( - item.SerialNumber or item.PortName or None - ) # assign either the serial number (if it exists) or port name. If neither exists, assign None + if isinstance(item, DotMap): + device_type_mapper[item.Name] = item.Type + device_sn[item.Name] = ( + item.SerialNumber or item.PortName or None + ) # assign either the serial number (if it exists) or port name. If neither exists, assign None + elif isinstance(item, str): # presocial + if meta_data.Devices[item].get("Type"): + device_type_mapper[item] = meta_data.Devices[item].get("Type") + device_sn[item] = meta_data.Devices[item].get("SerialNumber") or meta_data.Devices[item].get("PortName") or None with filename.open("w") as f: json.dump(device_type_mapper, f) From 668d9b7901ce74add12b338651dada786fdd189d Mon Sep 17 00:00:00 2001 From: JaerongA Date: Thu, 27 Apr 2023 04:50:22 +0000 Subject: [PATCH 40/54] add device removal time for each device table --- aeon/dj_pipeline/utils/load_metadata.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 2b66bb83..534acf65 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -182,7 +182,7 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): """ Make entries into device tables """ - + if experiment_name.startswith("oct"): ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath) return @@ -281,12 +281,18 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): table.Attribute.insert(table_attribute_entry, ignore_extra_fields=True) # Remove the currently installed devices that are absent in this config - device_removal_list.extend( - (table - table.RemovalTime - device_list & experiment_key - ).fetch("KEY")) + for device_type in streams.DeviceType.fetch("device_type"): + table = getattr(streams, device_type) - if device_removal_list: - table.RemovalTime.insert(device_removal_list, ignore_extra_fields=True) + device_removal_list.extend( + (table - table.RemovalTime - device_list & experiment_key + ).fetch("KEY")) + + if device_removal_list: + try: + table.RemovalTime.insert(device_removal_list, ignore_extra_fields=True) + except: + pass # region Get stream & device information From e340e64ee3ba4264ce5eee9e8f74511d9df3cf73 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Thu, 27 Apr 2023 04:53:37 +0000 Subject: [PATCH 41/54] fix streams populate error --- aeon/dj_pipeline/streams.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 435bd441..d76a8767 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -77,7 +77,7 @@ class ExperimentDevice(dj.Manual): definition = f""" # {device_title} placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-{aeon.__version__}) -> acquisition.Experiment - -> streams.Device + -> Device {device_type}_install_time : datetime(6) # time of the {device_type} placed and started operation at this position --- {device_type}_name : varchar(36) @@ -105,10 +105,10 @@ class RemovalTime(dj.Part): def get_device_stream_template(device_type: str, stream_type: str): """Returns table class template for DeviceDataStream""" - - ExperimentDevice = get_device_template(device_type) - exp_device_table_name = f"{device_type}" - + + context = inspect.currentframe().f_back.f_locals["context"] + ExperimentDevice = context[device_type] + # DeviceDataStream table(s) stream_detail = ( StreamType @@ -144,15 +144,15 @@ class DeviceDataStream(dj.Imported): @property def key_source(self): f""" - Only the combination of Chunk and {exp_device_table_name} with overlapping time - + Chunk(s) that started after {exp_device_table_name} install time and ended before {exp_device_table_name} remove time - + Chunk(s) that started after {exp_device_table_name} install time for {exp_device_table_name} that are not yet removed + Only the combination of Chunk and {device_type} with overlapping time + + Chunk(s) that started after {device_type} install time and ended before {device_type} remove time + + Chunk(s) that started after {device_type} install time for {device_type} that are not yet removed """ return ( acquisition.Chunk * ExperimentDevice.join(ExperimentDevice.RemovalTime, left=True) - & f"chunk_start >= {device_type}_install_time" - & f'chunk_start < IFNULL({device_type}_remove_time, "2200-01-01")' + & f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time" + & f'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")' ) def make(self, key): @@ -163,7 +163,7 @@ def make(self, key): key, directory_type=dir_type ) - device_name = (ExperimentDevice & key).fetch1(f"{device_type}_name") + device_name = (ExperimentDevice & key).fetch1(f"{dj.utils.from_camel_case(device_type)}_name") stream = self._stream_reader( **{ @@ -200,16 +200,17 @@ def make(self, key): # endregion -def main(): - context = inspect.currentframe().f_back.f_locals +def main(context=None): + if context is None: + context = inspect.currentframe().f_back.f_locals # Create tables. for device_info in (DeviceType).fetch(as_dict=True): table_class = get_device_template(device_info["device_type"]) context[table_class.__name__] = table_class schema(table_class, context=context) - + # Create DeviceDataStream tables. for device_info in (DeviceType.Stream).fetch(as_dict=True): table_class = get_device_stream_template( @@ -217,3 +218,5 @@ def main(): ) context[table_class.__name__] = table_class schema(table_class, context=context) + +main() \ No newline at end of file From e29c1d18c623d81ee8388cdea08c5cd7bf4fe5d8 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Thu, 27 Apr 2023 04:54:09 +0000 Subject: [PATCH 42/54] fix circular import error --- aeon/dj_pipeline/acquisition.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index 513c960f..b4163420 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -12,11 +12,6 @@ from . import get_schema_name, lab, subject from .utils import paths -from .utils.load_metadata import ( - extract_epoch_config, - ingest_epoch_metadata, - insert_device_types, -) logger = dj.logger schema = dj.schema(get_schema_name("acquisition")) @@ -275,7 +270,14 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): - if not specified, ingest all epochs Note: "start" and "end" are datetime specified a string in the format: "%Y-%m-%d %H:%M:%S" """ - from aeon.dj_pipeline import acquisition, streams + from aeon.dj_pipeline import streams + + from .utils.load_metadata import ( + extract_epoch_config, + ingest_epoch_metadata, + insert_device_types, + ) + device_name = _ref_device_mapping.get(experiment_name, "CameraTop") all_chunks, raw_data_dirs = _get_all_chunks(experiment_name, device_name) @@ -354,13 +356,14 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): if epoch_config: cls.Config.insert1(epoch_config) if metadata_yml_filepath and metadata_yml_filepath.exists(): + try: # Ingest streams.DeviceType, streams.Device and create device tables. insert_device_types( _device_schema_mapping[epoch_key["experiment_name"]], metadata_yml_filepath, ) - streams.main() # create device tables under streams schema + streams.main(context=streams.__dict__) # create device tables under streams schema with cls.connection.transaction: ingest_epoch_metadata( experiment_name, metadata_yml_filepath From 4793d8cdcb820ae82d7fa6f5a79ad33d70c15fb7 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 28 Apr 2023 23:44:59 +0000 Subject: [PATCH 43/54] fix bugs in device removal time entry --- aeon/dj_pipeline/utils/load_metadata.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 534acf65..0650a8f7 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -281,20 +281,20 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): table.Attribute.insert(table_attribute_entry, ignore_extra_fields=True) # Remove the currently installed devices that are absent in this config + device_removal = lambda device_type, device_entry: any(dj.utils. from_camel_case(device_type) in k for k in device_entry) # returns True if the device type is found in the attribute name + for device_type in streams.DeviceType.fetch("device_type"): table = getattr(streams, device_type) - + device_removal_list.extend( (table - table.RemovalTime - device_list & experiment_key - ).fetch("KEY")) + ).fetch("KEY")) # could be VideoSource or Patch + + for device_entry in device_removal_list: + if device_removal(device_type, device_entry): + table.RemovalTime.insert1(device_entry) + - if device_removal_list: - try: - table.RemovalTime.insert(device_removal_list, ignore_extra_fields=True) - except: - pass - - # region Get stream & device information def get_stream_entries(schema: DotMap) -> list[dict]: """Returns a list of dictionaries containing the stream entries for a given device, From 11db552748d26d828a6ab3c4a03928a366a9ff65 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Tue, 6 Jun 2023 20:48:50 +0000 Subject: [PATCH 44/54] add device table definitions posthoc --- aeon/dj_pipeline/streams.py | 111 ++++++++++++++++++++++++++++++++---- 1 file changed, 99 insertions(+), 12 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index d76a8767..518e8bcd 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -1,4 +1,5 @@ import inspect +import os import datajoint as dj import pandas as pd @@ -196,27 +197,113 @@ def make(self, key): return DeviceDataStream - # endregion +def get_device_table_definition(device_template): + """Returns table definition for ExperimentDevice. + + Args: + device_type (str): Device type (e.g., Patch, VideoSource) + + Returns: + device_table_def (str): Table definition for ExperimentDevice. + """ + + replacements = { + "ExperimentDevice": device_type, + "{device_title}": dj.utils.from_camel_case(device_type), + "{device_type}": dj.utils.from_camel_case(device_type), + "{aeon.__version__}": aeon.__version__ + } + + for old, new in replacements.items(): + device_table_def = device_table_def.replace(old, new) + return device_table_def + "\n\n" def main(context=None): + import re if context is None: context = inspect.currentframe().f_back.f_locals - # Create tables. + # Create DeviceType tables. for device_info in (DeviceType).fetch(as_dict=True): - table_class = get_device_template(device_info["device_type"]) - context[table_class.__name__] = table_class - schema(table_class, context=context) - + if device_info["device_type"] not in locals(): + table_class = get_device_template(device_info["device_type"]) + context[table_class.__name__] = table_class + schema(table_class, context=context) + + device_table_def = inspect.getsource(table_class) + replacements = { + "ExperimentDevice": device_info["device_type"], + "{device_title}": dj.utils.from_camel_case(device_info["device_type"]), + "{device_type}": dj.utils.from_camel_case(device_info["device_type"]), + "{aeon.__version__}": aeon.__version__ + } + for old, new in replacements.items(): + device_table_def = device_table_def.replace(old, new) + full_def = "\t@schema \n" + device_table_def + "\n\n" + if os.path.exists("existing_module.py"): + with open("existing_module.py", "r") as f: + existing_content = f.read() + + if full_def in existing_content: + continue + + with open("existing_module.py", "a") as f: + f.write(full_def) + else: + with open("existing_module.py", "w") as f: + full_def = """import datajoint as dj\n\n""" + full_def + f.write(full_def) + # Create DeviceDataStream tables. for device_info in (DeviceType.Stream).fetch(as_dict=True): - table_class = get_device_stream_template( - device_info["device_type"], device_info["stream_type"] - ) - context[table_class.__name__] = table_class - schema(table_class, context=context) - + + device_type = device_info['device_type'] + stream_type = device_info['stream_type'] + table_name = f"{device_type}{stream_type}" + + if table_name not in locals(): + table_class = get_device_stream_template( + device_type, stream_type) + context[table_class.__name__] = table_class + schema(table_class, context=context) + + device_stream_table_def = inspect.getsource(table_class) + + old_definition = f""" # Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) + -> {device_type} + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of {stream_type} data + """ + + replacements = { + "DeviceDataStream": f"{device_type}{stream_type}","ExperimentDevice": device_type, + 'f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time"': f"'chunk_start >= {device_type}_install_time'", + 'f\'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")\'': f'chunk_start < IFNULL({device_type}_removal_time, "2200-01-01")', + "{stream_type}": stream_type, + "{aeon.__version__}": aeon.__version__, + } + for old, new in replacements.items(): + new_definition = old_definition.replace(old, new) + + replacements["table_definition"] = '"""'+new_definition+'"""' + + for old, new in replacements.items(): + device_stream_table_def = device_stream_table_def.replace(old, new) + + full_def = "\t@schema \n" + device_stream_table_def + "\n\n" + + with open("existing_module.py", "r") as f: + existing_content = f.read() + + if full_def in existing_content: + continue + + with open("existing_module.py", "a") as f: + f.write(full_def) + main() \ No newline at end of file From b46fda73cfe4d1099556515639f6a6805d0c0c19 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Wed, 28 Jun 2023 15:12:10 +0000 Subject: [PATCH 45/54] add device type mapper --- aeon/dj_pipeline/create_experiments/device_type_mapper.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aeon/dj_pipeline/create_experiments/device_type_mapper.json b/aeon/dj_pipeline/create_experiments/device_type_mapper.json index 028ae594..5ffbee8f 100644 --- a/aeon/dj_pipeline/create_experiments/device_type_mapper.json +++ b/aeon/dj_pipeline/create_experiments/device_type_mapper.json @@ -1 +1 @@ -{"VideoController": "VideoController", "CameraTop": "VideoSource", "CameraWest": "VideoSource", "CameraEast": "VideoSource", "CameraNorth": "VideoSource", "CameraSouth": "VideoSource", "CameraPatch1": "VideoSource", "CameraPatch2": "VideoSource", "CameraNest": "VideoSource", "AudioAmbient": "AudioSource", "Patch1": "Patch", "Patch2": "Patch", "WeightNest": "WeightScale", "TrackingTop": "PositionTracking", "ActivityCenter": "ActivityTracking", "ActivityArena": "ActivityTracking", "ActivityNest": "ActivityTracking", "ActivityPatch1": "ActivityTracking", "ActivityPatch2": "ActivityTracking", "InNest": "RegionTracking", "InPatch1": "RegionTracking", "InPatch2": "RegionTracking", "ArenaCenter": "DistanceFromPoint", "InArena": "InRange", "InCorridor": "InRange", "ClockSynchronizer": null} \ No newline at end of file +{"VideoController": "VideoController", "CameraTop": "VideoSource", "CameraWest": "VideoSource", "CameraEast": "VideoSource", "CameraNorth": "VideoSource", "CameraSouth": "VideoSource", "CameraPatch1": "VideoSource", "CameraPatch2": "VideoSource", "CameraNest": "VideoSource", "AudioAmbient": "AudioSource", "Patch1": "UndergroundFeeder", "Patch2": "UndergroundFeeder", "WeightNest": "WeightScale", "TrackingTop": "PositionTracking", "ActivityCenter": "ActivityTracking", "ActivityArena": "ActivityTracking", "ActivityNest": "ActivityTracking", "ActivityPatch1": "ActivityTracking", "ActivityPatch2": "ActivityTracking", "InNest": "RegionTracking", "InPatch1": "RegionTracking", "InPatch2": "RegionTracking", "ArenaCenter": "DistanceFromPoint", "InArena": "InRange", "InCorridor": "InRange", "ClockSynchronizer": "Synchronizer"} \ No newline at end of file From 95c8dfcbab0a531bd2e92786d794cb902a479fff Mon Sep 17 00:00:00 2001 From: JaerongA Date: Wed, 28 Jun 2023 15:12:47 +0000 Subject: [PATCH 46/54] allow ingestion from multiple machines --- aeon/dj_pipeline/acquisition.py | 23 ++++++++++------ ...te_presocial_a2.py => create_presocial.py} | 27 ++++++++----------- 2 files changed, 26 insertions(+), 24 deletions(-) rename aeon/dj_pipeline/create_experiments/{create_presocial_a2.py => create_presocial.py} (64%) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index b4163420..585cf2bc 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -25,6 +25,8 @@ "exp0.2-r0": "CameraTop", "oct1.0-r0": "CameraTop", "presocial0.1-a2": "CameraTop", + "presocial0.1-a3": "CameraTop", + "presocial0.1-a4": "CameraTop", } _device_schema_mapping = { @@ -33,6 +35,8 @@ "exp0.2-r0": aeon_schema.exp02, "oct1.0-r0": aeon_schema.octagon01, "presocial0.1-a2": aeon_schema.presocial, + "presocial0.1-a3": aeon_schema.presocial, + "presocial0.1-a4": aeon_schema.presocial, } @@ -120,14 +124,17 @@ class Directory(dj.Part): @classmethod def get_data_directory(cls, experiment_key, directory_type="raw", as_posix=False): - repo_name, dir_path = ( - cls.Directory & experiment_key & {"directory_type": directory_type} - ).fetch1("repository_name", "directory_path") - data_directory = paths.get_repository_path(repo_name) / dir_path - if not data_directory.exists(): - return None - return data_directory.as_posix() if as_posix else data_directory - + + try: + repo_name, dir_path = ( + cls.Directory & experiment_key & {"directory_type": directory_type} + ).fetch1("repository_name", "directory_path") + data_directory = paths.get_repository_path(repo_name) / dir_path + if not data_directory.exists(): + return None + return data_directory.as_posix() if as_posix else data_directory + except dj.errors.DataJointError: + return @classmethod def get_data_directories( cls, experiment_key, directory_types=["raw"], as_posix=False diff --git a/aeon/dj_pipeline/create_experiments/create_presocial_a2.py b/aeon/dj_pipeline/create_experiments/create_presocial.py similarity index 64% rename from aeon/dj_pipeline/create_experiments/create_presocial_a2.py rename to aeon/dj_pipeline/create_experiments/create_presocial.py index 74d09a5c..038d5927 100644 --- a/aeon/dj_pipeline/create_experiments/create_presocial_a2.py +++ b/aeon/dj_pipeline/create_experiments/create_presocial.py @@ -1,9 +1,9 @@ from aeon.dj_pipeline import acquisition, lab, subject -experiment_type = "presocial" -experiment_name = "presocial0.1-a2" # AEON2 acquisition computer +experiment_type = "presocial0.1" +experiment_names = ["presocial0.1-a2", "presocial0.1-a3", "presocial0.1-a4"] location = "4th floor" - +computers = ["AEON2", "AEON3", "AEON4"] def create_new_experiment(): @@ -13,22 +13,23 @@ def create_new_experiment(): {"experiment_type": experiment_type}, skip_duplicates=True ) - acquisition.Experiment.insert1( - { + acquisition.Experiment.insert( + [{ "experiment_name": experiment_name, "experiment_start_time": "2023-02-25 00:00:00", - "experiment_description": "presocial experiment 0.1 in aeon2", + "experiment_description": "presocial experiment 0.1", "arena_name": "circle-2m", "lab": "SWC", "location": location, - "experiment_type": experiment_type, - }, + "experiment_type": experiment_type + } for experiment_name in experiment_names], skip_duplicates=True, ) acquisition.Experiment.Subject.insert( [ {"experiment_name": experiment_name, "subject": s} + for experiment_name in experiment_names for s in subject.Subject.fetch("subject") ], skip_duplicates=True, @@ -40,14 +41,8 @@ def create_new_experiment(): "experiment_name": experiment_name, "repository_name": "ceph_aeon", "directory_type": "raw", - "directory_path": "aeon/data/raw/AEON2/presocial0.1", - }, - { - "experiment_name": experiment_name, - "repository_name": "ceph_aeon", - "directory_type": "quality-control", - "directory_path": "aeon/data/qc/AEON2/presocial0.1", - }, + "directory_path": f"aeon/data/raw/{computer}/{experiment_type}" + } for experiment_name, computer in zip(experiment_names, computers) ], skip_duplicates=True, ) From ef71dc8490460de863ecf17b7f6f28c44a64abbd Mon Sep 17 00:00:00 2001 From: JaerongA Date: Wed, 28 Jun 2023 15:13:18 +0000 Subject: [PATCH 47/54] add device table definition --- aeon/dj_pipeline/streams.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 518e8bcd..1fb23604 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -198,30 +198,10 @@ def make(self, key): return DeviceDataStream # endregion -def get_device_table_definition(device_template): - """Returns table definition for ExperimentDevice. - - Args: - device_type (str): Device type (e.g., Patch, VideoSource) - - Returns: - device_table_def (str): Table definition for ExperimentDevice. - """ - - replacements = { - "ExperimentDevice": device_type, - "{device_title}": dj.utils.from_camel_case(device_type), - "{device_type}": dj.utils.from_camel_case(device_type), - "{aeon.__version__}": aeon.__version__ - } - - for old, new in replacements.items(): - device_table_def = device_table_def.replace(old, new) - return device_table_def + "\n\n" - def main(context=None): + import re if context is None: context = inspect.currentframe().f_back.f_locals @@ -254,7 +234,7 @@ def main(context=None): f.write(full_def) else: with open("existing_module.py", "w") as f: - full_def = """import datajoint as dj\n\n""" + full_def + full_def = """import datajoint as dj\nimport pandas as pd\nfrom aeon.dj_pipeline import acquisition\nfrom aeon.io import api as io_api\n\n""" + full_def f.write(full_def) # Create DeviceDataStream tables. @@ -282,8 +262,10 @@ def main(context=None): replacements = { "DeviceDataStream": f"{device_type}{stream_type}","ExperimentDevice": device_type, - 'f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time"': f"'chunk_start >= {device_type}_install_time'", - 'f\'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")\'': f'chunk_start < IFNULL({device_type}_removal_time, "2200-01-01")', + 'f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time"': f"'chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time'", + """f'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")'""": f"""'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")'""", + 'f"{dj.utils.from_camel_case(device_type)}_name"': f"'{dj.utils.from_camel_case(device_type)}_name'", + "{device_type}": device_type, "{stream_type}": stream_type, "{aeon.__version__}": aeon.__version__, } From 0c5ec8ca2d6f73532776422b5d45ce10218f99b1 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Thu, 29 Jun 2023 19:10:09 +0000 Subject: [PATCH 48/54] add stream object --- aeon/dj_pipeline/streams.py | 43 +++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index 1fb23604..fbdb93d8 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -125,12 +125,12 @@ def get_device_stream_template(device_type: str, stream_type: str): stream = reader(**stream_detail["stream_reader_kwargs"]) table_definition = f""" # Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) - -> {device_type} - -> acquisition.Chunk - --- - sample_count: int # number of data points acquired from this stream for a given chunk - timestamps: longblob # (datetime) timestamps of {stream_type} data - """ + -> {device_type} + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of {stream_type} data + """ for col in stream.columns: if col.startswith("_"): @@ -213,7 +213,7 @@ def main(context=None): context[table_class.__name__] = table_class schema(table_class, context=context) - device_table_def = inspect.getsource(table_class) + device_table_def = inspect.getsource(table_class).lstrip() replacements = { "ExperimentDevice": device_info["device_type"], "{device_title}": dj.utils.from_camel_case(device_info["device_type"]), @@ -222,7 +222,7 @@ def main(context=None): } for old, new in replacements.items(): device_table_def = device_table_def.replace(old, new) - full_def = "\t@schema \n" + device_table_def + "\n\n" + full_def = "@schema \n" + device_table_def + "\n\n" if os.path.exists("existing_module.py"): with open("existing_module.py", "r") as f: existing_content = f.read() @@ -234,7 +234,7 @@ def main(context=None): f.write(full_def) else: with open("existing_module.py", "w") as f: - full_def = """import datajoint as dj\nimport pandas as pd\nfrom aeon.dj_pipeline import acquisition\nfrom aeon.io import api as io_api\n\n""" + full_def + full_def = """import datajoint as dj\nimport pandas as pd\n\nimport aeon\nfrom aeon.dj_pipeline import acquisition\nfrom aeon.io import api as io_api\n\n""" + full_def f.write(full_def) # Create DeviceDataStream tables. @@ -250,15 +250,19 @@ def main(context=None): context[table_class.__name__] = table_class schema(table_class, context=context) - device_stream_table_def = inspect.getsource(table_class) + stream_obj = table_class.__dict__["_stream_reader"] + reader = stream_obj.__module__ + '.' + stream_obj.__name__ + stream_detail = table_class.__dict__["_stream_detail"] + + device_stream_table_def = inspect.getsource(table_class).lstrip() - old_definition = f""" # Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) - -> {device_type} - -> acquisition.Chunk - --- - sample_count: int # number of data points acquired from this stream for a given chunk - timestamps: longblob # (datetime) timestamps of {stream_type} data - """ + old_definition = f"""# Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) + -> {device_type} + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of {stream_type} data + """ replacements = { "DeviceDataStream": f"{device_type}{stream_type}","ExperimentDevice": device_type, @@ -276,8 +280,11 @@ def main(context=None): for old, new in replacements.items(): device_stream_table_def = device_stream_table_def.replace(old, new) + + device_stream_table_def = re.sub(r'_stream_reader\s*=\s*reader', f'_stream_reader = {reader}', device_stream_table_def) # insert reader + device_stream_table_def = re.sub(r'_stream_detail\s*=\s*stream_detail', f'_stream_detail = {stream_detail}', device_stream_table_def) # insert stream details - full_def = "\t@schema \n" + device_stream_table_def + "\n\n" + full_def = "@schema \n" + device_stream_table_def + "\n\n" with open("existing_module.py", "r") as f: existing_content = f.read() From 93d0f3d2c1b42d3610e864ed5d6f38fdd150028a Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Mon, 3 Jul 2023 16:30:41 -0500 Subject: [PATCH 49/54] refactor - explicit `streams_maker` --- aeon/dj_pipeline/acquisition.py | 16 +- aeon/dj_pipeline/populate/worker.py | 6 +- aeon/dj_pipeline/streams.py | 250 +----------------- aeon/dj_pipeline/streams_maker.py | 331 ++++++++++++++++++++++++ aeon/dj_pipeline/utils/load_metadata.py | 82 +++--- 5 files changed, 400 insertions(+), 285 deletions(-) create mode 100644 aeon/dj_pipeline/streams_maker.py diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index 585cf2bc..9a503a3a 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -15,7 +15,6 @@ logger = dj.logger schema = dj.schema(get_schema_name("acquisition")) -# streams = dj.VirtualModule("streams", get_schema_name("streams")) # ------------------- Some Constants -------------------------- @@ -124,7 +123,7 @@ class Directory(dj.Part): @classmethod def get_data_directory(cls, experiment_key, directory_type="raw", as_posix=False): - + try: repo_name, dir_path = ( cls.Directory & experiment_key & {"directory_type": directory_type} @@ -135,6 +134,7 @@ def get_data_directory(cls, experiment_key, directory_type="raw", as_posix=False return data_directory.as_posix() if as_posix else data_directory except dj.errors.DataJointError: return + @classmethod def get_data_directories( cls, experiment_key, directory_types=["raw"], as_posix=False @@ -277,14 +277,14 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): - if not specified, ingest all epochs Note: "start" and "end" are datetime specified a string in the format: "%Y-%m-%d %H:%M:%S" """ - from aeon.dj_pipeline import streams + from aeon.dj_pipeline import streams_maker from .utils.load_metadata import ( extract_epoch_config, ingest_epoch_metadata, insert_device_types, ) - + device_name = _ref_device_mapping.get(experiment_name, "CameraTop") all_chunks, raw_data_dirs = _get_all_chunks(experiment_name, device_name) @@ -363,15 +363,17 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): if epoch_config: cls.Config.insert1(epoch_config) if metadata_yml_filepath and metadata_yml_filepath.exists(): - + try: - # Ingest streams.DeviceType, streams.Device and create device tables. + # Insert new entries for streams.DeviceType, streams.Device. insert_device_types( _device_schema_mapping[epoch_key["experiment_name"]], metadata_yml_filepath, ) - streams.main(context=streams.__dict__) # create device tables under streams schema + # Define and instantiate new devices/stream tables under `streams` schema + streams_maker.main() with cls.connection.transaction: + # Insert devices' installation/removal/settings ingest_epoch_metadata( experiment_name, metadata_yml_filepath ) diff --git a/aeon/dj_pipeline/populate/worker.py b/aeon/dj_pipeline/populate/worker.py index 7b1678dc..395023b0 100644 --- a/aeon/dj_pipeline/populate/worker.py +++ b/aeon/dj_pipeline/populate/worker.py @@ -12,11 +12,13 @@ db_prefix, qc, report, - streams, + streams_maker, tracking, ) from aeon.dj_pipeline.utils import load_metadata +streams = streams_maker.main() + __all__ = [ "high_priority", "mid_priority", @@ -98,5 +100,5 @@ ) for attr in vars(streams).values(): - if is_djtable(attr) and hasattr(attr, "populate"): + if is_djtable(attr, dj.user_tables.AutoPopulate): streams_worker(attr) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index fbdb93d8..f642a62f 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -1,5 +1,5 @@ -import inspect -import os +#---- DO NOT MODIFY ---- +#---- THIS FILE IS AUTO-GENERATED BY `streams_maker.py` ---- import datajoint as dj import pandas as pd @@ -8,17 +8,11 @@ from aeon.dj_pipeline import acquisition, get_schema_name from aeon.io import api as io_api -logger = dj.logger +schema_name = get_schema_name('streams') +schema = dj.Schema() -# schema_name = f'u_{dj.config["database.user"]}_streams' # for testing -schema_name = get_schema_name("streams") -schema = dj.schema(schema_name) - -schema.spawn_missing_classes() - - -@schema +@schema class StreamType(dj.Lookup): """ Catalog of all steam types for the different device types used across Project Aeon @@ -38,7 +32,7 @@ class StreamType(dj.Lookup): """ -@schema +@schema class DeviceType(dj.Lookup): """ Catalog of all device types used across Project Aeon @@ -57,7 +51,7 @@ class Stream(dj.Part): """ -@schema +@schema class Device(dj.Lookup): definition = """ # Physical devices, of a particular type, identified by unique serial number device_serial_number: varchar(12) @@ -66,233 +60,3 @@ class Device(dj.Lookup): """ -# region Helper functions for creating device tables. - - -def get_device_template(device_type: str): - """Returns table class template for ExperimentDevice""" - device_title = device_type - device_type = dj.utils.from_camel_case(device_type) - - class ExperimentDevice(dj.Manual): - definition = f""" - # {device_title} placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-{aeon.__version__}) - -> acquisition.Experiment - -> Device - {device_type}_install_time : datetime(6) # time of the {device_type} placed and started operation at this position - --- - {device_type}_name : varchar(36) - """ - - class Attribute(dj.Part): - definition = """ # metadata/attributes (e.g. FPS, config, calibration, etc.) associated with this experimental device - -> master - attribute_name : varchar(32) - --- - attribute_value=null : longblob - """ - - class RemovalTime(dj.Part): - definition = f""" - -> master - --- - {device_type}_removal_time: datetime(6) # time of the {device_type} being removed - """ - - ExperimentDevice.__name__ = f"{device_title}" - - return ExperimentDevice - - -def get_device_stream_template(device_type: str, stream_type: str): - """Returns table class template for DeviceDataStream""" - - context = inspect.currentframe().f_back.f_locals["context"] - ExperimentDevice = context[device_type] - - # DeviceDataStream table(s) - stream_detail = ( - StreamType - & (DeviceType.Stream & {"device_type": device_type, "stream_type": stream_type}) - ).fetch1() - - for i, n in enumerate(stream_detail["stream_reader"].split(".")): - if i == 0: - reader = aeon - else: - reader = getattr(reader, n) - - stream = reader(**stream_detail["stream_reader_kwargs"]) - - table_definition = f""" # Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) - -> {device_type} - -> acquisition.Chunk - --- - sample_count: int # number of data points acquired from this stream for a given chunk - timestamps: longblob # (datetime) timestamps of {stream_type} data - """ - - for col in stream.columns: - if col.startswith("_"): - continue - table_definition += f"{col}: longblob\n\t\t\t" - - class DeviceDataStream(dj.Imported): - definition = table_definition - _stream_reader = reader - _stream_detail = stream_detail - - @property - def key_source(self): - f""" - Only the combination of Chunk and {device_type} with overlapping time - + Chunk(s) that started after {device_type} install time and ended before {device_type} remove time - + Chunk(s) that started after {device_type} install time for {device_type} that are not yet removed - """ - return ( - acquisition.Chunk - * ExperimentDevice.join(ExperimentDevice.RemovalTime, left=True) - & f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time" - & f'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")' - ) - - def make(self, key): - chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( - "chunk_start", "chunk_end", "directory_type" - ) - raw_data_dir = acquisition.Experiment.get_data_directory( - key, directory_type=dir_type - ) - - device_name = (ExperimentDevice & key).fetch1(f"{dj.utils.from_camel_case(device_type)}_name") - - stream = self._stream_reader( - **{ - k: v.format(**{k: device_name}) if k == "pattern" else v - for k, v in self._stream_detail["stream_reader_kwargs"].items() - } - ) - - stream_data = io_api.load( - root=raw_data_dir.as_posix(), - reader=stream, - start=pd.Timestamp(chunk_start), - end=pd.Timestamp(chunk_end), - ) - - self.insert1( - { - **key, - "sample_count": len(stream_data), - "timestamps": stream_data.index.values, - **{ - c: stream_data[c].values - for c in stream.columns - if not c.startswith("_") - }, - } - ) - - DeviceDataStream.__name__ = f"{device_type}{stream_type}" - - return DeviceDataStream - -# endregion - - -def main(context=None): - - import re - if context is None: - context = inspect.currentframe().f_back.f_locals - - # Create DeviceType tables. - for device_info in (DeviceType).fetch(as_dict=True): - if device_info["device_type"] not in locals(): - table_class = get_device_template(device_info["device_type"]) - context[table_class.__name__] = table_class - schema(table_class, context=context) - - device_table_def = inspect.getsource(table_class).lstrip() - replacements = { - "ExperimentDevice": device_info["device_type"], - "{device_title}": dj.utils.from_camel_case(device_info["device_type"]), - "{device_type}": dj.utils.from_camel_case(device_info["device_type"]), - "{aeon.__version__}": aeon.__version__ - } - for old, new in replacements.items(): - device_table_def = device_table_def.replace(old, new) - full_def = "@schema \n" + device_table_def + "\n\n" - if os.path.exists("existing_module.py"): - with open("existing_module.py", "r") as f: - existing_content = f.read() - - if full_def in existing_content: - continue - - with open("existing_module.py", "a") as f: - f.write(full_def) - else: - with open("existing_module.py", "w") as f: - full_def = """import datajoint as dj\nimport pandas as pd\n\nimport aeon\nfrom aeon.dj_pipeline import acquisition\nfrom aeon.io import api as io_api\n\n""" + full_def - f.write(full_def) - - # Create DeviceDataStream tables. - for device_info in (DeviceType.Stream).fetch(as_dict=True): - - device_type = device_info['device_type'] - stream_type = device_info['stream_type'] - table_name = f"{device_type}{stream_type}" - - if table_name not in locals(): - table_class = get_device_stream_template( - device_type, stream_type) - context[table_class.__name__] = table_class - schema(table_class, context=context) - - stream_obj = table_class.__dict__["_stream_reader"] - reader = stream_obj.__module__ + '.' + stream_obj.__name__ - stream_detail = table_class.__dict__["_stream_detail"] - - device_stream_table_def = inspect.getsource(table_class).lstrip() - - old_definition = f"""# Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) - -> {device_type} - -> acquisition.Chunk - --- - sample_count: int # number of data points acquired from this stream for a given chunk - timestamps: longblob # (datetime) timestamps of {stream_type} data - """ - - replacements = { - "DeviceDataStream": f"{device_type}{stream_type}","ExperimentDevice": device_type, - 'f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time"': f"'chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time'", - """f'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")'""": f"""'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")'""", - 'f"{dj.utils.from_camel_case(device_type)}_name"': f"'{dj.utils.from_camel_case(device_type)}_name'", - "{device_type}": device_type, - "{stream_type}": stream_type, - "{aeon.__version__}": aeon.__version__, - } - for old, new in replacements.items(): - new_definition = old_definition.replace(old, new) - - replacements["table_definition"] = '"""'+new_definition+'"""' - - for old, new in replacements.items(): - device_stream_table_def = device_stream_table_def.replace(old, new) - - device_stream_table_def = re.sub(r'_stream_reader\s*=\s*reader', f'_stream_reader = {reader}', device_stream_table_def) # insert reader - device_stream_table_def = re.sub(r'_stream_detail\s*=\s*stream_detail', f'_stream_detail = {stream_detail}', device_stream_table_def) # insert stream details - - full_def = "@schema \n" + device_stream_table_def + "\n\n" - - with open("existing_module.py", "r") as f: - existing_content = f.read() - - if full_def in existing_content: - continue - - with open("existing_module.py", "a") as f: - f.write(full_def) - -main() \ No newline at end of file diff --git a/aeon/dj_pipeline/streams_maker.py b/aeon/dj_pipeline/streams_maker.py new file mode 100644 index 00000000..0bc19ea5 --- /dev/null +++ b/aeon/dj_pipeline/streams_maker.py @@ -0,0 +1,331 @@ +import inspect +from pathlib import Path +import datajoint as dj +import pandas as pd +import re +import importlib + +import aeon +from aeon.dj_pipeline import acquisition, get_schema_name +from aeon.io import api as io_api + +logger = dj.logger + + +# schema_name = f'u_{dj.config["database.user"]}_streams' # for testing +schema_name = get_schema_name("streams") + +STREAMS_MODULE_NAME = "streams" +_STREAMS_MODULE_FILE = Path(__file__).parent / f"{STREAMS_MODULE_NAME}.py" + + +class StreamType(dj.Lookup): + """ + Catalog of all steam types for the different device types used across Project Aeon + One StreamType corresponds to one reader class in `aeon.io.reader` + The combination of `stream_reader` and `stream_reader_kwargs` should fully specify + the data loading routine for a particular device, using the `aeon.io.utils` + """ + + definition = """ # Catalog of all stream types used across Project Aeon + stream_type : varchar(20) + --- + stream_reader : varchar(256) # name of the reader class found in `aeon_mecha` package (e.g. aeon.io.reader.Video) + stream_reader_kwargs : longblob # keyword arguments to instantiate the reader class + stream_description='': varchar(256) + stream_hash : uuid # hash of dict(stream_reader_kwargs, stream_reader=stream_reader) + unique index (stream_hash) + """ + + +class DeviceType(dj.Lookup): + """ + Catalog of all device types used across Project Aeon + """ + + definition = """ # Catalog of all device types used across Project Aeon + device_type: varchar(36) + --- + device_description='': varchar(256) + """ + + class Stream(dj.Part): + definition = """ # Data stream(s) associated with a particular device type + -> master + -> StreamType + """ + + +class Device(dj.Lookup): + definition = """ # Physical devices, of a particular type, identified by unique serial number + device_serial_number: varchar(12) + --- + -> DeviceType + """ + + +# region Helper functions for creating device tables. + + +def get_device_template(device_type: str): + """Returns table class template for ExperimentDevice""" + device_title = device_type + device_type = dj.utils.from_camel_case(device_type) + + class ExperimentDevice(dj.Manual): + definition = f""" + # {device_title} placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-{aeon.__version__}) + -> acquisition.Experiment + -> Device + {device_type}_install_time : datetime(6) # time of the {device_type} placed and started operation at this position + --- + {device_type}_name : varchar(36) + """ + + class Attribute(dj.Part): + definition = """ # metadata/attributes (e.g. FPS, config, calibration, etc.) associated with this experimental device + -> master + attribute_name : varchar(32) + --- + attribute_value=null : longblob + """ + + class RemovalTime(dj.Part): + definition = f""" + -> master + --- + {device_type}_removal_time: datetime(6) # time of the {device_type} being removed + """ + + ExperimentDevice.__name__ = f"{device_title}" + + return ExperimentDevice + + +def get_device_stream_template(device_type: str, stream_type: str): + """Returns table class template for DeviceDataStream""" + + context = inspect.currentframe().f_back.f_locals["context"] + ExperimentDevice = context[device_type] + + # DeviceDataStream table(s) + stream_detail = ( + StreamType + & (DeviceType.Stream & {"device_type": device_type, "stream_type": stream_type}) + ).fetch1() + + for i, n in enumerate(stream_detail["stream_reader"].split(".")): + if i == 0: + reader = aeon + else: + reader = getattr(reader, n) + + stream = reader(**stream_detail["stream_reader_kwargs"]) + + table_definition = f""" # Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) + -> {device_type} + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of {stream_type} data + """ + + for col in stream.columns: + if col.startswith("_"): + continue + table_definition += f"{col}: longblob\n\t\t\t" + + class DeviceDataStream(dj.Imported): + definition = table_definition + _stream_reader = reader + _stream_detail = stream_detail + + @property + def key_source(self): + f""" + Only the combination of Chunk and {device_type} with overlapping time + + Chunk(s) that started after {device_type} install time and ended before {device_type} remove time + + Chunk(s) that started after {device_type} install time for {device_type} that are not yet removed + """ + return ( + acquisition.Chunk + * ExperimentDevice.join(ExperimentDevice.RemovalTime, left=True) + & f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time" + & f'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (ExperimentDevice & key).fetch1( + f"{dj.utils.from_camel_case(device_type)}_name" + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + DeviceDataStream.__name__ = f"{device_type}{stream_type}" + + return DeviceDataStream + + +# endregion + + +def main(create_tables=True): + + if not _STREAMS_MODULE_FILE.exists(): + with open(_STREAMS_MODULE_FILE, "w") as f: + imports_str = ( + "#---- DO NOT MODIFY ----\n" + "#---- THIS FILE IS AUTO-GENERATED BY `streams_maker.py` ----\n\n" + "import datajoint as dj\n" + "import pandas as pd\n\n" + "import aeon\n" + "from aeon.dj_pipeline import acquisition, get_schema_name\n" + "from aeon.io import api as io_api\n\n" + "schema_name = get_schema_name('streams')\n" + "schema = dj.Schema()\n\n\n" + ) + f.write(imports_str) + for table_class in (StreamType, DeviceType, Device): + device_table_def = inspect.getsource(table_class).lstrip() + full_def = "@schema \n" + device_table_def + "\n\n" + f.write(full_def) + + streams = importlib.import_module(f"aeon.dj_pipeline.{STREAMS_MODULE_NAME}") + streams.schema.activate(schema_name) + + if create_tables: + # Create DeviceType tables. + for device_info in streams.DeviceType.fetch(as_dict=True): + if hasattr(streams, device_info["device_type"]): + continue + + table_class = get_device_template(device_info["device_type"]) + # context[table_class.__name__] = table_class + # schema(table_class, context=context) + + device_table_def = inspect.getsource(table_class).lstrip() + replacements = { + "ExperimentDevice": device_info["device_type"], + "{device_title}": dj.utils.from_camel_case(device_info["device_type"]), + "{device_type}": dj.utils.from_camel_case(device_info["device_type"]), + "{aeon.__version__}": aeon.__version__, + } + for old, new in replacements.items(): + device_table_def = device_table_def.replace(old, new) + full_def = "@schema \n" + device_table_def + "\n\n" + with open(_STREAMS_MODULE_FILE, "r") as f: + existing_content = f.read() + + if full_def in existing_content: + continue + + with open(_STREAMS_MODULE_FILE, "a") as f: + f.write(full_def) + + # Create DeviceDataStream tables. + for device_info in streams.DeviceType.Stream.fetch(as_dict=True): + + device_type = device_info["device_type"] + stream_type = device_info["stream_type"] + table_name = f"{device_type}{stream_type}" + + if hasattr(streams, table_name): + continue + + table_class = get_device_stream_template(device_type, stream_type) + # context[table_class.__name__] = table_class + # schema(table_class, context=context) + + stream_obj = table_class.__dict__["_stream_reader"] + reader = stream_obj.__module__ + "." + stream_obj.__name__ + stream_detail = table_class.__dict__["_stream_detail"] + + device_stream_table_def = inspect.getsource(table_class).lstrip() + + old_definition = f"""# Raw per-chunk {stream_type} data stream from {device_type} (auto-generated with aeon_mecha-{aeon.__version__}) + -> {device_type} + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of {stream_type} data + """ + + replacements = { + "DeviceDataStream": f"{device_type}{stream_type}", + "ExperimentDevice": device_type, + 'f"chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time"': f"'chunk_start >= {dj.utils.from_camel_case(device_type)}_install_time'", + """f'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")'""": f"""'chunk_start < IFNULL({dj.utils.from_camel_case(device_type)}_removal_time, "2200-01-01")'""", + 'f"{dj.utils.from_camel_case(device_type)}_name"': f"'{dj.utils.from_camel_case(device_type)}_name'", + "{device_type}": device_type, + "{stream_type}": stream_type, + "{aeon.__version__}": aeon.__version__, + } + for old, new in replacements.items(): + new_definition = old_definition.replace(old, new) + + replacements["table_definition"] = '"""' + new_definition + '"""' + + for old, new in replacements.items(): + device_stream_table_def = device_stream_table_def.replace(old, new) + + device_stream_table_def = re.sub( + r"_stream_reader\s*=\s*reader", + f"_stream_reader = {reader}", + device_stream_table_def, + ) # insert reader + device_stream_table_def = re.sub( + r"_stream_detail\s*=\s*stream_detail", + f"_stream_detail = {stream_detail}", + device_stream_table_def, + ) # insert stream details + + full_def = "@schema \n" + device_stream_table_def + "\n\n" + + with open(_STREAMS_MODULE_FILE, "r") as f: + existing_content = f.read() + + if full_def in existing_content: + continue + + with open(_STREAMS_MODULE_FILE, "a") as f: + f.write(full_def) + + importlib.reload(streams) + streams.schema.activate(schema_name) + + return streams + + +streams = main() diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 0650a8f7..090b6b19 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -2,7 +2,6 @@ import inspect import json import pathlib -import re from collections import defaultdict from pathlib import Path @@ -11,16 +10,10 @@ import pandas as pd from dotmap import DotMap -from aeon.dj_pipeline import ( - acquisition, - dict_to_uuid, - get_schema_name, - lab, - streams, - subject, -) +from aeon.dj_pipeline import acquisition, dict_to_uuid, subject, streams_maker from aeon.io import api as io_api + _weight_scale_rate = 100 _weight_scale_nest = 1 _colony_csv_path = pathlib.Path("/ceph/aeon/aeon/colony/colony.csv") @@ -43,6 +36,8 @@ def insert_stream_types(): """Insert into streams.streamType table all streams in the dataset schema.""" from aeon.schema import dataset + streams = dj.VirtualModule("streams", streams_maker.STREAMS_MODULE_NAME) + schemas = [v for v in dataset.__dict__.values() if isinstance(v, DotMap)] for schema in schemas: @@ -64,6 +59,8 @@ def insert_stream_types(): def insert_device_types(schema: DotMap, metadata_yml_filepath: Path): """Use dataset.schema and metadata.yml to insert into streams.DeviceType and streams.Device. Only insert device types that were defined both in the device schema (e.g., exp02) and Metadata.yml. It then creates new device tables under streams schema.""" + streams = dj.VirtualModule("streams", streams_maker.STREAMS_MODULE_NAME) + device_info: dict[dict] = get_device_info(schema) device_type_mapper, device_sn = get_device_mapper(schema, metadata_yml_filepath) @@ -137,7 +134,7 @@ def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> di Returns: dict: epoch_config [dict] """ - + metadata_yml_filepath = pathlib.Path(metadata_yml_filepath) epoch_start = datetime.datetime.strptime( metadata_yml_filepath.parent.name, "%Y-%m-%dT%H-%M-%S" @@ -163,7 +160,9 @@ def extract_epoch_config(experiment_name: str, metadata_yml_filepath: str) -> di ) ) - if isinstance(devices, list): # In exp02, it is a list of dict. In presocial. It's a dict of dict. + if isinstance( + devices, list + ): # In exp02, it is a list of dict. In presocial. It's a dict of dict. devices: dict = { d.pop("Name"): d for d in devices } # {deivce_name: device_config} @@ -182,7 +181,8 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): """ Make entries into device tables """ - + streams = dj.VirtualModule("streams", streams_maker.STREAMS_MODULE_NAME) + if experiment_name.startswith("oct"): ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath) return @@ -209,14 +209,14 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): schema = acquisition._device_schema_mapping[experiment_name] device_type_mapper, _ = get_device_mapper(schema, metadata_yml_filepath) - + # Insert into each device table device_list = [] device_removal_list = [] - + for device_name, device_config in epoch_config["metadata"].items(): if table := getattr(streams, device_type_mapper.get(device_name) or "", None): - + device_sn = device_config.get("SerialNumber", device_config.get("PortName")) device_key = {"device_serial_number": device_sn} @@ -263,38 +263,50 @@ def ingest_epoch_metadata(experiment_name, metadata_yml_filepath): for entry in table_attribute_entry ] - if dict_to_uuid({config["attribute_name"]: config["attribute_value"] for config in current_device_config}) == dict_to_uuid( - {config["attribute_name"]: config["attribute_value"] for config in new_device_config} + if dict_to_uuid( + { + config["attribute_name"]: config["attribute_value"] + for config in current_device_config + } + ) == dict_to_uuid( + { + config["attribute_name"]: config["attribute_value"] + for config in new_device_config + } ): # Skip if none of the configuration has changed. continue # Remove old device - device_removal_list.append({ - **current_device_query.fetch1("KEY"), - f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ - "epoch_start" - ], - }) + device_removal_list.append( + { + **current_device_query.fetch1("KEY"), + f"{dj.utils.from_camel_case(table.__name__)}_removal_time": epoch_config[ + "epoch_start" + ], + } + ) # Insert into table. table.insert1(table_entry, skip_duplicates=True) table.Attribute.insert(table_attribute_entry, ignore_extra_fields=True) # Remove the currently installed devices that are absent in this config - device_removal = lambda device_type, device_entry: any(dj.utils. from_camel_case(device_type) in k for k in device_entry) # returns True if the device type is found in the attribute name - + device_removal = lambda device_type, device_entry: any( + dj.utils.from_camel_case(device_type) in k for k in device_entry + ) # returns True if the device type is found in the attribute name + for device_type in streams.DeviceType.fetch("device_type"): table = getattr(streams, device_type) device_removal_list.extend( - (table - table.RemovalTime - device_list & experiment_key - ).fetch("KEY")) # could be VideoSource or Patch - + (table - table.RemovalTime - device_list & experiment_key).fetch("KEY") + ) # could be VideoSource or Patch + for device_entry in device_removal_list: if device_removal(device_type, device_entry): table.RemovalTime.insert1(device_entry) - - + + # region Get stream & device information def get_stream_entries(schema: DotMap) -> list[dict]: """Returns a list of dictionaries containing the stream entries for a given device, @@ -474,10 +486,14 @@ def get_device_mapper(schema: DotMap, metadata_yml_filepath: Path): device_sn[item.Name] = ( item.SerialNumber or item.PortName or None ) # assign either the serial number (if it exists) or port name. If neither exists, assign None - elif isinstance(item, str): # presocial + elif isinstance(item, str): # presocial if meta_data.Devices[item].get("Type"): device_type_mapper[item] = meta_data.Devices[item].get("Type") - device_sn[item] = meta_data.Devices[item].get("SerialNumber") or meta_data.Devices[item].get("PortName") or None + device_sn[item] = ( + meta_data.Devices[item].get("SerialNumber") + or meta_data.Devices[item].get("PortName") + or None + ) with filename.open("w") as f: json.dump(device_type_mapper, f) @@ -491,7 +507,7 @@ def ingest_epoch_metadata_octagon(experiment_name, metadata_yml_filepath): """ Temporary ingestion routine to load devices' meta information for Octagon arena experiments """ - from aeon.dj_pipeline import streams + streams = dj.VirtualModule("streams", streams_maker.STREAMS_MODULE_NAME) oct01_devices = [ ("Metadata", "Metadata"), From 37cdfa26f54060feb66fb8532589ccb1aa7fa741 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Mon, 3 Jul 2023 16:52:22 -0500 Subject: [PATCH 50/54] update streams for presocial --- aeon/dj_pipeline/streams.py | 808 ++++++++++++++++++++++++++++++ aeon/dj_pipeline/streams_maker.py | 24 +- 2 files changed, 821 insertions(+), 11 deletions(-) diff --git a/aeon/dj_pipeline/streams.py b/aeon/dj_pipeline/streams.py index f642a62f..120b41fb 100644 --- a/aeon/dj_pipeline/streams.py +++ b/aeon/dj_pipeline/streams.py @@ -3,6 +3,7 @@ import datajoint as dj import pandas as pd +from uuid import UUID import aeon from aeon.dj_pipeline import acquisition, get_schema_name @@ -60,3 +61,810 @@ class Device(dj.Lookup): """ +@schema +class Patch(dj.Manual): + definition = f""" + # patch placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-unknown) + -> acquisition.Experiment + -> Device + patch_install_time : datetime(6) # time of the patch placed and started operation at this position + --- + patch_name : varchar(36) + """ + + class Attribute(dj.Part): + definition = """ # metadata/attributes (e.g. FPS, config, calibration, etc.) associated with this experimental device + -> master + attribute_name : varchar(32) + --- + attribute_value=null : longblob + """ + + class RemovalTime(dj.Part): + definition = f""" + -> master + --- + patch_removal_time: datetime(6) # time of the patch being removed + """ + + +@schema +class UndergroundFeeder(dj.Manual): + definition = f""" + # underground_feeder placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-unknown) + -> acquisition.Experiment + -> Device + underground_feeder_install_time : datetime(6) # time of the underground_feeder placed and started operation at this position + --- + underground_feeder_name : varchar(36) + """ + + class Attribute(dj.Part): + definition = """ # metadata/attributes (e.g. FPS, config, calibration, etc.) associated with this experimental device + -> master + attribute_name : varchar(32) + --- + attribute_value=null : longblob + """ + + class RemovalTime(dj.Part): + definition = f""" + -> master + --- + underground_feeder_removal_time: datetime(6) # time of the underground_feeder being removed + """ + + +@schema +class VideoSource(dj.Manual): + definition = f""" + # video_source placement and operation for a particular time period, at a certain location, for a given experiment (auto-generated with aeon_mecha-unknown) + -> acquisition.Experiment + -> Device + video_source_install_time : datetime(6) # time of the video_source placed and started operation at this position + --- + video_source_name : varchar(36) + """ + + class Attribute(dj.Part): + definition = """ # metadata/attributes (e.g. FPS, config, calibration, etc.) associated with this experimental device + -> master + attribute_name : varchar(32) + --- + attribute_value=null : longblob + """ + + class RemovalTime(dj.Part): + definition = f""" + -> master + --- + video_source_removal_time: datetime(6) # time of the video_source being removed + """ + + +@schema +class PatchBeamBreak(dj.Imported): + definition = """# Raw per-chunk BeamBreak data stream from Patch (auto-generated with aeon_mecha-unknown) + -> Patch + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of BeamBreak data + """ + _stream_reader = aeon.io.reader.BitmaskEvent + _stream_detail = {'stream_type': 'BeamBreak', 'stream_reader': 'aeon.io.reader.BitmaskEvent', 'stream_reader_kwargs': {'pattern': '{pattern}_32', 'value': 34, 'tag': 'BeamBroken'}, 'stream_description': '', 'stream_hash': UUID('b14171e6-d27d-117a-ae73-a16c4b5fc8a2')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and Patch with overlapping time + + Chunk(s) that started after Patch install time and ended before Patch remove time + + Chunk(s) that started after Patch install time for Patch that are not yet removed + """ + return ( + acquisition.Chunk + * Patch.join(Patch.RemovalTime, left=True) + & 'chunk_start >= patch_install_time' + & 'chunk_start < IFNULL(patch_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (Patch & key).fetch1( + 'patch_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class PatchDeliverPellet(dj.Imported): + definition = """# Raw per-chunk DeliverPellet data stream from Patch (auto-generated with aeon_mecha-unknown) + -> Patch + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of DeliverPellet data + """ + _stream_reader = aeon.io.reader.BitmaskEvent + _stream_detail = {'stream_type': 'DeliverPellet', 'stream_reader': 'aeon.io.reader.BitmaskEvent', 'stream_reader_kwargs': {'pattern': '{pattern}_35', 'value': 1, 'tag': 'TriggeredPellet'}, 'stream_description': '', 'stream_hash': UUID('c49dda51-2e38-8b49-d1d8-2e54ea928e9c')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and Patch with overlapping time + + Chunk(s) that started after Patch install time and ended before Patch remove time + + Chunk(s) that started after Patch install time for Patch that are not yet removed + """ + return ( + acquisition.Chunk + * Patch.join(Patch.RemovalTime, left=True) + & 'chunk_start >= patch_install_time' + & 'chunk_start < IFNULL(patch_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (Patch & key).fetch1( + 'patch_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class PatchDepletionState(dj.Imported): + definition = """# Raw per-chunk DepletionState data stream from Patch (auto-generated with aeon_mecha-unknown) + -> Patch + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of DepletionState data + """ + _stream_reader = aeon.schema.foraging._PatchState + _stream_detail = {'stream_type': 'DepletionState', 'stream_reader': 'aeon.schema.foraging._PatchState', 'stream_reader_kwargs': {'pattern': '{pattern}_State'}, 'stream_description': '', 'stream_hash': UUID('73025490-348c-18fd-d565-8e682b5b4bcd')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and Patch with overlapping time + + Chunk(s) that started after Patch install time and ended before Patch remove time + + Chunk(s) that started after Patch install time for Patch that are not yet removed + """ + return ( + acquisition.Chunk + * Patch.join(Patch.RemovalTime, left=True) + & 'chunk_start >= patch_install_time' + & 'chunk_start < IFNULL(patch_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (Patch & key).fetch1( + 'patch_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class PatchEncoder(dj.Imported): + definition = """# Raw per-chunk Encoder data stream from Patch (auto-generated with aeon_mecha-unknown) + -> Patch + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of Encoder data + """ + _stream_reader = aeon.io.reader.Encoder + _stream_detail = {'stream_type': 'Encoder', 'stream_reader': 'aeon.io.reader.Encoder', 'stream_reader_kwargs': {'pattern': '{pattern}_90'}, 'stream_description': '', 'stream_hash': UUID('45002714-c31d-b2b8-a6e6-6ae624385cc1')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and Patch with overlapping time + + Chunk(s) that started after Patch install time and ended before Patch remove time + + Chunk(s) that started after Patch install time for Patch that are not yet removed + """ + return ( + acquisition.Chunk + * Patch.join(Patch.RemovalTime, left=True) + & 'chunk_start >= patch_install_time' + & 'chunk_start < IFNULL(patch_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (Patch & key).fetch1( + 'patch_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class UndergroundFeederBeamBreak(dj.Imported): + definition = """# Raw per-chunk BeamBreak data stream from UndergroundFeeder (auto-generated with aeon_mecha-unknown) + -> UndergroundFeeder + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of BeamBreak data + """ + _stream_reader = aeon.io.reader.BitmaskEvent + _stream_detail = {'stream_type': 'BeamBreak', 'stream_reader': 'aeon.io.reader.BitmaskEvent', 'stream_reader_kwargs': {'pattern': '{pattern}_32', 'value': 34, 'tag': 'BeamBroken'}, 'stream_description': '', 'stream_hash': UUID('b14171e6-d27d-117a-ae73-a16c4b5fc8a2')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and UndergroundFeeder with overlapping time + + Chunk(s) that started after UndergroundFeeder install time and ended before UndergroundFeeder remove time + + Chunk(s) that started after UndergroundFeeder install time for UndergroundFeeder that are not yet removed + """ + return ( + acquisition.Chunk + * UndergroundFeeder.join(UndergroundFeeder.RemovalTime, left=True) + & 'chunk_start >= underground_feeder_install_time' + & 'chunk_start < IFNULL(underground_feeder_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (UndergroundFeeder & key).fetch1( + 'underground_feeder_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class UndergroundFeederDeliverPellet(dj.Imported): + definition = """# Raw per-chunk DeliverPellet data stream from UndergroundFeeder (auto-generated with aeon_mecha-unknown) + -> UndergroundFeeder + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of DeliverPellet data + """ + _stream_reader = aeon.io.reader.BitmaskEvent + _stream_detail = {'stream_type': 'DeliverPellet', 'stream_reader': 'aeon.io.reader.BitmaskEvent', 'stream_reader_kwargs': {'pattern': '{pattern}_35', 'value': 1, 'tag': 'TriggeredPellet'}, 'stream_description': '', 'stream_hash': UUID('c49dda51-2e38-8b49-d1d8-2e54ea928e9c')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and UndergroundFeeder with overlapping time + + Chunk(s) that started after UndergroundFeeder install time and ended before UndergroundFeeder remove time + + Chunk(s) that started after UndergroundFeeder install time for UndergroundFeeder that are not yet removed + """ + return ( + acquisition.Chunk + * UndergroundFeeder.join(UndergroundFeeder.RemovalTime, left=True) + & 'chunk_start >= underground_feeder_install_time' + & 'chunk_start < IFNULL(underground_feeder_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (UndergroundFeeder & key).fetch1( + 'underground_feeder_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class UndergroundFeederDepletionState(dj.Imported): + definition = """# Raw per-chunk DepletionState data stream from UndergroundFeeder (auto-generated with aeon_mecha-unknown) + -> UndergroundFeeder + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of DepletionState data + """ + _stream_reader = aeon.schema.foraging._PatchState + _stream_detail = {'stream_type': 'DepletionState', 'stream_reader': 'aeon.schema.foraging._PatchState', 'stream_reader_kwargs': {'pattern': '{pattern}_State'}, 'stream_description': '', 'stream_hash': UUID('73025490-348c-18fd-d565-8e682b5b4bcd')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and UndergroundFeeder with overlapping time + + Chunk(s) that started after UndergroundFeeder install time and ended before UndergroundFeeder remove time + + Chunk(s) that started after UndergroundFeeder install time for UndergroundFeeder that are not yet removed + """ + return ( + acquisition.Chunk + * UndergroundFeeder.join(UndergroundFeeder.RemovalTime, left=True) + & 'chunk_start >= underground_feeder_install_time' + & 'chunk_start < IFNULL(underground_feeder_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (UndergroundFeeder & key).fetch1( + 'underground_feeder_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class UndergroundFeederEncoder(dj.Imported): + definition = """# Raw per-chunk Encoder data stream from UndergroundFeeder (auto-generated with aeon_mecha-unknown) + -> UndergroundFeeder + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of Encoder data + """ + _stream_reader = aeon.io.reader.Encoder + _stream_detail = {'stream_type': 'Encoder', 'stream_reader': 'aeon.io.reader.Encoder', 'stream_reader_kwargs': {'pattern': '{pattern}_90'}, 'stream_description': '', 'stream_hash': UUID('45002714-c31d-b2b8-a6e6-6ae624385cc1')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and UndergroundFeeder with overlapping time + + Chunk(s) that started after UndergroundFeeder install time and ended before UndergroundFeeder remove time + + Chunk(s) that started after UndergroundFeeder install time for UndergroundFeeder that are not yet removed + """ + return ( + acquisition.Chunk + * UndergroundFeeder.join(UndergroundFeeder.RemovalTime, left=True) + & 'chunk_start >= underground_feeder_install_time' + & 'chunk_start < IFNULL(underground_feeder_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (UndergroundFeeder & key).fetch1( + 'underground_feeder_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class VideoSourcePosition(dj.Imported): + definition = """# Raw per-chunk Position data stream from VideoSource (auto-generated with aeon_mecha-unknown) + -> VideoSource + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of Position data + """ + _stream_reader = aeon.io.reader.Position + _stream_detail = {'stream_type': 'Position', 'stream_reader': 'aeon.io.reader.Position', 'stream_reader_kwargs': {'pattern': '{pattern}_200'}, 'stream_description': '', 'stream_hash': UUID('75f9f365-037a-1e9b-ad38-8b2b3783315d')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and VideoSource with overlapping time + + Chunk(s) that started after VideoSource install time and ended before VideoSource remove time + + Chunk(s) that started after VideoSource install time for VideoSource that are not yet removed + """ + return ( + acquisition.Chunk + * VideoSource.join(VideoSource.RemovalTime, left=True) + & 'chunk_start >= video_source_install_time' + & 'chunk_start < IFNULL(video_source_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (VideoSource & key).fetch1( + 'video_source_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class VideoSourceRegion(dj.Imported): + definition = """# Raw per-chunk Region data stream from VideoSource (auto-generated with aeon_mecha-unknown) + -> VideoSource + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of Region data + """ + _stream_reader = aeon.schema.foraging._RegionReader + _stream_detail = {'stream_type': 'Region', 'stream_reader': 'aeon.schema.foraging._RegionReader', 'stream_reader_kwargs': {'pattern': '{pattern}_201'}, 'stream_description': '', 'stream_hash': UUID('6234a429-8ae5-d7dc-41c8-602ac76da029')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and VideoSource with overlapping time + + Chunk(s) that started after VideoSource install time and ended before VideoSource remove time + + Chunk(s) that started after VideoSource install time for VideoSource that are not yet removed + """ + return ( + acquisition.Chunk + * VideoSource.join(VideoSource.RemovalTime, left=True) + & 'chunk_start >= video_source_install_time' + & 'chunk_start < IFNULL(video_source_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (VideoSource & key).fetch1( + 'video_source_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + +@schema +class VideoSourceVideo(dj.Imported): + definition = """# Raw per-chunk Video data stream from VideoSource (auto-generated with aeon_mecha-unknown) + -> VideoSource + -> acquisition.Chunk + --- + sample_count: int # number of data points acquired from this stream for a given chunk + timestamps: longblob # (datetime) timestamps of Video data + """ + _stream_reader = aeon.io.reader.Video + _stream_detail = {'stream_type': 'Video', 'stream_reader': 'aeon.io.reader.Video', 'stream_reader_kwargs': {'pattern': '{pattern}'}, 'stream_description': '', 'stream_hash': UUID('4246295b-789f-206d-b413-7af25b7548b2')} + + @property + def key_source(self): + f""" + Only the combination of Chunk and VideoSource with overlapping time + + Chunk(s) that started after VideoSource install time and ended before VideoSource remove time + + Chunk(s) that started after VideoSource install time for VideoSource that are not yet removed + """ + return ( + acquisition.Chunk + * VideoSource.join(VideoSource.RemovalTime, left=True) + & 'chunk_start >= video_source_install_time' + & 'chunk_start < IFNULL(video_source_removal_time, "2200-01-01")' + ) + + def make(self, key): + chunk_start, chunk_end, dir_type = (acquisition.Chunk & key).fetch1( + "chunk_start", "chunk_end", "directory_type" + ) + raw_data_dir = acquisition.Experiment.get_data_directory( + key, directory_type=dir_type + ) + + device_name = (VideoSource & key).fetch1( + 'video_source_name' + ) + + stream = self._stream_reader( + **{ + k: v.format(**{k: device_name}) if k == "pattern" else v + for k, v in self._stream_detail["stream_reader_kwargs"].items() + } + ) + + stream_data = io_api.load( + root=raw_data_dir.as_posix(), + reader=stream, + start=pd.Timestamp(chunk_start), + end=pd.Timestamp(chunk_end), + ) + + self.insert1( + { + **key, + "sample_count": len(stream_data), + "timestamps": stream_data.index.values, + **{ + c: stream_data[c].values + for c in stream.columns + if not c.startswith("_") + }, + } + ) + + diff --git a/aeon/dj_pipeline/streams_maker.py b/aeon/dj_pipeline/streams_maker.py index 0bc19ea5..8b78f3ff 100644 --- a/aeon/dj_pipeline/streams_maker.py +++ b/aeon/dj_pipeline/streams_maker.py @@ -102,16 +102,18 @@ class RemovalTime(dj.Part): return ExperimentDevice -def get_device_stream_template(device_type: str, stream_type: str): +def get_device_stream_template(device_type: str, stream_type: str, streams_module): """Returns table class template for DeviceDataStream""" - context = inspect.currentframe().f_back.f_locals["context"] - ExperimentDevice = context[device_type] + ExperimentDevice = getattr(streams_module, device_type) # DeviceDataStream table(s) stream_detail = ( - StreamType - & (DeviceType.Stream & {"device_type": device_type, "stream_type": stream_type}) + streams_module.StreamType + & ( + streams_module.DeviceType.Stream + & {"device_type": device_type, "stream_type": stream_type} + ) ).fetch1() for i, n in enumerate(stream_detail["stream_reader"].split(".")): @@ -209,7 +211,8 @@ def main(create_tables=True): "#---- DO NOT MODIFY ----\n" "#---- THIS FILE IS AUTO-GENERATED BY `streams_maker.py` ----\n\n" "import datajoint as dj\n" - "import pandas as pd\n\n" + "import pandas as pd\n" + "from uuid import UUID\n\n" "import aeon\n" "from aeon.dj_pipeline import acquisition, get_schema_name\n" "from aeon.io import api as io_api\n\n" @@ -232,8 +235,7 @@ def main(create_tables=True): continue table_class = get_device_template(device_info["device_type"]) - # context[table_class.__name__] = table_class - # schema(table_class, context=context) + streams.__dict__[table_class.__name__] = table_class device_table_def = inspect.getsource(table_class).lstrip() replacements = { @@ -264,9 +266,9 @@ def main(create_tables=True): if hasattr(streams, table_name): continue - table_class = get_device_stream_template(device_type, stream_type) - # context[table_class.__name__] = table_class - # schema(table_class, context=context) + table_class = get_device_stream_template( + device_type, stream_type, streams_module=streams + ) stream_obj = table_class.__dict__["_stream_reader"] reader = stream_obj.__module__ + "." + stream_obj.__name__ From 5678b4a2d6e1a2f2c64c55ac2c04950d942053d9 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 5 Jul 2023 17:04:51 -0500 Subject: [PATCH 51/54] update workers and epochs/chunks ingestion --- aeon/dj_pipeline/populate/process.py | 4 +- aeon/dj_pipeline/populate/worker.py | 72 +++++++++++++++++++--------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/aeon/dj_pipeline/populate/process.py b/aeon/dj_pipeline/populate/process.py index c83347cb..023425ad 100644 --- a/aeon/dj_pipeline/populate/process.py +++ b/aeon/dj_pipeline/populate/process.py @@ -36,7 +36,7 @@ from datajoint_utilities.dj_worker import parse_args from aeon.dj_pipeline.populate.worker import ( - high_priority, + acquisition_worker, mid_priority, streams_worker, logger, @@ -46,7 +46,7 @@ # ---- some wrappers to support execution as script or CLI configured_workers = { - "high_priority": high_priority, + "high_priority": acquisition_worker, "mid_priority": mid_priority, "streams_worker": streams_worker, } diff --git a/aeon/dj_pipeline/populate/worker.py b/aeon/dj_pipeline/populate/worker.py index 395023b0..1ca198a0 100644 --- a/aeon/dj_pipeline/populate/worker.py +++ b/aeon/dj_pipeline/populate/worker.py @@ -20,7 +20,7 @@ streams = streams_maker.main() __all__ = [ - "high_priority", + "acquisition_worker", "mid_priority", "streams_worker", "WorkerLog", @@ -30,34 +30,61 @@ # ---- Some constants ---- logger = dj.logger -_current_experiment = "exp0.2-r0" -worker_schema_name = db_prefix + "workerlog" +worker_schema_name = db_prefix + "worker" load_metadata.insert_stream_types() +# ---- Manage experiments for automated ingestion ---- + +schema = dj.Schema(worker_schema_name) + + +@schema +class AutomatedExperimentIngestion(dj.Manual): + definition = """ # experiments to undergo automated ingestion + -> acquisition.Experiment + """ + + +def ingest_colony_epochs_chunks(): + """ + Load and insert subjects from colony.csv + Ingest epochs and chunks + for experiments specified in AutomatedExperimentIngestion + """ + load_metadata.ingest_subject() + experiment_names = AutomatedExperimentIngestion.fetch("experiment_name") + for experiment_name in experiment_names: + acquisition.Epoch.ingest_epochs(experiment_name) + acquisition.Chunk.ingest_chunks(experiment_name) + + +def ingest_environment_visits(): + """ + Extract and insert complete visits + for experiments specified in AutomatedExperimentIngestion + """ + experiment_names = AutomatedExperimentIngestion.fetch("experiment_name") + analysis.ingest_environment_visits(experiment_names) + + # ---- Define worker(s) ---- -# configure a worker to process high-priority tasks -high_priority = DataJointWorker( - "high_priority", +# configure a worker to process `acquisition`-related tasks +acquisition_worker = DataJointWorker( + "acquisition_worker", worker_schema_name=worker_schema_name, db_prefix=db_prefix, run_duration=-1, sleep_duration=600, ) -high_priority(load_metadata.ingest_subject) -high_priority(acquisition.Epoch.ingest_epochs, experiment_name=_current_experiment) -high_priority(acquisition.Chunk.ingest_chunks, experiment_name=_current_experiment) -high_priority(acquisition.ExperimentLog) -high_priority(acquisition.SubjectEnterExit) -high_priority(acquisition.SubjectWeight) -high_priority(acquisition.FoodPatchEvent) -high_priority(acquisition.WheelState) -high_priority(acquisition.WeightMeasurement) -high_priority(acquisition.WeightMeasurementFiltered) - -high_priority( - analysis.ingest_environment_visits, experiment_names=[_current_experiment] -) +acquisition_worker(ingest_colony_epochs_chunks) +acquisition_worker(acquisition.ExperimentLog) +acquisition_worker(acquisition.SubjectEnterExit) +acquisition_worker(acquisition.SubjectWeight) +acquisition_worker(acquisition.FoodPatchEvent) +acquisition_worker(acquisition.WheelState) + +acquisition_worker(ingest_environment_visits) # configure a worker to process mid-priority tasks mid_priority = DataJointWorker( @@ -71,10 +98,9 @@ mid_priority(qc.CameraQC) mid_priority(tracking.CameraTracking) mid_priority(acquisition.FoodPatchWheel) +mid_priority(acquisition.WeightMeasurement) +mid_priority(acquisition.WeightMeasurementFiltered) -mid_priority( - analysis.visit.ingest_environment_visits, experiment_names=[_current_experiment] -) mid_priority(analysis.OverlapVisit) mid_priority(analysis.VisitSubjectPosition) From 4e162310933ec92b71b6908a5d5506a9c2743081 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Thu, 6 Jul 2023 16:35:14 -0500 Subject: [PATCH 52/54] update worker containers and config --- aeon/dj_pipeline/acquisition.py | 5 ++--- aeon/dj_pipeline/populate/worker.py | 14 ++++++------- aeon/dj_pipeline/utils/load_metadata.py | 3 ++- aeon/dj_pipeline/{ => utils}/streams_maker.py | 0 docker/docker-compose.yml | 20 +++++++++++-------- 5 files changed, 23 insertions(+), 19 deletions(-) rename aeon/dj_pipeline/{ => utils}/streams_maker.py (100%) diff --git a/aeon/dj_pipeline/acquisition.py b/aeon/dj_pipeline/acquisition.py index 9a503a3a..03a9a0f4 100644 --- a/aeon/dj_pipeline/acquisition.py +++ b/aeon/dj_pipeline/acquisition.py @@ -10,7 +10,7 @@ from aeon.io import reader as io_reader from aeon.schema import dataset as aeon_schema -from . import get_schema_name, lab, subject +from . import get_schema_name from .utils import paths logger = dj.logger @@ -277,8 +277,7 @@ def ingest_epochs(cls, experiment_name, start=None, end=None): - if not specified, ingest all epochs Note: "start" and "end" are datetime specified a string in the format: "%Y-%m-%d %H:%M:%S" """ - from aeon.dj_pipeline import streams_maker - + from .utils import streams_maker from .utils.load_metadata import ( extract_epoch_config, ingest_epoch_metadata, diff --git a/aeon/dj_pipeline/populate/worker.py b/aeon/dj_pipeline/populate/worker.py index 1ca198a0..b57c4c4e 100644 --- a/aeon/dj_pipeline/populate/worker.py +++ b/aeon/dj_pipeline/populate/worker.py @@ -12,10 +12,10 @@ db_prefix, qc, report, - streams_maker, tracking, ) -from aeon.dj_pipeline.utils import load_metadata +from aeon.dj_pipeline.utils import load_metadata, streams_maker + streams = streams_maker.main() @@ -75,7 +75,7 @@ def ingest_environment_visits(): worker_schema_name=worker_schema_name, db_prefix=db_prefix, run_duration=-1, - sleep_duration=600, + sleep_duration=1200, ) acquisition_worker(ingest_colony_epochs_chunks) acquisition_worker(acquisition.ExperimentLog) @@ -92,7 +92,7 @@ def ingest_environment_visits(): worker_schema_name=worker_schema_name, db_prefix=db_prefix, run_duration=-1, - sleep_duration=120, + sleep_duration=3600, ) mid_priority(qc.CameraQC) @@ -121,10 +121,10 @@ def ingest_environment_visits(): "streams_worker", worker_schema_name=worker_schema_name, db_prefix=db_prefix, - run_duration=1, - sleep_duration=600, + run_duration=-1, + sleep_duration=1200, ) for attr in vars(streams).values(): if is_djtable(attr, dj.user_tables.AutoPopulate): - streams_worker(attr) + streams_worker(attr, max_calls=10) diff --git a/aeon/dj_pipeline/utils/load_metadata.py b/aeon/dj_pipeline/utils/load_metadata.py index 090b6b19..84de0a85 100644 --- a/aeon/dj_pipeline/utils/load_metadata.py +++ b/aeon/dj_pipeline/utils/load_metadata.py @@ -10,7 +10,8 @@ import pandas as pd from dotmap import DotMap -from aeon.dj_pipeline import acquisition, dict_to_uuid, subject, streams_maker +from aeon.dj_pipeline import acquisition, dict_to_uuid, subject +from aeon.dj_pipeline.utils import streams_maker from aeon.io import api as io_api diff --git a/aeon/dj_pipeline/streams_maker.py b/aeon/dj_pipeline/utils/streams_maker.py similarity index 100% rename from aeon/dj_pipeline/streams_maker.py rename to aeon/dj_pipeline/utils/streams_maker.py diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a6d1868b..e2dce70a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -45,22 +45,26 @@ x-aeon-ingest-common: &aeon-ingest-common max-file: "5" services: - ingest_high: + acquisition_worker: <<: *aeon-ingest-common - command: ["aeon_ingest", "high_priority", "--duration=-1", "--sleep=1200"] + command: ["aeon_ingest", "acquisition_worker"] - ingest_mid: + streams_worker: <<: *aeon-ingest-common depends_on: - ingest_high: + acquisition_worker: condition: service_started deploy: mode: replicated - replicas: 2 - command: ["aeon_ingest", "mid_priority", "--duration=-1", "--sleep=3600"] + replicas: 3 + command: ["aeon_ingest", "streams_worker"] - dev: + ingest_mid: <<: *aeon-ingest-common + depends_on: + acquisition_worker: + condition: service_started deploy: mode: replicated - replicas: 0 + replicas: 2 + command: ["aeon_ingest", "mid_priority"] From 68cb397ba39b90e60a52824aaef8e8869047c4a0 Mon Sep 17 00:00:00 2001 From: JaerongA Date: Fri, 7 Jul 2023 17:28:23 -0500 Subject: [PATCH 53/54] enable import streams from aeon.dj_pipeline --- aeon/dj_pipeline/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aeon/dj_pipeline/__init__.py b/aeon/dj_pipeline/__init__.py index 4664cb93..ea805b57 100644 --- a/aeon/dj_pipeline/__init__.py +++ b/aeon/dj_pipeline/__init__.py @@ -16,6 +16,12 @@ repository_config = dj.config['custom'].get('repository_config', _default_repository_config) +try: + from .utils import streams_maker + streams = dj.VirtualModule("streams", streams_maker.STREAMS_MODULE_NAME) +except: + pass + def get_schema_name(name): return db_prefix + name From 3d66f67c5de2ab7c5be7b4d3ccf75899f28aecac Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Mon, 10 Jul 2023 12:50:54 -0500 Subject: [PATCH 54/54] Apply suggestions from code review Co-authored-by: JaerongA --- aeon/dj_pipeline/utils/streams_maker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aeon/dj_pipeline/utils/streams_maker.py b/aeon/dj_pipeline/utils/streams_maker.py index 8b78f3ff..46043e94 100644 --- a/aeon/dj_pipeline/utils/streams_maker.py +++ b/aeon/dj_pipeline/utils/streams_maker.py @@ -16,7 +16,7 @@ schema_name = get_schema_name("streams") STREAMS_MODULE_NAME = "streams" -_STREAMS_MODULE_FILE = Path(__file__).parent / f"{STREAMS_MODULE_NAME}.py" +_STREAMS_MODULE_FILE = Path(__file__).parent.parent / f"{STREAMS_MODULE_NAME}.py" class StreamType(dj.Lookup):