From 0752b169dac98c8a776726d2e2dce3d6a5ec238a Mon Sep 17 00:00:00 2001 From: Kyu Hyun Lee Date: Tue, 9 May 2023 23:56:29 -0700 Subject: [PATCH 1/5] Save LFP as pynwb.ecephys.LFP --- src/spyglass/common/common_filter.py | 7 +++++-- src/spyglass/lfp/v1/lfp.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/spyglass/common/common_filter.py b/src/spyglass/common/common_filter.py index 701f92d31..931000525 100644 --- a/src/spyglass/common/common_filter.py +++ b/src/spyglass/common/common_filter.py @@ -254,8 +254,11 @@ def filter_data_nwb( electrodes=electrode_table_region, timestamps=np.empty(output_shape_list[time_axis]), ) - # Add the electrical series to the scratch area - nwbf.add_scratch(es) + lfp = pynwb.ecephys.LFP(electrical_series=es) + ecephys_module = nwbf.create_processing_module( + name="ecephys", description="filtered extracellular electrophysiology data" + ) + ecephys_module.add(lfp) io.write(nwbf) # reload the NWB file to get the h5py objects for the data and the timestamps diff --git a/src/spyglass/lfp/v1/lfp.py b/src/spyglass/lfp/v1/lfp.py index 41b4953ed..8140f3fd8 100644 --- a/src/spyglass/lfp/v1/lfp.py +++ b/src/spyglass/lfp/v1/lfp.py @@ -533,8 +533,11 @@ def make(self, key): electrodes=electrode_table_region, timestamps=new_timestamps, ) - # Add the electrical series to the scratch area - nwbf.add_scratch(es) + lfp = pynwb.ecephys.LFP(electrical_series=es) + ecephys_module = nwbf.create_processing_module( + name="ecephys", description="processed LFP band data" + ) + ecephys_module.add(lfp) io.write(nwbf) filtered_data_object_id = es.object_id # From 9fb9d7bf65f583ac7bf7d21efbce8e88019ee621 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 10 May 2023 07:41:04 -0700 Subject: [PATCH 2/5] Fix formatting --- src/spyglass/common/common_filter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spyglass/common/common_filter.py b/src/spyglass/common/common_filter.py index 931000525..c5b8b235d 100644 --- a/src/spyglass/common/common_filter.py +++ b/src/spyglass/common/common_filter.py @@ -256,7 +256,8 @@ def filter_data_nwb( ) lfp = pynwb.ecephys.LFP(electrical_series=es) ecephys_module = nwbf.create_processing_module( - name="ecephys", description="filtered extracellular electrophysiology data" + name="ecephys", + description="filtered extracellular electrophysiology data" ) ecephys_module.add(lfp) io.write(nwbf) From 5243b21096c6ed91f84477547b8051ce9b4ca3ce Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 10 May 2023 07:43:18 -0700 Subject: [PATCH 3/5] Fix formatting --- src/spyglass/common/common_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spyglass/common/common_filter.py b/src/spyglass/common/common_filter.py index c5b8b235d..9d40c5f50 100644 --- a/src/spyglass/common/common_filter.py +++ b/src/spyglass/common/common_filter.py @@ -257,7 +257,7 @@ def filter_data_nwb( lfp = pynwb.ecephys.LFP(electrical_series=es) ecephys_module = nwbf.create_processing_module( name="ecephys", - description="filtered extracellular electrophysiology data" + description="filtered extracellular electrophysiology data", ) ecephys_module.add(lfp) io.write(nwbf) From d7b06d4828f81ad03b0e0bca1aa5bfe34ff8a9c6 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Mon, 24 Jul 2023 00:36:39 -0400 Subject: [PATCH 4/5] Remove nb_conda from conda environment files to fix installation error (#599) --- environment.yml | 1 - environment_position.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/environment.yml b/environment.yml index d7420426f..24e5ce7c3 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,6 @@ dependencies: - bottleneck - ipympl - tqdm - - nb_conda - pyfftw<=0.12.0 # used by ghostipy. install from conda-forge so that it works on Mac ARM processors - pip: - pubnub<6.4.0 diff --git a/environment_position.yml b/environment_position.yml index f06cc2348..7e43b59df 100644 --- a/environment_position.yml +++ b/environment_position.yml @@ -34,7 +34,6 @@ dependencies: - bottleneck - ipympl - tqdm - - nb_conda - ffmpeg - pytorch<1.12.0 - torchvision From 5a987769b362a852c96c4d0ea33a8a475d5aa40e Mon Sep 17 00:00:00 2001 From: Daniel Gramling <57648931+dpeg22@users.noreply.github.com> Date: Sun, 23 Jul 2023 21:41:33 -0700 Subject: [PATCH 5/5] Move LFPElectrodeGroup to lfp_electrode.py (#594) Co-authored-by: rly --- src/spyglass/lfp/__init__.py | 2 + src/spyglass/lfp/lfp_electrode.py | 57 ++++++++++++++++++++ src/spyglass/lfp/lfp_imported.py | 26 +++++++++ src/spyglass/lfp/lfp_merge.py | 7 +-- src/spyglass/lfp/v1/__init__.py | 1 - src/spyglass/lfp/v1/lfp.py | 72 +------------------------ src/spyglass/lfp_band/lfp_band_merge.py | 4 +- 7 files changed, 93 insertions(+), 76 deletions(-) create mode 100644 src/spyglass/lfp/lfp_electrode.py create mode 100644 src/spyglass/lfp/lfp_imported.py diff --git a/src/spyglass/lfp/__init__.py b/src/spyglass/lfp/__init__.py index f0b973f5b..ec2198930 100644 --- a/src/spyglass/lfp/__init__.py +++ b/src/spyglass/lfp/__init__.py @@ -1 +1,3 @@ from spyglass.lfp.lfp_merge import LFPOutput +from spyglass.lfp.lfp_electrode import LFPElectrodeGroup +from spyglass.lfp.lfp_imported import ImportedLFP diff --git a/src/spyglass/lfp/lfp_electrode.py b/src/spyglass/lfp/lfp_electrode.py new file mode 100644 index 000000000..5e4f1b193 --- /dev/null +++ b/src/spyglass/lfp/lfp_electrode.py @@ -0,0 +1,57 @@ +import datajoint as dj + +from spyglass.common.common_ephys import Electrode +from spyglass.common.common_session import Session + +schema = dj.schema("lfp_electrode") + + +@schema +class LFPElectrodeGroup(dj.Manual): + definition = """ + -> Session # the session to which this LFP belongs + lfp_electrode_group_name: varchar(200) # the name of this group of electrodes + """ + + class LFPElectrode(dj.Part): + definition = """ + -> LFPElectrodeGroup # the group of electrodes to be filtered + -> Electrode # the electrode to be filtered + """ + + @staticmethod + def create_lfp_electrode_group( + nwb_file_name: str, group_name: str, electrode_list: list[int] + ): + """Adds an LFPElectrodeGroup and the individual electrodes + + Parameters + ---------- + nwb_file_name : str + The name of the nwb file (e.g. the session) + group_name : str + The name of this group (< 200 char) + electrode_list : list + A list of the electrode ids to include in this group. + """ + # remove the session and then recreate the session and Electrode list + # check to see if the user allowed the deletion + key = { + "nwb_file_name": nwb_file_name, + "lfp_electrode_group_name": group_name, + } + LFPElectrodeGroup().insert1(key, skip_duplicates=True) + + # TODO: do this in a better way + all_electrodes = (Electrode() & {"nwb_file_name": nwb_file_name}).fetch( + as_dict=True + ) + primary_key = Electrode.primary_key + for e in all_electrodes: + # create a dictionary so we can insert the electrodes + if e["electrode_id"] in electrode_list: + lfpelectdict = {k: v for k, v in e.items() if k in primary_key} + lfpelectdict["lfp_electrode_group_name"] = group_name + LFPElectrodeGroup().LFPElectrode.insert1( + lfpelectdict, skip_duplicates=True + ) diff --git a/src/spyglass/lfp/lfp_imported.py b/src/spyglass/lfp/lfp_imported.py new file mode 100644 index 000000000..bdb5f22da --- /dev/null +++ b/src/spyglass/lfp/lfp_imported.py @@ -0,0 +1,26 @@ +import datajoint as dj + +from spyglass.common.common_interval import IntervalList +from spyglass.common.common_nwbfile import AnalysisNwbfile +from spyglass.common.common_session import Session +from spyglass.lfp.lfp_electrode import LFPElectrodeGroup + +schema = dj.schema("lfp_imported") + + +@schema +class ImportedLFP(dj.Imported): + definition = """ + -> Session # the session to which this LFP belongs + -> LFPElectrodeGroup # the group of electrodes to be filtered + -> IntervalList # the original set of times to be filtered + lfp_object_id: varchar(40) # the NWB object ID for loading this object from the file + --- + lfp_sampling_rate: float # the sampling rate, in samples/sec + -> AnalysisNwbfile + """ + + def make(self, key): + raise NotImplementedError( + "For `insert`, use `allow_direct_insert=True`" + ) diff --git a/src/spyglass/lfp/lfp_merge.py b/src/spyglass/lfp/lfp_merge.py index 6db8f8f85..36b0f696b 100644 --- a/src/spyglass/lfp/lfp_merge.py +++ b/src/spyglass/lfp/lfp_merge.py @@ -4,7 +4,8 @@ from spyglass.common.common_ephys import LFP as CommonLFP # noqa: F401 from spyglass.common.common_filter import FirFilterParameters # noqa: F401 from spyglass.common.common_interval import IntervalList # noqa: F401 -from spyglass.lfp.v1.lfp import LFPV1, ImportedLFPV1 # noqa: F401 +from spyglass.lfp.v1.lfp import LFPV1 # noqa: F401 +from spyglass.lfp.lfp_imported import ImportedLFP # noqa: F401 from spyglass.utils.dj_merge_tables import _Merge schema = dj.schema("lfp_merge") @@ -25,11 +26,11 @@ class LFPV1(dj.Part): -> LFPV1 """ - class ImportedLFPV1(dj.Part): + class ImportedLFP(dj.Part): definition = """ -> master --- - -> ImportedLFPV1 + -> ImportedLFP """ class CommonLFP(dj.Part): diff --git a/src/spyglass/lfp/v1/__init__.py b/src/spyglass/lfp/v1/__init__.py index 2530a2fc0..64ad9c72a 100644 --- a/src/spyglass/lfp/v1/__init__.py +++ b/src/spyglass/lfp/v1/__init__.py @@ -1,7 +1,6 @@ # flake8: noqa from spyglass.lfp.v1.lfp import ( LFPV1, - ImportedLFPV1, LFPElectrodeGroup, LFPSelection, ) diff --git a/src/spyglass/lfp/v1/lfp.py b/src/spyglass/lfp/v1/lfp.py index c44ff4986..109792d74 100644 --- a/src/spyglass/lfp/v1/lfp.py +++ b/src/spyglass/lfp/v1/lfp.py @@ -4,7 +4,8 @@ import numpy as np import pandas as pd -from spyglass.common.common_ephys import Electrode, Raw +from spyglass.common.common_ephys import Raw +from spyglass.lfp.lfp_electrode import LFPElectrodeGroup from spyglass.common.common_filter import FirFilterParameters from spyglass.common.common_interval import ( IntervalList, @@ -20,57 +21,6 @@ MIN_LFP_INTERVAL_DURATION = 1.0 # 1 second minimum interval duration -@schema -class LFPElectrodeGroup(dj.Manual): - definition = """ - -> Session # the session to which this LFP belongs - lfp_electrode_group_name: varchar(200) # the name of this group of electrodes - """ - - class LFPElectrode(dj.Part): - definition = """ - -> LFPElectrodeGroup # the group of electrodes to be filtered - -> Electrode # the electrode to be filtered - """ - - @staticmethod - def create_lfp_electrode_group( - nwb_file_name: str, group_name: str, electrode_list: list[int] - ): - """Adds an LFPElectrodeGroup and the individual electrodes - - Parameters - ---------- - nwb_file_name : str - The name of the nwb file (e.g. the session) - group_name : str - The name of this group (< 200 char) - electrode_list : list - A list of the electrode ids to include in this group. - """ - # remove the session and then recreate the session and Electrode list - # check to see if the user allowed the deletion - key = { - "nwb_file_name": nwb_file_name, - "lfp_electrode_group_name": group_name, - } - LFPElectrodeGroup().insert1(key, skip_duplicates=True) - - # TODO: do this in a better way - all_electrodes = (Electrode() & {"nwb_file_name": nwb_file_name}).fetch( - as_dict=True - ) - primary_key = Electrode.primary_key - for e in all_electrodes: - # create a dictionary so we can insert the electrodes - if e["electrode_id"] in electrode_list: - lfpelectdict = {k: v for k, v in e.items() if k in primary_key} - lfpelectdict["lfp_electrode_group_name"] = group_name - LFPElectrodeGroup().LFPElectrode.insert1( - lfpelectdict, skip_duplicates=True - ) - - @schema class LFPSelection(dj.Manual): """The user's selection of LFP data to be filtered @@ -226,21 +176,3 @@ def fetch1_dataframe(self, *attrs, **kwargs): nwb_lfp["lfp"].data, index=pd.Index(nwb_lfp["lfp"].timestamps, name="time"), ) - - -@schema -class ImportedLFPV1(dj.Imported): - definition = """ - -> Session # the session to which this LFP belongs - -> LFPElectrodeGroup # the group of electrodes to be filtered - -> IntervalList # the original set of times to be filtered - lfp_object_id: varchar(40) # the NWB object ID for loading this object from the file - --- - lfp_sampling_rate: float # the sampling rate, in samples/sec - -> AnalysisNwbfile - """ - - def make(self, key): - raise NotImplementedError( - "For `insert`, use `allow_direct_insert=True`" - ) diff --git a/src/spyglass/lfp_band/lfp_band_merge.py b/src/spyglass/lfp_band/lfp_band_merge.py index d5cafdcce..9c6ea1494 100644 --- a/src/spyglass/lfp_band/lfp_band_merge.py +++ b/src/spyglass/lfp_band/lfp_band_merge.py @@ -2,13 +2,13 @@ import pandas as pd from spyglass.lfp_band.v1.lfp_band import LFPBandV1 # noqa F401 -from spyglass.utils.dj_merge_tables import Merge +from spyglass.utils.dj_merge_tables import _Merge schema = dj.schema("lfp_band_merge") @schema -class LFPBandOutput(Merge): +class LFPBandOutput(_Merge): definition = """ merge_id: uuid ---