From 67832ca5640c44148dd01d0bc160de13d97da62e Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Fri, 26 Apr 2024 14:19:50 -0700 Subject: [PATCH 01/17] added notes for western blot --- src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md index 6d966b6..b87ec05 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md @@ -127,3 +127,11 @@ Many MSNs (ex. 'FOOD_FR1 TTL Left', 'FOOD_FR1 TTL Right', and 'FOOD_RI 30 LEFT') - DMS-Excitatory has some csv files w/ only session-aggregated info (total right rewards but not right reward times) ex. ChR2/121_280.CSV -- do you have individual session info for these animals? - RI 60 LEFT_STIM, RI 30 LEFT_STIM, and RK_C_FR1_BOTH_1hr msns show up in opto data but don't have associated files -- assumed to be the same as their right counterparts? + +## Western Blot +### Notes +- Excel file has subject_ids for Female DLS Actin, Female DLS DAT, Female DMS Actin, Female DMS DAT and their + and their corresponding data (area, mean, min, max, white-sample, corrected sample-blank) BUT no male data. +- Tif files have western blot images for male and female all conditions BUT only 1 subject/condition (Fig S3A has ~7animals/condition) +- Tif files are combined WT on the left DAT on the right --> will need to split. +- How to deal with this data? Western Blot extension? Skip? Just include the images? From 92eac5cac4520f2f5a8182e3e0a8f28daa59606d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 21 May 2024 12:43:22 -0700 Subject: [PATCH 02/17] switched to lab metadata fp table --- frozen_dependencies.txt | 4 ++-- requirements.txt | 2 +- .../seiler_2024/seiler_2024fiberphotometryinterface.py | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frozen_dependencies.txt b/frozen_dependencies.txt index 2b75166..8b77de8 100644 --- a/frozen_dependencies.txt +++ b/frozen_dependencies.txt @@ -86,7 +86,7 @@ more-itertools==10.2.0 multidict==6.0.4 natsort==8.4.0 ndx-events==0.2.0 -ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@2e80a7ac040ddca2873cce03aceaabd7c816a5f6 +ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@14e05cda86026db694c1c577918284f435b7ec10 ndx-grayscalevolume==0.0.2 ndx-icephys-meta==0.1.0 ndx-photometry @ git+https://github.com/catalystneuro/ndx-photometry.git@c1284c91a7e0a5dfd19f84bbf683e17fa607d4ec @@ -97,7 +97,7 @@ neuroconv==0.4.7 nodeenv==1.8.0 numcodecs==0.11.0 numpy==1.26.3 -nwbinspector==0.4.33 +nwbinspector==0.4.35 nwbwidgets==0.11.3 openpyxl==3.1.2 packaging==23.2 diff --git a/requirements.txt b/requirements.txt index efa0ceb..72f10da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ dandi pre-commit pytest ndx-events -ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@main +ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@add_fiber_photometry_lab_meta_data ipykernel tdt openpyxl diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py index d064657..1c74094 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py @@ -7,6 +7,7 @@ from neuroconv.tools import nwb_helpers from pathlib import Path from ndx_fiber_photometry import ( + FiberPhotometry, FiberPhotometryTable, FiberPhotometryResponseSeries, CommandedVoltageSeries, @@ -364,6 +365,10 @@ def add_to_nwbfile( description="The region of the FiberPhotometryTable corresponding to the DLS reference.", region=[3], ) + fiber_photometry_table_metadata = FiberPhotometry( + name="fiber_photometry", + fiber_photometry_table=fiber_photometry_table, + ) # Fiber Photometry Response Series dms_signal_series = FiberPhotometryResponseSeries( @@ -420,7 +425,7 @@ def add_to_nwbfile( else: nwbfile.add_acquisition(dms_commanded_voltage_series) nwbfile.add_acquisition(dls_commanded_voltage_series) - nwbfile.add_acquisition(fiber_photometry_table) + nwbfile.add_lab_meta_data(fiber_photometry_table_metadata) nwbfile.add_acquisition(dms_signal_series) nwbfile.add_acquisition(dms_reference_series) nwbfile.add_acquisition(dls_signal_series) From a30f7edf07dbde91832b7ca25a7729bd35a4f354 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 21 May 2024 13:13:01 -0700 Subject: [PATCH 03/17] consolidated optical fibers --- .../seiler_2024fiberphotometryinterface.py | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py index 1c74094..2b50d16 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py @@ -85,16 +85,8 @@ def add_to_nwbfile( ) # Optical Fibers - dms_fiber = OpticalFiber( - name="dms_fiber", - description="Fiber optic implants (Doric Lenses; 400 um, 0.48 NA) were placed above DMS (AP 0.8, ML 1.5, DV 2.8) and DLS (AP 0.1, ML 2.8, DV 3.5). The DMS implant was placed in the hemisphere receiving a medial SNc viral injection, while the DLS implant was placed in the hemisphere receiving a lateral SNc viral injection. Calcium signals from dopamine terminals in DMS and DLS were recorded during RI30, on the first and last days of RI60/RR20 training as well as on both footshock probes for each mouse. All recordings were done using a fiber photometry rig with optical components from Doric lenses controlled by a real-time processor from Tucker Davis Technologies (TDT; RZ5P). TDT Synapse software was used for data acquisition.", - manufacturer="Doric Lenses", - model="Fiber Optic Implant", - numerical_aperture=0.48, - core_diameter_in_um=400.0, - ) - dls_fiber = OpticalFiber( - name="dls_fiber", + optical_fiber = OpticalFiber( + name="optical_fiber", description="Fiber optic implants (Doric Lenses; 400 um, 0.48 NA) were placed above DMS (AP 0.8, ML 1.5, DV 2.8) and DLS (AP 0.1, ML 2.8, DV 3.5). The DMS implant was placed in the hemisphere receiving a medial SNc viral injection, while the DLS implant was placed in the hemisphere receiving a lateral SNc viral injection. Calcium signals from dopamine terminals in DMS and DLS were recorded during RI30, on the first and last days of RI60/RR20 training as well as on both footshock probes for each mouse. All recordings were done using a fiber photometry rig with optical components from Doric lenses controlled by a real-time processor from Tucker Davis Technologies (TDT; RZ5P). TDT Synapse software was used for data acquisition.", manufacturer="Doric Lenses", model="Fiber Optic Implant", @@ -257,7 +249,7 @@ def add_to_nwbfile( coordinates=(0.8, 1.5, 2.8), commanded_voltage_series=dms_commanded_signal_series, indicator=dms_fluorophore, - optical_fiber=dms_fiber, + optical_fiber=optical_fiber, excitation_source=dms_signal_excitation_source, photodetector=photodetector, excitation_filter=excitation_filter, @@ -269,7 +261,7 @@ def add_to_nwbfile( coordinates=(0.8, 1.5, 2.8), commanded_voltage_series=dms_commanded_reference_series, indicator=dms_fluorophore, - optical_fiber=dms_fiber, + optical_fiber=optical_fiber, excitation_source=dms_reference_excitation_source, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, @@ -281,7 +273,7 @@ def add_to_nwbfile( coordinates=(0.1, 2.8, 3.5), commanded_voltage_series=dls_commanded_signal_series, indicator=dls_fluorophore, - optical_fiber=dls_fiber, + optical_fiber=optical_fiber, excitation_source=dls_signal_excitation_source, photodetector=photodetector, excitation_filter=excitation_filter, @@ -293,7 +285,7 @@ def add_to_nwbfile( coordinates=(0.1, 2.8, 3.5), commanded_voltage_series=dls_commanded_reference_series, indicator=dls_fluorophore, - optical_fiber=dls_fiber, + optical_fiber=optical_fiber, excitation_source=dls_reference_excitation_source, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, @@ -306,7 +298,7 @@ def add_to_nwbfile( coordinates=(0.8, 1.5, 2.8), commanded_voltage_series=dms_commanded_voltage_series, indicator=dms_fluorophore, - optical_fiber=dms_fiber, + optical_fiber=optical_fiber, excitation_source=dms_signal_excitation_source, photodetector=photodetector, excitation_filter=excitation_filter, @@ -318,7 +310,7 @@ def add_to_nwbfile( coordinates=(0.8, 1.5, 2.8), commanded_voltage_series=dms_commanded_voltage_series, indicator=dms_fluorophore, - optical_fiber=dms_fiber, + optical_fiber=optical_fiber, excitation_source=dms_reference_excitation_source, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, @@ -330,7 +322,7 @@ def add_to_nwbfile( coordinates=(0.1, 2.8, 3.5), commanded_voltage_series=dls_commanded_voltage_series, indicator=dls_fluorophore, - optical_fiber=dls_fiber, + optical_fiber=optical_fiber, excitation_source=dls_signal_excitation_source, photodetector=photodetector, excitation_filter=excitation_filter, @@ -342,7 +334,7 @@ def add_to_nwbfile( coordinates=(0.1, 2.8, 3.5), commanded_voltage_series=dls_commanded_voltage_series, indicator=dls_fluorophore, - optical_fiber=dls_fiber, + optical_fiber=optical_fiber, excitation_source=dls_reference_excitation_source, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, @@ -404,8 +396,7 @@ def add_to_nwbfile( fiber_photometry_table_region=dls_reference_region, ) - nwbfile.add_device(dms_fiber) - nwbfile.add_device(dls_fiber) + nwbfile.add_device(optical_fiber) nwbfile.add_device(dms_signal_excitation_source) nwbfile.add_device(dms_reference_excitation_source) nwbfile.add_device(dls_signal_excitation_source) From c30450f58188f2d24e0e24f86bfac6e7d1a5df74 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 21 May 2024 13:30:12 -0700 Subject: [PATCH 04/17] consolidated excitation sources --- .../seiler_2024fiberphotometryinterface.py | 46 ++++++------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py index 2b50d16..d2fbe57 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py @@ -95,32 +95,16 @@ def add_to_nwbfile( ) # Excitation Sources - dms_signal_excitation_source = ExcitationSource( - name="dms_signal_excitation_source", + excitation_source_calcium_signal = ExcitationSource( + name="excitation_source_calcium_signal", description="465nm and 405nm LEDs were modulated at 211 Hz and 330 Hz, respectively, for DMS probes. 465nm and 405nm LEDs were modulated at 450 Hz and 270 Hz, respectively for DLS probes. LED currents were adjusted in order to return a voltage between 150-200mV for each signal, were offset by 5 mA, were demodulated using a 4 Hz lowpass frequency filter.", manufacturer="Doric Lenses", model="Connectorized LED", illumination_type="LED", excitation_wavelength_in_nm=465.0, ) - dms_reference_excitation_source = ExcitationSource( - name="dms_reference_excitation_source", - description="465nm and 405nm LEDs were modulated at 211 Hz and 330 Hz, respectively, for DMS probes. 465nm and 405nm LEDs were modulated at 450 Hz and 270 Hz, respectively for DLS probes. LED currents were adjusted in order to return a voltage between 150-200mV for each signal, were offset by 5 mA, were demodulated using a 4 Hz lowpass frequency filter.", - manufacturer="Doric Lenses", - model="Connectorized LED", - illumination_type="LED", - excitation_wavelength_in_nm=405.0, - ) - dls_signal_excitation_source = ExcitationSource( - name="dls_signal_excitation_source", - description="465nm and 405nm LEDs were modulated at 211 Hz and 330 Hz, respectively, for DMS probes. 465nm and 405nm LEDs were modulated at 450 Hz and 270 Hz, respectively for DLS probes. LED currents were adjusted in order to return a voltage between 150-200mV for each signal, were offset by 5 mA, were demodulated using a 4 Hz lowpass frequency filter.", - manufacturer="Doric Lenses", - model="Connectorized LED", - illumination_type="LED", - excitation_wavelength_in_nm=465.0, - ) - dls_reference_excitation_source = ExcitationSource( - name="dls_reference_excitation_source", + excitation_source_isosbestic_control = ExcitationSource( + name="excitation_source_isosbestic_control", description="465nm and 405nm LEDs were modulated at 211 Hz and 330 Hz, respectively, for DMS probes. 465nm and 405nm LEDs were modulated at 450 Hz and 270 Hz, respectively for DLS probes. LED currents were adjusted in order to return a voltage between 150-200mV for each signal, were offset by 5 mA, were demodulated using a 4 Hz lowpass frequency filter.", manufacturer="Doric Lenses", model="Connectorized LED", @@ -250,7 +234,7 @@ def add_to_nwbfile( commanded_voltage_series=dms_commanded_signal_series, indicator=dms_fluorophore, optical_fiber=optical_fiber, - excitation_source=dms_signal_excitation_source, + excitation_source=excitation_source_calcium_signal, photodetector=photodetector, excitation_filter=excitation_filter, emission_filter=emission_filter, @@ -262,7 +246,7 @@ def add_to_nwbfile( commanded_voltage_series=dms_commanded_reference_series, indicator=dms_fluorophore, optical_fiber=optical_fiber, - excitation_source=dms_reference_excitation_source, + excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, emission_filter=emission_filter, @@ -274,7 +258,7 @@ def add_to_nwbfile( commanded_voltage_series=dls_commanded_signal_series, indicator=dls_fluorophore, optical_fiber=optical_fiber, - excitation_source=dls_signal_excitation_source, + excitation_source=excitation_source_calcium_signal, photodetector=photodetector, excitation_filter=excitation_filter, emission_filter=emission_filter, @@ -286,7 +270,7 @@ def add_to_nwbfile( commanded_voltage_series=dls_commanded_reference_series, indicator=dls_fluorophore, optical_fiber=optical_fiber, - excitation_source=dls_reference_excitation_source, + excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, emission_filter=emission_filter, @@ -299,7 +283,7 @@ def add_to_nwbfile( commanded_voltage_series=dms_commanded_voltage_series, indicator=dms_fluorophore, optical_fiber=optical_fiber, - excitation_source=dms_signal_excitation_source, + excitation_source=excitation_source_calcium_signal, photodetector=photodetector, excitation_filter=excitation_filter, emission_filter=emission_filter, @@ -311,7 +295,7 @@ def add_to_nwbfile( commanded_voltage_series=dms_commanded_voltage_series, indicator=dms_fluorophore, optical_fiber=optical_fiber, - excitation_source=dms_reference_excitation_source, + excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, emission_filter=emission_filter, @@ -323,7 +307,7 @@ def add_to_nwbfile( commanded_voltage_series=dls_commanded_voltage_series, indicator=dls_fluorophore, optical_fiber=optical_fiber, - excitation_source=dls_signal_excitation_source, + excitation_source=excitation_source_calcium_signal, photodetector=photodetector, excitation_filter=excitation_filter, emission_filter=emission_filter, @@ -335,7 +319,7 @@ def add_to_nwbfile( commanded_voltage_series=dls_commanded_voltage_series, indicator=dls_fluorophore, optical_fiber=optical_fiber, - excitation_source=dls_reference_excitation_source, + excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, excitation_filter=isosbestic_excitation_filter, emission_filter=emission_filter, @@ -397,10 +381,8 @@ def add_to_nwbfile( ) nwbfile.add_device(optical_fiber) - nwbfile.add_device(dms_signal_excitation_source) - nwbfile.add_device(dms_reference_excitation_source) - nwbfile.add_device(dls_signal_excitation_source) - nwbfile.add_device(dls_reference_excitation_source) + nwbfile.add_device(excitation_source_calcium_signal) + nwbfile.add_device(excitation_source_isosbestic_control) nwbfile.add_device(photodetector) nwbfile.add_device(excitation_filter) nwbfile.add_device(isosbestic_excitation_filter) From 1e533bf44fd1a8321b9b676765d050f4fe3c0bec Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 21 May 2024 17:03:24 -0700 Subject: [PATCH 05/17] renamed commanded voltage series for consistency --- .../seiler_2024fiberphotometryinterface.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py index d2fbe57..a983160 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py @@ -161,16 +161,16 @@ def add_to_nwbfile( ) # Indicators (aka Fluorophores) - dms_fluorophore = Indicator( - name="dms_fluorophore", + dms_green_fluorophore = Indicator( + name="dms_green_fluorophore", description="Mice for fiber photometry experiments received infusions of 1ml of AAV5-CAG-FLEX-jGCaMP7b-WPRE (1.02e13 vg/mL, Addgene, lot 18-429) into lateral SNc (AP 3.1, ML 1.3, DV 4.2) in one hemisphere and medial SNc (AP 3.1, ML 0.8, DV 4.7) in the other. Hemispheres were counterbalanced between mice.", manufacturer="Addgene", label="GCaMP7b", injection_location="medial SNc", injection_coordinates_in_mm=(3.1, 0.8, 4.7), ) - dls_fluorophore = Indicator( - name="dls_fluorophore", + dls_green_fluorophore = Indicator( + name="dls_green_fluorophore", description="Mice for fiber photometry experiments received infusions of 1ml of AAV5-CAG-FLEX-jGCaMP7b-WPRE (1.02e13 vg/mL, Addgene, lot 18-429) into lateral SNc (AP 3.1, ML 1.3, DV 4.2) in one hemisphere and medial SNc (AP 3.1, ML 0.8, DV 4.7) in the other. Hemispheres were counterbalanced between mice.", manufacturer="Addgene", label="GCaMP7b", @@ -180,43 +180,43 @@ def add_to_nwbfile( # Commanded Voltage Series if has_demodulated_commanded_voltages: - dms_commanded_signal_series = CommandedVoltageSeries( - name="dms_commanded_signal", + commanded_voltage_series_dms_calcium_signal = CommandedVoltageSeries( + name="commanded_voltage_series_dms_calcium_signal", data=H5DataIO(tdt_photometry.streams["Fi1d"].data[0, :], compression=True), unit="volts", frequency=211.0, rate=tdt_photometry.streams["Fi1d"].fs, ) - dms_commanded_reference_series = CommandedVoltageSeries( - name="dms_commanded_reference", + commanded_voltage_series_dms_isosbestic_control = CommandedVoltageSeries( + name="commanded_voltage_series_dms_isosbestic_control", data=H5DataIO(tdt_photometry.streams["Fi1d"].data[1, :], compression=True), unit="volts", frequency=330.0, rate=tdt_photometry.streams["Fi1d"].fs, ) - dls_commanded_signal_series = CommandedVoltageSeries( - name="dls_commanded_signal", + commanded_voltage_series_dls_calcium_signal = CommandedVoltageSeries( + name="commanded_voltage_series_dls_calcium_signal", data=H5DataIO(tdt_photometry.streams["Fi1d"].data[3, :], compression=True), unit="volts", frequency=450.0, rate=tdt_photometry.streams["Fi1d"].fs, ) - dls_commanded_reference_series = CommandedVoltageSeries( - name="dls_commanded_reference", + commanded_voltage_series_dls_isosbestic_control = CommandedVoltageSeries( + name="commanded_voltage_series_dls_isosbestic_control", data=H5DataIO(tdt_photometry.streams["Fi1d"].data[2, :], compression=True), unit="volts", frequency=270.0, rate=tdt_photometry.streams["Fi1d"].fs, ) else: - dms_commanded_voltage_series = CommandedVoltageSeries( - name="dms_commanded_voltage", + commanded_voltage_series_dms = CommandedVoltageSeries( + name="commanded_voltage_series_dms", data=H5DataIO(tdt_photometry.streams["Fi1r"].data[0, :], compression=True), unit="volts", rate=tdt_photometry.streams["Fi1r"].fs, ) - dls_commanded_voltage_series = CommandedVoltageSeries( - name="dls_commanded_voltage", + commanded_voltage_series_dls = CommandedVoltageSeries( + name="commanded_voltage_series_dls", data=H5DataIO(tdt_photometry.streams["Fi1r"].data[1, :], compression=True), unit="volts", rate=tdt_photometry.streams["Fi1r"].fs, @@ -231,8 +231,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DMS", coordinates=(0.8, 1.5, 2.8), - commanded_voltage_series=dms_commanded_signal_series, - indicator=dms_fluorophore, + commanded_voltage_series=commanded_voltage_series_dms_calcium_signal, + indicator=dms_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_calcium_signal, photodetector=photodetector, @@ -243,8 +243,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DMS", coordinates=(0.8, 1.5, 2.8), - commanded_voltage_series=dms_commanded_reference_series, - indicator=dms_fluorophore, + commanded_voltage_series=commanded_voltage_series_dms_isosbestic_control, + indicator=dms_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, @@ -255,8 +255,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DLS", coordinates=(0.1, 2.8, 3.5), - commanded_voltage_series=dls_commanded_signal_series, - indicator=dls_fluorophore, + commanded_voltage_series=commanded_voltage_series_dls_calcium_signal, + indicator=dls_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_calcium_signal, photodetector=photodetector, @@ -267,8 +267,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DLS", coordinates=(0.1, 2.8, 3.5), - commanded_voltage_series=dls_commanded_reference_series, - indicator=dls_fluorophore, + commanded_voltage_series=commanded_voltage_series_dls_isosbestic_control, + indicator=dls_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, @@ -280,8 +280,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DMS", coordinates=(0.8, 1.5, 2.8), - commanded_voltage_series=dms_commanded_voltage_series, - indicator=dms_fluorophore, + commanded_voltage_series=commanded_voltage_series_dms, + indicator=dms_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_calcium_signal, photodetector=photodetector, @@ -292,8 +292,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DMS", coordinates=(0.8, 1.5, 2.8), - commanded_voltage_series=dms_commanded_voltage_series, - indicator=dms_fluorophore, + commanded_voltage_series=commanded_voltage_series_dms, + indicator=dms_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, @@ -304,8 +304,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DLS", coordinates=(0.1, 2.8, 3.5), - commanded_voltage_series=dls_commanded_voltage_series, - indicator=dls_fluorophore, + commanded_voltage_series=commanded_voltage_series_dls, + indicator=dls_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_calcium_signal, photodetector=photodetector, @@ -316,8 +316,8 @@ def add_to_nwbfile( fiber_photometry_table.add_row( location="DLS", coordinates=(0.1, 2.8, 3.5), - commanded_voltage_series=dls_commanded_voltage_series, - indicator=dls_fluorophore, + commanded_voltage_series=commanded_voltage_series_dls, + indicator=dls_green_fluorophore, optical_fiber=optical_fiber, excitation_source=excitation_source_isosbestic_control, photodetector=photodetector, @@ -388,16 +388,16 @@ def add_to_nwbfile( nwbfile.add_device(isosbestic_excitation_filter) nwbfile.add_device(emission_filter) nwbfile.add_device(dichroic_mirror) - nwbfile.add_device(dms_fluorophore) - nwbfile.add_device(dls_fluorophore) + nwbfile.add_device(dms_green_fluorophore) + nwbfile.add_device(dls_green_fluorophore) if has_demodulated_commanded_voltages: - nwbfile.add_acquisition(dms_commanded_signal_series) - nwbfile.add_acquisition(dms_commanded_reference_series) - nwbfile.add_acquisition(dls_commanded_signal_series) - nwbfile.add_acquisition(dls_commanded_reference_series) + nwbfile.add_acquisition(commanded_voltage_series_dms_calcium_signal) + nwbfile.add_acquisition(commanded_voltage_series_dms_isosbestic_control) + nwbfile.add_acquisition(commanded_voltage_series_dls_calcium_signal) + nwbfile.add_acquisition(commanded_voltage_series_dls_isosbestic_control) else: - nwbfile.add_acquisition(dms_commanded_voltage_series) - nwbfile.add_acquisition(dls_commanded_voltage_series) + nwbfile.add_acquisition(commanded_voltage_series_dms) + nwbfile.add_acquisition(commanded_voltage_series_dls) nwbfile.add_lab_meta_data(fiber_photometry_table_metadata) nwbfile.add_acquisition(dms_signal_series) nwbfile.add_acquisition(dms_reference_series) From 6ab82e1923933fffc66c205a209483646bdabd7f Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 21 May 2024 18:06:55 -0700 Subject: [PATCH 06/17] consolidated fiber photometry response series --- .../seiler_2024fiberphotometryinterface.py | 65 +++++-------------- 1 file changed, 17 insertions(+), 48 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py index a983160..ea49cb4 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024fiberphotometryinterface.py @@ -325,21 +325,9 @@ def add_to_nwbfile( emission_filter=emission_filter, dichroic_mirror=dichroic_mirror, ) - dms_signal_region = fiber_photometry_table.create_fiber_photometry_table_region( - description="The region of the FiberPhotometryTable corresponding to the DMS signal.", - region=[0], - ) - dms_reference_region = fiber_photometry_table.create_fiber_photometry_table_region( - description="The region of the FiberPhotometryTable corresponding to the DMS reference.", - region=[1], - ) - dls_signal_region = fiber_photometry_table.create_fiber_photometry_table_region( - description="The region of the FiberPhotometryTable corresponding to the DLS signal.", - region=[2], - ) - dls_reference_region = fiber_photometry_table.create_fiber_photometry_table_region( - description="The region of the FiberPhotometryTable corresponding to the DLS reference.", - region=[3], + fiber_photometry_table_region = fiber_photometry_table.create_fiber_photometry_table_region( + description="The region of the FiberPhotometryTable corresponding to the DMS calcium signal, DMS isosbestic control, DLS calcium signal, and DLS isosbestic control.", + region=[0, 1, 2, 3], ) fiber_photometry_table_metadata = FiberPhotometry( name="fiber_photometry", @@ -347,37 +335,21 @@ def add_to_nwbfile( ) # Fiber Photometry Response Series - dms_signal_series = FiberPhotometryResponseSeries( - name="dms_signal", - description="The fluorescence from the blue light excitation (465nm) corresponding to the calcium signal in the DMS.", - data=H5DataIO(tdt_photometry.streams["Dv1A"].data, compression=True), - unit="a.u.", - rate=tdt_photometry.streams["Dv1A"].fs, - fiber_photometry_table_region=dms_signal_region, + fiber_photometry_data = np.column_stack( + ( + tdt_photometry.streams["Dv1A"].data, + tdt_photometry.streams["Dv2A"].data, + tdt_photometry.streams["Dv3B"].data, + tdt_photometry.streams["Dv4B"].data, + ), ) - dms_reference_series = FiberPhotometryResponseSeries( - name="dms_reference", - description="The fluorescence from the UV light excitation (405nm) corresponding to the isosbestic reference in the DMS.", - data=H5DataIO(tdt_photometry.streams["Dv2A"].data, compression=True), + fiber_photometry_response_series = FiberPhotometryResponseSeries( + name="fiber_photometry_response_series", + description="The fluorescence from the DMS calcium signal, DMS isosbestic control, DLS calcium signal, and DLS isosbestic control.", + data=H5DataIO(fiber_photometry_data, compression=True), unit="a.u.", - rate=tdt_photometry.streams["Dv2A"].fs, - fiber_photometry_table_region=dms_reference_region, - ) - dls_signal_series = FiberPhotometryResponseSeries( - name="dls_signal", - description="The fluorescence from the blue light excitation (465nm) corresponding to the calcium signal in the DLS.", - data=H5DataIO(tdt_photometry.streams["Dv3B"].data, compression=True), - unit="a.u.", - rate=tdt_photometry.streams["Dv3B"].fs, - fiber_photometry_table_region=dls_signal_region, - ) - dls_reference_series = FiberPhotometryResponseSeries( - name="dls_reference", - description="The fluorescence from the UV light excitation (405nm) corresponding to the isosbestic reference in the DLS.", - data=H5DataIO(tdt_photometry.streams["Dv4B"].data, compression=True), - unit="a.u.", - rate=tdt_photometry.streams["Dv4B"].fs, - fiber_photometry_table_region=dls_reference_region, + rate=tdt_photometry.streams["Dv1A"].fs, + fiber_photometry_table_region=fiber_photometry_table_region, ) nwbfile.add_device(optical_fiber) @@ -399,7 +371,4 @@ def add_to_nwbfile( nwbfile.add_acquisition(commanded_voltage_series_dms) nwbfile.add_acquisition(commanded_voltage_series_dls) nwbfile.add_lab_meta_data(fiber_photometry_table_metadata) - nwbfile.add_acquisition(dms_signal_series) - nwbfile.add_acquisition(dms_reference_series) - nwbfile.add_acquisition(dls_signal_series) - nwbfile.add_acquisition(dls_reference_series) + nwbfile.add_acquisition(fiber_photometry_response_series) From b4034a98139c1f8b7a01b183a7e3433f98095af0 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 22 May 2024 16:28:41 -0700 Subject: [PATCH 07/17] initial setup of western blot --- frozen_dependencies.txt | 5 +- src/lerner_lab_to_nwb/seiler_2024/__init__.py | 3 +- .../seiler_2024_convert_session.py | 1258 +++++++++-------- .../seiler_2024/seiler_2024nwbconverter.py | 9 + .../seiler_2024westernblotinterface.py | 49 + 5 files changed, 715 insertions(+), 609 deletions(-) create mode 100644 src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py diff --git a/frozen_dependencies.txt b/frozen_dependencies.txt index a7dac87..8b77de8 100644 --- a/frozen_dependencies.txt +++ b/frozen_dependencies.txt @@ -43,7 +43,7 @@ fscacher==0.4.0 fsspec==2023.12.2 gast==0.4.0 h5py==3.10.0 -hdmf==3.12.2 +hdmf==3.13.0 hdmf_zarr==0.5.0 humanize==4.9.0 identify==2.5.33 @@ -86,6 +86,7 @@ more-itertools==10.2.0 multidict==6.0.4 natsort==8.4.0 ndx-events==0.2.0 +ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@14e05cda86026db694c1c577918284f435b7ec10 ndx-grayscalevolume==0.0.2 ndx-icephys-meta==0.1.0 ndx-photometry @ git+https://github.com/catalystneuro/ndx-photometry.git@c1284c91a7e0a5dfd19f84bbf683e17fa607d4ec @@ -96,7 +97,7 @@ neuroconv==0.4.7 nodeenv==1.8.0 numcodecs==0.11.0 numpy==1.26.3 -nwbinspector==0.4.33 +nwbinspector==0.4.35 nwbwidgets==0.11.3 openpyxl==3.1.2 packaging==23.2 diff --git a/src/lerner_lab_to_nwb/seiler_2024/__init__.py b/src/lerner_lab_to_nwb/seiler_2024/__init__.py index 15fccf3..87b121f 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/__init__.py +++ b/src/lerner_lab_to_nwb/seiler_2024/__init__.py @@ -2,4 +2,5 @@ from .seiler_2024fiberphotometryinterface import Seiler2024FiberPhotometryInterface from .seiler_2024optogeneticinterface import Seiler2024OptogeneticInterface from .seiler_2024excelmetadatainterface import Seiler2024ExcelMetadataInterface -from .seiler_2024nwbconverter import Seiler2024NWBConverter +from .seiler_2024westernblotinterface import Seiler2024WesternBlotInterface +from .seiler_2024nwbconverter import Seiler2024NWBConverter, Seiler2024WesternBlotNWBConverter diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py index 003bcc7..8ee2baa 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py @@ -6,7 +6,7 @@ from datetime import datetime from pytz import timezone -from lerner_lab_to_nwb.seiler_2024 import Seiler2024NWBConverter +from lerner_lab_to_nwb.seiler_2024 import Seiler2024NWBConverter, Seiler2024WesternBlotNWBConverter def session_to_nwb( @@ -168,619 +168,665 @@ def session_to_nwb( converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) -if __name__ == "__main__": - # Parameters for conversion - data_dir_path = Path("/Volumes/T7/CatalystNeuro/NWB/Lerner/raw_data") - output_dir_path = Path("/Volumes/T7/CatalystNeuro/NWB/Lerner/conversion_nwb") - stub_test = False - - if output_dir_path.exists(): - shutil.rmtree( - output_dir_path, ignore_errors=True - ) # ignore errors due to MacOS race condition (https://github.com/python/cpython/issues/81441) - - # No-shock example session - experiment_type = "FP" - experimental_group = "RR20" - subject_id = "95.259" - start_datetime = datetime(2019, 4, 9, 10, 34, 30) - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}" - ) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) - - # Shock session - experiment_type = "FP" - experimental_group = "RR20" - subject_id = "95.259" - start_datetime = datetime(2019, 4, 18, 10, 41, 42) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) - - # session with lots of trailing whitespace - experiment_type = "FP" - experimental_group = "PR" - subject_id = "141.308" - start_datetime = datetime(2019, 8, 1, 14, 1, 17) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) - - # session with missing medpc file - experiment_type = "FP" - experimental_group = "PS" - subject_id = "75.214" - start_datetime = datetime(2018, 10, 29, 12, 41, 44) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - "Subject": subject_id, - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / "MEDPC_RawFilesbyDate" - / f"{start_datetime.date().isoformat()}" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) - - # session with missing medpc file and missing subject info, but has csv file - experiment_type = "FP" - experimental_group = "PS" - subject_id = "75.214" - start_datetime = datetime(2018, 11, 9, 11, 46, 33) - box = "1" - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - "Box": box, - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / "MEDPC_RawFilesbyDate" - / f"{start_datetime.date().isoformat()}" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) - - # Behavior session from csv file - experiment_type = "FP" - experimental_group = "DPR" - subject_id = "87.239" - start_datetime = datetime(2019, 3, 19, 0, 0, 0) - session_conditions = {} - start_variable = "" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) - - # Fiber Photometry session - experiment_type = "FP" - experimental_group = "PR" - subject_id = "028.392" - start_datetime = datetime(2020, 7, 9, 13, 1, 26) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}" - ) - fiber_photometry_folder_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Photometry" - / f"Punishment Resistant" - / f"Early RI60" - / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200709-130922" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - fiber_photometry_folder_path=fiber_photometry_folder_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) +def western_blot_to_nwb( + *, + file_path: Union[str, Path], + output_dir_path: Union[str, Path], + verbose: bool = True, +): + """Convert a western blot to NWB. - # Fiber Photometry session without unrewarded port entries - experiment_type = "FP" - experimental_group = "PS" - subject_id = "249.391" - start_datetime = datetime(2020, 7, 21, 11, 42, 49) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}" - ) - fiber_photometry_folder_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Photometry" - / f"Punishment Sensitive" - / f"Late RI60" - / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200721-120136" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - fiber_photometry_folder_path=fiber_photometry_folder_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) + Parameters + ---------- + file_path : Union[str, Path] + Path to the western blot .tif file. + output_dir_path : Union[str, Path] + Path to the directory to save the NWB file. + verbose : bool, optional + Whether to print verbose output, by default True + """ + file_path = Path(file_path) + output_dir_path = Path(output_dir_path) + output_dir_path.mkdir(parents=True, exist_ok=True) - # Fiber Photometry session with only Fi1r (no Fi1d) - experiment_type = "FP" - experimental_group = "DPR" - subject_id = "333.393" - start_datetime = datetime(2020, 7, 13, 11, 57, 48) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}" - ) - fiber_photometry_folder_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Photometry" - / f"Delayed Punishment Resistant" - / f"Early" - / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200713-121027" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - fiber_photometry_folder_path=fiber_photometry_folder_path, - has_demodulated_commanded_voltages=False, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) + source_data = dict(WesternBlot={"file_path": str(file_path), "verbose": verbose}) + conversion_options = dict(WesternBlot={}) - # Fiber Photometry session with swapped left and right TTLs and missing Fi1d - experiment_type = "FP" - experimental_group = "PS" - subject_id = "140.306" - start_datetime = datetime(2019, 8, 9, 12, 10, 58) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" - / f"{subject_id}" - / f"{subject_id}" - ) - fiber_photometry_folder_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Photometry" - / f"Punishment Sensitive" - / f"Early RI60" - / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-190809-121107" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - fiber_photometry_folder_path=fiber_photometry_folder_path, - flip_ttls_lr=True, - has_demodulated_commanded_voltages=False, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) + converter = Seiler2024WesternBlotNWBConverter(source_data=source_data, verbose=verbose) + metadata = converter.get_metadata() - # Fiber Photometry session: Probe Test Habit Training TTL - experiment_type = "FP" - experimental_group = "PR" - subject_id = "89.247" - start_datetime = datetime(2019, 3, 8, 10, 59, 10) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Behavior" - / "MEDPC_RawFilesbyDate" - / f"{start_datetime.date().isoformat()} 89.247probe" - ) - fiber_photometry_folder_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / "Photometry" - / f"Punishment Resistant" - / f"Early RI60" - / f"Photo_89_247-190308-095258" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - fiber_photometry_folder_path=fiber_photometry_folder_path, - has_demodulated_commanded_voltages=False, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - stub_test=stub_test, - ) + # Update default metadata with the editable in the corresponding yaml file + editable_metadata_path = Path(__file__).parent / "seiler_2024_metadata.yaml" + editable_metadata = load_dict_from_file(editable_metadata_path) + metadata = dict_deep_update(metadata, editable_metadata) - # Example DMS-Inhibitory Opto session - experiment_type = "Opto" - experimental_group = "DMS-Inhibitory" - optogenetic_treatment = "NpHR" - subject_id = "112.415" - start_datetime = datetime(2020, 10, 21, 13, 8, 39) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / f"{experimental_group.replace('-', ' ')}" - / f"Group 1" - / f"Halo" - / f"{subject_id}" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - optogenetic_treatment=optogenetic_treatment, - stub_test=stub_test, - ) + cst = timezone("US/Central") + metadata["NWBFile"]["session_start_time"] = metadata["NWBFile"]["session_start_time"].replace(tzinfo=cst) - # Example DMS-Excitatory Opto session - experiment_type = "Opto" - experimental_group = "DMS-Excitatory" - optogenetic_treatment = "ChR2" - subject_id = "119.416" - start_datetime = datetime(2020, 10, 20, 13, 0, 57) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / f"{experimental_group.replace('-', ' ')}" - / f"{optogenetic_treatment}" - / f"{subject_id}" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - optogenetic_treatment=optogenetic_treatment, - stub_test=stub_test, - ) + nwbfile_path = output_dir_path / f"{file_path.stem}.nwb" - # Example DLS-Excitatory Opto session - experiment_type = "Opto" - experimental_group = "DLS-Excitatory" - optogenetic_treatment = "ChR2" - subject_id = "079.402" - start_datetime = datetime(2020, 6, 26, 13, 19, 27) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / f"{experimental_group.replace('-', ' ')}" - / f"{optogenetic_treatment}" - / f"{subject_id}" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - optogenetic_treatment=optogenetic_treatment, - stub_test=stub_test, - ) + # Run conversion + converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) - # Opto session with both left and right rewards - experiment_type = "Opto" - experimental_group = "DMS-Excitatory" - optogenetic_treatment = "ChR2" - subject_id = "281.402" - start_datetime = datetime(2020, 9, 23, 12, 36, 30) - session_conditions = { - "Start Date": start_datetime.strftime("%m/%d/%y"), - "Start Time": start_datetime.strftime("%H:%M:%S"), - } - start_variable = "Start Date" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / f"{experimental_group.replace('-', ' ')}" - / f"{optogenetic_treatment}" - / "2020-09-23_12h36m_Subject 281.402" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - optogenetic_treatment=optogenetic_treatment, - stub_test=stub_test, - ) - # Opto session from csv file - experiment_type = "Opto" - experimental_group = "DLS-Excitatory" - optogenetic_treatment = "ChR2" - subject_id = "290.407" - start_datetime = datetime(2020, 9, 23, 0, 0, 0) - session_conditions = {} - start_variable = "" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / f"{experimental_group.replace('-', ' ')}" - / f"{optogenetic_treatment}" - / f"{subject_id}" - / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - optogenetic_treatment=optogenetic_treatment, - stub_test=stub_test, - ) +if __name__ == "__main__": + # Parameters for conversion + data_dir_path = Path("/Volumes/T7/CatalystNeuro/NWB/Lerner/raw_data") + output_dir_path = Path("/Volumes/T7/CatalystNeuro/NWB/Lerner/conversion_nwb") + stub_test = False - # Opto session from csv file with scrambled optogenetic stimulation - experiment_type = "Opto" - experimental_group = "DLS-Excitatory" - optogenetic_treatment = "ChR2Scrambled" - subject_id = "276.405" - start_datetime = datetime(2020, 10, 1, 0, 0, 0) - session_conditions = {} - start_variable = "" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / f"{experimental_group.replace('-', ' ')}" - / "Scrambled" - / f"{subject_id.replace('.', '_')}" - / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - optogenetic_treatment=optogenetic_treatment, - stub_test=stub_test, - ) + if output_dir_path.exists(): + shutil.rmtree( + output_dir_path, ignore_errors=True + ) # ignore errors due to MacOS race condition (https://github.com/python/cpython/issues/81441) - # Opto session with mixed dtype - experiment_type = "Opto" - experimental_group = "DLS-Excitatory" - optogenetic_treatment = "ChR2" - subject_id = "299.405" - start_datetime = datetime(2020, 9, 11, 0, 0, 0) - session_conditions = {} - start_variable = "" - behavior_file_path = ( - data_dir_path - / f"{experiment_type} Experiments" - / f"{experimental_group.replace('-', ' ')}" - / f"{optogenetic_treatment}" - / f"{subject_id}" - / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - ) - session_to_nwb( - data_dir_path=data_dir_path, - output_dir_path=output_dir_path, - behavior_file_path=behavior_file_path, - subject_id=subject_id, - session_conditions=session_conditions, - start_variable=start_variable, - start_datetime=start_datetime, - experiment_type=experiment_type, - experimental_group=experimental_group, - optogenetic_treatment=optogenetic_treatment, - stub_test=stub_test, - ) + # # No-shock example session + # experiment_type = "FP" + # experimental_group = "RR20" + # subject_id = "95.259" + # start_datetime = datetime(2019, 4, 9, 10, 34, 30) + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}" + # ) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Shock session + # experiment_type = "FP" + # experimental_group = "RR20" + # subject_id = "95.259" + # start_datetime = datetime(2019, 4, 18, 10, 41, 42) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # session with lots of trailing whitespace + # experiment_type = "FP" + # experimental_group = "PR" + # subject_id = "141.308" + # start_datetime = datetime(2019, 8, 1, 14, 1, 17) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # session with missing medpc file + # experiment_type = "FP" + # experimental_group = "PS" + # subject_id = "75.214" + # start_datetime = datetime(2018, 10, 29, 12, 41, 44) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # "Subject": subject_id, + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / "MEDPC_RawFilesbyDate" + # / f"{start_datetime.date().isoformat()}" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # session with missing medpc file and missing subject info, but has csv file + # experiment_type = "FP" + # experimental_group = "PS" + # subject_id = "75.214" + # start_datetime = datetime(2018, 11, 9, 11, 46, 33) + # box = "1" + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # "Box": box, + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / "MEDPC_RawFilesbyDate" + # / f"{start_datetime.date().isoformat()}" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Behavior session from csv file + # experiment_type = "FP" + # experimental_group = "DPR" + # subject_id = "87.239" + # start_datetime = datetime(2019, 3, 19, 0, 0, 0) + # session_conditions = {} + # start_variable = "" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Fiber Photometry session + # experiment_type = "FP" + # experimental_group = "PR" + # subject_id = "028.392" + # start_datetime = datetime(2020, 7, 9, 13, 1, 26) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}" + # ) + # fiber_photometry_folder_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Photometry" + # / f"Punishment Resistant" + # / f"Early RI60" + # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200709-130922" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # fiber_photometry_folder_path=fiber_photometry_folder_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Fiber Photometry session without unrewarded port entries + # experiment_type = "FP" + # experimental_group = "PS" + # subject_id = "249.391" + # start_datetime = datetime(2020, 7, 21, 11, 42, 49) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}" + # ) + # fiber_photometry_folder_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Photometry" + # / f"Punishment Sensitive" + # / f"Late RI60" + # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200721-120136" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # fiber_photometry_folder_path=fiber_photometry_folder_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Fiber Photometry session with only Fi1r (no Fi1d) + # experiment_type = "FP" + # experimental_group = "DPR" + # subject_id = "333.393" + # start_datetime = datetime(2020, 7, 13, 11, 57, 48) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}" + # ) + # fiber_photometry_folder_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Photometry" + # / f"Delayed Punishment Resistant" + # / f"Early" + # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200713-121027" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # fiber_photometry_folder_path=fiber_photometry_folder_path, + # has_demodulated_commanded_voltages=False, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Fiber Photometry session with swapped left and right TTLs and missing Fi1d + # experiment_type = "FP" + # experimental_group = "PS" + # subject_id = "140.306" + # start_datetime = datetime(2019, 8, 9, 12, 10, 58) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}" + # ) + # fiber_photometry_folder_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Photometry" + # / f"Punishment Sensitive" + # / f"Early RI60" + # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-190809-121107" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # fiber_photometry_folder_path=fiber_photometry_folder_path, + # flip_ttls_lr=True, + # has_demodulated_commanded_voltages=False, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Fiber Photometry session: Probe Test Habit Training TTL + # experiment_type = "FP" + # experimental_group = "PR" + # subject_id = "89.247" + # start_datetime = datetime(2019, 3, 8, 10, 59, 10) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / "MEDPC_RawFilesbyDate" + # / f"{start_datetime.date().isoformat()} 89.247probe" + # ) + # fiber_photometry_folder_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Photometry" + # / f"Punishment Resistant" + # / f"Early RI60" + # / f"Photo_89_247-190308-095258" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # fiber_photometry_folder_path=fiber_photometry_folder_path, + # has_demodulated_commanded_voltages=False, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # # Example DMS-Inhibitory Opto session + # experiment_type = "Opto" + # experimental_group = "DMS-Inhibitory" + # optogenetic_treatment = "NpHR" + # subject_id = "112.415" + # start_datetime = datetime(2020, 10, 21, 13, 8, 39) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / f"{experimental_group.replace('-', ' ')}" + # / f"Group 1" + # / f"Halo" + # / f"{subject_id}" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # optogenetic_treatment=optogenetic_treatment, + # stub_test=stub_test, + # ) + + # # Example DMS-Excitatory Opto session + # experiment_type = "Opto" + # experimental_group = "DMS-Excitatory" + # optogenetic_treatment = "ChR2" + # subject_id = "119.416" + # start_datetime = datetime(2020, 10, 20, 13, 0, 57) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / f"{experimental_group.replace('-', ' ')}" + # / f"{optogenetic_treatment}" + # / f"{subject_id}" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # optogenetic_treatment=optogenetic_treatment, + # stub_test=stub_test, + # ) + + # # Example DLS-Excitatory Opto session + # experiment_type = "Opto" + # experimental_group = "DLS-Excitatory" + # optogenetic_treatment = "ChR2" + # subject_id = "079.402" + # start_datetime = datetime(2020, 6, 26, 13, 19, 27) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / f"{experimental_group.replace('-', ' ')}" + # / f"{optogenetic_treatment}" + # / f"{subject_id}" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # optogenetic_treatment=optogenetic_treatment, + # stub_test=stub_test, + # ) + + # # Opto session with both left and right rewards + # experiment_type = "Opto" + # experimental_group = "DMS-Excitatory" + # optogenetic_treatment = "ChR2" + # subject_id = "281.402" + # start_datetime = datetime(2020, 9, 23, 12, 36, 30) + # session_conditions = { + # "Start Date": start_datetime.strftime("%m/%d/%y"), + # "Start Time": start_datetime.strftime("%H:%M:%S"), + # } + # start_variable = "Start Date" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / f"{experimental_group.replace('-', ' ')}" + # / f"{optogenetic_treatment}" + # / "2020-09-23_12h36m_Subject 281.402" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # optogenetic_treatment=optogenetic_treatment, + # stub_test=stub_test, + # ) + + # # Opto session from csv file + # experiment_type = "Opto" + # experimental_group = "DLS-Excitatory" + # optogenetic_treatment = "ChR2" + # subject_id = "290.407" + # start_datetime = datetime(2020, 9, 23, 0, 0, 0) + # session_conditions = {} + # start_variable = "" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / f"{experimental_group.replace('-', ' ')}" + # / f"{optogenetic_treatment}" + # / f"{subject_id}" + # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # optogenetic_treatment=optogenetic_treatment, + # stub_test=stub_test, + # ) + + # # Opto session from csv file with scrambled optogenetic stimulation + # experiment_type = "Opto" + # experimental_group = "DLS-Excitatory" + # optogenetic_treatment = "ChR2Scrambled" + # subject_id = "276.405" + # start_datetime = datetime(2020, 10, 1, 0, 0, 0) + # session_conditions = {} + # start_variable = "" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / f"{experimental_group.replace('-', ' ')}" + # / "Scrambled" + # / f"{subject_id.replace('.', '_')}" + # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # optogenetic_treatment=optogenetic_treatment, + # stub_test=stub_test, + # ) + + # # Opto session with mixed dtype + # experiment_type = "Opto" + # experimental_group = "DLS-Excitatory" + # optogenetic_treatment = "ChR2" + # subject_id = "299.405" + # start_datetime = datetime(2020, 9, 11, 0, 0, 0) + # session_conditions = {} + # start_variable = "" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / f"{experimental_group.replace('-', ' ')}" + # / f"{optogenetic_treatment}" + # / f"{subject_id}" + # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # optogenetic_treatment=optogenetic_treatment, + # stub_test=stub_test, + # ) + + # Western blot + western_path = data_dir_path / "DATCre Western blot final images and analysis" + file_path = western_path / "Female_DLS_Actin.tif" + western_blot_to_nwb(file_path=file_path, output_dir_path=output_dir_path) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py index 566016f..43dac93 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py @@ -9,6 +9,7 @@ Seiler2024FiberPhotometryInterface, Seiler2024OptogeneticInterface, Seiler2024ExcelMetadataInterface, + Seiler2024WesternBlotInterface, ) from .medpc import read_medpc_file import numpy as np @@ -153,3 +154,11 @@ def run_conversion( verbose=self.verbose, ) as nwbfile_out: self.add_to_nwbfile(nwbfile_out, metadata, conversion_options) + + +class Seiler2024WesternBlotNWBConverter(NWBConverter): + """Western Blot conversion class.""" + + data_interface_classes = dict( + WesternBlot=Seiler2024WesternBlotInterface, + ) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py new file mode 100644 index 0000000..2030265 --- /dev/null +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py @@ -0,0 +1,49 @@ +"""Primary class for converting experiment-specific western blot data.""" +from pynwb.file import NWBFile +from neuroconv.basedatainterface import BaseDataInterface +from neuroconv.utils import DeepDict +from datetime import datetime +from pathlib import Path + + +class Seiler2024WesternBlotInterface(BaseDataInterface): + """Western Blot interface for seiler_2024 conversion""" + + keywords = ["western blot"] + + def __init__(self, file_path: str, verbose: bool = True): + """Initialize Seiler2024BehaviorInterface. + + Parameters + ---------- + file_path : str + Path to the .tif file. + verbose : bool, optional + Whether to print verbose output, by default True + """ + super().__init__( + file_path=file_path, + verbose=verbose, + ) + + def get_metadata(self) -> DeepDict: + metadata = super().get_metadata() + + # Western Blot does not have a start datetime --> using publication date of the paper: 2022-02-07 + metadata["NWBFile"]["session_start_time"] = datetime(2022, 2, 7, 0, 0, 0) + file_path = Path(self.source_data["file_path"]) + metadata["Subject"]["subject_id"] = file_path.stem + if "Female" in file_path.stem: + metadata["Subject"]["sex"] = "F" + elif "Male" in file_path.stem: + metadata["Subject"]["sex"] = "M" + else: + metadata["Subject"]["sex"] = "U" + return metadata + + def get_metadata_schema(self) -> dict: + metadata_schema = super().get_metadata_schema() + return metadata_schema + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict) -> None: + pass From 543bbfd1ea10b76782f132da4d3190afc9f67a77 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 22 May 2024 16:52:42 -0700 Subject: [PATCH 08/17] split western blots by subject (wt and dat-ires-cre) --- .../seiler_2024_convert_session.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py index 8ee2baa..46e5c05 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py @@ -5,6 +5,8 @@ from neuroconv.utils import load_dict_from_file, dict_deep_update from datetime import datetime from pytz import timezone +from tifffile import imread, imwrite +import matplotlib.pyplot as plt from lerner_lab_to_nwb.seiler_2024 import Seiler2024NWBConverter, Seiler2024WesternBlotNWBConverter @@ -209,6 +211,33 @@ def western_blot_to_nwb( converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) +def split_western_blot(*, file_path: Union[str, Path]): + """Split tif file into WT and DAT-IRES-Cre-het then writes back to two separate files. + + Parameters + ---------- + file_path : Union[str, Path] + Path to the western blot .tif file. + + Returns + ------- + wt_file_path : Path + Path to the WT western blot .tif file. + dat_file_path : Path + Path to the DAT-IRES-Cre-het western blot .tif file. + """ + file_path = Path(file_path) + western_blot = imread(file_path) + wt_western_blot = western_blot[:, :200] + dat_western_blot = western_blot[:, 200:] + wt_file_path = file_path.parent / f"{file_path.stem}_WT.tif" + dat_file_path = file_path.parent / f"{file_path.stem}_DAT-IRES-Cre-het.tif" + imwrite(wt_file_path, wt_western_blot) + imwrite(dat_file_path, dat_western_blot) + + return wt_file_path, dat_file_path + + if __name__ == "__main__": # Parameters for conversion data_dir_path = Path("/Volumes/T7/CatalystNeuro/NWB/Lerner/raw_data") @@ -829,4 +858,6 @@ def western_blot_to_nwb( # Western blot western_path = data_dir_path / "DATCre Western blot final images and analysis" file_path = western_path / "Female_DLS_Actin.tif" - western_blot_to_nwb(file_path=file_path, output_dir_path=output_dir_path) + wt_file_path, dat_file_path = split_western_blot(file_path=file_path) + western_blot_to_nwb(file_path=wt_file_path, output_dir_path=output_dir_path) + western_blot_to_nwb(file_path=dat_file_path, output_dir_path=output_dir_path) From 1f39b1528a9f06c3d2fb287deccefaec49ce9841 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 23 May 2024 08:59:45 -0700 Subject: [PATCH 09/17] added western blot to nwbfile --- .../seiler_2024westernblotinterface.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py index 2030265..7af6b83 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024westernblotinterface.py @@ -4,6 +4,8 @@ from neuroconv.utils import DeepDict from datetime import datetime from pathlib import Path +from tifffile import imread +from pynwb.image import GrayscaleImage, Images class Seiler2024WesternBlotInterface(BaseDataInterface): @@ -46,4 +48,16 @@ def get_metadata_schema(self) -> dict: return metadata_schema def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict) -> None: - pass + western_blot_image = imread(self.source_data["file_path"]) + + western_blot_image = GrayscaleImage( + name="western_blot_image", + data=western_blot_image, + description="Western blot images (quantified in a separate dataset) showing DAT and β-actin bands for WT and DATCre-het mice in DMS and DLS.", + ) + images = Images( + name="images", + images=[western_blot_image], + description="Western blot images (quantified in a separate dataset) showing DAT and β-actin bands for WT and DATCre-het mice in DMS and DLS.", + ) + nwbfile.add_acquisition(images) From fb160d2bc98042946a129bd5b31ebd27c96f2ea9 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 23 May 2024 11:33:34 -0700 Subject: [PATCH 10/17] added western blot to full dataset conversion --- .../seiler_2024_convert_dataset.py | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py index 2a7b3af..b9eb09f 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py @@ -11,7 +11,11 @@ import traceback import re -from lerner_lab_to_nwb.seiler_2024.seiler_2024_convert_session import session_to_nwb +from lerner_lab_to_nwb.seiler_2024.seiler_2024_convert_session import ( + session_to_nwb, + western_blot_to_nwb, + split_western_blot, +) from lerner_lab_to_nwb.seiler_2024.medpc import get_medpc_variables from lerner_lab_to_nwb.seiler_2024.medpc import read_medpc_file @@ -911,6 +915,36 @@ def get_raw_info(behavior_path): return raw_file_to_info +def western_dataset_to_nwb(*, data_dir_path: Path, output_dir_path: Path, verbose: bool = True): + """Convert all Western Blot data to NWB. + + Parameters + ---------- + data_dir_path : Path + The path to the directory containing the raw data. + output_dir_path : Path + The path to the directory where the NWB files will be saved. + verbose : bool, optional + Whether to print verbose output, by default True + """ + western_path = data_dir_path / "DATCre Western blot final images and analysis" + raw_western_file_names = [ + "Female_DLS_Actin.tif", + "Female_DLS_DAT.tif", + "Female_DMS_Actin.tif", + "Female_DMS_DAT.tif", + "Male_DLS_Actin.tif", + "Male_DLS_DAT.tif", + "Male_DMS_Actin.tif", + "Male_DMS_DAT.tif", + ] + for raw_western_file_name in raw_western_file_names: + raw_western_file_path = western_path / raw_western_file_name + wt_file_path, dat_file_path = split_western_blot(file_path=raw_western_file_path) + western_blot_to_nwb(file_path=wt_file_path, output_dir_path=output_dir_path, verbose=verbose) + western_blot_to_nwb(file_path=dat_file_path, output_dir_path=output_dir_path, verbose=verbose) + + if __name__ == "__main__": data_dir_path = Path("/Volumes/T7/CatalystNeuro/NWB/Lerner/raw_data") output_dir_path = Path("/Volumes/T7/CatalystNeuro/NWB/Lerner/conversion_nwb") @@ -926,3 +960,4 @@ def get_raw_info(behavior_path): stub_test=False, verbose=False, ) + western_dataset_to_nwb(data_dir_path=data_dir_path, output_dir_path=output_dir_path, verbose=False) From 4f88753f26fe2090d3bb51b18bbb751851031909 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 23 May 2024 11:35:42 -0700 Subject: [PATCH 11/17] uncomment convert session examples --- .../seiler_2024_convert_session.py | 1210 ++++++++--------- 1 file changed, 605 insertions(+), 605 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py index 46e5c05..7bf7b49 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py @@ -249,611 +249,611 @@ def split_western_blot(*, file_path: Union[str, Path]): output_dir_path, ignore_errors=True ) # ignore errors due to MacOS race condition (https://github.com/python/cpython/issues/81441) - # # No-shock example session - # experiment_type = "FP" - # experimental_group = "RR20" - # subject_id = "95.259" - # start_datetime = datetime(2019, 4, 9, 10, 34, 30) - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}" - # ) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Shock session - # experiment_type = "FP" - # experimental_group = "RR20" - # subject_id = "95.259" - # start_datetime = datetime(2019, 4, 18, 10, 41, 42) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # session with lots of trailing whitespace - # experiment_type = "FP" - # experimental_group = "PR" - # subject_id = "141.308" - # start_datetime = datetime(2019, 8, 1, 14, 1, 17) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # session with missing medpc file - # experiment_type = "FP" - # experimental_group = "PS" - # subject_id = "75.214" - # start_datetime = datetime(2018, 10, 29, 12, 41, 44) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # "Subject": subject_id, - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / "MEDPC_RawFilesbyDate" - # / f"{start_datetime.date().isoformat()}" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # session with missing medpc file and missing subject info, but has csv file - # experiment_type = "FP" - # experimental_group = "PS" - # subject_id = "75.214" - # start_datetime = datetime(2018, 11, 9, 11, 46, 33) - # box = "1" - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # "Box": box, - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / "MEDPC_RawFilesbyDate" - # / f"{start_datetime.date().isoformat()}" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Behavior session from csv file - # experiment_type = "FP" - # experimental_group = "DPR" - # subject_id = "87.239" - # start_datetime = datetime(2019, 3, 19, 0, 0, 0) - # session_conditions = {} - # start_variable = "" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Fiber Photometry session - # experiment_type = "FP" - # experimental_group = "PR" - # subject_id = "028.392" - # start_datetime = datetime(2020, 7, 9, 13, 1, 26) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}" - # ) - # fiber_photometry_folder_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Photometry" - # / f"Punishment Resistant" - # / f"Early RI60" - # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200709-130922" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # fiber_photometry_folder_path=fiber_photometry_folder_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Fiber Photometry session without unrewarded port entries - # experiment_type = "FP" - # experimental_group = "PS" - # subject_id = "249.391" - # start_datetime = datetime(2020, 7, 21, 11, 42, 49) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}" - # ) - # fiber_photometry_folder_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Photometry" - # / f"Punishment Sensitive" - # / f"Late RI60" - # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200721-120136" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # fiber_photometry_folder_path=fiber_photometry_folder_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Fiber Photometry session with only Fi1r (no Fi1d) - # experiment_type = "FP" - # experimental_group = "DPR" - # subject_id = "333.393" - # start_datetime = datetime(2020, 7, 13, 11, 57, 48) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}" - # ) - # fiber_photometry_folder_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Photometry" - # / f"Delayed Punishment Resistant" - # / f"Early" - # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200713-121027" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # fiber_photometry_folder_path=fiber_photometry_folder_path, - # has_demodulated_commanded_voltages=False, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Fiber Photometry session with swapped left and right TTLs and missing Fi1d - # experiment_type = "FP" - # experimental_group = "PS" - # subject_id = "140.306" - # start_datetime = datetime(2019, 8, 9, 12, 10, 58) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / f"{experimental_group}" - # / f"{subject_id}" - # / f"{subject_id}" - # ) - # fiber_photometry_folder_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Photometry" - # / f"Punishment Sensitive" - # / f"Early RI60" - # / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-190809-121107" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # fiber_photometry_folder_path=fiber_photometry_folder_path, - # flip_ttls_lr=True, - # has_demodulated_commanded_voltages=False, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Fiber Photometry session: Probe Test Habit Training TTL - # experiment_type = "FP" - # experimental_group = "PR" - # subject_id = "89.247" - # start_datetime = datetime(2019, 3, 8, 10, 59, 10) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Behavior" - # / "MEDPC_RawFilesbyDate" - # / f"{start_datetime.date().isoformat()} 89.247probe" - # ) - # fiber_photometry_folder_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / "Photometry" - # / f"Punishment Resistant" - # / f"Early RI60" - # / f"Photo_89_247-190308-095258" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # fiber_photometry_folder_path=fiber_photometry_folder_path, - # has_demodulated_commanded_voltages=False, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # stub_test=stub_test, - # ) - - # # Example DMS-Inhibitory Opto session - # experiment_type = "Opto" - # experimental_group = "DMS-Inhibitory" - # optogenetic_treatment = "NpHR" - # subject_id = "112.415" - # start_datetime = datetime(2020, 10, 21, 13, 8, 39) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / f"{experimental_group.replace('-', ' ')}" - # / f"Group 1" - # / f"Halo" - # / f"{subject_id}" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # optogenetic_treatment=optogenetic_treatment, - # stub_test=stub_test, - # ) - - # # Example DMS-Excitatory Opto session - # experiment_type = "Opto" - # experimental_group = "DMS-Excitatory" - # optogenetic_treatment = "ChR2" - # subject_id = "119.416" - # start_datetime = datetime(2020, 10, 20, 13, 0, 57) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / f"{experimental_group.replace('-', ' ')}" - # / f"{optogenetic_treatment}" - # / f"{subject_id}" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # optogenetic_treatment=optogenetic_treatment, - # stub_test=stub_test, - # ) - - # # Example DLS-Excitatory Opto session - # experiment_type = "Opto" - # experimental_group = "DLS-Excitatory" - # optogenetic_treatment = "ChR2" - # subject_id = "079.402" - # start_datetime = datetime(2020, 6, 26, 13, 19, 27) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / f"{experimental_group.replace('-', ' ')}" - # / f"{optogenetic_treatment}" - # / f"{subject_id}" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # optogenetic_treatment=optogenetic_treatment, - # stub_test=stub_test, - # ) - - # # Opto session with both left and right rewards - # experiment_type = "Opto" - # experimental_group = "DMS-Excitatory" - # optogenetic_treatment = "ChR2" - # subject_id = "281.402" - # start_datetime = datetime(2020, 9, 23, 12, 36, 30) - # session_conditions = { - # "Start Date": start_datetime.strftime("%m/%d/%y"), - # "Start Time": start_datetime.strftime("%H:%M:%S"), - # } - # start_variable = "Start Date" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / f"{experimental_group.replace('-', ' ')}" - # / f"{optogenetic_treatment}" - # / "2020-09-23_12h36m_Subject 281.402" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # optogenetic_treatment=optogenetic_treatment, - # stub_test=stub_test, - # ) - - # # Opto session from csv file - # experiment_type = "Opto" - # experimental_group = "DLS-Excitatory" - # optogenetic_treatment = "ChR2" - # subject_id = "290.407" - # start_datetime = datetime(2020, 9, 23, 0, 0, 0) - # session_conditions = {} - # start_variable = "" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / f"{experimental_group.replace('-', ' ')}" - # / f"{optogenetic_treatment}" - # / f"{subject_id}" - # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # optogenetic_treatment=optogenetic_treatment, - # stub_test=stub_test, - # ) - - # # Opto session from csv file with scrambled optogenetic stimulation - # experiment_type = "Opto" - # experimental_group = "DLS-Excitatory" - # optogenetic_treatment = "ChR2Scrambled" - # subject_id = "276.405" - # start_datetime = datetime(2020, 10, 1, 0, 0, 0) - # session_conditions = {} - # start_variable = "" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / f"{experimental_group.replace('-', ' ')}" - # / "Scrambled" - # / f"{subject_id.replace('.', '_')}" - # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # optogenetic_treatment=optogenetic_treatment, - # stub_test=stub_test, - # ) - - # # Opto session with mixed dtype - # experiment_type = "Opto" - # experimental_group = "DLS-Excitatory" - # optogenetic_treatment = "ChR2" - # subject_id = "299.405" - # start_datetime = datetime(2020, 9, 11, 0, 0, 0) - # session_conditions = {} - # start_variable = "" - # behavior_file_path = ( - # data_dir_path - # / f"{experiment_type} Experiments" - # / f"{experimental_group.replace('-', ' ')}" - # / f"{optogenetic_treatment}" - # / f"{subject_id}" - # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" - # ) - # session_to_nwb( - # data_dir_path=data_dir_path, - # output_dir_path=output_dir_path, - # behavior_file_path=behavior_file_path, - # subject_id=subject_id, - # session_conditions=session_conditions, - # start_variable=start_variable, - # start_datetime=start_datetime, - # experiment_type=experiment_type, - # experimental_group=experimental_group, - # optogenetic_treatment=optogenetic_treatment, - # stub_test=stub_test, - # ) + # No-shock example session + experiment_type = "FP" + experimental_group = "RR20" + subject_id = "95.259" + start_datetime = datetime(2019, 4, 9, 10, 34, 30) + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}" + ) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Shock session + experiment_type = "FP" + experimental_group = "RR20" + subject_id = "95.259" + start_datetime = datetime(2019, 4, 18, 10, 41, 42) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # session with lots of trailing whitespace + experiment_type = "FP" + experimental_group = "PR" + subject_id = "141.308" + start_datetime = datetime(2019, 8, 1, 14, 1, 17) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # session with missing medpc file + experiment_type = "FP" + experimental_group = "PS" + subject_id = "75.214" + start_datetime = datetime(2018, 10, 29, 12, 41, 44) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + "Subject": subject_id, + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / "MEDPC_RawFilesbyDate" + / f"{start_datetime.date().isoformat()}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # session with missing medpc file and missing subject info, but has csv file + experiment_type = "FP" + experimental_group = "PS" + subject_id = "75.214" + start_datetime = datetime(2018, 11, 9, 11, 46, 33) + box = "1" + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + "Box": box, + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / "MEDPC_RawFilesbyDate" + / f"{start_datetime.date().isoformat()}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Behavior session from csv file + experiment_type = "FP" + experimental_group = "DPR" + subject_id = "87.239" + start_datetime = datetime(2019, 3, 19, 0, 0, 0) + session_conditions = {} + start_variable = "" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Fiber Photometry session + experiment_type = "FP" + experimental_group = "PR" + subject_id = "028.392" + start_datetime = datetime(2020, 7, 9, 13, 1, 26) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}" + ) + fiber_photometry_folder_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Photometry" + / f"Punishment Resistant" + / f"Early RI60" + / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200709-130922" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + fiber_photometry_folder_path=fiber_photometry_folder_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Fiber Photometry session without unrewarded port entries + experiment_type = "FP" + experimental_group = "PS" + subject_id = "249.391" + start_datetime = datetime(2020, 7, 21, 11, 42, 49) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}" + ) + fiber_photometry_folder_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Photometry" + / f"Punishment Sensitive" + / f"Late RI60" + / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200721-120136" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + fiber_photometry_folder_path=fiber_photometry_folder_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Fiber Photometry session with only Fi1r (no Fi1d) + experiment_type = "FP" + experimental_group = "DPR" + subject_id = "333.393" + start_datetime = datetime(2020, 7, 13, 11, 57, 48) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}" + ) + fiber_photometry_folder_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Photometry" + / f"Delayed Punishment Resistant" + / f"Early" + / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200713-121027" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + fiber_photometry_folder_path=fiber_photometry_folder_path, + has_demodulated_commanded_voltages=False, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Fiber Photometry session with swapped left and right TTLs and missing Fi1d + experiment_type = "FP" + experimental_group = "PS" + subject_id = "140.306" + start_datetime = datetime(2019, 8, 9, 12, 10, 58) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / f"{experimental_group}" + / f"{subject_id}" + / f"{subject_id}" + ) + fiber_photometry_folder_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Photometry" + / f"Punishment Sensitive" + / f"Early RI60" + / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-190809-121107" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + fiber_photometry_folder_path=fiber_photometry_folder_path, + flip_ttls_lr=True, + has_demodulated_commanded_voltages=False, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Fiber Photometry session: Probe Test Habit Training TTL + experiment_type = "FP" + experimental_group = "PR" + subject_id = "89.247" + start_datetime = datetime(2019, 3, 8, 10, 59, 10) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Behavior" + / "MEDPC_RawFilesbyDate" + / f"{start_datetime.date().isoformat()} 89.247probe" + ) + fiber_photometry_folder_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / "Photometry" + / f"Punishment Resistant" + / f"Early RI60" + / f"Photo_89_247-190308-095258" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + fiber_photometry_folder_path=fiber_photometry_folder_path, + has_demodulated_commanded_voltages=False, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + stub_test=stub_test, + ) + + # Example DMS-Inhibitory Opto session + experiment_type = "Opto" + experimental_group = "DMS-Inhibitory" + optogenetic_treatment = "NpHR" + subject_id = "112.415" + start_datetime = datetime(2020, 10, 21, 13, 8, 39) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"Group 1" + / f"Halo" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Example DMS-Excitatory Opto session + experiment_type = "Opto" + experimental_group = "DMS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "119.416" + start_datetime = datetime(2020, 10, 20, 13, 0, 57) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Example DLS-Excitatory Opto session + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "079.402" + start_datetime = datetime(2020, 6, 26, 13, 19, 27) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Opto session with both left and right rewards + experiment_type = "Opto" + experimental_group = "DMS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "281.402" + start_datetime = datetime(2020, 9, 23, 12, 36, 30) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / "2020-09-23_12h36m_Subject 281.402" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Opto session from csv file + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "290.407" + start_datetime = datetime(2020, 9, 23, 0, 0, 0) + session_conditions = {} + start_variable = "" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / f"{subject_id}" + / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Opto session from csv file with scrambled optogenetic stimulation + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2Scrambled" + subject_id = "276.405" + start_datetime = datetime(2020, 10, 1, 0, 0, 0) + session_conditions = {} + start_variable = "" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / "Scrambled" + / f"{subject_id.replace('.', '_')}" + / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Opto session with mixed dtype + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "299.405" + start_datetime = datetime(2020, 9, 11, 0, 0, 0) + session_conditions = {} + start_variable = "" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / f"{subject_id}" + / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) # Western blot western_path = data_dir_path / "DATCre Western blot final images and analysis" From 9bd28abc4ba07a60d23320091bf3a70e05ddc3d7 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 23 May 2024 12:27:20 -0700 Subject: [PATCH 12/17] added appropriate wt/dat slices for other westerns --- .../seiler_2024/seiler_2024_convert_session.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py index 7bf7b49..e748ff8 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py @@ -226,10 +226,21 @@ def split_western_blot(*, file_path: Union[str, Path]): dat_file_path : Path Path to the DAT-IRES-Cre-het western blot .tif file. """ + raw_western_file_names_to_slices = { + "Female_DLS_Actin.tif": (slice(None, 200), slice(200, None)), + "Female_DLS_DAT.tif": (slice(50, 235), slice(235, None)), + "Female_DMS_Actin.tif": (slice(None, 230), slice(230, None)), + "Female_DMS_DAT.tif": (slice(55, 245), slice(245, None)), + "Male_DLS_Actin.tif": (slice(None, 260), slice(260, None)), + "Male_DLS_DAT.tif": (slice(40, 290), slice(290, None)), + "Male_DMS_Actin.tif": (slice(None, 250), slice(250, None)), + "Male_DMS_DAT.tif": (slice(50, 300), slice(300, None)), + } file_path = Path(file_path) western_blot = imread(file_path) - wt_western_blot = western_blot[:, :200] - dat_western_blot = western_blot[:, 200:] + wt_slice, dat_slice = raw_western_file_names_to_slices[file_path.name] + wt_western_blot = western_blot[:, wt_slice] + dat_western_blot = western_blot[:, dat_slice] wt_file_path = file_path.parent / f"{file_path.stem}_WT.tif" dat_file_path = file_path.parent / f"{file_path.stem}_DAT-IRES-Cre-het.tif" imwrite(wt_file_path, wt_western_blot) From 103cb03f0ff509e77710ac24a84f0aabb33b34a3 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 23 May 2024 14:37:37 -0700 Subject: [PATCH 13/17] removed missing subject id handling from metadata interface --- .../seiler_2024excelmetadatainterface.py | 51 ++++++++----------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024excelmetadatainterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024excelmetadatainterface.py index b021517..6f40d08 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024excelmetadatainterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024excelmetadatainterface.py @@ -41,36 +41,29 @@ def get_metadata(self) -> DeepDict: df["Mouse ID"] = df["Mouse ID"].str.replace("(DNL)", "") df["Mouse ID"] = df["Mouse ID"].str.strip() df.set_index("Mouse ID", inplace=True) + subject_df = df.loc[self.source_data["subject_id"]] - if self.source_data["subject_id"] in df.index: - subject_df = df.loc[self.source_data["subject_id"]] - - # Add metadata to metadata dict - excel_sex_to_nwb_sex = {"Male": "M", "Female": "F"} - metadata["Subject"]["sex"] = excel_sex_to_nwb_sex[subject_df["Sex"]] - metadata["NWBFile"]["surgery"] = subject_df["Surgical Manipulation"] - if not pd.isna(subject_df["Treatment"]): - metadata["NWBFile"]["stimulus_notes"] = subject_df["Treatment"] - if subject_df["Experiment"] == "Fiber Photometry": - metadata["NWBFile"]["virus"] = "AAV5-CAG-FLEX-jGCaMP7b-WPRE" - elif subject_df["Experiment"] == "DLS-Excitatory" or subject_df["Experiment"] == "DMS-Excitatory": - metadata["NWBFile"]["virus"] = "AAV5-EF1a-DIO-hChR2(H134R)-EYFP" - elif subject_df["Experiment"] == "DMS-Inhibitory" or subject_df["Experiment"] == "DMS-Inhibitory Group 2": - metadata["NWBFile"]["virus"] = "AAV5-EF1a-DIO-eNpHR3.0-EYFP" - if subject_df["Treatment"] == "Control": - metadata["NWBFile"]["virus"] = "AAV5-EF1a-DIO-EYFP" - metadata["NWBFile"]["notes"] = ( - f'Hemisphere with DMS: {subject_df["Hemisphere with DMS"]}\n' - f'Experiment: {subject_df["Experiment"]}\n' - f'Behavior: {subject_df["Behavior"]}\n' - f'Punishment Group: {str(subject_df["Punishment Group"]).replace("Resitant", "Resistant")}\n' - f'Did Not Learn: {subject_df["DNL"]}\n' - ) - else: # TODO: Ask Lerner lab about missing subjects - if self.verbose: - print(f"Subject ID {self.source_data['subject_id']} not found in metadata file.") - metadata["Subject"]["sex"] = "U" - + # Add metadata to metadata dict + excel_sex_to_nwb_sex = {"Male": "M", "Female": "F"} + metadata["Subject"]["sex"] = excel_sex_to_nwb_sex[subject_df["Sex"]] + metadata["NWBFile"]["surgery"] = subject_df["Surgical Manipulation"] + if not pd.isna(subject_df["Treatment"]): + metadata["NWBFile"]["stimulus_notes"] = subject_df["Treatment"] + if subject_df["Experiment"] == "Fiber Photometry": + metadata["NWBFile"]["virus"] = "AAV5-CAG-FLEX-jGCaMP7b-WPRE" + elif subject_df["Experiment"] == "DLS-Excitatory" or subject_df["Experiment"] == "DMS-Excitatory": + metadata["NWBFile"]["virus"] = "AAV5-EF1a-DIO-hChR2(H134R)-EYFP" + elif subject_df["Experiment"] == "DMS-Inhibitory" or subject_df["Experiment"] == "DMS-Inhibitory Group 2": + metadata["NWBFile"]["virus"] = "AAV5-EF1a-DIO-eNpHR3.0-EYFP" + if subject_df["Treatment"] == "Control": + metadata["NWBFile"]["virus"] = "AAV5-EF1a-DIO-EYFP" + metadata["NWBFile"]["notes"] = ( + f'Hemisphere with DMS: {subject_df["Hemisphere with DMS"]}\n' + f'Experiment: {subject_df["Experiment"]}\n' + f'Behavior: {subject_df["Behavior"]}\n' + f'Punishment Group: {str(subject_df["Punishment Group"]).replace("Resitant", "Resistant")}\n' + f'Did Not Learn: {subject_df["DNL"]}\n' + ) metadata["Subject"]["subject_id"] = self.source_data["subject_id"] return metadata From 97c8849ffba883bcecd2227ca0fcbf1e782a8699 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 23 May 2024 15:17:15 -0700 Subject: [PATCH 14/17] swapped example sessions with missing mouse_ids for equivalent ones with mouse_ids in the excel metadata file --- .../seiler_2024_convert_session.py | 21 ++++++++++--------- .../seiler_2024/seiler_2024_notes.md | 1 + 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py index 003bcc7..084c7de 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py @@ -182,7 +182,7 @@ def session_to_nwb( # No-shock example session experiment_type = "FP" experimental_group = "RR20" - subject_id = "95.259" + subject_id = "96.259" start_datetime = datetime(2019, 4, 9, 10, 34, 30) behavior_file_path = ( data_dir_path @@ -213,8 +213,8 @@ def session_to_nwb( # Shock session experiment_type = "FP" experimental_group = "RR20" - subject_id = "95.259" - start_datetime = datetime(2019, 4, 18, 10, 41, 42) + subject_id = "96.259" + start_datetime = datetime(2019, 4, 18, 9, 28, 20) session_conditions = { "Start Date": start_datetime.strftime("%m/%d/%y"), "Start Time": start_datetime.strftime("%H:%M:%S"), @@ -365,9 +365,9 @@ def session_to_nwb( # Fiber Photometry session experiment_type = "FP" - experimental_group = "PR" - subject_id = "028.392" - start_datetime = datetime(2020, 7, 9, 13, 1, 26) + experimental_group = "PS" + subject_id = "112.283" + start_datetime = datetime(2019, 6, 20, 9, 32, 4) session_conditions = { "Start Date": start_datetime.strftime("%m/%d/%y"), "Start Time": start_datetime.strftime("%H:%M:%S"), @@ -385,15 +385,16 @@ def session_to_nwb( data_dir_path / f"{experiment_type} Experiments" / "Photometry" - / f"Punishment Resistant" + / f"Punishment Sensitive" / f"Early RI60" - / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-200709-130922" + / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-190620-093542" ) session_to_nwb( data_dir_path=data_dir_path, output_dir_path=output_dir_path, behavior_file_path=behavior_file_path, fiber_photometry_folder_path=fiber_photometry_folder_path, + has_demodulated_commanded_voltages=False, subject_id=subject_id, session_conditions=session_conditions, start_variable=start_variable, @@ -635,8 +636,8 @@ def session_to_nwb( experiment_type = "Opto" experimental_group = "DLS-Excitatory" optogenetic_treatment = "ChR2" - subject_id = "079.402" - start_datetime = datetime(2020, 6, 26, 13, 19, 27) + subject_id = "242.388" + start_datetime = datetime(2020, 6, 26, 12, 10, 40) session_conditions = { "Start Date": start_datetime.strftime("%m/%d/%y"), "Start Time": start_datetime.strftime("%H:%M:%S"), diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md index 3539698..ea204cb 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md @@ -153,6 +153,7 @@ for that 1 session split across the two folders? ### Questions - Some of the subject_ids are not present in the metadata excel file -- pls provide - Some animals are missing the "Hemisphere with DMS" field -- pls provide +- Looks like some of the mouse ids in the rr20 section of the excel file are incorrect (all XX.259 instead of XX.257) ======= - Solution: Added missing MSNs; skipped RK_C_FR1_BOTH_1hr From d50e75b73fafe6cf2d309c43ed3cb8b719194cd0 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Fri, 24 May 2024 11:57:29 -0700 Subject: [PATCH 15/17] added partial subject handling to fp and dls excitatory --- .../seiler_2024_convert_dataset.py | 117 +++++++++++++----- 1 file changed, 89 insertions(+), 28 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py index 2a7b3af..e08e2cd 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py @@ -55,34 +55,51 @@ def dataset_to_nwb( verbose=verbose, ) session_to_nwb_args_per_session = fp_session_to_nwb_args_per_session + opto_session_to_nwb_args_per_session - - futures = [] - with ProcessPoolExecutor(max_workers=max_workers) as executor: - for session_to_nwb_kwargs in session_to_nwb_args_per_session: - experiment_type = session_to_nwb_kwargs["experiment_type"] - experimental_group = session_to_nwb_kwargs["experimental_group"] - subject_id = session_to_nwb_kwargs["subject_id"] - start_datetime = session_to_nwb_kwargs["start_datetime"] - optogenetic_treatment = session_to_nwb_kwargs.get("optogenetic_treatment", None) - if experiment_type == "FP": - exception_file_path = ( - output_dir_path - / f"ERROR_{experiment_type}_{experimental_group}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" - ) - elif experiment_type == "Opto": - exception_file_path = ( - output_dir_path - / f"ERROR_{experiment_type}_{experimental_group}_{optogenetic_treatment}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" - ) - futures.append( - executor.submit( - safe_session_to_nwb, - session_to_nwb_kwargs=session_to_nwb_kwargs, - exception_file_path=exception_file_path, - ) - ) - for _ in tqdm(as_completed(futures), total=len(futures)): - pass + unique_subject_ids = set() + for session_to_nwb_kwargs in session_to_nwb_args_per_session: + subject_id = session_to_nwb_kwargs["subject_id"] + unique_subject_ids.add(subject_id) + metadata_path = Path(data_dir_path / "MouseDemographics.xlsx") + df = pd.read_excel( + metadata_path, + sheet_name="Mouse Demographics", + dtype={"Mouse ID": str}, + ) + df["DNL"] = df["Mouse ID"].str.contains("(DNL)", regex=False) + df["Mouse ID"] = df["Mouse ID"].str.replace("(DNL)", "") + df["Mouse ID"] = df["Mouse ID"].str.strip() + mouse_ids = set(df["Mouse ID"]) + missing_subject_ids = unique_subject_ids - mouse_ids + for missing_subject_id in missing_subject_ids: + print(f"Missing metadata for {missing_subject_id}") + + # futures = [] + # with ProcessPoolExecutor(max_workers=max_workers) as executor: + # for session_to_nwb_kwargs in session_to_nwb_args_per_session: + # experiment_type = session_to_nwb_kwargs["experiment_type"] + # experimental_group = session_to_nwb_kwargs["experimental_group"] + # subject_id = session_to_nwb_kwargs["subject_id"] + # start_datetime = session_to_nwb_kwargs["start_datetime"] + # optogenetic_treatment = session_to_nwb_kwargs.get("optogenetic_treatment", None) + # if experiment_type == "FP": + # exception_file_path = ( + # output_dir_path + # / f"ERROR_{experiment_type}_{experimental_group}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" + # ) + # elif experiment_type == "Opto": + # exception_file_path = ( + # output_dir_path + # / f"ERROR_{experiment_type}_{experimental_group}_{optogenetic_treatment}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" + # ) + # futures.append( + # executor.submit( + # safe_session_to_nwb, + # session_to_nwb_kwargs=session_to_nwb_kwargs, + # exception_file_path=exception_file_path, + # ) + # ) + # for _ in tqdm(as_completed(futures), total=len(futures)): + # pass def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: Union[Path, str]): @@ -186,6 +203,12 @@ def fp_to_nwb( "Photo_140_306-190809-121107", "Photo_271_396-200707-125117", } + partial_subject_ids_to_subject_id = { + "300": "300.405", + "418": "418.404", + "299": "299.405", + "276": "276.405", + } raw_file_to_info = get_raw_info(behavior_path) # Iterate through file system to get necessary information for converting each session @@ -313,6 +336,8 @@ def fp_to_nwb( if box_number is not None: session_conditions["Box"] = box_number start_datetime = datetime.strptime(f"{start_date} {start_time}", "%m/%d/%y %H:%M:%S") + if photometry_subject_id in partial_subject_ids_to_subject_id: + photometry_subject_id = partial_subject_ids_to_subject_id[photometry_subject_id] session_to_nwb_args = dict( data_dir_path=data_dir_path, output_dir_path=output_dir_path, @@ -385,6 +410,8 @@ def fp_to_nwb( if box_number is not None: session_conditions["Box"] = box_number start_datetime = datetime.strptime(f"{start_date} {start_time}", "%m/%d/%y %H:%M:%S") + if subject_id in partial_subject_ids_to_subject_id: + subject_id = partial_subject_ids_to_subject_id[subject_id] session_to_nwb_args = dict( data_dir_path=data_dir_path, output_dir_path=output_dir_path, @@ -432,6 +459,35 @@ def opto_to_nwb( list[dict] A list of dictionaries containing the arguments for session_to_nwb for each session. """ + partial_subject_ids_to_subject_id = { + "268": "268.476", + "266": "266.477", + "244": "244.465", + "343": "343.483", + "419": "419.404", + "245": "245.464", + "342": "342.483", + "202": "202.465", + "313": "313.403", + "418": "418.404", + "340": "340.483", + "259": "259.478", + "264": "264.478", + "421": "421.404", + "417": "417.404", + "233": "233.469", + "261": "261.478", + "265": "265.476", + "311": "311.403", + "206": "206.468", + "243": "243.468", + "263": "263.477", + "338": "338.398", + "414": "414.405", + "300": "300.405", + "299": "299.405", + "276": "276.405", + } experiment_type = "Opto" experimental_group_to_optogenetic_treatments = { "DLS-Excitatory": ["ChR2", "EYFP", "ChR2Scrambled"], @@ -553,6 +609,8 @@ def opto_to_nwb( "Box": box_number, } start_datetime = datetime.strptime(f"{start_date} {start_time}", "%m/%d/%y %H:%M:%S") + if subject in partial_subject_ids_to_subject_id: + subject = partial_subject_ids_to_subject_id[subject] session_to_nwb_args = dict( data_dir_path=data_dir_path, output_dir_path=output_dir_path, @@ -613,6 +671,9 @@ def get_opto_subject_id(subject_path: Path): "263": "263.477", "338": "338.398", "414": "414.405", + "300": "300.405", + "299": "299.405", + "276": "276.405", } # fmt: off From 55b8d51697c30f2184a4fe0d7ccbbb2e9412e0d1 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Fri, 24 May 2024 12:57:54 -0700 Subject: [PATCH 16/17] added subjects to skip and metadata sheet corrections --- .../seiler_2024_convert_dataset.py | 94 ++++++++++--------- .../seiler_2024_convert_session.py | 2 +- .../seiler_2024/seiler_2024_notes.md | 39 +++++++- 3 files changed, 86 insertions(+), 49 deletions(-) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py index e08e2cd..ebba637 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py @@ -37,6 +37,22 @@ def dataset_to_nwb( verbose : bool, optional Whether to print verbose output, by default True """ + subjects_to_skip = { + "289.407", + "244.464", + "264.477", + "102.260", + "262.478", + "289.408", + "264.475", + "129.425", + "250.427", + "95.259", + "309.399", + "433.421", + "416.405", + "364.426", + } start_variable = "Start Date" data_dir_path = Path(data_dir_path) output_dir_path = Path(output_dir_path) @@ -55,51 +71,36 @@ def dataset_to_nwb( verbose=verbose, ) session_to_nwb_args_per_session = fp_session_to_nwb_args_per_session + opto_session_to_nwb_args_per_session - unique_subject_ids = set() - for session_to_nwb_kwargs in session_to_nwb_args_per_session: - subject_id = session_to_nwb_kwargs["subject_id"] - unique_subject_ids.add(subject_id) - metadata_path = Path(data_dir_path / "MouseDemographics.xlsx") - df = pd.read_excel( - metadata_path, - sheet_name="Mouse Demographics", - dtype={"Mouse ID": str}, - ) - df["DNL"] = df["Mouse ID"].str.contains("(DNL)", regex=False) - df["Mouse ID"] = df["Mouse ID"].str.replace("(DNL)", "") - df["Mouse ID"] = df["Mouse ID"].str.strip() - mouse_ids = set(df["Mouse ID"]) - missing_subject_ids = unique_subject_ids - mouse_ids - for missing_subject_id in missing_subject_ids: - print(f"Missing metadata for {missing_subject_id}") - - # futures = [] - # with ProcessPoolExecutor(max_workers=max_workers) as executor: - # for session_to_nwb_kwargs in session_to_nwb_args_per_session: - # experiment_type = session_to_nwb_kwargs["experiment_type"] - # experimental_group = session_to_nwb_kwargs["experimental_group"] - # subject_id = session_to_nwb_kwargs["subject_id"] - # start_datetime = session_to_nwb_kwargs["start_datetime"] - # optogenetic_treatment = session_to_nwb_kwargs.get("optogenetic_treatment", None) - # if experiment_type == "FP": - # exception_file_path = ( - # output_dir_path - # / f"ERROR_{experiment_type}_{experimental_group}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" - # ) - # elif experiment_type == "Opto": - # exception_file_path = ( - # output_dir_path - # / f"ERROR_{experiment_type}_{experimental_group}_{optogenetic_treatment}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" - # ) - # futures.append( - # executor.submit( - # safe_session_to_nwb, - # session_to_nwb_kwargs=session_to_nwb_kwargs, - # exception_file_path=exception_file_path, - # ) - # ) - # for _ in tqdm(as_completed(futures), total=len(futures)): - # pass + + futures = [] + with ProcessPoolExecutor(max_workers=max_workers) as executor: + for session_to_nwb_kwargs in session_to_nwb_args_per_session: + experiment_type = session_to_nwb_kwargs["experiment_type"] + experimental_group = session_to_nwb_kwargs["experimental_group"] + subject_id = session_to_nwb_kwargs["subject_id"] + if subject_id in subjects_to_skip: + continue + start_datetime = session_to_nwb_kwargs["start_datetime"] + optogenetic_treatment = session_to_nwb_kwargs.get("optogenetic_treatment", None) + if experiment_type == "FP": + exception_file_path = ( + output_dir_path + / f"ERROR_{experiment_type}_{experimental_group}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" + ) + elif experiment_type == "Opto": + exception_file_path = ( + output_dir_path + / f"ERROR_{experiment_type}_{experimental_group}_{optogenetic_treatment}_{subject_id}_{start_datetime.isoformat().replace(':', '-')}.txt" + ) + futures.append( + executor.submit( + safe_session_to_nwb, + session_to_nwb_kwargs=session_to_nwb_kwargs, + exception_file_path=exception_file_path, + ) + ) + for _ in tqdm(as_completed(futures), total=len(futures)): + pass def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: Union[Path, str]): @@ -208,6 +209,7 @@ def fp_to_nwb( "418": "418.404", "299": "299.405", "276": "276.405", + "262.259.478": "262.478", } raw_file_to_info = get_raw_info(behavior_path) @@ -487,6 +489,7 @@ def opto_to_nwb( "300": "300.405", "299": "299.405", "276": "276.405", + "262.259.478": "262.478", } experiment_type = "Opto" experimental_group_to_optogenetic_treatments = { @@ -674,6 +677,7 @@ def get_opto_subject_id(subject_path: Path): "300": "300.405", "299": "299.405", "276": "276.405", + "262.259.478": "262.478", } # fmt: off diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py index 084c7de..ab32de5 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py @@ -134,7 +134,7 @@ def session_to_nwb( conversion_options.update(dict(Optogenetic={})) # Add Excel-based Metadata - metadata_path = data_dir_path / "MouseDemographics.xlsx" + metadata_path = data_dir_path / "MouseDemographicsCorrected.xlsx" source_data.update( dict( Metadata={ diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md index ea204cb..08ff247 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md @@ -153,9 +153,42 @@ for that 1 session split across the two folders? ### Questions - Some of the subject_ids are not present in the metadata excel file -- pls provide - Some animals are missing the "Hemisphere with DMS" field -- pls provide -- Looks like some of the mouse ids in the rr20 section of the excel file are incorrect (all XX.259 instead of XX.257) -======= - - Solution: Added missing MSNs; skipped RK_C_FR1_BOTH_1hr +- Some of the mouse ids have typos (leading and trailing zeros) as well as some that appear incorrect (RR20 section) + So, I made the following corrections to metadata excel sheet: + Mouse ID corrections: + 79.402 --> 079.402 + 344.4 --> 344.400 + 432.42 --> 432.420 + 48.392 --> 048.392 + 98.259 --> 98.257 + 101.259 --> 101.260 + 97.259 --> 97.257 + 99.259 --> 99.257 + 100.259 --> 100.258 + 359.43 --> 359.430 + 28.392 --> 028.392 + 227.43 --> 227.430 + 262.478 --> 262.259 + 354.43 --> 354.430 + 430.42 --> 430.420 + 342.483 --> 342.400 + After these corrections the following mouse_ids are still missing from the excel sheet: + subjects_to_skip = { + "289.407", + "244.464", + "264.477", + "102.260", + "262.478", + "289.408", + "264.475", + "129.425", + "250.427", + "95.259", + "309.399", + "433.421", + "416.405", + "364.426", + } ### Active Questions - DMS-Excitatory has some csv files w/ only session-aggregated info (total right rewards but not right reward times) ex. ChR2/121_280.CSV -- do you have individual session info for these animals? From e357309d78aae70e0c6abf5078e5ea3967ab8d8a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Fri, 24 May 2024 14:40:16 -0700 Subject: [PATCH 17/17] swapped create_stimulation_timeseries for neuroconv version --- frozen_dependencies.txt | 13 +++-- requirements.txt | 4 +- .../seiler_2024optogeneticinterface.py | 51 +------------------ 3 files changed, 12 insertions(+), 56 deletions(-) diff --git a/frozen_dependencies.txt b/frozen_dependencies.txt index 8b77de8..4839da8 100644 --- a/frozen_dependencies.txt +++ b/frozen_dependencies.txt @@ -2,6 +2,7 @@ aiobotocore==2.11.1 aiohttp==3.9.2 aioitertools==0.11.0 aiosignal==1.3.1 +annotated-types==0.7.0 appdirs==1.4.4 appnope==0.1.4 arrow==1.3.0 @@ -23,11 +24,12 @@ comm==0.2.1 contourpy==1.2.0 cycler==0.12.1 dandi==0.59.1 -dandischema==0.8.4 +dandischema==0.8.2 debugpy==1.8.1 decorator==5.1.1 distlib==0.3.8 dnspython==2.5.0 +docstring_parser==0.16 email-validator==2.1.0.post1 entrypoints==0.4 et-xmlfile==1.1.0 @@ -44,7 +46,7 @@ fsspec==2023.12.2 gast==0.4.0 h5py==3.10.0 hdmf==3.13.0 -hdmf_zarr==0.5.0 +hdmf_zarr==0.7.0 humanize==4.9.0 identify==2.5.33 idna==3.6 @@ -93,7 +95,7 @@ ndx-photometry @ git+https://github.com/catalystneuro/ndx-photometry.git@c1284c9 ndx-spectrum==0.2.2 nest-asyncio==1.6.0 networkx==3.2.1 -neuroconv==0.4.7 +neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@f45e77e152369916459a764c918d4c5cbbf1fad1 nodeenv==1.8.0 numcodecs==0.11.0 numpy==1.26.3 @@ -116,9 +118,10 @@ ptyprocess==0.7.0 pure-eval==0.2.2 py2vega==0.6.1 pycryptodomex==3.20.0 -pydantic==1.10.14 +pydantic==2.7.1 +pydantic_core==2.18.2 Pygments==2.17.2 -pynwb==2.6.0 +pynwb==2.7.0 pyout==0.7.3 pyparsing==3.1.1 pytest==8.1.1 diff --git a/requirements.txt b/requirements.txt index 72f10da..090515c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ hdmf -neuroconv +neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main nwbwidgets nwbinspector dandi pre-commit pytest ndx-events -ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@add_fiber_photometry_lab_meta_data +ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@main ipykernel tdt openpyxl diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024optogeneticinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024optogeneticinterface.py index 2513b2f..d033524 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024optogeneticinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024optogeneticinterface.py @@ -4,6 +4,7 @@ from pynwb.ogen import OptogeneticSeries from neuroconv.basedatainterface import BaseDataInterface from neuroconv.utils import DeepDict +from neuroconv.tools.optogenetics import create_optogenetic_stimulation_timeseries from typing import Literal from hdmf.backends.hdf5.h5_utils import H5DataIO from datetime import datetime, time @@ -157,7 +158,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): location=f"Injection location: {opto_metadata['injection_location']} \n Stimulation location: {opto_metadata['stimulation_location']}", excitation_lambda=opto_metadata["excitation_lambda"], ) - timestamps, data = self.create_stimulation_timeseries( + timestamps, data = create_optogenetic_stimulation_timeseries( stimulation_onset_times=stim_times, duration=opto_metadata["duration"], frequency=opto_metadata["frequency"], @@ -172,51 +173,3 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): description=opto_metadata["ogen_series_description"], ) nwbfile.add_stimulus(ogen_series) - - def create_stimulation_timeseries( # TODO: Move to neuroconv - self, stimulation_onset_times: np.ndarray, duration: float, frequency: float, pulse_width: float, power: float - ) -> tuple[np.ndarray, np.ndarray]: - """Create a continuous stimulation time series from stimulation onset times and parameters. - - In the resulting data array, the offset time of each pulse is represented by a 0 power value. - - Parameters - ---------- - stimulation_onset_times : np.ndarray - Array of stimulation onset times. - duration : float - Duration of stimulation in seconds. - frequency : float - Frequency of stimulation in Hz. - pulse_width : float - Pulse width of stimulation in seconds. - power : float - Power of stimulation in W. - - Returns - ------- - np.ndarray - Stimulation timestamps. - np.ndarray - Instantaneous stimulation power. - - Notes - ----- - For continuous stimulation of a desired duration, simply set - ``` - pulse_width = duration - frequency = 1 / duration - ``` - """ - num_pulses = int(duration * frequency) - inter_pulse_interval = 1 / frequency - timestamps, data = [0], [0] - for onset_time in stimulation_onset_times: - for i in range(num_pulses): - pulse_onset_time = onset_time + i * inter_pulse_interval - timestamps.append(pulse_onset_time) - data.append(power) - pulse_offset_time = pulse_onset_time + pulse_width - timestamps.append(pulse_offset_time) - data.append(0) - return np.array(timestamps, dtype=np.float64), np.array(data, dtype=np.float64)