Merge pull request #25 from catalystneuro/final_feedback

Final feedback
catalystneuro · May 30, 2024 · c5135ca · c5135ca
2 parents ae83b75 + 74135bb
commit c5135ca
Show file tree

Hide file tree

Showing 8 changed files with 234 additions and 58 deletions.
diff --git a/frozen_dependencies.txt b/frozen_dependencies.txt
@@ -88,7 +88,7 @@ more-itertools==10.2.0
 multidict==6.0.4
 natsort==8.4.0
 ndx-events==0.2.0
-ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@14e05cda86026db694c1c577918284f435b7ec10
+ndx-fiber-photometry @ git+https://github.com/catalystneuro/ndx-fiber-photometry.git@94d80c2996cf3c71bd7e6f2b2c83f9b3fb9906ff
 ndx-grayscalevolume==0.0.2
 ndx-icephys-meta==0.1.0
 ndx-photometry @ git+https://github.com/catalystneuro/ndx-photometry.git@c1284c91a7e0a5dfd19f84bbf683e17fa607d4ec
@@ -99,7 +99,7 @@ neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@f45e77e1523699164
 nodeenv==1.8.0
 numcodecs==0.11.0
 numpy==1.26.3
-nwbinspector==0.4.35
+-e git+https://github.com/catalystneuro/nwbinspector.git@1f88732833709fc194ec46ebb82b0427137a2825#egg=nwbinspector
 nwbwidgets==0.11.3
 openpyxl==3.1.2
 packaging==23.2

diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py
@@ -207,6 +207,15 @@ def fp_to_nwb(
         "Photo_89_247-190308-095258",
         "Photo_140_306-190809-121107",
         "Photo_271_396-200707-125117",
+        "Photo_96_259-190417-160333",
+        "Photo_97_257-190417-134643",
+        "Photo_97_257-190506-120133",
+        "Photo_98_257-190424-114024",
+        "Photo_98_257-190510-095056",
+        "Photo_99_257-190506-130951",
+        "Photo_100_258-190423-122632",
+        "Photo_100_258-190509-133212",
+        "Photo_101_260-190425-120029",
     }
     partial_subject_ids_to_subject_id = {
         "300": "300.405",
@@ -274,11 +283,11 @@ def fp_to_nwb(
                 if (
                     (
                         photometry_subject_id == "334.394" and photometry_start_date == "07/21/20"
-                    )  # TODO: Ask Lerner Lab about this session
+                    )  # Skipping this session bc photometry data is corrupted
                     or (
                         photometry_subject_id == "99.257"
                         and photometry_start_date == "04/16/19"
-                        # TODO: Ask Lerner Lab about this session
+                        # Skipping this session bc missing behavior data
                     )
                     or (
                         photometry_subject_id == "64.205"
@@ -320,6 +329,11 @@ def fp_to_nwb(
                         and photometry_start_date == "10/31/18"
                         and experimental_subgroup.name == "Late RI60"
                     )
+                    or (
+                        photometry_subject_id == "96.259"
+                        and photometry_start_date == "05/06/19"
+                        and experimental_subgroup.name == "late"
+                    )  # This session is missing RNnR TTLs
                 ):
                     continue
                 assert (
@@ -535,7 +549,7 @@ def opto_to_nwb(
                     if not (
                         path.name.startswith(".")
                         or path.name.endswith(".csv")
-                        or path.name.endswith(".CSV")  # TODO: add support for session-aggregated CSV files
+                        or path.name.endswith(".CSV")  # session-aggregated CSV files are skipped
                     )
                 ]
                 for subject_path in subject_paths:
@@ -589,7 +603,7 @@ def opto_to_nwb(
     for file in raw_files_by_date_path.iterdir():
         if (
             file.name.startswith(".") or file.is_dir() or file.suffix == ".csv"
-        ):  # TODO: ask Lerner Lab about orphaned csvs
+        ):  # These .csv files are skipped bc they don't have subject info
             continue
         info = get_medpc_variables(file_path=file, variable_names=["Subject", "Start Date", "Start Time", "MSN", "Box"])
         for i in range(len(info["Subject"])):
@@ -819,8 +833,6 @@ def session_should_be_skipped(*, start_date, start_time, subject_id, msn):
         "FOOD_Magazine Training 1 hr",
         "RI_60_Left_Probability_AH_050619",
         "RI_60_Right_Probability_AH_050619",
-        "RR20_Right_AHJS",
-        "RR20_Left",
         "Extinction - 1 HR",
         "RR10_Left_AHJS",
         "Probe Test Habit Training CC",

diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py
@@ -156,6 +156,7 @@ def session_to_nwb(
     editable_metadata = load_dict_from_file(editable_metadata_path)
     metadata = dict_deep_update(metadata, editable_metadata)
 
+    metadata["NWBFile"]["session_description"] = metadata["Behavior"]["session_description"]
     session_start_time = metadata["NWBFile"]["session_start_time"]
     if optogenetic_treatment is None:
         session_id = f"{experiment_type}_{experimental_group}_{session_start_time.isoformat().replace(':', '-')}"
@@ -565,6 +566,50 @@ def split_western_blot(*, file_path: Union[str, Path]):
         experimental_group=experimental_group,
         stub_test=stub_test,
     )
+    # Fiber Photometry session with partial corruption AND missing Fi1d AND stitching two sessions together
+    experiment_type = "FP"
+    experimental_group = "PS"
+    subject_id = "139.298"
+    start_datetime = datetime(2019, 9, 12, 9, 33, 41)
+    session_conditions = {
+        "Start Date": start_datetime.strftime("%m/%d/%y"),
+        "Start Time": start_datetime.strftime("%H:%M:%S"),
+    }
+    start_variable = "Start Date"
+    behavior_file_path = (
+        data_dir_path
+        / f"{experiment_type} Experiments"
+        / "Behavior"
+        / f"{experimental_group}"
+        / f"{subject_id}"
+        / f"{subject_id}"
+    )
+    fiber_photometry_folder_path = (
+        data_dir_path
+        / f"{experiment_type} Experiments"
+        / "Photometry"
+        / f"Punishment Sensitive"
+        / f"Late RI60"
+        / f"Photo_{subject_id.split('.')[0]}_{subject_id.split('.')[1]}-190912-095034"
+    )
+    fiber_photometry_t2 = 2267.0
+    second_fiber_photometry_folder_path = fiber_photometry_folder_path.parent / "Photo_139_298-190912-103544"
+    session_to_nwb(
+        data_dir_path=data_dir_path,
+        output_dir_path=output_dir_path,
+        behavior_file_path=behavior_file_path,
+        fiber_photometry_folder_path=fiber_photometry_folder_path,
+        second_fiber_photometry_folder_path=second_fiber_photometry_folder_path,
+        fiber_photometry_t2=fiber_photometry_t2,
+        has_demodulated_commanded_voltages=False,
+        subject_id=subject_id,
+        session_conditions=session_conditions,
+        start_variable=start_variable,
+        start_datetime=start_datetime,
+        experiment_type=experiment_type,
+        experimental_group=experimental_group,
+        stub_test=stub_test,
+    )
 
     # Fiber Photometry session with swapped left and right TTLs and missing Fi1d
     experiment_type = "FP"

diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_metadata.yaml b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_metadata.yaml
@@ -1,7 +1,7 @@
 NWBFile:
   related_publications:
     - https://doi.org/10.1016/j.cub.2022.01.055
-  session_description:
+  experiment_description:
     Compulsive behavior is a defining feature of disorders such as substance use disorders.
     Current evidence suggests that corticostriatal circuits control the expression of established compulsions,
     but little is known about the mechanisms regulating the development of compulsions.
@@ -23,6 +23,17 @@ NWBFile:
     - Bianco, Joseph M.
     - Bridgemohan, Abigael S.
     - Lerner, Talia N.
+  keywords:
+    - dorsal striatum
+    - dopamine
+    - substantia nigra
+    - reward learning
+    - habit formation
+    - compulsive behavior
+    - punishment-resistant reward seeking
+    - fiber photometry
+    - optogenetics
+
 Subject:
   species: Mus musculus
   age: P10W/  # in ISO 8601, such as "P1W2D"
@@ -252,6 +263,14 @@ Behavior:
       E: duration_of_port_entry
       G: port_entry_times
       H: footshock_times
+    RR20_Left:
+      A: left_nose_poke_times
+      B: left_reward_times
+      C: right_nose_poke_times
+      D: right_reward_times
+      E: duration_of_port_entry
+      G: port_entry_times
+      H: footshock_times
     RR20Right:
       A: left_nose_poke_times
       B: left_reward_times
@@ -260,6 +279,14 @@ Behavior:
       E: duration_of_port_entry
       G: port_entry_times
       H: footshock_times
+    RR20_Right_AHJS:
+      A: left_nose_poke_times
+      B: left_reward_times
+      C: right_nose_poke_times
+      D: right_reward_times
+      E: duration_of_port_entry
+      G: port_entry_times
+      H: footshock_times
 
 Optogenetics:
   experimental_group_to_metadata:

diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md
@@ -48,6 +48,11 @@ In FP Experiments/Behavior/PS/140.306,
 - one of the sessions (ex. 09/06/19) has a bunch of garbage to the right of the 'A' variable
 - Solution: Handled by reader
 
+### Active Questions
+- In FP Experiments/Behavior/RR20/95.259/95.259, some of the sessions (ex. 04/09/19) have non-ascending reward port intervals
+ex. reward port entry = 985.750, reward port exit = 985.850 (duration = 0.1), next reward port entry = 985.800 (before previous exit)
+how is this possible?
+
 ### Resolved Questions
 In FP Experiments/Behavior/PS/139.298,
 - one of the sessions (ex. 09/20/19) is actually from subject 144.306
@@ -116,10 +121,16 @@ PrtN, RNnR, PrtR, LNPS, RNRW -- why is this mismatched with the expected epocs?
 for that 1 session split across the two folders?
     - Answer: Yes, occasionally the computer freezes or something and I need to restart the TDT recording while the MED program is unaffected. For my analysis I basically just stitched them together.
     - Solution: Added stitching functionality for optional second_folder_path
-
-### Active Questions
 - For DPR/334.394/07/21/20, 3 right nosepokes were made BUT photometry object still doesn't have a RNPS object
+    - Skip this session bc it is corrupted
 - RR20/99.257 on 04/16/19 has a photometry session but no matching behavior session on that day -- pls provide?
+    - Skip this session bc Lerner Lab can't find it
+
+### Active Questions
+- Some fiber photometry sessions (ex. FP_RR20_96.259_2019-04-17T16-03-52) have msns that I was told to skip (ex. RR20_Right_AHJS)
+  Should I still skip these, or just treat them like RR20Right?
+- In one of those sessions, FP Experiments/Photometry/RR20/late/Photo_96_259-190506-105642, doesn't have any RNnR TTLs,
+  even though it has plenty of right nose pokes in the behavioral file. Can you look into this?
 
 ## Optogenetics
 ### Notes
@@ -145,6 +156,36 @@ for that 1 session split across the two folders?
     - Solution: Metadata excel file has treatment info --> metadata["NWBFile"]["stimulus_notes"]
 - RI 60 LEFT_STIM, RI 30 LEFT_STIM, and RK_C_FR1_BOTH_1hr msns show up in opto data but don't have associated files -- assumed to be the same as their right counterparts?
 
+### Active Questions
+- DMS-Excitatory has some csv files w/ only session-aggregated info (total right rewards but not right reward times) ex. ChR2/121_280.CSV -- do you have individual session info for these animals?
+    - Lerner Lab does not have this data --> skip these sessions
+- Some csv files do not have any subject info (ex. DLS Excitatory/_08-28-20.csv) -- pls provide or we will need to skip these sessions
+    - Lerner Lab does not have this metadata --> skip these sessions
+- Some of the sessions in the DLS Excitatory medpc files organized by date don't have subject info -- pls provide or we will need to skip these sessions
+    Full List:
+        start_date ='08/28/20' start_time ='14:02:02'
+        start_date ='08/28/20' start_time ='15:19:38'
+        start_date ='08/28/20' start_time ='15:53:16'
+        start_date ='08/28/20' start_time ='16:02:33'
+        start_date ='08/28/20' start_time ='16:33:16'
+        start_date ='08/28/20' start_time ='16:38:51'
+        start_date ='09/03/20' start_time ='12:39:34'
+        start_date ='09/03/20' start_time ='12:39:34'
+        start_date ='09/03/20' start_time ='12:39:34'
+        start_date ='09/03/20' start_time ='12:39:34'
+        start_date ='08/31/20' start_time ='13:02:03'
+        start_date ='08/31/20' start_time ='13:02:03'
+        start_date ='08/31/20' start_time ='13:02:03'
+        start_date ='08/31/20' start_time ='13:02:03'
+        start_date ='08/31/20' start_time ='14:53:03'
+        start_date ='08/31/20' start_time ='14:53:03'
+        start_date ='08/31/20' start_time ='14:53:03'
+        start_date ='08/31/20' start_time ='15:34:14'
+        start_date ='09/22/20' start_time ='12:43:27'
+        start_date ='09/22/20' start_time ='12:43:27'
+        start_date ='09/22/20' start_time ='12:43:27'
+    - Lerner Lab does not have this metadata --> skip these sessions
+
 ## Western Blot
 ### Notes
 - Excel file has subject_ids for Female DLS Actin, Female DLS DAT, Female DMS Actin, Female DMS DAT and their
@@ -199,28 +240,16 @@ for that 1 session split across the two folders?
     }
 
 ### Active Questions
-- DMS-Excitatory has some csv files w/ only session-aggregated info (total right rewards but not right reward times) ex. ChR2/121_280.CSV -- do you have individual session info for these animals?
-- Some csv files do not have any subject info (ex. DLS Excitatory/_08-28-20.csv) -- pls provide or we will need to skip these sessions
-- Some of the sessions in the DLS Excitatory medpc files organized by date don't have subject info -- pls provide or we will need to skip these sessions
-    Full List:
-        start_date ='08/28/20' start_time ='14:02:02'
-        start_date ='08/28/20' start_time ='15:19:38'
-        start_date ='08/28/20' start_time ='15:53:16'
-        start_date ='08/28/20' start_time ='16:02:33'
-        start_date ='08/28/20' start_time ='16:33:16'
-        start_date ='08/28/20' start_time ='16:38:51'
-        start_date ='09/03/20' start_time ='12:39:34'
-        start_date ='09/03/20' start_time ='12:39:34'
-        start_date ='09/03/20' start_time ='12:39:34'
-        start_date ='09/03/20' start_time ='12:39:34'
-        start_date ='08/31/20' start_time ='13:02:03'
-        start_date ='08/31/20' start_time ='13:02:03'
-        start_date ='08/31/20' start_time ='13:02:03'
-        start_date ='08/31/20' start_time ='13:02:03'
-        start_date ='08/31/20' start_time ='14:53:03'
-        start_date ='08/31/20' start_time ='14:53:03'
-        start_date ='08/31/20' start_time ='14:53:03'
-        start_date ='08/31/20' start_time ='15:34:14'
-        start_date ='09/22/20' start_time ='12:43:27'
-        start_date ='09/22/20' start_time ='12:43:27'
-        start_date ='09/22/20' start_time ='12:43:27'
+None
+
+## Metadata
+### Notes
+- Some medpc filenames/sessions have incomplete or missing subject names (ex. 75 instead of 75.214) -- need to do some matching operation
+- Punishment Group has a typo for PR ('Punishment Resitant' instead of 'Punishment Resistant') -- I'll fix on my end
+
+### Resolved Questions
+- Some animals are missing the "Hemisphere with DMS" field -- pls provide
+    - Solution: Added missing MSNs; skipped RK_C_FR1_BOTH_1hr
+
+### Active Questions
+- Some of the subject_ids are not present in the metadata excel file -- pls provide