diff --git a/src/lerner_lab_to_nwb/seiler_2024/__init__.py b/src/lerner_lab_to_nwb/seiler_2024/__init__.py index f676fcc..c7a752c 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/__init__.py +++ b/src/lerner_lab_to_nwb/seiler_2024/__init__.py @@ -1,3 +1,4 @@ from .seiler_2024behaviorinterface import Seiler2024BehaviorInterface from .seiler_2024fiberphotometryinterface import Seiler2024FiberPhotometryInterface +from .seiler_2024optogeneticinterface import Seiler2024OptogeneticInterface from .seiler_2024nwbconverter import Seiler2024NWBConverter diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py index c08e50b..3fd0dc8 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py @@ -15,11 +15,96 @@ def dataset_to_nwb( data_dir_path: Union[str, Path], output_dir_path: Union[str, Path], stub_test: bool = False, verbose: bool = True ): - """Convert the entire dataset to NWB.""" - # Setup + """Convert the entire dataset to NWB. + + Parameters + ---------- + data_dir_path : Union[str, Path] + The path to the directory containing the raw data. + output_dir_path : Union[str, Path] + The path to the directory where the NWB files will be saved. + stub_test : bool, optional + Whether to run a stub test, by default False + verbose : bool, optional + Whether to print verbose output, by default True + """ start_variable = "Start Date" data_dir_path = Path(data_dir_path) output_dir_path = Path(output_dir_path) + fp_session_to_nwb_args_per_session = fp_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + start_variable=start_variable, + stub_test=stub_test, + verbose=verbose, + ) + opto_session_to_nwb_args_per_session = opto_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + start_variable=start_variable, + stub_test=stub_test, + verbose=verbose, + ) + session_to_nwb_args_per_session = fp_session_to_nwb_args_per_session + opto_session_to_nwb_args_per_session + + # Convert all sessions and handle missing Fi1d's + missing_fi1d_sessions = [] + missing_msn_errors = set() + for session_to_nwb_args in tqdm(session_to_nwb_args_per_session): + try: + session_to_nwb(**session_to_nwb_args) + except AttributeError as e: + if str(e) == "'StructType' object has no attribute 'Fi1d'": + missing_fi1d_sessions.append( + str(session_to_nwb_args["fiber_photometry_folder_path"]).split("Photometry/")[1] + ) + continue + else: + print( + f"Could not convert {session_to_nwb_args['experimental_group']}/{session_to_nwb_args['subject_id']}/{session_to_nwb_args['session_conditions']['Start Date']} {session_to_nwb_args['session_conditions']['Start Time']}" + ) + raise AttributeError(e) + except KeyError as e: + missing_msn_errors.add(str(e)) + except Exception as e: + print( + f"Could not convert {session_to_nwb_args['experimental_group']}/{session_to_nwb_args['subject_id']}/{session_to_nwb_args['session_conditions']['Start Date']} {session_to_nwb_args['session_conditions']['Start Time']}" + ) + raise Exception(e) + if missing_fi1d_sessions: + print("Missing Fi1d Sessions:") + for session in missing_fi1d_sessions: + print(session) + if missing_msn_errors: + print("Missing MSN errors:") + for error in missing_msn_errors: + print(error) + + +def fp_to_nwb( + *, data_dir_path: Path, output_dir_path: Path, start_variable: str, stub_test: bool = False, verbose: bool = True +): + """Convert the Fiber Photometry portion of the dataset to NWB. + + Parameters + ---------- + data_dir_path : Path + The path to the directory containing the raw data. + output_dir_path : Path + The path to the directory where the NWB files will be saved. + start_variable : str + The variable to use as the start variable for the session. + stub_test : bool, optional + Whether to run a stub test, by default False + verbose : bool, optional + Whether to print verbose output, by default True + + Returns + ------- + list[dict] + A list of dictionaries containing the arguments for session_to_nwb for each session. + """ + # Setup experiment_type = "FP" experimental_groups = ["DPR", "PR", "PS", "RR20"] experimental_group_to_long_name = { @@ -57,7 +142,7 @@ def dataset_to_nwb( photometry_start_date = datetime.strptime(photometry_start_date, "%y%m%d").strftime("%m/%d/%y") subject_dir = behavior_path / experimental_group / photometry_subject_id - header_variables = get_header_variables( + header_variables = get_fp_header_variables( subject_dir, photometry_subject_id, raw_file_to_info, start_variable ) start_dates, start_times, msns, file_paths, subjects, box_numbers = header_variables @@ -135,7 +220,7 @@ def dataset_to_nwb( subject_dirs = [subject_dir for subject_dir in experimental_group_path.iterdir() if subject_dir.is_dir()] for subject_dir in subject_dirs: subject_id = subject_dir.name - header_variables = get_header_variables(subject_dir, subject_id, raw_file_to_info, start_variable) + header_variables = get_fp_header_variables(subject_dir, subject_id, raw_file_to_info, start_variable) start_dates, start_times, msns, file_paths, subjects, box_numbers = header_variables for start_date, start_time, msn, file, subject, box_number in zip( start_dates, start_times, msns, file_paths, subjects, box_numbers @@ -177,32 +262,176 @@ def dataset_to_nwb( continue nwbfile_paths.add(nwbfile_path) session_to_nwb_args_per_session.append(session_to_nwb_args) + return session_to_nwb_args_per_session - # Convert all sessions and handle missing Fi1d's - missing_fi1d_sessions = [] - for session_to_nwb_args in tqdm(session_to_nwb_args_per_session): - try: - session_to_nwb(**session_to_nwb_args) - except AttributeError as e: - if str(e) == "'StructType' object has no attribute 'Fi1d'": - missing_fi1d_sessions.append( - str(session_to_nwb_args["fiber_photometry_folder_path"]).split("Photometry/")[1] - ) + +def opto_to_nwb( + *, data_dir_path: Path, output_dir_path: Path, start_variable: str, stub_test: bool = False, verbose: bool = True +): + """Convert the Optogenetic portion of the dataset to NWB. + + Parameters + ---------- + data_dir_path : Path + The path to the directory containing the raw data. + output_dir_path : Path + The path to the directory where the NWB files will be saved. + start_variable : str + The variable to use as the start variable for the session. + stub_test : bool, optional + Whether to run a stub test, by default False + verbose : bool, optional + Whether to print verbose output, by default True + + Returns + ------- + list[dict] + A list of dictionaries containing the arguments for session_to_nwb for each session. + """ + experiment_type = "Opto" + experimental_groups = ["DLS-Excitatory", "DMS-Excitatory", "DMS-Inhibitory"] + experimental_group_to_optogenetic_treatments = { + "DLS-Excitatory": ["ChR2", "EYFP", "ChR2Scrambled"], + "DMS-Excitatory": ["ChR2", "EYFP", "ChR2Scrambled"], + "DMS-Inhibitory": ["NpHR", "EYFP", "NpHRScrambled"], + } + optogenetic_treatment_to_folder_name = { + "ChR2": "ChR2", + "EYFP": "EYFP", + "ChR2Scrambled": "Scrambled", + "NpHR": "Halo", + "NpHRScrambled": "Scrambled", + } + experimental_group_to_subgroups = { + "DLS-Excitatory": [""], + "DMS-Excitatory": [""], + "DMS-Inhibitory": ["Group 1"], # TODO: Get group 2 data from Lerner Lab + } + opto_path = data_dir_path / f"{experiment_type} Experiments" + session_to_nwb_args_per_session: list[dict] = [] # Each dict contains the args for session_to_nwb for a session + nwbfile_paths = set() # Each path is the path to the nwb file created for a session + + for experimental_group in experimental_groups: + experimental_group_path = opto_path / experimental_group.replace("-", " ") + for subgroup in experimental_group_to_subgroups[experimental_group]: + subgroup_path = experimental_group_path / subgroup if subgroup else experimental_group_path + optogenetic_treatments = experimental_group_to_optogenetic_treatments[experimental_group] + for optogenetic_treatment in optogenetic_treatments: + optogenetic_treatment_path = subgroup_path / optogenetic_treatment_to_folder_name[optogenetic_treatment] + subject_paths = [ + path + for path in optogenetic_treatment_path.iterdir() + if not ( + path.name.startswith(".") + or path.name.endswith(".csv") + or path.name.endswith(".CSV") # TODO: add support for session-aggregated CSV files + ) + ] + for subject_path in subject_paths: + subject_id = ( + subject_path.name.split(" ")[1] if "Subject" in subject_path.name else subject_path.name + ) + header_variables = get_opto_header_variables(subject_path) + start_dates, start_times, msns, file_paths, subjects, box_numbers = header_variables + for start_date, start_time, msn, file, subject, box_number in zip( + start_dates, start_times, msns, file_paths, subjects, box_numbers + ): + if session_should_be_skipped( + start_date=start_date, + start_time=start_time, + subject_id=subject_id, + msn=msn, + ): + continue + session_conditions = { + "Start Date": start_date, + "Start Time": start_time, + } + if subject is not None: + session_conditions["Subject"] = subject + if box_number is not None: + session_conditions["Box"] = box_number + start_datetime = datetime.strptime(f"{start_date} {start_time}", "%m/%d/%y %H:%M:%S") + session_to_nwb_args = dict( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=file, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + verbose=verbose, + ) + nwbfile_path = ( + output_dir_path + / f"{experiment_type}_{experimental_group}__{optogenetic_treatment}_{subject_id}_{start_datetime.isoformat()}.nwb" + ) + if nwbfile_path in nwbfile_paths: + continue + nwbfile_paths.add(nwbfile_path) + session_to_nwb_args_per_session.append(session_to_nwb_args) + return session_to_nwb_args_per_session + + +def get_opto_header_variables(subject_path): + """Get the header variables for the Optogenetic portion of the dataset. + + Parameters + ---------- + subject_path : Path + The path to the subject directory. + + Returns + ------- + tuple + A tuple containing the start dates, start times, MSNs, file paths, subjects, and box numbers. + """ + if subject_path.is_file(): + medpc_file_path = subject_path + medpc_variables = get_medpc_variables( + file_path=medpc_file_path, variable_names=["Start Date", "Start Time", "MSN"] + ) + start_dates = medpc_variables["Start Date"] + start_times = medpc_variables["Start Time"] + msns = medpc_variables["MSN"] + file_paths = [medpc_file_path] * len(start_dates) + subjects = [None] * len(start_dates) + box_numbers = [None] * len(start_dates) + elif subject_path.is_dir(): + start_dates, start_times, msns, file_paths = [], [], [], [] + medpc_files, csv_files = [], [] + for file in subject_path.iterdir(): + if file.name.startswith("."): continue - else: - print( - f"Could not convert {session_to_nwb_args['experimental_group']}/{session_to_nwb_args['subject_id']}/{session_to_nwb_args['session_conditions']['Start Date']} {session_to_nwb_args['session_conditions']['Start Time']}" - ) - raise AttributeError(e) - except Exception as e: - print( - f"Could not convert {session_to_nwb_args['experimental_group']}/{session_to_nwb_args['subject_id']}/{session_to_nwb_args['session_conditions']['Start Date']} {session_to_nwb_args['session_conditions']['Start Time']}" - ) - raise Exception(e) - if missing_fi1d_sessions: - print("Missing Fi1d Sessions:") - for session in missing_fi1d_sessions: - print(session) + if file.suffix == ".csv" or file.suffix == ".CSV": + csv_files.append(file) + elif not file.name.startswith("."): + medpc_files.append(file) + for file in medpc_files: + medpc_variables = get_medpc_variables(file_path=file, variable_names=["Start Date", "Start Time", "MSN"]) + for start_date, start_time, msn in zip( + medpc_variables["Start Date"], medpc_variables["Start Time"], medpc_variables["MSN"] + ): + start_dates.append(start_date) + start_times.append(start_time) + msns.append(msn) + file_paths.append(file) + for file in csv_files: + start_date = file.stem.split("_")[1].replace("-", "/") + start_time = "00:00:00" + msn = "Unknown" + start_dates.append(start_date) + start_times.append(start_time) + msns.append(msn) + file_paths.append(file) + subjects = [None] * len(start_dates) + box_numbers = [None] * len(start_dates) + + return start_dates, start_times, msns, file_paths, subjects, box_numbers def session_should_be_skipped(*, start_date, start_time, subject_id, msn): @@ -275,7 +504,25 @@ def get_csv_session_dates(subject_dir): return csv_session_dates -def get_header_variables(subject_dir, subject_id, raw_file_to_info, start_variable): +def get_fp_header_variables(subject_dir, subject_id, raw_file_to_info, start_variable): + """Get the header variables for the Fiber Photometry portion of the dataset. + + Parameters + ---------- + subject_dir : Path + The path to the subject directory. + subject_id : str + The subject ID. + raw_file_to_info : dict + A dictionary mapping raw files to their information. + start_variable : str + The variable to use as the start variable for the session. + + Returns + ------- + tuple + A tuple containing the start dates, start times, MSNs, file paths, subjects, and box numbers. + """ medpc_file_path = subject_dir / f"{subject_id}" if medpc_file_path.exists(): # Medpc file with all the sessions for the subject is located in the subject directory medpc_variables = get_medpc_variables( diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py index 0c735e4..83a42ec 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_session.py @@ -18,7 +18,8 @@ def session_to_nwb( session_conditions: dict, start_variable: str, experiment_type: Literal["FP", "Opto"], - experimental_group: Literal["DPR", "PR", "PS", "RR20"], + experimental_group: Literal["DPR", "PR", "PS", "RR20", "DMS-Inhibitory", "DMS-Excitatory", "DLS-Excitatory"], + optogenetic_treatment: Optional[Literal["ChR2", "EYFP", "ChR2Scrambled", "NpHR", "NpHRScrambled"]] = None, fiber_photometry_folder_path: Optional[Union[str, Path]] = None, stub_test: bool = False, verbose: bool = True, @@ -44,8 +45,10 @@ def session_to_nwb( The name of the variable that starts the session (ex. 'Start Date'). experiment_type : Literal["FP", "Opto"] The type of experiment. - experimental_group : Literal["DPR", "PR", "PS", "RR20"] + experimental_group : Literal["DPR", "PR", "PS", "RR20", "DMS-Inhibitory", "DMS-Excitatory", "DLS-Excitatory"] The experimental group. + optogenetic_treatment : Optional[Literal["ChR2", "EYFP", "ChR2Scrambled", "NpHR", "NpHRScrambled"]], optional + The optogenetic treatment, by default None for FP sessions. stub_test : bool, optional Whether to run a stub test, by default False verbose : bool, optional @@ -58,9 +61,6 @@ def session_to_nwb( output_dir_path = output_dir_path / "nwb_stub" output_dir_path.mkdir(parents=True, exist_ok=True) - nwbfile_path = ( - output_dir_path / f"{experiment_type}_{experimental_group}_{subject_id}_{start_datetime.isoformat()}.nwb" - ) source_data = {} conversion_options = {} @@ -90,6 +90,22 @@ def session_to_nwb( ) conversion_options.update(dict(FiberPhotometry={})) + # Add Optogenetics + if experiment_type == "Opto": + source_data.update( + dict( + Optogenetic={ + "file_path": str(behavior_file_path), + "session_conditions": session_conditions, + "start_variable": start_variable, + "experimental_group": experimental_group, + "optogenetic_treatment": optogenetic_treatment, + "verbose": verbose, + } + ) + ) + conversion_options.update(dict(Optogenetic={})) + converter = Seiler2024NWBConverter(source_data=source_data, verbose=verbose) metadata = converter.get_metadata() @@ -98,6 +114,19 @@ def session_to_nwb( editable_metadata = load_dict_from_file(editable_metadata_path) metadata = dict_deep_update(metadata, editable_metadata) + start_datetime = metadata["NWBFile"]["session_start_time"] + if experiment_type == "FP": + nwbfile_path = ( + output_dir_path / f"{experiment_type}_{experimental_group}_{subject_id}_{start_datetime.isoformat()}.nwb" + ) + elif experiment_type == "Opto": + nwbfile_path = ( + output_dir_path + / f"{experiment_type}_{experimental_group}_{optogenetic_treatment}_{subject_id}_{start_datetime.isoformat()}.nwb" + ) + else: + raise ValueError(f"Invalid experiment type: {experiment_type}") + # Run conversion converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options) @@ -349,18 +378,237 @@ def session_to_nwb( # stub_test=stub_test, # ) - # Behavior session from csv file - experiment_type = "FP" - experimental_group = "DPR" - subject_id = "87.239" - start_datetime = datetime(2019, 3, 19, 0, 0, 0) + # # Behavior session from csv file + # experiment_type = "FP" + # experimental_group = "DPR" + # subject_id = "87.239" + # start_datetime = datetime(2019, 3, 19, 0, 0, 0) + # session_conditions = {} + # start_variable = "" + # behavior_file_path = ( + # data_dir_path + # / f"{experiment_type} Experiments" + # / "Behavior" + # / f"{experimental_group}" + # / f"{subject_id}" + # / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + # ) + # session_to_nwb( + # data_dir_path=data_dir_path, + # output_dir_path=output_dir_path, + # behavior_file_path=behavior_file_path, + # subject_id=subject_id, + # session_conditions=session_conditions, + # start_variable=start_variable, + # start_datetime=start_datetime, + # experiment_type=experiment_type, + # experimental_group=experimental_group, + # stub_test=stub_test, + # ) + + # Example DMS-Inhibitory Opto session + experiment_type = "Opto" + experimental_group = "DMS-Inhibitory" + optogenetic_treatment = "NpHR" + subject_id = "112.415" + start_datetime = datetime(2020, 10, 21, 13, 8, 39) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"Group 1" + / f"Halo" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Example DMS-Excitatory Opto session + experiment_type = "Opto" + experimental_group = "DMS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "119.416" + start_datetime = datetime(2020, 10, 20, 13, 0, 57) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Example DLS-Excitatory Opto session + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "079.402" + start_datetime = datetime(2020, 6, 26, 13, 19, 27) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / f"{subject_id}" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Opto session with both left and right rewards + experiment_type = "Opto" + experimental_group = "DMS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "281.402" + start_datetime = datetime(2020, 9, 23, 12, 36, 30) + session_conditions = { + "Start Date": start_datetime.strftime("%m/%d/%y"), + "Start Time": start_datetime.strftime("%H:%M:%S"), + } + start_variable = "Start Date" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / "2020-09-23_12h36m_Subject 281.402" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Opto session from csv file + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "290.407" + start_datetime = datetime(2020, 9, 23, 0, 0, 0) + session_conditions = {} + start_variable = "" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" + / f"{subject_id}" + / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Opto session from csv file with scrambled optogenetic stimulation + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2Scrambled" + subject_id = "276.405" + start_datetime = datetime(2020, 10, 1, 0, 0, 0) + session_conditions = {} + start_variable = "" + behavior_file_path = ( + data_dir_path + / f"{experiment_type} Experiments" + / f"{experimental_group.replace('-', ' ')}" + / "Scrambled" + / f"{subject_id.replace('.', '_')}" + / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" + ) + session_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=output_dir_path, + behavior_file_path=behavior_file_path, + subject_id=subject_id, + session_conditions=session_conditions, + start_variable=start_variable, + start_datetime=start_datetime, + experiment_type=experiment_type, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + stub_test=stub_test, + ) + + # Could not convert DLS-Excitatory/299.405/09/11/20 00:00:00 + # Opto session with mixed dtype + experiment_type = "Opto" + experimental_group = "DLS-Excitatory" + optogenetic_treatment = "ChR2" + subject_id = "299.405" + start_datetime = datetime(2020, 9, 11, 0, 0, 0) session_conditions = {} start_variable = "" behavior_file_path = ( data_dir_path / f"{experiment_type} Experiments" - / "Behavior" - / f"{experimental_group}" + / f"{experimental_group.replace('-', ' ')}" + / f"{optogenetic_treatment}" / f"{subject_id}" / f"{subject_id}_{start_datetime.strftime('%m-%d-%y')}.csv" ) @@ -374,5 +622,6 @@ def session_to_nwb( start_datetime=start_datetime, experiment_type=experiment_type, experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, stub_test=stub_test, ) diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_metadata.yaml b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_metadata.yaml index 9082dbc..7273088 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_metadata.yaml +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_metadata.yaml @@ -167,6 +167,20 @@ Behavior: D: right_reward_times E: duration_of_port_entry G: port_entry_times + RI 60 LEFT_STIM: + A: left_nose_poke_times + B: left_reward_times + C: right_nose_poke_times + D: right_reward_times + E: duration_of_port_entry + G: port_entry_times + RI 30 LEFT_STIM: + A: left_nose_poke_times + B: left_reward_times + C: right_nose_poke_times + D: right_reward_times + E: duration_of_port_entry + G: port_entry_times RI30 Left Scrambled: A: left_nose_poke_times B: left_reward_times @@ -223,3 +237,38 @@ Behavior: E: duration_of_port_entry G: port_entry_times H: footshock_times + +Optogenetics: + experimental_group_to_metadata: + DMS-Excitatory: + injection_location: medial SNc (AP -3.1, ML 0.8, DV -4.7) + stimulation_location: DMS (AP 0.8, ML 1.5, DV -2.8) + excitation_lambda: 460.0 # nm + ogen_site_description: Mice for DMS excitatory optogenetics experiments received 1 ml of AAV5-EF1a-DIO-hChR2(H134R)-EYFP (3.3e13 GC/mL, Addgene, lot v17652) or the control fluorophore-only virus AAV5-EF1a-DIO-EYFP (3.5e12 virus molecules/mL, UNC Vector Core, lot AV4310K) in medial (AP -3.1, ML 0.8, DV -4.7) and a single fiber optic implant (Prizmatix; 250mm core, 0.66 NA) over ipsilateral DMS (AP 0.8, ML 1.5, DV -2.8). Hemispheres were counterbalanced between mice. + ogen_series_description: During operant training (beginning with FR1), each rewarded nosepoke was paired with a train of blue light (460nm, 1 s, 20 Hz, 15 mW) generated by an LED light source and pulse generator (Prizmatix). A subset of mice ("ChR2 Scrambled") received the same train of light but paired with random nosepokes on a separate RI60 schedule. + duration: 1.0 # seconds + frequency: 20.0 # Hz + pulse_width: 0.01 # TODO: ask Lerner lab for pulse width + power: 0.015 # W + + DLS-Excitatory: + injection_location: lateral SNc (AP -3.1, ML 1.3, DV -4.2) + stimulation_location: DLS (AP -0.1, ML 2.8, DV -3.5) + excitation_lambda: 460.0 # nm + ogen_site_description: Mice for DLS excitatory optogenetics experiments received 1 ml of AAV5-EF1a-DIO-hChR2(H134R)-EYFP (3.3e13 GC/mL, Addgene, lot v17652) or the control fluorophore-only virus AAV5-EF1a-DIO-EYFP (3.5e12 virus molecules/mL, UNC Vector Core, lot AV4310K) in lateral SNc (AP -3.1, ML 1.3, DV -4.2) and a single fiber optic implant (Prizmatix; 250mm core, 0.66 NA) over ipsilateral DLS (AP -0.1, ML 2.8, DV -3.5). Hemispheres were counterbalanced between mice. + ogen_series_description: During operant training (beginning with FR1), each rewarded nosepoke was paired with a train of blue light (460nm, 1 s, 20 Hz, 15 mW) generated by an LED light source and pulse generator (Prizmatix). A subset of mice ("ChR2 Scrambled") received the same train of light but paired with random nosepokes on a separate RI60 schedule. + duration: 1.0 # seconds + frequency: 20.0 # Hz + pulse_width: 0.01 # TODO: ask Lerner lab for pulse width + power: 0.015 # W + + DMS-Inhibitory: + injection_location: bilateral medial SNc (AP -3.1, ML ± 0.8, DV -4.7) + stimulation_location: bilateral DMS (AP 0.8, ML ± 1.5, DV -2.8) + excitation_lambda: 625.0 # nm + ogen_site_description: Mice for DMS inhibitory optogenetics experiments received 1 ml per side of AAV5-EF1a-DIO-eNpHR3.0-EYFP (1.1e13 GC/mL, Addgene, lot v32533) or the control fluorophore-only virus AAV5-EF1a-DIO-EYFP (3.5e12 virus molecules/mL, UNC Vector Core, lot AV4310K) in bilateral medial SNc (AP -3.1, ML 0.8, DV -4.7) and bilateral fiber optic implants (Prizmatix; 500mm core, 0.66 NA) in DMS (AP 0.8, ML ± 1.5, DV -2.8). + ogen_series_description: There were two groups of inhibitory optogenetics animals. Group 1 received inhibitory stimulation during operant training beginning with FR1, Since a subset of animals in this group were unable to learn the operant task, we also ran another group (Group 2) that received inhibitory stimulation during operant training beginning with RI30. These groups are combined for analysis of behaviors occurring after RI training has begun. For both groups, each rewarded nosepoke was paired with a continuous pulse of orange/red light (625nm, 1 s, 15 mW) generated by an LED light source and pulse generator (Prizmatix). A subset of mice ("NpHR Scrambled") received the same continuous pulse of light but paired with random nosepokes on a separate RI60 schedule. + duration: 1.0 # seconds + frequency: 1.0 # Hz + pulse_width: 1.0 # seconds + power: 0.015 # W diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md index 0b68a36..c69c07c 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_notes.md @@ -133,4 +133,19 @@ Many MSNs (ex. 'FOOD_FR1 TTL Left', 'FOOD_FR1 TTL Right', and 'FOOD_RI 30 LEFT') Punishment Sensitive/Late RI60/Photo_348_393-200730-113125 ## Optogenetics -TODO +### Notes +- Optogenetic pulses are either paired directly with reward times or optogenetic_stimulus_times variable in medpc file + for "scrambled" trials. +- timing info can be found in paper (460nm, 1 s, 20 Hz, 15 mW for excitatory and 625nm, 1 s, 15 mW for inhibitory) +- Some of the opto csv sessions have start times (ex. DLS Excitatory/ChR2/290.407/290.407_09-23-20.csv) -- added optional parsing +- Some of the sessions (ex. DLS-Excitatory/079.402/06/27/20) don't have any reward/stim times + +### Questions +- need to ask for more specific info about the device (data sheet) +- Need pulse width for excitatory optogenetics +- DMS-Inhibitory Group 2 is missing +- DLS-Excitatory has a bunch of files (medpc and csv) organized by date not belonging to any optogenetic treatment group folder + (ChR2, EYFP, Scrambled). Which treatment did these sessions receive? +- DMS-Excitatory has some csv files w/ only session-aggregated info (total right rewards but not right reward times) + ex. ChR2/121_280.CSV -- do you have individual session info for these animals? +- RI 60 LEFT_STIM, RI 30 LEFT_STIM, and RK_C_FR1_BOTH_1hr msns show up in opto data but don't have associated files -- assumed to be the same as their right counterparts? diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024behaviorinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024behaviorinterface.py index 84df75b..5e06dab 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024behaviorinterface.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024behaviorinterface.py @@ -90,14 +90,37 @@ def get_metadata(self) -> DeepDict: "Probe Test Habit Training TTL": "OmissionProbe", # TODO: Confirm with Lerner Lab "RI 30 RIGHT_STIM": "RI30", "RI 60 RIGHT STIM": "RI60", + "RI 60 LEFT_STIM": "RI60", + "RI 30 LEFT_STIM": "RI30", } if self.source_data["from_csv"]: - start_date = datetime.strptime(Path(self.source_data["file_path"]).stem.split("_")[1], "%m-%d-%y") - start_time = time(0, 0, 0) - training_stage = "Unknown" - subject = Path(self.source_data["file_path"]).stem.split("_")[0] - msn = "Unknown" - box = "Unknown" + session_dtypes = { + "Start Date": str, + "End Date": str, + "Start Time": str, + "End Time": str, + "MSN": str, + "Experiment": str, + "Subject": str, + "Box": str, + } + session_df = pd.read_csv(self.source_data["file_path"], dtype=session_dtypes) + start_date = ( + session_df["Start Date"][0] + if "Start Date" in session_df.columns + else Path(self.source_data["file_path"]).stem.split("_")[1].replace("-", "/") + ) + start_date = datetime.strptime(start_date, "%m/%d/%y").date() + start_time = session_df["Start Time"][0] if "Start Time" in session_df.columns else "00:00:00" + start_time = datetime.strptime(start_time, "%H:%M:%S").time() + msn = session_df["MSN"][0] if "MSN" in session_df.columns else "Unknown" + training_stage = msn_to_training_stage[msn] if "MSN" in session_df.columns else "Unknown" + subject = ( + session_df["Subject"][0] + if "Subject" in session_df.columns + else Path(self.source_data["file_path"]).stem.split("_")[0] + ) + box = session_df["Box"][0] if "Box" in session_df.columns else "Unknown" else: session_dict = read_medpc_file( file_path=self.source_data["file_path"], @@ -163,7 +186,17 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict) -> None: "RightRewardTs": "right_reward_times", "LeftRewardTs": "left_reward_times", } - session_df = pd.read_csv(self.source_data["file_path"]) + session_dtypes = { + "Start Date": str, + "End Date": str, + "Start Time": str, + "End Time": str, + "MSN": str, + "Experiment": str, + "Subject": str, + "Box": str, + } + session_df = pd.read_csv(self.source_data["file_path"], dtype=session_dtypes) session_dict = {} for csv_name, dict_name in csv_name_to_dict_name.items(): session_dict[dict_name] = np.trim_zeros(session_df[csv_name].dropna().values, trim="b") diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py index b5a122c..0c3cf43 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024nwbconverter.py @@ -4,8 +4,11 @@ from pynwb import NWBFile from neuroconv.tools.nwb_helpers import make_or_load_nwbfile -from lerner_lab_to_nwb.seiler_2024 import Seiler2024BehaviorInterface -from lerner_lab_to_nwb.seiler_2024 import Seiler2024FiberPhotometryInterface +from lerner_lab_to_nwb.seiler_2024 import ( + Seiler2024BehaviorInterface, + Seiler2024FiberPhotometryInterface, + Seiler2024OptogeneticInterface, +) from .medpc import read_medpc_file import numpy as np from tdt import read_block @@ -19,6 +22,7 @@ class Seiler2024NWBConverter(NWBConverter): data_interface_classes = dict( Behavior=Seiler2024BehaviorInterface, FiberPhotometry=Seiler2024FiberPhotometryInterface, + Optogenetic=Seiler2024OptogeneticInterface, ) def temporally_align_data_interfaces(self, metadata: dict): diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024optogeneticinterface.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024optogeneticinterface.py new file mode 100644 index 0000000..b2736e7 --- /dev/null +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024optogeneticinterface.py @@ -0,0 +1,199 @@ +"""Primary class for converting experiment-specific optogenetic stimulation.""" +import numpy as np +from pynwb.file import NWBFile +from pynwb.ogen import OptogeneticSeries +from neuroconv.basedatainterface import BaseDataInterface +from neuroconv.utils import DeepDict +from typing import Literal +from hdmf.backends.hdf5.h5_utils import H5DataIO +from datetime import datetime, time +from pathlib import Path +import pandas as pd + +from .medpc import read_medpc_file + + +class Seiler2024OptogeneticInterface(BaseDataInterface): + """Optogenetic interface for seiler_2024 conversion.""" + + keywords = ["optogenetics"] + + def __init__( + self, + file_path: str, + session_conditions: dict, + start_variable: str, + experimental_group: Literal["DMS-Inhibitory", "DMS-Excitatory", "DLS-Excitatory"], + optogenetic_treatment: Literal["ChR2", "EYFP", "ChR2Scrambled", "NpHR", "NpHRScrambled"], + verbose: bool = True, + ): + """Initialize Seiler2024OptogeneticInterface. + + Parameters + ---------- + file_path : str + Path to the MedPC file. Or path to the CSV file. + session_conditions : dict + The conditions that define the session. The keys are the names of the single-line variables (ex. 'Start Date') + and the values are the values of those variables for the desired session (ex. '11/09/18'). + start_variable : str + The name of the variable that starts the session (ex. 'Start Date'). + experimental_group : Literal["DMS-Inhibitory", "DMS-Excitatory", "DLS-Excitatory"] + The experimental group. + optogenetic_treatment : Literal["ChR2", "EYFP", "ChR2Scrambled", "NpHR", "NpHRScrambled"] + The optogenetic treatment. + verbose : bool, optional + Whether to print verbose output, by default True + """ + from_csv = file_path.endswith(".csv") + super().__init__( + file_path=file_path, + session_conditions=session_conditions, + start_variable=start_variable, + experimental_group=experimental_group, + optogenetic_treatment=optogenetic_treatment, + from_csv=from_csv, + verbose=verbose, + ) + + def get_metadata(self) -> DeepDict: + metadata = super().get_metadata() + return metadata + + def get_metadata_schema(self) -> dict: + metadata_schema = super().get_metadata_schema() + return metadata_schema + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): + # Read stim times from medpc file or csv + if self.source_data["from_csv"]: + csv_name_to_dict_name = { + "RightRewardTs": "right_reward_times", + "LeftRewardTs": "left_reward_times", + } + session_dtypes = { + "Start Date": str, + "End Date": str, + "Start Time": str, + "End Time": str, + "MSN": str, + "Experiment": str, + } + session_df = pd.read_csv(self.source_data["file_path"], dtype=session_dtypes) + session_dict = {} + for csv_name, dict_name in csv_name_to_dict_name.items(): + session_dict[dict_name] = np.trim_zeros(session_df[csv_name].dropna().values, trim="b") + if "Z" in session_df.columns: + session_dict["optogenetic_stimulation_times"] = np.trim_zeros(session_df["Z"].dropna().values, trim="b") + else: + msn = metadata["Behavior"]["msn"] + medpc_name_to_dict_name = metadata["Behavior"]["msn_to_medpc_name_to_dict_name"][msn] + opto_dict_names = {"left_reward_times", "right_reward_times", "optogenetic_stimulation_times"} + medpc_name_to_dict_name = { + medpc_name: dict_name + for medpc_name, dict_name in medpc_name_to_dict_name.items() + if dict_name in opto_dict_names + } + dict_name_to_type = {dict_name: np.ndarray for dict_name in medpc_name_to_dict_name.values()} + session_dict = read_medpc_file( + file_path=self.source_data["file_path"], + medpc_name_to_dict_name=medpc_name_to_dict_name, + dict_name_to_type=dict_name_to_type, + session_conditions=self.source_data["session_conditions"], + start_variable=self.source_data["start_variable"], + ) + if "optogenetic_stimulation_times" in session_dict: # stim times are recorded for scrambled trials + session_dict["stim_times"] = session_dict.pop("optogenetic_stimulation_times") + else: # otherwise, stim is delivered on either left or right reward -- usually interleaved + stim_times = [] + if len(session_dict["left_reward_times"]) > 0: + stim_times.extend(session_dict.pop("left_reward_times")) + if len(session_dict["right_reward_times"]) > 0: + stim_times.extend(session_dict.pop("right_reward_times")) + if not stim_times: # sessions without reward/stim times are skipped with a warning + if self.verbose: + print(f"No optogenetic stimulation times found for {metadata['NWBFile']['session_id']}") + return + session_dict["stim_times"] = np.sort(stim_times) + stim_times = session_dict["stim_times"] + + # Create optogenetic series and add to nwbfile + opto_metadata = metadata["Optogenetics"]["experimental_group_to_metadata"][ + self.source_data["experimental_group"] + ] + device = nwbfile.create_device( # TODO: Ask Lerner Lab for data sheet + name="LED_and_pulse_generator", + description="LED and pulse generator used for optogenetic stimulation.", + manufacturer="Prizmatix", + ) + ogen_site = nwbfile.create_ogen_site( + name="OptogeneticStimulusSite", + device=device, + description=opto_metadata["ogen_site_description"], + location=f"Injection location: {opto_metadata['injection_location']} \n Stimulation location: {opto_metadata['stimulation_location']}", + excitation_lambda=opto_metadata["excitation_lambda"], + ) + timestamps, data = self.create_stimulation_timeseries( + stimulation_onset_times=stim_times, + duration=opto_metadata["duration"], + frequency=opto_metadata["frequency"], + pulse_width=opto_metadata["pulse_width"], + power=opto_metadata["power"], + ) + ogen_series = OptogeneticSeries( + name="OptogeneticSeries", + site=ogen_site, + data=H5DataIO(data, compression=True), + timestamps=H5DataIO(timestamps, compression=True), + description=opto_metadata["ogen_series_description"], + comments=f"Optogenetic Treatment: {self.source_data['optogenetic_treatment']}", + ) + nwbfile.add_stimulus(ogen_series) + + def create_stimulation_timeseries( # TODO: Move to neuroconv + self, stimulation_onset_times: np.ndarray, duration: float, frequency: float, pulse_width: float, power: float + ) -> tuple[np.ndarray, np.ndarray]: + """Create a continuous stimulation time series from stimulation onset times and parameters. + + In the resulting data array, the offset time of each pulse is represented by a 0 power value. + + Parameters + ---------- + stimulation_onset_times : np.ndarray + Array of stimulation onset times. + duration : float + Duration of stimulation in seconds. + frequency : float + Frequency of stimulation in Hz. + pulse_width : float + Pulse width of stimulation in seconds. + power : float + Power of stimulation in W. + + Returns + ------- + np.ndarray + Stimulation timestamps. + np.ndarray + Instantaneous stimulation power. + + Notes + ----- + For continuous stimulation of a desired duration, simply set + ``` + pulse_width = duration + frequency = 1 / duration + ``` + """ + num_pulses = int(duration * frequency) + inter_pulse_interval = 1 / frequency + timestamps, data = [0], [0] + for onset_time in stimulation_onset_times: + for i in range(num_pulses): + pulse_onset_time = onset_time + i * inter_pulse_interval + timestamps.append(pulse_onset_time) + data.append(power) + pulse_offset_time = pulse_onset_time + pulse_width + timestamps.append(pulse_offset_time) + data.append(0) + return np.array(timestamps, dtype=np.float64), np.array(data, dtype=np.float64)