diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc20cc6e..78b882663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Update docs to reflect new notebooks. #776 - Add overview of Spyglass to docs. #779 - Update linting for Black 24. #808 +- Steamline dependency management. #822 ### Pipelines @@ -28,11 +29,6 @@ - Add SpikeSorting V1 pipeline. #651 - Move modules into spikesorting.v0 #807 - Add MUA analysis to spike sorting pipeline -- LFP: Minor fixes to LFPBandV1 populator and `make`. #706, #795 - -### Pipelines - -- Spike sorting: Add SpikeSorting V1 pipeline. #651 - LFP: - Minor fixes to LFPBandV1 populator and `make`. #706, #795 - LFPV1: Fix error for multiple lfp settings on same data #775 @@ -53,7 +49,7 @@ - Add fetch class functionality to `Merge` table. #783, #786 - Add ability to filter sorted units in decoding #807 - Rename SortedSpikesGroup.SortGroup to SortedSpikesGroup.Units #807 - - Change methods with load_... to fetch_... for consistency #807 + - Change methods with load\_... to fetch\_... for consistency #807 - Use merge table methods to access part methods #807 ## [0.4.3] (November 7, 2023) diff --git a/environment.yml b/environment.yml index c8c79d4c6..c1b2a25fd 100644 --- a/environment.yml +++ b/environment.yml @@ -1,38 +1,50 @@ +# 1. Install a conda distribution. +# https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html +# 2. Run: `mamba env create -f environment.yml` +# 3. Activate: `conda activate spyglass` +# +# (lines intentionally left blank) +# +# name: spyglass channels: - conda-forge - defaults + # - pytorch # dlc-only - franklab - edeno dependencies: - - python>=3.9,<3.10 + - bottleneck + # - cudatoolkit=11.3 # dlc-only + # - ffmpeg # dlc-only + - ipympl - jupyterlab>=3.* - - pydotplus - - dask + # - libgcc # dlc-only + - matplotlib + - non_local_detector + - numpy<1.24 - pip - position_tools - - numpy<1.24 - - track_linearization>=2.3 + - pybind11 # req by mountainsort4 -> isosplit5 + - pydotplus + - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM + - python>=3.9,<3.10 + - pytorch<1.12.0 - ripple_detection - - non_local_detector - - matplotlib - seaborn - - bottleneck - - ipympl - - tqdm - - pyfftw<=0.12.0 # used by ghostipy. install from conda-forge so that it works on Mac ARM processors + # - torchaudio # dlc-only + # - torchvision # dlc-only + - track_linearization>=2.3 - pip: - - pubnub<6.4.0 - - spikeinterface>=0.98.2,<0.99 - - pynwb>=2.2.0,<3 - - hdmf>=3.4.6 + - "black[jupyter]" - datajoint>=0.13.6 - - ghostipy - - pymysql + # - deeplabcut<2.3.0 # dlc-only + - ghostipy # for common_filter + - ndx-franklab-novela>=0.1.0 + - mountainsort4 + - panel<=1.3.5 # See panel #6325 + - pubnub<=6.4.0 + - pynwb>=2.2.0,<3 - sortingview>=0.11 - - figurl-jupyter - - git+https://github.com/LorenFrankLab/ndx-franklab-novela.git - - pyyaml - - click - - "black[jupyter]" + - spikeinterface>=0.98.2,<0.99 - . diff --git a/environment_dlc.yml b/environment_dlc.yml new file mode 100644 index 000000000..e175e7408 --- /dev/null +++ b/environment_dlc.yml @@ -0,0 +1,50 @@ +# 1. INSTALL CORRECT DRIVER for your GPU-equipped machine. +# see https://stackoverflow.com/questions/30820513/what-is-the-correct-version-of-cuda-for-my-nvidia-driver/30820690 +# 2. install: mamba env create -f environment_dlc.yml +# update existing install: mamba env update -f environment_dlc.yml +# 2. After installing... +# run: conda activate spyglass-dlc +# run: mamba env config vars set LD_LIBRARY_PATH=~/path/to//envs/spyglass-position/lib/ +# run: mamba install -c conda-forge wxpython +name: spyglass-dlc +channels: + - conda-forge + - defaults + - pytorch # dlc-only + - franklab + - edeno +dependencies: + - bottleneck + - cudatoolkit=11.3 # dlc-only + - ffmpeg # dlc-only + - ipympl + - jupyterlab>=3.* + - libgcc # dlc-only + - matplotlib + - non_local_detector + - numpy<1.24 + - pip>=20.2.* + - position_tools + - pybind11 # req by mountainsort4 -> isosplit5 + - pydotplus>=2.0.* + - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM + - python>=3.9,<3.10 + - pytorch<1.12.0 + - ripple_detection + - seaborn + - torchaudio # dlc-only + - torchvision # dlc-only + - track_linearization>=2.3 + - pip: + - "black[jupyter]" + - datajoint>=0.13.6 + - deeplabcut<2.3.0 # dlc-only + - ghostipy # for common_filter + - ndx-franklab-novela>=0.1.0 + - mountainsort4 + - panel<=1.3.5 # See panel #6325 + - pubnub<=6.4.0 + - pynwb>=2.2.0,<3 + - sortingview>=0.11 + - spikeinterface>=0.98.2,<0.99 + - .[dlc] diff --git a/environment_position.yml b/environment_position.yml deleted file mode 100644 index 125754062..000000000 --- a/environment_position.yml +++ /dev/null @@ -1,61 +0,0 @@ -# FIRST: INSTALL CORRECT DRIVER for GPU, see https://stackoverflow.com/questions/30820513/what-is-the-correct-version-of-cuda-for-my-nvidia-driver/30820690 -# -# NOTE: Perform install on a GPU-equipped machine -# -# install: mamba env create -f environment_position.yml -# update: mamba env update -f environment_position.yml -# after installing do the following: -# run: conda activate spyglass-position -# run: mamba env config vars set LD_LIBRARY_PATH=~/path/to//envs/spyglass-position/lib/ -# run: mamba install -c conda-forge wxpython -# cd into spyglass and run: pip install -e .[position] - -name: spyglass-position -channels: - - conda-forge - - pytorch - - franklab - - edeno -dependencies: - - python>=3.9, <3.10 - - jupyterlab>=3.* - - pydotplus - - libgcc - - dask>=2.30 - - pip - - position_tools - - track_linearization>=2.3 - - ripple_detection - - non_local_detector - - ipython - - matplotlib>=3.3 - - seaborn - - bottleneck - - ipympl - - tqdm - - ffmpeg - - pytorch<1.12.0 - - torchvision - - torchaudio - - cudatoolkit=11.3 - - tensorflow>=2.0 - - numba>=0.48.0 - - pyfftw<=0.12.0 # used by ghostipy. install from conda-forge so that it works on Mac ARM processors - - pybind11 #To avoid isosplit5 build error - - pip: - - pubnub<6.4.0 - - mountainsort4 - - spikeinterface>=0.98.2,<0.99 - - pynwb>=2.2.0,<3 - - hdmf>=3.4.6 - - datajoint>=0.13.6 - - ghostipy - - pymysql>=1.0.* - - sortingview>=0.11 - - figurl-jupyter - - git+https://github.com/LorenFrankLab/ndx-franklab-novela.git - - pyyaml - - click - - "black[jupyter]" - - deeplabcut<2.3.0 - - .[position] diff --git a/notebooks/01_Insert_Data.ipynb b/notebooks/01_Insert_Data.ipynb index de31ea7c8..e68623e72 100644 --- a/notebooks/01_Insert_Data.ipynb +++ b/notebooks/01_Insert_Data.ipynb @@ -945,8 +945,9 @@ "metadata": {}, "outputs": [], "source": [ + "team_name = \"My Team\"\n", "sgc.LabTeam().create_new_team(\n", - " team_name=\"My Team\", # Should be unique\n", + " team_name=team_name, # Should be unique\n", " team_members=[\"Firstname Lastname\", \"Firstname2 Lastname2\"],\n", " team_description=\"test\", # Optional\n", ")" @@ -2252,6 +2253,69 @@ "session_entry" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`Session.Experimenter` is used for permissions checks when deleting. The\n", + "session will need to have an experimenter in order avoid an error being\n", + "raised during this check.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sess_key = (sgc.Session & {\"nwb_file_name\": nwb_copy_file_name}).fetch(\n", + " \"KEY\", as_dict=True\n", + ")[0]\n", + "exp_key = (sgc.LabMember).fetch(\"KEY\", as_dict=True)[0]\n", + "sgc.Session.Experimenter.insert1(\n", + " dict(**sess_key, **exp_key), skip_duplicates=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Even with the experimenter specified, there are still delete protections\n", + "in place. To see an example, uncomment the cell below.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# session_entry.delete()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To delete, you'll need to share a team with the session experimenter.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "your_name = \"YourFirst YourLast\"\n", + "parts = your_name.split(\" \")\n", + "sgc.LabMember.insert1([your_name, parts[0], parts[1]])\n", + "sgc.LabMember.LabMemberInfo.insert1(\n", + " [your_name, \"your_gmail\", dj.config[\"database.user\"], 0]\n", + ")\n", + "sgc.LabTeam.LabTeamMember.insert1([team_name, your_name])" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -2311,7 +2375,7 @@ } ], "source": [ - "session_entry.delete()" + "# session_entry.delete()" ] }, { diff --git a/notebooks/02_Data_Sync.ipynb b/notebooks/02_Data_Sync.ipynb index 74f01a46b..485a4c920 100644 --- a/notebooks/02_Data_Sync.ipynb +++ b/notebooks/02_Data_Sync.ipynb @@ -20,11 +20,11 @@ "source": [ "This notebook will cover ...\n", "\n", - "1. [General Kachery information](#kachery) \n", - "2. Setting up Kachery as a [host](#host-setup). If you'll use an existing host, \n", - " skip this.\n", - "3. Setting up Kachery in your [database](#database-setup). If you're using an \n", - " existing database, skip this.\n", + "1. [General Kachery information](#kachery)\n", + "2. Setting up Kachery as a [host](#host-setup). If you'll use an existing host,\n", + " skip this.\n", + "3. Setting up Kachery in your [database](#database-setup). If you're using an\n", + " existing database, skip this.\n", "4. Adding Kachery [data](#data-setup).\n" ] }, @@ -46,7 +46,7 @@ "\n", "- To set up your Spyglass environment and database, see\n", " [the Setup notebook](./00_Setup.ipynb)\n", - "- To fully demonstrate syncing features, we'll need to run some basic analyses. \n", + "- To fully demonstrate syncing features, we'll need to run some basic analyses.\n", " This can either be done with code in this notebook or by running another\n", " notebook (e.g., [LFP](./12_LFP.ipynb))\n", "- For additional info on DataJoint syntax, including table definitions and\n", @@ -54,9 +54,9 @@ " [these additional tutorials](https://github.com/datajoint/datajoint-tutorials)\n", "\n", "Let's start by importing the `spyglass` package and testing that your environment\n", - " is properly configured for kachery sharing\n", + "is properly configured for kachery sharing\n", "\n", - "If you haven't already done so, be sure to set up your Spyglass base directory and Kachery sharing directory with [Setup](./00_Setup.ipynb)" + "If you haven't already done so, be sure to set up your Spyglass base directory and Kachery sharing directory with [Setup](./00_Setup.ipynb)\n" ] }, { @@ -76,7 +76,6 @@ "source": [ "import os\n", "import datajoint as dj\n", - "import pandas as pd\n", "\n", "# change to the upper level folder to detect dj_local_conf.json\n", "if os.path.basename(os.getcwd()) == \"notebooks\":\n", @@ -128,7 +127,8 @@ ")\n", "lfp.v1.LFPV1().populate()\n", "```\n", - "" + "\n", + "\n" ] }, { @@ -154,8 +154,8 @@ "makes it possible to share analysis results, stored in NWB files. When a user\n", "tries to access a file, Spyglass does the following:\n", "\n", - "1. Try to load from the local file system/store. \n", - "2. If unavailable, check if it is in the relevant sharing table (i.e., \n", + "1. Try to load from the local file system/store.\n", + "2. If unavailable, check if it is in the relevant sharing table (i.e.,\n", " `NwbKachery` or `AnalysisNWBKachery`).\n", "3. If present, attempt to download from the associated Kachery Resource to the user's spyglass analysis directory.\n", "\n", @@ -182,27 +182,27 @@ "2. `franklab.collaborator`: File sharing with collaborating labs.\n", "3. `franklab.public`: Public file sharing (not yet active)\n", "\n", - "Setting your zone can either be done as as an environment variable or an item \n", + "Setting your zone can either be done as as an environment variable or an item\n", "in a DataJoint config. Spyglass will automatically handle setting the appropriate zone when downloading\n", "database files through kachery\n", "\n", "- Environment variable:\n", "\n", - " ```bash\n", - " export KACHERY_ZONE=franklab.default\n", - " export KACHERY_CLOUD_DIR=/stelmo/nwb/.kachery-cloud\n", - " ```\n", + " ```bash\n", + " export KACHERY_ZONE=franklab.default\n", + " export KACHERY_CLOUD_DIR=/stelmo/nwb/.kachery-cloud\n", + " ```\n", "\n", "- DataJoint Config:\n", "\n", - " ```json\n", - " \"custom\": {\n", - " \"kachery_zone\": \"franklab.default\",\n", - " \"kachery_dirs\": {\n", - " \"cloud\": \"/your/base/path/.kachery-cloud\"\n", - " }\n", - " }\n", - " ```" + " ```json\n", + " \"custom\": {\n", + " \"kachery_zone\": \"franklab.default\",\n", + " \"kachery_dirs\": {\n", + " \"cloud\": \"/your/base/path/.kachery-cloud\"\n", + " }\n", + " }\n", + " ```\n" ] }, { @@ -213,7 +213,7 @@ "\n", "- If you are a member of a team with a pre-existing database and zone who will be sharing data, please skip to `Sharing Data`\n", "\n", - "- If you are a collaborator outside your team's network and need to access files shared with you, please skip to `Accessing Shared Data`" + "- If you are a collaborator outside your team's network and need to access files shared with you, please skip to `Accessing Shared Data`\n" ] }, { @@ -231,13 +231,13 @@ "See\n", "[instructions](https://github.com/flatironinstitute/kachery-cloud/blob/main/doc/create_kachery_zone.md)\n", "for setting up new Kachery Zones, including creating a cloud bucket and\n", - "registering it with the Kachery team. \n", + "registering it with the Kachery team.\n", "\n", - "_Notes:_ \n", + "_Notes:_\n", "\n", "- Bucket names cannot include periods, so we substitute a dash, as in\n", " `franklab-default`.\n", - "- You only need to create an API token for your first zone." + "- You only need to create an API token for your first zone.\n" ] }, { @@ -245,16 +245,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Resources" + "### Resources\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "\n", "See [instructions](https://github.com/scratchrealm/kachery-resource/blob/main/README.md)\n", - "for setting up zone resources. This allows for sharing files on demand. We \n", + "for setting up zone resources. This allows for sharing files on demand. We\n", "suggest using the same name for the zone and resource.\n", "\n", "_Note:_ For each zone, you need to run the local daemon that listens for\n", @@ -270,7 +269,7 @@ "For convenience, we recommend saving this code as a bash script which can be executed by the local daemon. For franklab member, these scripts can be found in the directory `/home/loren/bin/`:\n", "\n", "- run_restart_kachery_collab.sh\n", - "- run_restart_kachery_default.sh" + "- run_restart_kachery_default.sh\n" ] }, { @@ -285,9 +284,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Once you have a hosted zone running, we need to add its information to the Spyglass database. \n", + "Once you have a hosted zone running, we need to add its information to the Spyglass database.\n", "This will allow spyglass to manage linking files from our analysis tables to kachery.\n", - "First, we'll check existing Zones." + "First, we'll check existing Zones.\n" ] }, { @@ -405,9 +404,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To add a new hosted Zone, we need to prepare an entry for the `KacheryZone` table. \n", + "To add a new hosted Zone, we need to prepare an entry for the `KacheryZone` table.\n", "Note that the `kacherycloud_dir` key should be the path for the server daemon _hosting_ the zone,\n", - " and is not required to be present on the client machine:" + "and is not required to be present on the client machine:\n" ] }, { @@ -433,7 +432,7 @@ "metadata": {}, "source": [ "Use caution when inserting into an active database, as it could interfere with\n", - "ongoing work." + "ongoing work.\n" ] }, { @@ -442,14 +441,14 @@ "metadata": {}, "outputs": [], "source": [ - "sgs.KacheryZone().insert1(zone_key)" + "sgs.KacheryZone().insert1(zone_key, skip_duplicates=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Sharing Data" + "## Sharing Data\n" ] }, { @@ -459,8 +458,8 @@ "source": [ "Once the zone exists, we can add `AnalysisNWB` files we want to share with members of the zone.\n", "\n", - "The `AnalysisNwbFileKachery` table links analysis files made within other spyglass tables with a `uri` \n", - "used by kachery. We can view files already made available through kachery here:" + "The `AnalysisNwbFileKachery` table links analysis files made within other spyglass tables with a `uri`\n", + "used by kachery. We can view files already made available through kachery here:\n" ] }, { @@ -602,8 +601,8 @@ "\n", "To do so we first add these entries to the `AnalysisNwbfileKacherySelection` table.\n", "\n", - "_Note:_ This step depends on having previously run an analysis on the example \n", - "file." + "_Note:_ This step depends on having previously run an analysis on the example\n", + "file.\n" ] }, { @@ -635,7 +634,7 @@ }, "source": [ "With those files in the selection table, we can add them as links to the zone by\n", - "populating the `AnalysisNwbfileKachery` table:" + "populating the `AnalysisNwbfileKachery` table:\n" ] }, { @@ -653,10 +652,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Alternatively, we can share data based on its source table in the database using the helper function `share_data_to_kachery()` \n", + "Alternatively, we can share data based on its source table in the database using the helper function `share_data_to_kachery()`\n", "\n", - "This will take a list of tables and add all associated analysis files for entries corresponding with a passed restriction. \n", - "Here, we are sharing LFP and position data for the Session \"minirec20230622_.nwb\"" + "This will take a list of tables and add all associated analysis files for entries corresponding with a passed restriction.\n", + "Here, we are sharing LFP and position data for the Session \"minirec20230622\\_.nwb\"\n" ] }, { @@ -674,7 +673,7 @@ "share_data_to_kachery(\n", " table_list=tables,\n", " restriction=restriction,\n", - " zone_name=\"franklab.collaborators\",\n", + " zone_name=zone_name,\n", ")" ] }, @@ -682,7 +681,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Managing access" + "## Managing access\n" ] }, { @@ -696,13 +695,13 @@ "tags": [] }, "source": [ - "If all of that worked, \n", + "If all of that worked,\n", "\n", "1. Go to https://kachery-gateway.figurl.org/admin?zone=your_zone\n", - " (changing your_zone to the name of your zone)\n", + " (changing your_zone to the name of your zone)\n", "2. Go to the Admin/Authorization Settings tab\n", - "3. Add the GitHub login names and permissions for the users you want to share \n", - " with. \n", + "3. Add the GitHub login names and permissions for the users you want to share\n", + " with.\n", "\n", "If those users can connect to your database, they should now be able to use the\n", "`.fetch_nwb()` method to download any `AnalysisNwbfiles` that have been shared\n", @@ -717,14 +716,14 @@ " CuratedSpikeSorting & {\"nwb_file_name\": \"minirec20230622_.nwb\"}\n", ").fetch()[0]\n", "sort = (CuratedSpikeSorting & test_sort).fetch_nwb()\n", - "```" + "```\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Accessing Shared Data" + "## Accessing Shared Data\n" ] }, { @@ -733,7 +732,7 @@ "source": [ "If you are a collaborator accessing datasets, you first need to be given access to the zone by a collaborator admin (see above).\n", "\n", - "If you know the uri for the dataset you are accessing you can test this process below (example is for members of `franklab.collaborators`)" + "If you know the uri for the dataset you are accessing you can test this process below (example is for members of `franklab.collaborators`)\n" ] }, { @@ -759,11 +758,11 @@ "source": [ "In normal use, spyglass will manage setting the zone and uri when accessing files.\n", "In general, the easiest way to access data valueswill be through the `fetch1_dataframe()`\n", - "function part of many of the spyglass tables. In brief this will check for the appropriate\n", + "function part of many of the spyglass tables. In brief this will check for the appropriate\n", "nwb analysis file in your local directory, and if not found, attempt to download it from the appropriate kachery zone.\n", - "It will then parse the relevant information from that nwb file into a pandas dataframe. \n", + "It will then parse the relevant information from that nwb file into a pandas dataframe.\n", "\n", - "We will look at an example with data from the `LFPV1` table:" + "We will look at an example with data from the `LFPV1` table:\n" ] }, { @@ -897,7 +896,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can access the data using `fetch1_dataframe()`" + "We can access the data using `fetch1_dataframe()`\n" ] }, { @@ -1294,7 +1293,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Up Next" + "# Up Next\n" ] }, { @@ -1302,7 +1301,7 @@ "metadata": {}, "source": [ "In the [next notebook](./03_Merge_Tables.ipynb), we'll explore the details of a\n", - "table tier unique to Spyglass, Merge Tables." + "table tier unique to Spyglass, Merge Tables.\n" ] } ], diff --git a/notebooks/11_CurationV0.ipynb b/notebooks/11_CurationV0.ipynb index 78cb5caa6..b8f99e430 100644 --- a/notebooks/11_CurationV0.ipynb +++ b/notebooks/11_CurationV0.ipynb @@ -28,7 +28,7 @@ " [this notebook](./00_Setup.ipynb)\n", "- For a more detailed introduction to DataJoint with inserts, see\n", " [this notebook](./01_Insert_Data.ipynb)\n", - "- [The Spike Sorting notebook](./02_Spike_Sorting.ipynb) is a mandatory\n", + "- [The Spike Sorting notebook](./10_Spike_SortingV0.ipynb) is a mandatory\n", " prerequisite to Curation.\n" ] }, @@ -288,7 +288,7 @@ "source": [ "## Up Next\n", "\n", - "Next, we'll turn our attention to [LFP data](./12_LFP.ipynb) data." + "Next, we'll turn our attention to [LFP data](./12_LFP.ipynb) data.\n" ] } ], diff --git a/notebooks/21_DLC.ipynb b/notebooks/21_DLC.ipynb index 9bd492720..ffc0d450c 100644 --- a/notebooks/21_DLC.ipynb +++ b/notebooks/21_DLC.ipynb @@ -478,7 +478,7 @@ } ], "source": [ - "team_name = \"LorenLab\"\n", + "team_name = sgc.LabTeam.fetch(\"team_name\")[0] # If on lab DB, \"LorenLab\"\n", "project_name = \"tutorial_scratch_DG\"\n", "frames_per_video = 100\n", "bodyparts = [\"redLED_C\", \"greenLED\", \"redLED_L\", \"redLED_R\", \"tailBase\"]\n", diff --git a/notebooks/22_DLC_Loop.ipynb b/notebooks/22_DLC_Loop.ipynb index 103b39525..804d1a021 100644 --- a/notebooks/22_DLC_Loop.ipynb +++ b/notebooks/22_DLC_Loop.ipynb @@ -5,7 +5,7 @@ "id": "a93a1550-8a67-4346-a4bf-e5a136f3d903", "metadata": {}, "source": [ - "## Position- DeepLabCut from Scratch" + "## Position- DeepLabCut from Scratch\n" ] }, { @@ -13,7 +13,7 @@ "id": "13dd3267", "metadata": {}, "source": [ - "### Overview" + "### Overview\n" ] }, { @@ -41,7 +41,7 @@ "- processing the pose estimation output to extract a centroid and orientation\n", "- inserting the resulting information into the `PositionOutput` table\n", "\n", - "**Note 2: Make sure you are running this within the spyglass-position Conda environment (instructions for install are in the environment_position.yml)**" + "**Note 2: Make sure you are running this within the spyglass-position Conda environment (instructions for install are in the environment_position.yml)**\n" ] }, { @@ -60,6 +60,7 @@ "metadata": {}, "source": [ "### Table of Contents\n", + "\n", "[`DLCProject`](#DLCProject1)
\n", "[`DLCModelTraining`](#DLCModelTraining1)
\n", "[`DLCModel`](#DLCModel1)
\n", @@ -69,7 +70,7 @@ "[`DLCOrientation`](#DLCOrientation1)
\n", "[`DLCPosV1`](#DLCPosV1-1)
\n", "[`DLCPosVideo`](#DLCPosVideo1)
\n", - "[`PositionOutput`](#PositionOutput1)
" + "[`PositionOutput`](#PositionOutput1)
\n" ] }, { @@ -77,7 +78,7 @@ "id": "70a0a678", "metadata": {}, "source": [ - "__You can click on any header to return to the Table of Contents__" + "**You can click on any header to return to the Table of Contents**\n" ] }, { @@ -85,7 +86,7 @@ "id": "c9b98c3d", "metadata": {}, "source": [ - "### Imports" + "### Imports\n" ] }, { @@ -144,7 +145,7 @@ "id": "5e6221a3-17e5-45c0-aa40-2fd664b02219", "metadata": {}, "source": [ - "#### [DLCProject](#TableOfContents) " + "#### [DLCProject](#TableOfContents) \n" ] }, { @@ -171,7 +172,7 @@ "id": "50c9f1c9", "metadata": {}, "source": [ - "### Body Parts" + "### Body Parts\n" ] }, { @@ -179,7 +180,7 @@ "id": "96637cb9-519d-41e1-8bfd-69f68dc66b36", "metadata": {}, "source": [ - "We'll begin by looking at the `BodyPart` table, which stores standard names of body parts used in DLC models throughout the lab with a concise description." + "We'll begin by looking at the `BodyPart` table, which stores standard names of body parts used in DLC models throughout the lab with a concise description.\n" ] }, { @@ -207,7 +208,7 @@ " ],\n", " skip_duplicates=True,\n", ")\n", - "```" + "```\n" ] }, { @@ -215,7 +216,7 @@ "id": "57b590d3", "metadata": {}, "source": [ - "### Define videos and camera name (optional) for training set" + "### Define videos and camera name (optional) for training set\n" ] }, { @@ -227,7 +228,7 @@ "\n", "The list can either contain dictionaries identifying behavioral videos for NWB files that have already been added to Spyglass, or absolute file paths to the videos you want to use.\n", "\n", - "For this tutorial, we'll use two videos for which we already have frames labeled." + "For this tutorial, we'll use two videos for which we already have frames labeled.\n" ] }, { @@ -238,7 +239,7 @@ "Defining camera name is optional: it should be done in cases where there are multiple cameras streaming per epoch, but not necessary otherwise.
\n", "example:\n", "`camera_name = \"HomeBox_camera\" \n", - " `" + " `\n" ] }, { @@ -303,7 +304,7 @@ "_NOTE:_ If only `base` is specified as shown above, spyglass will assume the\n", "relative directories shown.\n", "\n", - "You can check the result of this setup process with..." + "You can check the result of this setup process with...\n" ] }, { @@ -331,7 +332,7 @@ " **\"tutorial_scratch_yourinitials\"**\n", "- `bodyparts` is a list of body parts for which we want to extract position.\n", " The pre-labeled frames we're using include the bodyparts listed below.\n", - "- Number of frames to extract/label as `frames_per_video`. Note that the DLC creators recommend having 200 frames as the minimum total number for each project." + "- Number of frames to extract/label as `frames_per_video`. Note that the DLC creators recommend having 200 frames as the minimum total number for each project.\n" ] }, { @@ -341,7 +342,7 @@ "metadata": {}, "outputs": [], "source": [ - "team_name = \"LorenLab\"\n", + "team_name = sgc.LabTeam.fetch(\"team_name\")[0] # If on lab DB, \"LorenLab\"\n", "project_name = \"tutorial_scratch_DG\"\n", "frames_per_video = 100\n", "bodyparts = [\"redLED_C\", \"greenLED\", \"redLED_L\", \"redLED_R\", \"tailBase\"]\n", @@ -360,7 +361,7 @@ "id": "f5d83452-48eb-4669-89eb-a6beb1f2d051", "metadata": {}, "source": [ - "Now that we've initialized our project we'll need to extract frames which we will then label. " + "Now that we've initialized our project we'll need to extract frames which we will then label.\n" ] }, { @@ -380,9 +381,10 @@ "metadata": {}, "source": [ "This is the line used to label the frames you extracted, if you wish to use the DLC GUI on the computer you are currently using.\n", + "\n", "```#comment this line out after frames are labeled for your project\n", "sgp.DLCProject().run_label_frames(project_key)\n", - "```" + "```\n" ] }, { @@ -391,17 +393,18 @@ "metadata": {}, "source": [ "Otherwise, it is best/easiest practice to label the frames on your local computer (like a MacBook) that can run DeepLabCut's GUI well. Instructions:
\n", + "\n", "1. Install DLC on your local (preferably into a 'Src' folder): https://deeplabcut.github.io/DeepLabCut/docs/installation.html\n", "2. Upload frames extracted and saved in nimbus (should be `/nimbus/deeplabcut//labeled-data`) AND the project's associated config file (should be `/nimbus/deeplabcut//config.yaml`) to Box (we get free with UCSF)\n", "3. Download labeled-data and config files on your local from Box\n", "4. Create a 'projects' folder where you installed DeepLabCut; create a new folder with your complete project name there; save the downloaded files there.\n", - "4. Edit the config.yaml file: line 9 defining `project_path` needs to be the file path where it is saved on your local (ex: `/Users/lorenlab/Src/DeepLabCut/projects/tutorial_sratch_DG-LorenLab-2023-08-16`)\n", - "5. Open the DLC GUI through terminal \n", - "
(ex: `conda activate miniconda/envs/DEEPLABCUT_M1`\n", - "\t\t
`pythonw -m deeplabcut`)\n", - "6. Load an existing project; choose the config.yaml file\n", - "7. Label frames; labeling tutorial: https://www.youtube.com/watch?v=hsA9IB5r73E.\n", - "8. Once all frames are labeled, you should re-upload labeled-data folder back to Box and overwrite it in the original nimbus location so that your completed frames are ready to be used in the model." + "5. Edit the config.yaml file: line 9 defining `project_path` needs to be the file path where it is saved on your local (ex: `/Users/lorenlab/Src/DeepLabCut/projects/tutorial_sratch_DG-LorenLab-2023-08-16`)\n", + "6. Open the DLC GUI through terminal\n", + "
(ex: `conda activate miniconda/envs/DEEPLABCUT_M1`\n", + "
`pythonw -m deeplabcut`)\n", + "7. Load an existing project; choose the config.yaml file\n", + "8. Label frames; labeling tutorial: https://www.youtube.com/watch?v=hsA9IB5r73E.\n", + "9. Once all frames are labeled, you should re-upload labeled-data folder back to Box and overwrite it in the original nimbus location so that your completed frames are ready to be used in the model.\n" ] }, { @@ -409,7 +412,7 @@ "id": "c12dd229-2f8b-455a-a7b1-a20916cefed9", "metadata": {}, "source": [ - "Now we can check the `DLCProject.File` part table and see all of our training files and videos there!" + "Now we can check the `DLCProject.File` part table and see all of our training files and videos there!\n" ] }, { @@ -429,7 +432,7 @@ "source": [ "
\n", " This step and beyond should be run on a GPU-enabled machine.\n", - "
" + "\n" ] }, { @@ -518,7 +521,7 @@ "id": "6b6cc709", "metadata": {}, "source": [ - "Next we'll modify the `project_key` from above to include the necessary entries for `DLCModelTraining`" + "Next we'll modify the `project_key` from above to include the necessary entries for `DLCModelTraining`\n" ] }, { @@ -540,7 +543,7 @@ "source": [ "We can insert an entry into `DLCModelTrainingSelection` and populate `DLCModelTraining`.\n", "\n", - "_Note:_ You can stop training at any point using `I + I` or interrupt the Kernel. \n", + "_Note:_ You can stop training at any point using `I + I` or interrupt the Kernel.\n", "\n", "The maximum total number of training iterations is 1030000; you can end training before this amount if the loss rate (lr) and total loss plateau and are very close to 0.\n" ] @@ -587,7 +590,7 @@ "id": "da004b3e", "metadata": {}, "source": [ - "Here we'll make sure that the entry made it into the table properly!" + "Here we'll make sure that the entry made it into the table properly!\n" ] }, { @@ -609,7 +612,7 @@ "source": [ "Populating `DLCModelTraining` automatically inserts the entry into\n", "`DLCModelSource`, which is used to select between models trained using Spyglass\n", - "vs. other tools." + "vs. other tools.\n" ] }, { @@ -627,7 +630,7 @@ "id": "92cb8969", "metadata": {}, "source": [ - "The `source` field will only accept _\"FromImport\"_ or _\"FromUpstream\"_ as entries. Let's checkout the `FromUpstream` part table attached to `DLCModelSource` below." + "The `source` field will only accept _\"FromImport\"_ or _\"FromUpstream\"_ as entries. Let's checkout the `FromUpstream` part table attached to `DLCModelSource` below.\n" ] }, { @@ -651,7 +654,7 @@ "information for all trained models.\n", "\n", "First, we'll need to determine a set of parameters for our model to select the\n", - "correct model file. Here is the default:" + "correct model file. Here is the default:\n" ] }, { @@ -692,7 +695,7 @@ "metadata": {}, "source": [ "We can insert sets of parameters into `DLCModelSelection` and populate\n", - "`DLCModel`." + "`DLCModel`.\n" ] }, { @@ -734,7 +737,7 @@ "id": "f8f1b839", "metadata": {}, "source": [ - "Again, let's make sure that everything looks correct in `DLCModel`." + "Again, let's make sure that everything looks correct in `DLCModel`.\n" ] }, { @@ -752,7 +755,7 @@ "id": "02202650", "metadata": {}, "source": [ - "## Loop Begins" + "## Loop Begins\n" ] }, { @@ -760,7 +763,7 @@ "id": "dd886971", "metadata": {}, "source": [ - "We can view all `VideoFile` entries with the specidied `camera_ name` for this project to ensure the rat whose position you wish to model is in this table `matching_rows`" + "We can view all `VideoFile` entries with the specidied `camera_ name` for this project to ensure the rat whose position you wish to model is in this table `matching_rows`\n" ] }, { @@ -782,7 +785,7 @@ "source": [ "The `DLCPoseEstimationSelection` insertion step will convert your .h264 video to an .mp4 first and save it in `/nimbus/deeplabcut/video`. If this video already exists here, the insertion will never complete.\n", "\n", - "We first delete any .mp4 that exists for this video from the nimbus folder:" + "We first delete any .mp4 that exists for this video from the nimbus folder:\n" ] }, { @@ -801,6 +804,7 @@ "metadata": {}, "source": [ "If the first insertion step (for pose estimation task) fails in either trigger or load mode for an epoch, run the following lines:\n", + "\n", "```\n", "(pose_estimation_key = sgp.DLCPoseEstimationSelection.insert_estimation_task(\n", " {\n", @@ -809,7 +813,7 @@ " \"video_file_num\": video_file_num,\n", " **model_key,\n", " }).delete()\n", - "```" + "```\n" ] }, { @@ -817,12 +821,12 @@ "id": "7eb99b6f", "metadata": {}, "source": [ - "This loop will generate posiiton data for all epochs associated with the pre-defined camera in one day, for one rat (based on the NWB file; see ***)\n", + "This loop will generate posiiton data for all epochs associated with the pre-defined camera in one day, for one rat (based on the NWB file; see \\*\\*\\*)\n", "
The output should print Pose Estimation and Centroid plots for each epoch.\n", "\n", "- It defines `col1val` as each `nwb_file_name` entry in the table, one at a time.\n", - "- Next, it sees if the trial on which you are testing this model is in the string for the current `col1val`; if not, it re-defines `col1val` as the next `nwb_file_name` entry and re-tries this step. \n", - "- If the previous step works, it then saves `col2val` and `col3val` as the `epoch` and the `video_file_num`, respectively, based on the nwb_file_name. From there, it iterates through the insertion and population steps required to extract position data, which we see laid out in notebook 05_DLC.ipynb." + "- Next, it sees if the trial on which you are testing this model is in the string for the current `col1val`; if not, it re-defines `col1val` as the next `nwb_file_name` entry and re-tries this step.\n", + "- If the previous step works, it then saves `col2val` and `col3val` as the `epoch` and the `video_file_num`, respectively, based on the nwb_file_name. From there, it iterates through the insertion and population steps required to extract position data, which we see laid out in notebook 05_DLC.ipynb.\n" ] }, { @@ -950,7 +954,8 @@ "metadata": {}, "source": [ "### _CONGRATULATIONS!!_\n", - "Please treat yourself to a nice tea break :-)" + "\n", + "Please treat yourself to a nice tea break :-)\n" ] }, { @@ -958,7 +963,7 @@ "id": "c71c90a2", "metadata": {}, "source": [ - "### [Return To Table of Contents](#TableOfContents)
" + "### [Return To Table of Contents](#TableOfContents)
\n" ] } ], diff --git a/notebooks/24_Linearization.ipynb b/notebooks/24_Linearization.ipynb index 208bf2d45..d9f58c9d2 100644 --- a/notebooks/24_Linearization.ipynb +++ b/notebooks/24_Linearization.ipynb @@ -125,7 +125,7 @@ "source": [ "from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename\n", "\n", - "nwb_file_name = \"chimi20200216_new.nwb\"\n", + "nwb_file_name = \"minirec20230622.nwb\" # detailed data: chimi20200216_new.nwb\n", "nwb_copy_file_name = get_nwb_copy_filename(nwb_file_name)\n", "nwb_copy_file_name" ] @@ -341,8 +341,8 @@ "\n", "pos_key = {\n", " \"nwb_file_name\": nwb_copy_file_name,\n", - " \"interval_list_name\": \"pos 1 valid times\",\n", - " \"position_info_param_name\": \"default\",\n", + " \"interval_list_name\": \"pos 0 valid times\", # For chimi, \"pos 1 valid times\"\n", + " \"trodes_pos_params_name\": \"single_led_upsampled\", # For chimi, \"default\"\n", "}\n", "\n", "# Note: You'll have to change the part table to the one where your data came from\n", diff --git a/notebooks/30_LFP.ipynb b/notebooks/30_LFP.ipynb index d9eb5eaac..6449a8648 100644 --- a/notebooks/30_LFP.ipynb +++ b/notebooks/30_LFP.ipynb @@ -2491,7 +2491,11 @@ "of the electrodes we band pass filtered\n", "\n", "_Note:_ Much of the code below could be replaced by a function calls that would\n", - "return the data from each electrical series\n" + "return the data from each electrical series.\n", + "\n", + "_Note:_ If you see an error `Qt: Session Management Error`, try running the\n", + "following unix command: `export -n SESSION_MANAGER`.\n", + "[See also](https://stackoverflow.com/questions/986964/qt-session-management-error)\n" ] }, { @@ -2515,7 +2519,7 @@ "metadata": {}, "outputs": [], "source": [ - "lfp_eseries = lfp.LFPOutput.fetch_nwb(lfp_key)[0][\"lfp\"]\n", + "lfp_eseries = lfp.LFPOutput().fetch_nwb(lfp_key)[0][\"lfp\"]\n", "lfp_elect_indices = sgc.get_electrode_indices(\n", " lfp_eseries, lfp_band_electrode_ids\n", ")\n", diff --git a/notebooks/32_Ripple_Detection.ipynb b/notebooks/32_Ripple_Detection.ipynb index e9ac8348c..01c833cac 100644 --- a/notebooks/32_Ripple_Detection.ipynb +++ b/notebooks/32_Ripple_Detection.ipynb @@ -35,7 +35,7 @@ " inserts, see\n", " [the Insert Data notebook](./01_Insert_Data.ipynb)\n", "\n", - "Ripple detection depends on a set of LFPs, the parameters used for detection and the speed of the animal. You will need `RippleLFPSelection`, `RippleParameters`, and `PositionOutput` to be populated accordingly." + "Ripple detection depends on a set of LFPs, the parameters used for detection and the speed of the animal. You will need `RippleLFPSelection`, `RippleParameters`, and `PositionOutput` to be populated accordingly.\n" ] }, { @@ -106,7 +106,7 @@ "metadata": {}, "source": [ "First, we'll pick the electrodes on which we'll run ripple detection on, using\n", - "`RippleLFPSelection.set_lfp_electrodes`" + "`RippleLFPSelection.set_lfp_electrodes`\n" ] }, { @@ -155,12 +155,12 @@ "id": "355190ce-a553-44d6-8260-7eda67f9407f", "metadata": {}, "source": [ - "We'll need the `nwb_file_name`, an `electrode_list`, and to a `group_name`. \n", + "We'll need the `nwb_file_name`, an `electrode_list`, and to a `group_name`.\n", "\n", "- By default, `group_name` is set to CA1 for ripple detection, but we could\n", " alternatively use PFC.\n", "- We use `nwb_file_name` to explore which electrodes are available for the\n", - " `electrode_list`." + " `electrode_list`.\n" ] }, { @@ -172,7 +172,10 @@ "source": [ "nwb_file_name = \"tonks20211103_.nwb\"\n", "interval_list_name = \"test interval\"\n", - "filter_name = \"Ripple 150-250 Hz\"" + "filter_name = \"Ripple 150-250 Hz\"\n", + "if not sgc.Session & {\"nwb_file_name\": nwb_file_name}:\n", + " # This error will be raised when notebooks auto-run with 'minirec'\n", + " raise ValueError(f\"Session with nwb_file_name={nwb_file_name} not found\")" ] }, { @@ -181,7 +184,7 @@ "id": "ac9c694f-06dd-47e8-9379-ec37397758f8", "metadata": {}, "source": [ - "Now we can look at `electrode_id` in the `Electrode` table:" + "Now we can look at `electrode_id` in the `Electrode` table:\n" ] }, { @@ -541,7 +544,7 @@ "id": "c400f47e-a21b-451a-8721-35191161dc6e", "metadata": {}, "source": [ - "For ripple detection, we want only tetrodes, and only the first good wire on each tetrode. We will assume that is the first wire on each tetrode. I will do this using pandas syntax but you could use datajoint to filter this table as well. Here is the filtered table." + "For ripple detection, we want only tetrodes, and only the first good wire on each tetrode. We will assume that is the first wire on each tetrode. I will do this using pandas syntax but you could use datajoint to filter this table as well. Here is the filtered table.\n" ] }, { @@ -892,7 +895,7 @@ "id": "007840a2-8b8b-4d56-9bf6-be5160aa41e6", "metadata": {}, "source": [ - "We only want the electrode_id to put in the `electrode_list`:" + "We only want the electrode_id to put in the `electrode_list`:\n" ] }, { @@ -926,7 +929,7 @@ "\n", "We can insert into `RippleLFPSelection` and the `RippleLFPElectrode` part table,\n", "passing the key for the entry from `LFPBandV1`, our `electrode_list`, and the\n", - "`group_name` into `set_lfp_electrodes`" + "`group_name` into `set_lfp_electrodes`\n" ] }, { @@ -1212,7 +1215,7 @@ "id": "0b27717a-d4a4-4293-9ea9-cf759b09039e", "metadata": {}, "source": [ - "Here's the ripple selection key we'll use downstream" + "Here's the ripple selection key we'll use downstream\n" ] }, { @@ -1338,7 +1341,7 @@ "id": "80e88984-0581-4f3d-88cf-164c24885569", "metadata": {}, "source": [ - "Here are the default ripple parameters:" + "Here are the default ripple parameters:\n" ] }, { @@ -1379,7 +1382,7 @@ "- `speed_name`: the name of the speed parameters in `IntervalPositionInfo`\n", "\n", "For the `Kay_ripple_detector` (options are currently Kay and Karlsson, see `ripple_detection` package for specifics) the parameters are:\n", - " \n", + "\n", "- `speed_threshold` (cm/s): maximum speed the animal can move\n", "- `minimum_duration` (s): minimum time above threshold\n", "- `zscore_threshold` (std): minimum value to be considered a ripple, in standard\n", @@ -1397,7 +1400,7 @@ "## Check interval speed\n", "\n", "The speed for this interval should exist under the default position parameter\n", - "set and for a given interval." + "set and for a given interval.\n" ] }, { @@ -1615,7 +1618,7 @@ "id": "1bc905e6-211a-4292-9f13-dfff906479a0", "metadata": {}, "source": [ - "We'll use the `head_speed` above as part of `RippleParameters`." + "We'll use the `head_speed` above as part of `RippleParameters`.\n" ] }, { @@ -1624,7 +1627,7 @@ "id": "674d6896-8a93-4a51-ad45-f5a4218f007e", "metadata": {}, "source": [ - "## Run Ripple Detection \n" + "## Run Ripple Detection\n" ] }, { @@ -1632,8 +1635,7 @@ "id": "a5c3f1b4", "metadata": {}, "source": [ - "\n", - "Now we can put everything together." + "Now we can put everything together.\n" ] }, { @@ -1686,7 +1688,7 @@ "id": "500d96cd-e558-4036-bc93-27704cb3f855", "metadata": {}, "source": [ - "And then `fetch1_dataframe` for ripple times" + "And then `fetch1_dataframe` for ripple times\n" ] }, { @@ -2046,7 +2048,7 @@ "source": [ "## Up Next\n", "\n", - "Next, we'll [extract mark indicator](./31_Extract_Mark_Indicators.ipynb)." + "Next, we'll [extract mark indicator](./31_Extract_Mark_Indicators.ipynb).\n" ] } ], diff --git a/notebooks/41_Extracting_Clusterless_Waveform_Features.ipynb b/notebooks/41_Extracting_Clusterless_Waveform_Features.ipynb index 09cc22bb2..820da6496 100644 --- a/notebooks/41_Extracting_Clusterless_Waveform_Features.ipynb +++ b/notebooks/41_Extracting_Clusterless_Waveform_Features.ipynb @@ -7,7 +7,7 @@ "_Developer Note:_ if you may make a PR in the future, be sure to copy this\n", "notebook, and use the `gitignore` prefix `temp` to avoid future conflicts.\n", "\n", - "This is one notebook in a multi-part series on clusterless decoding in Spyglass \n", + "This is one notebook in a multi-part series on clusterless decoding in Spyglass\n", "\n", "- To set up your Spyglass environment and database, see\n", " [the Setup notebook](./00_Setup.ipynb)\n", @@ -22,7 +22,7 @@ "\n", "The goal of this notebook is to populate the `UnitWaveformFeatures` table, which depends `SpikeSortingOutput`. This table contains the features of the waveforms of each unit.\n", "\n", - "While clusterless decoding avoids actual spike sorting, we need to pass through these tables to maintain (relative) pipeline simplicity. Pass-through tables keep spike sorting and clusterless waveform extraction as similar as possible, by using shared steps. Here, \"spike sorting\" involves simple thresholding (sorter: clusterless_thresholder)." + "While clusterless decoding avoids actual spike sorting, we need to pass through these tables to maintain (relative) pipeline simplicity. Pass-through tables keep spike sorting and clusterless waveform extraction as similar as possible, by using shared steps. Here, \"spike sorting\" involves simple thresholding (sorter: clusterless_thresholder).\n" ] }, { @@ -45,7 +45,7 @@ "source": [ "First, if you haven't inserted the the `mediumnwb20230802.wnb` file into the database, you should do so now. This is the file that we will use for the decoding tutorials.\n", "\n", - "It is a truncated version of the full NWB file, so it will run faster, but bigger than the minirec file we used in the previous tutorials so that decoding makes sense." + "It is a truncated version of the full NWB file, so it will run faster, but bigger than the minirec file we used in the previous tutorials so that decoding makes sense.\n" ] }, { @@ -99,7 +99,7 @@ "\n", "We first set the `SortGroup` to define which contacts are sorted together.\n", "\n", - "We then setup for spike sorting by bandpass filtering and whitening the data via the `SpikeSortingRecording` table." + "We then setup for spike sorting by bandpass filtering and whitening the data via the `SpikeSortingRecording` table.\n" ] }, { @@ -166,7 +166,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Next we do artifact detection. Here we skip it by setting the `artifact_param_name` to `None`, but in practice you should detect artifacts as it will affect the decoding." + "Next we do artifact detection. Here we skip it by setting the `artifact_param_name` to `None`, but in practice you should detect artifacts as it will affect the decoding.\n" ] }, { @@ -226,7 +226,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now we run the \"spike sorting\", which in our case is simply thresholding the signal to find spikes. We use the `SpikeSorting` table to store the results. Note that `sorter_param_name` defines the parameters for thresholding the signal." + "Now we run the \"spike sorting\", which in our case is simply thresholding the signal to find spikes. We use the `SpikeSorting` table to store the results. Note that `sorter_param_name` defines the parameters for thresholding the signal.\n" ] }, { @@ -289,7 +289,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For clusterless decoding we do not need any manual curation, but for the sake of the pipeline, we need to store the output of the thresholding in the `CurationV1` table and insert this into the `SpikeSortingOutput` table." + "For clusterless decoding we do not need any manual curation, but for the sake of the pipeline, we need to store the output of the thresholding in the `CurationV1` table and insert this into the `SpikeSortingOutput` table.\n" ] }, { @@ -324,6 +324,7 @@ "Finally, we extract the waveform features of each SortGroup. This is done by the `UnitWaveformFeatures` table.\n", "\n", "To set this up, we use the `WaveformFeaturesParams` to define the time around the spike that we want to use for feature extraction, and which features to extract. Here is an example of the parameters used for extraction the amplitude of the negative peak of the waveform:\n", + "\n", "```python\n", "\n", "waveform_extraction_params = {\n", @@ -343,8 +344,8 @@ "\n", "We see that we want 0.5 ms of time before and after the peak of the negative spike. We also see that we want to extract the amplitude of the negative peak, and that we do not want to estimate the peak time (since we know it is at 0 ms).\n", "\n", - "\n", "You can define other features to extract such as spatial location of the spike:\n", + "\n", "```python\n", "waveform_extraction_params = {\n", " \"ms_before\": 0.5,\n", @@ -361,7 +362,10 @@ " \"spike location\": {}\n", "}\n", "\n", - "```\n" + "```\n", + "\n", + "_Note_: Members of the Frank Lab can use \"ampl_10_jobs_v2\" instead of \"amplitude\"\n", + "for significant speed improvements.\n" ] }, { @@ -490,7 +494,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now that we've inserted the waveform features parameters, we need to define which parameters to use for each SortGroup. This is done by the `UnitWaveformFeaturesSelection` table. We need to link the primary key `merge_id` from the `SpikeSortingOutput` table to a features parameter set." + "Now that we've inserted the waveform features parameters, we need to define which parameters to use for each SortGroup. This is done by the `UnitWaveformFeaturesSelection` table. We need to link the primary key `merge_id` from the `SpikeSortingOutput` table to a features parameter set.\n" ] }, { @@ -615,7 +619,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "First we find the units we need:" + "First we find the units we need:\n" ] }, { @@ -674,7 +678,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Then we link them with the features parameters:" + "Then we link them with the features parameters:\n" ] }, { @@ -806,7 +810,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Finally, we extract the waveform features, by populating the `UnitWaveformFeatures` table:" + "Finally, we extract the waveform features, by populating the `UnitWaveformFeatures` table:\n" ] }, { @@ -1621,7 +1625,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now that we've extracted the data, we can inspect the results. Let's fetch the data:" + "Now that we've extracted the data, we can inspect the results. Let's fetch the data:\n" ] }, { @@ -1669,7 +1673,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Let's look at the features shape. This is a list corresponding to tetrodes, with each element being a numpy array of shape (n_spikes, n_features). The features in this case are the amplitude of each tetrode wire at the negative peak of the waveform." + "Let's look at the features shape. This is a list corresponding to tetrodes, with each element being a numpy array of shape (n_spikes, n_features). The features in this case are the amplitude of each tetrode wire at the negative peak of the waveform.\n" ] }, { @@ -1716,7 +1720,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can plot the amplitudes to see if there is anything that looks neural and to look for outliers:" + "We can plot the amplitudes to see if there is anything that looks neural and to look for outliers:\n" ] }, { diff --git a/notebooks/43_Decoding_SortedSpikes.ipynb b/notebooks/43_Decoding_SortedSpikes.ipynb index 9bbd01592..dc0cc9b61 100644 --- a/notebooks/43_Decoding_SortedSpikes.ipynb +++ b/notebooks/43_Decoding_SortedSpikes.ipynb @@ -163,12 +163,17 @@ ], "source": [ "from spyglass.spikesorting.analysis.v1.group import UnitSelectionParams\n", + "\n", "UnitSelectionParams().insert_default()\n", "\n", "# look at the filter set we'll use here\n", "unit_filter_params_name = \"default_exclusion\"\n", - "print((UnitSelectionParams()\n", - " & {\"unit_filter_params_name\":unit_filter_params_name}).fetch1())\n", + "print(\n", + " (\n", + " UnitSelectionParams()\n", + " & {\"unit_filter_params_name\": unit_filter_params_name}\n", + " ).fetch1()\n", + ")\n", "# look at full table\n", "UnitSelectionParams()" ] @@ -318,7 +323,7 @@ "sorter_keys = {\n", " \"nwb_file_name\": nwb_copy_file_name,\n", " \"sorter\": \"mountainsort4\",\n", - " \"curation_id\": 1\n", + " \"curation_id\": 1,\n", "}\n", "# check the set of sorting we'll use\n", "(sgs.SpikeSortingSelection & sorter_keys) * SpikeSortingOutput.CurationV1" @@ -846,6 +851,7 @@ "}\n", "\n", "from spyglass.decoding import SortedSpikesDecodingSelection\n", + "\n", "SortedSpikesDecodingSelection.insert1(\n", " selection_key,\n", " skip_duplicates=True,\n", @@ -1502,7 +1508,6 @@ } ], "source": [ - "\n", "results = (SortedSpikesDecodingV1 & selection_key).fetch_results()\n", "results" ] diff --git a/notebooks/py_scripts/01_Insert_Data.py b/notebooks/py_scripts/01_Insert_Data.py index c1fec99a9..975ed4ac5 100644 --- a/notebooks/py_scripts/01_Insert_Data.py +++ b/notebooks/py_scripts/01_Insert_Data.py @@ -171,8 +171,9 @@ # privileges. # +team_name = "My Team" sgc.LabTeam().create_new_team( - team_name="My Team", # Should be unique + team_name=team_name, # Should be unique team_members=["Firstname Lastname", "Firstname2 Lastname2"], team_description="test", # Optional ) @@ -377,11 +378,45 @@ session_entry = sgc.Session & {"nwb_file_name": nwb_copy_file_name} session_entry +# `Session.Experimenter` is used for permissions checks when deleting. The +# session will need to have an experimenter in order avoid an error being +# raised during this check. +# + +sess_key = (sgc.Session & {"nwb_file_name": nwb_copy_file_name}).fetch( + "KEY", as_dict=True +)[0] +exp_key = (sgc.LabMember).fetch("KEY", as_dict=True)[0] +sgc.Session.Experimenter.insert1( + dict(**sess_key, **exp_key), skip_duplicates=True +) + +# Even with the experimenter specified, there are still delete protections +# in place. To see an example, uncomment the cell below. +# + +# + +# session_entry.delete() +# - + +# To delete, you'll need to share a team with the session experimenter. +# + +your_name = "YourFirst YourLast" +parts = your_name.split(" ") +sgc.LabMember.insert1([your_name, parts[0], parts[1]]) +sgc.LabMember.LabMemberInfo.insert1( + [your_name, "your_gmail", dj.config["database.user"], 0] +) +sgc.LabTeam.LabTeamMember.insert1([team_name, your_name]) + # By default, DataJoint is cautious about deletes and will prompt before deleting. # To delete, uncomment the cell below and respond `yes` in the prompt. # -session_entry.delete() +# + +# session_entry.delete() +# - # We can check that delete worked, both for `Session` and `IntervalList` # diff --git a/notebooks/py_scripts/02_Data_Sync.py b/notebooks/py_scripts/02_Data_Sync.py index 0ffe123c1..3902f9f73 100644 --- a/notebooks/py_scripts/02_Data_Sync.py +++ b/notebooks/py_scripts/02_Data_Sync.py @@ -22,9 +22,9 @@ # # 1. [General Kachery information](#kachery) # 2. Setting up Kachery as a [host](#host-setup). If you'll use an existing host, -# skip this. +# skip this. # 3. Setting up Kachery in your [database](#database-setup). If you're using an -# existing database, skip this. +# existing database, skip this. # 4. Adding Kachery [data](#data-setup). # @@ -46,14 +46,14 @@ # [these additional tutorials](https://github.com/datajoint/datajoint-tutorials) # # Let's start by importing the `spyglass` package and testing that your environment -# is properly configured for kachery sharing +# is properly configured for kachery sharing # # If you haven't already done so, be sure to set up your Spyglass base directory and Kachery sharing directory with [Setup](./00_Setup.ipynb) +# # + import os import datajoint as dj -import pandas as pd # change to the upper level folder to detect dj_local_conf.json if os.path.basename(os.getcwd()) == "notebooks": @@ -101,7 +101,9 @@ # ) # lfp.v1.LFPV1().populate() # ``` +# # +# # ## Kachery # @@ -140,27 +142,29 @@ # # - Environment variable: # -# ```bash -# export KACHERY_ZONE=franklab.default -# export KACHERY_CLOUD_DIR=/stelmo/nwb/.kachery-cloud -# ``` +# ```bash +# export KACHERY_ZONE=franklab.default +# export KACHERY_CLOUD_DIR=/stelmo/nwb/.kachery-cloud +# ``` # # - DataJoint Config: # -# ```json -# "custom": { -# "kachery_zone": "franklab.default", -# "kachery_dirs": { -# "cloud": "/your/base/path/.kachery-cloud" -# } -# } -# ``` +# ```json +# "custom": { +# "kachery_zone": "franklab.default", +# "kachery_dirs": { +# "cloud": "/your/base/path/.kachery-cloud" +# } +# } +# ``` +# # ## Host Setup # # - If you are a member of a team with a pre-existing database and zone who will be sharing data, please skip to `Sharing Data` # # - If you are a collaborator outside your team's network and need to access files shared with you, please skip to `Accessing Shared Data` +# # ### Zones # @@ -175,10 +179,11 @@ # - Bucket names cannot include periods, so we substitute a dash, as in # `franklab-default`. # - You only need to create an API token for your first zone. +# # ### Resources - # + # See [instructions](https://github.com/scratchrealm/kachery-resource/blob/main/README.md) # for setting up zone resources. This allows for sharing files on demand. We # suggest using the same name for the zone and resource. @@ -197,6 +202,7 @@ # # - run_restart_kachery_collab.sh # - run_restart_kachery_default.sh +# # ## Database Setup # @@ -204,12 +210,14 @@ # Once you have a hosted zone running, we need to add its information to the Spyglass database. # This will allow spyglass to manage linking files from our analysis tables to kachery. # First, we'll check existing Zones. +# sgs.KacheryZone() # To add a new hosted Zone, we need to prepare an entry for the `KacheryZone` table. # Note that the `kacherycloud_dir` key should be the path for the server daemon _hosting_ the zone, -# and is not required to be present on the client machine: +# and is not required to be present on the client machine: +# # + zone_name = config.get("KACHERY_ZONE") @@ -226,15 +234,18 @@ # Use caution when inserting into an active database, as it could interfere with # ongoing work. +# -sgs.KacheryZone().insert1(zone_key) +sgs.KacheryZone().insert1(zone_key, skip_duplicates=True) # ## Sharing Data +# # Once the zone exists, we can add `AnalysisNWB` files we want to share with members of the zone. # # The `AnalysisNwbFileKachery` table links analysis files made within other spyglass tables with a `uri` # used by kachery. We can view files already made available through kachery here: +# sgs.AnalysisNwbfileKachery() @@ -244,6 +255,7 @@ # # _Note:_ This step depends on having previously run an analysis on the example # file. +# # + nwb_copy_filename = "minirec20230622_.nwb" @@ -263,13 +275,15 @@ # With those files in the selection table, we can add them as links to the zone by # populating the `AnalysisNwbfileKachery` table: +# sgs.AnalysisNwbfileKachery.populate() # Alternatively, we can share data based on its source table in the database using the helper function `share_data_to_kachery()` # # This will take a list of tables and add all associated analysis files for entries corresponding with a passed restriction. -# Here, we are sharing LFP and position data for the Session "minirec20230622_.nwb" +# Here, we are sharing LFP and position data for the Session "minirec20230622\_.nwb" +# # + from spyglass.sharing import share_data_to_kachery @@ -281,20 +295,21 @@ share_data_to_kachery( table_list=tables, restriction=restriction, - zone_name="franklab.collaborators", + zone_name=zone_name, ) # - # ## Managing access +# # + [markdown] jupyter={"outputs_hidden": true} # If all of that worked, # # 1. Go to https://kachery-gateway.figurl.org/admin?zone=your_zone -# (changing your_zone to the name of your zone) +# (changing your_zone to the name of your zone) # 2. Go to the Admin/Authorization Settings tab # 3. Add the GitHub login names and permissions for the users you want to share -# with. +# with. # # If those users can connect to your database, they should now be able to use the # `.fetch_nwb()` method to download any `AnalysisNwbfiles` that have been shared @@ -310,13 +325,16 @@ # ).fetch()[0] # sort = (CuratedSpikeSorting & test_sort).fetch_nwb() # ``` +# # - # ## Accessing Shared Data +# # If you are a collaborator accessing datasets, you first need to be given access to the zone by a collaborator admin (see above). # # If you know the uri for the dataset you are accessing you can test this process below (example is for members of `franklab.collaborators`) +# # + import kachery_cloud as kcl @@ -332,11 +350,12 @@ # In normal use, spyglass will manage setting the zone and uri when accessing files. # In general, the easiest way to access data valueswill be through the `fetch1_dataframe()` -# function part of many of the spyglass tables. In brief this will check for the appropriate +# function part of many of the spyglass tables. In brief this will check for the appropriate # nwb analysis file in your local directory, and if not found, attempt to download it from the appropriate kachery zone. # It will then parse the relevant information from that nwb file into a pandas dataframe. # # We will look at an example with data from the `LFPV1` table: +# # + from spyglass.lfp.v1 import LFPV1 @@ -349,6 +368,7 @@ # - # We can access the data using `fetch1_dataframe()` +# ( LFPV1 @@ -359,6 +379,8 @@ ).fetch1_dataframe() # # Up Next +# # In the [next notebook](./03_Merge_Tables.ipynb), we'll explore the details of a # table tier unique to Spyglass, Merge Tables. +# diff --git a/notebooks/py_scripts/11_CurationV0.py b/notebooks/py_scripts/11_CurationV0.py index f728512a2..80e8cdc16 100644 --- a/notebooks/py_scripts/11_CurationV0.py +++ b/notebooks/py_scripts/11_CurationV0.py @@ -27,7 +27,7 @@ # [this notebook](./00_Setup.ipynb) # - For a more detailed introduction to DataJoint with inserts, see # [this notebook](./01_Insert_Data.ipynb) -# - [The Spike Sorting notebook](./02_Spike_Sorting.ipynb) is a mandatory +# - [The Spike Sorting notebook](./10_Spike_SortingV0.ipynb) is a mandatory # prerequisite to Curation. # @@ -109,3 +109,4 @@ # ## Up Next # # Next, we'll turn our attention to [LFP data](./12_LFP.ipynb) data. +# diff --git a/notebooks/py_scripts/21_DLC.py b/notebooks/py_scripts/21_DLC.py index 996dd1764..63ffe4d0c 100644 --- a/notebooks/py_scripts/21_DLC.py +++ b/notebooks/py_scripts/21_DLC.py @@ -211,7 +211,7 @@ # - Number of frames to extract/label as `frames_per_video`. Note that the DLC creators recommend having 200 frames as the minimum total number for each project. # -team_name = "LorenLab" +team_name = sgc.LabTeam.fetch("team_name")[0] # If on lab DB, "LorenLab" project_name = "tutorial_scratch_DG" frames_per_video = 100 bodyparts = ["redLED_C", "greenLED", "redLED_L", "redLED_R", "tailBase"] diff --git a/notebooks/py_scripts/21_Position_DLC_1.py b/notebooks/py_scripts/21_Position_DLC_1.py deleted file mode 100644 index cb89065d1..000000000 --- a/notebooks/py_scripts/21_Position_DLC_1.py +++ /dev/null @@ -1,393 +0,0 @@ -# --- -# jupyter: -# jupytext: -# text_representation: -# extension: .py -# format_name: light -# format_version: '1.5' -# jupytext_version: 1.16.0 -# kernelspec: -# display_name: Python 3 (ipykernel) -# language: python -# name: python3 -# --- - -# # Position - DeepLabCut from Scratch -# - -# ### Overview -# - -# _Developer Note:_ if you may make a PR in the future, be sure to copy this -# notebook, and use the `gitignore` prefix `temp` to avoid future conflicts. -# -# This is one notebook in a multi-part series on Spyglass. -# -# - To set up your Spyglass environment and database, see -# [the Setup notebook](./00_Setup.ipynb) -# - For additional info on DataJoint syntax, including table definitions and -# inserts, see -# [the Insert Data notebook](./01_Insert_Data.ipynb) -# -# This tutorial will extract position via DeepLabCut (DLC). It will walk through... -# -# - creating a DLC project -# - extracting and labeling frames -# - training your model -# -# If you have a pre-trained project, you can either skip to the -# [next tutorial](./22_Position_DLC_2.ipynb) to load it into the database, or skip -# to the [following tutorial](./23_Position_DLC_3.ipynb) to start pose estimation -# with a model that is already inserted. -# - -# Here is a schematic showing the tables used in this pipeline. -# -# ![dlc_scratch.png|2000x900](./../notebook-images/dlc_scratch.png) -# - -# ### Table of Contents -# - -# - [Imports](#imports) -# - [`DLCProject`](#DLCProject1) -# - [`DLCModelTraining`](#DLCModelTraining1) -# - [`DLCModel`](#DLCModel1) -# -# **You can click on any header to return to the Table of Contents** -# - -# ### Imports -# - -# + -import os -import datajoint as dj -from pprint import pprint - -import spyglass.common as sgc -import spyglass.position.v1 as sgp - -# change to the upper level folder to detect dj_local_conf.json -if os.path.basename(os.getcwd()) == "notebooks": - os.chdir("..") -dj.config.load("dj_local_conf.json") # load config for database connection info - -# ignore datajoint+jupyter async warnings -import warnings - -warnings.simplefilter("ignore", category=DeprecationWarning) -warnings.simplefilter("ignore", category=ResourceWarning) -# - - -# #### [DLCProject](#TableOfContents) -# - -#
-# Notes:
    -#
  • -# The cells within this DLCProject step need to be performed -# in a local Jupyter notebook to allow for use of the frame labeling GUI -#
    • -#
  • -# Please do not add to the BodyPart table in the production -# database unless necessary. -#
    • -#
-#
-# - -# ### Body Parts -# - -# We'll begin by looking at the `BodyPart` table, which stores standard names of body parts used in DLC models throughout the lab with a concise description. -# - -# If the bodyparts you plan to use in your model are not yet in the table, here is code to add bodyparts: -# -# ```python -# sgp.BodyPart.insert( -# [ -# {"bodypart": "bp_1", "bodypart_description": "concise descrip"}, -# {"bodypart": "bp_2", "bodypart_description": "concise descrip"}, -# ], -# skip_duplicates=True, -# ) -# ``` -# - -# To train a model, we'll need to extract frames, which we can label as training data. We can construct a list of videos from which we'll extract frames. -# -# The list can either contain dictionaries identifying behavioral videos for NWB files that have already been added to Spyglass, or absolute file paths to the videos you want to use. -# -# For this tutorial, we'll use two videos for which we already have frames labeled. -# - -sgp.BodyPart() - -# ### Define camera name and videos for training set -# -# Defining camera name is optional: it should be done in cases where there are multiple cameras streaming per epoch, but not necessary otherwise. -# - -# example: -# `camera_name = "HomeBox_camera" -# ` -# - -# _NOTE:_ The official release of Spyglass does not yet support multicamera -# projects. You can monitor progress on the effort to add this feature by checking -# [this PR](https://github.com/LorenFrankLab/spyglass/pull/684) or use -# [this experimental branch](https://github.com/dpeg22/spyglass/tree/add-multi-camera), -# which only takes the keys nwb_file_name and epoch in the video_list variable. -# - -video_list = [ - {"nwb_file_name": "J1620210529_.nwb", "epoch": 2}, - {"nwb_file_name": "peanut20201103_.nwb", "epoch": 4}, -] - -# ### Path variables -# -# The position pipeline also keeps track of paths for project, video, and output. -# Just like we saw in [Setup](./00_Setup.ipynb), you can manage these either with -# environmental variables... -# -# ```bash -# export DLC_PROJECT_DIR="/nimbus/deeplabcut/projects" -# export DLC_VIDEO_DIR="/nimbus/deeplabcut/video" -# export DLC_OUTPUT_DIR="/nimbus/deeplabcut/output" -# ``` -# -# -# -# Or these can be set in your datajoint config: -# -# ```json -# { -# "custom": { -# "dlc_dirs": { -# "base": "/nimbus/deeplabcut/", -# "project": "/nimbus/deeplabcut/projects", -# "video": "/nimbus/deeplabcut/video", -# "output": "/nimbus/deeplabcut/output" -# } -# } -# } -# ``` -# -# _NOTE:_ If only `base` is specified as shown above, spyglass will assume the -# relative directories shown. -# -# You can check the result of this setup process with... -# - -# + -from spyglass.settings import config - -config -# - - -# _NOTE:_ The official release of Spyglass does not yet support master branch only takes the keys nwb_file_name and epoch in the video_list variable. EB is circumventing this by running this on daniel's (dpeg22) branch "add-multi-camera" -# - -# Before creating our project, we need to define a few variables. -# -# - A team name, as shown in `LabTeam` for setting permissions. Here, we'll -# use "LorenLab". -# - A `project_name`, as a unique identifier for this DLC project. Here, we'll use -# **"tutorial_scratch_yourinitials"** -# - `bodyparts` is a list of body parts for which we want to extract position. -# The pre-labeled frames we're using include the bodyparts listed below. -# - Number of frames to extract/label as `frames_per_video`. A true project might -# use 200, but we'll use 100 for efficiency. -# - -team_name = "LorenLab" -project_name = "tutorial_scratch_DG" -frames_per_video = 100 -bodyparts = ["redLED_C", "greenLED", "redLED_L", "redLED_R", "tailBase"] -project_key = sgp.DLCProject.insert_new_project( - project_name=project_name, - bodyparts=bodyparts, - lab_team=team_name, - frames_per_video=frames_per_video, - video_list=video_list, - skip_duplicates=True, -) - -# After initializing our project, we would typically extract and label frames. Use the following commands to pull up the DLC GUI: -# - -sgp.DLCProject().run_extract_frames(project_key) -sgp.DLCProject().run_label_frames(project_key) - -# In order to use pre-labeled frames, you'll need to change the values in the -# labeled-data files. You can do that using the `import_labeled_frames` method, -# which expects: -# -# - `project_key` from your new project. -# - The absolute path to the project directory from which we'll import labeled -# frames. -# - The filenames, without extension, of the videos from which we want frames. -# - -sgp.DLCProject.import_labeled_frames( - project_key.copy(), - import_project_path="/nimbus/deeplabcut/projects/tutorial_model-LorenLab-2022-07-15/", - video_filenames=["20201103_peanut_04_r2", "20210529_J16_02_r1"], - skip_duplicates=True, -) - -#
-# This step and beyond should be run on a GPU-enabled machine. -#
-# - -# #### [DLCModelTraining](#ToC) -# -# Please make sure you're running this notebook on a GPU-enabled machine. -# -# Now that we've imported existing frames, we can get ready to train our model. -# -# First, we'll need to define a set of parameters for `DLCModelTrainingParams`, which will get used by DeepLabCut during training. Let's start with `gputouse`, -# which determines which GPU core to use. -# -# The cell below determines which core has space and set the `gputouse` variable -# accordingly. -# - -sgp.dlc_utils.get_gpu_memory() - -# Set GPU core: -# - -gputouse = 1 ## 1-9 - -# Now we'll define the rest of our parameters and insert the entry. -# -# To see all possible parameters, try: -# -# ```python -# sgp.DLCModelTrainingParams.get_accepted_params() -# ``` -# - -training_params_name = "tutorial" -sgp.DLCModelTrainingParams.insert_new_params( - paramset_name=training_params_name, - params={ - "trainingsetindex": 0, - "shuffle": 1, - "gputouse": gputouse, - "net_type": "resnet_50", - "augmenter_type": "imgaug", - }, - skip_duplicates=True, -) - -# Next we'll modify the `project_key` to include the entries for -# `DLCModelTraining` -# - -# project_key['project_path'] = os.path.dirname(project_key['config_path']) -if "config_path" in project_key: - del project_key["config_path"] - -# We can insert an entry into `DLCModelTrainingSelection` and populate `DLCModelTraining`. -# -# _Note:_ You can stop training at any point using `I + I` or interrupt the Kernel -# - -sgp.DLCModelTrainingSelection.heading - -# + jupyter={"outputs_hidden": true} -sgp.DLCModelTrainingSelection().insert1( - { - **project_key, - "dlc_training_params_name": training_params_name, - "training_id": 0, - "model_prefix": "", - } -) -model_training_key = ( - sgp.DLCModelTrainingSelection - & { - **project_key, - "dlc_training_params_name": training_params_name, - } -).fetch1("KEY") -sgp.DLCModelTraining.populate(model_training_key) -# - - -# Here we'll make sure that the entry made it into the table properly! -# - -sgp.DLCModelTraining() & model_training_key - -# Populating `DLCModelTraining` automatically inserts the entry into -# `DLCModelSource`, which is used to select between models trained using Spyglass -# vs. other tools. -# - -sgp.DLCModelSource() & model_training_key - -# The `source` field will only accept _"FromImport"_ or _"FromUpstream"_ as entries. Let's checkout the `FromUpstream` part table attached to `DLCModelSource` below. -# - -sgp.DLCModelSource.FromUpstream() & model_training_key - -# #### [DLCModel](#TableOfContents) -# -# Next we'll populate the `DLCModel` table, which holds all the relevant -# information for all trained models. -# -# First, we'll need to determine a set of parameters for our model to select the -# correct model file. Here is the default: -# - -pprint(sgp.DLCModelParams.get_default()) - -# Here is the syntax to add your own parameter set: -# -# ```python -# dlc_model_params_name = "make_this_yours" -# params = { -# "params": {}, -# "shuffle": 1, -# "trainingsetindex": 0, -# "model_prefix": "", -# } -# sgp.DLCModelParams.insert1( -# {"dlc_model_params_name": dlc_model_params_name, "params": params}, -# skip_duplicates=True, -# ) -# ``` -# - -# We can insert sets of parameters into `DLCModelSelection` and populate -# `DLCModel`. -# - -temp_model_key = (sgp.DLCModelSource & model_training_key).fetch1("KEY") -sgp.DLCModelSelection().insert1( - {**temp_model_key, "dlc_model_params_name": "default"}, skip_duplicates=True -) -model_key = (sgp.DLCModelSelection & temp_model_key).fetch1("KEY") -sgp.DLCModel.populate(model_key) - -# Again, let's make sure that everything looks correct in `DLCModel`. -# - -sgp.DLCModel() & model_key - -# ### Next Steps -# -# With our trained model in place, we're ready to move on to pose estimation -# (notebook coming soon!). -# -# -# - -# ### [Return To Table of Contents](#TableOfContents)
-# diff --git a/notebooks/py_scripts/22_DLC_Loop.py b/notebooks/py_scripts/22_DLC_Loop.py index b37e1c6ba..23c493263 100644 --- a/notebooks/py_scripts/22_DLC_Loop.py +++ b/notebooks/py_scripts/22_DLC_Loop.py @@ -13,8 +13,10 @@ # --- # ## Position- DeepLabCut from Scratch +# # ### Overview +# # _Developer Note:_ if you may make a PR in the future, be sure to copy this # notebook, and use the `gitignore` prefix `temp` to avoid future conflicts. @@ -37,6 +39,7 @@ # - inserting the resulting information into the `PositionOutput` table # # **Note 2: Make sure you are running this within the spyglass-position Conda environment (instructions for install are in the environment_position.yml)** +# # Here is a schematic showing the tables used in this pipeline. # @@ -44,6 +47,7 @@ # # ### Table of Contents +# # [`DLCProject`](#DLCProject1)
# [`DLCModelTraining`](#DLCModelTraining1)
# [`DLCModel`](#DLCModel1)
@@ -54,10 +58,13 @@ # [`DLCPosV1`](#DLCPosV1-1)
# [`DLCPosVideo`](#DLCPosVideo1)
# [`PositionOutput`](#PositionOutput1)
+# -# __You can click on any header to return to the Table of Contents__ +# **You can click on any header to return to the Table of Contents** +# # ### Imports +# # %load_ext autoreload # %autoreload 2 @@ -87,6 +94,7 @@ # - # #### [DLCProject](#TableOfContents) +# #
# Notes:
    @@ -103,8 +111,10 @@ # # ### Body Parts +# # We'll begin by looking at the `BodyPart` table, which stores standard names of body parts used in DLC models throughout the lab with a concise description. +# sgp.BodyPart() @@ -119,19 +129,23 @@ # skip_duplicates=True, # ) # ``` +# # ### Define videos and camera name (optional) for training set +# # To train a model, we'll need to extract frames, which we can label as training data. We can construct a list of videos from which we'll extract frames. # # The list can either contain dictionaries identifying behavioral videos for NWB files that have already been added to Spyglass, or absolute file paths to the videos you want to use. # # For this tutorial, we'll use two videos for which we already have frames labeled. +# # Defining camera name is optional: it should be done in cases where there are multiple cameras streaming per epoch, but not necessary otherwise.
    # example: # `camera_name = "HomeBox_camera" # ` +# # _NOTE:_ The official release of Spyglass does not yet support multicamera # projects. You can monitor progress on the effort to add this feature by checking @@ -178,6 +192,7 @@ # relative directories shown. # # You can check the result of this setup process with... +# # + from spyglass.settings import config @@ -194,8 +209,9 @@ # - `bodyparts` is a list of body parts for which we want to extract position. # The pre-labeled frames we're using include the bodyparts listed below. # - Number of frames to extract/label as `frames_per_video`. Note that the DLC creators recommend having 200 frames as the minimum total number for each project. +# -team_name = "LorenLab" +team_name = sgc.LabTeam.fetch("team_name")[0] # If on lab DB, "LorenLab" project_name = "tutorial_scratch_DG" frames_per_video = 100 bodyparts = ["redLED_C", "greenLED", "redLED_L", "redLED_R", "tailBase"] @@ -209,35 +225,42 @@ ) # Now that we've initialized our project we'll need to extract frames which we will then label. +# # comment this line out after you finish frame extraction for each project sgp.DLCProject().run_extract_frames(project_key) # This is the line used to label the frames you extracted, if you wish to use the DLC GUI on the computer you are currently using. +# # ```#comment this line out after frames are labeled for your project # sgp.DLCProject().run_label_frames(project_key) # ``` +# # Otherwise, it is best/easiest practice to label the frames on your local computer (like a MacBook) that can run DeepLabCut's GUI well. Instructions:
    +# # 1. Install DLC on your local (preferably into a 'Src' folder): https://deeplabcut.github.io/DeepLabCut/docs/installation.html # 2. Upload frames extracted and saved in nimbus (should be `/nimbus/deeplabcut//labeled-data`) AND the project's associated config file (should be `/nimbus/deeplabcut//config.yaml`) to Box (we get free with UCSF) # 3. Download labeled-data and config files on your local from Box # 4. Create a 'projects' folder where you installed DeepLabCut; create a new folder with your complete project name there; save the downloaded files there. -# 4. Edit the config.yaml file: line 9 defining `project_path` needs to be the file path where it is saved on your local (ex: `/Users/lorenlab/Src/DeepLabCut/projects/tutorial_sratch_DG-LorenLab-2023-08-16`) -# 5. Open the DLC GUI through terminal -#
    (ex: `conda activate miniconda/envs/DEEPLABCUT_M1` -#
    `pythonw -m deeplabcut`) -# 6. Load an existing project; choose the config.yaml file -# 7. Label frames; labeling tutorial: https://www.youtube.com/watch?v=hsA9IB5r73E. -# 8. Once all frames are labeled, you should re-upload labeled-data folder back to Box and overwrite it in the original nimbus location so that your completed frames are ready to be used in the model. +# 5. Edit the config.yaml file: line 9 defining `project_path` needs to be the file path where it is saved on your local (ex: `/Users/lorenlab/Src/DeepLabCut/projects/tutorial_sratch_DG-LorenLab-2023-08-16`) +# 6. Open the DLC GUI through terminal +#
    (ex: `conda activate miniconda/envs/DEEPLABCUT_M1` +#
    `pythonw -m deeplabcut`) +# 7. Load an existing project; choose the config.yaml file +# 8. Label frames; labeling tutorial: https://www.youtube.com/watch?v=hsA9IB5r73E. +# 9. Once all frames are labeled, you should re-upload labeled-data folder back to Box and overwrite it in the original nimbus location so that your completed frames are ready to be used in the model. +# # Now we can check the `DLCProject.File` part table and see all of our training files and videos there! +# sgp.DLCProject.File & project_key #
    # This step and beyond should be run on a GPU-enabled machine. #
    +# # #### [DLCModelTraining](#ToC) # @@ -282,6 +305,7 @@ ) # Next we'll modify the `project_key` from above to include the necessary entries for `DLCModelTraining` +# # project_key['project_path'] = os.path.dirname(project_key['config_path']) if "config_path" in project_key: @@ -314,16 +338,19 @@ sgp.DLCModelTraining.populate(model_training_key) # Here we'll make sure that the entry made it into the table properly! +# sgp.DLCModelTraining() & model_training_key # Populating `DLCModelTraining` automatically inserts the entry into # `DLCModelSource`, which is used to select between models trained using Spyglass # vs. other tools. +# sgp.DLCModelSource() & model_training_key # The `source` field will only accept _"FromImport"_ or _"FromUpstream"_ as entries. Let's checkout the `FromUpstream` part table attached to `DLCModelSource` below. +# sgp.DLCModelSource.FromUpstream() & model_training_key @@ -334,6 +361,7 @@ # # First, we'll need to determine a set of parameters for our model to select the # correct model file. Here is the default: +# sgp.DLCModelParams.get_default() @@ -356,6 +384,7 @@ # We can insert sets of parameters into `DLCModelSelection` and populate # `DLCModel`. +# temp_model_key = (sgp.DLCModelSource & model_training_key).fetch1("KEY") @@ -368,12 +397,15 @@ sgp.DLCModel.populate(model_key) # Again, let's make sure that everything looks correct in `DLCModel`. +# sgp.DLCModel() & model_key # ## Loop Begins +# # We can view all `VideoFile` entries with the specidied `camera_ name` for this project to ensure the rat whose position you wish to model is in this table `matching_rows` +# camera_name = "SleepBox_camera" matching_rows = sgc.VideoFile() & {"camera_name": camera_name} @@ -382,10 +414,12 @@ # The `DLCPoseEstimationSelection` insertion step will convert your .h264 video to an .mp4 first and save it in `/nimbus/deeplabcut/video`. If this video already exists here, the insertion will never complete. # # We first delete any .mp4 that exists for this video from the nimbus folder: +# # ! find /nimbus/deeplabcut/video -type f -name '*20230606_SC38*' -delete # change based on date and rat with which you are training the model # If the first insertion step (for pose estimation task) fails in either trigger or load mode for an epoch, run the following lines: +# # ``` # (pose_estimation_key = sgp.DLCPoseEstimationSelection.insert_estimation_task( # { @@ -395,13 +429,15 @@ # **model_key, # }).delete() # ``` +# -# This loop will generate posiiton data for all epochs associated with the pre-defined camera in one day, for one rat (based on the NWB file; see ***) +# This loop will generate posiiton data for all epochs associated with the pre-defined camera in one day, for one rat (based on the NWB file; see \*\*\*) #
    The output should print Pose Estimation and Centroid plots for each epoch. # # - It defines `col1val` as each `nwb_file_name` entry in the table, one at a time. # - Next, it sees if the trial on which you are testing this model is in the string for the current `col1val`; if not, it re-defines `col1val` as the next `nwb_file_name` entry and re-tries this step. # - If the previous step works, it then saves `col2val` and `col3val` as the `epoch` and the `video_file_num`, respectively, based on the nwb_file_name. From there, it iterates through the insertion and population steps required to extract position data, which we see laid out in notebook 05_DLC.ipynb. +# for row in matching_rows: col1val = row["nwb_file_name"] @@ -515,6 +551,9 @@ continue # ### _CONGRATULATIONS!!_ +# # Please treat yourself to a nice tea break :-) +# # ### [Return To Table of Contents](#TableOfContents)
    +# diff --git a/notebooks/py_scripts/24_Linearization.py b/notebooks/py_scripts/24_Linearization.py index 90ec20eae..ef7c67d77 100644 --- a/notebooks/py_scripts/24_Linearization.py +++ b/notebooks/py_scripts/24_Linearization.py @@ -75,7 +75,7 @@ # + from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename -nwb_file_name = "chimi20200216_new.nwb" +nwb_file_name = "minirec20230622.nwb" # detailed data: chimi20200216_new.nwb nwb_copy_file_name = get_nwb_copy_filename(nwb_file_name) nwb_copy_file_name # - @@ -89,8 +89,8 @@ pos_key = { "nwb_file_name": nwb_copy_file_name, - "interval_list_name": "pos 1 valid times", - "position_info_param_name": "default", + "interval_list_name": "pos 0 valid times", # For chimi, "pos 1 valid times" + "trodes_pos_params_name": "single_led_upsampled", # For chimi, "default" } # Note: You'll have to change the part table to the one where your data came from diff --git a/notebooks/py_scripts/30_LFP.py b/notebooks/py_scripts/30_LFP.py index 42d452a39..67cbc583a 100644 --- a/notebooks/py_scripts/30_LFP.py +++ b/notebooks/py_scripts/30_LFP.py @@ -295,7 +295,11 @@ # of the electrodes we band pass filtered # # _Note:_ Much of the code below could be replaced by a function calls that would -# return the data from each electrical series +# return the data from each electrical series. +# +# _Note:_ If you see an error `Qt: Session Management Error`, try running the +# following unix command: `export -n SESSION_MANAGER`. +# [See also](https://stackoverflow.com/questions/986964/qt-session-management-error) # orig_eseries = (sgc.Raw() & {"nwb_file_name": nwb_file_name}).fetch_nwb()[0][ @@ -306,7 +310,7 @@ ) orig_timestamps = np.asarray(orig_eseries.timestamps) -lfp_eseries = lfp.LFPOutput.fetch_nwb(lfp_key)[0]["lfp"] +lfp_eseries = lfp.LFPOutput().fetch_nwb(lfp_key)[0]["lfp"] lfp_elect_indices = sgc.get_electrode_indices( lfp_eseries, lfp_band_electrode_ids ) diff --git a/notebooks/py_scripts/32_Ripple_Detection.py b/notebooks/py_scripts/32_Ripple_Detection.py index 6e994bf30..a2b6be30d 100644 --- a/notebooks/py_scripts/32_Ripple_Detection.py +++ b/notebooks/py_scripts/32_Ripple_Detection.py @@ -30,6 +30,7 @@ # [the Insert Data notebook](./01_Insert_Data.ipynb) # # Ripple detection depends on a set of LFPs, the parameters used for detection and the speed of the animal. You will need `RippleLFPSelection`, `RippleParameters`, and `PositionOutput` to be populated accordingly. +# # ## Imports # @@ -66,6 +67,7 @@ # First, we'll pick the electrodes on which we'll run ripple detection on, using # `RippleLFPSelection.set_lfp_electrodes` +# # ?sgr.RippleLFPSelection.set_lfp_electrodes @@ -75,12 +77,17 @@ # alternatively use PFC. # - We use `nwb_file_name` to explore which electrodes are available for the # `electrode_list`. +# nwb_file_name = "tonks20211103_.nwb" interval_list_name = "test interval" filter_name = "Ripple 150-250 Hz" +if not sgc.Session & {"nwb_file_name": nwb_file_name}: + # This error will be raised when notebooks auto-run with 'minirec' + raise ValueError(f"Session with nwb_file_name={nwb_file_name} not found") # Now we can look at `electrode_id` in the `Electrode` table: +# electrodes = ( (sgc.Electrode() & {"nwb_file_name": nwb_file_name}) @@ -97,6 +104,7 @@ electrodes # For ripple detection, we want only tetrodes, and only the first good wire on each tetrode. We will assume that is the first wire on each tetrode. I will do this using pandas syntax but you could use datajoint to filter this table as well. Here is the filtered table. +# hpc_names = ["ca1", "hippocampus", "CA1", "Hippocampus"] electrodes.loc[ @@ -104,6 +112,7 @@ ] # We only want the electrode_id to put in the `electrode_list`: +# # + electrode_list = np.unique( @@ -125,6 +134,7 @@ # We can insert into `RippleLFPSelection` and the `RippleLFPElectrode` part table, # passing the key for the entry from `LFPBandV1`, our `electrode_list`, and the # `group_name` into `set_lfp_electrodes` +# # + group_name = "CA1_test" @@ -148,6 +158,7 @@ sgr.RippleLFPSelection.RippleLFPElectrode() # Here's the ripple selection key we'll use downstream +# rip_sel_key = (sgrip.RippleLFPSelection & lfp_band_key).fetch1("KEY") @@ -157,6 +168,7 @@ sgr.RippleParameters() # Here are the default ripple parameters: +# (sgrip.RippleParameters() & {"ripple_param_name": "default"}).fetch1() @@ -177,6 +189,7 @@ # # The speed for this interval should exist under the default position parameter # set and for a given interval. +# pos_key = sgp.PositionOutput.merge_get_part( { @@ -188,12 +201,13 @@ (sgp.PositionOutput & pos_key).fetch1_dataframe() # We'll use the `head_speed` above as part of `RippleParameters`. +# # ## Run Ripple Detection # -# # Now we can put everything together. +# key = { "ripple_param_name": "default", @@ -203,6 +217,7 @@ sgrip.RippleTimesV1().populate(key) # And then `fetch1_dataframe` for ripple times +# ripple_times = (sgrip.RippleTimesV1() & key).fetch1_dataframe() ripple_times @@ -210,3 +225,4 @@ # ## Up Next # # Next, we'll [extract mark indicator](./31_Extract_Mark_Indicators.ipynb). +# diff --git a/notebooks/py_scripts/41_Extracting_Clusterless_Waveform_Features.py b/notebooks/py_scripts/41_Extracting_Clusterless_Waveform_Features.py index 3880f94c4..a569bad36 100644 --- a/notebooks/py_scripts/41_Extracting_Clusterless_Waveform_Features.py +++ b/notebooks/py_scripts/41_Extracting_Clusterless_Waveform_Features.py @@ -31,6 +31,7 @@ # The goal of this notebook is to populate the `UnitWaveformFeatures` table, which depends `SpikeSortingOutput`. This table contains the features of the waveforms of each unit. # # While clusterless decoding avoids actual spike sorting, we need to pass through these tables to maintain (relative) pipeline simplicity. Pass-through tables keep spike sorting and clusterless waveform extraction as similar as possible, by using shared steps. Here, "spike sorting" involves simple thresholding (sorter: clusterless_thresholder). +# # + from pathlib import Path @@ -44,6 +45,7 @@ # First, if you haven't inserted the the `mediumnwb20230802.wnb` file into the database, you should do so now. This is the file that we will use for the decoding tutorials. # # It is a truncated version of the full NWB file, so it will run faster, but bigger than the minirec file we used in the previous tutorials so that decoding makes sense. +# # + from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename @@ -77,6 +79,7 @@ # We first set the `SortGroup` to define which contacts are sorted together. # # We then setup for spike sorting by bandpass filtering and whitening the data via the `SpikeSortingRecording` table. +# # + import spyglass.spikesorting.v1 as sgs @@ -103,6 +106,7 @@ # - # Next we do artifact detection. Here we skip it by setting the `artifact_param_name` to `None`, but in practice you should detect artifacts as it will affect the decoding. +# # + recording_ids = ( @@ -122,6 +126,7 @@ # - # Now we run the "spike sorting", which in our case is simply thresholding the signal to find spikes. We use the `SpikeSorting` table to store the results. Note that `sorter_param_name` defines the parameters for thresholding the signal. +# # + group_keys = [] @@ -144,6 +149,7 @@ # - # For clusterless decoding we do not need any manual curation, but for the sake of the pipeline, we need to store the output of the thresholding in the `CurationV1` table and insert this into the `SpikeSortingOutput` table. +# # + from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput @@ -155,7 +161,7 @@ for sorting_id in sorting_ids: try: sgs.CurationV1.insert_curation(sorting_id=sorting_id) - except KeyError as e: + except KeyError: pass SpikeSortingOutput.insert( @@ -168,6 +174,7 @@ # Finally, we extract the waveform features of each SortGroup. This is done by the `UnitWaveformFeatures` table. # # To set this up, we use the `WaveformFeaturesParams` to define the time around the spike that we want to use for feature extraction, and which features to extract. Here is an example of the parameters used for extraction the amplitude of the negative peak of the waveform: +# # ```python # # waveform_extraction_params = { @@ -187,8 +194,8 @@ # # We see that we want 0.5 ms of time before and after the peak of the negative spike. We also see that we want to extract the amplitude of the negative peak, and that we do not want to estimate the peak time (since we know it is at 0 ms). # -# # You can define other features to extract such as spatial location of the spike: +# # ```python # waveform_extraction_params = { # "ms_before": 0.5, @@ -207,6 +214,9 @@ # # ``` # +# _Note_: Members of the Frank Lab can use "ampl_10_jobs_v2" instead of "amplitude" +# for significant speed improvements. +# # + from spyglass.decoding.v1.waveform_features import WaveformFeaturesParams @@ -240,6 +250,7 @@ # - # Now that we've inserted the waveform features parameters, we need to define which parameters to use for each SortGroup. This is done by the `UnitWaveformFeaturesSelection` table. We need to link the primary key `merge_id` from the `SpikeSortingOutput` table to a features parameter set. +# # + from spyglass.decoding.v1.waveform_features import UnitWaveformFeaturesSelection @@ -248,6 +259,7 @@ # - # First we find the units we need: +# # + from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput @@ -264,6 +276,7 @@ # - # Then we link them with the features parameters: +# # + selection_keys = [ @@ -279,6 +292,7 @@ # - # Finally, we extract the waveform features, by populating the `UnitWaveformFeatures` table: +# # + from spyglass.decoding.v1.waveform_features import UnitWaveformFeatures @@ -289,17 +303,20 @@ UnitWaveformFeatures & selection_keys # Now that we've extracted the data, we can inspect the results. Let's fetch the data: +# spike_times, spike_waveform_features = ( UnitWaveformFeatures & selection_keys ).fetch_data() # Let's look at the features shape. This is a list corresponding to tetrodes, with each element being a numpy array of shape (n_spikes, n_features). The features in this case are the amplitude of each tetrode wire at the negative peak of the waveform. +# for features in spike_waveform_features: print(features.shape) # We can plot the amplitudes to see if there is anything that looks neural and to look for outliers: +# # + import matplotlib.pyplot as plt diff --git a/pyproject.toml b/pyproject.toml index fa50e8ddd..b5feb6950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,28 +35,28 @@ keywords = [ "sortingview", ] dependencies = [ - "pydotplus", + "black[jupyter]", + "bottleneck", "dask", - "position_tools", - "track_linearization>=2.3", - "non_local_detector", - "ripple_detection", + "datajoint>=0.13.6", + # "ghostipy", # removed from list bc M1 users need to install pyfftw first + "hdmf>=3.4.6", + "ipympl", "matplotlib", - "seaborn", - "bottleneck", + "ndx_franklab_novela>=0.1.0", + "non_local_detector", "numpy<1.24", - "ipympl", - "tqdm", - "pubnub<6.4.0", # TODO: remove this when sortingview is updated + "opencv-python", + "panel<=1.3.4", # See panel #6325 + "position_tools", + "pubnub<6.4.0", # TODO: remove this when sortingview is updated + "pydotplus", "pynwb>=2.2.0,<3", - "hdmf>=3.4.6", - "datajoint>=0.13.6", - "pymysql", + "ripple_detection", + "seaborn", "sortingview>=0.11", - "pyyaml", - "click", "spikeinterface>=0.98.2,<0.99", - "ndx_franklab_novela>=0.1.0", + "track_linearization>=2.3", ] dynamic = ["version"] @@ -68,17 +68,21 @@ spyglass_cli = "spyglass.cli:cli" "Bug Tracker" = "https://github.com/LorenFrankLab/spyglass/issues" [project.optional-dependencies] -position = ["ffmpeg", "numba>=0.54", "deeplabcut<2.3.0"] +dlc = ["ffmpeg", "numba>=0.54", "deeplabcut<2.3.0"] test = [ + "click", # for CLI subpackage only "docker", # for tests in a container - "pytest", # unit testing - "pytest-cov", # code coverage + "ghostipy", "kachery", # database access "kachery-client", - "kachery-cloud", + "kachery-cloud>=0.4.0", + "pre-commit", # linting + "pytest", # unit testing + "pytest-cov", # code coverage ] docs = [ "hatch", # Get version from env + "jupytext==1.16.0", # Convert notebooks to .py "mike", # Docs versioning "mkdocs", # Docs core "mkdocs-exclude", # Docs exclude files @@ -121,7 +125,7 @@ addopts = [ # "--pdb", # drop into debugger on failure "-p no:warnings", # "--no-teardown", # don't teardown the database after tests - "--quiet-spy", # don't show logging from spyglass + "--quiet-spy", # don't show logging from spyglass "--show-capture=no", "--pdbcls=IPython.terminal.debugger:TerminalPdb", # use ipython debugger "--cov=spyglass", @@ -150,3 +154,7 @@ omit = [ # which submodules have no tests "*/spikesorting/*", # "*/utils/*", ] + +[tool.ruff] # CB: Propose replacing flake8 with ruff to delete setup.cfg +line-length = 80 +ignore = ["F401" , "E402", "E501"] diff --git a/setup.cfg b/setup.cfg index bf615cc86..bbea83f2f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ -[flake8] +[flake8] # Planned for removal in favor of Ruff max-line-length = 80 max-complexity = 17 exclude = diff --git a/src/spyglass/cli/cli.py b/src/spyglass/cli/cli.py index 5ccc7aaee..a07919da2 100644 --- a/src/spyglass/cli/cli.py +++ b/src/spyglass/cli/cli.py @@ -1,8 +1,15 @@ from typing import Union -import click import yaml +try: + import click +except ImportError: + raise ImportError( + "spyglass.cli.cli requires the 'click' package. " + "You can install it with 'pip install click'." + ) + @click.group(help="Spyglass command-line client") def cli(): diff --git a/src/spyglass/common/common_lab.py b/src/spyglass/common/common_lab.py index ca9d4359a..a6a162b2b 100644 --- a/src/spyglass/common/common_lab.py +++ b/src/spyglass/common/common_lab.py @@ -262,6 +262,7 @@ def decompose_name(full_name: str) -> tuple: """Return the full, first and last name parts of a full name. Names capitalized. Decomposes either comma- or space-separated. + If name includes spaces, must use exactly one comma. Parameters ---------- @@ -272,16 +273,33 @@ def decompose_name(full_name: str) -> tuple: ------- tuple A tuple of the full, first, last name parts. + + Raises + ------ + ValueError + When full_name is in an unsupported format """ - if full_name.count(", ") != 1 and full_name.count(" ") != 1: + + full_trimmed = "" if not full_name else full_name.strip() + + delim = ( + ", " + if full_trimmed.count(", ") == 1 + else " " if full_trimmed.count(" ") == 1 else None + ) + + parts = full_trimmed.split(delim) + + if delim is None or len(parts) != 2: # catch unsupported format raise ValueError( - f"Names should be stored as 'last, first'. Skipping {full_name}" + f"Name has unsupported format for {full_name}. \n\t" + + "Must use exactly one comma+space (i.e., ', ') or space.\n\t" + + "And provide exactly two name parts.\n\t" + + "For example, 'First Last' or 'Last1 Last2, First1 First2' " + + "if name(s) includes spaces." ) - elif ", " in full_name: - last, first = full_name.title().split(", ") - else: - first, last = full_name.title().split(" ") + first, last = parts if delim == " " else parts[::-1] full = f"{first} {last}" return full, first, last diff --git a/src/spyglass/decoding/decoding_merge.py b/src/spyglass/decoding/decoding_merge.py index 1e15ba859..de890590f 100644 --- a/src/spyglass/decoding/decoding_merge.py +++ b/src/spyglass/decoding/decoding_merge.py @@ -1,4 +1,3 @@ -import inspect from itertools import chain from pathlib import Path diff --git a/src/spyglass/linearization/merge.py b/src/spyglass/linearization/merge.py index f6d0837b1..ee4feae78 100644 --- a/src/spyglass/linearization/merge.py +++ b/src/spyglass/linearization/merge.py @@ -1,8 +1,8 @@ import datajoint as dj -from spyglass.linearization.v0.main import ( - IntervalLinearizedPosition, -) # noqa F401 +from spyglass.linearization.v0.main import ( # noqa F401 + IntervalLinearizedPosition as LinearizedPositionV0, +) from spyglass.linearization.v1.main import LinearizedPositionV1 # noqa F401 from spyglass.utils import SpyglassMixin, _Merge @@ -21,7 +21,7 @@ class LinearizedPositionV0(SpyglassMixin, dj.Part): # noqa: F811 definition = """ -> LinearizedPositionOutput --- - -> IntervalLinearizedPosition + -> LinearizedPositionV0 """ class LinearizedPositionV1(SpyglassMixin, dj.Part): # noqa: F811 diff --git a/src/spyglass/linearization/v1/main.py b/src/spyglass/linearization/v1/main.py index 4c135e884..a45ffe03e 100644 --- a/src/spyglass/linearization/v1/main.py +++ b/src/spyglass/linearization/v1/main.py @@ -117,7 +117,7 @@ def make(self, key): orig_key = copy.deepcopy(key) logger.info(f"Computing linear position for: {key}") - position_nwb = PositionOutput.fetch_nwb( + position_nwb = PositionOutput().fetch_nwb( {"merge_id": key["pos_merge_id"]} )[0] key["analysis_file_name"] = AnalysisNwbfile().create( diff --git a/src/spyglass/position/v1/position_dlc_project.py b/src/spyglass/position/v1/position_dlc_project.py index 32615c0a5..0cdb89a8d 100644 --- a/src/spyglass/position/v1/position_dlc_project.py +++ b/src/spyglass/position/v1/position_dlc_project.py @@ -282,7 +282,14 @@ def insert_new_project( video_filename=video[1], )[0].as_posix() for video in videos_to_convert + if video[0] is not None ] + if len(videos) < 1: + raise ValueError( + f"no .mp4 videos found in {videos_to_convert[0][0]}" + + f" for key: {video_list[0]}" + ) + # If not dict, assume list of video file paths that may or may not need to be converted else: videos = [] diff --git a/src/spyglass/settings.py b/src/spyglass/settings.py index 007ec9160..d1753f6af 100644 --- a/src/spyglass/settings.py +++ b/src/spyglass/settings.py @@ -544,14 +544,20 @@ def dlc_output_dir(self) -> str: if sg_config.load_failed: # Failed to load logger.warning("Failed to load SpyglassConfig. Please set up config file.") config = {} # Let __intit__ fetch empty config for first time setup - config, prepopulate, test_mode, base_dir, raw_dir, analysis_dir = ( - {}, - False, - False, - None, - None, - None, - ) + prepopulate = False + test_mode = False + debug_mode = False + base_dir = None + raw_dir = None + recording_dir = None + temp_dir = None + analysis_dir = None + sorting_dir = None + waveform_dir = None + video_dir = None + dlc_project_dir = None + dlc_video_dir = None + dlc_output_dir = None else: config = sg_config.config base_dir = sg_config.base_dir diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index b748267ad..f15183dfe 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -658,6 +658,8 @@ def merge_get_parent( @property def source_class_dict(self) -> dict: + # NOTE: fails if table is aliased in dj.Part but not merge script + # i.e., must import aliased table as part name if not self._source_class_dict: module = getmodule(self) self._source_class_dict = { diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 7a69ba75e..0e18e3a5c 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -3,10 +3,12 @@ from typing import Dict, List, Union import datajoint as dj +from datajoint.errors import DataJointError from datajoint.expression import QueryExpression from datajoint.logging import logger as dj_logger from datajoint.table import Table from datajoint.utils import get_master, user_choice +from pymysql.err import DataError from spyglass.utils.dj_chains import TableChain, TableChains from spyglass.utils.dj_helper_fn import fetch_nwb