From d0da3c4bf33178104010d45c9c3128c8bfedd13b Mon Sep 17 00:00:00 2001 From: CBroz1 Date: Wed, 7 Feb 2024 14:13:33 -0600 Subject: [PATCH] Change dependencies. See details. Across `pyproject.toml` and both `environment.yml`: - Alphabetical sort to allow for easier comparison across toml/yaml files. - Remove dependencies required by other dependencies to reduce build time. - Move `black` to `test` dependencies. Could be `dev` instead? - Move `click` to `test` dependencies. CLI should be its own package. - Add ruff config to `pyproject.toml`. Flake8 replacement to allow removal of `setup.cfg`. - Remove `tqdm`, as it is required by `datajoint` Standard environment: - Add comments for all dependencies specific to position environment to allow for easier comparison across yaml files. - Remove `dask`, as it is required by `ghostipy` - Remove `hdmf`, as it is required by `pynwb` - Remove `pymysql`, as it is required by `datajoint` - Remove `pyyaml`, as it is required by `sortingview` - Remove `pydotplus` from `pip` section, as it is already in `conda` section Position environment: - Rename `_position` or `-position` to `_dlc` or `-dlc` to be more precise. - Remove `ipython`, `numba`, and `tensorflow`, as they are all required by `deeplabcut` - Add items present in standard environment but not in position environment. --- CHANGELOG.md | 6 +--- environment.yml | 59 ++++++++++++++++++++------------ environment_dlc.yml | 53 +++++++++++++++++++++++++++++ environment_position.yml | 61 ---------------------------------- pyproject.toml | 42 ++++++++++++----------- setup.cfg | 2 +- src/spyglass/cli/cli.py | 9 ++++- src/spyglass/utils/dj_mixin.py | 2 ++ 8 files changed, 125 insertions(+), 109 deletions(-) create mode 100644 environment_dlc.yml delete mode 100644 environment_position.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc20cc6e..4ac7f2094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Update docs to reflect new notebooks. #776 - Add overview of Spyglass to docs. #779 - Update linting for Black 24. #808 +- Steamline dependency management. #822 (?) ### Pipelines @@ -28,11 +29,6 @@ - Add SpikeSorting V1 pipeline. #651 - Move modules into spikesorting.v0 #807 - Add MUA analysis to spike sorting pipeline -- LFP: Minor fixes to LFPBandV1 populator and `make`. #706, #795 - -### Pipelines - -- Spike sorting: Add SpikeSorting V1 pipeline. #651 - LFP: - Minor fixes to LFPBandV1 populator and `make`. #706, #795 - LFPV1: Fix error for multiple lfp settings on same data #775 diff --git a/environment.yml b/environment.yml index c8c79d4c6..8e314ceb0 100644 --- a/environment.yml +++ b/environment.yml @@ -1,38 +1,53 @@ +# 1. Install a conda distribution. +# https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html +# 2. Run: `mamba env create -f environment.yml` +# 3. Activate: `conda activate spyglass` +# +# (lines intentionally left blank) +# +# name: spyglass channels: - conda-forge - defaults + # - pytorch # dlc-only - franklab - edeno dependencies: - - python>=3.9,<3.10 + - bottleneck + # - cudatoolkit=11.3 # dlc-only + # - cupy # dlc-only + # - dask-cuda # dlc-only + # - ffmpeg # dlc-only + - ipympl - jupyterlab>=3.* - - pydotplus - - dask + # - libgcc # dlc-only + - matplotlib + - non_local_detector + - numpy<1.24 - pip - position_tools - - numpy<1.24 - - track_linearization>=2.3 + # - pybind11 # dlc-only + - pydotplus + - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM + - python>=3.9,<3.10 + - pytorch<1.12.0 + - replay_trajectory_classification - ripple_detection - - non_local_detector - - matplotlib - seaborn - - bottleneck - - ipympl - - tqdm - - pyfftw<=0.12.0 # used by ghostipy. install from conda-forge so that it works on Mac ARM processors + # - torchaudio # dlc-only + # - torchvision # dlc-only + - track_linearization>=2.3 + # - trajectory_analysis_tools # dlc-only - pip: - - pubnub<6.4.0 - - spikeinterface>=0.98.2,<0.99 - - pynwb>=2.2.0,<3 - - hdmf>=3.4.6 - datajoint>=0.13.6 - - ghostipy - - pymysql - - sortingview>=0.11 + # - deeplabcut<2.3.0 # dlc-only - figurl-jupyter - - git+https://github.com/LorenFrankLab/ndx-franklab-novela.git - - pyyaml - - click - - "black[jupyter]" + - ghostipy # for common_filter + - ndx-franklab-novela>=0.1.0 + - mountainsort4 + - pubnub<=6.4.0 + - pynwb>=2.2.0,<3 + - sortingview>=0.11 + - spikeinterface>=0.98.2,<0.99 - . diff --git a/environment_dlc.yml b/environment_dlc.yml new file mode 100644 index 000000000..dd2234a36 --- /dev/null +++ b/environment_dlc.yml @@ -0,0 +1,53 @@ +# 1. INSTALL CORRECT DRIVER for your GPU-equipped machine. +# see https://stackoverflow.com/questions/30820513/what-is-the-correct-version-of-cuda-for-my-nvidia-driver/30820690 +# 2. install: mamba env create -f environment_dlc.yml +# update existing install: mamba env update -f environment_dlc.yml +# 2. After installing... +# run: conda activate spyglass-dlc +# run: mamba env config vars set LD_LIBRARY_PATH=~/path/to//envs/spyglass-position/lib/ +# run: mamba install -c conda-forge wxpython +name: spyglass-dlc +channels: + - conda-forge + - defaults + - pytorch # dlc-only + - franklab + - edeno +dependencies: + - bottleneck + - cudatoolkit=11.3 # dlc-only + - cupy # dlc-only + - dask-cuda # dlc-only + - ffmpeg # dlc-only + - ipympl + - jupyterlab>=3.* + - libgcc # dlc-only + - matplotlib + - non_local_detector + - numpy<1.24 + - pip>=20.2.* + - position_tools + - pybind11 #To avoid isosplit5 build error. Req by mountainsort4 + - pydotplus>=2.0.* + - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM + - python>=3.9,<3.10 + - pytorch<1.12.0 + - replay_trajectory_classification + - ripple_detection + - seaborn + - torchaudio # dlc-only + - torchvision # dlc-only + - track_linearization>=2.3 + - trajectory_analysis_tools # dlc-only + - pip: + - datajoint>=0.13.6 + - deeplabcut<2.3.0 + - figurl-jupyter + - ghostipy # for common_filter + - ndx-franklab-novela>=0.1.0 + - mountainsort4 + - pubnub<=6.4.0 + - pynwb>=2.2.0,<3 + - sortingview>=0.11 + - spikeinterface>=0.98.2,<0.99 + - .[dlc] diff --git a/environment_position.yml b/environment_position.yml deleted file mode 100644 index 125754062..000000000 --- a/environment_position.yml +++ /dev/null @@ -1,61 +0,0 @@ -# FIRST: INSTALL CORRECT DRIVER for GPU, see https://stackoverflow.com/questions/30820513/what-is-the-correct-version-of-cuda-for-my-nvidia-driver/30820690 -# -# NOTE: Perform install on a GPU-equipped machine -# -# install: mamba env create -f environment_position.yml -# update: mamba env update -f environment_position.yml -# after installing do the following: -# run: conda activate spyglass-position -# run: mamba env config vars set LD_LIBRARY_PATH=~/path/to//envs/spyglass-position/lib/ -# run: mamba install -c conda-forge wxpython -# cd into spyglass and run: pip install -e .[position] - -name: spyglass-position -channels: - - conda-forge - - pytorch - - franklab - - edeno -dependencies: - - python>=3.9, <3.10 - - jupyterlab>=3.* - - pydotplus - - libgcc - - dask>=2.30 - - pip - - position_tools - - track_linearization>=2.3 - - ripple_detection - - non_local_detector - - ipython - - matplotlib>=3.3 - - seaborn - - bottleneck - - ipympl - - tqdm - - ffmpeg - - pytorch<1.12.0 - - torchvision - - torchaudio - - cudatoolkit=11.3 - - tensorflow>=2.0 - - numba>=0.48.0 - - pyfftw<=0.12.0 # used by ghostipy. install from conda-forge so that it works on Mac ARM processors - - pybind11 #To avoid isosplit5 build error - - pip: - - pubnub<6.4.0 - - mountainsort4 - - spikeinterface>=0.98.2,<0.99 - - pynwb>=2.2.0,<3 - - hdmf>=3.4.6 - - datajoint>=0.13.6 - - ghostipy - - pymysql>=1.0.* - - sortingview>=0.11 - - figurl-jupyter - - git+https://github.com/LorenFrankLab/ndx-franklab-novela.git - - pyyaml - - click - - "black[jupyter]" - - deeplabcut<2.3.0 - - .[position] diff --git a/pyproject.toml b/pyproject.toml index fa50e8ddd..8894de466 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,28 +35,26 @@ keywords = [ "sortingview", ] dependencies = [ - "pydotplus", + "bottleneck", "dask", - "position_tools", - "track_linearization>=2.3", - "non_local_detector", - "ripple_detection", + "datajoint>=0.13.6", + # "ghostipy", # removed from list bc M1 users need to install pyfftw first + "hdmf>=3.4.6", + "ipympl", "matplotlib", - "seaborn", - "bottleneck", + "ndx_franklab_novela>=0.1.0", + "non_local_detector", "numpy<1.24", - "ipympl", - "tqdm", - "pubnub<6.4.0", # TODO: remove this when sortingview is updated + "opencv-python", + "position_tools", + "pubnub<6.4.0", # TODO: remove this when sortingview is updated + "pydotplus", "pynwb>=2.2.0,<3", - "hdmf>=3.4.6", - "datajoint>=0.13.6", - "pymysql", + "ripple_detection", + "seaborn", "sortingview>=0.11", - "pyyaml", - "click", "spikeinterface>=0.98.2,<0.99", - "ndx_franklab_novela>=0.1.0", + "track_linearization>=2.3", ] dynamic = ["version"] @@ -68,14 +66,16 @@ spyglass_cli = "spyglass.cli:cli" "Bug Tracker" = "https://github.com/LorenFrankLab/spyglass/issues" [project.optional-dependencies] -position = ["ffmpeg", "numba>=0.54", "deeplabcut<2.3.0"] +dlc = ["ffmpeg", "numba>=0.54", "deeplabcut<2.3.0"] test = [ + "black[jupyter]", # code formatting, rather than a 'dev' install + "click", # for CLI subpackage only "docker", # for tests in a container "pytest", # unit testing "pytest-cov", # code coverage "kachery", # database access "kachery-client", - "kachery-cloud", + "kachery-cloud>=0.4.0", ] docs = [ "hatch", # Get version from env @@ -121,7 +121,7 @@ addopts = [ # "--pdb", # drop into debugger on failure "-p no:warnings", # "--no-teardown", # don't teardown the database after tests - "--quiet-spy", # don't show logging from spyglass + "--quiet-spy", # don't show logging from spyglass "--show-capture=no", "--pdbcls=IPython.terminal.debugger:TerminalPdb", # use ipython debugger "--cov=spyglass", @@ -150,3 +150,7 @@ omit = [ # which submodules have no tests "*/spikesorting/*", # "*/utils/*", ] + +[tool.ruff] # CB: Propose replacing flake8 with ruff to delete setup.cfg +line-length = 80 +ignore = ["F401" , "E402", "E501"] diff --git a/setup.cfg b/setup.cfg index bf615cc86..bbea83f2f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ -[flake8] +[flake8] # Planned for removal in favor of Ruff max-line-length = 80 max-complexity = 17 exclude = diff --git a/src/spyglass/cli/cli.py b/src/spyglass/cli/cli.py index 5ccc7aaee..a07919da2 100644 --- a/src/spyglass/cli/cli.py +++ b/src/spyglass/cli/cli.py @@ -1,8 +1,15 @@ from typing import Union -import click import yaml +try: + import click +except ImportError: + raise ImportError( + "spyglass.cli.cli requires the 'click' package. " + "You can install it with 'pip install click'." + ) + @click.group(help="Spyglass command-line client") def cli(): diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 7a69ba75e..0e18e3a5c 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -3,10 +3,12 @@ from typing import Dict, List, Union import datajoint as dj +from datajoint.errors import DataJointError from datajoint.expression import QueryExpression from datajoint.logging import logger as dj_logger from datajoint.table import Table from datajoint.utils import get_master, user_choice +from pymysql.err import DataError from spyglass.utils.dj_chains import TableChain, TableChains from spyglass.utils.dj_helper_fn import fetch_nwb