diff --git a/.github/workflows/test-package-build.yml b/.github/workflows/test-package-build.yml index 4b968802e..3d87548fc 100644 --- a/.github/workflows/test-package-build.yml +++ b/.github/workflows/test-package-build.yml @@ -80,15 +80,19 @@ jobs: # - name: Run tests # run: pytest --doctest-modules -v --pyargs spyglass publish: + name: Upload release to PyPI runs-on: ubuntu-latest needs: [test-package] + environment: + name: pypi + url: https://pypi.org/p/spyglass-neuro + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v3 with: name: dist path: dist/ - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.vscode/settings.json b/.vscode/settings.json index 2c05895e3..54a7c2424 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,11 +4,18 @@ "files.trimFinalNewlines": true, "editor.multiCursorModifier": "ctrlCmd", "autoDocstring.docstringFormat": "numpy", - "python.formatting.provider": "none", "remote.SSH.remoteServerListenOnSocket": true, "git.confirmSync": false, "python.analysis.typeCheckingMode": "off", "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + }, }, + "isort.args": [ + "--profile", + "black" + ], } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index db2d7182d..ce2c5d367 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ - Migrate `config` helper scripts to Spyglass codebase. #662 - Revise contribution guidelines. #655 -- Minor bug fixes. #656, #657, #659, #651 +- Minor bug fixes. #656, #657, #659, #651, #671 +- Add setup instruction specificity. ## [0.4.2] (October 10, 2023) diff --git a/notebooks/00_Setup.ipynb b/notebooks/00_Setup.ipynb index ff462b42b..ecf012f27 100644 --- a/notebooks/00_Setup.ipynb +++ b/notebooks/00_Setup.ipynb @@ -41,7 +41,11 @@ "1. [Python 3.9](https://wiki.python.org/moin/BeginnersGuide/Download).\n", "2. [mamba](https://mamba.readthedocs.io/en/latest/installation.html) as a\n", " replacement for conda. Spyglass installation is significantly faster with\n", - " mamba.\n", + " mamba. \n", + " ```bash\n", + " wget \"https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh\"\n", + " bash Miniforge3-$(uname)-$(uname -m).sh\n", + " ```\n", "3. [VS Code](https://code.visualstudio.com/docs/python/python-tutorial) with\n", " relevant python extensions, including\n", " [Jupyter](https://code.visualstudio.com/docs/datascience/jupyter-notebooks).\n", @@ -119,14 +123,26 @@ "id": "580d3feb", "metadata": {}, "source": [ - "Members of the Frank Lab can run the `dj_config.py` helper script to generate\n", - "a config like the one below. Outside users should copy/paste `dj_local_conf_example` and adjust values accordingly.\n", + "Members of the Frank Lab will need to use DataJoint 0.14.2 (currently in\n", + "pre-release) in order to change their password on the MySQL 8 server. DataJoint\n", + "0.14.2\n", + "\n", + "```bash\n", + "git clone https://github.com/datajoint/datajoint-python\n", + "pip install ./datajoint-python\n", + "```\n", + "\n", + "Members of the lab can run the `dj_config.py` helper script to generate a config\n", + "like the one below. \n", "\n", "```bash\n", "cd spyglass\n", "python config/dj_config.py \n", "```\n", "\n", + "Outside users should copy/paste `dj_local_conf_example` and adjust values\n", + "accordingly.\n", + "\n", "The base path (formerly `SPYGLASS_BASE_DIR`) is the directory where all data\n", "will be saved. See also\n", "[docs](https://lorenfranklab.github.io/spyglass/0.4/installation/) for more\n", @@ -172,7 +188,16 @@ " }\n", " }\n", "}\n", - "```\n" + "```\n", + "\n", + "If you see an error saying `Could not find SPYGLASS_BASE_DIR`, try loading your\n", + "config before importing Spyglass.\n", + "\n", + "```python\n", + "import datajoint as dj\n", + "dj.load('/path/to/config')\n", + "import spyglass\n", + "```" ] }, { diff --git a/notebooks/py_scripts/00_Setup.py b/notebooks/py_scripts/00_Setup.py index 583777de8..bdc46d164 100644 --- a/notebooks/py_scripts/00_Setup.py +++ b/notebooks/py_scripts/00_Setup.py @@ -5,7 +5,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.15.2 +# jupytext_version: 1.14.5 # kernelspec: # display_name: Python 3 (ipykernel) # language: python @@ -39,6 +39,10 @@ # 2. [mamba](https://mamba.readthedocs.io/en/latest/installation.html) as a # replacement for conda. Spyglass installation is significantly faster with # mamba. +# ```bash +# wget "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" +# bash Miniforge3-$(uname)-$(uname -m).sh +# ``` # 3. [VS Code](https://code.visualstudio.com/docs/python/python-tutorial) with # relevant python extensions, including # [Jupyter](https://code.visualstudio.com/docs/datascience/jupyter-notebooks). @@ -96,14 +100,26 @@ # ### Existing Database # -# Members of the Frank Lab can run the `dj_config.py` helper script to generate -# a config like the one below. Outside users should copy/paste `dj_local_conf_example` and adjust values accordingly. +# Members of the Frank Lab will need to use DataJoint 0.14.2 (currently in +# pre-release) in order to change their password on the MySQL 8 server. DataJoint +# 0.14.2 +# +# ```bash +# git clone https://github.com/datajoint/datajoint-python +# pip install ./datajoint-python +# ``` +# +# Members of the lab can run the `dj_config.py` helper script to generate a config +# like the one below. # # ```bash # # cd spyglass # python config/dj_config.py # ``` # +# Outside users should copy/paste `dj_local_conf_example` and adjust values +# accordingly. +# # The base path (formerly `SPYGLASS_BASE_DIR`) is the directory where all data # will be saved. See also # [docs](https://lorenfranklab.github.io/spyglass/0.4/installation/) for more @@ -151,6 +167,14 @@ # } # ``` # +# If you see an error saying `Could not find SPYGLASS_BASE_DIR`, try loading your +# config before importing Spyglass. +# +# ```python +# import datajoint as dj +# dj.load('/path/to/config') +# import spyglass +# ``` # ### Running your own database # diff --git a/src/spyglass/common/common_position.py b/src/spyglass/common/common_position.py index 62ecf5284..3ebab0403 100644 --- a/src/spyglass/common/common_position.py +++ b/src/spyglass/common/common_position.py @@ -214,22 +214,14 @@ def generate_pos_components( } @staticmethod - def calculate_position_info( - spatial_df: pd.DataFrame, - meters_to_pixels: float, - position_smoothing_duration, - led1_is_front, - is_upsampled, - upsampling_sampling_rate, - upsampling_interpolation_method, + def _fix_kwargs( orient_smoothing_std_dev=None, speed_smoothing_std_dev=None, max_LED_separation=None, max_plausible_speed=None, **kwargs, ): - CM_TO_METERS = 100 - + """Handles discrepancies between common and v1 param names.""" if not orient_smoothing_std_dev: orient_smoothing_std_dev = kwargs.get( "head_orient_smoothing_std_dev" @@ -244,36 +236,149 @@ def calculate_position_info( [speed_smoothing_std_dev, max_LED_separation, max_plausible_speed] ): raise ValueError( - "Missing required parameters:\n\t" + "Missing at least one required parameter:\n\t" + f"speed_smoothing_std_dev: {speed_smoothing_std_dev}\n\t" + f"max_LED_separation: {max_LED_separation}\n\t" + f"max_plausible_speed: {max_plausible_speed}" ) + return ( + orient_smoothing_std_dev, + speed_smoothing_std_dev, + max_LED_separation, + max_plausible_speed, + ) - # Accepts x/y 'loc' or 'loc1' format for first pos. Renames to 'loc' - DEFAULT_COLS = ["xloc", "yloc", "xloc2", "yloc2", "xloc1", "yloc1"] - ALTERNATIVE_COLS = ["xloc1", "xloc2", "yloc1", "yloc2"] - - if all([c in spatial_df.columns for c in DEFAULT_COLS[:4]]): - # move the 4 position columns to front, continue - spatial_df = spatial_df[DEFAULT_COLS[:4]] - elif all([c in spatial_df.columns for c in ALTERNATIVE_COLS]): - # move the 4 position columns to front, rename to default, continue - spatial_df = spatial_df[ALTERNATIVE_COLS] - spatial_df.columns = DEFAULT_COLS[:4] - else: - cols = list(spatial_df.columns) - if len(cols) != 4 or not all([c in DEFAULT_COLS for c in cols]): - choice = dj.utils.user_choice( - "Unexpected columns in raw position. Assume " - + f"{DEFAULT_COLS[:4]}?\n{spatial_df}\n" + @staticmethod + def _fix_col_names(spatial_df): + """Renames columns in spatial dataframe according to previous norm + + Accepts unnamed first led, 1 or 0 indexed. + Prompts user for confirmation of renaming unexpected columns. + For backwards compatibility, renames to "xloc", "yloc", "xloc2", "yloc2" + """ + + DEFAULT_COLS = ["xloc", "yloc", "xloc2", "yloc2"] + ONE_IDX_COLS = ["xloc1", "yloc1", "xloc2", "yloc2"] + ZERO_IDX_COLS = ["xloc0", "yloc0", "xloc1", "yloc1"] + + input_cols = list(spatial_df.columns) + + has_default = all([c in input_cols for c in DEFAULT_COLS]) + has_0_idx = all([c in input_cols for c in ZERO_IDX_COLS]) + has_1_idx = all([c in input_cols for c in ONE_IDX_COLS]) + + # if unexpected columns, ask user to confirm + if len(input_cols) != 4 or not (has_default or has_0_idx or has_1_idx): + choice = dj.utils.user_choice( + "Unexpected columns in raw position. Assume " + + f"{DEFAULT_COLS[:4]}?\n{spatial_df}\n" + ) + if choice.lower() not in ["yes", "y"]: + raise ValueError( + f"Unexpected columns in raw position: {input_cols}" ) - if choice.lower() not in ["yes", "y"]: - raise ValueError( - f"Unexpected columns in raw position: {cols}" - ) - # rename first 4 columns, keep rest. Rest dropped below - spatial_df.columns = DEFAULT_COLS[:4] + cols[4:] + spatial_df.columns = DEFAULT_COLS + input_cols[4:] + + # Ensure data order, only 4 col + spatial_df = ( + spatial_df[DEFAULT_COLS] + if has_default + else spatial_df[ZERO_IDX_COLS] + if has_0_idx + else spatial_df[ONE_IDX_COLS] + ) + + # rename to default + spatial_df.columns = DEFAULT_COLS + + return spatial_df + + @staticmethod + def _upsample( + front_LED, + back_LED, + time, + sampling_rate, + upsampling_sampling_rate, + upsampling_interpolation_method, + **kwargs, + ): + position_df = pd.DataFrame( + { + "time": time, + "back_LED_x": back_LED[:, 0], + "back_LED_y": back_LED[:, 1], + "front_LED_x": front_LED[:, 0], + "front_LED_y": front_LED[:, 1], + } + ).set_index("time") + + upsampling_start_time = time[0] + upsampling_end_time = time[-1] + + n_samples = ( + int( + np.ceil( + (upsampling_end_time - upsampling_start_time) + * upsampling_sampling_rate + ) + ) + + 1 + ) + new_time = np.linspace( + upsampling_start_time, upsampling_end_time, n_samples + ) + new_index = pd.Index( + np.unique(np.concatenate((position_df.index, new_time))), + name="time", + ) + position_df = ( + position_df.reindex(index=new_index) + .interpolate(method=upsampling_interpolation_method) + .reindex(index=new_time) + ) + + time = np.asarray(position_df.index) + back_LED = np.asarray(position_df.loc[:, ["back_LED_x", "back_LED_y"]]) + front_LED = np.asarray( + position_df.loc[:, ["front_LED_x", "front_LED_y"]] + ) + + sampling_rate = upsampling_sampling_rate + + return front_LED, back_LED, time, sampling_rate + + def calculate_position_info( + self, + spatial_df: pd.DataFrame, + meters_to_pixels: float, + position_smoothing_duration, + led1_is_front, + is_upsampled, + upsampling_sampling_rate, + upsampling_interpolation_method, + orient_smoothing_std_dev=None, + speed_smoothing_std_dev=None, + max_LED_separation=None, + max_plausible_speed=None, + **kwargs, + ): + CM_TO_METERS = 100 + + ( + orient_smoothing_std_dev, + speed_smoothing_std_dev, + max_LED_separation, + max_plausible_speed, + ) = self._fix_kwargs( + orient_smoothing_std_dev, + speed_smoothing_std_dev, + max_LED_separation, + max_plausible_speed, + **kwargs, + ) + + spatial_df = self._fix_col_names(spatial_df) # Get spatial series properties time = np.asarray(spatial_df.index) # seconds position = np.asarray(spatial_df.iloc[:, :4]) # meters @@ -340,51 +445,15 @@ def calculate_position_info( ) if is_upsampled: - position_df = pd.DataFrame( - { - "time": time, - "back_LED_x": back_LED[:, 0], - "back_LED_y": back_LED[:, 1], - "front_LED_x": front_LED[:, 0], - "front_LED_y": front_LED[:, 1], - } - ).set_index("time") - - upsampling_start_time = time[0] - upsampling_end_time = time[-1] - - n_samples = ( - int( - np.ceil( - (upsampling_end_time - upsampling_start_time) - * upsampling_sampling_rate - ) - ) - + 1 - ) - new_time = np.linspace( - upsampling_start_time, upsampling_end_time, n_samples - ) - new_index = pd.Index( - np.unique(np.concatenate((position_df.index, new_time))), - name="time", - ) - position_df = ( - position_df.reindex(index=new_index) - .interpolate(method=upsampling_interpolation_method) - .reindex(index=new_time) + front_LED, back_LED, time, sampling_rate = self._upsample( + front_LED, + back_LED, + time, + sampling_rate, + upsampling_sampling_rate, + upsampling_interpolation_method, ) - time = np.asarray(position_df.index) - back_LED = np.asarray( - position_df.loc[:, ["back_LED_x", "back_LED_y"]] - ) - front_LED = np.asarray( - position_df.loc[:, ["front_LED_x", "front_LED_y"]] - ) - - sampling_rate = upsampling_sampling_rate - # Calculate position, orientation, velocity, speed position = get_centriod(back_LED, front_LED) # cm diff --git a/src/spyglass/position/v1/position_trodes_position.py b/src/spyglass/position/v1/position_trodes_position.py index 5a3c5b9b1..4f0500949 100644 --- a/src/spyglass/position/v1/position_trodes_position.py +++ b/src/spyglass/position/v1/position_trodes_position.py @@ -205,12 +205,12 @@ def make(self, key): @staticmethod def generate_pos_components(*args, **kwargs): - return IntervalPositionInfo.generate_pos_components(*args, **kwargs) + return IntervalPositionInfo().generate_pos_components(*args, **kwargs) @staticmethod def calculate_position_info(*args, **kwargs): """Calculate position info from 2D spatial series.""" - return IntervalPositionInfo.calculate_position_info(*args, **kwargs) + return IntervalPositionInfo().calculate_position_info(*args, **kwargs) def fetch_nwb(self, *attrs, **kwargs): return fetch_nwb(