Merge branch 'SpikeInterface:main' into main

.github/actions/build-test-environment/action.yml

@@ -21,7 +21,7 @@ runs:
         python -m pip install -U pip  # Official recommended way
         source ${{ github.workspace }}/test_env/bin/activate
         pip install tabulate  # This produces summaries at the end
-        pip install -e .[test,extractors,streaming_extractors,full]
+        pip install -e .[test,extractors,streaming_extractors,test_extractors,full]
       shell: bash
     - name: Force installation of latest dev from key-packages when running dev (not release)
       run: |

doc/api.rst

@@ -60,14 +60,17 @@ spikeinterface.core
     .. autofunction:: select_segment_sorting
     .. autofunction:: read_binary
     .. autofunction:: read_zarr
+    .. autofunction:: apply_merges_to_sorting
+    .. autofunction:: spike_vector_to_spike_trains
+    .. autofunction:: random_spikes_selection
+
 
 Low-level
 ~~~~~~~~~
 
 .. automodule:: spikeinterface.core
     :noindex:
 
-    .. autoclass:: BaseWaveformExtractorExtension
     .. autoclass:: ChunkRecordingExecutor
 
 spikeinterface.extractors
@@ -335,14 +338,60 @@ spikeinterface.curation
 spikeinterface.generation
 -------------------------
 
+Core
+~~~~
+.. automodule:: spikeinterface.generation
+
+    .. autofunction:: generate_recording
+    .. autofunction:: generate_sorting
+    .. autofunction:: generate_snippets
+    .. autofunction:: generate_templates
+    .. autofunction:: generate_recording_by_size
+    .. autofunction:: generate_ground_truth_recording
+    .. autofunction:: add_synchrony_to_sorting
+    .. autofunction:: synthesize_random_firings
+    .. autofunction:: inject_some_duplicate_units
+    .. autofunction:: inject_some_split_units
+    .. autofunction:: synthetize_spike_train_bad_isi
+    .. autofunction:: inject_templates
+    .. autofunction:: noise_generator_recording
+    .. autoclass:: InjectTemplatesRecording
+    .. autoclass:: NoiseGeneratorRecording
+
+Drift
+~~~~~
 .. automodule:: spikeinterface.generation
 
+    .. autofunction:: generate_drifting_recording
+    .. autofunction:: generate_displacement_vector
+    .. autofunction:: make_one_displacement_vector
     .. autofunction:: make_linear_displacement
     .. autofunction:: move_dense_templates
     .. autofunction:: interpolate_templates
     .. autoclass:: DriftingTemplates
     .. autoclass:: InjectDriftingTemplatesRecording
 
+Hybrid
+~~~~~~
+.. automodule:: spikeinterface.generation
+
+    .. autofunction:: generate_hybrid_recording
+    .. autofunction:: estimate_templates_from_recording
+    .. autofunction:: select_templates
+    .. autofunction:: scale_template_to_range
+    .. autofunction:: relocate_templates
+    .. autofunction:: fetch_template_object_from_database
+    .. autofunction:: fetch_templates_database_info
+    .. autofunction:: list_available_datasets_in_template_database
+    .. autofunction:: query_templates_from_database
+
+
+Noise
+~~~~~
+.. automodule:: spikeinterface.generation
+
+    .. autofunction:: generate_noise
+
 
 spikeinterface.sortingcomponents
 --------------------------------

doc/how_to/analyse_neuropixels.rst → doc/how_to/analyze_neuropixels.rst

@@ -1,4 +1,4 @@
-Analyse Neuropixels datasets
+Analyze Neuropixels datasets
 ============================
 
 This example shows how to perform Neuropixels-specific analysis,
@@ -218,7 +218,7 @@ We need to specify which one to read:
 
 
 
-.. image:: analyse_neuropixels_files/analyse_neuropixels_8_1.png
+.. image:: analyze_neuropixels_files/analyze_neuropixels_8_1.png
 
 
 Preprocess the recording
@@ -286,7 +286,7 @@ is lazy, so you can change the previsous cell (parameters, step order,
 
 
 
-.. image:: analyse_neuropixels_files/analyse_neuropixels_13_0.png
+.. image:: analyze_neuropixels_files/analyze_neuropixels_13_0.png
 
 
 .. code:: ipython3
@@ -306,7 +306,7 @@ is lazy, so you can change the previsous cell (parameters, step order,
 
 
 
-.. image:: analyse_neuropixels_files/analyse_neuropixels_14_1.png
+.. image:: analyze_neuropixels_files/analyze_neuropixels_14_1.png
 
 
 Should we save the preprocessed data to a binary file?
@@ -389,7 +389,7 @@ Noise levels can be estimated on the scaled traces or on the raw
 
 
 
-.. image:: analyse_neuropixels_files/analyse_neuropixels_21_1.png
+.. image:: analyze_neuropixels_files/analyze_neuropixels_21_1.png
 
 
 Detect and localize peaks
@@ -480,7 +480,7 @@ documentation for motion estimation and correction for more details.
 
 
 
-.. image:: analyse_neuropixels_files/analyse_neuropixels_26_1.png
+.. image:: analyze_neuropixels_files/analyze_neuropixels_26_1.png
 
 
 .. code:: ipython3
@@ -502,7 +502,7 @@ documentation for motion estimation and correction for more details.
 
 
 
-.. image:: analyse_neuropixels_files/analyse_neuropixels_27_1.png
+.. image:: analyze_neuropixels_files/analyze_neuropixels_27_1.png
 
 
 Run a spike sorter

doc/how_to/benchmark_with_hybrid_recordings.rst

@@ -9,7 +9,7 @@ with known spiking activity. The template (aka average waveforms) of the
 injected units can be from previous spike sorted data. In this example,
 we will be using an open database of templates that we have constructed
 from the International Brain Laboratory - Brain Wide Map (available on
-`DANDI <https://dandiarchive.org/dandiset/000409?search=IBL&page=2&sortOption=0&sortDir=-1&showDrafts=true&showEmpty=false&pos=9>`__).
+`DANDI <https://dandiarchive.org/dandiset/000409?search=IBL&page=2&sortOption=0&sortDir=-1&showDrafts=true&showEmpty=false&pos=9>`_).
 
 Importantly, recordings from long-shank probes, such as Neuropixels,
 usually experience drifts. Such drifts have to be taken into account in

doc/how_to/index.rst

@@ -8,7 +8,7 @@ Guides on how to solve specific, short problems in SpikeInterface. Learn how to.
 
     viewers
     handle_drift
-    analyse_neuropixels
+    analyze_neuropixels
     load_matlab_data
     combine_recordings
     process_by_channel_group

doc/modules/generation.rst

@@ -1,9 +1,28 @@
 Generation module
 =================
 
-The :py:mod:`spikeinterface.generation` provides functions to generate recordings containing spikes.
-This module proposes several approaches for this including purely synthetic recordings as well as "hybrid" recordings (where templates come from true datasets).
+The :py:mod:`spikeinterface.generation` provides functions to generate recordings containing spikes,
+which can be used as "ground-truth" for benchmarking spike sorting algorithms.
 
+There are several approaches to generating such recordings.
+One possibility is to generate purely synthetic recordings. Another approach is to use real
+recordings and add synthetic spikes to them, to make "hybrid" recordings.
+The advantage of the former is that the ground-truth is known exactly, which is useful for benchmarking.
+The advantage of the latter is that the spikes are added to real noise, which can be more realistic.
 
-The :py:mod:`spikeinterface.core.generate` already provides functions for generating synthetic data but this module will supply an extended and more complex
-machinery, for instance generating recordings that possess various types of drift.
+For hybrid recordings, the main challenge is to generate realistic spike templates.
+We therefore built an open database of templates that we have constructed from the International
+Brain Laboratory - Brain Wide Map (available on
+`DANDI <https://dandiarchive.org/dandiset/000409?search=IBL&page=2&sortOption=0&sortDir=-1&showDrafts=true&showEmpty=false&pos=9>`_).
+You can check out this collection of over 600 templates from this `web app <https://spikeinterface.github.io/hybrid_template_library/>`_.
+
+The :py:mod:`spikeinterface.generation` module offers tools to interact with this database to select and download templates,
+manupulating (e.g. rescaling and relocating them), and construct hybrid recordings with them.
+Importantly, recordings from long-shank probes, such as Neuropixels, usually experience drifts.
+Such drifts can be taken into account in order to smoothly inject spikes into the recording.
+
+The :py:mod:`spikeinterface.generation` also includes functions to generate different kinds of drift signals and drifting
+recordings, as well as generating synthetic noise profiles of various types.
+
+Some of the generation functions are defined in the :py:mod:`spikeinterface.core.generate` module, but also exposed at the
+:py:mod:`spikeinterface.generation` level for convenience.

src/spikeinterface/comparison/groundtruthstudy.py

@@ -9,9 +9,6 @@
 import numpy as np
 
 from spikeinterface.core import load_extractor, create_sorting_analyzer, load_sorting_analyzer
-from spikeinterface.core.core_tools import SIJsonEncoder
-from spikeinterface.core.job_tools import split_job_kwargs
-
 from spikeinterface.sorters import run_sorter_jobs, read_sorter_folder
 
 from spikeinterface.qualitymetrics import compute_quality_metrics
@@ -54,6 +51,7 @@ def __init__(self, study_folder):
         self.cases = {}
         self.sortings = {}
         self.comparisons = {}
+        self.colors = None
 
         self.scan_folder()
 
@@ -175,6 +173,22 @@ def remove_sorting(self, key):
             if f.exists():
                 f.unlink()
 
+    def set_colors(self, colors=None, map_name="tab20"):
+        from spikeinterface.widgets import get_some_colors
+
+        if colors is None:
+            case_keys = list(self.cases.keys())
+            self.colors = get_some_colors(
+                case_keys, map_name=map_name, color_engine="matplotlib", shuffle=False, margin=0
+            )
+        else:
+            self.colors = colors
+
+    def get_colors(self):
+        if self.colors is None:
+            self.set_colors()
+        return self.colors
+
     def run_sorters(self, case_keys=None, engine="loop", engine_kwargs={}, keep=True, verbose=False):
         if case_keys is None:
             case_keys = self.cases.keys()

src/spikeinterface/core/__init__.py

@@ -101,7 +101,7 @@
     get_chunk_with_margin,
     order_channels_by_depth,
 )
-from .sorting_tools import spike_vector_to_spike_trains, random_spikes_selection
+from .sorting_tools import spike_vector_to_spike_trains, random_spikes_selection, apply_merges_to_sorting
 
 from .waveform_tools import extract_waveforms_to_buffers, estimate_templates, estimate_templates_with_accumulator
 from .snippets_tools import snippets_from_sorting

src/spikeinterface/core/base.py

@@ -7,7 +7,6 @@
 import weakref
 import json
 import pickle
-import os
 import random
 import string
 from packaging.version import parse
@@ -41,7 +40,7 @@ class BaseExtractor:
     # This replaces the old key_properties
     # These are annotations/properties that always need to be
     # dumped (for instance locations, groups, is_fileterd, etc.)
-    _main_annotations = []
+    _main_annotations = ["name"]
     _main_properties = []
 
     # these properties are skipped by default in copy_metadata
@@ -79,6 +78,19 @@ def __init__(self, main_ids: Sequence) -> None:
         # preferred context for multiprocessing
         self._preferred_mp_context = None
 
+    @property
+    def name(self):
+        name = self._annotations.get("name", None)
+        return name if name is not None else self.__class__.__name__
+
+    @name.setter
+    def name(self, value):
+        if value is not None:
+            self.annotate(name=value)
+        else:
+            # we remove the annotation if it exists
+            _ = self._annotations.pop("name", None)
+
     def get_num_segments(self) -> int:
         # This is implemented in BaseRecording or BaseSorting
         raise NotImplementedError
@@ -128,8 +140,18 @@ def ids_to_indices(
                 indices = np.arange(len(self._main_ids))
         else:
             assert isinstance(ids, (list, np.ndarray, tuple)), "'ids' must be a list, np.ndarray or tuple"
+
+            non_existent_ids = [id for id in ids if id not in self._main_ids]
+            if non_existent_ids:
+                error_msg = (
+                    f"IDs {non_existent_ids} are not channel ids of the extractor. \n"
+                    f"Available ids are {self._main_ids} with dtype {self._main_ids.dtype}"
+                )
+                raise ValueError(error_msg)
+
             _main_ids = self._main_ids.tolist()
             indices = np.array([_main_ids.index(id) for id in ids], dtype=int)
+
             if prefer_slice:
                 if np.all(np.diff(indices) == 1):
                     indices = slice(indices[0], indices[-1] + 1)
@@ -928,13 +950,14 @@ def save_to_folder(
         folder.mkdir(parents=True, exist_ok=False)
 
         # dump provenance
-        provenance_file = folder / f"provenance.json"
         if self.check_serializability("json"):
+            provenance_file = folder / f"provenance.json"
+            self.dump(provenance_file)
+        elif self.check_serializability("pickle"):
+            provenance_file = folder / f"provenance.pkl"
             self.dump(provenance_file)
         else:
-            provenance_file.write_text(
-                json.dumps({"warning": "the provenace is not json serializable!!!"}), encoding="utf8"
-            )
+            warnings.warn("The extractor is not serializable to file. The provenance will not be saved.")
 
         self.save_metadata_to_folder(folder)
 
@@ -1001,7 +1024,6 @@ def save_to_zarr(
         cached: ZarrExtractor
             Saved copy of the extractor.
         """
-        import zarr
         from .zarrextractors import read_zarr
 
         save_kwargs.pop("format", None)