Fix conflicts

.github/import_test.py

@@ -26,18 +26,17 @@
     time_taken_list = []
     for _ in range(n_samples):
         script_to_execute = (
-                f"import timeit \n"
-                f"import_statement = '{import_statement}' \n"
-                f"time_taken = timeit.timeit(import_statement, number=1) \n"
-                f"print(time_taken) \n"
-               )
+            f"import timeit \n"
+            f"import_statement = '{import_statement}' \n"
+            f"time_taken = timeit.timeit(import_statement, number=1) \n"
+            f"print(time_taken) \n"
+        )
 
         result = subprocess.run(["python", "-c", script_to_execute], capture_output=True, text=True)
 
         if result.returncode != 0:
-            error_message  = (
-                f"Error when running {import_statement} \n"
-                f"Error in subprocess: {result.stderr.strip()}\n"
+            error_message = (
+                f"Error when running {import_statement} \n" f"Error in subprocess: {result.stderr.strip()}\n"
             )
             exceptions.append(error_message)
             break
@@ -46,15 +45,25 @@
         time_taken_list.append(time_taken)
 
     for time in time_taken_list:
-        if time > 1.5:
-            exceptions.append(f"Importing {import_statement} took too long: {time:.2f} seconds")
+        import_time_threshold = 2.0  # Most of the times is sub-second but there outliers
+        if time >= import_time_threshold:
+            exceptions.append(
+                f"Importing {import_statement} took: {time:.2f} s. Should be <: {import_time_threshold} s."
+            )
             break
 
+
     if time_taken_list:
-        avg_time_taken = sum(time_taken_list) / len(time_taken_list)
-        std_dev_time_taken = math.sqrt(sum((x - avg_time_taken) ** 2 for x in time_taken_list) / len(time_taken_list))
+        avg_time = sum(time_taken_list) / len(time_taken_list)
+        std_time = math.sqrt(sum((x - avg_time) ** 2 for x in time_taken_list) / len(time_taken_list))
         times_list_str = ", ".join(f"{time:.2f}" for time in time_taken_list)
-        markdown_output += f"| `{import_statement}` | {avg_time_taken:.2f} | {std_dev_time_taken:.2f} | {times_list_str} |\n"
+        markdown_output += f"| `{import_statement}` | {avg_time:.2f} | {std_time:.2f} | {times_list_str} |\n"
+
+        import_time_threshold = 1.0
+        if avg_time > import_time_threshold:
+            exceptions.append(
+                f"Importing {import_statement} took: {avg_time:.2f} s in average. Should be <: {import_time_threshold} s."
+            )
 
 if exceptions:
     raise Exception("\n".join(exceptions))

.github/workflows/installation-tips-test.yml

@@ -28,8 +28,9 @@ jobs:
       with:
         python-version: '3.10'
     - name: Test Conda Environment Creation
-      uses: conda-incubator/setup-miniconda@v2.2.0
+      uses: conda-incubator/setup-miniconda@v3
       with:
+        miniconda-version: "latest"
         environment-file: ./installation_tips/full_spikeinterface_environment_${{ matrix.label }}.yml
         activate-environment: si_env
     - name: Check Installation Tips

doc/how_to/index.rst

@@ -1,5 +1,5 @@
 How to Guides
-=========
+=============
 
 Guides on how to solve specific, short problems in SpikeInterface. Learn how to...
 
@@ -12,3 +12,4 @@ Guides on how to solve specific, short problems in SpikeInterface. Learn how to.
     load_matlab_data
     combine_recordings
     process_by_channel_group
+    load_your_data_into_sorting

doc/how_to/load_your_data_into_sorting.rst

@@ -0,0 +1,154 @@
+Load Your Own Data into a Sorting
+=================================
+
+Why make a :code:`Sorting`?
+
+SpikeInterface contains pre-build readers for the output of many common sorters.
+However, what if you have sorting output that is not in a standard format (e.g.
+old csv file)? If this is the case you can make your own Sorting object to load
+your data into SpikeInterface. This means you can still easily apply various
+downstream analyses to your results (e.g. building correlograms or for generating
+a :code:`SortingAnalyzer``).
+
+The Sorting object is a core object within SpikeInterface that acts as a convenient
+way to interface with sorting results, no matter which sorter was used to generate
+them. **At a fundamental level it is a series of spike times and a series of labels
+for each unit and a sampling frequency for transforming frames to time.** Below, we will show you have
+to take your existing data and load it as a SpikeInterface :code:`Sorting` object.
+
+
+Reading a standard spike sorting format into a :code:`Sorting`
+-------------------------------------------------------------
+
+For most spike sorting output formats the :code:`Sorting` is automatically generated. For example one could do
+
+.. code-block:: python
+
+    from spikeinterface.extractors import read_phy
+
+    # For kilosort/phy output files we can use the read_phy
+    # most formats will have a read_xx that can used.
+    phy_sorting = read_phy('path/to/folder')
+
+And voilà you now have your :code:`Sorting` object generated and can use it for further analysis. For all the
+current formats see :ref:`compatible_formats`.
+
+
+
+Loading your own data into a :code:`Sorting`
+-------------------------------------------
+
+
+This :code:`Sorting` contains important information about your spike trains including:
+
+  * spike times: the peaks of the extracellular potentials expressed in samples/frames these can
+    be converted to seconds under the hood using the sampling_frequency
+  * spike labels: the neuron id for each spike, can also be called cluster ids or unit ids
+    Stored as the :code:`unit_ids` in SpikeInterface
+  * sampling_frequency: the rate at which the recording equipment was run at. Note this is the
+    frequency and not the period. This value allows for switching between samples/frames to seconds
+
+
+There are 3 options for loading your own data into a sorting object
+
+With lists of spike trains and spike labels
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In this case we need a list of spike times unit labels, sampling_frequency and optional unit_ids
+if you want specific labels to be used (in this case we only create the :code:`Sorting` based on
+the requested unit_ids).
+
+.. code-block:: python
+
+    import numpy as np
+    from spikeinterface.core import NumpySorting
+
+    # in this case we are making a monosegment sorting
+    # we have four spikes that are spread among two neurons
+    my_sorting = NumpySorting.from_times_labels(
+        times_list=[
+            np.array([1000,12000,15000,22000])   # Note these are samples/frames not times in seconds
+            ],
+        labels_list=[
+            np.array(["a","b","a","b"])
+            ],
+        sampling_frequency=30_000.0
+        )
+
+
+With a unit dictionary
+^^^^^^^^^^^^^^^^^^^^^^
+
+We can also use a dictionary where each unit is a key and its spike times are values.
+This is entered as either a list of dicts with each dict being a segment or as a single
+dict for monosegment. We still need to separately specify the sampling_frequency
+
+.. code-block:: python
+
+    from spikeinterface.core import NumpySorting
+
+    my_sorting = NumpySorting.from_unit_dict(
+        units_dict_list={
+            '0': [1000,15000],
+            '1': [12000,22000],
+            },
+        sampling_frequency=30_000.0
+        )
+
+
+With Neo SpikeTrains
+^^^^^^^^^^^^^^^^^^^^
+
+Finally since SpikeInterface is tightly integrated with the Neo project you can create
+a sorting from :code:`Neo.SpikeTrain` objects. See :doc:`Neo documentation<neo:index>` for more information on
+using :code:`Neo.SpikeTrain`'s.
+
+.. code-block:: python
+
+    from spikeinterface.core import NumpySorting
+
+    # neo_spiketrain is a Neo spiketrain object
+    my_sorting = NumpySorting.from_neo_spiketrain_list(
+        neo_spiketrain,
+        sampling_frequency=30_000.0,
+        )
+
+
+Loading multisegment data into a :code:`Sorting`
+-----------------------------------------------
+
+One of the great advantages of SpikeInterface :code:`Sorting` objects is that they can also handle
+multisegment recordings and sortings (e.g. you have a baseline, stimulus, post-stimulus). The
+exact same machinery can be used to generate your sorting, but in this case we do a list of arrays instead of
+a single list. Let's go through one example for using :code:`from_times_labels`:
+
+.. code-block:: python
+
+    import numpy as np
+    from spikeinterface.core import NumpySorting
+
+    # in this case we are making three-segment sorting
+    # we have four spikes that are spread among two neurons
+    # in each segment
+    my_sorting = NumpySorting.from_times_labels(
+        times_list=[
+            np.array([1000,12000,15000,22000]),
+            np.array([30000,33000, 41000, 47000]),
+            np.array([50000,53000,64000,70000]),
+            ],
+        labels_list=[
+            np.array([0,1,0,1]),
+            np.array([0,0,1,1]),
+            np.array([1,0,1,0]),
+        ],
+        sampling_frequency=30_000.0
+        )
+
+
+Next steps
+----------
+
+Now that we've created a Sorting object you can combine it with a Recording to make a
+:ref:`SortingAnalyzer<sphx_glr_tutorials_core_plot_4_sorting_analyzer.py>`
+or start visualizing using plotting functions from our widgets model such as
+:py:func:`~spikeinterface.widgets.plot_crosscorrelograms`.

doc/modules/curation.rst

@@ -41,6 +41,111 @@ The merging and splitting operations are handled by the :py:class:`~spikeinterfa
     # here is the final clean sorting
     clean_sorting = cs.sorting
 
+Manual curation format
+----------------------
+
+SpikeInterface internally supports a JSON-based manual curation format.
+When manual curation is necessary, modifying a dataset in place is a bad practice.
+Instead, to ensure the reproducibility of the spike sorting pipelines, we have introduced a simple and JSON-based manual curation format.
+This format defines at the moment : merges + deletions + manual tags.
+The simple file can be kept along side the output of a sorter and applied on the result to have a "clean" result.
+
+This format has two part:
+
+  * **definition** with the folowing keys:
+
+    * "format_version" : format specification
+    * "unit_ids" : the list of unit_ds
+    * "label_definitions" : list of label categories and possible labels per category.
+                            Every category can be *exclusive=True* onely one label or *exclusive=False* several labels possible
+
+  * **manual output** curation with the folowing keys:
+
+    * "manual_labels"
+    * "merged_unit_groups"
+    * "removed_units"
+
+Here is the description of the format with a simple example:
+
+.. code-block:: json
+
+    {
+        # the first part of the format is the definitation
+        "format_version": "1",
+        "unit_ids": [
+            "u1",
+            "u2",
+            "u3",
+            "u6",
+            "u10",
+            "u14",
+            "u20",
+            "u31",
+            "u42"
+        ],
+        "label_definitions": {
+            "quality": {
+                "label_options": [
+                    "good",
+                    "noise",
+                    "MUA",
+                    "artifact"
+                ],
+                "exclusive": true
+            },
+            "putative_type": {
+                "label_options": [
+                    "excitatory",
+                    "inhibitory",
+                    "pyramidal",
+                    "mitral"
+                ],
+                "exclusive": false
+            }
+        },
+        # the second part of the format is manual action
+        "manual_labels": [
+            {
+                "unit_id": "u1",
+                "quality": [
+                    "good"
+                ]
+            },
+            {
+                "unit_id": "u2",
+                "quality": [
+                    "noise"
+                ],
+                "putative_type": [
+                    "excitatory",
+                    "pyramidal"
+                ]
+            },
+            {
+                "unit_id": "u3",
+                "putative_type": [
+                    "inhibitory"
+                ]
+            }
+        ],
+        "merged_unit_groups": [
+            [
+                "u3",
+                "u6"
+            ],
+            [
+                "u10",
+                "u14",
+                "u20"
+            ]
+        ],
+        "removed_units": [
+            "u31",
+            "u42"
+        ]
+    }
+
+
 
 Automatic curation tools
 ------------------------

doc/modules/motion_correction.rst

@@ -163,21 +163,19 @@ The high-level :py:func:`~spikeinterface.preprocessing.correct_motion()` is inte
                                     max_distance_um=150.0, **job_kwargs)
 
     # Step 2: motion inference
-    motion, temporal_bins, spatial_bins = estimate_motion(recording=rec,
-                                                          peaks=peaks,
-                                                          peak_locations=peak_locations,
-                                                          method="decentralized",
-                                                          direction="y",
-                                                          bin_duration_s=2.0,
-                                                          bin_um=5.0,
-                                                          win_step_um=50.0,
-                                                          win_sigma_um=150.0)
+    motion = estimate_motion(recording=rec,
+                             peaks=peaks,
+                             peak_locations=peak_locations,
+                             method="decentralized",
+                             direction="y",
+                             bin_duration_s=2.0,
+                             bin_um=5.0,
+                             win_step_um=50.0,
+                             win_sigma_um=150.0)
 
     # Step 3: motion interpolation
     # this step is lazy
     rec_corrected = interpolate_motion(recording=rec, motion=motion,
-                                       temporal_bins=temporal_bins,
-                                       spatial_bins=spatial_bins,
                                        border_mode="remove_channels",
                                        spatial_interpolation_method="kriging",
                                        sigma_um=30.)
@@ -220,8 +218,6 @@ different preprocessing chains: one for motion correction and one for spike sort
     rec_corrected2 = interpolate_motion(
                       recording=rec2,
                       motion=motion_info['motion'],
-                      temporal_bins=motion_info['temporal_bins'],
-                      spatial_bins=motion_info['spatial_bins'],
                       **motion_info['parameters']['interpolate_motion_kwargs'])
 
     sorting = run_sorter(sorter_name="montainsort5", recording=rec_corrected2)

doc/releases/0.100.5.rst

@@ -0,0 +1,12 @@
+.. _release0.100.5:
+
+SpikeInterface 0.100.5 release notes
+------------------------------------
+
+6th April 2024
+
+Minor release with bug fixes
+
+* Open Ephys: Use discovered recording ids to load sync timestamps (#2655)
+* Fix channel gains in NwbRecordingExtractor with backend (#2661)
+* Fix depth location in spikes on traces map (#2676)