Merge branch 'main' into spike_location_with_true_channel

.github/actions/build-test-environment/action.yml

@@ -37,6 +37,11 @@ runs:
     - name: git-annex install
       run: |
         wget https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz
+        mkdir /home/runner/work/installation
+        mv git-annex-standalone-amd64.tar.gz /home/runner/work/installation/
+        workdir=$(pwd)
+        cd /home/runner/work/installation
         tar xvzf git-annex-standalone-amd64.tar.gz
         echo "$(pwd)/git-annex.linux" >> $GITHUB_PATH
+        cd $workdir
       shell: bash

.github/actions/install-wine/action.yml

@@ -0,0 +1,21 @@
+name: Install packages
+description: This action installs the package and its dependencies for testing
+
+inputs:
+  python-version:
+    description: 'Python version to set up'
+    required: false
+  os:
+    description: 'Operating system to set up'
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install wine (needed for Plexon2)
+      run: |
+        sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
+        sudo dpkg --add-architecture i386
+        sudo apt-get update -qq
+        sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
+      shell: bash

.github/workflows/full-test.yml

@@ -75,6 +75,10 @@ jobs:
               echo "Extractors changed"
               echo "EXTRACTORS_CHANGED=true" >> $GITHUB_OUTPUT
             fi
+            if [[ $file == *"plexon2"* ]]; then
+              echo "Plexon2 changed"
+              echo "PLEXON2_CHANGED=true" >> $GITHUB_OUTPUT
+            fi
             if [[ $file == *"/preprocessing/"* ]]; then
               echo "Preprocessing changed"
               echo "PREPROCESSING_CHANGED=true" >> $GITHUB_OUTPUT
@@ -122,6 +126,9 @@ jobs:
           done
       - name: Set execute permissions on run_tests.sh
         run: chmod +x .github/run_tests.sh
+      - name: Install Wine (Plexon2)
+        if: ${{ steps.modules-changed.outputs.PLEXON2_CHANGED == 'true' }}
+        uses: ./.github/actions/install-wine
       - name: Test core
         run: ./.github/run_tests.sh core
       - name: Test extractors

.github/workflows/installation-tips-test.yml

@@ -0,0 +1,33 @@
+name: Creates Conda Install for Installation Tips
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * 0"  # Weekly at noon UTC on Sundays
+
+jobs:
+  installation-tips-testing:
+    name: Build Conda Env on ${{ matrix.os }} OS
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            label: linux_dandi
+          - os: macos-latest
+            label: mac
+          - os: windows-latest
+            label: windows
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Test Conda Environment Creation
+      uses: conda-incubator/[email protected]
+      with:
+        environment-file: ./installation_tips/full_spikeinterface_environment_${{ matrix.label }}.yml

.github/workflows/test_containers_singularity_gpu.yml

@@ -46,5 +46,6 @@ jobs:
       - name: Run test singularity containers with GPU
         env:
           REPO_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          SPIKEINTERFACE_DEV_PATH: ${{ github.workspace }}
         run: |
           pytest -vv --capture=tee-sys -rA src/spikeinterface/sorters/external/tests/test_singularity_containers_gpu.py

.gitignore

@@ -188,3 +188,4 @@ test_folder/
 
 # Mac OS
 .DS_Store
+test_data.json

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
     -   id: black
         files: ^src/

README.md

@@ -59,15 +59,17 @@ With SpikeInterface, users can:
 - post-process sorted datasets.
 - compare and benchmark spike sorting outputs.
 - compute quality metrics to validate and curate spike sorting outputs.
-- visualize recordings and spike sorting outputs in several ways (matplotlib, sortingview, in jupyter)
-- export report and export to phy
-- offer a powerful Qt-based viewer in separate package [spikeinterface-gui](https://github.com/SpikeInterface/spikeinterface-gui)
-- have some powerful sorting components to build your own sorter.
+- visualize recordings and spike sorting outputs in several ways (matplotlib, sortingview, jupyter, ephyviewer)
+- export a report and/or export to phy
+- offer a powerful Qt-based viewer in a separate package [spikeinterface-gui](https://github.com/SpikeInterface/spikeinterface-gui)
+- have powerful sorting components to build your own sorter.
 
 
 ## Documentation
 
-Detailed documentation for spikeinterface can be found [here](https://spikeinterface.readthedocs.io/en/latest).
+Detailed documentation of the latest PyPI release of SpikeInterface can be found [here](https://spikeinterface.readthedocs.io/en/0.98.2).
+
+Detailed documentation of the development version of SpikeInterface can be found [here](https://spikeinterface.readthedocs.io/en/latest).
 
 Several tutorials to get started can be found in [spiketutorials](https://github.com/SpikeInterface/spiketutorials).
 
@@ -77,9 +79,9 @@ and sorting components.
 You can also have a look at the [spikeinterface-gui](https://github.com/SpikeInterface/spikeinterface-gui).
 
 
-## How to install spikeinteface
+## How to install spikeinterface
 
-You can install the new `spikeinterface` version with pip:
+You can install the latest version of `spikeinterface` version with pip:
 
 ```bash
 pip install spikeinterface[full]
@@ -94,7 +96,7 @@ To install all interactive widget backends, you can use:
 ```
 
 
-To get the latest updates, you can install `spikeinterface` from sources:
+To get the latest updates, you can install `spikeinterface` from source:
 
 ```bash
 git clone https://github.com/SpikeInterface/spikeinterface.git

doc/api.rst

@@ -19,6 +19,8 @@ spikeinterface.core
     .. autofunction:: extract_waveforms
     .. autofunction:: load_waveforms
     .. autofunction:: compute_sparsity
+    .. autoclass:: ChannelSparsity
+        :members:
     .. autoclass:: BinaryRecordingExtractor
     .. autoclass:: ZarrRecordingExtractor
     .. autoclass:: BinaryFolderRecording
@@ -48,17 +50,15 @@ spikeinterface.core
     .. autofunction:: get_template_extremum_channel
     .. autofunction:: get_template_extremum_channel_peak_shift
     .. autofunction:: get_template_extremum_amplitude
-
-..
-    .. autofunction:: read_binary
-    .. autofunction:: read_zarr
     .. autofunction:: append_recordings
     .. autofunction:: concatenate_recordings
     .. autofunction:: split_recording
     .. autofunction:: select_segment_recording
     .. autofunction:: append_sortings
     .. autofunction:: split_sorting
     .. autofunction:: select_segment_sorting
+    .. autofunction:: read_binary
+    .. autofunction:: read_zarr
 
 Low-level
 ~~~~~~~~~
@@ -67,7 +67,6 @@ Low-level
     :noindex:
 
     .. autoclass:: BaseWaveformExtractorExtension
-    .. autoclass:: ChannelSparsity
     .. autoclass:: ChunkRecordingExecutor
 
 spikeinterface.extractors
@@ -83,6 +82,7 @@ NEO-based
     .. autofunction:: read_alphaomega_event
     .. autofunction:: read_axona
     .. autofunction:: read_biocam
+    .. autofunction:: read_binary
     .. autofunction:: read_blackrock
     .. autofunction:: read_ced
     .. autofunction:: read_intan
@@ -91,16 +91,21 @@ NEO-based
     .. autofunction:: read_mcsraw
     .. autofunction:: read_neuralynx
     .. autofunction:: read_neuralynx_sorting
+    .. autofunction:: read_neuroexplorer
     .. autofunction:: read_neuroscope
     .. autofunction:: read_nix
     .. autofunction:: read_openephys
     .. autofunction:: read_openephys_event
     .. autofunction:: read_plexon
     .. autofunction:: read_plexon_sorting
+    .. autofunction:: read_plexon2
+    .. autofunction:: read_plexon2_sorting
     .. autofunction:: read_spike2
     .. autofunction:: read_spikegadgets
     .. autofunction:: read_spikeglx
     .. autofunction:: read_tdt
+    .. autofunction:: read_zarr
+
 
 Non-NEO-based
 ~~~~~~~~~~~~~
@@ -212,8 +217,10 @@ spikeinterface.sorters
     .. autofunction:: print_sorter_versions
     .. autofunction:: get_sorter_description
     .. autofunction:: run_sorter
+    .. autofunction:: run_sorter_jobs
     .. autofunction:: run_sorters
     .. autofunction:: run_sorter_by_property
+    .. autofunction:: read_sorter_folder
 
 Low level
 ~~~~~~~~~

doc/conf.py

@@ -67,6 +67,8 @@
     'numpydoc',
     "sphinx.ext.intersphinx",
     "sphinx.ext.extlinks",
+    "IPython.sphinxext.ipython_directive",
+    "IPython.sphinxext.ipython_console_highlighting"
 ]
 
 numpydoc_show_class_members = False
@@ -116,14 +118,15 @@
     'examples_dirs': ['../examples/modules_gallery'],
     'gallery_dirs': ['modules_gallery', ],  # path where to save gallery generated examples
     'subsection_order': ExplicitOrder([
-                                       '../examples/modules_gallery/core/',
-                                       '../examples/modules_gallery/extractors/',
+                                       '../examples/modules_gallery/core',
+                                       '../examples/modules_gallery/extractors',
                                        '../examples/modules_gallery/qualitymetrics',
                                        '../examples/modules_gallery/comparison',
                                        '../examples/modules_gallery/widgets',
                                        ]),
     'within_subsection_order': FileNameSortKey,
     'ignore_pattern': '/generate_',
+    'nested_sections': False,
 }
 
 intersphinx_mapping = {

doc/development/development.rst

@@ -1,5 +1,5 @@
 Development
-==========
+===========
 
 How to contribute
 -----------------
@@ -14,15 +14,15 @@ There are various ways to contribute to SpikeInterface as a user or developer. S
 * Writing unit tests to expand code coverage and use case scenarios.
 * Reporting bugs and issues.
 
-We use a forking workflow <https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow>_ to manage contributions. Here's a summary of the steps involved, with more details available in the provided link:
+We use a forking workflow `<https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow>`_ to manage contributions. Here's a summary of the steps involved, with more details available in the provided link:
 
 * Fork the SpikeInterface repository.
 * Create a new branch (e.g., :code:`git switch -c my-contribution`).
 * Modify the code, commit, and push changes to your fork.
 * Open a pull request from the "Pull Requests" tab of your fork to :code:`spikeinterface/main`.
 * By following this process, we can review the code and even make changes as necessary.
 
-While we appreciate all the contributions please be mindful of the cost of reviewing pull requests <https://rgommers.github.io/2019/06/the-cost-of-an-open-source-contribution/>_ .
+While we appreciate all the contributions please be mindful of the cost of reviewing pull requests `<https://rgommers.github.io/2019/06/the-cost-of-an-open-source-contribution/>`_ .
 
 
 How to run tests locally
@@ -201,7 +201,7 @@ Implement a new extractor
 SpikeInterface already supports over 30 file formats, but the acquisition system you use might not be among the
 supported formats list (***ref***). Most of the extractord rely on the `NEO <https://github.com/NeuralEnsemble/python-neo>`_
 package to read information from files.
-Therefore, to implement a new extractor to handle the unsupported format, we recommend make a new `neo.rawio `_ class.
+Therefore, to implement a new extractor to handle the unsupported format, we recommend make a new :code:`neo.rawio.BaseRawIO` class (see `example <https://github.com/NeuralEnsemble/python-neo/blob/master/neo/rawio/examplerawio.py#L44>`_).
 Once that is done, the new class can be easily wrapped into SpikeInterface as an extension of the
 :py:class:`~spikeinterface.extractors.neoextractors.neobaseextractors.NeoBaseRecordingExtractor`
 (for :py:class:`~spikeinterface.core.BaseRecording` objects) or