Merge branch 'main' into add_colon_spaces

.github/workflows/core-test.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           git config --global user.email "[email protected]"
@@ -31,7 +31,7 @@ jobs:
           pip install -e .[test_core]
       - name: Test core with pytest
         run: |
-          pytest -vv -ra --durations=0 --durations-min=0.001 src/spikeinterface/core | tee report.txt; test ${PIPESTATUS[0]} -eq 0 || exit 1
+          pytest -m "core" -vv -ra --durations=0 --durations-min=0.001 | tee report.txt; test $? -eq 0 || exit 1
         shell: bash  # Necessary for pipeline to work on windows
       - name: Build test summary
         run: |

conftest.py

@@ -25,31 +25,27 @@ def pytest_sessionstart(session):
     for mark_name in mark_names:
         (pytest.global_test_folder / mark_name).mkdir()
 
-
 def pytest_collection_modifyitems(config, items):
     """
     This function marks (in the pytest sense) the tests according to their name and file_path location
     Marking them in turn allows the tests to be run by using the pytest -m marker_name option.
     """
 
-
-    # python 3.4/3.5 compat: rootdir = pathlib.Path(str(config.rootdir))
     rootdir = Path(config.rootdir)
-
+    modules_location = rootdir / "src" / "spikeinterface"
     for item in items:
-        rel_path = Path(item.fspath).relative_to(rootdir)
-        if "sorters" in str(rel_path):
-            if "/internal/" in str(rel_path):
+        rel_path = Path(item.fspath).relative_to(modules_location)
+        module = rel_path.parts[0]
+        if module == "sorters":
+            if "internal" in rel_path.parts:
                 item.add_marker("sorters_internal")
-            elif "/external/" in str(rel_path):
+            elif "external" in rel_path.parts:
                 item.add_marker("sorters_external")
             else:
                 item.add_marker("sorters")
         else:
-            for mark_name in mark_names:
-                if f"/{mark_name}/" in str(rel_path):
-                    mark = getattr(pytest.mark, mark_name)
-                    item.add_marker(mark)
+            item.add_marker(module)
+
 
 
 def pytest_sessionfinish(session, exitstatus):

src/spikeinterface/comparison/tests/test_groundtruthcomparison.py

@@ -1,7 +1,6 @@
 import numpy as np
 from numpy.testing import assert_array_equal
 
-import pandas as pd
 
 from spikeinterface.extractors import NumpySorting
 from spikeinterface.comparison import compare_sorter_to_ground_truth
@@ -57,6 +56,8 @@ def test_compare_sorter_to_ground_truth():
             "pooled_with_average",
         ]
         for method in methods:
+            import pandas as pd
+
             perf_df = sc.get_performance(method=method, output="pandas")
             assert isinstance(perf_df, (pd.Series, pd.DataFrame))
             perf_dict = sc.get_performance(method=method, output="dict")

src/spikeinterface/curation/tests/sv-sorting-curation.json

@@ -1 +1 @@
-{"labelsByUnit":{"#2":["mua"],"#3":["mua"],"#4":["mua"],"#5":["accept"],"#6":["accept"],"#7":["accept"],"#8":["artifact"],"#9":["artifact"]},"mergeGroups":[["#8","#9"]]}
+{"labelsByUnit":{"2":["mua"],"3":["mua"],"4":["mua"],"5":["accept"],"6":["accept"],"7":["accept"],"8":["artifact"],"9":["artifact"]},"mergeGroups":[[8,9]]}

src/spikeinterface/curation/tests/test_sortingview_curation.py

@@ -5,8 +5,8 @@
 import numpy as np
 
 import spikeinterface as si
+from spikeinterface.core import generate_sorting
 import spikeinterface.extractors as se
-from spikeinterface.extractors import read_mearec
 from spikeinterface import set_global_tmp_folder
 from spikeinterface.postprocessing import (
     compute_correlograms,
@@ -34,8 +34,6 @@
 # def generate_sortingview_curation_dataset():
 #     import spikeinterface.widgets as sw
 
-#     local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5")
-#     recording, sorting = read_mearec(local_path)
 
 #     sorting_analyzer = si.create_sorting_analyzer(sorting, recording, format="memory")
 #     sorting_analyzer.compute("random_spikes")
@@ -50,23 +48,22 @@
 #     w = sw.plot_sorting_summary(sorting_analyzer, curation=True, backend="sortingview")
 
 #     # curation_link:
-#     # https://figurl.org/f?v=gs://figurl/spikesortingview-10&d=sha1://bd53f6b707f8121cadc901562a89b67aec81cc81&label=SpikeInterface%20-%20Sorting%20Summary
+#     # https://figurl.org/f?v=npm://@fi-sci/figurl-sortingview@12/dist&d=sha1://058ab901610aa9d29df565595a3cc2a81a1b08e5
 
 
 @pytest.mark.skipif(ON_GITHUB and not KACHERY_CLOUD_SET, reason="Kachery cloud secrets not available")
 def test_gh_curation():
     """
     Test curation using GitHub URI.
     """
-    local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5")
-    _, sorting = read_mearec(local_path)
+    sorting = generate_sorting(num_units=10)
     # curated link:
-    # https://figurl.org/f?v=gs://figurl/spikesortingview-10&d=sha1://bd53f6b707f8121cadc901562a89b67aec81cc81&label=SpikeInterface%20-%20Sorting%20Summary&s={%22sortingCuration%22:%22gh://alejoe91/spikeinterface/fix-codecov/spikeinterface/curation/tests/sv-sorting-curation.json%22}
+    # https://figurl.org/f?v=npm://@fi-sci/figurl-sortingview@12/dist&d=sha1://058ab901610aa9d29df565595a3cc2a81a1b08e5
     gh_uri = "gh://SpikeInterface/spikeinterface/main/src/spikeinterface/curation/tests/sv-sorting-curation.json"
     sorting_curated_gh = apply_sortingview_curation(sorting, uri_or_json=gh_uri, verbose=True)
 
     assert len(sorting_curated_gh.unit_ids) == 9
-    assert "#8-#9" in sorting_curated_gh.unit_ids
+    assert 1, 2 in sorting_curated_gh.unit_ids
     assert "accept" in sorting_curated_gh.get_property_keys()
     assert "mua" in sorting_curated_gh.get_property_keys()
     assert "artifact" in sorting_curated_gh.get_property_keys()
@@ -86,18 +83,17 @@ def test_sha1_curation():
     """
     Test curation using SHA1 URI.
     """
-    local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5")
-    _, sorting = read_mearec(local_path)
+    sorting = generate_sorting(num_units=10)
 
     # from SHA1
     # curated link:
-    # https://figurl.org/f?v=gs://figurl/spikesortingview-10&d=sha1://bd53f6b707f8121cadc901562a89b67aec81cc81&label=SpikeInterface%20-%20Sorting%20Summary&s={%22sortingCuration%22:%22sha1://1182ba19671fcc7d3f8e0501b0f8c07fb9736c22%22}
-    sha1_uri = "sha1://1182ba19671fcc7d3f8e0501b0f8c07fb9736c22"
+    # https://figurl.org/f?v=npm://@fi-sci/figurl-sortingview@12/dist&d=sha1://058ab901610aa9d29df565595a3cc2a81a1b08e5
+    sha1_uri = "sha1://449a428e8824eef9ad9bcc3241e45a2cee02d381"
     sorting_curated_sha1 = apply_sortingview_curation(sorting, uri_or_json=sha1_uri, verbose=True)
     # print(f"From SHA: {sorting_curated_sha1}")
 
     assert len(sorting_curated_sha1.unit_ids) == 9
-    assert "#8-#9" in sorting_curated_sha1.unit_ids
+    assert 1, 2 in sorting_curated_sha1.unit_ids
     assert "accept" in sorting_curated_sha1.get_property_keys()
     assert "mua" in sorting_curated_sha1.get_property_keys()
     assert "artifact" in sorting_curated_sha1.get_property_keys()
@@ -116,16 +112,15 @@ def test_json_curation():
     """
     Test curation using a JSON file.
     """
-    local_path = si.download_dataset(remote_path="mearec/mearec_test_10s.h5")
-    _, sorting = read_mearec(local_path)
+    sorting = generate_sorting(num_units=10)
 
     # from curation.json
     json_file = parent_folder / "sv-sorting-curation.json"
     # print(f"Sorting: {sorting.get_unit_ids()}")
     sorting_curated_json = apply_sortingview_curation(sorting, uri_or_json=json_file, verbose=True)
 
     assert len(sorting_curated_json.unit_ids) == 9
-    assert "#8-#9" in sorting_curated_json.unit_ids
+    assert 1, 2 in sorting_curated_json.unit_ids
     assert "accept" in sorting_curated_json.get_property_keys()
     assert "mua" in sorting_curated_json.get_property_keys()
     assert "artifact" in sorting_curated_json.get_property_keys()

src/spikeinterface/extractors/tests/test_iblextractors.py

@@ -4,7 +4,6 @@
 import numpy as np
 from numpy.testing import assert_array_equal
 import pytest
-import requests
 
 from spikeinterface.extractors import read_ibl_recording, read_ibl_sorting, IblRecordingExtractor
 
@@ -16,6 +15,7 @@
 class TestDefaultIblRecordingExtractorApBand(TestCase):
     @classmethod
     def setUpClass(cls):
+        import requests
         from one.api import ONE
 
         cls.eid = EID

src/spikeinterface/extractors/tests/test_nwbextractors.py

@@ -1,17 +1,10 @@
 import unittest
-import unittest
 from pathlib import Path
-from tempfile import mkdtemp
-from datetime import datetime
+
 
 import pytest
 import numpy as np
-from pynwb import NWBHDF5IO
-from hdmf_zarr import NWBZarrIO
-from pynwb.ecephys import ElectricalSeries, LFP, FilteredEphys
-from pynwb.testing.mock.file import mock_NWBFile
-from pynwb.testing.mock.device import mock_Device
-from pynwb.testing.mock.ecephys import mock_ElectricalSeries, mock_ElectrodeGroup, mock_electrodes
+
 from spikeinterface.extractors import NwbRecordingExtractor, NwbSortingExtractor
 
 from spikeinterface.extractors.tests.common_tests import RecordingCommonTestSuite, SortingCommonTestSuite
@@ -30,10 +23,12 @@ class NwbSortingTest(SortingCommonTestSuite, unittest.TestCase):
     entities = []
 
 
-from pynwb.testing.mock.ecephys import mock_ElectrodeGroup
-
-
 def nwbfile_with_ecephys_content():
+    from pynwb.ecephys import ElectricalSeries, LFP, FilteredEphys
+    from pynwb.testing.mock.file import mock_NWBFile
+    from pynwb.testing.mock.device import mock_Device
+    from pynwb.testing.mock.ecephys import mock_ElectricalSeries, mock_ElectrodeGroup
+
     to_micro_volts = 1e6
 
     nwbfile = mock_NWBFile()
@@ -160,6 +155,9 @@ def nwbfile_with_ecephys_content():
 
 
 def _generate_nwbfile(backend, file_path):
+    from pynwb import NWBHDF5IO
+    from hdmf_zarr import NWBZarrIO
+
     nwbfile = nwbfile_with_ecephys_content()
     if backend == "hdf5":
         io_class = NWBHDF5IO
@@ -367,6 +365,9 @@ def test_failure_with_wrong_electrical_series_path(generate_nwbfile, use_pynwb):
 
 @pytest.mark.parametrize("use_pynwb", [True, False])
 def test_sorting_extraction_of_ragged_arrays(tmp_path, use_pynwb):
+    from pynwb import NWBHDF5IO
+    from pynwb.testing.mock.file import mock_NWBFile
+
     nwbfile = mock_NWBFile()
 
     # Add the spikes
@@ -433,6 +434,10 @@ def test_sorting_extraction_of_ragged_arrays(tmp_path, use_pynwb):
 
 @pytest.mark.parametrize("use_pynwb", [True, False])
 def test_sorting_extraction_start_time(tmp_path, use_pynwb):
+
+    from pynwb import NWBHDF5IO
+    from pynwb.testing.mock.file import mock_NWBFile
+
     nwbfile = mock_NWBFile()
 
     # Add the spikes
@@ -477,6 +482,12 @@ def test_sorting_extraction_start_time(tmp_path, use_pynwb):
 
 @pytest.mark.parametrize("use_pynwb", [True, False])
 def test_sorting_extraction_start_time_from_series(tmp_path, use_pynwb):
+    from pynwb import NWBHDF5IO
+    from pynwb.testing.mock.file import mock_NWBFile
+    from pynwb.ecephys import ElectricalSeries, LFP, FilteredEphys
+
+    from pynwb.testing.mock.ecephys import mock_electrodes
+
     nwbfile = mock_NWBFile()
     electrical_series_name = "ElectricalSeries"
     t_start = 10.0
@@ -530,6 +541,8 @@ def test_sorting_extraction_start_time_from_series(tmp_path, use_pynwb):
 @pytest.mark.parametrize("use_pynwb", [True, False])
 def test_multiple_unit_tables(tmp_path, use_pynwb):
     from pynwb.misc import Units
+    from pynwb import NWBHDF5IO
+    from pynwb.testing.mock.file import mock_NWBFile
 
     nwbfile = mock_NWBFile()
 

src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py

@@ -1,54 +1,15 @@
 from pathlib import Path
 import pickle
-from tabnanny import check
 
 import pytest
 import numpy as np
-import h5py
 
 from spikeinterface import load_extractor
 from spikeinterface.core.testing import check_recordings_equal
 from spikeinterface.core.testing import check_recordings_equal, check_sortings_equal
 from spikeinterface.extractors import NwbRecordingExtractor, NwbSortingExtractor
 
 
-@pytest.mark.streaming_extractors
-@pytest.mark.skipif("ros3" not in h5py.registered_drivers(), reason="ROS3 driver not installed")
-def test_recording_s3_nwb_ros3(tmp_path):
-    file_path = (
-        "https://dandi-api-staging-dandisets.s3.amazonaws.com/blobs/5f4/b7a/5f4b7a1f-7b95-4ad8-9579-4df6025371cc"
-    )
-    rec = NwbRecordingExtractor(file_path, stream_mode="ros3")
-
-    start_frame = 0
-    end_frame = 300
-    num_frames = end_frame - start_frame
-
-    num_seg = rec.get_num_segments()
-    num_chans = rec.get_num_channels()
-    dtype = rec.get_dtype()
-
-    for segment_index in range(num_seg):
-        num_samples = rec.get_num_samples(segment_index=segment_index)
-
-        full_traces = rec.get_traces(segment_index=segment_index, start_frame=start_frame, end_frame=end_frame)
-        assert full_traces.shape == (num_frames, num_chans)
-        assert full_traces.dtype == dtype
-
-    if rec.has_scaleable_traces():
-        trace_scaled = rec.get_traces(segment_index=segment_index, return_scaled=True, end_frame=2)
-        assert trace_scaled.dtype == "float32"
-
-    tmp_file = tmp_path / "test_ros3_recording.pkl"
-    with open(tmp_file, "wb") as f:
-        pickle.dump(rec, f)
-
-    with open(tmp_file, "rb") as f:
-        reloaded_recording = pickle.load(f)
-
-    check_recordings_equal(rec, reloaded_recording)
-
-
 @pytest.mark.streaming_extractors
 @pytest.mark.parametrize("cache", [True, False])  # Test with and without cache
 def test_recording_s3_nwb_fsspec(tmp_path, cache):
@@ -154,37 +115,6 @@ def test_recording_s3_nwb_remfile_file_like(tmp_path):
     check_recordings_equal(rec, rec2)
 
 
-@pytest.mark.streaming_extractors
-@pytest.mark.skipif("ros3" not in h5py.registered_drivers(), reason="ROS3 driver not installed")
-def test_sorting_s3_nwb_ros3(tmp_path):
-    file_path = "https://dandiarchive.s3.amazonaws.com/blobs/84b/aa4/84baa446-cf19-43e8-bdeb-fc804852279b"
-    # we provide the 'sampling_frequency' because the NWB file does not the electrical series
-    sort = NwbSortingExtractor(file_path, sampling_frequency=30000, stream_mode="ros3", t_start=0)
-
-    start_frame = 0
-    end_frame = 300
-    num_frames = end_frame - start_frame
-
-    num_seg = sort.get_num_segments()
-    num_units = len(sort.unit_ids)
-
-    for segment_index in range(num_seg):
-        for unit in sort.unit_ids:
-            spike_train = sort.get_unit_spike_train(unit_id=unit, segment_index=segment_index)
-            assert len(spike_train) > 0
-            assert spike_train.dtype == "int64"
-            assert np.all(spike_train >= 0)
-
-    tmp_file = tmp_path / "test_ros3_sorting.pkl"
-    with open(tmp_file, "wb") as f:
-        pickle.dump(sort, f)
-
-    with open(tmp_file, "rb") as f:
-        reloaded_sorting = pickle.load(f)
-
-    check_sortings_equal(reloaded_sorting, sort)
-
-
 @pytest.mark.streaming_extractors
 @pytest.mark.parametrize("cache", [True, False])  # Test with and without cache
 def test_sorting_s3_nwb_fsspec(tmp_path, cache):

src/spikeinterface/postprocessing/tests/common_extension_tests.py

@@ -2,9 +2,7 @@
 
 import pytest
 import numpy as np
-import pandas as pd
 import shutil
-import platform
 from pathlib import Path
 
 from spikeinterface.core import generate_ground_truth_recording

src/spikeinterface/preprocessing/motion.py

@@ -5,7 +5,6 @@
 
 import numpy as np
 import json
-import copy
 
 from spikeinterface.core import get_noise_levels, fix_job_kwargs
 from spikeinterface.core.job_tools import _shared_job_kwargs_doc

src/spikeinterface/preprocessing/tests/test_detect_bad_channels.py

@@ -1,6 +1,5 @@
 import pytest
 import numpy as np
-import scipy.stats
 
 from spikeinterface import NumpyRecording, get_random_data_chunks
 from probeinterface import generate_linear_probe
@@ -167,6 +166,8 @@ def test_detect_bad_channels_ibl(num_channels):
         channel_flags_ibl[:, i] = channel_flags
 
     # Take the mode of the chunk estimates as final result. Convert to binary good / bad channel output.
+    import scipy.stats
+
     bad_channel_labels_ibl, _ = scipy.stats.mode(channel_flags_ibl, axis=1, keepdims=False)
 
     # Compare