diff --git a/.github/actions/install-wine/action.yml b/.github/actions/install-wine/action.yml index 3ae08ecd34..85e70b471d 100644 --- a/.github/actions/install-wine/action.yml +++ b/.github/actions/install-wine/action.yml @@ -2,20 +2,29 @@ name: Install packages description: This action installs the package and its dependencies for testing inputs: - python-version: - description: 'Python version to set up' - required: false os: description: 'Operating system to set up' - required: false + required: true runs: using: "composite" steps: - - name: Install wine (needed for Plexon2) + - name: Install wine on Linux + if: runner.os == 'Linux' run: | sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list sudo dpkg --add-architecture i386 sudo apt-get update -qq sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine shell: bash + - name: Install wine on macOS + if: runner.os == 'macOS' + run: | + brew install --cask xquartz + brew install --cask wine-stable + shell: bash + + - name: Skip installation on Windows + if: ${{ inputs.os == 'Windows' }} + run: echo "Skipping Wine installation on Windows. Not necessary." + shell: bash diff --git a/.github/determine_testing_environment.py b/.github/determine_testing_environment.py index 4945ccc807..0c0c5ef95b 100644 --- a/.github/determine_testing_environment.py +++ b/.github/determine_testing_environment.py @@ -36,50 +36,47 @@ file_is_in_src = changed_file.parts[0] == "src" - if not file_is_in_src: - - if changed_file.name == "pyproject.toml": - pyproject_toml_changed = True - - else: - if changed_file.name == "neobaseextractor.py": - neobaseextractor_changed = True - elif changed_file.name == "plexon2.py": - extractors_changed = True - elif "core" in changed_file.parts: - conditions_changed = True - elif "extractors" in changed_file.parts: - extractors_changed = True - elif "preprocessing" in changed_file.parts: - preprocessing_changed = True - elif "postprocessing" in changed_file.parts: - postprocessing_changed = True - elif "qualitymetrics" in changed_file.parts: - qualitymetrics_changed = True - elif "comparison" in changed_file.parts: - comparison_changed = True - elif "curation" in changed_file.parts: - curation_changed = True - elif "widgets" in changed_file.parts: - widgets_changed = True - elif "exporters" in changed_file.parts: - exporters_changed = True - elif "sortingcomponents" in changed_file.parts: - sortingcomponents_changed = True - elif "generation" in changed_file.parts: - generation_changed = True - elif "sorters" in changed_file.parts: - if "external" in changed_file.parts: - sorters_external_changed = True - elif "internal" in changed_file.parts: - sorters_internal_changed = True - else: - sorters_changed = True + + if changed_file.name == "pyproject.toml": + pyproject_toml_changed = True + elif changed_file.name == "neobaseextractor.py": + neobaseextractor_changed = True + elif changed_file.name == "plexon2.py": + plexon2_changed = True + elif "core" in changed_file.parts: + core_changed = True + elif "extractors" in changed_file.parts: + extractors_changed = True + elif "preprocessing" in changed_file.parts: + preprocessing_changed = True + elif "postprocessing" in changed_file.parts: + postprocessing_changed = True + elif "qualitymetrics" in changed_file.parts: + qualitymetrics_changed = True + elif "comparison" in changed_file.parts: + comparison_changed = True + elif "curation" in changed_file.parts: + curation_changed = True + elif "widgets" in changed_file.parts: + widgets_changed = True + elif "exporters" in changed_file.parts: + exporters_changed = True + elif "sortingcomponents" in changed_file.parts: + sortingcomponents_changed = True + elif "generation" in changed_file.parts: + generation_changed = True + elif "sorters" in changed_file.parts: + if "external" in changed_file.parts: + sorters_external_changed = True + elif "internal" in changed_file.parts: + sorters_internal_changed = True + else: + sorters_changed = True run_everything = core_changed or pyproject_toml_changed or neobaseextractor_changed run_generation_tests = run_everything or generation_changed -run_extractor_tests = run_everything or extractors_changed +run_extractor_tests = run_everything or extractors_changed or plexon2_changed run_preprocessing_tests = run_everything or preprocessing_changed run_postprocessing_tests = run_everything or postprocessing_changed run_qualitymetrics_tests = run_everything or qualitymetrics_changed diff --git a/.github/workflows/all-tests.yml b/.github/workflows/all-tests.yml index cce73a9008..8dfea50faf 100644 --- a/.github/workflows/all-tests.yml +++ b/.github/workflows/all-tests.yml @@ -78,9 +78,8 @@ jobs: run: pytest -m "core" shell: bash - - name: Install Other Testing Dependencies + - name: Install Dependencies for Timing Display run: | - pip install -e .[test] pip install tabulate pip install pandas shell: bash @@ -91,6 +90,7 @@ jobs: run: echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT - name: Cache datasets + if: env.RUN_EXTRACTORS_TESTS == 'true' id: cache-datasets uses: actions/cache/restore@v4 with: @@ -119,6 +119,9 @@ jobs: fi git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency + - name : Install Plexon dependencies + if: env.INSTALL_PLEXON_DEPENDENCIES == 'true' + uses: ./.github/actions/install-wine - name: Set execute permissions on run_tests.sh shell: bash @@ -130,16 +133,21 @@ jobs: HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_plugin_path_maxwell if: env.RUN_EXTRACTORS_TESTS == 'true' run: | - pip install -e .[extractors,streaming_extractors] + pip install -e .[extractors,streaming_extractors,test_extractors] ./.github/run_tests.sh "extractors and not streaming_extractors" --no-virtual-env - name: Test preprocessing shell: bash if: env.RUN_PREPROCESSING_TESTS == 'true' run: | - pip install -e .[preprocessing] + pip install -e .[preprocessing,test_preprocessing] ./.github/run_tests.sh "preprocessing and not deepinterpolation" --no-virtual-env + - name: Install remaining testing dependencies # TODO: Remove this step once we have better modularization + shell: bash + run: | + pip install -e .[test] + - name: Test postprocessing shell: bash if: env.RUN_POSTPROCESSING_TESTS == 'true' diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml deleted file mode 100644 index ed2f28dc23..0000000000 --- a/.github/workflows/full-test.yml +++ /dev/null @@ -1,167 +0,0 @@ -name: Full spikeinterface tests - -on: - pull_request: - types: [synchronize, opened, reopened] - branches: - - main - -concurrency: # Cancel previous workflows on the same pull request - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: # For the sortingview backend - KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }} - KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }} - -jobs: - full-tests-depending-on-changed-files: - name: Test on (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # "macos-latest", "windows-latest" - os: ["ubuntu-latest", ] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Get current year-month - id: date - run: echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT - - name: Get ephy_testing_data current head hash - # the key depends on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git - id: vars - run: | - echo "HASH_EPHY_DATASET=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT - - name: Restore cached gin data for extractors tests - uses: actions/cache/restore@v4 - id: cache-datasets - env: - # the key depends on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git - HASH_EPHY_DATASET: git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1 - with: - path: ~/spikeinterface_datasets - key: ${{ runner.os }}-datasets-${{ steps.vars.outputs.HASH_EPHY_DATASET }} - restore-keys: ${{ runner.os }}-datasets - - name: Install packages - uses: ./.github/actions/build-test-environment - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v41 - - name: Module changes - id: modules-changed - run: | - for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - if [[ $file == *"pyproject.toml" ]]; then - echo "pyproject.toml changed" - echo "CORE_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/core/"* || $file == *"/extractors/neoextractors/neobaseextractor.py" ]]; then - echo "Core changed" - echo "CORE_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/extractors/"* ]]; then - echo "Extractors changed" - echo "EXTRACTORS_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"plexon2"* ]]; then - echo "Plexon2 changed" - echo "PLEXON2_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/preprocessing/"* ]]; then - echo "Preprocessing changed" - echo "PREPROCESSING_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/postprocessing/"* ]]; then - echo "Postprocessing changed" - echo "POSTPROCESSING_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/qualitymetrics/"* ]]; then - echo "Quality metrics changed" - echo "QUALITYMETRICS_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/sorters/"* && $file != *"/sorters/internal/"* && $file != *"/sorters/external/"* ]]; then - echo "Sorters changed" - echo "SORTERS_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/sorters/external"* ]]; then - echo "External sorters changed" - echo "SORTERS_EXTERNAL_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/sorters/internal"* ]]; then - echo "Internal sorters changed" - echo "SORTERS_INTERNAL_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/comparison/"* ]]; then - echo "Comparison changed" - echo "COMPARISON_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/curation/"* ]]; then - echo "Curation changed" - echo "CURATION_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/widgets/"* ]]; then - echo "Widgets changed" - echo "WIDGETS_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/exporters/"* ]]; then - echo "Exporters changed" - echo "EXPORTERS_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/sortingcomponents/"* ]]; then - echo "Sortingcomponents changed" - echo "SORTINGCOMPONENTS_CHANGED=true" >> $GITHUB_OUTPUT - fi - if [[ $file == *"/generation/"* ]]; then - echo "Generation changed" - echo "GENERATION_CHANGED=true" >> $GITHUB_OUTPUT - fi - done - - name: Set execute permissions on run_tests.sh - run: chmod +x .github/run_tests.sh - - name: Install Wine (Plexon2) - if: ${{ steps.modules-changed.outputs.PLEXON2_CHANGED == 'true' }} - uses: ./.github/actions/install-wine - - name: Test core - run: ./.github/run_tests.sh core - - name: Test extractors - env: - HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_plugin_path_maxwell - if: ${{ steps.modules-changed.outputs.EXTRACTORS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh "extractors and not streaming_extractors" - - name: Test preprocessing - if: ${{ steps.modules-changed.outputs.PREPROCESSING_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh "preprocessing and not deepinterpolation" - - name: Test postprocessing - if: ${{ steps.modules-changed.outputs.POSTPROCESSING_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh postprocessing - - name: Test quality metrics - if: ${{ steps.modules-changed.outputs.QUALITYMETRICS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh qualitymetrics - - name: Test core sorters - if: ${{ steps.modules-changed.outputs.SORTERS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh sorters - - name: Test comparison - if: ${{ steps.modules-changed.outputs.COMPARISON_CHANGED == 'true' || steps.modules-changed.outputs.GENERATION_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh comparison - - name: Test curation - if: ${{ steps.modules-changed.outputs.CURATION_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh curation - - name: Test widgets - if: ${{ steps.modules-changed.outputs.WIDGETS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' || steps.modules-changed.outputs.QUALITYMETRICS_CHANGED == 'true' || steps.modules-changed.outputs.PREPROCESSING_CHANGED == 'true'}} - run: ./.github/run_tests.sh widgets - - name: Test exporters - if: ${{ steps.modules-changed.outputs.EXPORTERS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' || steps.modules-changed.outputs.WIDGETS_CHANGED == 'true' }} - run: ./.github/run_tests.sh exporters - - name: Test sortingcomponents - if: ${{ steps.modules-changed.outputs.SORTINGCOMPONENTS_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh sortingcomponents - - name: Test internal sorters - if: ${{ steps.modules-changed.outputs.SORTERS_INTERNAL_CHANGED == 'true' || steps.modules-changed.outputs.SORTINGCOMPONENTS_CHANGED || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh sorters_internal - - name: Test generation - if: ${{ steps.modules-changed.outputs.GENERATION_CHANGED == 'true' || steps.modules-changed.outputs.CORE_CHANGED == 'true' }} - run: ./.github/run_tests.sh generation diff --git a/pyproject.toml b/pyproject.toml index 69f4067d13..7108f121e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ streaming_extractors = [ "s3fs" ] + preprocessing = [ "scipy", ] @@ -124,10 +125,23 @@ test_core = [ # for github test : probeinterface and neo from master # for release we need pypi, so this need to be commented - # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git", - # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git", + "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git", + "neo @ git+https://github.com/NeuralEnsemble/python-neo.git", +] + +test_extractors = [ + # Functions to download data in neo test suite + "pooch>=1.8.2", + "datalad>=1.0.2", + "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git", + "neo @ git+https://github.com/NeuralEnsemble/python-neo.git", ] +test_preprocessing = [ + "ibllib>=2.36.0", # for IBL +] + + test = [ "pytest", "pytest-dependency", @@ -145,9 +159,7 @@ test = [ # for sortingview backend "sortingview", - # Download data - "pooch>=1.8.2", - "datalad>=1.0.2", + ## install tridesclous for testing ## "tridesclous>=1.6.8", diff --git a/src/spikeinterface/extractors/neoextractors/plexon2.py b/src/spikeinterface/extractors/neoextractors/plexon2.py index c7351a308b..6c9160f13b 100644 --- a/src/spikeinterface/extractors/neoextractors/plexon2.py +++ b/src/spikeinterface/extractors/neoextractors/plexon2.py @@ -13,8 +13,8 @@ class Plexon2RecordingExtractor(NeoBaseRecordingExtractor): Parameters ---------- - file_path : str - The file path to load the recordings from. + file_path : str | Path + The file path of the plexon2 file. It should have the .pl2 extension. stream_id : str, default: None If there are several streams, specify the stream id you want to load. stream_name : str, default: None @@ -23,7 +23,7 @@ class Plexon2RecordingExtractor(NeoBaseRecordingExtractor): If True, the names of the signals are used as channel ids. If False, the channel ids are a combination of the source id and the channel index. - Example for widegain signals: + Example for wideband signals: names: ["WB01", "WB02", "WB03", "WB04"] ids: ["source3.1" , "source3.2", "source3.3", "source3.4"] all_annotations : bool, default: False