Merge branch 'main' of https://github.com/GazzolaLab/MiV-OS into main

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = E203, E266, E501, W503, F403, F401, W191, C901
+ignore = E203, E266, E501, W503, F403, F401, W191, C901, E704, E701
 max-line-length = 120
 max-complexity = 18
 select = B,C,E,F,W,T4,B9

.github/workflows/ci.yml

@@ -0,0 +1,55 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, update-** ]
+  pull_request:
+    branches: [ '**' ]
+
+jobs:
+  report-coverage:  # Report coverage from python 3.8 and mac-os. May change later
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        os: [ubuntu-latest]
+        include:
+          - os: ubuntu-latest
+            path: ~/.cache/pip
+    steps:
+      - uses: actions/checkout@v4
+      - uses: mpi4py/setup-mpi@v1
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-in-project: true
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+      - uses: FedericoCarboni/setup-ffmpeg@v3
+        id: setup-ffmpeg
+        with:
+          ffmpeg-version: release
+          architecture: ''
+          linking-type: static
+          github-token: ${{ github.server_url == 'https://github.com' && github.token || '' }}
+      - name: Install dependencies
+        run: |
+          #poetry lock --no-update
+          poetry install --all-extras --with=algorithmExtension,sortingExtension,mpi
+      # Set environment variables for coverage test. Coverage test is done using python 3.8
+      - name: Test MiV-OS using pytest
+        run: |
+          make test_ci
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          env_vars: OS,PYTHON
+          fail_ci_if_error: true
+          flags: unittests
+          name: codecov-umbrella
+          verbose: true
+          file: ./coverage.xml

.github/workflows/main.yml

@@ -12,12 +12,12 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
-        os: [macos-latest] #, ubuntu-latest]
+        os: [macos-13, ubuntu-latest]
         include:
-          - os: macos-latest
+          - os: macos-13
             path: ~/Library/Caches/pip
-          #- os: ubuntu-latest
-          #  path: ~/.cache/pip
+          - os: ubuntu-latest
+            path: ~/.cache/pip
           #- os: windows-latest
           #  path: ~\AppData\Local\pip\Cache
     defaults:
@@ -31,14 +31,27 @@ jobs:
         with:
           virtualenvs-in-project: true
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'poetry'
       - uses: FedericoCarboni/setup-ffmpeg@v3
         id: setup-ffmpeg
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          # like "6.1.0". At the moment semver specifiers (i.e. >=6.1.0) are supported
+          # only on Windows, on other platforms they are allowed but version is matched
+          # exactly regardless.
+          ffmpeg-version: release
+          # Target architecture of the ffmpeg executable to install. Defaults to the
+          # system architecture. Only x64 and arm64 are supported (arm64 only on Linux).
+          architecture: ''
+          # Linking type of the binaries. Use "shared" to download shared binaries and
+          # "static" for statically linked ones. Shared builds are currently only available
+          # for windows releases. Defaults to "static"
+          linking-type: static
+          # As of version 3 of this action, builds are no longer downloaded from GitHub
+          # except on Windows: https://github.com/GyanD/codexffmpeg/releases.
+          github-token: ${{ github.server_url == 'https://github.com' && github.token || '' }}
       # - name: Set up cache
       #   uses: actions/cache@v3
       #   with:
@@ -55,56 +68,3 @@ jobs:
         run: |
           source $VENV
           make test
-  report-coverage:  # Report coverage from python 3.8 and mac-os. May change later
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-version: ["3.10"]
-        os: [macos-latest]
-        include:
-          - os: macos-latest
-            path: ~/Library/Caches/pip
-    steps:
-      - uses: actions/checkout@v4
-      - uses: mpi4py/setup-mpi@v1
-      - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-in-project: true
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/[email protected]
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
-      - uses: FedericoCarboni/setup-ffmpeg@v3
-        id: setup-ffmpeg
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      # - name: Set up cache
-      #   uses: actions/cache@v3
-      #   with:
-      #     path: ${{ matrix.path }}
-      #     key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}
-      #     restore-keys: |
-      #       ${{ runner.os }}-pip-
-      - name: Install dependencies
-        run: |
-          #poetry lock --no-update
-          poetry install --all-extras --with=algorithmExtension,sortingExtension,mpi
-      # Set environment variables for coverage test. Coverage test is done using python 3.8
-      - name: Run style checks
-        run: |
-          make check-codestyle
-      - name: Test MiV-OS using pytest
-        if: startsWith(runner.os, 'macOS')
-        run: |
-          make test_ci
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          env_vars: OS,PYTHON
-          fail_ci_if_error: true
-          flags: unittests
-          name: codecov-umbrella
-          verbose: true

.github/workflows/publish-to-pypi.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.x"
       - name: Install Poetry

miv/core/datatype/collapsable.py

@@ -5,8 +5,7 @@
 
 class _Collapsable(Protocol):
     @classmethod
-    def from_collapse(self) -> None:
-        ...
+    def from_collapse(self) -> None: ...
 
 
 class CollapseExtendableMixin:

miv/core/datatype/protocol.py

@@ -4,26 +4,21 @@
 
 
 class Extendable(Protocol):
-    def extend(self, other) -> None:
-        ...
+    def extend(self, other) -> None: ...
 
 
 ChannelWiseSelf = TypeVar("ChannelWiseSelf", bound="ChannelWise")
 
 
 class ChannelWise(Protocol):
     @property
-    def number_of_channels(self) -> int:
-        ...
+    def number_of_channels(self) -> int: ...
 
-    def append(self, other) -> None:
-        ...
+    def append(self, other) -> None: ...
 
-    def insert(self, index, other) -> None:
-        ...
+    def insert(self, index, other) -> None: ...
 
-    def __setitem__(self, index, other) -> None:
-        ...
+    def __setitem__(self, index, other) -> None: ...
 
     def select(self, indices: List[int]) -> ChannelWiseSelf:
         """

miv/core/datatype/pure_python.py

@@ -9,8 +9,7 @@
 
 class RawValuesProtocol(Protocol):
     @staticmethod
-    def is_valid(value) -> bool:
-        ...
+    def is_valid(value) -> bool: ...
 
 
 class ValuesMixin(BaseChainingMixin):

miv/core/datatype/spikestamps.py

@@ -142,6 +142,14 @@ def neo(self):
             for arr in self.data
         ]
 
+    def flatten(self):
+        """Flatten spikestamps into a single array. One can plot the spikestamps using this array with scatter plot."""
+        x, y = [], []
+        for idx, arr in enumerate(self.data):
+            x.extend(arr)
+            y.extend([idx] * len(arr))
+        return np.array(x), np.array(y)
+
     def binning(
         self,
         bin_size: float = 1 * pq.ms,

miv/core/functools.py

@@ -0,0 +1,112 @@
+__all__ = ["ParallelGeneratorFetch"]
+
+import os
+import sys
+from multiprocessing import Process, Queue
+from queue import Empty
+
+
+class ExceptionItem:
+    def __init__(self, exception):
+        self.exception = exception
+
+
+class ParallelGeneratorException(Exception):
+    pass
+
+
+class ParallelGeneratorFetch:
+    def __init__(self, orig_gen, max_lookahead=None, get_timeout=10):
+        """
+        Creates a parallel generator from a normal one.
+        The elements will be prefetched up to max_lookahead
+        ahead of the consumer. If max_lookahead is None,
+        everything will be fetched.
+
+        The get_timeout parameter is the number of seconds
+        after which we check that the subprocess is still
+        alive, when waiting for an element to be generated.
+
+        Any exception raised in the generator will
+        be forwarded to this parallel generator.
+        """
+        if max_lookahead:
+            self.queue = Queue(max_lookahead)
+        else:
+            self.queue = Queue()
+
+        def wrapped():
+            try:
+                for item in orig_gen:
+                    self.queue.put(item)
+                raise StopIteration()
+            except Exception as e:
+                self.queue.put(ExceptionItem(e))
+
+        self.get_timeout = get_timeout
+        self.ppid = None  # pid of the parent process
+        self.process = Process(target=wrapped)
+        self.process_started = False
+
+    def finish_if_possible(self):
+        """
+        We can only terminate the child process from the parent process
+        """
+        if self.ppid == os.getpid() and self.process:  # and self.process.is_alive():
+            self.process.terminate()
+            self.process = None
+            self.queue = None
+            self.ppid = None
+
+    def __enter__(self):
+        """
+        Starts the process
+        """
+        self.ppid = os.getpid()
+        self.process.start()
+        self.process_started = True
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """
+        Kills the process
+        """
+        assert self.process_started and self.ppid is None or self.ppid == os.getpid()
+        self.finish_if_possible()
+
+    def __next__(self):
+        return self.next()
+
+    def __iter__(self):
+        return self
+
+    def __del__(self):
+        self.finish_if_possible()
+
+    def next(self):
+        if not self.process_started:
+            raise ParallelGeneratorException(
+                """The generator has not been started.
+                   Please use "with ParallelGenerator(..) as g:"
+                """
+            )
+        try:
+            item_received = False
+            while not item_received:
+                try:
+                    item = self.queue.get(timeout=self.get_timeout)
+                    item_received = True
+                except Empty:
+                    # check that the process is still alive
+                    if not self.process.is_alive():
+                        raise ParallelGeneratorException(
+                            "The generator died unexpectedly."
+                        )
+
+            if type(item) == ExceptionItem:
+                raise item.exception
+            return item
+
+        except Exception:
+            self.finish_if_possible()
+            raise