diff --git a/.github/environment-ci.yml b/.github/environment-ci.yml new file mode 100644 index 0000000..c3c85e0 --- /dev/null +++ b/.github/environment-ci.yml @@ -0,0 +1,17 @@ +name: pescador-dev +channels: + - conda-forge + - defaults +dependencies: + # required + - pip + - six>=1.8 + - pyzmq>=18.0 + - decorator>=4.0 + + # optional, but required for testing + - pytest + - pytest-cov + - pytest-timeout>=1.2 + - coverage + - scipy diff --git a/.github/environment-minimal.yml b/.github/environment-minimal.yml new file mode 100644 index 0000000..c3ce1bf --- /dev/null +++ b/.github/environment-minimal.yml @@ -0,0 +1,17 @@ +name: pescador-dev +channels: + - conda-forge + - defaults +dependencies: + # required + - pip + - six~=1.8 + - pyzmq~=18.0 + - decorator~=4.0 + + # optional, but required for testing + - pytest + - pytest-cov + - pytest-timeout>=1.2 + - coverage + - scipy diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9d350a1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,110 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: True + +jobs: + test: + name: "Python ${{ matrix.python-version }} on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python-version: "3.7" + envfile: ".github/environment-minimal.yml" + channel-priority: "flexible" + name: "Minimal dependencies" + + - os: ubuntu-latest + python-version: "3.8" + channel-priority: "strict" + envfile: ".github/environment-ci.yml" + + - os: ubuntu-latest + python-version: "3.9" + channel-priority: "strict" + envfile: ".github/environment-ci.yml" + + - os: ubuntu-latest + python-version: "3.10" + channel-priority: "strict" + envfile: ".github/environment-ci.yml" + + - os: ubuntu-latest + python-version: "3.11" + channel-priority: "strict" + envfile: ".github/environment-ci.yml" + + - os: macos-latest + python-version: "3.11" + channel-priority: "strict" + envfile: ".github/environment-ci.yml" + + - os: windows-latest + python-version: "3.11" + channel-priority: "strict" + envfile: ".github/environment-ci.yml" + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Cache conda + uses: actions/cache@v3 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles( matrix.envfile ) }} + + - name: Install Conda environment + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + add-pip-as-python-dependency: true + auto-activate-base: false + activate-environment: test + channel-priority: ${{ matrix.channel-priority }} + environment-file: ${{ matrix.envfile }} + use-only-tar-bz2: false # IMPORTANT: This needs to be set for caching to work properly! + + - name: Conda info + shell: bash -l {0} + run: | + conda info -a + conda list + + - name: Install pescador + shell: bash -l {0} + run: python -m pip install --upgrade-strategy only-if-needed -e .[tests] + + - name: Run pytest + shell: bash -l {0} + run: pytest + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + directory: ./coverage/reports/ + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: true + verbose: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a385523..0000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -language: python - -cache: - directories: - - $HOME/env - -notifications: - email: false - -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - -before_install: - - bash .travis_dependencies.sh - - export PATH="$HOME/env/miniconda$TRAVIS_PYTHON_VERSION/bin:$PATH"; - - hash -r - - source activate test-environment - -install: - # install your own package into the environment - - pip install -e .[tests] - -script: - - pytest - -after_success: - - coveralls - - pip uninstall -y pescador - -after_failure: - - pip uninstall -y pescador diff --git a/.travis_dependencies.sh b/.travis_dependencies.sh deleted file mode 100644 index ba30791..0000000 --- a/.travis_dependencies.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh - -ENV_NAME="test-environment" -set -e - -conda_create () -{ - - hash -r - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda config --add channels pypi - conda info -a - deps='pip numpy scipy pyzmq' - - conda create -q -n $ENV_NAME "python=$TRAVIS_PYTHON_VERSION" $deps -} - -src="$HOME/env/miniconda$TRAVIS_PYTHON_VERSION" -if [ ! -f "$HOME/env/miniconda.sh" ]; then - mkdir -p $HOME/env - pushd $HOME/env - - # Download miniconda packages - wget http://repo.continuum.io/miniconda/Miniconda-3.16.0-Linux-x86_64.sh -O miniconda.sh; - - # Install both environments - bash miniconda.sh -b -p $src - - export PATH="$src/bin:$PATH" - conda_create - - source activate $ENV_NAME - - pip install coveralls - - source deactivate - - popd -else - echo "Using cached dependencies" -fi diff --git a/README.md b/README.md index 93f675f..42ef418 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ pescador ======== [![PyPI](https://img.shields.io/pypi/v/pescador.svg)](https://pypi.python.org/pypi/pescador) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pescador/badges/version.svg)](https://anaconda.org/conda-forge/pescador) -[![Build Status](https://travis-ci.org/pescadores/pescador.svg?branch=master)](https://travis-ci.org/pescadores/pescador) -[![Coverage Status](https://coveralls.io/repos/pescadores/pescador/badge.svg)](https://coveralls.io/r/pescadores/pescador) +[![Build Status](https://github.com/pescadores/pescador/actions/workflows/ci.yml/badge.svg)](https://github.com/pescadores/pescador/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/pescadores/pescador/graph/badge.svg?token=aCgfizw6O5)](https://codecov.io/gh/pescadores/pescador) [![Documentation Status](https://readthedocs.org/projects/pescador/badge/?version=latest)](https://readthedocs.org/projects/pescador/?badge=latest) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.400700.svg)](https://doi.org/10.5281/zenodo.400700) diff --git a/pescador/version.py b/pescador/version.py index 4b569d2..8eded2e 100644 --- a/pescador/version.py +++ b/pescador/version.py @@ -2,5 +2,5 @@ # -*- coding: utf-8 -*- """Version info""" -short_version = '2.1' -version = '2.1.0' +short_version = '3.0' +version = '3.0.0-dev' diff --git a/pescador/zmq_stream.py b/pescador/zmq_stream.py index feee2c0..9a56af7 100644 --- a/pescador/zmq_stream.py +++ b/pescador/zmq_stream.py @@ -16,22 +16,12 @@ import multiprocessing as mp import zmq import numpy as np -import six -import sys -import warnings try: import ujson as json except ImportError: import json -try: - # joblib <= 0.9.4 - from joblib.parallel import SafeFunction -except ImportError: - # joblib >= 0.10.0 - from joblib._parallel_backends import SafeFunction - from .core import Streamer from .exceptions import DataError @@ -39,11 +29,6 @@ __all__ = ['ZMQStreamer'] -# A hack to support buffers in py3 -if six.PY3: - buffer = memoryview - - def zmq_send_data(socket, data, flags=0, copy=True, track=False): """Send data, e.g. {key: np.ndarray}, with metadata""" @@ -82,12 +67,9 @@ def zmq_recv_data(socket, flags=0, copy=True, track=False): raise StopIteration for header, payload in zip(headers, msg[1:]): - data[header['key']] = np.frombuffer(buffer(payload), + data[header['key']] = np.frombuffer(memoryview(payload), dtype=header['dtype']) data[header['key']].shape = header['shape'] - if six.PY2: - # Legacy python won't let us preserve alignment, skip this step - continue data[header['key']].flags['ALIGNED'] = header['aligned'] return data @@ -178,10 +160,6 @@ def iterate(self, max_iter=None): """ context = zmq.Context() - if six.PY2: - warnings.warn('zmq_stream cannot preserve numpy array alignment ' - 'in Python 2', RuntimeWarning) - try: socket = context.socket(zmq.PAIR) @@ -191,7 +169,7 @@ def iterate(self, max_iter=None): max_tries=self.max_tries) terminate = mp.Event() - worker = mp.Process(target=SafeFunction(zmq_worker), + worker = mp.Process(target=zmq_worker, args=[port, self.streamer, terminate], kwargs=dict(copy=self.copy, max_iter=max_iter)) @@ -206,13 +184,10 @@ def iterate(self, max_iter=None): except StopIteration: pass - except: - # pylint: disable-msg=W0702 - six.reraise(*sys.exc_info()) - finally: terminate.set() - worker.join(self.timeout) + if worker.is_alive(): + worker.join(self.timeout) if worker.is_alive(): worker.terminate() context.destroy() diff --git a/setup.cfg b/setup.cfg index 8d118aa..677003d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,46 @@ [tool:pytest] -addopts = --cov-report term-missing --cov pescador +addopts = --cov-report term-missing --cov pescador --cov-report=xml + +[metadata] +name = pescador +version = attr: pescador.version.version +description = Multiplex generators for incremental learning +author = Pescador developers +author_email = brian.mcfee@nyu.edu +url = https://github.com/pescadores/pescador +download_url = https://github.com/pescadores/pescador/releases +long_description = Multiplex generators for incremental learning +license = ISC +python_requires = ">=3.7" +classifiers = + License :: OSI Approved :: ISC License (ISCL) + Programming Language :: Python + Development Status :: 3 - Alpha + Intended Audience :: Developers + Topic :: Multimedia :: Sound/Audio :: Analysis + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + +[options] +packages = find: +keywords = machine learning +install_requires = + six >= 1.8 + pyzmq >= 18.0 + numpy >= 1.9 + decorator >= 4.0 + +[options.extras_require] +docs = + numpydoc >= 0.6 + sphinx-gallery >= 0.1.10 +tests = + pytest + pytest-timeout>=1.2 + pytest-cov + scipy + diff --git a/setup.py b/setup.py index 2fe0ad3..6b40b52 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import setup -import imp - -version = imp.load_source('pescador.version', 'pescador/version.py') - -setup( - name='pescador', - version=version.version, - description='Multiplex generators for incremental learning', - author='Pescador developers', - author_email='brian.mcfee@nyu.edu', - url='https://github.com/pescadores/pescador', - download_url='https://github.com/pescadores/pescador/releases', - packages=find_packages(), - long_description='Multiplex generators for incremental learning', - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", - classifiers=[ - "License :: OSI Approved :: ISC License (ISCL)", - "Programming Language :: Python", - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Topic :: Multimedia :: Sound/Audio :: Analysis", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - ], - keywords='machine learning', - license='ISC', - install_requires=[ - 'joblib>=0.9', - 'six>=1.8', - 'pyzmq>=15.0', - 'numpy>=1.9', - 'decorator>=4.0' - ], - extras_require={ - 'docs': ['numpydoc>=0.6', - 'sphinx-gallery>=0.1.10'], - 'tests': ['pytest<=4.0', 'pytest-timeout>=1.2', 'pytest-cov'] - } -) +if __name__ == '__main__': + setup() diff --git a/tests/test_maps.py b/tests/test_maps.py index 9fa4fb4..8e062a0 100644 --- a/tests/test_maps.py +++ b/tests/test_maps.py @@ -124,16 +124,18 @@ def test_keras_tuples(sample_data): pass -@pytest.mark.parametrize('n_cache', [2, 4, 8, 64, - pytest.mark.xfail(-1, - raises=pescador.PescadorError)]) -@pytest.mark.parametrize('prob', [0.1, 0.5, 1, - pytest.mark.xfail(-1, - raises=pescador.PescadorError), - pytest.mark.xfail(0, - raises=pescador.PescadorError), - pytest.mark.xfail(1.5, - raises=pescador.PescadorError)]) +@pytest.mark.parametrize( + 'n_cache', + [2, 4, 8, 64, + pytest.param(-1, marks=pytest.mark.xfail(raises=pescador.PescadorError))] +) +@pytest.mark.parametrize( + 'prob', + [0.1, 0.5, 1, + pytest.param(-1, marks=pytest.mark.xfail(raises=pescador.PescadorError)), + pytest.param(0, marks=pytest.mark.xfail(raises=pescador.PescadorError)), + pytest.param(1.5, marks=pytest.mark.xfail(raises=pescador.PescadorError))] +) def test_cache(n_cache, prob): data = list(range(32)) diff --git a/tests/test_mux.py b/tests/test_mux.py index 60ccbe5..f0ec291 100644 --- a/tests/test_mux.py +++ b/tests/test_mux.py @@ -88,16 +88,9 @@ def test_mux_single_infinite(mux_class): functools.partial(pescador.mux.StochasticMux, n_active=1, rate=256), pescador.mux.ShuffledMux, pescador.mux.RoundRobinMux, - pytest.mark.xfail(pescador.mux.ChainMux, - reason="ChainMux can accept an empty iterable or " - "generator, and will simply return empty.", - strict=True), -], - ids=["StochasticMux-exhaustive", - "ShuffledMux", - "RoundRobinMux", - "ChainMux" - ]) + pytest.param(pescador.mux.ChainMux, marks=pytest.mark.xfail(reason="ChainMux can accept an empty iterable or generator, and will simply return empty.", strict=True)), +], ids=["StochasticMux-exhaustive", "ShuffledMux", "RoundRobinMux", "ChainMux"]) + def test_mux_empty(mux_class): "Make sure an empty list of streamers raises an error." with pytest.raises(pescador.PescadorError): @@ -241,8 +234,8 @@ def test_deepcopy__randomseed(self, mux_class, random_state): class TestStochasticMux: @pytest.mark.parametrize( 'mode', ['with_replacement', 'single_active', 'exhaustive', - pytest.mark.xfail('foo', raises=pescador.PescadorError), - pytest.mark.xfail(None, raises=pescador.PescadorError)]) + pytest.param('foo', marks=pytest.mark.xfail(raises=pescador.PescadorError)), + pytest.param(None, marks=pytest.mark.xfail(raises=pescador.PescadorError))]) @pytest.mark.parametrize('n_samples', [10]) def test_valid_modes(self, mode, n_samples): """Simply tests the modes to make sure they work.""" @@ -254,8 +247,7 @@ def test_valid_modes(self, mode, n_samples): assert len(output) == n_samples def test_multiple_copies(self): - """Check that the Mux class can be activated multiple times successfully. - """ + """Check that the Mux class can be activated multiple times successfully.""" ab = pescador.Streamer('ab') cde = pescador.Streamer('cde') fghi = pescador.Streamer('fghi') @@ -267,8 +259,7 @@ def test_multiple_copies(self): # No streamers should be active until we actually start the generators assert mux.active == 0 - # grab one sample each to make sure we've actually started the - # generator + # grab one sample each to make sure we've actually started the generators _ = next(gen1) _ = next(gen2) assert mux.active == 2 @@ -292,21 +283,17 @@ class TestStochasticMux_WithReplacement: @pytest.mark.parametrize('n_samples', [10, 20, 80]) @pytest.mark.parametrize('n_active', [1, 2, 4]) @pytest.mark.parametrize('rate', [1.0, 2.0, 8.0]) - @pytest.mark.parametrize('random_state', - [None, - 1000, - np.random.RandomState(seed=1000), - pytest.mark.xfail( - 'foo', raises=pescador.PescadorError, - strict=True), - ]) - def test_mux_replacement(self, mux_class, n_streams, n_samples, n_active, - rate, random_state): + @pytest.mark.parametrize('random_state', [ + None, + 1000, + np.random.RandomState(seed=1000), + pytest.param('foo', marks=pytest.mark.xfail(raises=pescador.PescadorError, strict=True)), + ]) + def test_mux_replacement(self, mux_class, n_streams, n_samples, n_active, rate, random_state): streamers = [pescador.Streamer(T.infinite_generator) for _ in range(n_streams)] - mux = mux_class(streamers, n_active, rate=rate, - random_state=random_state) + mux = mux_class(streamers, n_active, rate=rate, random_state=random_state) estimate = list(mux.iterate(n_samples)) # Make sure we get the right number of samples @@ -327,11 +314,13 @@ def test_mux_k_greater_n(self, mux_class, n_samples, rate, random_state): a = pescador.Streamer('a') b = pescador.Streamer('b') - mux = mux_class([a, b], 6, rate, random_state=random_state) + mux = mux_class([a, b], 6, rate=rate, random_state=random_state) result = list(mux.iterate(n_samples)) assert len(result) == n_samples + + class TestStochasticMux_Exhaustive: @pytest.mark.parametrize('mux_class', [ functools.partial(pescador.mux.StochasticMux, mode="exhaustive")], diff --git a/tests/test_utils.py b/tests/test_utils.py index 7160c55..37da23f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -70,11 +70,13 @@ def __zip_generator(n, size1, size2): @pytest.mark.parametrize( - 'n1,n2', [pytest.mark.xfail((5, 10), raises=pescador.util.PescadorError), - pytest.mark.xfail((5, 15), raises=pescador.util.PescadorError), - pytest.mark.xfail((10, 5), raises=pescador.util.PescadorError), - pytest.mark.xfail((15, 5), raises=pescador.util.PescadorError), - (5, 5), (10, 10), (15, 15)]) + 'n1, n2', + [pytest.param(5, 10, marks=pytest.mark.xfail(raises=pescador.util.PescadorError)), + pytest.param(5, 15, marks=pytest.mark.xfail(raises=pescador.util.PescadorError)), + pytest.param(10, 5, marks=pytest.mark.xfail(raises=pescador.util.PescadorError)), + pytest.param(15, 5, marks=pytest.mark.xfail(raises=pescador.util.PescadorError)), + (5, 5), (10, 10), (15, 15)] +) def test_batch_length(n1, n2): generator, n = __zip_generator(3, n1, n2), n1 diff --git a/tests/test_zmq_stream.py b/tests/test_zmq_stream.py index 79a12ad..fd61770 100644 --- a/tests/test_zmq_stream.py +++ b/tests/test_zmq_stream.py @@ -44,13 +44,13 @@ def test_zmq_align(): assert b2[key].flags['ALIGNED'] -@pytest.mark.xfail(raises=pescador.PescadorError) -def test_zmq_bad_type(): +def __bad_generator(): + for _ in range(100): + yield dict(X=list(range(100))) - def __bad_generator(): - for _ in range(100): - yield dict(X=list(range(100))) +@pytest.mark.xfail(raises=pescador.PescadorError) +def test_zmq_bad_type(): stream = pescador.Streamer(__bad_generator)