diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7edc99d2..00000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[report] -show_missing = true -omit = - panoptes_aggregation/tests/* - panoptes_aggregation/scripts/gui*.py - panoptes_aggregation/scripts/no_gooey.py - panoptes_aggregation/scripts/path_type.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..18f6ccfd --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +format = pylint +exclude = dist/*,cover/*,build/*,.ropeproject/*,docs/*,__init__.py +ignore = E402,E501,E722,E741,W503,E128,BLK100,B006,B001,B023,B028 diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 6384415b..4bde7158 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -8,16 +8,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install flit - name: Build - run: python setup.py sdist bdist_wheel + run: flit build - name: Publish to PyPi uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index dfed20f2..4b63f767 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -8,16 +8,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install flit - name: Build - run: python setup.py sdist bdist_wheel + run: flit build - name: Publish to Test PyPi uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/python-versions.yml b/.github/workflows/python-versions.yml index f9b498c7..fe359eeb 100644 --- a/.github/workflows/python-versions.yml +++ b/.github/workflows/python-versions.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] # , "3.10"] include 3.10 later on (requires test runner update) + python-version: [3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -30,9 +30,11 @@ jobs: - name: Run tests env: TRAVIS: true # one test is skipped on CI and looks for this env value - run: nosetests + run: | + coverage run + coverage report - name: Coveralls - if: ${{ matrix.python-version == 3.9 }} + if: ${{ matrix.python-version == 3.10 }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --service=github diff --git a/.hound.yml b/.hound.yml index de5adb31..09d8c1ba 100644 --- a/.hound.yml +++ b/.hound.yml @@ -1,3 +1,3 @@ flake8: enabled: true - config_file: setup.cfg + config_file: .flake8 diff --git a/Dockerfile b/Dockerfile index 9838f54e..5740f767 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,14 @@ WORKDIR /usr/src/aggregation RUN apt-get update && apt-get -y upgrade && \ apt-get install --no-install-recommends -y \ - build-essential libgeos-dev && \ + build-essential libgeos-dev git && \ apt-get clean && rm -rf /var/lib/apt/lists/* # install dependencies -COPY setup.py . +RUN mkdir -p panoptes_aggregation/version +COPY pyproject.toml README.md ./ +COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/ +COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/ RUN pip install --upgrade pip RUN pip install .[online,test,doc] diff --git a/Dockerfile.bin_cmds b/Dockerfile.bin_cmds index f500dcf5..ddeefb35 100644 --- a/Dockerfile.bin_cmds +++ b/Dockerfile.bin_cmds @@ -5,10 +5,13 @@ ENV LANG=C.UTF-8 WORKDIR /usr/src/aggregation -RUN apt-get update && apt-get install --no-install-recommends -y libgeos-dev +RUN apt-get update && apt-get install --no-install-recommends -y libgeos-dev git # install dependencies -COPY setup.py . +RUN mkdir -p panoptes_aggregation/version +COPY pyproject.toml README.md ./ +COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/ +COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/ RUN pip install --upgrade pip RUN pip install .[test] diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index fe75f5c5..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include panoptes_aggregation/scripts/icons/* -include LICENSE diff --git a/README.md b/README.md index aa1e1e7d..8555a44d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can find the [latest documentation](https://aggregation-caesar.zooniverse.or --- ## Installing for offline use -### With your own python install (python 3 only) +### With your own python install (python 3.8 or higher only) Instal the latest stable release: ```bash pip install panoptes_aggregation @@ -40,31 +40,9 @@ Or for the latest development build from GitHub: pip install -U git+https://github.com/zooniverse/aggregation-for-caesar.git#egg=panoptes-aggregation[gui] ``` -#### Anaconda build of python -If your are using the anaconda version of python some of the dependencies should be installed using the `conda` package manager before installing `panoptes_aggregation`: +On linux systems you may need to install GTK3: ```bash -conda install -c conda-forge python-levenshtein hdbscan wxpython -conda install psutil -``` - -#### Mac Anaconda build -If you are installing this code on a Mac using the anaconda build of python and you want to use the GUI instead of the command line you will have to update one line of the of code in the `panoptes_aggregation_gui` script. Change the first line from: -```python -#!/path/to/anaconda/python/bin/python -``` -to: -```python -#!/bin/bash /path/to/anaconda/python/bin/python.app -``` - -You can find the location of this file with the command: -```bash -which panoptes_aggregation_gui -``` - -You will also need to run: -```bash -conda install python.app +sudo apt-get install build-essential libgtk-3-dev ``` ### With Docker @@ -93,7 +71,7 @@ docker run -it --rm --name config_workflow_panoptes -v "$PWD":/usr/src/aggregati --- ## Installing for online use -The docker file included is ready to be deployed on any server. Once deployed, the extractors will be available on the `/extractors/` routes and the reducers will be available on the `/reducers/` routes. Any keywords passed into these functions should be included as url parameters on the route (e.g. `https://aggregation-caesar.zooniverse.org/extractors/point_extractor_by_frame?task=T0`). For more complex keywords (e.g. `detals` for subtasks), python's [urllib.parse.urlencode](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode) can be used to translate a keyword list into the proper url encoding. +The docker file included is ready to be deployed on any server. Once deployed, the extractors will be available on the `/extractors/` routes and the reducers will be available on the `/reducers/` routes. Any keywords passed into these functions should be included as url parameters on the route (e.g. `https://aggregation-caesar.zooniverse.org/extractors/point_extractor_by_frame?task=T0`). For more complex keywords (e.g. `details` for subtasks), python's [urllib.parse.urlencode](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode) can be used to translate a keyword list into the proper url encoding. The documentation will be built and available on the `/docs` route. diff --git a/docs/source/Contributing.md b/docs/source/Contributing.md index d1c9a84e..551fe7a1 100644 --- a/docs/source/Contributing.md +++ b/docs/source/Contributing.md @@ -55,8 +55,8 @@ The code is auto-documented using [sphinx](http://www.sphinx-doc.org/en/stable/i 4. Build the docs with the `make_docs.sh` bash script ### 5. Make sure everything still works -1. run `nosetests` and ensure all tests still pass -2. (optional) `nosetests --cover-html` to compile an html page for checking what parts of the code are not covered +1. run `coverage run` and ensure all tests still pass +2. (optional) run `coverage report` to check tests coverage in each file --- @@ -99,8 +99,8 @@ The code is auto-documented using [sphinx](http://www.sphinx-doc.org/en/stable/i 4. Build the docs with the `make_docs.sh` bash script ### 4. Make sure everything still works -1. run `nosetests` and ensure all tests still pass (coverage is automatically reported) -2. (optional) `nosetests --cover-html` to compile an html page for checking what parts of the code are not covered +1. run `coverage run` and ensure all tests still pass +2. (optional) `coverage report` to checking what parts of the code are not covered --- diff --git a/panoptes_aggregation/__init__.py b/panoptes_aggregation/__init__.py index 7fa8060e..dd7d3850 100644 --- a/panoptes_aggregation/__init__.py +++ b/panoptes_aggregation/__init__.py @@ -3,9 +3,20 @@ # warnings.filterwarnings("always") warnings.filterwarnings("ignore", message="numpy.dtype size changed") warnings.filterwarnings("ignore", message="numpy.ufunc size changed") -from . import extractors -from . import reducers -from . import running_reducers -from . import scripts -from . import version -__version__ = version.__version__ +from .version import __version__ + + +# work around until https://github.com/pypa/flit/pull/382 (or similar) is merged into flit +def within_flit(): + import traceback + for frame in traceback.extract_stack(): + if frame.name == "get_docstring_and_version_via_import": + return True + return False + + +if not within_flit(): + from . import extractors + from . import reducers + from . import running_reducers + from . import scripts diff --git a/panoptes_aggregation/reducers/shape_metric_IoU.py b/panoptes_aggregation/reducers/shape_metric_IoU.py index 22859809..57b20747 100644 --- a/panoptes_aggregation/reducers/shape_metric_IoU.py +++ b/panoptes_aggregation/reducers/shape_metric_IoU.py @@ -102,7 +102,9 @@ def IoU_metric(params1, params2, shape): ''' geo1 = panoptes_to_geometry(params1, shape) geo2 = panoptes_to_geometry(params2, shape) - intersection = geo1.intersection(geo2).area + intersection = 0 + if geo1.intersects(geo2): + intersection = geo1.intersection(geo2).area union = geo1.union(geo2).area if union == 0: # catch divide by zero (i.e. cases when neither shape has an area) diff --git a/panoptes_aggregation/routes.py b/panoptes_aggregation/routes.py index bf39d2e7..857e99ce 100644 --- a/panoptes_aggregation/routes.py +++ b/panoptes_aggregation/routes.py @@ -1,7 +1,9 @@ try: from flask import jsonify, request, Flask - from flask.json import JSONEncoder + from flask.json.provider import JSONProvider from flask_cors import CORS + from json import JSONEncoder + import json from panoptes_aggregation import panoptes import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration @@ -17,6 +19,7 @@ import numpy as np +# see https://stackoverflow.com/a/75666126 class MyEncoder(JSONEncoder): def default(self, obj): if isinstance(obj, np.integer): @@ -31,6 +34,15 @@ def default(self, obj): return super(MyEncoder, self).default(obj) +# see https://stackoverflow.com/a/75666126 +class CustomJSONProvider(JSONProvider): + def dumps(self, obj, **kwargs): + return json.dumps(obj, **kwargs, cls=MyEncoder) + + def loads(self, s, **kwargs): + return json.loads(s, **kwargs) + + def request_wrapper(name): ''' Example use of process_wrapper: @@ -68,7 +80,8 @@ def make_application(): instance_relative_config=True, static_url_path='', static_folder='../docs/build/html') - application.json_encoder = MyEncoder + application.json_provider_class = CustomJSONProvider + application.json = CustomJSONProvider(application) CORS( application, origins=[ diff --git a/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py b/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py index a1d9d8df..1fbc6faa 100644 --- a/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py +++ b/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py @@ -125,7 +125,7 @@ def setUp(self): self.reduced_dataframe_survey = pandas.read_csv(StringIO(reduced_csv_survey)) def test_first_filter(self): - '''Test frist filter''' + '''Test first filter''' extracted_dataframe = pandas.read_csv(self.extracted_csv_question, parse_dates=['created_at']) task_T0_csv = extracted_dataframe[(extracted_dataframe.task == 'T0') & (extracted_dataframe.subject_id == 1)] expected = task_T0_csv[[True, False]] diff --git a/panoptes_aggregation/version/__init__.py b/panoptes_aggregation/version/__init__.py index d6497a81..fa721b49 100644 --- a/panoptes_aggregation/version/__init__.py +++ b/panoptes_aggregation/version/__init__.py @@ -1 +1 @@ -__version__ = '4.0.0' +__version__ = '4.1.0' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..08081d97 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "panoptes_aggregation" +description = "Aggregation code for Zooniverse panoptes projects." +authors = [ + {name = "Coleman Krawczyk", email = "coleman@zooniverse.org"}, +] +readme = "README.md" +license = {file = "LICENSE"} +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "License :: OSI Approved :: Apache Software License" +] +dynamic = ["version"] +requires-python = ">=3.8,<3.12" +dependencies = [ + "beautifulsoup4>=4.8.1,<4.13", + "collatex@git+https://github.com/interedition/collatex.git@174035dda549ac3f967919a8475fdeb8dbb48f40#subdirectory=collatex-pythonport", + "hdbscan>=0.8.20,<=0.8.33", + "lxml>=4.4,<4.10", + "numpy>=1.22.0,<1.25.3", + "packaging>=20.1,<23.2", + "pandas>=1.4.0,<2.1.1", + "progressbar2>=3.39,<4.3", + "python-levenshtein>=0.21.0,<0.22", + "python-slugify>=7.0.0,<8.1", + "pyyaml>=6.0,<6.1", + "scikit-learn>=1.2.0,<1.3.1", + "scipy>=1.10.0,<1.10.2", + "werkzeug>=2.3.0,<2.3.8", + "shapely>=2.0,<2.0.2" +] + +[project.optional-dependencies] +online = [ + "flask>=2.3,<2.4", + "flask-cors>=3.0,<3.1", + "panoptes-client>=1.6,<1.7", + "requests>=2.28,<2.32", + "gunicorn>=20.0,<20.2", + "sentry-sdk[flask]>=1.0,<1.30", + "newrelic>=8.4.0,<8.8.1", + "gitpython>=3.0.0,<3.2" +] +gui = [ + "Gooey>=1.0.8.1,<1.1" +] +doc = [ + "matplotlib>=3.5.1,<3.8", + "myst-nb>=0.13.2,<0.18", + "sphinx>=5.2.0,<7.3", + "sphinxcontrib-httpdomain>=1.7.0,<1.9", + "sphinx_rtd_theme>=0.4.3,<1.3" +] +test = [ + "coverage>=4.5.3,<7.3", + "coveralls>=3.0.0,<3.3.2", + "flake8>=6.0,<6.1", + "flake8-black>=0.3.4,<0.4", + "flake8-bugbear>=23.5,<23.8", + "pytest>=7.1.2,<7.4.2", + "pytest-subtests>=0.10.0,<0.11.1" +] + +[project.scripts] +panoptes_aggregation = "panoptes_aggregation.scripts.aggregation_parser:main" + +[project.gui-scripts] +panoptes_aggregation_gui = "panoptes_aggregation.scripts.gui:gui" + +[project.urls] +Documentation = "https://aggregation-caesar.zooniverse.org/docs" +Source = "https://github.com/zooniverse/aggregation-for-caesar" + +[tool.flit.sdist] +include = [ + "panoptes_aggregation/scripts/icons/*", + "LICENSE" +] +exclude = [ + "docs/", + "kubernetes", + "make_docs.sh" +] + +[tool.coverage.run] +omit = [ + "*test*", + "panoptes_aggregation/scripts/gui*.py", + "panoptes_aggregation/scripts/no_gooey.py", + "panoptes_aggregation/scripts/path_type.py" +] +source = ["panoptes_aggregation"] +command_line = "-m pytest" + +[tool.coverage.report] +show_missing = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ddd939c2..00000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -format = pylint -exclude = dist/*,cover/*,build/*,.ropeproject/*,docs/*,__init__.py -ignore = E402,E501,E722,E741,W503,E128,BLK100,B006,B001 - -[nosetests] -with-coverage = 1 -cover-package = panoptes_aggregation -cover-erase = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index f30d64cb..00000000 --- a/setup.py +++ /dev/null @@ -1,103 +0,0 @@ -from setuptools import setup, find_packages -import re -import os - -here = os.path.abspath(os.path.dirname(__file__)) - -try: - with open(os.path.join(here, 'README.md'), 'r') as fh: - long_description = fh.read() -except FileNotFoundError: - long_description = '' - -try: - with open(os.path.join(here, 'panoptes_aggregation/version/__init__.py'), 'r') as fp: - version_file = fp.read() - version_match = re.search( - r"^__version__ = ['\"]([^'\"]*)['\"]", - version_file, - re.M - ) - if version_match: - VERSION = version_match.group(1) - else: - raise RuntimeError("Unable to find version string.") -except FileNotFoundError: - VERSION = '0.0.0' - - -setup( - name='panoptes_aggregation', - python_requires='>=3', - version=VERSION, - description='Aggregation code for Zooniverse panoptes projects.', - long_description=long_description, - long_description_content_type='text/markdown', - license='Apache License 2.0', - classifiers=[ - 'Programming Language :: Python :: 3 :: Only', - 'License :: OSI Approved :: Apache Software License' - ], - url='https://github.com/zooniverse/aggregation-for-caesar', - author='Coleman Krawczyk', - author_email='coleman@zooniverse.org', - test_suite='nose.collector', - tests_require=['nose'], - entry_points={ - 'console_scripts': [ - 'panoptes_aggregation = panoptes_aggregation.scripts.aggregation_parser:main' - ], - 'gui_scripts': [ - 'panoptes_aggregation_gui = panoptes_aggregation.scripts.gui:gui' - ] - }, - packages=find_packages(), - include_package_data=True, - extras_require={ - 'online': [ - 'flask>=1.0,<2.3', - 'flask-cors>=3.0,<3.1', - 'panoptes-client>=1.1,<1.7', - 'requests>=2.4.2,<2.31', - 'gunicorn>=20.0,<20.2', - 'sentry-sdk[flask]>=0.13.5,<1.25', - 'newrelic>=5.4.0,<8.8.1', - 'gitpython>=3.0.0,<3.2' - ], - 'doc': [ - 'matplotlib>=3.5.1,<3.8', - 'myst-nb>=0.13.2,<0.18', - 'sphinx>=2.2.2,<6.2', - 'sphinxcontrib-httpdomain>=1.7.0,<1.9', - 'sphinx_rtd_theme>=0.4.3,<1.3' - ], - 'test': [ - 'nose>=1.3.7,<1.4', - 'coverage>=4.5.3,<7.3', - 'coveralls>=3.0.0,<3.3.2', - 'flake8>=3.7,<6.1', - 'flake8-black>=0.1.1,<0.4', - 'flake8-bugbear>=20.1.2,<23.3' - ], - 'gui': [ - 'Gooey>=1.0.3,<1.1' - ] - }, - install_requires=[ - 'beautifulsoup4>=4.8.1,<4.13', - 'collatex>=2.2,<2.3', - 'hdbscan>=0.8.20,<=0.8.33', - 'lxml>=4.4,<4.10', - 'numpy>=1.21.5,<1.25.2', - 'packaging>=20.1,<23.1', - 'pandas>=1.0.0,<1.5.4', - 'progressbar2>=3.39,<4.3', - 'python-levenshtein>=0.12.0,<0.21', - 'python-slugify>=3.0.0,<8.1', - 'pyyaml>=5.1,<6.1', - 'scikit-learn>=1.0.0,<1.2.3', - 'scipy>=1.2,<1.10.2', - 'werkzeug>=0.14,<2.3.5', - 'shapely>=1.7.1,<2.0.2', - ] -)