Merge branch 'zooniverse:master' into leveling_up_fix

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+format = pylint
+exclude = dist/*,cover/*,build/*,.ropeproject/*,docs/*,__init__.py
+ignore = E402,E501,E722,E741,W503,E128,BLK100,B006,B001,B023,B028

.github/workflows/publish-to-pypi.yml

@@ -8,16 +8,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.7
+    - name: Set up Python 3.9
       uses: actions/setup-python@v4
       with:
-        python-version: 3.7
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel
+        pip install flit
     - name: Build
-      run: python setup.py sdist bdist_wheel
+      run: flit build
     - name: Publish to PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
       with:

.github/workflows/publish-to-test-pypi.yml

@@ -8,16 +8,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.7
+    - name: Set up Python 3.9
       uses: actions/setup-python@v4
       with:
-        python-version: 3.7
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel
+        pip install flit
     - name: Build
-      run: python setup.py sdist bdist_wheel
+      run: flit build
     - name: Publish to Test PyPi
       uses: pypa/gh-action-pypi-publish@release/v1
       with:

.github/workflows/python-versions.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9] # , "3.10"] include 3.10 later on (requires test runner update)
+        python-version: [3.8, 3.9, "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
@@ -30,9 +30,11 @@ jobs:
     - name: Run tests
       env:
         TRAVIS: true  # one test is skipped on CI and looks for this env value
-      run: nosetests
+      run: |
+        coverage run
+        coverage report
     - name: Coveralls
-      if: ${{ matrix.python-version == 3.9 }}
+      if: ${{ matrix.python-version == 3.10 }}
       env: 
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: coveralls --service=github

.hound.yml

@@ -1,3 +1,3 @@
 flake8:
   enabled: true
-  config_file: setup.cfg
+  config_file: .flake8

Dockerfile

@@ -5,11 +5,14 @@ WORKDIR /usr/src/aggregation
 
 RUN apt-get update && apt-get -y upgrade && \
     apt-get install --no-install-recommends -y \
-    build-essential libgeos-dev && \
+    build-essential libgeos-dev git && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # install dependencies
-COPY setup.py .
+RUN mkdir -p panoptes_aggregation/version
+COPY pyproject.toml README.md ./
+COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/
+COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/
 RUN pip install --upgrade pip
 RUN pip install .[online,test,doc]
 

Dockerfile.bin_cmds

@@ -5,10 +5,13 @@ ENV LANG=C.UTF-8
 
 WORKDIR /usr/src/aggregation
 
-RUN apt-get update && apt-get install  --no-install-recommends -y libgeos-dev
+RUN apt-get update && apt-get install  --no-install-recommends -y libgeos-dev git
 
 # install dependencies
-COPY setup.py .
+RUN mkdir -p panoptes_aggregation/version
+COPY pyproject.toml README.md ./
+COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/
+COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/
 RUN pip install --upgrade pip
 RUN pip install .[test]
 

README.md

@@ -13,7 +13,7 @@ You can find the [latest documentation](https://aggregation-caesar.zooniverse.or
 ---
 
 ## Installing for offline use
-### With your own python install (python 3 only)
+### With your own python install (python 3.8 or higher only)
 Instal the latest stable release:
 ```bash
 pip install panoptes_aggregation
@@ -40,31 +40,9 @@ Or for the latest development build from GitHub:
 pip install -U git+https://github.com/zooniverse/aggregation-for-caesar.git#egg=panoptes-aggregation[gui]
 ```
 
-#### Anaconda build of python
-If your are using the anaconda version of python some of the dependencies should be installed using the `conda` package manager before installing `panoptes_aggregation`:
+On linux systems you may need to install GTK3:
 ```bash
-conda install -c conda-forge python-levenshtein hdbscan wxpython
-conda install psutil
-```
-
-#### Mac Anaconda build
-If you are installing this code on a Mac using the anaconda build of python and you want to use the GUI instead of the command line you will have to update one line of the of code in the `panoptes_aggregation_gui` script.  Change the first line from:
-```python
-#!/path/to/anaconda/python/bin/python
-```
-to:
-```python
-#!/bin/bash /path/to/anaconda/python/bin/python.app
-```
-
-You can find the location of this file with the command:
-```bash
-which panoptes_aggregation_gui
-```
-
-You will also need to run:
-```bash
-conda install python.app
+sudo apt-get install build-essential libgtk-3-dev
 ```
 
 ### With Docker
@@ -93,7 +71,7 @@ docker run -it --rm --name config_workflow_panoptes -v "$PWD":/usr/src/aggregati
 ---
 
 ## Installing for online use
-The docker file included is ready to be deployed on any server.  Once deployed, the extractors will be available on the `/extractors/<name of extractor function>` routes and the reducers will be available on the `/reducers/<name of reducer function>` routes.  Any keywords passed into these functions should be included as url parameters on the route (e.g. `https://aggregation-caesar.zooniverse.org/extractors/point_extractor_by_frame?task=T0`).  For more complex keywords (e.g. `detals` for subtasks), python's [urllib.parse.urlencode](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode) can be used to translate a keyword list into the proper url encoding.
+The docker file included is ready to be deployed on any server.  Once deployed, the extractors will be available on the `/extractors/<name of extractor function>` routes and the reducers will be available on the `/reducers/<name of reducer function>` routes.  Any keywords passed into these functions should be included as url parameters on the route (e.g. `https://aggregation-caesar.zooniverse.org/extractors/point_extractor_by_frame?task=T0`).  For more complex keywords (e.g. `details` for subtasks), python's [urllib.parse.urlencode](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode) can be used to translate a keyword list into the proper url encoding.
 
 The documentation will be built and available on the `/docs` route.
 

docs/source/Contributing.md

@@ -55,8 +55,8 @@ The code is auto-documented using [sphinx](http://www.sphinx-doc.org/en/stable/i
 4. Build the docs with the `make_docs.sh` bash script
 
 ### 5. Make sure everything still works
-1. run `nosetests` and ensure all tests still pass
-2. (optional) `nosetests --cover-html` to compile an html page for checking what parts of the code are not covered
+1. run `coverage run` and ensure all tests still pass
+2. (optional) run `coverage report` to check tests coverage in each file
 
 ---
 
@@ -99,8 +99,8 @@ The code is auto-documented using [sphinx](http://www.sphinx-doc.org/en/stable/i
 4. Build the docs with the `make_docs.sh` bash script
 
 ### 4. Make sure everything still works
-1. run `nosetests` and ensure all tests still pass (coverage is automatically reported)
-2. (optional) `nosetests --cover-html` to compile an html page for checking what parts of the code are not covered
+1. run `coverage run` and ensure all tests still pass
+2. (optional) `coverage report` to checking what parts of the code are not covered
 
 ---
 

panoptes_aggregation/__init__.py

@@ -3,9 +3,20 @@
 # warnings.filterwarnings("always")
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
 warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
-from . import extractors
-from . import reducers
-from . import running_reducers
-from . import scripts
-from . import version
-__version__ = version.__version__
+from .version import __version__
+
+
+# work around until https://github.com/pypa/flit/pull/382 (or similar) is merged into flit
+def within_flit():
+    import traceback
+    for frame in traceback.extract_stack():
+        if frame.name == "get_docstring_and_version_via_import":
+            return True
+    return False
+
+
+if not within_flit():
+    from . import extractors
+    from . import reducers
+    from . import running_reducers
+    from . import scripts

panoptes_aggregation/reducers/shape_metric_IoU.py

@@ -102,7 +102,9 @@ def IoU_metric(params1, params2, shape):
     '''
     geo1 = panoptes_to_geometry(params1, shape)
     geo2 = panoptes_to_geometry(params2, shape)
-    intersection = geo1.intersection(geo2).area
+    intersection = 0
+    if geo1.intersects(geo2):
+        intersection = geo1.intersection(geo2).area
     union = geo1.union(geo2).area
     if union == 0:
         # catch divide by zero (i.e. cases when neither shape has an area)

panoptes_aggregation/routes.py

@@ -1,7 +1,9 @@
 try:
     from flask import jsonify, request, Flask
-    from flask.json import JSONEncoder
+    from flask.json.provider import JSONProvider
     from flask_cors import CORS
+    from json import JSONEncoder
+    import json
     from panoptes_aggregation import panoptes
     import sentry_sdk
     from sentry_sdk.integrations.flask import FlaskIntegration
@@ -17,6 +19,7 @@
 import numpy as np
 
 
+# see https://stackoverflow.com/a/75666126
 class MyEncoder(JSONEncoder):
     def default(self, obj):
         if isinstance(obj, np.integer):
@@ -31,6 +34,15 @@ def default(self, obj):
             return super(MyEncoder, self).default(obj)
 
 
+# see https://stackoverflow.com/a/75666126
+class CustomJSONProvider(JSONProvider):
+    def dumps(self, obj, **kwargs):
+        return json.dumps(obj, **kwargs, cls=MyEncoder)
+
+    def loads(self, s, **kwargs):
+        return json.loads(s, **kwargs)
+
+
 def request_wrapper(name):
     '''
     Example use of process_wrapper:
@@ -68,7 +80,8 @@ def make_application():
                         instance_relative_config=True,
                         static_url_path='',
                         static_folder='../docs/build/html')
-    application.json_encoder = MyEncoder
+    application.json_provider_class = CustomJSONProvider
+    application.json = CustomJSONProvider(application)
     CORS(
         application,
         origins=[

panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py

@@ -125,7 +125,7 @@ def setUp(self):
         self.reduced_dataframe_survey = pandas.read_csv(StringIO(reduced_csv_survey))
 
     def test_first_filter(self):
-        '''Test frist filter'''
+        '''Test first filter'''
         extracted_dataframe = pandas.read_csv(self.extracted_csv_question, parse_dates=['created_at'])
         task_T0_csv = extracted_dataframe[(extracted_dataframe.task == 'T0') & (extracted_dataframe.subject_id == 1)]
         expected = task_T0_csv[[True, False]]

panoptes_aggregation/version/__init__.py

@@ -1 +1 @@
-__version__ = '4.0.0'
+__version__ = '4.1.0'

pyproject.toml

@@ -0,0 +1,100 @@
+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "panoptes_aggregation"
+description = "Aggregation code for Zooniverse panoptes projects."
+authors = [
+    {name = "Coleman Krawczyk", email = "[email protected]"},
+]
+readme = "README.md"
+license = {file = "LICENSE"}
+classifiers = [
+    "Programming Language :: Python :: 3 :: Only",
+    "License :: OSI Approved :: Apache Software License"
+]
+dynamic = ["version"]
+requires-python = ">=3.8,<3.12"
+dependencies = [
+    "beautifulsoup4>=4.8.1,<4.13",
+    "collatex@git+https://github.com/interedition/collatex.git@174035dda549ac3f967919a8475fdeb8dbb48f40#subdirectory=collatex-pythonport",
+    "hdbscan>=0.8.20,<=0.8.33",
+    "lxml>=4.4,<4.10",
+    "numpy>=1.22.0,<1.25.3",
+    "packaging>=20.1,<23.2",
+    "pandas>=1.4.0,<2.1.1",
+    "progressbar2>=3.39,<4.3",
+    "python-levenshtein>=0.21.0,<0.22",
+    "python-slugify>=7.0.0,<8.1",
+    "pyyaml>=6.0,<6.1",
+    "scikit-learn>=1.2.0,<1.3.1",
+    "scipy>=1.10.0,<1.10.2",
+    "werkzeug>=2.3.0,<2.3.8",
+    "shapely>=2.0,<2.0.2"
+]
+
+[project.optional-dependencies]
+online = [
+    "flask>=2.3,<2.4",
+    "flask-cors>=3.0,<3.1",
+    "panoptes-client>=1.6,<1.7",
+    "requests>=2.28,<2.32",
+    "gunicorn>=20.0,<20.2",
+    "sentry-sdk[flask]>=1.0,<1.30",
+    "newrelic>=8.4.0,<8.8.1",
+    "gitpython>=3.0.0,<3.2"
+]
+gui = [
+    "Gooey>=1.0.8.1,<1.1"
+]
+doc = [
+    "matplotlib>=3.5.1,<3.8",
+    "myst-nb>=0.13.2,<0.18",
+    "sphinx>=5.2.0,<7.3",
+    "sphinxcontrib-httpdomain>=1.7.0,<1.9",
+    "sphinx_rtd_theme>=0.4.3,<1.3"
+]
+test = [
+    "coverage>=4.5.3,<7.3",
+    "coveralls>=3.0.0,<3.3.2",
+    "flake8>=6.0,<6.1",
+    "flake8-black>=0.3.4,<0.4",
+    "flake8-bugbear>=23.5,<23.8",
+    "pytest>=7.1.2,<7.4.2",
+    "pytest-subtests>=0.10.0,<0.11.1"
+]
+
+[project.scripts]
+panoptes_aggregation = "panoptes_aggregation.scripts.aggregation_parser:main"
+
+[project.gui-scripts]
+panoptes_aggregation_gui = "panoptes_aggregation.scripts.gui:gui"
+
+[project.urls]
+Documentation = "https://aggregation-caesar.zooniverse.org/docs"
+Source = "https://github.com/zooniverse/aggregation-for-caesar"
+
+[tool.flit.sdist]
+include = [
+    "panoptes_aggregation/scripts/icons/*",
+    "LICENSE"
+]
+exclude = [
+    "docs/",
+    "kubernetes",
+    "make_docs.sh"
+]
+
+[tool.coverage.run]
+omit = [
+    "*test*",
+    "panoptes_aggregation/scripts/gui*.py",
+    "panoptes_aggregation/scripts/no_gooey.py",
+    "panoptes_aggregation/scripts/path_type.py"
+]
+source = ["panoptes_aggregation"]
+command_line = "-m pytest"
+
+[tool.coverage.report]
+show_missing = true