From 60994079729b09ca4350ccf05b367426d42d4f4b Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:03:13 +0100 Subject: [PATCH 01/21] Convert setup tools to flit Convert `setup.py` to `pyproject.toml` and move various dev tool configs to their new expected locations. --- .coveragerc | 7 ---- .flake8 | 4 ++ MANIFEST.in | 2 - pyproject.toml | 100 +++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 9 ----- setup.py | 103 ------------------------------------------------- 6 files changed, 104 insertions(+), 121 deletions(-) delete mode 100644 .coveragerc create mode 100644 .flake8 delete mode 100644 MANIFEST.in create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7edc99d2..00000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[report] -show_missing = true -omit = - panoptes_aggregation/tests/* - panoptes_aggregation/scripts/gui*.py - panoptes_aggregation/scripts/no_gooey.py - panoptes_aggregation/scripts/path_type.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..18f6ccfd --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +format = pylint +exclude = dist/*,cover/*,build/*,.ropeproject/*,docs/*,__init__.py +ignore = E402,E501,E722,E741,W503,E128,BLK100,B006,B001,B023,B028 diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index fe75f5c5..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include panoptes_aggregation/scripts/icons/* -include LICENSE diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..233e81a5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "panoptes_aggregation" +description = "Aggregation code for Zooniverse panoptes projects." +authors = [ + {name = "Coleman Krawczyk", email = "coleman@zooniverse.org"}, +] +readme = "README.md" +license = {file = "LICENSE"} +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "License :: OSI Approved :: Apache Software License" +] +dynamic = ["version"] +requires-python = ">=3.7,<3.11" +dependencies = [ + "beautifulsoup4>=4.8.1,<4.13", + "collatex>=2.2,<2.3", + "hdbscan>=0.8.20,<=0.8.33", + "lxml>=4.4,<4.10", + "numpy>=1.22.0,<1.25.2", + "packaging>=20.1,<23.1", + "pandas>=1.0.0,<1.5.4", + "progressbar2>=3.39,<4.3", + "python-levenshtein>=0.12.0,<0.21", + "python-slugify>=3.0.0,<8.1", + "pyyaml>=5.1,<6.1", + "scikit-learn>=1.0.0,<1.2.3", + "scipy>=1.6.1,<1.10.2", + "werkzeug>=0.14,<2.3.5", + "shapely>=1.7.1,<2.0.2" +] + +[project.optional-dependencies] +online = [ + "flask>=2.0,<2.3", + "flask-cors>=3.0,<3.1", + "panoptes-client>=1.1,<1.7", + "requests>=2.4.2,<2.31", + "gunicorn>=20.0,<20.2", + "sentry-sdk[flask]>=0.13.5,<1.25", + "newrelic>=5.4.0,<8.8.1", + "gitpython>=3.0.0,<3.2" +] +gui = [ + "Gooey>=1.0.8.1,<1.1" +] +doc = [ + "matplotlib>=3.5.1,<3.8", + "myst-nb>=0.13.2,<0.18", + "sphinx>=2.2.2,<6.2", + "sphinxcontrib-httpdomain>=1.7.0,<1.9", + "sphinx_rtd_theme>=0.4.3,<1.3" +] +test = [ + "coverage>=4.5.3,<7.3", + "coveralls>=3.0.0,<3.3.2", + "flake8>=3.7,<6.1", + "flake8-black>=0.1.1,<0.4", + "flake8-bugbear>=20.1.2,<23.3", + "pytest>=7.1.2,<7.4.2", + "pytest-subtests>=0.8.0,<0.11.1" +] + +[project.scripts] +panoptes_aggregation = "panoptes_aggregation.scripts.aggregation_parser:main" + +[project.gui-scripts] +panoptes_aggregation_gui = "panoptes_aggregation.scripts.gui:gui" + +[project.urls] +Documentation = "https://aggregation-caesar.zooniverse.org/docs" +Source = "https://github.com/zooniverse/aggregation-for-caesar" + +[tool.flit.sdist] +include = [ + "panoptes_aggregation/scripts/icons/*", + "LICENSE" +] +exclude = [ + "docs/", + "kubernetes", + "make_docs.sh" +] + +[tool.coverage.run] +omit = [ + "*test*", + "panoptes_aggregation/scripts/gui*.py", + "panoptes_aggregation/scripts/no_gooey.py", + "panoptes_aggregation/scripts/path_type.py" +] +source = ["panoptes_aggregation"] +command_line = "-m pytest" + +[tool.coverage.report] +show_missing = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ddd939c2..00000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -format = pylint -exclude = dist/*,cover/*,build/*,.ropeproject/*,docs/*,__init__.py -ignore = E402,E501,E722,E741,W503,E128,BLK100,B006,B001 - -[nosetests] -with-coverage = 1 -cover-package = panoptes_aggregation -cover-erase = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index f30d64cb..00000000 --- a/setup.py +++ /dev/null @@ -1,103 +0,0 @@ -from setuptools import setup, find_packages -import re -import os - -here = os.path.abspath(os.path.dirname(__file__)) - -try: - with open(os.path.join(here, 'README.md'), 'r') as fh: - long_description = fh.read() -except FileNotFoundError: - long_description = '' - -try: - with open(os.path.join(here, 'panoptes_aggregation/version/__init__.py'), 'r') as fp: - version_file = fp.read() - version_match = re.search( - r"^__version__ = ['\"]([^'\"]*)['\"]", - version_file, - re.M - ) - if version_match: - VERSION = version_match.group(1) - else: - raise RuntimeError("Unable to find version string.") -except FileNotFoundError: - VERSION = '0.0.0' - - -setup( - name='panoptes_aggregation', - python_requires='>=3', - version=VERSION, - description='Aggregation code for Zooniverse panoptes projects.', - long_description=long_description, - long_description_content_type='text/markdown', - license='Apache License 2.0', - classifiers=[ - 'Programming Language :: Python :: 3 :: Only', - 'License :: OSI Approved :: Apache Software License' - ], - url='https://github.com/zooniverse/aggregation-for-caesar', - author='Coleman Krawczyk', - author_email='coleman@zooniverse.org', - test_suite='nose.collector', - tests_require=['nose'], - entry_points={ - 'console_scripts': [ - 'panoptes_aggregation = panoptes_aggregation.scripts.aggregation_parser:main' - ], - 'gui_scripts': [ - 'panoptes_aggregation_gui = panoptes_aggregation.scripts.gui:gui' - ] - }, - packages=find_packages(), - include_package_data=True, - extras_require={ - 'online': [ - 'flask>=1.0,<2.3', - 'flask-cors>=3.0,<3.1', - 'panoptes-client>=1.1,<1.7', - 'requests>=2.4.2,<2.31', - 'gunicorn>=20.0,<20.2', - 'sentry-sdk[flask]>=0.13.5,<1.25', - 'newrelic>=5.4.0,<8.8.1', - 'gitpython>=3.0.0,<3.2' - ], - 'doc': [ - 'matplotlib>=3.5.1,<3.8', - 'myst-nb>=0.13.2,<0.18', - 'sphinx>=2.2.2,<6.2', - 'sphinxcontrib-httpdomain>=1.7.0,<1.9', - 'sphinx_rtd_theme>=0.4.3,<1.3' - ], - 'test': [ - 'nose>=1.3.7,<1.4', - 'coverage>=4.5.3,<7.3', - 'coveralls>=3.0.0,<3.3.2', - 'flake8>=3.7,<6.1', - 'flake8-black>=0.1.1,<0.4', - 'flake8-bugbear>=20.1.2,<23.3' - ], - 'gui': [ - 'Gooey>=1.0.3,<1.1' - ] - }, - install_requires=[ - 'beautifulsoup4>=4.8.1,<4.13', - 'collatex>=2.2,<2.3', - 'hdbscan>=0.8.20,<=0.8.33', - 'lxml>=4.4,<4.10', - 'numpy>=1.21.5,<1.25.2', - 'packaging>=20.1,<23.1', - 'pandas>=1.0.0,<1.5.4', - 'progressbar2>=3.39,<4.3', - 'python-levenshtein>=0.12.0,<0.21', - 'python-slugify>=3.0.0,<8.1', - 'pyyaml>=5.1,<6.1', - 'scikit-learn>=1.0.0,<1.2.3', - 'scipy>=1.2,<1.10.2', - 'werkzeug>=0.14,<2.3.5', - 'shapely>=1.7.1,<2.0.2', - ] -) From 9400f39c013776049e0d6c61021ebcf5555314ad Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:04:31 +0100 Subject: [PATCH 02/21] Update base import for flit For flit to grab the version number from the code it needs to import the code **before** it is installed! Adjust the __init__.py file so that can be done without crashing. --- panoptes_aggregation/__init__.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/panoptes_aggregation/__init__.py b/panoptes_aggregation/__init__.py index 7fa8060e..dd7d3850 100644 --- a/panoptes_aggregation/__init__.py +++ b/panoptes_aggregation/__init__.py @@ -3,9 +3,20 @@ # warnings.filterwarnings("always") warnings.filterwarnings("ignore", message="numpy.dtype size changed") warnings.filterwarnings("ignore", message="numpy.ufunc size changed") -from . import extractors -from . import reducers -from . import running_reducers -from . import scripts -from . import version -__version__ = version.__version__ +from .version import __version__ + + +# work around until https://github.com/pypa/flit/pull/382 (or similar) is merged into flit +def within_flit(): + import traceback + for frame in traceback.extract_stack(): + if frame.name == "get_docstring_and_version_via_import": + return True + return False + + +if not within_flit(): + from . import extractors + from . import reducers + from . import running_reducers + from . import scripts From 24b9880c188774b6355c4b6833ca2e6a1a63332c Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:04:37 +0100 Subject: [PATCH 03/21] bump version --- panoptes_aggregation/version/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoptes_aggregation/version/__init__.py b/panoptes_aggregation/version/__init__.py index d6497a81..fa721b49 100644 --- a/panoptes_aggregation/version/__init__.py +++ b/panoptes_aggregation/version/__init__.py @@ -1 +1 @@ -__version__ = '4.0.0' +__version__ = '4.1.0' From ce607d1056c95430b6e4d9f2f2d09a0380ee29ed Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:05:07 +0100 Subject: [PATCH 04/21] Update docker files Not much changed except the number of files needed to do dependency install up front. --- Dockerfile | 5 ++++- Dockerfile.bin_cmds | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9838f54e..5d6d984b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,10 @@ RUN apt-get update && apt-get -y upgrade && \ apt-get clean && rm -rf /var/lib/apt/lists/* # install dependencies -COPY setup.py . +RUN mkdir -p panoptes_aggregation/version +COPY pyproject.toml README.md ./ +COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/ +COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/ RUN pip install --upgrade pip RUN pip install .[online,test,doc] diff --git a/Dockerfile.bin_cmds b/Dockerfile.bin_cmds index f500dcf5..4913c045 100644 --- a/Dockerfile.bin_cmds +++ b/Dockerfile.bin_cmds @@ -8,7 +8,10 @@ WORKDIR /usr/src/aggregation RUN apt-get update && apt-get install --no-install-recommends -y libgeos-dev # install dependencies -COPY setup.py . +RUN mkdir -p panoptes_aggregation/version +COPY pyproject.toml README.md ./ +COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/ +COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/ RUN pip install --upgrade pip RUN pip install .[test] From b4d3b1ec293d6ef6e86ec68a92e29ade8c734a68 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:05:47 +0100 Subject: [PATCH 05/21] Update GHA for flit Mostly how to build the package before publishing and running the unit tests. --- .github/workflows/publish-to-pypi.yml | 8 ++++---- .github/workflows/publish-to-test-pypi.yml | 8 ++++---- .github/workflows/python-versions.yml | 8 +++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 6384415b..72e206f3 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -8,16 +8,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install setuptools wheel flit - name: Build - run: python setup.py sdist bdist_wheel + run: flit build - name: Publish to PyPi uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index dfed20f2..24ca5609 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -8,16 +8,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install setuptools wheel flit - name: Build - run: python setup.py sdist bdist_wheel + run: flit build - name: Publish to Test PyPi uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/python-versions.yml b/.github/workflows/python-versions.yml index f9b498c7..b191bf03 100644 --- a/.github/workflows/python-versions.yml +++ b/.github/workflows/python-versions.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] # , "3.10"] include 3.10 later on (requires test runner update) + python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -30,9 +30,11 @@ jobs: - name: Run tests env: TRAVIS: true # one test is skipped on CI and looks for this env value - run: nosetests + run: | + coverage run + coverage report - name: Coveralls - if: ${{ matrix.python-version == 3.9 }} + if: ${{ matrix.python-version == "3.10" }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --service=github From cd7b8cd84cb309524175555e8e7344ae96a20085 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:08:50 +0100 Subject: [PATCH 06/21] Only need to install flit to build --- .github/workflows/publish-to-pypi.yml | 2 +- .github/workflows/publish-to-test-pypi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 72e206f3..4bde7158 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -15,7 +15,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel flit + pip install flit - name: Build run: flit build - name: Publish to PyPi diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 24ca5609..4b63f767 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -15,7 +15,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel flit + pip install flit - name: Build run: flit build - name: Publish to Test PyPi From 5b9d42d5777d88f9838745b7926e3b9c586763da Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:20:38 +0100 Subject: [PATCH 07/21] Fix type on GHA --- .github/workflows/python-versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-versions.yml b/.github/workflows/python-versions.yml index b191bf03..508a4c28 100644 --- a/.github/workflows/python-versions.yml +++ b/.github/workflows/python-versions.yml @@ -34,7 +34,7 @@ jobs: coverage run coverage report - name: Coveralls - if: ${{ matrix.python-version == "3.10" }} + if: ${{ matrix.python-version == 3.10 }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --service=github From 2b2ac13df40b00b123a1cae6cad466cb6e1aa858 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:28:23 +0100 Subject: [PATCH 08/21] drop python 3.7 tests --- .github/workflows/python-versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-versions.yml b/.github/workflows/python-versions.yml index 508a4c28..fe359eeb 100644 --- a/.github/workflows/python-versions.yml +++ b/.github/workflows/python-versions.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 From 2d8fcb5900dbd6345e657b470ac9dc507df1ba71 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 14:53:50 +0100 Subject: [PATCH 09/21] Update lower bounds on version for python 3.11 Pip backtrace for python 3.11 can try in "install" very old package version that are not compatible with 3.11 and crash the installer. Bump up lower bounds to try and reduce this happening. --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 233e81a5..ec755a17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,8 +30,8 @@ dependencies = [ "pyyaml>=5.1,<6.1", "scikit-learn>=1.0.0,<1.2.3", "scipy>=1.6.1,<1.10.2", - "werkzeug>=0.14,<2.3.5", - "shapely>=1.7.1,<2.0.2" + "werkzeug>=2.3.0,<2.3.5", + "shapely>=1.8.5,<2.0.2" ] [project.optional-dependencies] @@ -51,7 +51,7 @@ gui = [ doc = [ "matplotlib>=3.5.1,<3.8", "myst-nb>=0.13.2,<0.18", - "sphinx>=2.2.2,<6.2", + "sphinx>=5.2.0,<6.2", "sphinxcontrib-httpdomain>=1.7.0,<1.9", "sphinx_rtd_theme>=0.4.3,<1.3" ] From b6c3646d240dd872dc61ed93d8b5e8805d16c461 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Thu, 7 Sep 2023 16:50:20 +0100 Subject: [PATCH 10/21] Update package ranges The latest pip with python 3.11 does backtracing for dependency resolution, this can go faster if package ranges are smaller. Collatex has a fix on master for python 3.10 and 3.11 but not released to PyPi. For the moment the package will grab the code from git. --- pyproject.toml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ec755a17..a351d5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,34 +15,34 @@ classifiers = [ "License :: OSI Approved :: Apache Software License" ] dynamic = ["version"] -requires-python = ">=3.7,<3.11" +requires-python = ">=3.8,<3.12" dependencies = [ "beautifulsoup4>=4.8.1,<4.13", - "collatex>=2.2,<2.3", + "collatex@git+https://github.com/interedition/collatex.git@174035dda549ac3f967919a8475fdeb8dbb48f40#subdirectory=collatex-pythonport", "hdbscan>=0.8.20,<=0.8.33", "lxml>=4.4,<4.10", "numpy>=1.22.0,<1.25.2", - "packaging>=20.1,<23.1", - "pandas>=1.0.0,<1.5.4", + "packaging>=20.1,<23.2", + "pandas>=1.4.0,<1.5.4", "progressbar2>=3.39,<4.3", - "python-levenshtein>=0.12.0,<0.21", - "python-slugify>=3.0.0,<8.1", - "pyyaml>=5.1,<6.1", - "scikit-learn>=1.0.0,<1.2.3", - "scipy>=1.6.1,<1.10.2", - "werkzeug>=2.3.0,<2.3.5", - "shapely>=1.8.5,<2.0.2" + "python-levenshtein>=0.21.0,<0.22", + "python-slugify>=7.0.0,<8.1", + "pyyaml>=6.0,<6.1", + "scikit-learn>=1.2.0,<1.3.1", + "scipy>=1.10.0,<1.10.2", + "werkzeug>=2.3.0,<2.3.8", + "shapely>=2.0,<2.0.2" ] [project.optional-dependencies] online = [ - "flask>=2.0,<2.3", + "flask>=2.0,<2.4", "flask-cors>=3.0,<3.1", - "panoptes-client>=1.1,<1.7", - "requests>=2.4.2,<2.31", + "panoptes-client>=1.6,<1.7", + "requests>=2.28,<2.32", "gunicorn>=20.0,<20.2", - "sentry-sdk[flask]>=0.13.5,<1.25", - "newrelic>=5.4.0,<8.8.1", + "sentry-sdk[flask]>=1.0,<1.30", + "newrelic>=8.4.0,<8.8.1", "gitpython>=3.0.0,<3.2" ] gui = [ @@ -51,18 +51,18 @@ gui = [ doc = [ "matplotlib>=3.5.1,<3.8", "myst-nb>=0.13.2,<0.18", - "sphinx>=5.2.0,<6.2", + "sphinx>=5.2.0,<7.3", "sphinxcontrib-httpdomain>=1.7.0,<1.9", "sphinx_rtd_theme>=0.4.3,<1.3" ] test = [ "coverage>=4.5.3,<7.3", "coveralls>=3.0.0,<3.3.2", - "flake8>=3.7,<6.1", - "flake8-black>=0.1.1,<0.4", - "flake8-bugbear>=20.1.2,<23.3", + "flake8>=6.0,<6.1", + "flake8-black>=0.3.4,<0.4", + "flake8-bugbear>=23.5,<23.8", "pytest>=7.1.2,<7.4.2", - "pytest-subtests>=0.8.0,<0.11.1" + "pytest-subtests>=0.10.0,<0.11.1" ] [project.scripts] From b2f7b2c8bb4443dd12299a80f42f4c146fcdfafc Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 11 Sep 2023 11:48:56 +0100 Subject: [PATCH 11/21] Update to latest pandas and numpy --- panoptes_aggregation/scripts/reduce_panoptes_csv.py | 1 - panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py | 3 +-- pyproject.toml | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/panoptes_aggregation/scripts/reduce_panoptes_csv.py b/panoptes_aggregation/scripts/reduce_panoptes_csv.py index b6685bae..93259df6 100644 --- a/panoptes_aggregation/scripts/reduce_panoptes_csv.py +++ b/panoptes_aggregation/scripts/reduce_panoptes_csv.py @@ -93,7 +93,6 @@ def reduce_csv( with extracted_csv as extracted_csv_in: extracted = pandas.read_csv( extracted_csv_in, - infer_datetime_format=True, parse_dates=['created_at'], encoding='utf-8' ) diff --git a/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py b/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py index a1d9d8df..d946146a 100644 --- a/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py +++ b/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py @@ -112,7 +112,6 @@ def setUp(self): self.extracted_csv_question = StringIO(extracted_csv_question) self.extracted_dataframe_question = pandas.read_csv( StringIO(extracted_csv_question), - infer_datetime_format=True, parse_dates=['created_at'], encoding='utf-8' ) @@ -125,7 +124,7 @@ def setUp(self): self.reduced_dataframe_survey = pandas.read_csv(StringIO(reduced_csv_survey)) def test_first_filter(self): - '''Test frist filter''' + '''Test first filter''' extracted_dataframe = pandas.read_csv(self.extracted_csv_question, parse_dates=['created_at']) task_T0_csv = extracted_dataframe[(extracted_dataframe.task == 'T0') & (extracted_dataframe.subject_id == 1)] expected = task_T0_csv[[True, False]] diff --git a/pyproject.toml b/pyproject.toml index a351d5a0..e8713f42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,9 @@ dependencies = [ "collatex@git+https://github.com/interedition/collatex.git@174035dda549ac3f967919a8475fdeb8dbb48f40#subdirectory=collatex-pythonport", "hdbscan>=0.8.20,<=0.8.33", "lxml>=4.4,<4.10", - "numpy>=1.22.0,<1.25.2", + "numpy>=1.22.0,<1.25.3", "packaging>=20.1,<23.2", - "pandas>=1.4.0,<1.5.4", + "pandas>=2.1.0,<2.1.1", "progressbar2>=3.39,<4.3", "python-levenshtein>=0.21.0,<0.22", "python-slugify>=7.0.0,<8.1", From d2b0c7687d626849dc9ee3b13876907a8f608d84 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 11 Sep 2023 11:54:04 +0100 Subject: [PATCH 12/21] Extend pandas lower bound to keep python 3.8 support The latest version of pandas drop 3.8 support, we want to keep that around for now. --- panoptes_aggregation/scripts/reduce_panoptes_csv.py | 1 + panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py | 1 + pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/panoptes_aggregation/scripts/reduce_panoptes_csv.py b/panoptes_aggregation/scripts/reduce_panoptes_csv.py index 93259df6..b6685bae 100644 --- a/panoptes_aggregation/scripts/reduce_panoptes_csv.py +++ b/panoptes_aggregation/scripts/reduce_panoptes_csv.py @@ -93,6 +93,7 @@ def reduce_csv( with extracted_csv as extracted_csv_in: extracted = pandas.read_csv( extracted_csv_in, + infer_datetime_format=True, parse_dates=['created_at'], encoding='utf-8' ) diff --git a/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py b/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py index d946146a..1fbc6faa 100644 --- a/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py +++ b/panoptes_aggregation/tests/scripts_tests/test_reduce_csv.py @@ -112,6 +112,7 @@ def setUp(self): self.extracted_csv_question = StringIO(extracted_csv_question) self.extracted_dataframe_question = pandas.read_csv( StringIO(extracted_csv_question), + infer_datetime_format=True, parse_dates=['created_at'], encoding='utf-8' ) diff --git a/pyproject.toml b/pyproject.toml index e8713f42..fbeef5e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "lxml>=4.4,<4.10", "numpy>=1.22.0,<1.25.3", "packaging>=20.1,<23.2", - "pandas>=2.1.0,<2.1.1", + "pandas>=1.4.0,<2.1.1", "progressbar2>=3.39,<4.3", "python-levenshtein>=0.21.0,<0.22", "python-slugify>=7.0.0,<8.1", From 69a8f9506d38e7bab73a4be845e16ceff2960ef0 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 11 Sep 2023 14:48:17 +0100 Subject: [PATCH 13/21] Update flask code From flask 2.3 the json encoder has been removed, need to use a different method to use the custom encoder. --- panoptes_aggregation/routes.py | 11 ++++++++++- pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/panoptes_aggregation/routes.py b/panoptes_aggregation/routes.py index bf39d2e7..c0ef4dda 100644 --- a/panoptes_aggregation/routes.py +++ b/panoptes_aggregation/routes.py @@ -1,7 +1,9 @@ try: from flask import jsonify, request, Flask - from flask.json import JSONEncoder + from flask.json.provider import JSONProvider from flask_cors import CORS + from json import JSONEncoder + import json from panoptes_aggregation import panoptes import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration @@ -31,6 +33,11 @@ def default(self, obj): return super(MyEncoder, self).default(obj) +class CustomJSONProvider(JSONProvider): + def dumps(self, obj, **kwargs): + return json.dumps(obj, **kwargs, cls=MyEncoder) + + def request_wrapper(name): ''' Example use of process_wrapper: @@ -69,6 +76,8 @@ def make_application(): static_url_path='', static_folder='../docs/build/html') application.json_encoder = MyEncoder + application.json_provider_class = CustomJSONProvider + application.json = CustomJSONProvider(application) CORS( application, origins=[ diff --git a/pyproject.toml b/pyproject.toml index fbeef5e5..08081d97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ [project.optional-dependencies] online = [ - "flask>=2.0,<2.4", + "flask>=2.3,<2.4", "flask-cors>=3.0,<3.1", "panoptes-client>=1.6,<1.7", "requests>=2.28,<2.32", From ce04be063815fde1241a95785c8b5067f6cc723e Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 11 Sep 2023 15:02:09 +0100 Subject: [PATCH 14/21] Update hound config --- .hound.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.hound.yml b/.hound.yml index de5adb31..09d8c1ba 100644 --- a/.hound.yml +++ b/.hound.yml @@ -1,3 +1,3 @@ flake8: enabled: true - config_file: setup.cfg + config_file: .flake8 From 6fd68b7f1e065ae869309942e0c24bd815fb812e Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 11 Sep 2023 15:59:47 +0100 Subject: [PATCH 15/21] remove old flask json encoder method --- panoptes_aggregation/routes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/panoptes_aggregation/routes.py b/panoptes_aggregation/routes.py index c0ef4dda..033f461d 100644 --- a/panoptes_aggregation/routes.py +++ b/panoptes_aggregation/routes.py @@ -19,6 +19,7 @@ import numpy as np +# see https://stackoverflow.com/a/75666126 class MyEncoder(JSONEncoder): def default(self, obj): if isinstance(obj, np.integer): @@ -33,10 +34,14 @@ def default(self, obj): return super(MyEncoder, self).default(obj) +# see https://stackoverflow.com/a/75666126 class CustomJSONProvider(JSONProvider): def dumps(self, obj, **kwargs): return json.dumps(obj, **kwargs, cls=MyEncoder) + def loads(self, s: str | bytes, **kwargs): + return json.loads(s, **kwargs) + def request_wrapper(name): ''' @@ -75,7 +80,6 @@ def make_application(): instance_relative_config=True, static_url_path='', static_folder='../docs/build/html') - application.json_encoder = MyEncoder application.json_provider_class = CustomJSONProvider application.json = CustomJSONProvider(application) CORS( From a8875fe607da9bbcb4aa6208efa03f7e660124ed Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 11 Sep 2023 16:08:38 +0100 Subject: [PATCH 16/21] fix json loads def --- panoptes_aggregation/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoptes_aggregation/routes.py b/panoptes_aggregation/routes.py index 033f461d..857e99ce 100644 --- a/panoptes_aggregation/routes.py +++ b/panoptes_aggregation/routes.py @@ -39,7 +39,7 @@ class CustomJSONProvider(JSONProvider): def dumps(self, obj, **kwargs): return json.dumps(obj, **kwargs, cls=MyEncoder) - def loads(self, s: str | bytes, **kwargs): + def loads(self, s, **kwargs): return json.loads(s, **kwargs) From 5818eb0f7c4254ccf6223be435f1243160bf6eaa Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Tue, 12 Sep 2023 09:31:34 +0100 Subject: [PATCH 17/21] Update docker files to include git This is needed to install collatex from git. --- Dockerfile | 2 +- Dockerfile.bin_cmds | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d6d984b..5740f767 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /usr/src/aggregation RUN apt-get update && apt-get -y upgrade && \ apt-get install --no-install-recommends -y \ - build-essential libgeos-dev && \ + build-essential libgeos-dev git && \ apt-get clean && rm -rf /var/lib/apt/lists/* # install dependencies diff --git a/Dockerfile.bin_cmds b/Dockerfile.bin_cmds index 4913c045..ddeefb35 100644 --- a/Dockerfile.bin_cmds +++ b/Dockerfile.bin_cmds @@ -5,7 +5,7 @@ ENV LANG=C.UTF-8 WORKDIR /usr/src/aggregation -RUN apt-get update && apt-get install --no-install-recommends -y libgeos-dev +RUN apt-get update && apt-get install --no-install-recommends -y libgeos-dev git # install dependencies RUN mkdir -p panoptes_aggregation/version From c4b91e96d301920cd88f0a0c82dad15fb704381d Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Tue, 12 Sep 2023 09:32:40 +0100 Subject: [PATCH 18/21] Update readme with new install instructions Conda envs no longer need some packages to be installed from conda, pip can do it all :D Also the GUI no longer needs a file edit to launch :D --- README.md | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/README.md b/README.md index aa1e1e7d..74d8c850 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can find the [latest documentation](https://aggregation-caesar.zooniverse.or --- ## Installing for offline use -### With your own python install (python 3 only) +### With your own python install (python 3.8 or higher only) Instal the latest stable release: ```bash pip install panoptes_aggregation @@ -40,33 +40,6 @@ Or for the latest development build from GitHub: pip install -U git+https://github.com/zooniverse/aggregation-for-caesar.git#egg=panoptes-aggregation[gui] ``` -#### Anaconda build of python -If your are using the anaconda version of python some of the dependencies should be installed using the `conda` package manager before installing `panoptes_aggregation`: -```bash -conda install -c conda-forge python-levenshtein hdbscan wxpython -conda install psutil -``` - -#### Mac Anaconda build -If you are installing this code on a Mac using the anaconda build of python and you want to use the GUI instead of the command line you will have to update one line of the of code in the `panoptes_aggregation_gui` script. Change the first line from: -```python -#!/path/to/anaconda/python/bin/python -``` -to: -```python -#!/bin/bash /path/to/anaconda/python/bin/python.app -``` - -You can find the location of this file with the command: -```bash -which panoptes_aggregation_gui -``` - -You will also need to run: -```bash -conda install python.app -``` - ### With Docker [https://docs.docker.com/get-started/](https://docs.docker.com/get-started/) From c085f037833fbe8285e55e9bf41156b1a151b509 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Tue, 12 Sep 2023 09:33:33 +0100 Subject: [PATCH 19/21] Check for shape intersection before trying to calculate its area Removes on warning for non overlapping shapes. --- panoptes_aggregation/reducers/shape_metric_IoU.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/panoptes_aggregation/reducers/shape_metric_IoU.py b/panoptes_aggregation/reducers/shape_metric_IoU.py index 22859809..57b20747 100644 --- a/panoptes_aggregation/reducers/shape_metric_IoU.py +++ b/panoptes_aggregation/reducers/shape_metric_IoU.py @@ -102,7 +102,9 @@ def IoU_metric(params1, params2, shape): ''' geo1 = panoptes_to_geometry(params1, shape) geo2 = panoptes_to_geometry(params2, shape) - intersection = geo1.intersection(geo2).area + intersection = 0 + if geo1.intersects(geo2): + intersection = geo1.intersection(geo2).area union = geo1.union(geo2).area if union == 0: # catch divide by zero (i.e. cases when neither shape has an area) From 614c9cb18988b043fb5bc4b27a90d81ccc88b5f7 Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Wed, 13 Sep 2023 09:36:10 +0100 Subject: [PATCH 20/21] Update how to run tests in Contributing.md --- README.md | 2 +- docs/source/Contributing.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 74d8c850..952a5cfb 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ docker run -it --rm --name config_workflow_panoptes -v "$PWD":/usr/src/aggregati --- ## Installing for online use -The docker file included is ready to be deployed on any server. Once deployed, the extractors will be available on the `/extractors/` routes and the reducers will be available on the `/reducers/` routes. Any keywords passed into these functions should be included as url parameters on the route (e.g. `https://aggregation-caesar.zooniverse.org/extractors/point_extractor_by_frame?task=T0`). For more complex keywords (e.g. `detals` for subtasks), python's [urllib.parse.urlencode](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode) can be used to translate a keyword list into the proper url encoding. +The docker file included is ready to be deployed on any server. Once deployed, the extractors will be available on the `/extractors/` routes and the reducers will be available on the `/reducers/` routes. Any keywords passed into these functions should be included as url parameters on the route (e.g. `https://aggregation-caesar.zooniverse.org/extractors/point_extractor_by_frame?task=T0`). For more complex keywords (e.g. `details` for subtasks), python's [urllib.parse.urlencode](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode) can be used to translate a keyword list into the proper url encoding. The documentation will be built and available on the `/docs` route. diff --git a/docs/source/Contributing.md b/docs/source/Contributing.md index d1c9a84e..551fe7a1 100644 --- a/docs/source/Contributing.md +++ b/docs/source/Contributing.md @@ -55,8 +55,8 @@ The code is auto-documented using [sphinx](http://www.sphinx-doc.org/en/stable/i 4. Build the docs with the `make_docs.sh` bash script ### 5. Make sure everything still works -1. run `nosetests` and ensure all tests still pass -2. (optional) `nosetests --cover-html` to compile an html page for checking what parts of the code are not covered +1. run `coverage run` and ensure all tests still pass +2. (optional) run `coverage report` to check tests coverage in each file --- @@ -99,8 +99,8 @@ The code is auto-documented using [sphinx](http://www.sphinx-doc.org/en/stable/i 4. Build the docs with the `make_docs.sh` bash script ### 4. Make sure everything still works -1. run `nosetests` and ensure all tests still pass (coverage is automatically reported) -2. (optional) `nosetests --cover-html` to compile an html page for checking what parts of the code are not covered +1. run `coverage run` and ensure all tests still pass +2. (optional) `coverage report` to checking what parts of the code are not covered --- From 6cd0e0b52db54599c017b31aa893d6ebf8de60aa Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 6 Nov 2023 13:36:04 +0000 Subject: [PATCH 21/21] Update readme with GTK3 instructions --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 952a5cfb..8555a44d 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,11 @@ Or for the latest development build from GitHub: pip install -U git+https://github.com/zooniverse/aggregation-for-caesar.git#egg=panoptes-aggregation[gui] ``` +On linux systems you may need to install GTK3: +```bash +sudo apt-get install build-essential libgtk-3-dev +``` + ### With Docker [https://docs.docker.com/get-started/](https://docs.docker.com/get-started/)