Test ESMF from conda and remainder PyPI deps

.github/workflows/main.yml

@@ -23,7 +23,7 @@ jobs:
         uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
@@ -34,74 +34,120 @@ jobs:
         run: |
           tox -e black
 
-  testing:
-    name: Test with Python${{ matrix.python-version }}
+  testing-pypi:
+    name: Test with Python${{ matrix.python-version }} (PyPI)
     needs: black
     runs-on: ubuntu-latest
+    env:
+      COVERALLS_PARALLEL: true
+      COVERALLS_SERVICE_NAME: github
+      esmf-version: 8.4.2
     strategy:
-      fail-fast: false
       matrix:
         include:
           - python-version: "3.9"
-#            tox-build: "py39"
+            tox-build: "py39-esmpy-coveralls"
           - python-version: "3.10"
-#            tox-build: "py310"
+            tox-build: "py310-esmpy-coveralls"
           - python-version: "3.11"
-#            tox-build: "py311"
+            tox-build: "py311-esmpy-coveralls"
     defaults:
       run:
         shell: bash -l {0}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Conda (Micromamba) with Python ${{ matrix.python-version }}
         uses: mamba-org/setup-micromamba@v1
         with:
           cache-downloads: true
-          environment-file: environment.yml
+          environment-name: xscen-pypi
+          create-args: >-
+            esmf=${{ env.esmf-version }}
+            mamba
+            python=${{ matrix.python-version }}
+            tox
+      - name: Test with tox
+        run: |
+          tox -e ${{ matrix.tox-build }}
+        env:
+          ESMF_VERSION: ${{ env.esmf-version }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }}
+
+  #      - name: Compile language catalogs
+#        run: |
+#          make translate
+#      - name: Install esmpy
+#        run: |
+#          pip install git+https://github.com/esmf-org/esmf.git@v${{ matrix.esmf-version }}#subdirectory=src/addon/esmpy
+#      - name: Install xscen
+#        run: |
+#          pip install --editable ".[dev]"
+#      - name: Check versions
+#        run: |
+#          pip list
+#          pip check
+#      - name: Test with pytest
+#        run: |
+#          pytest tests
+#      - name: Report coverage
+#        run: |
+#          coveralls
+#        env:
+#          ESMF_VERSION: ${{ matrix.esmf-version }}
+#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#          COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }}
+#          COVERALLS_PARALLEL: true
+#          COVERALLS_SERVICE_NAME: github
+
+  testing-conda:
+    name: Test with Python${{ matrix.python-version }} (Anaconda)
+    needs: black
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Conda (Micromamba) with Python ${{ matrix.python-version }}
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          cache-downloads: true
+          environment-file: environment-dev.yml
           create-args: >-
-            coveralls
             mamba
             python=${{ matrix.python-version }}
-            pytest
-            pytest-cov
-            xdoctest
       - name: Conda and Mamba versions
         run: |
           mamba --version
           echo "micromamba $(micromamba --version)"
       - name: Compile catalogs and install xscen
         run: |
           make translate
-          pip install --editable .
+          pip install --no-deps --editable .
       - name: Check versions
         run: |
           conda list
           pip check
       - name: Test with pytest
         run: |
-          pytest --cov xscen
+          pytest tests
       - name: Report coverage
         run: |
           coveralls
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }}
+          COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }}-conda
           COVERALLS_PARALLEL: true
           COVERALLS_SERVICE_NAME: github
 
-#      - name: Install tox-current-env
-#        run: |
-#          pip install tox tox-conda tox-current-env
-#      - name: Test with tox
-#        run: |
-#          tox -vv -e ${{ matrix.tox-build }} --current-env
-#        env:
-#          CONDA_EXE: mamba
-#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
   finish:
     needs:
-      - testing
+      - testing-pypi
+      - testing-conda
     runs-on: ubuntu-latest
     container: python:3-slim
     steps:

.readthedocs.yml

@@ -13,6 +13,7 @@ build:
     python: "mambaforge-22.9"
   jobs:
     post_create_environment:
+      - make translate
       - pip install . --no-deps
     pre_build:
       - sphinx-apidoc -o docs/apidoc --private --module-first xscen

HISTORY.rst

@@ -4,7 +4,7 @@ History
 
 v0.8.0 (unreleased)
 -------------------
-Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal Bourgault (:user:`aulemahal`), Juliette Lavoie (:user:`juliettelavoie`), Sarah-Claude Bourdeau-Goulet (:user:`sarahclaude`).
+Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal Bourgault (:user:`aulemahal`), Juliette Lavoie (:user:`juliettelavoie`), Sarah-Claude Bourdeau-Goulet (:user:`sarahclaude`), Trevor James Smith (:user:`Zeitsperre`).
 
 Announcements
 ^^^^^^^^^^^^^
@@ -19,7 +19,7 @@ New features and enhancements
 * ``xs.save_to_table`` and ``xs.io.to_table`` to transform datasets and arrays to DataFrames, but with support for multi-columns, multi-sheets and localized table of content generation.
 * Better ``xs.extract.resample`` : support for weighted resampling operations when starting with frequencies coarser than daily and missing timesteps/values handling. (:issue:`80`, :issue:`93`, :pull:`265`).
 * New argument ``attribute_weights`` to ``generate_weights`` to allow for custom weights. (:pull:`252`).
-* ``xs.io.round_bits`` to round floating point variable up to a number of bits, allowing for a better compression. This can be combined with the saving step through argument "bitround" of ``save_to_netcdf`` and ``save_to_zarr``. (:pull:`266`).
+* ``xs.io.round_bits`` to round floating point variable up to a number of bits, allowing for a better compression. This can be combined with the saving step through argument ``"bitround"`` of ``save_to_netcdf`` and ``save_to_zarr``. (:pull:`266`).
 
 Breaking changes
 ^^^^^^^^^^^^^^^^
@@ -34,15 +34,23 @@ Bug fixes
 Internal changes
 ^^^^^^^^^^^^^^^^
 * Continued work on adding tests. (:pull:`251`).
-* Fixed pre-commit's pretty-format-json so it ignores notebooks. (:pull:`254`).
+* Fixed `pre-commit`'s `pretty-format-json` hook so that it ignores notebooks. (:pull:`254`).
 * Fixed the labeler so docs/CI isn't automatically added for contributions by new collaborators. (:pull:`254`).
 * Made it so that `tests` are no longer treated as an installable package. (:pull:`248`).
-* Renamed the pytest marker from `requires_docs` to `requires_netcdf`. (:pull:`248`).
+* Renamed the pytest marker from ``requires_docs`` to ``requires_netcdf``. (:pull:`248`).
 * Included the documentation in the source distribution, while excluding the NetCDF files. (:pull:`248`).
-* Reduced the size of the files in /docs/notebooks/samples and changed the Notebooks and tests accordingly. (:issue:`247`, :pull:`248`).
-* Added a new `xscen.testing` module with the `datablock_3d` function previously located in `/tests/conftest.py`. (:pull:`248`).
+* Reduced the size of the files in ``/docs/notebooks/samples`` and changed the notebooks and tests accordingly. (:issue:`247`, :pull:`248`).
+* Added a new `xscen.testing` module with the `datablock_3d` function previously located in ``/tests/conftest.py``. (:pull:`248`).
 * New function `xscen.testing.fake_data` to generate fake data for testing. (:pull:`248`).
 * xESMF 0.8 Regridder and SpatialAverager argument ``out_chunks`` is now accepted by ``xs.regrid_dataset``  and ``xs.spatial_mean``. (:pull:`260`).
+* Testing, Packaging, and CI adjustments. (:pull:`274`):
+    * `xscen` builds now install in a `tox` environment with `conda`-provided `ESMF` in GitHub Workflows.
+    * `tox` now offers a method for installing esmpy from a tag/branch (via ESMF_VERSION environment variable).
+    * `$ make translate` is now called on ReadTheDocs and within `tox`.
+    * Linters are now called by order of most common failures first, to speed up the CI.
+    * `Manifest.in` is much more specific about what is installed.
+    * Re-adds a dev recipe to the `setup.py`.
+
 
 v0.7.1 (2023-08-23)
 -------------------

MANIFEST.in

@@ -4,10 +4,14 @@ include HISTORY.rst
 include LICENSE
 include README.rst
 
-recursive-include xscen *.json *.yml *.py *.csv *.mo
-recursive-include tests *
-recursive-include docs notebooks *.rst conf.py Makefile make.bat *.jpg *.png *.gif *.ipynb *.csv *.json *.yml *.md
-recursive-include docs notebooks samples *.csv *.json
+recursive-include xscen *.py *.yml
+recursive-include xscen CVs *.json
+recursive-include xscen data fr *.yml *.csv
+recursive-include xscen data fr LC_MESSAGES *.mo *.po
+recursive-include tests *.py
+recursive-include docs conf.py Makefile make.bat *.png *.rst *.yml
+recursive-include docs notebooks *.ipynb
+recursive-include docs notebooks samples *.csv *.json *.yml
 
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]

Makefile

@@ -53,13 +53,13 @@ clean-test: ## remove test and coverage artifacts
 	rm -fr htmlcov/
 	rm -fr .pytest_cache
 
-lint/flake8: ## check style with flake8
-	flake8 --config=setup.cfg xscen tests
-
 lint/black: ## check style with black
 	black --check xscen tests
 
-lint: lint/flake8 lint/black ## check style
+lint/flake8: ## check style with flake8
+	flake8 --config=setup.cfg xscen tests
+
+lint: lint/black lint/flake8 ## check style
 
 test: ## run tests quickly with the default Python
 	pytest

environment-dev.yml

@@ -19,22 +19,21 @@ dependencies:
   - matplotlib
   - netCDF4
   - numpy
-  - pandas >= 2
+  - pandas >=2.0
   - parse
   - pyyaml
   - rechunker
-  - shapely >= 2
+  - shapely >=2.0
   - sparse
   - toolz
   # FIXME: Unpin xarray when xclim 0.46 is released
-  - xarray < 2023.09.0
+  - xarray <2023.09.0
   - xclim >=0.43.0
   - xesmf >=0.7
   - zarr
   # Opt
   - nc-time-axis >=1.3.1
   - pyarrow >=1.0.0
-  - pip
   # Dev
   - babel
   - bumpversion
@@ -58,8 +57,8 @@ dependencies:
   - xdoctest
   - pip
   # Testing
-  - tox <4.0 # 2022-12-12: tox v4.0 is incompatible with tox-conda plugin
-  - tox-conda >=0.10.2
+  - tox # <4.0 # 2022-12-12: tox v4.0 is incompatible with tox-conda plugin
+  # - tox-conda >=0.10.2
   # packaging
   - build
   - wheel

environment.yml

@@ -26,7 +26,8 @@ dependencies:
   - shapely >= 2
   - sparse
   - toolz
-  - xarray
+  # FIXME: Unpin xarray when xclim 0.46 is released
+  - xarray <2023.09.0
   - xclim >=0.43.0
   - xesmf >=0.7
   - zarr

setup.cfg

@@ -3,13 +3,13 @@ current_version = 0.7.15-beta
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+).(?P<patch>\d+)(\-(?P<release>[a-z]+))?
-serialize = 
+serialize =
 	{major}.{minor}.{patch}-{release}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:release]
 optional_value = gamma
-values = 
+values =
 	beta
 	gamma
 
@@ -34,27 +34,28 @@ test = pytest
 
 [tool:pytest]
 collect_ignore = ['setup.py']
-addopts = 
+addopts =
+    --color=yes
 	--cov=xscen
 	--ignore-glob='*.ipynb_checkpoints'
-	--verbose
 	--strict-markers
-filterwarnings = 
+	--verbose
+filterwarnings =
 	ignore::UserWarning
 usefixtures = xdoctest_namespace
-markers = 
+markers =
 	requires_netcdf: marks tests that require netcdf files to run
 
 [flake8]
-exclude = 
+exclude =
 	.git,
 	docs,
 	build,
 	.eggs,
 	docs/conf.py,
 max-line-length = 88
 max-complexity = 12
-ignore = 
+ignore =
 	C901
 	E203
 	E231
@@ -64,9 +65,9 @@ ignore =
 	F403
 	W503
 	W504
-per-file-ignores = 
+per-file-ignores =
 	tests/*:E402
-rst-roles = 
+rst-roles =
 	mod,
 	py:attr,
 	py:attribute,
@@ -82,7 +83,7 @@ rst-roles =
 
 [coverage:run]
 relative_files = True
-omit = 
+omit =
 	docs/notebooks
 	tests