diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3d4791c9..ba3a1a5f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.9" @@ -34,37 +34,92 @@ jobs: run: | tox -e black - testing: - name: Test with Python${{ matrix.python-version }} + testing-pypi: + name: Test with Python${{ matrix.python-version }} (PyPI) needs: black runs-on: ubuntu-latest + env: + COVERALLS_PARALLEL: true + COVERALLS_SERVICE_NAME: github + esmf-version: 8.4.2 strategy: - fail-fast: false matrix: include: - python-version: "3.9" -# tox-build: "py39" + tox-build: "py39-esmpy-coveralls" - python-version: "3.10" -# tox-build: "py310" + tox-build: "py310-esmpy-coveralls" - python-version: "3.11" -# tox-build: "py311" + tox-build: "py311-esmpy-coveralls" defaults: run: shell: bash -l {0} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Conda (Micromamba) with Python ${{ matrix.python-version }} uses: mamba-org/setup-micromamba@v1 with: cache-downloads: true - environment-file: environment.yml + environment-name: xscen-pypi + create-args: >- + esmf=${{ env.esmf-version }} + mamba + python=${{ matrix.python-version }} + tox + - name: Test with tox + run: | + tox -e ${{ matrix.tox-build }} + env: + ESMF_VERSION: ${{ env.esmf-version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }} + + # - name: Compile language catalogs +# run: | +# make translate +# - name: Install esmpy +# run: | +# pip install git+https://github.com/esmf-org/esmf.git@v${{ matrix.esmf-version }}#subdirectory=src/addon/esmpy +# - name: Install xscen +# run: | +# pip install --editable ".[dev]" +# - name: Check versions +# run: | +# pip list +# pip check +# - name: Test with pytest +# run: | +# pytest tests +# - name: Report coverage +# run: | +# coveralls +# env: +# ESMF_VERSION: ${{ matrix.esmf-version }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }} +# COVERALLS_PARALLEL: true +# COVERALLS_SERVICE_NAME: github + + testing-conda: + name: Test with Python${{ matrix.python-version }} (Anaconda) + needs: black + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11"] + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v4 + - name: Setup Conda (Micromamba) with Python ${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@v1 + with: + cache-downloads: true + environment-file: environment-dev.yml create-args: >- - coveralls mamba python=${{ matrix.python-version }} - pytest - pytest-cov - xdoctest - name: Conda and Mamba versions run: | mamba --version @@ -72,36 +127,27 @@ jobs: - name: Compile catalogs and install xscen run: | make translate - pip install --editable . + pip install --no-deps --editable . - name: Check versions run: | conda list pip check - name: Test with pytest run: | - pytest --cov xscen + pytest tests - name: Report coverage run: | coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }} + COVERALLS_FLAG_NAME: run-Python${{ matrix.python-version }}-conda COVERALLS_PARALLEL: true COVERALLS_SERVICE_NAME: github -# - name: Install tox-current-env -# run: | -# pip install tox tox-conda tox-current-env -# - name: Test with tox -# run: | -# tox -vv -e ${{ matrix.tox-build }} --current-env -# env: -# CONDA_EXE: mamba -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - finish: needs: - - testing + - testing-pypi + - testing-conda runs-on: ubuntu-latest container: python:3-slim steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index 4192147e..6fad0b5c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -13,6 +13,7 @@ build: python: "mambaforge-22.9" jobs: post_create_environment: + - make translate - pip install . --no-deps pre_build: - sphinx-apidoc -o docs/apidoc --private --module-first xscen diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 00470414..221d3df3 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -68,7 +68,7 @@ Ready to contribute? Here's how to set up `xscen` for local development. $ cd xscen/ $ mamba env create -f environment-dev.yml - $ pip install -e . + $ pip install -e ".[dev]" 3. As xscen was installed in editable mode, we also need to compile the translation catalogs manually: @@ -89,18 +89,21 @@ Ready to contribute? Here's how to set up `xscen` for local development. Now you can make your changes locally. -6. When you're done making changes, check that your changes pass flake8, black, and the - tests, including testing other Python versions with tox:: +6. When you're done making changes, check that your changes pass `flake8`, `black`, and the + tests, including testing other Python versions with `tox`:: $ tox - To get flake8, black, and tox, just pip install them into your virtualenv. + Alternatively, you can run the tests using `make`:: + + $ make lint + $ make test + + Running `make lint` and `make test` demands that your runtime/dev environment have all necessary development dependencies installed. .. warning:: - Due to some dependencies only being available via Anaconda/conda-forge, `tox` will only work if both `tox` and `tox-conda` - are installed in a conda-based environment. Running `pytest` demands that your runtime/dev environment have all necessary - dependencies installed. + Due to some dependencies only being available via Anaconda/conda-forge or built from source, `tox`-based testing will only work if `ESMF`_ is available in your system path. This also requires that the `ESMF_VERSION` environment variable (matching the version of ESMF installed) be accessible within your shell as well (e.g.: `$ export ESMF_VERSION=8.5.0`). 7. Commit your changes and push your branch to GitHub:: @@ -188,3 +191,5 @@ Then run:: $ bumpversion patch # possible: major / minor / patch $ git push $ git push --tags + +.. _`ESMF`: http://earthsystemmodeling.org/download/ diff --git a/HISTORY.rst b/HISTORY.rst index 9281407b..1a00b585 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,7 +4,7 @@ History v0.8.0 (unreleased) ------------------- -Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal Bourgault (:user:`aulemahal`), Juliette Lavoie (:user:`juliettelavoie`), Sarah-Claude Bourdeau-Goulet (:user:`sarahclaude`). +Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal Bourgault (:user:`aulemahal`), Juliette Lavoie (:user:`juliettelavoie`), Sarah-Claude Bourdeau-Goulet (:user:`sarahclaude`), Trevor James Smith (:user:`Zeitsperre`). Announcements ^^^^^^^^^^^^^ @@ -19,7 +19,7 @@ New features and enhancements * ``xs.save_to_table`` and ``xs.io.to_table`` to transform datasets and arrays to DataFrames, but with support for multi-columns, multi-sheets and localized table of content generation. * Better ``xs.extract.resample`` : support for weighted resampling operations when starting with frequencies coarser than daily and missing timesteps/values handling. (:issue:`80`, :issue:`93`, :pull:`265`). * New argument ``attribute_weights`` to ``generate_weights`` to allow for custom weights. (:pull:`252`). -* ``xs.io.round_bits`` to round floating point variable up to a number of bits, allowing for a better compression. This can be combined with the saving step through argument "bitround" of ``save_to_netcdf`` and ``save_to_zarr``. (:pull:`266`). +* ``xs.io.round_bits`` to round floating point variable up to a number of bits, allowing for a better compression. This can be combined with the saving step through argument ``"bitround"`` of ``save_to_netcdf`` and ``save_to_zarr``. (:pull:`266`). Breaking changes ^^^^^^^^^^^^^^^^ @@ -35,15 +35,23 @@ Bug fixes Internal changes ^^^^^^^^^^^^^^^^ * Continued work on adding tests. (:pull:`251`). -* Fixed pre-commit's pretty-format-json so it ignores notebooks. (:pull:`254`). +* Fixed `pre-commit`'s `pretty-format-json` hook so that it ignores notebooks. (:pull:`254`). * Fixed the labeler so docs/CI isn't automatically added for contributions by new collaborators. (:pull:`254`). * Made it so that `tests` are no longer treated as an installable package. (:pull:`248`). -* Renamed the pytest marker from `requires_docs` to `requires_netcdf`. (:pull:`248`). +* Renamed the pytest marker from ``requires_docs`` to ``requires_netcdf``. (:pull:`248`). * Included the documentation in the source distribution, while excluding the NetCDF files. (:pull:`248`). -* Reduced the size of the files in /docs/notebooks/samples and changed the Notebooks and tests accordingly. (:issue:`247`, :pull:`248`). -* Added a new `xscen.testing` module with the `datablock_3d` function previously located in `/tests/conftest.py`. (:pull:`248`). +* Reduced the size of the files in ``/docs/notebooks/samples`` and changed the notebooks and tests accordingly. (:issue:`247`, :pull:`248`). +* Added a new `xscen.testing` module with the `datablock_3d` function previously located in ``/tests/conftest.py``. (:pull:`248`). * New function `xscen.testing.fake_data` to generate fake data for testing. (:pull:`248`). * xESMF 0.8 Regridder and SpatialAverager argument ``out_chunks`` is now accepted by ``xs.regrid_dataset`` and ``xs.spatial_mean``. (:pull:`260`). +* Testing, Packaging, and CI adjustments. (:pull:`274`): + * `xscen` builds now install in a `tox` environment with `conda`-provided `ESMF` in GitHub Workflows. + * `tox` now offers a method for installing esmpy from a tag/branch (via ESMF_VERSION environment variable). + * `$ make translate` is now called on ReadTheDocs and within `tox`. + * Linters are now called by order of most common failures first, to speed up the CI. + * `Manifest.in` is much more specific about what is installed. + * Re-adds a dev recipe to the `setup.py`. + v0.7.1 (2023-08-23) ------------------- diff --git a/MANIFEST.in b/MANIFEST.in index 227cd926..c7612de0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,10 +4,14 @@ include HISTORY.rst include LICENSE include README.rst -recursive-include xscen *.json *.yml *.py *.csv *.mo -recursive-include tests * -recursive-include docs notebooks *.rst conf.py Makefile make.bat *.jpg *.png *.gif *.ipynb *.csv *.json *.yml *.md -recursive-include docs notebooks samples *.csv *.json +recursive-include xscen *.py *.yml +recursive-include xscen CVs *.json +recursive-include xscen data fr *.yml *.csv +recursive-include xscen data fr LC_MESSAGES *.mo *.po +recursive-include tests *.py +recursive-include docs conf.py Makefile make.bat *.png *.rst *.yml +recursive-include docs notebooks *.ipynb +recursive-include docs notebooks samples *.csv *.json *.yml recursive-exclude * __pycache__ recursive-exclude * *.py[co] diff --git a/Makefile b/Makefile index c57529b3..425d9f43 100644 --- a/Makefile +++ b/Makefile @@ -53,13 +53,13 @@ clean-test: ## remove test and coverage artifacts rm -fr htmlcov/ rm -fr .pytest_cache -lint/flake8: ## check style with flake8 - flake8 --config=setup.cfg xscen tests - lint/black: ## check style with black black --check xscen tests -lint: lint/flake8 lint/black ## check style +lint/flake8: ## check style with flake8 + flake8 --config=setup.cfg xscen tests + +lint: lint/black lint/flake8 ## check style test: ## run tests quickly with the default Python pytest diff --git a/environment-dev.yml b/environment-dev.yml index 1d97332a..9650fe0b 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -19,22 +19,21 @@ dependencies: - matplotlib - netCDF4 - numpy - - pandas >= 2 + - pandas >=2.0 - parse - pyyaml - rechunker - - shapely >= 2 + - shapely >=2.0 - sparse - toolz # FIXME: Unpin xarray when xclim 0.46 is released - - xarray < 2023.09.0 + - xarray <2023.09.0 - xclim >=0.43.0 - xesmf >=0.7 - zarr # Opt - nc-time-axis >=1.3.1 - pyarrow >=1.0.0 - - pip # Dev - babel - bumpversion @@ -58,8 +57,8 @@ dependencies: - xdoctest - pip # Testing - - tox <4.0 # 2022-12-12: tox v4.0 is incompatible with tox-conda plugin - - tox-conda >=0.10.2 + - tox # <4.0 # 2022-12-12: tox v4.0 is incompatible with tox-conda plugin + # - tox-conda >=0.10.2 # packaging - build - wheel diff --git a/setup.cfg b/setup.cfg index 6e3eb34e..3e41fe0e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,13 +3,13 @@ current_version = 0.7.17-beta commit = True tag = False parse = (?P\d+)\.(?P\d+).(?P\d+)(\-(?P[a-z]+))? -serialize = +serialize = {major}.{minor}.{patch}-{release} {major}.{minor}.{patch} [bumpversion:part:release] optional_value = gamma -values = +values = beta gamma @@ -34,19 +34,19 @@ test = pytest [tool:pytest] collect_ignore = ['setup.py'] -addopts = +addopts = + --color=yes --cov=xscen --ignore-glob='*.ipynb_checkpoints' - --verbose --strict-markers -filterwarnings = + --verbose +filterwarnings = ignore::UserWarning -usefixtures = xdoctest_namespace -markers = +markers = requires_netcdf: marks tests that require netcdf files to run [flake8] -exclude = +exclude = .git, docs, build, @@ -54,7 +54,7 @@ exclude = docs/conf.py, max-line-length = 88 max-complexity = 12 -ignore = +ignore = C901 E203 E231 @@ -64,9 +64,9 @@ ignore = F403 W503 W504 -per-file-ignores = +per-file-ignores = tests/*:E402 -rst-roles = +rst-roles = mod, py:attr, py:attribute, @@ -82,7 +82,7 @@ rst-roles = [coverage:run] relative_files = True -omit = +omit = docs/notebooks tests diff --git a/setup.py b/setup.py index d3ce6c68..84fa7fdf 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,30 @@ def run(self): "zarr", ] -dev_requirements = ["pytest", "pytest-cov", "xdoctest"] +dev_requirements = [ + "black", + "flake8", + "flake8-rst-docstrings", + "pytest", + "pytest-cov", +] + +docs_requirements = [ + "ipykernel", + "ipython", + "jupyter_client", + "nbsphinx", + "nc-time-axis>=1.3.1", + "nbval", + "sphinx", + "sphinx-autoapi", + "sphinx-rtd-theme>=1.0", + "sphinxcontrib-napoleon", + "sphinx-codeautolink", + "sphinx-copybutton", +] + +setup_requirements = ["babel"] setup( author="Gabriel Rondeau-Genesse", @@ -98,9 +121,9 @@ def run(self): "Changelog": "https://xscen.readthedocs.io/en/stable/history.html", "Issue tracker": "https://github.com/Ouranosinc/xscen/issues", }, - setup_requires=["babel"], + setup_requires=setup_requirements, test_suite="tests", - extras_require={"dev": dev_requirements}, + extras_require={"dev": dev_requirements, "docs": docs_requirements}, url="https://github.com/Ouranosinc/xscen", version="0.7.17-beta", zip_safe=False, diff --git a/tox.ini b/tox.ini index d02b038a..cdf7d9ed 100644 --- a/tox.ini +++ b/tox.ini @@ -2,11 +2,12 @@ min_version = 4.0 envlist = black - py{39,310,311} + py{39,310,311}-esmpy docs - coveralls requires = - pip >= 21.0 + babel + pip >= 23.0 + setuptools >= 65.0 opts = --verbose @@ -17,9 +18,11 @@ download = true conda_channels = conda_env = deps = + black flake8 flake8-rst-docstrings - black +commands_pre = + pip list commands = make lint allowlist_externals = @@ -29,6 +32,7 @@ allowlist_externals = description = Run Build of xscen Documentation conda_deps = conda_env = environment-dev.yml +extras = commands = make docs allowlist_externals = @@ -38,18 +42,29 @@ allowlist_externals = description = Run documentation linters and doctests with pytest under {basepython} conda_deps = conda_env = environment-dev.yml +extras = + dev commands = - pytest --no-cov --nbval docs/notebooks --durations=10 - pytest --xdoctest xscen --durations=10 + pytest --no-cov --nbval docs/notebooks --durations=10 {posargs} + pytest --xdoctest xscen --durations=10 {posargs} [testenv] description = Run tests with pytest under {basepython} (Anaconda distribution) setenv = - PYTEST_ADDOPTS = "--color=yes" + COV_CORE_SOURCE = + PYTEST_ADDOPTS = --color=yes --cov=xscen --strict-markers --verbose PYTHONPATH = {toxinidir} passenv = + COVERALLS_* + ESMFMKFILE + ESMF_VERSION GITHUB_* download = true +deps = + coveralls: coveralls + esmpy: git+https://github.com/esmf-org/esmf.git@v{env:ESMF_VERSION}\#subdirectory=src/addon/esmpy +extras = + dev conda_channels = conda-forge defaults @@ -63,4 +78,10 @@ commands_pre = pip list pip check commands = - pytest --cov xscen + make translate + pytest {posargs} + coveralls: - coveralls +allowlist_externals = + make + pip + git