From d895a8ee127d80d41d0be39de2b597dd068b2b08 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 24 Nov 2023 17:00:11 -0500 Subject: [PATCH 01/22] update cookiecutter, maintain setuptools in lieu of flit, add trusted publishing, migrate settings to pyproject.toml and .flake8, rename black to lint, consistency in Makefile calls, add actions-version-updater --- .cruft.json | 3 +- .flake8 | 32 ++ .../workflows/actions-versions-updater.yml | 24 ++ .github/workflows/main.yml | 33 ++- .github/workflows/publish-pypi.yml | 12 +- .github/workflows/tag-testpypi.yml | 33 ++- .pre-commit-config.yaml | 35 ++- HISTORY.rst => CHANGES.rst | 6 +- CONTRIBUTING.rst | 155 ++++++---- MANIFEST.in | 26 -- Makefile | 16 +- docs/changes.rst | 1 + docs/history.rst | 1 - docs/index.rst | 2 +- docs/installation.rst | 4 +- environment-dev.yml | 3 +- environment.yml | 2 +- pyproject.toml | 277 ++++++++++++++++++ setup.cfg | 97 ------ setup.py | 118 +------- tox.ini | 7 +- xscen/__init__.py | 2 +- 22 files changed, 537 insertions(+), 352 deletions(-) create mode 100644 .flake8 create mode 100644 .github/workflows/actions-versions-updater.yml rename HISTORY.rst => CHANGES.rst (99%) delete mode 100644 MANIFEST.in create mode 100644 docs/changes.rst delete mode 100644 docs/history.rst create mode 100644 pyproject.toml diff --git a/.cruft.json b/.cruft.json index 386b9e5f..307c687c 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "/home/tjs/git/cookiecutter-pypackage", - "commit": "2fc25aaaf59ce346529c90c032c34baa30a45def", + "commit": "e67ba8180e050ffd1c29e38cf2ad3929004fdd8f", "checkout": null, "context": { "cookiecutter": { @@ -14,6 +14,7 @@ "version": "0.7.20-beta", "use_pytest": "y", "use_black": "y", + "use_conda": "y", "add_pyup_badge": "n", "make_docs": "y", "command_line_interface": "Click", diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..12028909 --- /dev/null +++ b/.flake8 @@ -0,0 +1,32 @@ +[flake8] +exclude = + .eggs, + .git, + build, + docs/conf.py, + tests +ignore = + AZ100, + AZ200, + AZ300, + C, + D, + E, + F, + W503 + W504 +per-file-ignores = + tests/*:E402 +rst-roles = + doc, + mod, + py:attr, + py:attribute, + py:class, + py:const, + py:data, + py:func, + py:meth, + py:mod, + py:obj, + py:ref diff --git a/.github/workflows/actions-versions-updater.yml b/.github/workflows/actions-versions-updater.yml new file mode 100644 index 00000000..bb05b24e --- /dev/null +++ b/.github/workflows/actions-versions-updater.yml @@ -0,0 +1,24 @@ +name: GitHub Actions Version Updater + +on: + schedule: + # 12:00 AM on the first of every month + - cron: '0 0 1 * *' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + with: + # This requires a personal access token with the privileges to push directly to `main` + token: ${{ secrets.WORKFLOW_TOKEN }} + persist-credentials: true + - name: Run GitHub Actions Version Updater + uses: saadmk11/github-actions-version-updater@v0.8.1 + with: + token: ${{ secrets.WORKFLOW_TOKEN }} + committer_email: 'bumpversion[bot]@ouranos.ca' + committer_username: 'update-github-actions[bot]' + pull_request_title: '[bot] Update GitHub Action Versions' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c7d1127..1cca4204 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,12 +12,13 @@ on: pull_request: jobs: - black: - name: Code Style Compliance + lint: + name: Lint (Python${{ matrix.python-version }}) runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} + strategy: + matrix: + python-version: + - "3.x" steps: - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.11.0 @@ -26,15 +27,15 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.x" - name: Install tox run: | python -m pip install tox - name: Run linting suite run: | - python -m tox -e black + python -m tox -e lint - testing-tox: + test-pypi: name: Test with Python${{ matrix.python-version }} (PyPI/tox) needs: black runs-on: ubuntu-latest @@ -100,9 +101,9 @@ jobs: # COVERALLS_PARALLEL: true # COVERALLS_SERVICE_NAME: github - testing-conda: + test-conda: name: Test with Python${{ matrix.python-version }} (Anaconda) - needs: black + needs: lint runs-on: ubuntu-latest strategy: matrix: @@ -127,14 +128,14 @@ jobs: - name: Compile catalogs and install xscen run: | make translate - python -m pip install --no-deps --editable . + python -m pip install --no-deps . - name: Check versions run: | conda list python -m pip check || true - name: Test with pytest run: | - python -m pytest tests + python -m pytest --cov xscen - name: Report coverage run: | python -m coveralls @@ -146,15 +147,15 @@ jobs: finish: needs: - - testing-tox - - testing-conda + - test-pypi + - test-conda runs-on: ubuntu-latest container: python:3-slim steps: - name: Coveralls Finished run: | - pip install --upgrade coveralls - coveralls --finish + python -m pip install --upgrade coveralls + python -m coveralls --finish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_SERVICE_NAME: github diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 0a1c0840..b1cb4219 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -9,20 +9,22 @@ jobs: build-n-publish-pypi: name: Build and publish Python 🐍 distributions 📦 to PyPI runs-on: ubuntu-latest + environment: production + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python3 uses: actions/setup-python@v4 with: python-version: "3.x" - name: Install packaging libraries run: | - python -m pip install build wheel + python -m pip install babel build setuptools wheel - name: Build a binary wheel and a source tarball run: | + make translate python -m build --sdist --wheel - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/tag-testpypi.yml b/.github/workflows/tag-testpypi.yml index 874dc515..bd977c3d 100644 --- a/.github/workflows/tag-testpypi.yml +++ b/.github/workflows/tag-testpypi.yml @@ -3,28 +3,49 @@ name: "Publish Python 🐍 distributions 📦 to TestPyPI" on: push: tags: - - '*' + - 'v*.*' # Push events to matching v*, i.e. v1.0, v20.15.10 jobs: - build-n-publish-testpypi: + release: + name: Create Release from tag + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '.0') + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Create Release + uses: softprops/action-gh-release@v1 + env: + # This token is provided by Actions, you do not need to create your own token + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + name: Release ${{ github.ref }} + draft: true + prerelease: false + + deploy-testpypi: name: Build and publish Python 🐍 distributions 📦 to TestPyPI runs-on: ubuntu-latest + environment: staging + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python3 uses: actions/setup-python@v4 with: python-version: "3.x" - name: Install packaging libraries run: | - python -m pip install build wheel + python -m pip install babel build setuptools wheel - name: Build a binary wheel and a source tarball run: | + make translate python -m build --sdist --wheel - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ skip_existing: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 16fcbcaa..643eb32b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,7 @@ repos: - id: pretty-format-json args: [ '--autofix', '--no-ensure-ascii', '--no-sort-keys' ] exclude: .ipynb + - id: check-toml - id: check-yaml args: [ '--allow-multiple-documents' ] exclude: conda/xscen/meta.yaml @@ -27,28 +28,30 @@ repos: rev: v1.10.0 hooks: - id: rst-inline-touching-normal + - repo: https://github.com/pappasam/toml-sort + rev: v0.23.1 + hooks: + - id: toml-sort-fix - repo: https://github.com/psf/black-pre-commit-mirror rev: 23.11.0 hooks: - id: black exclude: ^docs/ - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - additional_dependencies: [ 'flake8-rst-docstrings' ] - args: [ '--config=setup.cfg' ] - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: - id: isort - args: [ '--settings-file=setup.cfg' ] exclude: ^docs/ - - repo: https://github.com/pycqa/pydocstyle - rev: 6.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.4 hooks: - - id: pydocstyle - args: [ '--config=setup.cfg' ] + - id: ruff + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + additional_dependencies: [ 'flake8-alphabetize', 'flake8-rst-docstrings' ] + args: [ '--config=.flake8' ] - repo: https://github.com/keewis/blackdoc rev: v0.3.9 hooks: @@ -83,11 +86,11 @@ repos: - id: detect-secrets exclude: .cruft.json|docs/notebooks args: [ '--baseline=.secrets.baseline' ] -# Currently commented because Git needs to be updated on our work environment. -# - repo: https://github.com/mgedmin/check-manifest -# rev: "0.49" -# hooks: -# - id: check-manifest + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.27.1 + hooks: + - id: check-github-workflows + - id: check-readthedocs - repo: meta hooks: - id: check-hooks-apply diff --git a/HISTORY.rst b/CHANGES.rst similarity index 99% rename from HISTORY.rst rename to CHANGES.rst index db9af8ed..e97a4751 100644 --- a/HISTORY.rst +++ b/CHANGES.rst @@ -1,6 +1,6 @@ -======= -History -======= +========= +Changelog +========= v0.8.0 (unreleased) ------------------- diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 221d3df3..a249e021 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -4,8 +4,7 @@ Contributing ============ -Contributions are welcome, and they are greatly appreciated! Every little bit -helps, and credit will always be given. +Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. You can contribute in many ways: @@ -26,21 +25,17 @@ If you are reporting a bug, please include: Fix Bugs ~~~~~~~~ -Look through the GitHub issues for bugs. Anything tagged with "bug" and "help -wanted" is open to whoever wants to implement it. +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help wanted" is open to whoever wants to implement it. Implement Features ~~~~~~~~~~~~~~~~~~ -Look through the GitHub issues for features. Anything tagged with "enhancement" -and "help wanted" is open to whoever wants to implement it. +Look through the GitHub issues for features. Anything tagged with "enhancement" and "help wanted" is open to whoever wants to implement it. Write Documentation ~~~~~~~~~~~~~~~~~~~ -xscen could always use more documentation, whether as part of the -official xscen docs, in docstrings, or even on the web in blog posts, -articles, and such. +xscen could always use more documentation, whether as part of the official xscen docs, in docstrings, or even on the web in blog posts, articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ @@ -57,24 +52,23 @@ If you are proposing a feature: Get Started! ------------ -Ready to contribute? Here's how to set up `xscen` for local development. +Ready to contribute? Here's how to set up ``xscen`` for local development. -1. Clone the repo locally:: +#. Clone the repo locally:: $ git clone git@github.com:Ouranosinc/xscen.git +#. Install your local copy into a development environment. Using ``mamba``, you can create a new development environment with:: -2. Install your local copy into an isolated environment. We usually use `mamba` or `conda` for this:: - - $ cd xscen/ $ mamba env create -f environment-dev.yml - $ pip install -e ".[dev]" + $ conda activate xscen + $ python -m pip install --editable ".[dev]" -3. As xscen was installed in editable mode, we also need to compile the translation catalogs manually: +#. As xscen was installed in editable mode, we also need to compile the translation catalogs manually: $ make translate -4. To ensure a consistent style, please install the pre-commit hooks to your repo:: +#. To ensure a consistent style, please install the pre-commit hooks to your repo:: $ pre-commit install @@ -83,16 +77,23 @@ Ready to contribute? Here's how to set up `xscen` for local development. $ pre-commit run -a -5. Create a branch for local development:: +#. Create a branch for local development:: $ git checkout -b name-of-your-bugfix-or-feature Now you can make your changes locally. -6. When you're done making changes, check that your changes pass `flake8`, `black`, and the - tests, including testing other Python versions with `tox`:: +#. When you're done making changes, check that your changes pass ``black``, ``blackdoc``, ``flake8``, ``isort``, ``ruff``, and the tests, including testing other Python versions with tox:: + + $ black --check xscen tests + $ isort --check xscen tests + $ ruff xscen tests + $ flake8 xscen tests + $ blackdoc --check xscen docs + $ python -m pytest + $ tox - $ tox + To get ``black``, ``blackdoc``, ``flake8``, ``isort``, ``ruff``, and tox, just pip install them into your virtualenv. Alternatively, you can run the tests using `make`:: @@ -101,57 +102,61 @@ Ready to contribute? Here's how to set up `xscen` for local development. Running `make lint` and `make test` demands that your runtime/dev environment have all necessary development dependencies installed. -.. warning:: + .. warning:: - Due to some dependencies only being available via Anaconda/conda-forge or built from source, `tox`-based testing will only work if `ESMF`_ is available in your system path. This also requires that the `ESMF_VERSION` environment variable (matching the version of ESMF installed) be accessible within your shell as well (e.g.: `$ export ESMF_VERSION=8.5.0`). + Due to some dependencies only being available via Anaconda/conda-forge or built from source, `tox`-based testing will only work if `ESMF`_ is available in your system path. This also requires that the `ESMF_VERSION` environment variable (matching the version of ESMF installed) be accessible within your shell as well (e.g.: `$ export ESMF_VERSION=8.5.0`). -7. Commit your changes and push your branch to GitHub:: +#. Commit your changes and push your branch to GitHub:: $ git add . $ git commit -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature -8. If you are editing the docs, compile and open them with:: +#. If you are editing the docs, compile and open them with:: $ make docs # or to simply generate the html $ cd docs/ $ make html -.. note:: + .. note:: - When building the documentation, the default behaviour is to evaluate notebooks ('nbsphinx_execute = "always"'), rather than simply parse the content ('nbsphinx_execute = "never"'). Due to their complexity, this can sometimes be a very computationally demanding task and should only be performed when necessary (i.e.: when the notebooks have been modified). + When building the documentation, the default behaviour is to evaluate notebooks ('nbsphinx_execute = "always"'), rather than simply parse the content ('nbsphinx_execute = "never"'). Due to their complexity, this can sometimes be a very computationally demanding task and should only be performed when necessary (i.e.: when the notebooks have been modified). - In order to speed up documentation builds, setting a value for the environment variable "SKIP_NOTEBOOKS" (e.g. "$ export SKIP_NOTEBOOKS=1") will prevent the notebooks from being evaluated on all subsequent "$ tox -e docs" or "$ make docs" invocations. + In order to speed up documentation builds, setting a value for the environment variable "SKIP_NOTEBOOKS" (e.g. "$ export SKIP_NOTEBOOKS=1") will prevent the notebooks from being evaluated on all subsequent "$ tox -e docs" or "$ make docs" invocations. -9. Submit a pull request through the GitHub website. +#. Submit a pull request through the GitHub website. .. _translating-xscen: Translating xscen ~~~~~~~~~~~~~~~~~ -If your additions to xscen play with plain text attributes like "long_name" or "description", you should also provide -French translations for those fields. To manage translations, xscen uses python's `gettext` with the help of `babel`. + +If your additions to ``xscen` play with plain text attributes like "long_name" or "description", you should also provide +French translations for those fields. To manage translations, xscen uses python's ``gettext`` with the help of ``babel``. To update an attribute while enabling translation, use :py:func:`utils.add_attr` instead of a normal set-item. For example: -.. code python - ds.attrs['description'] = "The English description" + .. code-block:: python + + ds.attrs["description"] = "The English description" + +becomes: + + .. code-block:: python -becomes + from xscen.utils import add_attr -.. code python - from xscen.utils import add_attr - def _(s): - return s + def _(s): + return s - add_attr(ds, "description", _("English description of {a}"), a="var") + + add_attr(ds, "description", _("English description of {a}"), a="var") See also :py:func:`update_attr` for the special case where an attribute is updated using its previous version. -Once the code is implemented and translatable strings are marked as such, we need to extract them and catalog them -in the French translation map. From the root directory of xscen, run:: +Once the code is implemented and translatable strings are marked as such, we need to extract them and catalog them in the French translation map. From the root directory of xscen, run:: $ make findfrench @@ -159,19 +164,20 @@ Then go edit ``xscen/xscen/data/fr/LC_MESSAGES/xscen.po`` with the correct Frenc $ make translate -will compile the edited catalogs, allowing python to detect and use them. +This will compile the edited catalogs, allowing python to detect and use them. Pull Request Guidelines ----------------------- Before you submit a pull request, check that it meets these guidelines: -1. The pull request should include tests. -2. If the pull request adds functionality, the docs should be updated. Put - your new functionality into a function with a docstring, and add the - feature to the list in README.rst. -3. The pull request should not break the templates. -4. The pull request should work for all supported major Python versions (3.9, 3.10, and 3.11). +#. The pull request should include tests. + +#. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in ``README.rst``. + +#. The pull request should not break the templates. + +#. The pull request should work for Python 3.8, 3.9, 3.10, and 3.11. Check that the tests pass for all supported Python versions. Tips ---- @@ -183,13 +189,52 @@ $ pytest tests.test_xscen Versioning/Tagging ------------------ -A reminder for the maintainers on how to deploy. -Make sure all your changes are committed (including an entry in HISTORY.rst). -The templates must also be tested manually before each release. -Then run:: +A reminder for the maintainers on how to deploy. This section is only relevant for maintainers when they are producing a new point release for the package. + +#. Create a new branch from `main` (e.g. `release-0.2.0`). +#. Update the `CHANGES.rst` file to change the `Unreleased` section to the current date. +#. Create a pull request from your branch to `main`. +#. Once the pull request is merged, create a new release on GitHub. On the main branch, run: + + .. code-block:: shell + + $ bump-my-version bump minor # In most cases, we will be releasing a minor version + $ git push + $ git push --tags + + This will trigger the CI to build the package and upload it to TestPyPI. In order to upload to PyPI, this can be done by publishing a new version on GitHub. This will then trigger the workflow to build and upload the package to PyPI. + +#. Once the release is published, it will go into a `staging` mode on Github Actions. Once the tests pass, admins can approve the release (an e-mail will be sent) and it will be published on PyPI. + +.. note:: + + The ``bump-version.yml`` GitHub workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. It is not necessary to manually bump the version in your branch when merging (non-release) pull requests. + +.. warning:: + + It is important to be aware that any changes to files found within the ``xscen`` folder (with the exception of ``xscen/__init__.py``) will trigger the ``bump-version.yml`` workflow. Be careful not to commit changes to files in this folder when preparing a new release. + +Packaging +--------- + +When a new version has been minted (features have been successfully integrated test coverage and stability is adequate), maintainers should update the ``pip``-installable package (wheel and source release) on PyPI as well as the binary on conda-forge. + +The simple approach +~~~~~~~~~~~~~~~~~~~ + +The simplest approach to packaging for general support (pip wheels) requires the following packages installed: + * build + * setuptools + * twine + * wheel + +From the command line on your Linux distribution, simply run the following from the clone's main dev branch:: + + # To build the packages (sources and wheel) + $ python -m build --sdist --wheel + + # To upload to PyPI + $ twine upload dist/* -$ bumpversion patch # possible: major / minor / patch -$ git push -$ git push --tags .. _`ESMF`: http://earthsystemmodeling.org/download/ diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index c7612de0..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,26 +0,0 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst - -recursive-include xscen *.py *.yml -recursive-include xscen CVs *.json -recursive-include xscen data fr *.yml *.csv -recursive-include xscen data fr LC_MESSAGES *.mo *.po -recursive-include tests *.py -recursive-include docs conf.py Makefile make.bat *.png *.rst *.yml -recursive-include docs notebooks *.ipynb -recursive-include docs notebooks samples *.csv *.json *.yml - -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] -recursive-exclude conda *.yml -recursive-exclude templates *.csv *.json *.py *.yml -recursive-exclude docs notebooks samples tutorial *.nc - -exclude .* -exclude Makefile -exclude environment.yml -exclude environment-dev.yml -exclude tox.ini diff --git a/Makefile b/Makefile index 425d9f43..d9cd398a 100644 --- a/Makefile +++ b/Makefile @@ -53,16 +53,19 @@ clean-test: ## remove test and coverage artifacts rm -fr htmlcov/ rm -fr .pytest_cache +lint/flake8: ## check style with flake8 + ruff xscen tests + flake8 --config=.flake8 xscen tests + lint/black: ## check style with black black --check xscen tests - -lint/flake8: ## check style with flake8 - flake8 --config=setup.cfg xscen tests + blackdoc --check xscen docs + isort --check xscen tests lint: lint/black lint/flake8 ## check style test: ## run tests quickly with the default Python - pytest + python -m pytest test-all: ## run tests on every Python version with tox tox @@ -88,7 +91,10 @@ dist: clean ## builds source and wheel package ls -l dist install: clean ## install the package to the active Python's site-packages - python setup.py install + python -m pip install . + +dev: clean ## install the package in editable mode with all development dependencies + python -m pip install --editable ".[dev]" findfrench: ## Extract phrases and update the French translation catalog (this doesn't translate) python setup.py extract_messages diff --git a/docs/changes.rst b/docs/changes.rst new file mode 100644 index 00000000..d9e113ec --- /dev/null +++ b/docs/changes.rst @@ -0,0 +1 @@ +.. include:: ../CHANGES.rst diff --git a/docs/history.rst b/docs/history.rst deleted file mode 100644 index 25064996..00000000 --- a/docs/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst index 2a2be703..fb02e17c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,7 +38,7 @@ Features api contributing authors - history + changes .. toctree:: :maxdepth: 2 diff --git a/docs/installation.rst b/docs/installation.rst index 670f7321..fe8e93ce 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -21,7 +21,7 @@ If for some reason you wish to install the `PyPI` version of `xscen` into an exi .. code-block:: console - $ pip install xscen + $ python -m pip install xscen Development Installation (Anaconda + pip) ----------------------------------------- @@ -45,7 +45,7 @@ Finally, perform an `--editable` install of xscen and compile the translation ca .. code-block:: console - $ pip install -e . + $ python -m pip install -e . $ make translate .. _Github repo: https://github.com/Ouranosinc/xscen diff --git a/environment-dev.yml b/environment-dev.yml index f66b3cd8..5be64a4c 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -57,8 +57,7 @@ dependencies: - xdoctest - pip # Testing - - tox # <4.0 # 2022-12-12: tox v4.0 is incompatible with tox-conda plugin - # - tox-conda >=0.10.2 + - tox # packaging - build - wheel diff --git a/environment.yml b/environment.yml index ccd95857..9de31b74 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - clisops >=0.10 - dask - flox - - fsspec < 2023.10.0 + - fsspec <2023.10.0 - geopandas - h5netcdf - h5py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a8a684ee --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,277 @@ +[build-system] +requires = ["setuptools", "babel", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "xscen" +authors = [ + {name = "Gabriel Rondeau-Genesse", email = "rondeau-genesse.gabriel@ouranos.ca"} +] +maintainers = [] +description = "A climate change scenario-building analysis framework, built with xclim/xarray." +readme = "README.rst" +requires-python = ">=3.9.0" +keywords = ["xscen"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering :: Atmospheric Science" +] +dynamic = ["version"] +dependencies = [ + "cartopy", + "cftime", + "cf_xarray>=0.7.6", + "clisops>=0.10", + "dask", + "flox", + "fsspec<2023.10.0", + "geopandas", + "h5netcdf", + "h5py", + "intake-esm>=2023.07.07", + "matplotlib", + "netCDF4", + "numpy", + "pandas>=2.0", + "parse", + # Used when opening catalogs. + "pyarrow", + "pyyaml", + "rechunker", + "shapely>=2.0", + "sparse", + "toolz", + "xarray", + "xclim>=0.46.0", + "xesmf>=0.7", + "zarr" +] + +[project.optional-dependencies] +dev = [ + # Dev tools and testing + "pip>=23.1.2", + "bump-my-version>=0.12.0", + "watchdog>=3.0.0", + "flake8>=6.1.0", + "flake8-rst-docstrings>=0.3.0", + "tox>=4.5.1", + "coverage>=6.2.2,<7.0.0", + "coveralls>=3.3.1", + "click>=8.1.7", + "pytest>=7.3.1", + "pytest-cov>=4.0.0", + "black>=23.10.1", + "blackdoc>=0.3.9", + "isort>=5.12.0", + "pre-commit>=3.3.2" +] +docs = [ + # Documentation and examples + "ipykernel", + "ipython", + "jupyter_client", + "nbsphinx", + "pandoc", + "sphinx", + "sphinx-autoapi", + "sphinx-codeautolink", + "sphinx-copybutton", + "sphinx-rtd-theme>=1.0", + "sphinxcontrib-napoleon" +] + +[project.urls] +"About Ouranos" = "https://www.ouranos.ca/en/" +"Changelog" = "https://xscen.readthedocs.io/en/stable/changes.html" +"Homepage" = "https://xscen.readthedocs.io/" +"Issue tracker" = "https://github.com/Ouranosinc/xscen/issues" +"Source" = "https://github.com/Ouranosinc/xscen" + +[tool] + +[tool.black] +target-version = [ + "py38", + "py39", + "py310", + "py311" +] + +[tool.bumpversion] +current_version = "0.7.20-beta" +commit = true +tag = false +tag_name = "{new_version}" +allow_dirty = false +parse = "(?P\\d+)\\.(?P\\d+).(?P\\d+)(\\-(?P[a-z]+))?" +serialize = [ + "{major}.{minor}.{patch}-{release}", + "{major}.{minor}.{patch}" +] + +[[tool.bumpversion.files]] +filename = "xscen/__init__.py" +search = "__version__ = \"{current_version}\"" +replace = "__version__ = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "tests/test_xscen.py" +search = "__version__ = \"{current_version}\"" +replace = "__version__ = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = ".cruft.json" +search = "\"version\": \"{current_version}\"" +replace = "\"version\": \"{new_version}\"" + +[[tool.bumpversion.parts.release]] +optional_value = "gamma" +values = [ + "beta", + "gamma" +] + +[tool.coverage.run] +relative_files = true +include = ["xscen/*"] +omit = ["docs/notebooks/*.ipynb", "tests/*.py"] + +[tool.isort] +append_only = true +known_first_party = "xscen" +profile = "black" +py_version = 38 + +[tool.mypy] +python_version = 3.8 +show_error_codes = true +warn_return_any = true +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = [] +ignore_missing_imports = true + +[tool.pytest.ini_options] +addopts = [ + "--color=yes", + "--cov=xscen", + "--ignore-glob='*.ipynb_checkpoints'", + "--strict-markers", + "--verbose" +] +filterwarnings = ["ignore::UserWarning"] +testpaths = "tests" +markers = ["requires_netcdf: marks tests that require netcdf files to run"] + +[tool.ruff] +src = [""] +line-length = 150 +target-version = "py38" +exclude = [ + ".eggs", + ".git", + "build", + "docs", + "tests" +] +ignore = [ + "D205", + "D400", + "D401" +] +select = [ + "C9", + "D", + "E", + "F", + "W" +] + +[tool.ruff.flake8-bandit] +check-typed-exception = true + +[tool.ruff.format] +line-ending = "auto" + +[tool.ruff.isort] +known-first-party = ["xscen"] +case-sensitive = true +detect-same-package = false +lines-after-imports = 1 +no-lines-before = ["future", "standard-library"] + +[tool.ruff.mccabe] +max-complexity = 15 + +[tool.ruff.per-file-ignores] +"xscen/**/__init__.py" = ["F401", "F403"] + +[tool.ruff.pycodestyle] +max-doc-length = 180 + +[tool.ruff.pydocstyle] +convention = "numpy" + +[tool.setuptools] +license-files = ["LICENSE"] +include-package-data = true + +[tool.setuptools.dynamic] +version = {attr = "xscen.__version__"} + +[tool.setuptools.packages.find] +include = [ + ".zenodo.json", + "AUTHORS.rst", + "CHANGES.rst", + "CONTRIBUTING.rst", + "LICENSE", + "Makefile", + "README.rst", + "docs/*.rst", + "docs/Makefile", + "docs/_static/_images/*.png", + "docs/conf.py", + "docs/make.bat", + "docs/notebooks/.ipynb", + "docs/notebooks/samples/*.csv", + "docs/notebooks/samples/*.json", + "docs/notebooks/samples/*.yml", + "environment.yml", + "environment-dev.yml", + "tests/*.py", + "xscen*", + "tox.ini" +] +exclude = [ + "*.py[co]", + "__pycache__", + ".coveralls.yml", + ".editorconfig", + ".flake8", + ".gitignore", + ".pre-commit-config.yaml", + ".readthedocs.yml", + ".yamllint.yaml", + "Makefile", + "conda", + "docs/_*", + "docs/modules.rst", + "docs/xscen*.rst", + "docs/notesbnooks/samples/tutorial/**/*.nc", + "templates" +] +namespaces = true diff --git a/setup.cfg b/setup.cfg index 541cadb1..41eb2465 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,100 +1,3 @@ -[bumpversion] -current_version = 0.7.20-beta -commit = True -tag = False -parse = (?P\d+)\.(?P\d+).(?P\d+)(\-(?P[a-z]+))? -serialize = - {major}.{minor}.{patch}-{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = gamma -values = - beta - gamma - -[bumpversion:file:setup.py] -search = version="{current_version}" -replace = version="{new_version}" - -[bumpversion:file:xscen/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[bumpversion:file:tests/test_xscen.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[bumpversion:file:.cruft.json] -search = "version": "{current_version}", -replace = "version": "{new_version}", - -[aliases] -test = pytest - -[tool:pytest] -collect_ignore = ['setup.py'] -addopts = - --color=yes - --cov=xscen - --ignore-glob='*.ipynb_checkpoints' - --strict-markers - --verbose -filterwarnings = - ignore::UserWarning -markers = - requires_netcdf: marks tests that require netcdf files to run - -[flake8] -exclude = - .git, - docs, - build, - .eggs, - docs/conf.py, -max-line-length = 88 -max-complexity = 12 -ignore = - C901 - E203 - E231 - E266 - E501 - F401 - F403 - W503 - W504 -per-file-ignores = - tests/*:E402 -rst-roles = - mod, - py:attr, - py:attribute, - py:class, - py:const, - py:data, - py:func, - py:meth, - py:mod, - py:obj, - py:ref, - ref - -[coverage:run] -relative_files = True -omit = - docs/notebooks - tests - -[isort] -profile = black -append_only = true -known_first_party = xscen - -[pydocstyle] -convention = numpy -match = ((?!test_|conf).)*\.py - [extract_messages] output_file = xscen.pot keywords = _ gettext ngettext diff --git a/setup.py b/setup.py index f34e9227..aa4c0ee3 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,16 @@ -#!/usr/bin/env python -"""The setup script.""" -from setuptools import find_packages, setup +"""Custom installation process for xscen translations.""" + +from babel.messages.frontend import compile_catalog +from setuptools import setup +from setuptools.build_meta import * # noqa: F403, F401 from setuptools.command.install import install class InstallWithCompile(install): """Injection of the catalog compilation in the installation process.""" - # Taken from https://stackoverflow.com/a/41120180/9291575 - def run(self): """Install the package, but compile the i18n catalogs first.""" - from babel.messages.frontend import compile_catalog - compiler = compile_catalog(self.distribution) option_dict = self.distribution.get_option_dict("compile_catalog") compiler.domain = [option_dict["domain"][1]] @@ -21,112 +19,8 @@ def run(self): super().run() -with open("README.rst") as readme_file: - readme = readme_file.read() - -# Copied from xclim. Needed for replacing roles with hyperlinks in documentation on PyPI -# hyperlink_replacements = { -# r":issue:`([0-9]+)`": r"`GH/\1 `_", -# r":pull:`([0-9]+)`": r"`PR/\1 `_", -# r":user:`([a-zA-Z0-9_.-]+)`": r"`@\1 `_", -# } -# for search, replacement in hyperlink_replacements.items(): -# history = re.sub(search, replacement, history) - -# This list are the minimum requirements for xscen -# this is only meant to make `pip check` work -# xscen dependencies can only be installed through conda-forge. -# Don't forget to sync changes between environment.yml, environment-dev.yml, and setup.py! -# Also consider updating the list in xs.utils.show_versions if you add a new package. -requirements = [ - "cartopy", - "cftime", - "cf_xarray>=0.7.6", - "clisops>=0.10", - "dask", - "flox", - "fsspec<2023.10.0", - "geopandas", - "h5netcdf", - "h5py", - "intake-esm>=2023.07.07", - "matplotlib", - "netCDF4", - "numpy", - "pandas>=2.0", - "parse", - "pyarrow", # Used when opening catalogs. - "pyyaml", - "rechunker", - "shapely>=2.0", - "sparse", - "toolz", - "xarray", - "xclim>=0.43", - "xesmf>=0.7.0", - "zarr", -] - -dev_requirements = [ - "black", - "flake8", - "flake8-rst-docstrings", - "pytest", - "pytest-cov", -] - -docs_requirements = [ - "ipykernel", - "ipython", - "jupyter_client", - "nbsphinx", - "nc-time-axis>=1.3.1", - "nbval", - "sphinx", - "sphinx-autoapi", - "sphinx-rtd-theme>=1.0", - "sphinxcontrib-napoleon", - "sphinx-codeautolink", - "sphinx-copybutton", -] - -setup_requirements = ["babel"] - setup( - author="Gabriel Rondeau-Genesse", - author_email="rondeau-genesse.gabriel@ouranos.ca", - python_requires=">=3.9", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering :: Atmospheric Science", - ], cmdclass={"install": InstallWithCompile}, - description="A climate change scenario-building analysis framework, built with xclim/xarray.", - install_requires=requirements, - long_description=readme, - long_description_content_type="text/x-rst", - include_package_data=True, - keywords="xscen", message_extractors={"xscen": [("**.py", "python", None)]}, - name="xscen", - packages=find_packages(include=["xscen", "xscen.*"]), - project_urls={ - "About Ouranos": "https://www.ouranos.ca/en/", - "Changelog": "https://xscen.readthedocs.io/en/stable/history.html", - "Issue tracker": "https://github.com/Ouranosinc/xscen/issues", - }, - setup_requires=setup_requirements, - test_suite="tests", - extras_require={"dev": dev_requirements, "docs": docs_requirements}, - url="https://github.com/Ouranosinc/xscen", - version="0.7.20-beta", - zip_safe=False, + setup_requires=["babel"], ) diff --git a/tox.ini b/tox.ini index cdf7d9ed..e982c993 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] min_version = 4.0 envlist = - black + lint py{39,310,311}-esmpy docs requires = @@ -11,7 +11,7 @@ requires = opts = --verbose -[testenv:black] +[testenv:lint] description = Check for Code Compliance skip_install = True download = true @@ -19,8 +19,11 @@ conda_channels = conda_env = deps = black + blackdoc + isort flake8 flake8-rst-docstrings + ruff commands_pre = pip list commands = diff --git a/xscen/__init__.py b/xscen/__init__.py index ac3fa080..2ccd5262 100644 --- a/xscen/__init__.py +++ b/xscen/__init__.py @@ -1,4 +1,4 @@ -"""Top-level package for xscen.""" +"""A climate change scenario-building analysis framework, built with xclim/xarray.""" import warnings # Import the submodules From 2d96cdba8540d73bf004b4d7729b755c58e1abba Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 24 Nov 2023 17:09:51 -0500 Subject: [PATCH 02/22] ci adjustments --- .github/labeler.yml | 1 + .github/workflows/bump-version.yml | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 8815f1ae..5a7577d8 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -8,6 +8,7 @@ CI: - .cruft.json - .editorconfig + - .flake8 - .gitignore - .pre-commit-config.yaml - .readthedocs.yml diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 92e7fce4..1f062b12 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -7,6 +7,7 @@ on: paths-ignore: - .cruft.json - .editorconfig + - .flake8 - .github/**.yml - .gitignore - .gitlab-ci.yml @@ -14,19 +15,19 @@ on: - .pre-commit-config.yaml - .yamllint.yaml - AUTHORS.rst + - CHANGES.rst - CONTRIBUTING.rst - - docs/notebooks + - MANIFEST.in + - Makefile - docs/*.py - docs/*.rst + - docs/notebooks - environment-dev.yml - environment.yml - - HISTORY.rst - - Makefile - - MANIFEST.in - - requirements_upstream.txt + - pyproject.toml - setup.cfg - setup.py - - tests/**.py + - tests/*.py - tox.ini - xscen/__init__.py @@ -48,9 +49,9 @@ jobs: run: echo "current_version=$(grep -E '__version__' xscen/__init__.py | cut -d ' ' -f3)" - name: Bump Patch Version run: | - python -m pip install bump2version + python -m pip install bump-my-version echo "Bumping version" - python -m bump2version patch + python -m bump-my-version bump patch echo "new_version=$(grep -E '__version__' xscen/__init__.py | cut -d ' ' -f3)" - name: Push Changes uses: ad-m/github-push-action@master From 7fb15cbd768b35eeb9bfc8fe7f7c4d4b3af5ef08 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:01:42 -0500 Subject: [PATCH 03/22] flake8 and ruff adjutsments --- .pre-commit-config.yaml | 1 - .../1-basic_workflow_with_config/workflow1.py | 40 ++++++---- xscen/__init__.py | 2 +- xscen/aggregate.py | 58 ++++++++------ xscen/biasadjust.py | 8 +- xscen/catutils.py | 6 +- xscen/config.py | 2 +- xscen/diagnostics.py | 32 +++++--- xscen/ensembles.py | 33 +++++--- xscen/extract.py | 80 ++++++++++++------- xscen/indicators.py | 28 ++++--- xscen/io.py | 20 +++-- xscen/regrid.py | 4 +- xscen/scripting.py | 10 +-- xscen/spatial.py | 4 +- xscen/utils.py | 10 +-- 16 files changed, 201 insertions(+), 137 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 643eb32b..b6e6c14b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,6 @@ repos: args: [ '--target-version=py39' ] additional_dependencies: [ 'black==23.11.0' ] - id: nbqa-isort - args: [ "--settings-file=setup.cfg" ] additional_dependencies: [ 'isort==5.12.0' ] - repo: https://github.com/kynan/nbstripout rev: 0.6.1 diff --git a/templates/1-basic_workflow_with_config/workflow1.py b/templates/1-basic_workflow_with_config/workflow1.py index 6fd82a1d..6897bc2f 100644 --- a/templates/1-basic_workflow_with_config/workflow1.py +++ b/templates/1-basic_workflow_with_config/workflow1.py @@ -1,7 +1,6 @@ """Template of a typical workflow.""" import atexit import logging -import os import xarray as xr from dask import config as dskconf @@ -11,7 +10,8 @@ from xscen.config import CONFIG # Load configuration -# paths1.yml is used to add private information to your workflow, such as file paths, without running the risk of them being pushed to a public Github repo. +# paths1.yml is used to add private information to your workflow, such as file paths, +# without running the risk of them being pushed to a public GitHub repo. # For this to work as intended, 'paths1.yml' should be included in your .gitignore. # config1.yml (or any number of those) can then contain the rest of the configuration. # All configuration files are merged together into a single CONFIG instance when you call xs.load_config @@ -23,7 +23,8 @@ if "logging" in CONFIG: logger = logging.getLogger("xscen") -# The workflow is made to be able to restart where it left off, in case of a crash or if you needed to stop it for some reason. +# The workflow is made to be able to restart where it left off, +# in case of a crash or if you needed to stop it for some reason. # To achieve this, it checks if the results of a task are already in the project catalog before doing it. # When a task is completed, the produced data files are added to the project catalog. # Files manually removed from disk will be noticed by the workflow and they will be produced again. @@ -60,7 +61,8 @@ # --- EXTRACT--- # Check if that step is in list of tasks before doing it, hence when commented in the config it will be skipped. if "extract" in CONFIG["tasks"]: - # Iterate on types of data to extract (reconstruction, simulation) and get the respective dictionary from the config + # Iterate on types of data to extract (reconstruction, simulation) + # and get the respective dictionary from the config for source_type, type_dict in CONFIG["extract"].items(): # Filter the catalog to get only the datasets that match the arguments in the config. # Arguments are not passed automatically, because the config is different for each type of data. @@ -70,7 +72,8 @@ # Iterate over the datasets that matched the search # 'ds_id' is the ID of the dataset, 'dc' is the sub-catalog for this dataset for ds_id, dc in cat.items(): - # These are some relevant attributes that are used to check if the task was done already and to write the output path. + # These are some relevant attributes that are used to check if the task was done already + # and to write the output path. cur = { "id": ds_id, "xrfreq": "D", @@ -83,7 +86,8 @@ Client(**type_dict["dask"], **daskkws), xs.measure_time(name=f"extract {cur}", logger=logger), ): - # Extract and create a dictionary of datasets from the sub-catalog, with the requested domain, period, and frequency. + # Extract and create a dictionary of datasets from the sub-catalog, + # with the requested domain, period, and frequency. ds_dict = xs.extract_dataset( catalog=dc, # Once again, we manually pass the arguments from the config to the function. @@ -92,7 +96,8 @@ # Iterate over the different frequencies for key_freq, ds in ds_dict.items(): - # For future steps in the workflow, we can save a lot of time by stacking the spatial coordinates into a single dimension + # For future steps in the workflow, we can save a lot of time + # by stacking the spatial coordinates into a single dimension # and dropping the NaNs. This is especially useful for big datasets. if type_dict.get("stack_drop_nans", False): ds = xs.utils.stack_drop_nans( @@ -104,7 +109,8 @@ # Prepare the filename for the zarr file, using the format specified in paths1.yml path = CONFIG["paths"]["task"].format(**cur) # Save to zarr - # Once again, we manually pass the arguments from the config to the function, as they will differ for each task. + # Once again, we manually pass the arguments from the config to the function, + # as they will differ for each task. xs.save_to_zarr(ds=ds, filename=path, **type_dict["save"]) pcat.update_from_ds(ds=ds, path=path) @@ -123,7 +129,8 @@ # --- REGRID --- if "regrid" in CONFIG["tasks"]: # Search the ProjectCatalog for the results of the previous step, then iterate over each dataset. - # We usually don't have to rely on search_data_catalogs anymore after the initial extraction, because the content of the ProjectCatalog is smaller and more manageable. + # We usually don't have to rely on search_data_catalogs anymore after the initial extraction, + # because the content of the ProjectCatalog is smaller and more manageable. # In most cases, we can just use the search function with the 'type' and 'processing_level' attributes. input_dict = pcat.search(**CONFIG["regrid"]["inputs"]).to_dataset_dict(**tdd) for key_input, ds_input in input_dict.items(): @@ -143,7 +150,8 @@ ) # Perform the regridding - # Most arguments are passed automatically from the config, but we still need to manually pass the input and the grid. + # Most arguments are passed automatically from the config, + # but we still need to manually pass the input and the grid. ds_regrid = xs.regrid_dataset( ds=ds_input, ds_grid=ds_grid, @@ -205,8 +213,10 @@ # - Convert the data to a standard calendar and fill the missing values. if "cleanup" in CONFIG["tasks"]: # In the previous step, we bias adjusted tasmax, dtr, and pr. - # Here, we can use search_data_catalogs with ['tasmin', 'tasmax', 'pr'] to instruct intake-esm to compute 'tasmin' from 'tasmax' and 'dtr'. - # You can find more details on DerivedVariables here: https://xscen.readthedocs.io/en/latest/notebooks/1_catalog.html#Derived-variables + # Here, we can use search_data_catalogs with ['tasmin', 'tasmax', 'pr'] + # to instruct intake-esm to compute 'tasmin' from 'tasmax' and 'dtr'. + # You can find more details on DerivedVariables + # here: https://xscen.readthedocs.io/en/latest/notebooks/1_catalog.html#Derived-variables cu_cats = xs.search_data_catalogs(**CONFIG["cleanup"]["search_data_catalogs"]) for cu_id, cu_cat in cu_cats.items(): cur = { @@ -219,7 +229,8 @@ Client(**CONFIG["cleanup"]["dask"], **daskkws), xs.measure_time(name=f"{cur}", logger=logger), ): - # Alongside search_data_catalogs, extract_dataset will compute 'tasmin' and put the requested variables back together in one dataset. + # Alongside search_data_catalogs, extract_dataset will compute 'tasmin' + # and put the requested variables back together in one dataset. freq_dict = xs.extract_dataset(catalog=cu_cat) # Iterate over the frequencies (usually just 'D') @@ -271,7 +282,8 @@ # --- DIAGNOSTICS --- if "diagnostics" in CONFIG["tasks"]: - # The properties and measures that we want to compute are different for each type of data (ref, sim, scen), so we need to iterate over them. + # The properties and measures that we want to compute are different for each type of data (ref, sim, scen), + # so we need to iterate over them. for kind, kind_dict in CONFIG["diagnostics"]["kind"].items(): # Search for the right datasets and iterate over them dict_input = pcat.search(**kind_dict["inputs"]).to_dataset_dict(**tdd) diff --git a/xscen/__init__.py b/xscen/__init__.py index 2ccd5262..6d342bfd 100644 --- a/xscen/__init__.py +++ b/xscen/__init__.py @@ -55,10 +55,10 @@ __version__ = "0.7.20-beta" -# monkeypatch so that warnings.warn() doesn't mention itself def warning_on_one_line( message: str, category: Warning, filename: str, lineno: int, file=None, line=None ): # noqa: D103 + """Monkeypatch Reformat warning so that `warnings.warn` doesn't mention itself.""" return f"{filename}:{lineno}: {category.__name__}: {message}\n" diff --git a/xscen/aggregate.py b/xscen/aggregate.py index 3d872fd0..231dde08 100644 --- a/xscen/aggregate.py +++ b/xscen/aggregate.py @@ -30,8 +30,8 @@ __all__ = [ "climatological_mean", "compute_deltas", - "spatial_mean", "produce_horizon", + "spatial_mean", ] @@ -67,7 +67,8 @@ def climatological_mean( interval : int Interval (in years) at which to provide an output. periods : list of str or list of lists of str, optional - Either [start, end] or list of [start, end] of continuous periods to be considered. This is needed when the time axis of ds contains some jumps in time. + Either [start, end] or list of [start, end] of continuous periods to be considered. + This is needed when the time axis of ds contains some jumps in time. If None, the dataset will be considered continuous. to_level : str, optional The processing level to assign to the output. @@ -202,7 +203,7 @@ def climatological_mean( @parse_config -def compute_deltas( +def compute_deltas( # noqa: C901 ds: xr.Dataset, reference_horizon: Union[str, xr.Dataset], *, @@ -217,7 +218,8 @@ def compute_deltas( ds : xr.Dataset Dataset to use for the computation. reference_horizon : str or xr.Dataset - Either a YYYY-YYYY string corresponding to the 'horizon' coordinate of the reference period, or a xr.Dataset containing the climatological mean. + Either a YYYY-YYYY string corresponding to the 'horizon' coordinate of the reference period, + or a xr.Dataset containing the climatological mean. kind : str or dict ['+', '/', '%'] Whether to provide absolute, relative, or percentage deltas. Can also be a dictionary separated per variable name. @@ -368,7 +370,7 @@ def compute_deltas( @parse_config -def spatial_mean( +def spatial_mean( # noqa: C901 ds: xr.Dataset, method: str, *, @@ -388,9 +390,11 @@ def spatial_mean( Dataset to use for the computation. method : str 'cos-lat' will weight the area covered by each pixel using an approximation based on latitude. - 'interp_centroid' will find the region's centroid (if coordinates are not fed through kwargs), then perform a .interp() over the spatial dimensions of the Dataset. + 'interp_centroid' will find the region's centroid (if coordinates are not fed through kwargs), + then perform a .interp() over the spatial dimensions of the Dataset. The coordinate can also be directly fed to .interp() through the 'kwargs' argument below. - 'xesmf' will make use of xESMF's SpatialAverager. This will typically be more precise, especially for irregular regions, but can be much slower than other methods. + 'xesmf' will make use of xESMF's SpatialAverager. This will typically be more precise, + especially for irregular regions, but can be much slower than other methods. spatial_subset : bool, optional If True, xscen.spatial.subset will be called prior to the other operations. This requires the 'region' argument. If None, this will automatically become True if 'region' is provided and the subsetting method is either 'cos-lat' or 'mean'. @@ -398,7 +402,8 @@ def spatial_mean( Description of the region and the subsetting method (required fields listed in the Notes). If method=='interp_centroid', this is used to find the region's centroid. If method=='xesmf', the bounding box or shapefile is given to SpatialAverager. - Can also be "global", for global averages. This is simply a shortcut for `{'name': 'global', 'method': 'bbox', 'lon_bnds' [-180, 180], 'lat_bnds': [-90, 90]}`. + Can also be "global", for global averages. + This is simply a shortcut for `{'name': 'global', 'method': 'bbox', 'lon_bnds' [-180, 180], 'lat_bnds': [-90, 90]}`. kwargs : dict, optional Arguments to send to either mean(), interp() or SpatialAverager(). For SpatialAverager, one can give `skipna` or `out_chunks` here, to be passed to the averager call itself. @@ -436,7 +441,8 @@ def spatial_mean( kwargs = kwargs or {} if method == "mean": warnings.warn( - "xs.spatial_mean with method=='mean' is deprecated and will be abandoned in a future release. Use method=='cos-lat' instead for a more robust but similar method.", + "xs.spatial_mean with method=='mean' is deprecated and will be abandoned in a future release. " + "Use method=='cos-lat' instead for a more robust but similar method.", category=FutureWarning, ) elif method == "interp_coord": @@ -693,7 +699,7 @@ def spatial_mean( @parse_config -def produce_horizon( +def produce_horizon( # noqa: C901 ds: xr.Dataset, indicators: Union[ str, @@ -708,7 +714,9 @@ def produce_horizon( to_level: Optional[str] = "horizons", period: Optional[list] = None, ) -> xr.Dataset: - """Compute indicators, then the climatological mean, and finally unstack dates in order to have a single dataset with all indicators of different frequencies. + """ + Compute indicators, then the climatological mean, and finally unstack dates in order + to have a single dataset with all indicators of different frequencies. Once this is done, the function drops 'time' in favor of 'horizon'. This function computes the indicators and does an interannual mean. @@ -716,21 +724,21 @@ def produce_horizon( Parameters ---------- - ds: xr.Dataset + ds : xr.Dataset Input dataset with a time dimension. - indicators: Union[str, os.PathLike, Sequence[Indicator], Sequence[Tuple[str, Indicator]], ModuleType] + indicators : Union[str, os.PathLike, Sequence[Indicator], Sequence[Tuple[str, Indicator]], ModuleType] Indicators to compute. It will be passed to the `indicators` argument of `xs.compute_indicators`. - periods: list of str or list of lists of str, optional + periods : list of str or list of lists of str, optional Either [start, end] or list of [start_year, end_year] for the period(s) to be evaluated. If both periods and warminglevels are None, the full time series will be used. - warminglevels: dict, optional + warminglevels : dict, optional Dictionary of arguments to pass to `py:func:xscen.subset_warming_level`. If 'wl' is a list, the function will be called for each value and produce multiple horizons. If both periods and warminglevels are None, the full time series will be used. - to_level: str, optional + to_level : str, optional The processing level to assign to the output. - If there is only one horizon, you can use "{wl}", "{period0}" and "{period1}" in the string to dynamically include - that information in the processing level. + If there is only one horizon, you can use "{wl}", "{period0}" and "{period1}" in the string to dynamically + include that information in the processing level. Returns ------- @@ -739,7 +747,8 @@ def produce_horizon( """ if "warminglevel" in ds and len(ds.warminglevel) != 1: raise ValueError( - "Input dataset should only have `warminglevel` dimension of length 1. If you want to use produce_horizon for multiple warming levels, " + "Input dataset should only have `warminglevel` dimension of length 1. " + "If you want to use produce_horizon for multiple warming levels, " "extract the full time series and use the `warminglevels` argument instead." ) if period is not None: @@ -777,7 +786,8 @@ def produce_horizon( ds_sub = ds.sel(time=slice(period[0], period[1])) else: warnings.warn( - f"The requested period {period} is not fully covered by the input dataset. The requested period will be skipped." + f"The requested period {period} is not fully covered by the input dataset. " + "The requested period will be skipped." ) ds_sub = None else: @@ -812,7 +822,8 @@ def produce_horizon( new_dim = "month" else: raise ValueError( - f"Frequency {freq} is not supported or recognized. Please use annual (AS), seasonal (QS), monthly (MS), or fixed (fx) frequency." + f"Frequency {freq} is not supported or recognized." + "Please use annual (AS), seasonal (QS), monthly (MS), or fixed (fx) frequency." ) ds_mean = unstack_dates( ds_mean, @@ -865,8 +876,9 @@ def produce_horizon( ] ): warnings.warn( - f"The attributes for variable {v} are not the same for all horizons, probably because the periods were not of the same length. " - f"Attributes will be kept from the first horizon, but this might not be the most appropriate." + f"The attributes for variable {v} are not the same for all horizons, " + "probably because the periods were not of the same length. " + "Attributes will be kept from the first horizon, but this might not be the most appropriate." ) out = xr.concat(out, dim="horizon") diff --git a/xscen/biasadjust.py b/xscen/biasadjust.py index 77e2eeae..a420824a 100644 --- a/xscen/biasadjust.py +++ b/xscen/biasadjust.py @@ -220,10 +220,10 @@ def adjust( The project to assign to the output. moving_yearly_window: dict, optional Arguments to pass to `xclim.sdba.construct_moving_yearly_window`. - If not None, `construct_moving_yearly_window` will be called on dsim (and scen in - xclim_adjust_args if it exists) before adjusting and `unpack_moving_yearly_window` - will be called on the output after the adjustment. - `construct_moving_yearly_window` stacks windows of the dataArray in a new 'movingwin' dimension and `unpack_moving_yearly_window` unpack it to a normal time series. + If not None, `construct_moving_yearly_window` will be called on dsim (and scen in xclim_adjust_args if it exists) + before adjusting and `unpack_moving_yearly_window` will be called on the output after the adjustment. + `construct_moving_yearly_window` stacks windows of the dataArray in a new 'movingwin' dimension. + `unpack_moving_yearly_window` unpacks it to a normal time series. align_on: str, optional `align_on` argument for the fonction `xclim.core.calendar.convert_calendar`. diff --git a/xscen/catutils.py b/xscen/catutils.py index c3908262..14549384 100644 --- a/xscen/catutils.py +++ b/xscen/catutils.py @@ -252,7 +252,7 @@ def _name_parser( } -def _parse_dir( +def _parse_dir( # noqa: C901 root: Union[os.PathLike, str], patterns: list[str], dirglob: Optional[str] = None, @@ -432,7 +432,7 @@ def _parse_first_ds( @parse_config -def parse_directory( +def parse_directory( # noqa:C901 directories: list[Union[str, os.PathLike]], patterns: list[str], *, @@ -695,7 +695,7 @@ def parse_directory( return df -def parse_from_ds( +def parse_from_ds( # noqa: C901 obj: Union[str, os.PathLike, xr.Dataset], names: Sequence[str], attrs_map: Optional[Mapping[str, str]] = None, diff --git a/xscen/config.py b/xscen/config.py index 3c3106cc..1fc4e4db 100644 --- a/xscen/config.py +++ b/xscen/config.py @@ -93,7 +93,7 @@ def update_from_list(self, pairs): for key, valstr in pairs: try: val = ast.literal_eval(valstr) - except (ValueError, SyntaxError): + except (SyntaxError, ValueError): val = valstr self.set(key, val) diff --git a/xscen/diagnostics.py b/xscen/diagnostics.py index b64042f6..5e84c11d 100644 --- a/xscen/diagnostics.py +++ b/xscen/diagnostics.py @@ -30,9 +30,9 @@ __all__ = [ "health_checks", - "properties_and_measures", "measures_heatmap", "measures_improvement", + "properties_and_measures", ] @@ -42,7 +42,7 @@ def _(s): @parse_config -def health_checks( +def health_checks( # noqa: C901 ds: Union[xr.Dataset, xr.DataArray], *, structure: Optional[dict] = None, @@ -78,24 +78,30 @@ def health_checks( Dictionary containing the expected variables and units. cfchecks: dict, optional Dictionary where the key is the variable to check and the values are the cfchecks. - The cfchecks themselves must be a dictionary with the keys being the cfcheck names and the values being the arguments to pass to the cfcheck. + The cfchecks themselves must be a dictionary with the keys being the cfcheck names + and the values being the arguments to pass to the cfcheck. See `xclim.core.cfchecks` for more details. freq: str, optional Expected frequency, written as the result of xr.infer_freq(ds.time). missing: dict or str or list of str, optional - String, list of strings, or dictionary where the key is the method to check for missing data and the values are the arguments to pass to the method. - The methods are: "missing_any", "at_least_n_valid", "missing_pct", "missing_wmo". See :py:func:`xclim.core.missing` for more details. + String, list of strings, or dictionary where the key is the method to check for missing data + and the values are the arguments to pass to the method. + The methods are: "missing_any", "at_least_n_valid", "missing_pct", "missing_wmo". + See :py:func:`xclim.core.missing` for more details. flags: dict, optional Dictionary where the key is the variable to check and the values are the flags. - The flags themselves must be a dictionary with the keys being the data_flags names and the values being the arguments to pass to the data_flags. - If `None` is passed instead of a dictionary, then xclim's default flags for the given variable are run. See :py:data:`xclim.core.utils.VARIABLES`. - See :py:func:`xclim.core.dataflags.data_flags` for the list of possible flags. + The flags themselves must be a dictionary with the keys being the data_flags names + and the values being the arguments to pass to the data_flags. + If `None` is passed instead of a dictionary, then xclim's default flags for the given variable are run. + See :py:data:`xclim.core.utils.VARIABLES`. + See also :py:func:`xclim.core.dataflags.data_flags` for the list of possible flags. flags_kwargs: dict, optional Additional keyword arguments to pass to the data_flags ("dims" and "freq"). return_flags: bool Whether to return the Dataset created by data_flags. raise_on: list of str, optional - Whether to raise an error if a check fails, else there will only be a warning. The possible values are the names of the checks. + Whether to raise an error if a check fails, else there will only be a warning. + The possible values are the names of the checks. Use ["all"] to raise on all checks. Returns @@ -290,7 +296,7 @@ def _message(): # TODO: just measures? @parse_config -def properties_and_measures( +def properties_and_measures( # noqa: C901 ds: xr.Dataset, properties: Union[ str, @@ -333,7 +339,7 @@ def properties_and_measures( If not None, calls `xscen.utils.change_units` on ds before computing properties using this dictionary for the `variables_and_units` argument. It can be useful to convert units before computing the properties, because it is sometimes - easier to convert the units of the variables than the units of the properties (eg. variance). + easier to convert the units of the variables than the units of the properties (e.g. variance). to_level_prop : str processing_level to give the first output (prop) to_level_meas : str @@ -446,8 +452,8 @@ def measures_heatmap( List or dictionary of datasets of measures of properties. If it is a dictionary, the keys will be used to name the rows. If it is a list, the rows will be given a number. - to_level: str - processing_level to assign to the output + to_level : str + The processing_level to assign to the output. Returns ------- diff --git a/xscen/ensembles.py b/xscen/ensembles.py index d72944c7..01b3e296 100644 --- a/xscen/ensembles.py +++ b/xscen/ensembles.py @@ -61,8 +61,9 @@ def ensemble_stats( See Also -------- - xclim.ensembles._base.create_ensemble, xclim.ensembles._base.ensemble_percentiles, xclim.ensembles._base.ensemble_mean_std_max_min, xclim.ensembles._robustness.change_significance, xclim.ensembles._robustness.robustness_coefficient, - + xclim.ensembles._base.create_ensemble, xclim.ensembles._base.ensemble_percentiles, + xclim.ensembles._base.ensemble_mean_std_max_min, xclim.ensembles._robustness.change_significance, + xclim.ensembles._robustness.robustness_coefficient, """ create_kwargs = create_kwargs or {} @@ -141,7 +142,7 @@ def ensemble_stats( return ens_stats -def generate_weights( +def generate_weights( # noqa: C901 datasets: Union[dict, list], *, independence_level: str = "model", @@ -162,22 +163,27 @@ def generate_weights( A dictionary can be passed instead of a list, in which case the keys are used for the 'realization' coordinate. Tip: With a project catalog, you can do: `datasets = pcat.search(**search_dict).to_dataset_dict()`. independence_level : str - 'model': Weights using the method '1 model - 1 Vote', where every unique combination of 'source' and 'driving_model' is considered a model. + 'model': Weights using the method '1 model - 1 Vote', + where every unique combination of 'source' and 'driving_model' is considered a model. 'GCM': Weights using the method '1 GCM - 1 Vote' 'institution': Weights using the method '1 institution - 1 Vote' balance_experiments : bool - If True, each experiment will be given a total weight of 1 (prior to subsequent weighting made through `attribute_weights`). + If True, each experiment will be given a total weight of 1 + (prior to subsequent weighting made through `attribute_weights`). This option requires the 'cat:experiment' attribute to be present in all datasets. attribute_weights : dict, optional - Nested dictionaries of weights to apply to each dataset. These weights are applied after the independence weighting. + Nested dictionaries of weights to apply to each dataset. + These weights are applied after the independence weighting. The first level of keys are the attributes for which weights are being given. The second level of keys are unique entries for the attribute, with the value being either an individual weight - or a xr.DataArray. If a DataArray is used, its dimensions must be the same non-stationary coordinate as the datasets (ex: time, horizon) and the attribute being weighted (ex: experiment). - A `others` key can be used to give the same weight to all entries not specifically named in the dictionnary. + or a xr.DataArray. If a DataArray is used, its dimensions must be the same non-stationary coordinate + as the datasets (ex: time, horizon) and the attribute being weighted (ex: experiment). + A `others` key can be used to give the same weight to all entries not specifically named in the dictionary. Example #1: {'source': {'MPI-ESM-1-2-HAM': 0.25, 'MPI-ESM1-2-HR': 0.5}}, Example #2: {'experiment': {'ssp585': xr.DataArray, 'ssp126': xr.DataArray}, 'institution': {'CCCma': 0.5, 'others': 1}} skipna : bool - If True, weights will be computed from attributes only. If False, weights will be computed from the number of non-missing values. + If True, weights will be computed from attributes only. + If False, weights will be computed from the number of non-missing values. skipna=False requires either a 'time' or 'horizon' dimension in the datasets. v_for_skipna : str, optional Variable to use for skipna=False. If None, the first variable in the first dataset is used. @@ -241,7 +247,8 @@ def generate_weights( for k in other_dims: if len(other_dims[k]) > 0: warnings.warn( - f"Dataset {k} has dimensions that are not 'time' or 'horizon': {other_dims[k]}. The first indexes of these dimensions will be used to compute the weights." + f"Dataset {k} has dimensions that are not 'time' or 'horizon': {other_dims[k]}. " + "The first indexes of these dimensions will be used to compute the weights." ) datasets[k] = datasets[k].isel({d: 0 for d in other_dims[k]}) @@ -264,7 +271,8 @@ def generate_weights( > 1 ): raise NotImplementedError( - "Weighting `source` and/or `driving_model` through `attribute_weights` is not yet implemented when given a mix of GCMs and RCMs." + "Weighting `source` and/or `driving_model` through `attribute_weights` " + "is not yet implemented when given a mix of GCMs and RCMs." ) # More easily manage GCMs and RCMs @@ -341,7 +349,8 @@ def generate_weights( extra_dim[0].name ) - # Check that the extra dimension is the same for all datasets. If not, modify the datasets to make them the same. + # Check that the extra dimension is the same for all datasets. + # If not, modify the datasets to make them the same. if not all(extra_dimension.equals(extra_dim[d]) for d in range(len(extra_dim))): warnings.warn( f"Extra dimension {extra_dimension.name} is not the same for all datasets. Reindexing." diff --git a/xscen/extract.py b/xscen/extract.py index be322eaa..df94c6ae 100644 --- a/xscen/extract.py +++ b/xscen/extract.py @@ -31,9 +31,9 @@ __all__ = [ "extract_dataset", + "get_warming_level", "resample", "search_data_catalogs", - "get_warming_level", "subset_warming_level", ] @@ -79,7 +79,7 @@ def clisops_subset(ds: xr.Dataset, region: dict) -> xr.Dataset: @parse_config -def extract_dataset( +def extract_dataset( # noqa: C901 catalog: DataCatalog, *, variables_and_freqs: Optional[dict] = None, @@ -93,7 +93,9 @@ def extract_dataset( resample_methods: Optional[dict] = None, mask: Union[bool, xr.Dataset, xr.DataArray] = False, ) -> dict: - """Take one element of the output of `search_data_catalogs` and returns a dataset, performing conversions and resampling as needed. + """ + Take one element of the output of `search_data_catalogs` and returns a dataset, + performing conversions and resampling as needed. Nothing is written to disk within this function. @@ -350,7 +352,8 @@ def extract_dataset( ds_mask = out_dict["fx"]["mask"].copy() else: raise ValueError( - "No mask found. Either pass a xr.Dataset/xr.DataArray to the `mask` argument or pass a `dc` that includes a dataset with a variable named `mask`." + "No mask found. Either pass a xr.Dataset/xr.DataArray to the `mask` argument " + "or pass a `dc` that includes a dataset with a variable named `mask`." ) # iter over all xrfreq to apply the mask @@ -362,7 +365,7 @@ def extract_dataset( return out_dict -def resample( +def resample( # noqa: C901 da: xr.DataArray, target_frequency: str, *, @@ -389,9 +392,10 @@ def resample( using the mapping in CVs/resampling_methods.json. If the variable is not found there, "mean" is used by default. missing: {'mask', 'drop'} or dict, optional - If 'mask' or 'drop', target periods that would have been computed from fewer timesteps than expected are masked or dropped, using a threshold of 5% of missing data. - For example, the first season of a `target_frequency` of "QS-DEC" will be masked or dropped if data only starts in January. - If a dict, it points to a xclim check missing method which will mask periods according to their number of NaN values. + If 'mask' or 'drop', target periods that would have been computed from fewer timesteps than expected + are masked or dropped, using a threshold of 5% of missing data. + E.g. the first season of a `target_frequency` of "QS-DEC" will be masked or dropped if data starts in January. + If a dict, points to a xclim check missing method which will mask periods according to the number of NaN values. The dict must contain a "method" field corresponding to the xclim method name and may contain any other args to pass. Options are documented in :py:mod:`xclim.core.missing`. @@ -450,7 +454,7 @@ def resample( days_per_period = ( days_per_step.resample(time=target_frequency) .sum() # Total number of days per period - .sel(time=days_per_step.time, method="ffill") # Upsample to initial freq + .sel(time=days_per_step.time, method="ffill") # Up-sample to initial freq .assign_coords( time=days_per_step.time ) # Not sure why we need this, but time coord is from the resample even after sel @@ -490,7 +494,7 @@ def resample( ds = ds.resample(time=target_frequency).mean(dim="time", keep_attrs=True) # Based on Vector Magnitude and Direction equations - # For example: https://www.khanacademy.org/math/precalculus/x9e81a4f98389efdf:vectors/x9e81a4f98389efdf:component-form/a/vector-magnitude-and-direction-review + # For example: https://www.khanacademy.org/math/precalculus/x9e81a4f98389efdf:vectors/x9e81a4f98389efdf:component-form/a/vector-magnitude-and-direction-review # noqa: E501 uas_attrs = ds["uas"].attrs vas_attrs = ds["vas"].attrs @@ -583,7 +587,7 @@ def resample( @parse_config -def search_data_catalogs( +def search_data_catalogs( # noqa: C901 data_catalogs: Union[ str, os.PathLike, DataCatalog, list[Union[str, os.PathLike, DataCatalog]] ], @@ -607,17 +611,22 @@ def search_data_catalogs( Parameters ---------- data_catalogs : str, os.PathLike, DataCatalog, or a list of those - DataCatalog (or multiple, in a list) or paths to JSON/CSV data catalogs. They must use the same columns and aggregation options. + DataCatalog (or multiple, in a list) or paths to JSON/CSV data catalogs. + They must use the same columns and aggregation options. variables_and_freqs : dict - Variables and freqs to search for, following a 'variable: xr-freq-compatible-str' format. A list of strings can also be provided. + Variables and freqs to search for, following a 'variable: xr-freq-compatible-str' format. + A list of strings can also be provided. other_search_criteria : dict, optional Other criteria to search for in the catalogs' columns, following a 'column_name: list(subset)' format. - You can also pass 'require_all_on: list(columns_name)' in order to only return results that correspond to all other criteria across the listed columns. + You can also pass 'require_all_on: list(columns_name)' in order to only return results that correspond to + all other criteria across the listed columns. More details available at https://intake-esm.readthedocs.io/en/stable/how-to/enforce-search-query-criteria-via-require-all-on.html . exclusions : dict, optional - Same as other_search_criteria, but for eliminating results. Any result that matches any of the exclusions will be removed. + Same as other_search_criteria, but for eliminating results. + Any result that matches any of the exclusions will be removed. match_hist_and_fut: bool - If True, historical and future simulations will be combined into the same line, and search results lacking one of them will be rejected. + If True, historical and future simulations will be combined into the same line, + and search results lacking one of them will be rejected. periods : list of str or list of lists of str, optional Either [start, end] or list of [start, end] for the periods to be evaluated. coverage_kwargs : dict, optional @@ -846,7 +855,8 @@ def search_data_catalogs( same_frq | (lower_frq & allow_resampling) ] - # For each dataset (id - xrfreq - processing_level - domain - variable), make sure that file availability covers the requested time periods + # For each dataset (id * xrfreq * processing_level * domain * variable), + # make sure that file availability covers the requested time periods if periods is not None and len(varcat) > 0: valid_tp = [] for var, group in varcat.df.groupby( @@ -861,7 +871,8 @@ def search_data_catalogs( varcat.esmcat._df = pd.concat(valid_tp) # We now select the coarsest timedelta for each variable - # we need to re-iterate over variables in case we used the registry (and thus there are multiple variables in varcat) + # We need to re-iterate over variables in case we used the registry + # (and thus there are multiple variables in varcat) rows = [] for var, group in varcat.df.groupby("variable"): rows.append(group[group.timedelta == group.timedelta.max()]) @@ -915,7 +926,7 @@ def search_data_catalogs( @parse_config -def get_warming_level( +def get_warming_level( # noqa: C901 realization: Union[xr.Dataset, dict, str, list], wl: float, *, @@ -925,17 +936,21 @@ def get_warming_level( tas_csv: Optional[str] = None, return_horizon: bool = True, ) -> Union[dict, list[str], str]: - """Use the IPCC Atlas method to return the window of time over which the requested level of global warming is first reached. + """ + Use the IPCC Atlas method to return the window of time + over which the requested level of global warming is first reached. Parameters ---------- realization : xr.Dataset, dict, str or list of those Model to be evaluated. Needs the four fields mip_era, source, experiment and member, - as a dict or in a Dataset's attributes. Strings should follow this formatting: {mip_era}_{source}_{experiment}_{member}. + as a dict or in a Dataset's attributes. + Strings should follow this formatting: {mip_era}_{source}_{experiment}_{member}. Lists of dicts, strings or Datasets are also accepted, in which case the output will be a dict. Regex wildcards (.*) are accepted, but may lead to unexpected results. - Datasets should include the catalogue attributes (starting by "cat:") required to create such a string: 'cat:mip_era', 'cat:experiment', - 'cat:member', and either 'cat:source' for global models or 'cat:driving_model' for regional models. + Datasets should include the catalogue attributes (starting by "cat:") required to create such a string: + 'cat:mip_era', 'cat:experiment', 'cat:member', + and either 'cat:source' for global models or 'cat:driving_model' for regional models. e.g. 'CMIP5_CanESM2_rcp85_r1i1p1' wl : float Warming level. @@ -958,8 +973,9 @@ def get_warming_level( Returns ------- dict, list or str - If `realization` is a Dataset, a dict or a string, the output will follow the format indicated by `return_horizon`. - If `realization` is a list, the output will be a dictionary where the keys are the selected columns from the csv and the values follow the format indicated by `return_period`. + If `realization` is a Dataset, dict or string, the output will follow the format indicated by `return_horizon`. + If `realization` is a list, the output will be a dictionary where the keys are the selected columns from the csv + and the values follow the format indicated by `return_period`. """ tas_baseline_period = standardize_periods( tas_baseline_period or ["1850", "1900"], multiple=False @@ -1101,7 +1117,9 @@ def subset_warming_level( wl_dim: str = "+{wl}Cvs{period0}-{period1}", **kwargs, ): - """Subsets the input dataset with only the window of time over which the requested level of global warming is first reached, using the IPCC Atlas method. + """ + Subsets the input dataset with only the window of time over which the requested level of global warming + is first reached, using the IPCC Atlas method. Parameters ---------- @@ -1112,7 +1130,7 @@ def subset_warming_level( 'cat:source' for global models or 'cat:driving_institution' (optional) + 'cat:driving_model' for regional models. wl : float Warming level. - eg. 2 for a global warming level of +2 degree Celsius above the mean temperature of the `tas_baseline_period`. + e.g. 2 for a global warming level of +2 degree Celsius above the mean temperature of the `tas_baseline_period`. to_level : The processing level to assign to the output. Use "{wl}", "{period0}" and "{period1}" in the string to dynamically include @@ -1127,7 +1145,7 @@ def subset_warming_level( Instructions on how to search for warming levels. The keyword arguments are passed to `get_warming_level()` - Valid keyword aguments are: + Valid keyword arguments are: window : int tas_baseline_period : list ignore_member : bool @@ -1240,8 +1258,10 @@ def _dispatch_historical_to_future( if pd.isna(sdf.activity).any(): warnings.warn( f"np.NaN was found in the activity column of {group}. The rows with np.NaN activity will be skipped." - "If you want them to be included in the historical and future matching, please put a valid activity (https://xscen.readthedocs.io/en/latest/columns.html)." - "For example, xscen expects experiment `historical` to have `CMIP` activity and experiments `sspXYZ` to have `ScenarioMIP` activity. " + "If you want them to be included in the historical and future matching, " + "please put a valid activity (https://xscen.readthedocs.io/en/latest/columns.html)." + "For example, xscen expects experiment `historical` to have `CMIP` activity " + "and experiments `sspXYZ` to have `ScenarioMIP` activity. " ) for activity_id in set(sdf.activity) - {"HighResMip", np.NaN}: sub_sdf = sdf[sdf.activity == activity_id] diff --git a/xscen/indicators.py b/xscen/indicators.py index bab6d28d..befc9e93 100644 --- a/xscen/indicators.py +++ b/xscen/indicators.py @@ -63,7 +63,7 @@ def load_xclim_module( @parse_config -def compute_indicators( +def compute_indicators( # noqa: C901 ds: xr.Dataset, indicators: Union[ str, @@ -93,12 +93,14 @@ def compute_indicators( Can be the indicator module directly, or a sequence of indicators or a sequence of tuples (indicator name, indicator) as returned by `iter_indicators()`. periods : list of str or list of lists of str, optional - Either [start, end] or list of [start, end] of continuous periods over which to compute the indicators. This is needed when the time axis of ds contains some jumps in time. + Either [start, end] or list of [start, end] of continuous periods over which to compute the indicators. + This is needed when the time axis of ds contains some jumps in time. If None, the dataset will be considered continuous. - restrict_years: + restrict_years : bool If True, cut the time axis to be within the same years as the input. This is mostly useful for frequencies that do not start in January, such as QS-DEC. - In that instance, `xclim` would start on previous_year-12-01 (DJF), with a NaN. `restrict_years` will cut that first timestep. + In that instance, `xclim` would start on previous_year-12-01 (DJF), with a NaN. + `restrict_years` will cut that first timestep. This should have no effect on YS and MS indicators. to_level : str, optional The processing level to assign to the output. @@ -249,21 +251,21 @@ def registry_from_module( Parameters ---------- module : ModuleType - A module of xclim. + A module of xclim. registry : DerivedVariableRegistry, optional - If given, this registry is extended, instead of creating a new one. + If given, this registry is extended, instead of creating a new one. variable_column : str - The name of the variable column (the name used in the query). + The name of the variable column (the name used in the query). Returns ------- DerivedVariableRegistry - A variable registry where each indicator and each of its output has been registered. - If an indicator returns multiple values, each of them is mapped individually, as - the DerivedVariableRegistry only supports single output function. - Each indicator was wrapped into a new function that only accepts a dataset and - returns it with the extra variable appended. This means all other parameters are - given their defaults. + A variable registry where each indicator and each of its output has been registered. + If an indicator returns multiple values, each of them is mapped individually, as + the DerivedVariableRegistry only supports single output function. + Each indicator was wrapped into a new function that only accepts a dataset and + returns it with the extra variable appended. This means all other parameters are + given their defaults. """ dvr = registry or DerivedVariableRegistry() for name, ind in module.iter_indicators(): diff --git a/xscen/io.py b/xscen/io.py index 32bfa0f8..9d722106 100644 --- a/xscen/io.py +++ b/xscen/io.py @@ -37,8 +37,8 @@ "rechunk", "rechunk_for_saving", "round_bits", - "save_to_table", "save_to_netcdf", + "save_to_table", "save_to_zarr", "subset_maxsize", "to_table", @@ -69,7 +69,7 @@ def get_engine(file: Union[str, os.PathLike]) -> str: return engine -def estimate_chunks( +def estimate_chunks( # noqa: C901 ds: Union[str, os.PathLike, xr.Dataset], dims: list, target_mb: float = 50, @@ -369,10 +369,12 @@ def save_to_netcdf( dimension names. Rechunking is only done on *data* variables sharing dimensions with this argument. bitround : bool or int or dict - If not False, float variables are bit-rounded by dropping a certain number of bits from their mantissa, allowing for a much better compression. + If not False, float variables are bit-rounded by dropping a certain number of bits from their mantissa, + allowing for a much better compression. If an int, this is the number of bits to keep for all float variables. If a dict, a mapping from variable name to the number of bits to keep. - If True, the number of bits to keep is guessed based on the variable's name, defaulting to 12, which yields a relative error below 0.013%. + If True, the number of bits to keep is guessed based on the variable's name, defaulting to 12, + which yields a relative error below 0.013%. compute : bool Whether to start the computation or return a delayed object. netcdf_kwargs : dict, optional @@ -409,7 +411,7 @@ def save_to_netcdf( @parse_config -def save_to_zarr( +def save_to_zarr( # noqa: C901 ds: xr.Dataset, filename: Union[str, os.PathLike], *, @@ -448,10 +450,12 @@ def save_to_zarr( encoding : dict, optional If given, skipped variables are popped in place. bitround : bool or int or dict - If not False, float variables are bit-rounded by dropping a certain number of bits from their mantissa, allowing for a much better compression. + If not False, float variables are bit-rounded by dropping a certain number of bits from their mantissa, + allowing for a much better compression. If an int, this is the number of bits to keep for all float variables. If a dict, a mapping from variable name to the number of bits to keep. - If True, the number of bits to keep is guessed based on the variable's name, defaulting to 12, which yields a relative error of 0.012%. + If True, the number of bits to keep is guessed based on the variable's name, defaulting to 12, + which yields a relative error of 0.012%. itervar : bool If True, (data) variables are written one at a time, appending to the zarr. If False, this function computes, no matter what was passed to kwargs. @@ -652,7 +656,7 @@ def to_table( pd.DataFrame or dict DataFrame with a MultiIndex with levels `row` and MultiColumn with levels `column`. If `sheet` is given, the output is dictionary with keys for each unique "sheet" dimensions tuple, values are DataFrames. - The DataFrames are always sorted with level priority as given in `row` and in ascending order,. + The DataFrames are always sorted with level priority as given in `row` and in ascending order. """ if isinstance(ds, xr.Dataset): da = ds.to_array(name="data") diff --git a/xscen/regrid.py b/xscen/regrid.py index 6ed5dd05..b627c1e0 100644 --- a/xscen/regrid.py +++ b/xscen/regrid.py @@ -19,11 +19,11 @@ # TODO: Implement support for an OBS2SIM kind of interpolation -__all__ = ["regrid_dataset", "create_mask"] +__all__ = ["create_mask", "regrid_dataset"] @parse_config -def regrid_dataset( +def regrid_dataset( # noqa: C901 ds: xr.Dataset, ds_grid: xr.Dataset, weights_location: Union[str, os.PathLike], diff --git a/xscen/scripting.py b/xscen/scripting.py index 6e264ec2..4495d4ff 100644 --- a/xscen/scripting.py +++ b/xscen/scripting.py @@ -26,14 +26,14 @@ logger = logging.getLogger(__name__) __all__ = [ + "TimeoutException", + "measure_time", + "move_and_delete", + "save_and_update", "send_mail", "send_mail_on_exit", - "measure_time", - "timeout", - "TimeoutException", "skippable", - "save_and_update", - "move_and_delete", + "timeout", ] diff --git a/xscen/spatial.py b/xscen/spatial.py index 3618c546..e7fdbe28 100644 --- a/xscen/spatial.py +++ b/xscen/spatial.py @@ -17,8 +17,8 @@ from .config import parse_config __all__ = [ - "creep_weights", "creep_fill", + "creep_weights", "subset", ] @@ -124,7 +124,7 @@ def _dot(arr, wei): ) -def subset( +def subset( # noqa: C901 ds: xr.Dataset, region: Optional[dict] = None, *, diff --git a/xscen/utils.py b/xscen/utils.py index 00efad4f..87c4819e 100644 --- a/xscen/utils.py +++ b/xscen/utils.py @@ -43,8 +43,8 @@ "stack_drop_nans", "standardize_periods", "translate_time_chunk", - "unstack_fill_nan", "unstack_dates", + "unstack_fill_nan", "update_attr", ] @@ -151,7 +151,7 @@ def add_attr(ds: Union[xr.Dataset, xr.DataArray], attr: str, new: str, **fmt): ds.attrs[f"{attr}_{loc}"] = TRANSLATOR[loc](new).format(**fmt) -def date_parser( +def date_parser( # noqa: C901 date: Union[str, cftime.datetime, pd.Timestamp, datetime, pd.Period], *, end_of_period: Union[bool, str] = False, @@ -216,7 +216,7 @@ def _parse_date(date, fmts): except (KeyError, ValueError): try: date = pd.Timestamp(date) - except (pd._libs.tslibs.parsing.DateParseError, ValueError): + except (ValueError, pd._libs.tslibs.parsing.DateParseError): date = pd.NaT elif isinstance(date, cftime.datetime): for n in range(3): @@ -716,7 +716,7 @@ def change_units(ds: xr.Dataset, variables_and_units: dict) -> xr.Dataset: return ds -def clean_up( +def clean_up( # noqa: C901 ds: xr.Dataset, *, variables_and_units: Optional[dict] = None, @@ -1325,7 +1325,7 @@ def season_sort_key(idx: pd.Index, name: Optional[str] = None): if (name or getattr(idx, "name", None)) == "month": m = list(xr.coding.cftime_offsets._MONTH_ABBREVIATIONS.values()) return idx.map(m.index) - except (ValueError, TypeError): + except (TypeError, ValueError): # ValueError if string not in seasons, or value not in months # TypeError if season element was not a string. pass From 4e3193f217a32e51370463f5688a160fc1b0f04a Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:03:57 -0500 Subject: [PATCH 04/22] lint --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1cca4204..f37a9092 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,7 +37,7 @@ jobs: test-pypi: name: Test with Python${{ matrix.python-version }} (PyPI/tox) - needs: black + needs: lint runs-on: ubuntu-latest env: COVERALLS_PARALLEL: true From 9684c4fdae673fc41eb50c38ce7503b0f388bfdd Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:15:25 -0500 Subject: [PATCH 05/22] linting over tests --- pyproject.toml | 4 ++-- tests/test_diagnostics.py | 1 - tests/test_scripting.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a8a684ee..85827d91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,8 +184,7 @@ exclude = [ ".eggs", ".git", "build", - "docs", - "tests" + "docs" ] ignore = [ "D205", @@ -218,6 +217,7 @@ max-complexity = 15 [tool.ruff.per-file-ignores] "xscen/**/__init__.py" = ["F401", "F403"] +"tests/**/*.py" = ["D100", "D101", "D102", "D103"] [tool.ruff.pycodestyle] max-doc-length = 180 diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 401c44e6..d11cd029 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -1,7 +1,6 @@ import numpy as np import pytest import xarray as xr -import xclim as xc from xclim.testing.helpers import test_timeseries as timeseries import xscen as xs diff --git a/tests/test_scripting.py b/tests/test_scripting.py index 6d7e6827..489afbdc 100644 --- a/tests/test_scripting.py +++ b/tests/test_scripting.py @@ -51,7 +51,7 @@ def test_save_and_update(self): assert ( cat.df.path[1] == root - + "/simulation/raw/CMIP6/ScenarioMIP/global/CCCma/CanESM5/ssp585/r1i1p1f1/yr/tas/tas_yr_CMIP6_ScenarioMIP_global_CCCma_CanESM5_ssp585_r1i1p1f1_2000-2049.nc" + + "/simulation/raw/CMIP6/ScenarioMIP/global/CCCma/CanESM5/ssp585/r1i1p1f1/yr/tas/tas_yr_CMIP6_ScenarioMIP_global_CCCma_CanESM5_ssp585_r1i1p1f1_2000-2049.nc" # noqa: E501 ) assert cat.df.source[1] == "CanESM5" From cc4c2fa4623a21d2610ecf9c316ddceea74d3771 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:24:44 -0500 Subject: [PATCH 06/22] update manifest --- pyproject.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 85827d91..60e600c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -253,7 +253,12 @@ include = [ "environment.yml", "environment-dev.yml", "tests/*.py", - "xscen*", + "xscen/**/*.py", + "xscen/**/*.yml", + "xscen/CVs/*.json", + "xscen/data/*.csv", + "xscen/data/**/*.mo", + "xscen/data/**/*.po", "tox.ini" ] exclude = [ From 441e74b4861669797994de3b1a8f966e417be254 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:21:09 -0500 Subject: [PATCH 07/22] use MANIFEST.in to select sdist files, adjustments to package data, add __init__.py to xclim_modules --- MANIFEST.in | 35 +++++++++++ pyproject.toml | 105 +++++++++++++++++--------------- xscen/xclim_modules/__init__.py | 1 + 3 files changed, 93 insertions(+), 48 deletions(-) create mode 100644 MANIFEST.in create mode 100644 xscen/xclim_modules/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..88c7195b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,35 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include Makefile +include README.rst +include .zenodo.json + +recursive-include xscen *.py *.yml +recursive-include xscen CVs *.json +recursive-include xscen data fr *.yml *.csv +recursive-include xscen data fr LC_MESSAGES *.mo *.po +recursive-include tests *.py +recursive-include docs conf.py Makefile make.bat *.png *.rst *.yml +recursive-include docs notebooks *.ipynb +recursive-include docs notebooks samples *.csv *.json *.yml + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] +recursive-exclude .github *.md *.yml +recursive-exclude conda *.yaml +recursive-exclude templates *.csv *.json *.py *.rst *.yml + +exclude .coveralls.yml +exclude .cruft.json +exclude .editorconfig +exclude .flake8 +exclude .gitignore +exclude .pre-commit-config.yaml +exclude .readthedocs.yml +exclude .secrets.baseline +exclude .yamllint.yaml +exclude environment.yml +exclude environment-dev.yml +exclude tox.ini diff --git a/pyproject.toml b/pyproject.toml index 60e600c7..5e11c73f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,10 @@ [build-system] -requires = ["setuptools", "babel", "wheel"] +requires = [ + "setuptools>=60", + "setuptools-scm>=8.0", + "babel", + "wheel" +] build-backend = "setuptools.build_meta" [project] @@ -233,50 +238,54 @@ include-package-data = true version = {attr = "xscen.__version__"} [tool.setuptools.packages.find] -include = [ - ".zenodo.json", - "AUTHORS.rst", - "CHANGES.rst", - "CONTRIBUTING.rst", - "LICENSE", - "Makefile", - "README.rst", - "docs/*.rst", - "docs/Makefile", - "docs/_static/_images/*.png", - "docs/conf.py", - "docs/make.bat", - "docs/notebooks/.ipynb", - "docs/notebooks/samples/*.csv", - "docs/notebooks/samples/*.json", - "docs/notebooks/samples/*.yml", - "environment.yml", - "environment-dev.yml", - "tests/*.py", - "xscen/**/*.py", - "xscen/**/*.yml", - "xscen/CVs/*.json", - "xscen/data/*.csv", - "xscen/data/**/*.mo", - "xscen/data/**/*.po", - "tox.ini" -] -exclude = [ - "*.py[co]", - "__pycache__", - ".coveralls.yml", - ".editorconfig", - ".flake8", - ".gitignore", - ".pre-commit-config.yaml", - ".readthedocs.yml", - ".yamllint.yaml", - "Makefile", - "conda", - "docs/_*", - "docs/modules.rst", - "docs/xscen*.rst", - "docs/notesbnooks/samples/tutorial/**/*.nc", - "templates" -] -namespaces = true +where = ["."] +include = ["xscen"] + +# [tool.setuptools.packages.find] +# include = [ +# ".zenodo.json", +# "AUTHORS.rst", +# "CHANGES.rst", +# "CONTRIBUTING.rst", +# "LICENSE", +# "Makefile", +# "README.rst", +# "docs/*.rst", +# "docs/Makefile", +# "docs/_static/_images/*.png", +# "docs/conf.py", +# "docs/make.bat", +# "docs/notebooks/.ipynb", +# "docs/notebooks/samples/*.csv", +# "docs/notebooks/samples/*.json", +# "docs/notebooks/samples/*.yml", +# "environment.yml", +# "environment-dev.yml", +# "setup.cfg", +# "setup.py", +# "tests/*.py", +# "xscen/**/*.py", +# "xscen/**/*.yml", +# "xscen/CVs/*.json", +# "xscen/data/*.csv", +# "xscen/data/**/*.mo", +# "xscen/data/**/*.po", +# "tox.ini" +# ] +# exclude = [ +# "*.py[co]", +# "__pycache__", +# ".coveralls.yml", +# ".editorconfig", +# ".flake8", +# ".gitignore", +# ".pre-commit-config.yaml", +# ".readthedocs.yml", +# ".yamllint.yaml", +# "conda/xscen/*.yml", +# "docs/_*", +# "docs/modules.rst", +# "docs/xscen*.rst", +# "docs/notesbnooks/samples/tutorial/**/*.nc", +# "templates" +# ] diff --git a/xscen/xclim_modules/__init__.py b/xscen/xclim_modules/__init__.py new file mode 100644 index 00000000..d7852d47 --- /dev/null +++ b/xscen/xclim_modules/__init__.py @@ -0,0 +1 @@ +"""xclim extension module.""" From cd58f3cb00bbccd2428e66b9206160ed4893ee84 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:35:51 -0500 Subject: [PATCH 08/22] dependency synchronisation --- environment-dev.yml | 20 +++++++++++++------- pyproject.toml | 23 +++++++++++++---------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 5be64a4c..58c8a9c7 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -36,28 +36,34 @@ dependencies: - pyarrow >=1.0.0 # Dev - babel - - bumpversion - - coveralls + - black >=23.10.1 + - bump-my-version >=0.12.0 + - coverage>=6.2.2,<7.0.0 + - coveralls>=3.3.1 + - flake8 >=6.1.0 + - flake8-rst-docstrings>=0.3.0 - ipykernel - ipython + - isort >=5.12.0 - jupyter_client - nbsphinx - nbval - pandoc - pooch - - pre-commit - - pytest - - pytest-cov + - pre-commit >=3.3.2 + - pytest >=7.3.1 + - pytest-cov >=4.0.0 + - ruff >=0.1.0 - sphinx - sphinx-autoapi - sphinx-rtd-theme >=1.0 - sphinxcontrib-napoleon - sphinx-codeautolink - sphinx-copybutton - - xdoctest + - watchdog >=3.0.0 - pip # Testing - - tox + - tox >=4.5.1 # packaging - build - wheel diff --git a/pyproject.toml b/pyproject.toml index 5e11c73f..03d81c66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,20 +67,22 @@ dependencies = [ dev = [ # Dev tools and testing "pip>=23.1.2", + "babel", + "black>=23.11.0", + "blackdoc>=0.3.9", "bump-my-version>=0.12.0", - "watchdog>=3.0.0", - "flake8>=6.1.0", - "flake8-rst-docstrings>=0.3.0", - "tox>=4.5.1", "coverage>=6.2.2,<7.0.0", "coveralls>=3.3.1", - "click>=8.1.7", - "pytest>=7.3.1", - "pytest-cov>=4.0.0", - "black>=23.10.1", - "blackdoc>=0.3.9", + "flake8-rst-docstrings>=0.3.0", + "flake8>=6.1.0", "isort>=5.12.0", - "pre-commit>=3.3.2" + "pooch", + "pre-commit>=3.3.2", + "pytest-cov>=4.0.0", + "pytest>=7.3.1", + "ruff>=0.1.0", + "tox>=4.5.1", + "watchdog>=3.0.0" ] docs = [ # Documentation and examples @@ -88,6 +90,7 @@ docs = [ "ipython", "jupyter_client", "nbsphinx", + "nbval", "pandoc", "sphinx", "sphinx-autoapi", From 45268dee25dc0553381d0cb653463fe9682c0791 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:31:51 -0500 Subject: [PATCH 09/22] babel is now a hard dependency, setup_requires handled in pyproject.toml, --colored tox output --- pyproject.toml | 2 +- setup.py | 1 - tox.ini | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 03d81c66..7ce323ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ + "babel", "cartopy", "cftime", "cf_xarray>=0.7.6", @@ -67,7 +68,6 @@ dependencies = [ dev = [ # Dev tools and testing "pip>=23.1.2", - "babel", "black>=23.11.0", "blackdoc>=0.3.9", "bump-my-version>=0.12.0", diff --git a/setup.py b/setup.py index aa4c0ee3..127a38be 100644 --- a/setup.py +++ b/setup.py @@ -22,5 +22,4 @@ def run(self): setup( cmdclass={"install": InstallWithCompile}, message_extractors={"xscen": [("**.py", "python", None)]}, - setup_requires=["babel"], ) diff --git a/tox.ini b/tox.ini index e982c993..575c54bf 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ requires = pip >= 23.0 setuptools >= 65.0 opts = + --colored --verbose [testenv:lint] From 66a03258e93bf91e6696e0f52b37d124102187a0 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:04:53 -0500 Subject: [PATCH 10/22] MANIFEST.in fixes, publish_release_notes update, re-add xdoctest, saner tox builds --- .github/workflows/main.yml | 2 +- MANIFEST.in | 10 ++++----- environment-dev.yml | 1 + pyproject.toml | 4 ++-- tox.ini | 8 ++++--- xscen/utils.py | 46 +++++++++++++++++++++++--------------- 6 files changed, 42 insertions(+), 29 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f37a9092..77cf5976 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,7 +5,7 @@ on: branches: - main paths-ignore: - - HISTORY.rst + - CHANGES.rst - setup.cfg - setup.py - xscen/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in index 88c7195b..50d4a6fd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,13 +7,13 @@ include README.rst include .zenodo.json recursive-include xscen *.py *.yml -recursive-include xscen CVs *.json -recursive-include xscen data fr *.yml *.csv -recursive-include xscen data fr LC_MESSAGES *.mo *.po +recursive-include xscen/CVs *.json +recursive-include xscen/data/fr *.yml *.csv +recursive-include xscen/data/fr/LC_MESSAGES *.mo *.po recursive-include tests *.py recursive-include docs conf.py Makefile make.bat *.png *.rst *.yml -recursive-include docs notebooks *.ipynb -recursive-include docs notebooks samples *.csv *.json *.yml +recursive-include docs/notebooks *.ipynb +recursive-include docs/notebooks/samples *.csv *.json *.yml recursive-exclude * __pycache__ recursive-exclude * *.py[co] diff --git a/environment-dev.yml b/environment-dev.yml index 58c8a9c7..8607219c 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -61,6 +61,7 @@ dependencies: - sphinx-codeautolink - sphinx-copybutton - watchdog >=3.0.0 + - xdoctest - pip # Testing - tox >=4.5.1 diff --git a/pyproject.toml b/pyproject.toml index 7ce323ac..4899c484 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,8 @@ dev = [ "pytest>=7.3.1", "ruff>=0.1.0", "tox>=4.5.1", - "watchdog>=3.0.0" + "watchdog>=3.0.0", + "xdoctest" ] docs = [ # Documentation and examples @@ -91,7 +92,6 @@ docs = [ "jupyter_client", "nbsphinx", "nbval", - "pandoc", "sphinx", "sphinx-autoapi", "sphinx-codeautolink", diff --git a/tox.ini b/tox.ini index 575c54bf..d6c91fc8 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ min_version = 4.0 envlist = lint py{39,310,311}-esmpy - docs + docs-esmpy requires = babel pip >= 23.0 @@ -32,22 +32,24 @@ commands = allowlist_externals = make -[testenv:docs] +[testenv:docs{,-esmpy}] description = Run Build of xscen Documentation conda_deps = conda_env = environment-dev.yml extras = + docs commands = make docs allowlist_externals = make -[testenv:doctests] +[testenv:doctests{,-esmpy}] description = Run documentation linters and doctests with pytest under {basepython} conda_deps = conda_env = environment-dev.yml extras = dev + docs commands = pytest --no-cov --nbval docs/notebooks --durations=10 {posargs} pytest --xdoctest xscen --durations=10 {posargs} diff --git a/xscen/utils.py b/xscen/utils.py index 87c4819e..aa6c3e75 100644 --- a/xscen/utils.py +++ b/xscen/utils.py @@ -935,16 +935,21 @@ def _search(a, b): def publish_release_notes( - style: str = "md", file: Optional[Union[os.PathLike, StringIO, TextIO]] = None + style: str = "md", + file: Optional[Union[os.PathLike, StringIO, TextIO]] = None, + changes: Union[str, os.PathLike] = None, ) -> Optional[str]: """Format release history in Markdown or ReStructuredText. Parameters ---------- - style: {"rst", "md"} - Use ReStructuredText formatting or Markdown. Default: Markdown. - file: {os.PathLike, StringIO, TextIO}, optional - If provided, prints to the given file-like object. Otherwise, returns a string. + style : {"rst", "md"} + Use ReStructuredText (`rst`) or Markdown (`md`) formatting. Default: Markdown. + file : {os.PathLike, StringIO, TextIO, None} + If provided, prints to the given file-like object. Otherwise, returns a string. + changes : {str, os.PathLike}, optional + If provided, manually points to the file where the changelog can be found. + Assumes a relative path otherwise. Returns ------- @@ -955,13 +960,16 @@ def publish_release_notes( This function exists solely for development purposes. Adapted from xclim.testing.utils.publish_release_notes. """ - history_file = Path(__file__).parent.parent.joinpath("HISTORY.rst") + if isinstance(changes, (str, Path)): + changes_file = Path(changes).absolute() + else: + changes_file = Path(__file__).absolute().parents[2].joinpath("CHANGES.rst") - if not history_file.exists(): - raise FileNotFoundError("History file not found in xscen file tree.") + if not changes_file.exists(): + raise FileNotFoundError("Changes file not found in xscen file tree.") - with open(history_file) as hf: - history = hf.read() + with open(changes_file) as f: + changes = f.read() if style == "rst": hyperlink_replacements = { @@ -979,32 +987,34 @@ def publish_release_notes( raise NotImplementedError() for search, replacement in hyperlink_replacements.items(): - history = re.sub(search, replacement, history) + changes = re.sub(search, replacement, changes) if style == "md": - history = history.replace("=======\nHistory\n=======", "# History") + changes = changes.replace("=========\nChangelog\n=========", "# Changelog") titles = {r"\n(.*?)\n([\-]{1,})": "-", r"\n(.*?)\n([\^]{1,})": "^"} for title_expression, level in titles.items(): - found = re.findall(title_expression, history) + found = re.findall(title_expression, changes) for grouping in found: fixed_grouping = ( str(grouping[0]).replace("(", r"\(").replace(")", r"\)") ) search = rf"({fixed_grouping})\n([\{level}]{'{' + str(len(grouping[1])) + '}'})" replacement = f"{'##' if level=='-' else '###'} {grouping[0]}" - history = re.sub(search, replacement, history) + changes = re.sub(search, replacement, changes) link_expressions = r"[\`]{1}([\w\s]+)\s<(.+)>`\_" - found = re.findall(link_expressions, history) + found = re.findall(link_expressions, changes) for grouping in found: search = rf"`{grouping[0]} <.+>`\_" replacement = f"[{str(grouping[0]).strip()}]({grouping[1]})" - history = re.sub(search, replacement, history) + changes = re.sub(search, replacement, changes) if not file: - return history - print(history, file=file) + return + if isinstance(file, (Path, os.PathLike)): + file = Path(file).open("w") + print(changes, file=file) def unstack_dates( From 353534ac1f68f86b21b682777c335832debb29b2 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:21:58 -0500 Subject: [PATCH 11/22] update CHANGES.rst --- CHANGES.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e97a4751..723e8e94 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,7 @@ Contributors to this version: Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal Announcements ^^^^^^^^^^^^^ -* N/A +* `xscen` now adheres to PEPs 517/518/621 using the `setuptools` and `setuptools-scm` backend for building and packaging. (:pull:`292`). New features and enhancements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -24,6 +24,7 @@ New features and enhancements Breaking changes ^^^^^^^^^^^^^^^^ * ``experiment_weights`` argument in ``generate_weights`` was renamed to ``balance_experiments``. (:pull:`252`). +* ``bump2version`` version-bumping utility was replaced by ``bump-my-version``. (:pull:`292`). Bug fixes ^^^^^^^^^ @@ -55,6 +56,17 @@ Internal changes * Re-adds a dev recipe to the `setup.py`. * Multiple improvements to the docstrings and type annotations. (:pull:`282`). * `pip check` in conda builds in GitHub workflows have been temporarily set to always pass. (:pull:`288`). +* The `cookiecutter` template has been updated to the latest commit via `cruft`. (:pull:`292`): + * `setup.py` has been mostly hollowed-out, save for the `babel`-related translation function. + * `pyproject.toml` has been added, with most package configurations migrated into it. + * `HISTORY.rst` has been renamed to `CHANGES.rst`. + * `actions-version-updater.yml` has been added to automate the versioning of the package. + * `pre-commit` hooks have been updated to the latest versions; `check-toml` and `toml-sort` have been added to cleanup the `pyproject.toml` file, and `check-json-schema` has been added to ensure GitHub and ReadTheDocs workflow files are valid. + * `ruff` has been added to the linting tools to replace most `flake8` and `pydocstyle` verifications. + * `tox` builds are more pure Python environment/PyPI-friendly. + * `xscen` now uses `Trusted Publishing` for TestPyPI and PyPI uploads. +* Linting checks now examine the testing folder, function complexity, and alphabetical order of `__all__` lists. (:pull:`292`). +* ``publish_release_notes`` now uses better logic for finding and reformatting the `CHANGES.rst` file. (:pull:`292`). v0.7.1 (2023-08-23) ------------------- From 7294d7221a4ecffff9e9051c67dfd29680b04546 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:31:21 -0500 Subject: [PATCH 12/22] remove duplicate import --- xscen/catalog.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xscen/catalog.py b/xscen/catalog.py index 4eb077ad..60679895 100644 --- a/xscen/catalog.py +++ b/xscen/catalog.py @@ -17,7 +17,6 @@ import intake_esm import pandas as pd import tlz -import xarray import xarray as xr from intake_esm.cat import ESMCatalogModel @@ -736,7 +735,7 @@ def update( def update_from_ds( self, - ds: xarray.Dataset, + ds: xr.Dataset, path: Union[os.PathLike, str], info_dict: Optional[dict] = None, **info_kwargs, From c6829c86383a8fa8923baa7f76390138b67323e2 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:40:47 -0500 Subject: [PATCH 13/22] update exceptions --- .github/workflows/bump-version.yml | 3 +++ .github/workflows/main.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 1f062b12..01e26b11 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -13,12 +13,14 @@ on: - .gitlab-ci.yml - .gitmodules - .pre-commit-config.yaml + - .secrets.baseline - .yamllint.yaml - AUTHORS.rst - CHANGES.rst - CONTRIBUTING.rst - MANIFEST.in - Makefile + - conda/xscen/*.yaml - docs/*.py - docs/*.rst - docs/notebooks @@ -27,6 +29,7 @@ on: - pyproject.toml - setup.cfg - setup.py + - templates - tests/*.py - tox.ini - xscen/__init__.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 77cf5976..c2975451 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,6 +6,8 @@ on: - main paths-ignore: - CHANGES.rst + - README.rst + - pyproject.toml - setup.cfg - setup.py - xscen/__init__.py From 397fe0bf0640534313766a0732268755ad6d6d53 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:54:08 -0500 Subject: [PATCH 14/22] Apply suggestions from code review Co-authored-by: RondeauG <38501935+RondeauG@users.noreply.github.com> --- CONTRIBUTING.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index a249e021..642a0688 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -61,7 +61,7 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. #. Install your local copy into a development environment. Using ``mamba``, you can create a new development environment with:: $ mamba env create -f environment-dev.yml - $ conda activate xscen + $ mamba activate xscen-dev $ python -m pip install --editable ".[dev]" #. As xscen was installed in editable mode, we also need to compile the translation catalogs manually: @@ -171,13 +171,13 @@ Pull Request Guidelines Before you submit a pull request, check that it meets these guidelines: -#. The pull request should include tests. +#. The pull request should include tests, and those should aim to cover all new lines of code. You can use `--cov-report html --cov .` during the call to `pytest` to generate an HTML report and analyse the current coverage of your tests. #. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in ``README.rst``. #. The pull request should not break the templates. -#. The pull request should work for Python 3.8, 3.9, 3.10, and 3.11. Check that the tests pass for all supported Python versions. +#. The pull request should work for Python 3.9, 3.10, and 3.11. Check that the tests pass for all supported Python versions. Tips ---- From 58d6806182eab718e5003c177d16b06c6f5630b1 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:53:27 -0500 Subject: [PATCH 15/22] fix .gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 02d4b88c..e591c797 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # XSCEN-specific paths.yml -docs/modules.rst -docs/xscen*.rst +docs/apidoc/modules.rst +docs/apidoc/xscen*.rst docs/notebooks/_data # Byte-compiled / optimized / DLL files From 75642c52ece4967a044b13f3b1d1011736335494 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 29 Nov 2023 16:15:24 -0500 Subject: [PATCH 16/22] update cookiecutter --- .cruft.json | 2 +- .github/workflows/bump-version.yml | 2 +- CONTRIBUTING.rst | 109 ++++++++++++++++++++--------- pyproject.toml | 5 +- 4 files changed, 81 insertions(+), 37 deletions(-) diff --git a/.cruft.json b/.cruft.json index 307c687c..066b66ab 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "/home/tjs/git/cookiecutter-pypackage", - "commit": "e67ba8180e050ffd1c29e38cf2ad3929004fdd8f", + "commit": "6bdf69e3dddcfa96942023b4c7885f59cbe75039", "checkout": null, "context": { "cookiecutter": { diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 01e26b11..81f8f4c4 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -54,7 +54,7 @@ jobs: run: | python -m pip install bump-my-version echo "Bumping version" - python -m bump-my-version bump patch + python -m bump-my-version bump --tag patch echo "new_version=$(grep -E '__version__' xscen/__init__.py | cut -d ' ' -f3)" - name: Push Changes uses: ad-m/github-push-action@master diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 642a0688..21d68624 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -52,55 +52,69 @@ If you are proposing a feature: Get Started! ------------ +.. note:: + + If you are new to using GitHub and `git`, please read `this guide `_ first. + +.. warning:: + + Anaconda Python users: Due to the complexity of some packages, the default dependency solver can take a long time to resolve the environment. Consider running the following commands in order to speed up the process:: + + $ conda install -n base conda-libmamba-solver + $ conda config --set solver libmamba + + For more information, please see the following link: https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community + + Alternatively, you can use the `mamba `_ package manager, which is a drop-in replacement for ``conda``. If you are already using `mamba`, replace the following commands with ``mamba`` instead of ``conda``. + Ready to contribute? Here's how to set up ``xscen`` for local development. #. Clone the repo locally:: $ git clone git@github.com:Ouranosinc/xscen.git -#. Install your local copy into a development environment. Using ``mamba``, you can create a new development environment with:: +#. Install your local copy into a development environment. You can create a new Anaconda development environment with:: - $ mamba env create -f environment-dev.yml - $ mamba activate xscen-dev + $ conda env create -f environment-dev.yml + $ conda activate xscen-dev $ python -m pip install --editable ".[dev]" + This installs ``xscen`` in an "editable" state, meaning that changes to the code are immediately seen by the environment. + #. As xscen was installed in editable mode, we also need to compile the translation catalogs manually: $ make translate -#. To ensure a consistent style, please install the pre-commit hooks to your repo:: +#. To ensure a consistent coding style, install the ``pre-commit`` hooks to your local clone:: $ pre-commit install - Special style and formatting checks will be run when you commit your changes. You - can always run the hooks on their own with: + On commit, ``pre-commit`` will check that ``black``, ``blackdoc``, ``isort``, ``flake8``, and ``ruff`` checks are passing, perform automatic fixes if possible, and warn of violations that require intervention. If your commit fails the checks initially, simply fix the errors and re-commit. + + You can also run the hooks manually with: $ pre-commit run -a + If you want to skip the ``pre-commit`` hooks temporarily, you can pass the ``--no-verify`` flag to `$ git commit`. + #. Create a branch for local development:: $ git checkout -b name-of-your-bugfix-or-feature - Now you can make your changes locally. + Now you can make your changes locally. -#. When you're done making changes, check that your changes pass ``black``, ``blackdoc``, ``flake8``, ``isort``, ``ruff``, and the tests, including testing other Python versions with tox:: +#. When you're done making changes, we **strongly** suggest running the tests in your environment or with the help of ``tox``:: - $ black --check xscen tests - $ isort --check xscen tests - $ ruff xscen tests - $ flake8 xscen tests - $ blackdoc --check xscen docs $ python -m pytest ++ # Or, to run multiple build tests $ tox - To get ``black``, ``blackdoc``, ``flake8``, ``isort``, ``ruff``, and tox, just pip install them into your virtualenv. - - Alternatively, you can run the tests using `make`:: + Alternatively, you can run the tests using `make`:: $ make lint $ make test - Running `make lint` and `make test` demands that your runtime/dev environment have all necessary development dependencies installed. + Running `make lint` and `make test` demands that your runtime/dev environment have all necessary development dependencies installed. .. warning:: @@ -112,12 +126,19 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. $ git commit -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature -#. If you are editing the docs, compile and open them with:: + If ``pre-commit`` hooks fail, try re-committing your changes (or, if need be, you can skip them with `$ git commit --no-verify`). +#. Submit a `Pull Request `_ through the GitHub website. + +#. When pushing your changes to your branch on GitHub, the documentation will automatically be tested to reflect the changes in your Pull Request. This build process can take several minutes at times. If you are actively making changes that affect the documentation and wish to save time, you can compile and test your changes beforehand locally with:: + + # To generate the html and open it in your browser $ make docs - # or to simply generate the html - $ cd docs/ - $ make html + # To uniquely generate the html + $ make autodoc + $ make -C docs html + # To simply test that the docs pass build checks + $ tox -e docs .. note:: @@ -125,7 +146,13 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. In order to speed up documentation builds, setting a value for the environment variable "SKIP_NOTEBOOKS" (e.g. "$ export SKIP_NOTEBOOKS=1") will prevent the notebooks from being evaluated on all subsequent "$ tox -e docs" or "$ make docs" invocations. -#. Submit a pull request through the GitHub website. +#. Once your Pull Request has been accepted and merged to the ``main`` branch, several automated workflows will be triggered: + + - The ``bump-version.yml`` workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. **It is not necessary to manually bump the version in your branch when merging (non-release) pull requests.** + - `ReadTheDocs` will automatically build the documentation and publish it to the `latest` branch of `xscen` documentation website. + - If your branch is not a fork (ie: you are a maintainer), your branch will be automatically deleted. + + You will have contributed your first changes to ``xscen``! .. _translating-xscen: @@ -171,9 +198,9 @@ Pull Request Guidelines Before you submit a pull request, check that it meets these guidelines: -#. The pull request should include tests, and those should aim to cover all new lines of code. You can use `--cov-report html --cov .` during the call to `pytest` to generate an HTML report and analyse the current coverage of your tests. +#. The pull request should include tests and should aim to provide `code coverage `_ all new lines of code. You can use the ``--cov-report html --cov xscen`` flags during the call to ``pytest`` to generate an HTML report and analyse the current test coverage. -#. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in ``README.rst``. +#. If the pull request adds functionality, the docs should also be updated. Put your new functionality into a function with a docstring, and add the feature to the list in ``README.rst``. #. The pull request should not break the templates. @@ -186,33 +213,49 @@ To run a subset of tests:: $ pytest tests.test_xscen +To run specific code style checks:: + + $ black --check xscen tests + $ isort --check xscen tests + $ blackdoc --check xscen docs + $ ruff xscen tests + $ flake8 xscen tests + +To get ``black``, ``isort ``blackdoc``, ``ruff``, and ``flake8`` (with plugins ``flake8-alphabetize`` and ``flake8-rst-docstrings``) simply `$ pip install` them into your environment. + Versioning/Tagging ------------------ -A reminder for the maintainers on how to deploy. This section is only relevant for maintainers when they are producing a new point release for the package. +A reminder for the **maintainers** on how to deploy. This section is only relevant when producing a new point release for the package. + +.. warning:: + + It is important to be aware that any changes to files found within the ``xscen`` folder (with the exception of ``xscen/__init__.py``) will trigger the ``bump-version.yml`` workflow. Be careful not to commit changes to files in this folder when preparing a new release. #. Create a new branch from `main` (e.g. `release-0.2.0`). #. Update the `CHANGES.rst` file to change the `Unreleased` section to the current date. -#. Create a pull request from your branch to `main`. -#. Once the pull request is merged, create a new release on GitHub. On the main branch, run: +#. Bump the version in your branch to the next version (e.g. `v0.1.0 -> v0.2.0`):: .. code-block:: shell $ bump-my-version bump minor # In most cases, we will be releasing a minor version $ git push - $ git push --tags - This will trigger the CI to build the package and upload it to TestPyPI. In order to upload to PyPI, this can be done by publishing a new version on GitHub. This will then trigger the workflow to build and upload the package to PyPI. +#. Create a pull request from your branch to `main`. +#. Once the pull request is merged, create a new release on GitHub. On the main branch, run: + + .. code-block:: shell -#. Once the release is published, it will go into a `staging` mode on Github Actions. Once the tests pass, admins can approve the release (an e-mail will be sent) and it will be published on PyPI. + $ git tag v0.2.0 + $ git push --tags -.. note:: + This will trigger a GitHub workflow to build the package and upload it to TestPyPI. At the same time, the GitHub workflow will create a draft release on GitHub. Assuming that the workflow passes, the final release can then be published on GitHub by finalizing the draft release. - The ``bump-version.yml`` GitHub workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. It is not necessary to manually bump the version in your branch when merging (non-release) pull requests. +#. Once the release is published, the `publish-pypi.yml` workflow will go into an `awaiting approval` mode on Github Actions. Only authorized users may approve this workflow (notifications will be sent) to trigger the upload to PyPI. .. warning:: - It is important to be aware that any changes to files found within the ``xscen`` folder (with the exception of ``xscen/__init__.py``) will trigger the ``bump-version.yml`` workflow. Be careful not to commit changes to files in this folder when preparing a new release. + Uploads to PyPI can **never** be overwritten. If you make a mistake, you will need to bump the version and re-release the package. If the package uploaded to PyPI is broken, you should modify the GitHub release to mark the package as broken, as well as yank the package (mark the version "broken") on PyPI. Packaging --------- diff --git a/pyproject.toml b/pyproject.toml index 4899c484..167661da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ dev = [ "bump-my-version>=0.12.0", "coverage>=6.2.2,<7.0.0", "coveralls>=3.3.1", + "flake8-alphabetize>=0.0.21", "flake8-rst-docstrings>=0.3.0", "flake8>=6.1.0", "isort>=5.12.0", @@ -287,8 +288,8 @@ include = ["xscen"] # ".yamllint.yaml", # "conda/xscen/*.yml", # "docs/_*", -# "docs/modules.rst", -# "docs/xscen*.rst", +# "docs/apidoc/modules.rst", +# "docs/apidoc/xscen*.rst", # "docs/notesbnooks/samples/tutorial/**/*.nc", # "templates" # ] From aad73ebaa8d8215e50010da072937a1bb6fd4519 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 29 Nov 2023 16:25:50 -0500 Subject: [PATCH 17/22] update fix docs, pin xarray temporarily --- CONTRIBUTING.rst | 2 +- pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 21d68624..e34879ca 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -106,7 +106,7 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. #. When you're done making changes, we **strongly** suggest running the tests in your environment or with the help of ``tox``:: $ python -m pytest -+ # Or, to run multiple build tests + # Or, to run multiple build tests $ tox Alternatively, you can run the tests using `make`:: diff --git a/pyproject.toml b/pyproject.toml index 167661da..f0573a08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,8 @@ dependencies = [ "shapely>=2.0", "sparse", "toolz", - "xarray", + # FIXME: Unpin xarray before releasing! + "xarray<2023.11.0", "xclim>=0.46.0", "xesmf>=0.7", "zarr" From e9a7bec2b637a2f1362e0606774ee191d9d0ae05 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 30 Nov 2023 13:02:14 -0500 Subject: [PATCH 18/22] Apply suggestions from code review Co-authored-by: Pascal Bourgault --- CONTRIBUTING.rst | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e34879ca..d01289de 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -159,7 +159,7 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. Translating xscen ~~~~~~~~~~~~~~~~~ -If your additions to ``xscen` play with plain text attributes like "long_name" or "description", you should also provide +If your additions to ``xscen`` play with plain text attributes like "long_name" or "description", you should also provide French translations for those fields. To manage translations, xscen uses python's ``gettext`` with the help of ``babel``. To update an attribute while enabling translation, use :py:func:`utils.add_attr` instead of a normal set-item. For example: diff --git a/pyproject.toml b/pyproject.toml index f0573a08..a1f64383 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -189,7 +189,7 @@ markers = ["requires_netcdf: marks tests that require netcdf files to run"] [tool.ruff] src = [""] line-length = 150 -target-version = "py38" +target-version = "py39" exclude = [ ".eggs", ".git", From 13c0251cccf30f29c71ab7544a715b586be1d761 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 30 Nov 2023 14:13:11 -0500 Subject: [PATCH 19/22] Apply suggestions from code review Co-authored-by: Pascal Bourgault --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a1f64383..8538642f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,7 +113,6 @@ docs = [ [tool.black] target-version = [ - "py38", "py39", "py310", "py311" @@ -162,10 +161,10 @@ omit = ["docs/notebooks/*.ipynb", "tests/*.py"] append_only = true known_first_party = "xscen" profile = "black" -py_version = 38 +py_version = 39 [tool.mypy] -python_version = 3.8 +python_version = 3.9 show_error_codes = true warn_return_any = true warn_unused_configs = true From d017b84d677224fb329bd66b4268f5422364e90b Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 30 Nov 2023 14:58:25 -0500 Subject: [PATCH 20/22] fast-forward cookiecutter --- .cruft.json | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- CONTRIBUTING.rst | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.cruft.json b/.cruft.json index 066b66ab..46102582 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "/home/tjs/git/cookiecutter-pypackage", - "commit": "6bdf69e3dddcfa96942023b4c7885f59cbe75039", + "commit": "64eceda7d95aeb8937fa9961989d3d617a525c04", "checkout": null, "context": { "cookiecutter": { diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fb8880e4..93ae5e0f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,7 +6,7 @@ - [ ] (If applicable) Documentation has been added / updated (for bug fixes / features). - [ ] (If applicable) Tests have been added. - [ ] This PR does not seem to break the templates. -- [ ] HISTORY.rst has been updated (with summary of main changes). +- [ ] CHANGES.rst has been updated (with summary of main changes). - [ ] Link to issue (:issue:`number`) and pull request (:pull:`number`) has been added. ### What kind of change does this PR introduce? diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d01289de..00850289 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -89,7 +89,7 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. $ pre-commit install - On commit, ``pre-commit`` will check that ``black``, ``blackdoc``, ``isort``, ``flake8``, and ``ruff`` checks are passing, perform automatic fixes if possible, and warn of violations that require intervention. If your commit fails the checks initially, simply fix the errors and re-commit. + On commit, ``pre-commit`` will check that ``black``, ``blackdoc``, ``isort``, ``flake8``, and ``ruff`` checks are passing, perform automatic fixes if possible, and warn of violations that require intervention. If your commit fails the checks initially, simply fix the errors, re-add the files, and re-commit. You can also run the hooks manually with: @@ -134,7 +134,7 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. # To generate the html and open it in your browser $ make docs - # To uniquely generate the html + # To only generate the html $ make autodoc $ make -C docs html # To simply test that the docs pass build checks @@ -148,7 +148,7 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. #. Once your Pull Request has been accepted and merged to the ``main`` branch, several automated workflows will be triggered: - - The ``bump-version.yml`` workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. **It is not necessary to manually bump the version in your branch when merging (non-release) pull requests.** + - The ``bump-version.yml`` workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. **It is not recommended to manually bump the version in your branch when merging (non-release) pull requests (this will cause the version to be bumped twice).** - `ReadTheDocs` will automatically build the documentation and publish it to the `latest` branch of `xscen` documentation website. - If your branch is not a fork (ie: you are a maintainer), your branch will be automatically deleted. @@ -198,7 +198,7 @@ Pull Request Guidelines Before you submit a pull request, check that it meets these guidelines: -#. The pull request should include tests and should aim to provide `code coverage `_ all new lines of code. You can use the ``--cov-report html --cov xscen`` flags during the call to ``pytest`` to generate an HTML report and analyse the current test coverage. +#. The pull request should include tests and should aim to provide `code coverage `_ for all new lines of code. You can use the ``--cov-report html --cov xscen`` flags during the call to ``pytest`` to generate an HTML report and analyse the current test coverage. #. If the pull request adds functionality, the docs should also be updated. Put your new functionality into a function with a docstring, and add the feature to the list in ``README.rst``. @@ -221,7 +221,7 @@ To run specific code style checks:: $ ruff xscen tests $ flake8 xscen tests -To get ``black``, ``isort ``blackdoc``, ``ruff``, and ``flake8`` (with plugins ``flake8-alphabetize`` and ``flake8-rst-docstrings``) simply `$ pip install` them into your environment. +To get ``black``, ``isort ``blackdoc``, ``ruff``, and ``flake8`` (with plugins ``flake8-alphabetize`` and ``flake8-rst-docstrings``) simply install them with `pip` (or `conda`) into your environment. Versioning/Tagging ------------------ From 78dd693af3134052de561b41a0ee8c2b98c0b44a Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 30 Nov 2023 14:59:38 -0500 Subject: [PATCH 21/22] update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 723e8e94..b2fe9195 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,7 +24,6 @@ New features and enhancements Breaking changes ^^^^^^^^^^^^^^^^ * ``experiment_weights`` argument in ``generate_weights`` was renamed to ``balance_experiments``. (:pull:`252`). -* ``bump2version`` version-bumping utility was replaced by ``bump-my-version``. (:pull:`292`). Bug fixes ^^^^^^^^^ @@ -67,6 +66,7 @@ Internal changes * `xscen` now uses `Trusted Publishing` for TestPyPI and PyPI uploads. * Linting checks now examine the testing folder, function complexity, and alphabetical order of `__all__` lists. (:pull:`292`). * ``publish_release_notes`` now uses better logic for finding and reformatting the `CHANGES.rst` file. (:pull:`292`). +* ``bump2version`` version-bumping utility was replaced by ``bump-my-version``. (:pull:`292`). v0.7.1 (2023-08-23) ------------------- From 139c65e62752beabca33dc47a21020349a844729 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 30 Nov 2023 15:04:46 -0500 Subject: [PATCH 22/22] CONTRIBUTING.rst formatting --- CONTRIBUTING.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 00850289..e4751146 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -79,9 +79,9 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. $ conda activate xscen-dev $ python -m pip install --editable ".[dev]" - This installs ``xscen`` in an "editable" state, meaning that changes to the code are immediately seen by the environment. + This installs ``xscen`` in an "editable" state, meaning that changes to the code are immediately seen by the environment. -#. As xscen was installed in editable mode, we also need to compile the translation catalogs manually: +#. As xscen was installed in editable mode, we also need to compile the translation catalogs manually:: $ make translate @@ -89,19 +89,19 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. $ pre-commit install - On commit, ``pre-commit`` will check that ``black``, ``blackdoc``, ``isort``, ``flake8``, and ``ruff`` checks are passing, perform automatic fixes if possible, and warn of violations that require intervention. If your commit fails the checks initially, simply fix the errors, re-add the files, and re-commit. + On commit, ``pre-commit`` will check that ``black``, ``blackdoc``, ``isort``, ``flake8``, and ``ruff`` checks are passing, perform automatic fixes if possible, and warn of violations that require intervention. If your commit fails the checks initially, simply fix the errors, re-add the files, and re-commit. - You can also run the hooks manually with: + You can also run the hooks manually with:: $ pre-commit run -a - If you want to skip the ``pre-commit`` hooks temporarily, you can pass the ``--no-verify`` flag to `$ git commit`. + If you want to skip the ``pre-commit`` hooks temporarily, you can pass the ``--no-verify`` flag to `$ git commit`. #. Create a branch for local development:: $ git checkout -b name-of-your-bugfix-or-feature - Now you can make your changes locally. + Now you can make your changes locally. #. When you're done making changes, we **strongly** suggest running the tests in your environment or with the help of ``tox``:: @@ -109,12 +109,12 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. # Or, to run multiple build tests $ tox - Alternatively, you can run the tests using `make`:: + Alternatively, you can run the tests using `make`:: $ make lint $ make test - Running `make lint` and `make test` demands that your runtime/dev environment have all necessary development dependencies installed. + Running `make lint` and `make test` demands that your runtime/dev environment have all necessary development dependencies installed. .. warning:: @@ -126,7 +126,7 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. $ git commit -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature - If ``pre-commit`` hooks fail, try re-committing your changes (or, if need be, you can skip them with `$ git commit --no-verify`). + If ``pre-commit`` hooks fail, try re-committing your changes (or, if need be, you can skip them with `$ git commit --no-verify`). #. Submit a `Pull Request `_ through the GitHub website. @@ -148,11 +148,11 @@ Ready to contribute? Here's how to set up ``xscen`` for local development. #. Once your Pull Request has been accepted and merged to the ``main`` branch, several automated workflows will be triggered: - - The ``bump-version.yml`` workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. **It is not recommended to manually bump the version in your branch when merging (non-release) pull requests (this will cause the version to be bumped twice).** - - `ReadTheDocs` will automatically build the documentation and publish it to the `latest` branch of `xscen` documentation website. - - If your branch is not a fork (ie: you are a maintainer), your branch will be automatically deleted. + - The ``bump-version.yml`` workflow will automatically bump the patch version when pull requests are pushed to the ``main`` branch on GitHub. **It is not recommended to manually bump the version in your branch when merging (non-release) pull requests (this will cause the version to be bumped twice).** + - `ReadTheDocs` will automatically build the documentation and publish it to the `latest` branch of `xscen` documentation website. + - If your branch is not a fork (ie: you are a maintainer), your branch will be automatically deleted. - You will have contributed your first changes to ``xscen``! +You will have contributed your first changes to ``xscen``! .. _translating-xscen: