From 6a5b2e3cb525445ed39460e7009bddbdbbc675e1 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:00:52 -0500 Subject: [PATCH 1/9] add builds for windows and macOS --- .github/workflows/main.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f3dac00c..20667f6d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -113,26 +113,43 @@ jobs: contains(github.event.pull_request.labels.*.name, 'approved') || (github.event.review.state == 'approved') || (github.event_name == 'push') - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: include: + # Windows builds + - tox-env: py39-coverage + python-version: "3.9" + markers: -m 'not slow' + os: windows-latest + # macOS builds + - tox-env: py310-coverage + python-version: "3.10" + markers: -m 'not slow' + os: macos-latest + # Linux builds - tox-env: py39-coverage-sbck python-version: "3.9" markers: -m 'not slow' + os: ubuntu-latest - tox-env: py310-coverage # No markers -- includes slow tests python-version: "3.10" + os: ubuntu-latest - tox-env: py311-coverage-sbck python-version: "3.11" markers: -m 'not slow' + os: ubuntu-latest - tox-env: py312-coverage-numba python-version: "3.12" markers: -m 'not slow' + os: ubuntu-latest - tox-env: notebooks_doctests python-version: "3.10" + os: ubuntu-latest - tox-env: offline-prefetch python-version: "3.11" markers: -m 'not slow and not requires_internet' + os: ubuntu-latest steps: - name: Harden Runner uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 @@ -168,7 +185,7 @@ jobs: python -m tox -e ${{ matrix.tox-env }} -- ${{ matrix.markers }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_FLAG_NAME: run-{{ matrix.tox-env }} + COVERALLS_FLAG_NAME: run-{{ matrix.tox-env }}-{{ matrix.os }} COVERALLS_PARALLEL: true COVERALLS_SERVICE_NAME: github From 4e721db4e358f8b5928437406f53d05ca622388f Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:51:04 -0500 Subject: [PATCH 2/9] properly handle lockfile --- tests/conftest.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e8e3bf0e0..5d970f187 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -474,16 +474,19 @@ def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace): ): if helpers.PREFETCH_TESTING_DATA: print("`XCLIM_PREFETCH_TESTING_DATA` set. Prefetching testing data...") - if worker_id in "master": + if worker_id in ["master"]: helpers.populate_testing_data(branch=helpers.TESTDATA_BRANCH) else: _default_cache_dir.mkdir(exist_ok=True, parents=True) - test_data_being_written = FileLock(_default_cache_dir.joinpath(".lock")) - with test_data_being_written as fl: + lockfile = _default_cache_dir.joinpath(".lock") + test_data_being_written = FileLock(lockfile) + with test_data_being_written: # This flag prevents multiple calls from re-attempting to download testing data in the same pytest run helpers.populate_testing_data(branch=helpers.TESTDATA_BRANCH) _default_cache_dir.joinpath(".data_written").touch() - fl.acquire() + with test_data_being_written.acquire(): + if lockfile.exists(): + lockfile.unlink() shutil.copytree(_default_cache_dir, threadsafe_data_dir) helpers.generate_atmos(threadsafe_data_dir) xdoctest_namespace.update(helpers.add_example_file_paths(threadsafe_data_dir)) From 8495f74c7fb4d4da7475693fc86353ba79ce8476 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:51:31 -0500 Subject: [PATCH 3/9] add OS to build name --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20667f6d0..065098d82 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -108,7 +108,7 @@ jobs: test-pypi: needs: lint - name: test-${{ matrix.tox-env }} (Python${{ matrix.python-version }}) + name: ${{ matrix.tox-env }} (Python${{ matrix.python-version }}, ${{ matrix.os }}) if: | contains(github.event.pull_request.labels.*.name, 'approved') || (github.event.review.state == 'approved') || From 04d114cf03022ee2f72380f0509d4a1304da431f Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:17:13 -0500 Subject: [PATCH 4/9] fnctl is not supported on Windows, prefetch testing data --- .github/workflows/main.yml | 2 +- tests/conftest.py | 11 ++++++++++- tox.ini | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 065098d82..9cdbe9acc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -118,7 +118,7 @@ jobs: matrix: include: # Windows builds - - tox-env: py39-coverage + - tox-env: py39-prefetch-coverage python-version: "3.9" markers: -m 'not slow' os: windows-latest diff --git a/tests/conftest.py b/tests/conftest.py index 5d970f187..ee8d8b4e9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import os import re import shutil +import sys import time import warnings from datetime import datetime as dt @@ -474,7 +475,15 @@ def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace): ): if helpers.PREFETCH_TESTING_DATA: print("`XCLIM_PREFETCH_TESTING_DATA` set. Prefetching testing data...") - if worker_id in ["master"]: + if ( + sys.platform == "win32" + and not _default_cache_dir.joinpath(helpers.TESTDATA_BRANCH).exists() + ): + raise OSError( + "UNIX-style file-locking is not supported on Windows. " + "Consider running `$ xclim prefetch_testing_data` to download testing data." + ) + elif worker_id in ["master"]: helpers.populate_testing_data(branch=helpers.TESTDATA_BRANCH) else: _default_cache_dir.mkdir(exist_ok=True, parents=True) diff --git a/tox.ini b/tox.ini index afed81075..95e8b9980 100644 --- a/tox.ini +++ b/tox.ini @@ -113,6 +113,7 @@ commands_pre = xclim show_version_info python -m pip check commands = + prefetch: xclim prefetch_testing_data doctest: pytest --no-cov --rootdir=tests/ --xdoctest xclim pytest {posargs} commands_post = From 6a35c55441840c8d90687cb5d6e83929f812897b Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:34:07 -0500 Subject: [PATCH 5/9] simplify logic --- tests/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ee8d8b4e9..f2329b64b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -475,10 +475,7 @@ def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace): ): if helpers.PREFETCH_TESTING_DATA: print("`XCLIM_PREFETCH_TESTING_DATA` set. Prefetching testing data...") - if ( - sys.platform == "win32" - and not _default_cache_dir.joinpath(helpers.TESTDATA_BRANCH).exists() - ): + if sys.platform == "win32": raise OSError( "UNIX-style file-locking is not supported on Windows. " "Consider running `$ xclim prefetch_testing_data` to download testing data." From 7c9ba25716ecb7d566bcd97cd1f65e80899d006e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:42:15 -0500 Subject: [PATCH 6/9] adapt md5 test for Windows --- tests/test_testing_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py index c696ae3bf..17a6f2f9e 100644 --- a/tests/test_testing_utils.py +++ b/tests/test_testing_utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import platform +import sys from pathlib import Path import numpy as np @@ -100,7 +101,11 @@ def test_md5_sum(self): test_data = Path(__file__).parent / "data" callendar = test_data / "callendar_1938.txt" md5_sum = utilities.file_md5_checksum(callendar) - assert md5_sum == "9a5d9f94d76d4f9d9b7aaadbe8cbf541" # noqa + if sys.platform == "win32": + # Windows has a different line ending (CR-LF) than Unix (LF) + assert md5_sum == "38083271c2d4c85dea6bd6baf23d34de" # noqa + else: + assert md5_sum == "9a5d9f94d76d4f9d9b7aaadbe8cbf541" # noqa class TestReleaseSupportFuncs: From 3e4b8ecc27ffb11a0d7a077b446c0bd446588b66 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 14 Feb 2024 18:01:10 -0500 Subject: [PATCH 7/9] run xclim --help --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 95e8b9980..b126a8e6d 100644 --- a/tox.ini +++ b/tox.ini @@ -112,6 +112,7 @@ commands_pre = python -m pip list xclim show_version_info python -m pip check + xclim --help commands = prefetch: xclim prefetch_testing_data doctest: pytest --no-cov --rootdir=tests/ --xdoctest xclim From d27f9a995e0e875dc8fd29f3921cbe672d9d07b1 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:26:26 -0500 Subject: [PATCH 8/9] update CHANGES.rst --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index f86f97581..670bebdd4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -47,6 +47,7 @@ Bug fixes * Fix the daily case `freq='D'` of ``xclim.stats.preprocess_standardized_index`` (:issue:`1602` :pull:`1607`). * Several spelling mistakes have been corrected within the documentation and codebase. (:pull:`1576`). * Added missing ``xclim.ensembles.robustness_fractions`` and ``xclim.ensembles.robistness_categoris`` in api doc section. (:pull:`1630`). +* Fixed an issue that can occur when fetching the testing data and running tests on Windows systems. Adapted a few existing tests for Windows support. (:pull:`1648`). Internal changes ^^^^^^^^^^^^^^^^ @@ -64,6 +65,8 @@ Internal changes * Some small adjustments to code organization to address `pylint` errors. * `dev` formatting tools (`black`, `blackdoc`, `isort`) are now pinned to their `pre-commit` hook version equivalents in both ``pyproject.toml`` and ``tox.ini``. (:pull:`1626`). * `black`, `isort`, and `pyupgrade` code formatters no longer target Python3.8 coding style conventions. (:pull:`1565`). +* The GitHub Workflows now include builds to run tests against both Windows and MacOS. (:pull:`1648`). +* `prefetch` is now available as a `tox` environment modifier in order to download the testing data before launching `pytest` (e.g. `py3x-prefetch`). This is . (:pull:`1648`). v0.47.0 (2023-12-01) -------------------- From c4e4f25d7e391acd4fbaa14cdf7689d47ff5d80f Mon Sep 17 00:00:00 2001 From: "bumpversion[bot]" Date: Thu, 15 Feb 2024 18:56:42 +0000 Subject: [PATCH 9/9] =?UTF-8?q?Bump=20version:=200.47.5-dev.16=20=E2=86=92?= =?UTF-8?q?=200.47.5-dev.17?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- xclim/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 64ee9531c..a5bedb7f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,7 +124,7 @@ target-version = [ ] [tool.bumpversion] -current_version = "0.47.5-dev.16" +current_version = "0.47.5-dev.17" commit = true commit_args = "--no-verify" tag = false diff --git a/xclim/__init__.py b/xclim/__init__.py index 76ffe52bf..37e47c8c2 100644 --- a/xclim/__init__.py +++ b/xclim/__init__.py @@ -16,7 +16,7 @@ __author__ = """Travis Logan""" __email__ = "logan.travis@ouranos.ca" -__version__ = "0.47.5-dev.16" +__version__ = "0.47.5-dev.17" _module_data = _files("xclim.data")