From 69ed976677d1d861c73ab1111929a1fb2a415eff Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 8 Sep 2023 17:10:46 -0400 Subject: [PATCH 01/22] cached testing data should always attempt to be copied to pytest threadsafe cache location --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6bce17adb..4448ea289 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -451,7 +451,7 @@ def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace): helpers.populate_testing_data(branch=helpers.TESTDATA_BRANCH) _default_cache_dir.joinpath(".data_written").touch() fl.acquire() - shutil.copytree(_default_cache_dir, threadsafe_data_dir) + shutil.copytree(_default_cache_dir, threadsafe_data_dir) helpers.generate_atmos(threadsafe_data_dir) xdoctest_namespace.update(helpers.add_example_file_paths(threadsafe_data_dir)) From ac97bd5e3a2e0449f07114a43fc585b23ecad485 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 8 Sep 2023 17:14:26 -0400 Subject: [PATCH 02/22] add requires_internet pytest marker, apply to request-based tests --- pyproject.toml | 3 ++- tests/test_testing_utils.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ad85a86f..cd32d52be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,5 +205,6 @@ usefixtures = "xdoctest_namespace" doctest_optionflags = ["NORMALIZE_WHITESPACE", "IGNORE_EXCEPTION_DETAIL", "NUMBER", "ELLIPSIS"] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", - "requires_docs: mark tests that can only be run with documentation present (deselect with '-m \"not requires_docs\"')" + "requires_docs: mark tests that can only be run with documentation present (deselect with '-m \"not requires_docs\"')", + "requires_internet: mark tests that require internet access (deselect with '-m \"not requires_internet\"')" ] diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py index 767a79bd3..454b7b0a4 100644 --- a/tests/test_testing_utils.py +++ b/tests/test_testing_utils.py @@ -28,6 +28,7 @@ def test_timeseries_made_up_variable(self): class TestFileRequests: + @pytest.mark.requires_internet def test_get_failure(self, tmp_path): bad_repo_address = "https://github.com/beard/of/zeus/" with pytest.raises(HTTPError): @@ -39,6 +40,7 @@ def test_get_failure(self, tmp_path): tmp_path, ) + @pytest.mark.requires_internet def test_open_dataset_with_bad_file(self, tmp_path): cmip3_folder = tmp_path.joinpath("main", "cmip3") cmip3_folder.mkdir(parents=True) @@ -78,7 +80,7 @@ def test_open_testdata(self): ) assert ds.lon.size == 128 - # Not that this test is super slow, but there is no need in spamming github's API for no reason. + # Not that this test is super slow, but there is no real value in spamming GitHub's API for no reason. @pytest.mark.slow @pytest.mark.xfail(reason="Test is rate limited by GitHub.") def test_list_datasets(self): From 99d50e80eabf31432a5980a746d3410b76984995 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 8 Sep 2023 17:56:46 -0400 Subject: [PATCH 03/22] add pytest-socket to dev requirements, add tox recipe for offline testing via pytest-socket, handle SocketBlockedError when performing requests for verification of files already on disk when disabling sockets --- environment.yml | 1 + pyproject.toml | 1 + tox.ini | 2 ++ xclim/testing/utils.py | 8 ++++++++ 4 files changed, 12 insertions(+) diff --git a/environment.yml b/environment.yml index 34defe4a7..92392b10a 100644 --- a/environment.yml +++ b/environment.yml @@ -55,6 +55,7 @@ dependencies: - pylint - pytest - pytest-cov + - pytest-socket - pytest-xdist>=3.2 - sphinx - sphinx-autodoc-typehints diff --git a/pyproject.toml b/pyproject.toml index cd32d52be..486fe5f98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ dev = [ "pylint", "pytest", "pytest-cov", + "pytest-socket", "pytest-xdist[psutil] >=3.2", "tokenize-rt", "tox", diff --git a/tox.ini b/tox.ini index dd8c18c2b..f15bd0135 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ env_list = py38 py39-upstream-doctest py310-slow + py311-offline requires = pip >= 21.0 opts = -vv @@ -80,6 +81,7 @@ description = Run tests with pytest under {basepython} setenv = COV_CORE_SOURCE = PYTEST_ADDOPTS = --numprocesses=logical --durations=10 + offline: PYTEST_ADDOPTS = --numprocesses=logical --durations=10 --disable-socket --allow-unix-socket coverage: PYTEST_ADDOPTS = --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing PYTHONPATH = {toxinidir} passenv = diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index e53ff087e..ce7f113b6 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -27,6 +27,11 @@ from xarray import Dataset from xarray import open_dataset as _open_dataset +try: + from pytest_socket import SocketBlockedError +except ImportError: + SocketBlockedError = None + _xclim_deps = [ "xclim", "xarray", @@ -211,6 +216,9 @@ def _get( except (HTTPError, URLError): msg = f"{md5_name.as_posix()} not accessible online. Unable to determine validity with upstream repo." warnings.warn(msg) + except SocketBlockedError: + msg = f"Unable to access {md5_name.as_posix()} online. Testing suite is being run with `--disable-socket`." + warnings.warn(msg) if not local_file.is_file(): # This will always leave this directory on disk. From 65a93eada359a7cd493605a2462e9d68f478984e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 8 Sep 2023 18:12:38 -0400 Subject: [PATCH 04/22] Add an offline test build --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 782beaf08..cb73d927a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,10 +90,12 @@ jobs: python-version: "3.8" - tox-env: py39-coverage-sbck-eofs python-version: "3.9" - - tox-env: notebooks_doctests + - tox-env: py310-coverage-offline python-version: "3.10" - tox-env: py311-coverage-sbck python-version: "3.11" + - tox-env: notebooks_doctests + python-version: "3.10" steps: - uses: actions/checkout@v3.6.0 - name: Install Eigen3 From 0aed5eb006f532eab61657f8152e56909972e7bb Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:19:20 -0400 Subject: [PATCH 05/22] pytest markers must now be passed directly with tox, offline build on `tox` and GitHub Workflows is now its own separate build configuration, `tox` config now offers the `test` label for easier testing, `black` configuration in `tox` and GitHub Workflows is now called `lint` since it contains much more than `black` checks. --- .github/workflows/main.yml | 20 +++++++++++++------- tox.ini | 23 +++++++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb73d927a..a5e6a4b6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ env: XCLIM_TESTDATA_BRANCH: v2023.6.28 jobs: - black: + lint: name: Black (Python${{ matrix.python-version }}) runs-on: ubuntu-latest if: | @@ -49,11 +49,11 @@ jobs: - name: Run pylint run: pylint --rcfile=pylintrc --disable=import-error --exit-zero xclim - name: Run linting suite - run: tox -e black + run: tox -e lint test-py39: name: test-${{ matrix.tox-env }} (Python${{ matrix.python-version }}) - needs: black + needs: lint if: | (github.event_name == 'pull_request') && !contains(github.event.pull_request.labels.*.name, 'approved') runs-on: ubuntu-latest @@ -76,7 +76,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test-pypi: - needs: black + needs: lint name: test-${{ matrix.tox-env }} (Python${{ matrix.python-version }}) if: | contains(github.event.pull_request.labels.*.name, 'approved') || @@ -88,14 +88,20 @@ jobs: include: - tox-env: py38-coverage-eofs python-version: "3.8" + markers: '-- -m "not slow"' - tox-env: py39-coverage-sbck-eofs python-version: "3.9" - - tox-env: py310-coverage-offline + markers: '-- -m "not slow"' + - tox-env: py310-coverage # No markers -- includes slow tests python-version: "3.10" - tox-env: py311-coverage-sbck python-version: "3.11" + markers: '-- -m "not slow"' - tox-env: notebooks_doctests python-version: "3.10" + - tox-env: offline + python-version: "3.11" + markers: '-- -m "not slow and not requires_internet"' steps: - uses: actions/checkout@v3.6.0 - name: Install Eigen3 @@ -110,7 +116,7 @@ jobs: - name: Install tox run: pip install tox~=4.0 - name: Test with tox - run: tox -e ${{ matrix.tox-env }} + run: tox -e ${{ matrix.tox-env }} ${{ matrix.markers }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-{{ matrix.tox-env }} @@ -118,7 +124,7 @@ jobs: COVERALLS_SERVICE_NAME: github test-conda: - needs: black + needs: lint name: test-conda-${{ matrix.tox-env }} (Python${{ matrix.python-version }}) if: | contains(github.event.pull_request.labels.*.name, 'approved') || diff --git a/tox.ini b/tox.ini index f15bd0135..2ffdf50e2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,22 @@ [tox] min_version = 4.0 env_list = - black + lint docs notebooks_doctests + offline ; opt-slow py38 py39-upstream-doctest - py310-slow - py311-offline + py310 + py311 +labels = + test = py38, py39-upstream-doctest, py310, py311, notebooks_doctests, offline requires = pip >= 21.0 opts = -vv -[testenv:black] +[testenv:lint] description = Run code quality compliance tests under {basepython} skip_install = True extras = @@ -64,7 +67,7 @@ allowlist_externals = [testenv:notebooks_doctests{-coverage,}] description = Run notebooks and doctests with pytest under {basepython} commands = - pytest --no-cov --nbval --dist=loadscope --rootdir=tests/ docs/notebooks --ignore=docs/notebooks/example.ipynb + pytest --no-cov --nbval --dist=loadscope --rootdir=tests/ --ignore=docs/notebooks/example.ipynb docs/notebooks pytest --rootdir=tests/ --xdoctest xclim # Requires tox-conda compatible with tox@v4.0 @@ -76,12 +79,17 @@ commands = ; !slow: pytest xclim -m "not slow" --durations=10 ; slow: pytest xclim --durations=10 +[testenv:offline{-coverage,}] +description = Run tests with pytest under {basepython}, preventing socket connections (except for unix sockets for async support) +commands: + python -c 'print("Running offline tests with positional arguments \"not requires_internet\". This can be overridden with: tox -e offline -- -m \"some other marker statement\"")' + pytest --disable-socket --allow-unix-socket {posargs:-m "not requires_internet"} + [testenv] description = Run tests with pytest under {basepython} setenv = COV_CORE_SOURCE = PYTEST_ADDOPTS = --numprocesses=logical --durations=10 - offline: PYTEST_ADDOPTS = --numprocesses=logical --durations=10 --disable-socket --allow-unix-socket coverage: PYTEST_ADDOPTS = --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing PYTHONPATH = {toxinidir} passenv = @@ -107,8 +115,7 @@ commands_pre = python -m pip check commands = doctest: pytest --no-cov --rootdir=tests/ --xdoctest xclim - !slow: pytest -m "not slow" - slow: pytest + pytest {posargs} commands_post = coverage: - coveralls allowlist_externals = From 6564ffb90900532f7578d99b88c23bb1900be4d1 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:21:13 -0400 Subject: [PATCH 06/22] Contributing documentation now has a section about offline testing, updated pytest and tox usage to reflect new conventions --- CONTRIBUTING.rst | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 33f030a88..56985318f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -146,12 +146,32 @@ Ready to contribute? Here's how to set up `xclim` for local development. $ pydocstyle --config=setup.cfg xclim xclim $ yamllint --config-file .yamllint.yaml xclim -6. When unit/doc tests are added or notebooks updated, use ``$ pytest`` to run them. Alternatively, one can use ``$ tox`` to run all testing suites as would github do when the PR is submitted and new commits are pushed:: +6. When features or bug fixes have been contributed, unit tests and doctests have been added, or notebooks have been updated, use ``$ pytest`` to test them:: $ pytest --no-cov --nbval --dist=loadscope --rootdir=tests/ docs/notebooks --ignore=docs/notebooks/example.ipynb # for notebooks, exclusively. $ pytest --no-cov --rootdir=tests/ --xdoctest xclim # for doctests, exclusively. $ pytest # for all unit tests, excluding doctests and notebooks. - $ tox # run all testing suites + $ pytest -m "not slow" # for all unit tests, excluding doctests, notebooks, and "slow" marked tests. + + Alternatively, one can use ``$ tox`` to run very specific testing configurations, as GitHub Workflows would do when a Pull Request is submitted and new commits are pushed:: + + $ tox -e py38 # run tests on Python 3.8 + $ tox -e py39-upstream-doctest # run tests on Python 3.9, including doctests, with upstream dependencies + $ tox -e py310 -- -m "not slow # run tests on Python 3.10, excluding "slow" marked tests + $ tox -e py311 # run tests on Python 3.11 + $ tox -e notebooks_doctests # run tests using the base Python on doctests and evaluate all notebooks + $ tox -e offline # run tests using the base Python, excluding tests requiring internet access + + $ tox -m test # run all builds listed above + +.. warning:: + Starting from `xclim` v0.46.0, when running tests with `tox`, any `pytest` markers passed to `pyXX` builds (e.g. `-m "not slow"`) must be passed to `tox` directly. This can be done as follows:: + + $ tox -e py38 -- -m "not slow" + + The exceptions to this rule are: + `notebooks_doctests`: this configuration does not pass test markers to its `pytest` call. + `offline`: this configuration runs by default with the `-m "not requires_internet"` test marker. Be aware that running `tox` and manually setting a `pytest` marker will override this default. .. note:: `xclim` tests are organized to support the `pytest-xdist `_ plugin for distributed testing across workers or CPUs. In order to benefit from multiple processes, add the flag `--numprocesses=auto` or `-n auto` to your `pytest` calls. @@ -264,6 +284,19 @@ If you wish to test a specific branch using GitHub CI, this can be set in `.gith In order for a Pull Request to be allowed to merge to main development branch, this variable must match the latest tagged commit name on `Ouranosinc/xclim-testdata`. We suggest merging changed testing data first, tagging a new version of `xclim-testdata`, then re-running tests on your Pull Request at `Ouranosinc/xclim` with the newest tag. +Running Tests Offline +~~~~~~~~~~~~~~~~~~~~~ + +`xclim` testing is normally performed with the assumption that the machine running the tests has internet access. Many calls to `xclim` functions will attempt to download data or verify checksums from the `Ouranosinc/xclim-testdata` repository. This can be problematic for developers working on features when internet access is not immediately available. + +In order to bypass these remote validation checks, the testing can be run offline by either running pytest with the following options:: + + $ pytest --disable-socket --allow-unix-socket -m "not requires_internet" + # or, alternatively: + $ tox -e offline + +These options will disable all network calls and skip tests marked with the `requires_internet` marker. The `--allow-unix-socket` option is required to allow the `pytest-xdist` plugin to function properly. + Tips ---- From 861214cb4dae9478f5b0dcfc78f7fceb1e43962e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:34:44 -0400 Subject: [PATCH 07/22] update changes to reflect new features and builds for offline testing --- CHANGES.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index c059d400c..2d84200c5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,27 @@ Changelog ========= +v0.46.0 (unreleased) +-------------------- +Contributors to this version: Trevor James Smith (:user:`Zeitsperre`). + +New features and enhancements +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* The testing suite now offers a means of running tests without internet access. This requires a local copy of `xclim-testdata` to be present in the user's home directory and for certain `pytest` options and markers to be set. For more information, see the contributing documentation section `Running Tests Offline`. (:issue:`1468`, :pull:`1473`). + +Bug fixes +^^^^^^^^^ +* Fixed an error in the `pytest` configuration that prevented copying of testing data to thread-safe caches of workers under certain conditions (this should always occur). (:pull:`1473`). + +Breaking changes +^^^^^^^^^^^^^^^^ +* For better transparency and control in development, the `tox` configuration has been adapted to allow passing of markers directly to the `pytest` call. Positional arguments must be passed to tox after the `--` separator to select/deselect tests (e.g. ``'tox -e py38 -- -m "not slow"'``). (:pull:`1473`). +* `pytest-socket` is now a required development dependency for running the `"offline"` configurations of the `tox` testing suite. This has been added to the `dev` installation recipe. (:issue:`1468`, :pull:`1473`). + +Internal changes +^^^^^^^^^^^^^^^^ +* Added handling for `pytest-socket`'s ``SocketBlockedError`` in ``xclim.testing.open_dataset`` when attempting to fetch md5 validation files for cached testing data while explicitly disabling internet sockets. (:issue:`1468`, :pull:`1473`). + v0.45.0 (2023-09-05) -------------------- Contributors to this version: David Huard (:user:`huard`), Trevor James Smith (:user:`Zeitsperre`), Pascal Bourgault (:user:`aulemahal`), Juliette Lavoie (:user:`juliettelavoie`), Gabriel Rondeau-Genesse (:user:`RondeauG`), Marco Braun (:user:`vindelico`), Éric Dupuis (:user:`coxipi`). From 29dddb96d270041ddb9bbda3c5293835c9b69f07 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:56:16 -0400 Subject: [PATCH 08/22] try prefetching testdata for offline build --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5e6a4b6e..b33159afd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -115,6 +115,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install tox run: pip install tox~=4.0 + - name: Prefetch testing data + if: contains(matrix.tox-env, 'offline') + run: | + python -c "import xclim.testing.helpers; xclim.testing.helpers.populate_testing_data()" - name: Test with tox run: tox -e ${{ matrix.tox-env }} ${{ matrix.markers }} env: From b0fd6d15673eb3cd4c7ac73ae0a968072097a1a3 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:11:27 -0400 Subject: [PATCH 09/22] need to install xclim unfortunately --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b33159afd..6a9b92cfc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -118,6 +118,7 @@ jobs: - name: Prefetch testing data if: contains(matrix.tox-env, 'offline') run: | + pip install . python -c "import xclim.testing.helpers; xclim.testing.helpers.populate_testing_data()" - name: Test with tox run: tox -e ${{ matrix.tox-env }} ${{ matrix.markers }} From d4395c2a139e1f9185172931f10c24d4673d8ba9 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 11:29:42 -0400 Subject: [PATCH 10/22] add missing testing data to the prefetch file list, minor testing data naming cleanup --- tests/test_cffwis.py | 20 ++++++++++---------- tests/test_missing.py | 32 ++++---------------------------- tests/test_testing_utils.py | 2 +- xclim/testing/helpers.py | 10 ++++++++++ 4 files changed, 25 insertions(+), 39 deletions(-) diff --git a/tests/test_cffwis.py b/tests/test_cffwis.py index d0c262e5a..f9f3a3cb4 100644 --- a/tests/test_cffwis.py +++ b/tests/test_cffwis.py @@ -25,8 +25,6 @@ ) from xclim.indices.run_length import run_bounds -fwi_url = "FWI/cffdrs_test_fwi.nc" - class TestCFFWIS: # The following were computed with cffdrs 1.8.18, on the test_wDC data. @@ -44,6 +42,8 @@ class TestCFFWIS: ], } + fwi_test_dataset = "FWI/cffdrs_test_fwi.nc" + @classmethod def _get_cffdrs_fire_season(cls, key=None): def to_xr(arr): @@ -56,7 +56,7 @@ def to_xr(arr): return {key: to_xr(arr) for key, arr in cls.cffdrs_fire_season.items()} def test_fine_fuel_moisture_code(self, open_dataset): - fwi_data = open_dataset(fwi_url) + fwi_data = open_dataset(self.fwi_test_dataset) ffmc = np.full(fwi_data.time.size + 1, np.nan) ffmc[0] = 85 for i, t in enumerate(fwi_data.time): @@ -71,7 +71,7 @@ def test_fine_fuel_moisture_code(self, open_dataset): np.testing.assert_allclose(ffmc[1:], fwi_data.ffmc.isel(test=0), rtol=1e-6) def test_duff_moisture_code(self, open_dataset): - fwi_data = open_dataset(fwi_url) + fwi_data = open_dataset(self.fwi_test_dataset) dmc = np.full(fwi_data.time.size + 1, np.nan) dmc[0] = 6 for i, t in enumerate(fwi_data.time): @@ -87,7 +87,7 @@ def test_duff_moisture_code(self, open_dataset): np.testing.assert_allclose(dmc[1:], fwi_data.dmc.isel(test=0), rtol=1e-6) def test_drought_code(self, open_dataset): - fwi_data = open_dataset(fwi_url) + fwi_data = open_dataset(self.fwi_test_dataset) dc = np.full(fwi_data.time.size + 1, np.nan) dc[0] = 15 for i, t in enumerate(fwi_data.time): @@ -102,7 +102,7 @@ def test_drought_code(self, open_dataset): np.testing.assert_allclose(dc[1:], fwi_data.dc.isel(test=0), rtol=1e-6) def test_initial_spread_index(self, open_dataset): - fwi_data = open_dataset(fwi_url) + fwi_data = open_dataset(self.fwi_test_dataset) isi = np.full(fwi_data.time.size, np.nan) for i, t in enumerate(fwi_data.time): isi[i] = initial_spread_index( @@ -112,7 +112,7 @@ def test_initial_spread_index(self, open_dataset): np.testing.assert_allclose(isi, fwi_data.isi.isel(test=0), rtol=1e-6) def test_build_up_index(self, open_dataset): - fwi_data = open_dataset(fwi_url) + fwi_data = open_dataset(self.fwi_test_dataset) bui = np.full(fwi_data.time.size, np.nan) for i, t in enumerate(fwi_data.time): bui[i] = build_up_index( @@ -152,7 +152,7 @@ def test_overwintering_drought_code_indice(self, inputs, exp): np.testing.assert_allclose(out, exp, rtol=1e-6) def test_fire_weather_index(self, open_dataset): - fwi_data = open_dataset(fwi_url) + fwi_data = open_dataset(self.fwi_test_dataset) fwi = np.full(fwi_data.time.size, np.nan) for i, t in enumerate(fwi_data.time): fwi[i] = fire_weather_index( @@ -164,11 +164,11 @@ def test_fire_weather_index(self, open_dataset): def test_day_length(self): assert _day_length(44, 1) == 6.5 - def test_day_lengh_factor(self): + def test_day_length_factor(self): assert _day_length_factor(44, 1) == -1.6 def test_cffwis_indicator(self, open_dataset): - fwi_data = open_dataset(fwi_url) + fwi_data = open_dataset(self.fwi_test_dataset) fwi_data.lat.attrs["units"] = "degrees_north" dc, dmc, ffmc, isi, bui, fwi = atmos.cffwis_indices( tas=fwi_data.tas, diff --git a/tests/test_missing.py b/tests/test_missing.py index 86fccf6cf..0026af5f0 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -1,7 +1,5 @@ from __future__ import annotations -from pathlib import Path - import numpy as np import pandas as pd import pytest @@ -143,8 +141,7 @@ def test_no_freq(self, tasmin_series): np.testing.assert_array_equal(miss, True) def test_hydro(self, open_dataset): - fn = Path("Raven", "q_sim.nc") - ds = open_dataset(fn) + ds = open_dataset("Raven/q_sim.nc") miss = missing.missing_any(ds.q_sim, freq="YS") np.testing.assert_array_equal(miss[:-1], False) np.testing.assert_array_equal(miss[-1], True) @@ -261,16 +258,7 @@ def test_any(self, pr_hr_series): out = missing.missing_any(pr, "D", src_timestep="H") np.testing.assert_array_equal( out, - [ - True, - ] - + 8 - * [ - False, - ] - + [ - True, - ], + [True] + 8 * [False] + [True], ) def test_pct(self, pr_hr_series): @@ -278,13 +266,7 @@ def test_pct(self, pr_hr_series): out = missing.missing_pct(pr, "D", src_timestep="H", tolerance=0.1) np.testing.assert_array_equal( out, - 9 - * [ - False, - ] - + [ - True, - ], + 9 * [False] + [True], ) def test_at_least_n_valid(self, pr_hr_series): @@ -292,11 +274,5 @@ def test_at_least_n_valid(self, pr_hr_series): out = missing.at_least_n_valid(pr, "D", src_timestep="H", n=20) np.testing.assert_array_equal( out, - 9 - * [ - False, - ] - + [ - True, - ], + 9 * [False] + [True], ) diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py index 454b7b0a4..8eaa1eff6 100644 --- a/tests/test_testing_utils.py +++ b/tests/test_testing_utils.py @@ -76,7 +76,7 @@ def test_open_dataset_with_bad_file(self, tmp_path): def test_open_testdata(self): ds = utilities.open_dataset( - Path("cmip5", "tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712").as_posix() + Path("cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712") ) assert ds.lon.size == 128 diff --git a/xclim/testing/helpers.py b/xclim/testing/helpers.py index 49d08beae..a332906b1 100644 --- a/xclim/testing/helpers.py +++ b/xclim/testing/helpers.py @@ -113,6 +113,7 @@ def populate_testing_data( return data_entries = [ + "CanESM2_365day/pr_day_CanESM2_rcp85_r1i1p1_na10kgrid_qm-moving-50bins-detrend_2095.nc", "ERA5/daily_surface_cancities_1990-1993.nc", "EnsembleReduce/TestEnsReduceCriteria.nc", "EnsembleStats/BCCAQv2+ANUSPLIN300_ACCESS1-0_historical+rcp45_r1i1p1_1950-2100_tg_mean_YS.nc", @@ -120,11 +121,20 @@ def populate_testing_data( "EnsembleStats/BCCAQv2+ANUSPLIN300_CCSM4_historical+rcp45_r1i1p1_1950-2100_tg_mean_YS.nc", "EnsembleStats/BCCAQv2+ANUSPLIN300_CCSM4_historical+rcp45_r2i1p1_1950-2100_tg_mean_YS.nc", "EnsembleStats/BCCAQv2+ANUSPLIN300_CNRM-CM5_historical+rcp45_r1i1p1_1970-2050_tg_mean_YS.nc", + "FWI/GFWED_sample_2017.nc", + "FWI/cffdrs_test_fwi.nc", + "FWI/cffdrs_test_wDC.nc", + "HadGEM2-CC_360day/pr_day_HadGEM2-CC_rcp85_r1i1p1_na10kgrid_qm-moving-50bins-detrend_2095.nc", "NRCANdaily/nrcan_canada_daily_pr_1990.nc", "NRCANdaily/nrcan_canada_daily_tasmax_1990.nc", "NRCANdaily/nrcan_canada_daily_tasmin_1990.nc", + "SpatialAnalogs/dissimilarity", + "SpatialAnalogs/indicators", "cmip3/tas.sresb1.giss_model_e_r.run1.atm.da.nc", + "cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712", "sdba/CanESM2_1950-2100.nc", + "sdba/nrcan_1950-2013.nc", + "uncertainty_partitioning/cmip5_pr_global_mon.nc", ] data = dict() From fa73f46920c6ac3b5e0cb55cfa34b0501ea60e0e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 11:58:46 -0400 Subject: [PATCH 11/22] fix prefetch filename typos, much more accurate error descriptions for testdata fetching utilities --- xclim/testing/helpers.py | 6 +++--- xclim/testing/utils.py | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/xclim/testing/helpers.py b/xclim/testing/helpers.py index a332906b1..181ae8794 100644 --- a/xclim/testing/helpers.py +++ b/xclim/testing/helpers.py @@ -128,10 +128,10 @@ def populate_testing_data( "NRCANdaily/nrcan_canada_daily_pr_1990.nc", "NRCANdaily/nrcan_canada_daily_tasmax_1990.nc", "NRCANdaily/nrcan_canada_daily_tasmin_1990.nc", - "SpatialAnalogs/dissimilarity", - "SpatialAnalogs/indicators", + "SpatialAnalogs/dissimilarity.nc", + "SpatialAnalogs/indicators.nc", "cmip3/tas.sresb1.giss_model_e_r.run1.atm.da.nc", - "cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712", + "cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712.nc", "sdba/CanESM2_1950-2100.nc", "sdba/nrcan_1950-2013.nc", "uncertainty_partitioning/cmip5_pr_global_mon.nc", diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index ce7f113b6..b65f76f05 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -213,8 +213,17 @@ def _get( "Attempting new download." ) warnings.warn(msg) - except (HTTPError, URLError): - msg = f"{md5_name.as_posix()} not accessible online. Unable to determine validity with upstream repo." + except HTTPError: + msg = ( + f"{md5_name.as_posix()} not accessible in remote repository. " + "Unable to determine validity with upstream repo." + ) + warnings.warn(msg) + except URLError: + msg = ( + f"{md5_name.as_posix()} not found in remote repository. " + "Unable to determine validity with upstream repo." + ) warnings.warn(msg) except SocketBlockedError: msg = f"Unable to access {md5_name.as_posix()} online. Testing suite is being run with `--disable-socket`." @@ -227,13 +236,34 @@ def _get( url = "/".join((github_url, "raw", branch, fullname.as_posix())) logger.info(f"Fetching remote file: {fullname.as_posix()}") - urlretrieve(url, local_file) # nosec + try: + urlretrieve(url, local_file) # nosec + except HTTPError as e: + msg = f"{fullname.as_posix()} not accessible in remote repository. Aborting file retrieval." + raise FileNotFoundError(msg) from e + except URLError as e: + msg = ( + f"{fullname.as_posix()} not found in remote repository. " + "Verify filename and repository address. Aborting file retrieval." + ) + raise FileNotFoundError(msg) from e + except SocketBlockedError as e: + msg = ( + f"Unable to access {md5_name.as_posix()} online. Testing suite is being run with `--disable-socket`." + f"If you intend to run tests with this option enabled, please download the file beforehand with the " + f"following command:\n`xclim.testing.helpers.populate_testing_data()`." + ) + raise FileNotFoundError(msg) from e try: url = "/".join((github_url, "raw", branch, md5_name.as_posix())) logger.info(f"Fetching remote file md5: {md5_name.as_posix()}") urlretrieve(url, md5_file) # nosec - except HTTPError as e: - msg = f"{md5_name.as_posix()} not found. Aborting file retrieval." + except (HTTPError, URLError) as e: + msg = ( + f"{md5_name.as_posix()} not accessible online. " + "Unable to determine validity of file from upstream repo. " + "Aborting file retrieval." + ) local_file.unlink() raise FileNotFoundError(msg) from e From eeaa1787835190f9935a5710247d3a1bba1f2085 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:41:02 -0400 Subject: [PATCH 12/22] setting `XCLIM_PREFETCH_TESTING_DATA` as well as `ref_hist_sim_tuto` will now bypass `--disable-socket` for testing purposes, added more missing prefetch files, update `test_get_failure` expectations, `gather_session_data` now has a much clearer explanation docstring --- tests/conftest.py | 17 ++++++++++++++--- tests/test_sdba/conftest.py | 2 +- tests/test_testing_utils.py | 2 +- xclim/testing/helpers.py | 2 ++ xclim/testing/utils.py | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4448ea289..61ee11b55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -430,17 +430,28 @@ def ensemble_dataset_objects() -> dict: @pytest.fixture(scope="session", autouse=True) -def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace): +def gather_session_data( + threadsafe_data_dir, worker_id, xdoctest_namespace, socket_enabled +): """Gather testing data on pytest run. When running pytest with multiple workers, one worker will copy data remotely to _default_cache_dir while - other workers wait using lockfile. Once the lock is released, all workers will copy data to their local - threadsafe_data_dir.""" + other workers wait using lockfile. Once the lock is released, all workers will then copy data to their local + threadsafe_data_dir.As this fixture is scoped to the session, it will only run once per pytest run. + + Additionally, this fixture is also used to generate the `atmosds` synthetic testing dataset as well as add the + example file paths to the xdoctest_namespace, used when running doctests. + + Finally, this fixture is also forced socket enabled, meaning that it will run regardless of whether the tests + are being run with `--disable-socket` or not. + """ if ( not _default_cache_dir.joinpath(helpers.TESTDATA_BRANCH).exists() or helpers.PREFETCH_TESTING_DATA ): + if helpers.PREFETCH_TESTING_DATA: + print("`XCLIM_PREFETCH_TESTING_DATA` set. Prefetching testing data...") if worker_id in "master": helpers.populate_testing_data(branch=helpers.TESTDATA_BRANCH) else: diff --git a/tests/test_sdba/conftest.py b/tests/test_sdba/conftest.py index c78417645..b0e036ba8 100644 --- a/tests/test_sdba/conftest.py +++ b/tests/test_sdba/conftest.py @@ -105,7 +105,7 @@ def qds_month(): @pytest.fixture -def ref_hist_sim_tuto(): +def ref_hist_sim_tuto(socket_enabled): """Return ref, hist, sim time series of air temperature.""" def _ref_hist_sim_tuto(sim_offset=3, delta=0.1, smth_win=3, trend=True): diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py index 8eaa1eff6..d4a56c9ac 100644 --- a/tests/test_testing_utils.py +++ b/tests/test_testing_utils.py @@ -31,7 +31,7 @@ class TestFileRequests: @pytest.mark.requires_internet def test_get_failure(self, tmp_path): bad_repo_address = "https://github.com/beard/of/zeus/" - with pytest.raises(HTTPError): + with pytest.raises(FileNotFoundError): utilities._get( Path("san_diego", "60_percent_of_the_time_it_works_everytime"), bad_repo_address, diff --git a/xclim/testing/helpers.py b/xclim/testing/helpers.py index 181ae8794..faf99f1c5 100644 --- a/xclim/testing/helpers.py +++ b/xclim/testing/helpers.py @@ -128,11 +128,13 @@ def populate_testing_data( "NRCANdaily/nrcan_canada_daily_pr_1990.nc", "NRCANdaily/nrcan_canada_daily_tasmax_1990.nc", "NRCANdaily/nrcan_canada_daily_tasmin_1990.nc", + "Raven/q_sim.nc", "SpatialAnalogs/dissimilarity.nc", "SpatialAnalogs/indicators.nc", "cmip3/tas.sresb1.giss_model_e_r.run1.atm.da.nc", "cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712.nc", "sdba/CanESM2_1950-2100.nc", + "sdba/ahccd_1950-2013.nc", "sdba/nrcan_1950-2013.nc", "uncertainty_partitioning/cmip5_pr_global_mon.nc", ] diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index b65f76f05..d98de5e74 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -249,7 +249,7 @@ def _get( raise FileNotFoundError(msg) from e except SocketBlockedError as e: msg = ( - f"Unable to access {md5_name.as_posix()} online. Testing suite is being run with `--disable-socket`." + f"Unable to access {fullname.as_posix()} online. Testing suite is being run with `--disable-socket`. " f"If you intend to run tests with this option enabled, please download the file beforehand with the " f"following command:\n`xclim.testing.helpers.populate_testing_data()`." ) From 95c1a52441242b14c846746bd54ebda75c2f9292 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:43:14 -0400 Subject: [PATCH 13/22] manually call to populate_testing_data is no longer required --- .github/workflows/main.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a9b92cfc..a5e6a4b6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -115,11 +115,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install tox run: pip install tox~=4.0 - - name: Prefetch testing data - if: contains(matrix.tox-env, 'offline') - run: | - pip install . - python -c "import xclim.testing.helpers; xclim.testing.helpers.populate_testing_data()" - name: Test with tox run: tox -e ${{ matrix.tox-env }} ${{ matrix.markers }} env: From 0ccee74a1c56d5b80a3a94319e4968c5e8b140a5 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:11:49 -0400 Subject: [PATCH 14/22] add testing data prefetch step to tox recipe, `socket_enabled` is function-scoped, mark `test_reduce_dims` as `requires_internet` as it relies on remote xarray tutorial data --- tests/conftest.py | 7 +------ tests/test_sdba/conftest.py | 7 ++++++- tests/test_sdba/test_adjustment.py | 1 + tox.ini | 1 + 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 61ee11b55..29a40899a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -430,9 +430,7 @@ def ensemble_dataset_objects() -> dict: @pytest.fixture(scope="session", autouse=True) -def gather_session_data( - threadsafe_data_dir, worker_id, xdoctest_namespace, socket_enabled -): +def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace): """Gather testing data on pytest run. When running pytest with multiple workers, one worker will copy data remotely to _default_cache_dir while @@ -441,9 +439,6 @@ def gather_session_data( Additionally, this fixture is also used to generate the `atmosds` synthetic testing dataset as well as add the example file paths to the xdoctest_namespace, used when running doctests. - - Finally, this fixture is also forced socket enabled, meaning that it will run regardless of whether the tests - are being run with `--disable-socket` or not. """ if ( diff --git a/tests/test_sdba/conftest.py b/tests/test_sdba/conftest.py index b0e036ba8..4b519d4d6 100644 --- a/tests/test_sdba/conftest.py +++ b/tests/test_sdba/conftest.py @@ -106,7 +106,12 @@ def qds_month(): @pytest.fixture def ref_hist_sim_tuto(socket_enabled): - """Return ref, hist, sim time series of air temperature.""" + """Return ref, hist, sim time series of air temperature. + + socket_enabled is a fixture that enables the use of the internet to download the tutorial dataset while the + `--disable-socket` flag has been called. This fixture will crash if the `air_temperature` tutorial file is + not on disk while the internet is unavailable. + """ def _ref_hist_sim_tuto(sim_offset=3, delta=0.1, smth_win=3, trend=True): ds = xr.tutorial.open_dataset("air_temperature") diff --git a/tests/test_sdba/test_adjustment.py b/tests/test_sdba/test_adjustment.py index 40bbb1f5f..6462115b4 100644 --- a/tests/test_sdba/test_adjustment.py +++ b/tests/test_sdba/test_adjustment.py @@ -70,6 +70,7 @@ def test_time_and_from_ds(self, series, group, dec, tmp_path, random): p2 = loci2.adjust(sim) np.testing.assert_array_equal(p, p2) + @pytest.mark.requires_internet def test_reduce_dims(self, ref_hist_sim_tuto): ref, hist, sim = ref_hist_sim_tuto() hist = hist.expand_dims(member=[0, 1]) diff --git a/tox.ini b/tox.ini index 2ffdf50e2..47861697d 100644 --- a/tox.ini +++ b/tox.ini @@ -82,6 +82,7 @@ commands = [testenv:offline{-coverage,}] description = Run tests with pytest under {basepython}, preventing socket connections (except for unix sockets for async support) commands: + python -c "import xclim.testing.helpers; xclim.testing.helpers.populate_testing_data()" python -c 'print("Running offline tests with positional arguments \"not requires_internet\". This can be overridden with: tox -e offline -- -m \"some other marker statement\"")' pytest --disable-socket --allow-unix-socket {posargs:-m "not requires_internet"} From f0f55a0d6d2c0c31cb190199a5c779c1b7e57bb5 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:59:12 -0400 Subject: [PATCH 15/22] remove unneeded import --- docs/notebooks/cli.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/notebooks/cli.ipynb b/docs/notebooks/cli.ipynb index 2c37d12d1..9027f82e6 100644 --- a/docs/notebooks/cli.ipynb +++ b/docs/notebooks/cli.ipynb @@ -93,7 +93,6 @@ "from __future__ import annotations\n", "\n", "import warnings\n", - "from pathlib import Path\n", "\n", "import numpy as np\n", "import pandas as pd\n", From 7fd52a9876d551a50a3a9dc9feb2fec92c7150fa Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:03:25 -0400 Subject: [PATCH 16/22] add a CLI command for fetching the testing data, allow `offline` to optionally prefetch data (mainly for GitHub CI purposes), update contributing documentation to reflect changes --- .github/workflows/main.yml | 2 +- CONTRIBUTING.rst | 29 ++++++++++++++++++++++------- tox.ini | 10 ++++++---- xclim/cli.py | 38 +++++++++++++++++++++++++++++++++++--- xclim/testing/utils.py | 2 +- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5e6a4b6e..7aed26f74 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,7 +99,7 @@ jobs: markers: '-- -m "not slow"' - tox-env: notebooks_doctests python-version: "3.10" - - tox-env: offline + - tox-env: offline-prefetch python-version: "3.11" markers: '-- -m "not slow and not requires_internet"' steps: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 56985318f..6d63dae45 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -254,8 +254,7 @@ Before you submit a pull request, please follow these guidelines: Updating Testing Data ~~~~~~~~~~~~~~~~~~~~~ -If your code changes require changes to the testing data of `xclim` (i.e.: modifications to existing datasets or new datasets), -these changes must be made via a Pull Request at https://github.com/Ouranosinc/xclim-testdata. +If your code changes require changes to the testing data of `xclim` (i.e.: modifications to existing datasets or new datasets), these changes must be made via a Pull Request at https://github.com/Ouranosinc/xclim-testdata. `xclim` allows for developers to test specific branches/versions of `xclim-testdata` via the `XCLIM_TESTDATA_BRANCH` environment variable, either through export, e.g.:: @@ -273,6 +272,20 @@ or by setting the variable at runtime:: This will ensure that tests load the testing data from this branch before running. +If you anticipate not having internet access, we suggest prefetching the testing data from `Ouranosinc/xclim-testdata` and storing it in your local cache. This can be done by running the following console command:: + + $ xclim prefetch_testing_data + +If your development branch relies on a specific branch of `Ouranosinc/xclim-testdata`, you can specify this using environment variables:: + + $ export XCLIM_TESTDATA_BRANCH="my_new_branch_of_testing_data" + $ xclim prefetch_testing_data + +or, alternatively, with the `--branch` option:: + + $ xclim prefetch_testing_data --branch my_new_branch_of_testing_data + + If you wish to test a specific branch using GitHub CI, this can be set in `.github/workflows/main.yml`: .. code-block:: yaml @@ -284,15 +297,17 @@ If you wish to test a specific branch using GitHub CI, this can be set in `.gith In order for a Pull Request to be allowed to merge to main development branch, this variable must match the latest tagged commit name on `Ouranosinc/xclim-testdata`. We suggest merging changed testing data first, tagging a new version of `xclim-testdata`, then re-running tests on your Pull Request at `Ouranosinc/xclim` with the newest tag. -Running Tests Offline -~~~~~~~~~~~~~~~~~~~~~ +Running Tests in Offline Mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -`xclim` testing is normally performed with the assumption that the machine running the tests has internet access. Many calls to `xclim` functions will attempt to download data or verify checksums from the `Ouranosinc/xclim-testdata` repository. This can be problematic for developers working on features when internet access is not immediately available. +`xclim` testing is designed with the assumption that the machine running the tests has internet access. Many calls to `xclim` functions will attempt to download data or verify checksums from the `Ouranosinc/xclim-testdata` repository. This can be problematic for developers working on features where internet access is not reliably available. -In order to bypass these remote validation checks, the testing can be run offline by either running pytest with the following options:: +If you wish to ensure that your feature or bugfix can be developed without internet access, the testing can be run in offline mode by running pytest with the following options:: $ pytest --disable-socket --allow-unix-socket -m "not requires_internet" - # or, alternatively: + +or, alternatively, using `tox` :: + $ tox -e offline These options will disable all network calls and skip tests marked with the `requires_internet` marker. The `--allow-unix-socket` option is required to allow the `pytest-xdist` plugin to function properly. diff --git a/tox.ini b/tox.ini index 47861697d..c6599eb17 100644 --- a/tox.ini +++ b/tox.ini @@ -4,14 +4,14 @@ env_list = lint docs notebooks_doctests - offline + offline-prefetch ; opt-slow py38 py39-upstream-doctest py310 py311 labels = - test = py38, py39-upstream-doctest, py310, py311, notebooks_doctests, offline + test = py38, py39-upstream-doctest, py310, py311, notebooks_doctests, offline-prefetch requires = pip >= 21.0 opts = -vv @@ -79,12 +79,14 @@ commands = ; !slow: pytest xclim -m "not slow" --durations=10 ; slow: pytest xclim --durations=10 -[testenv:offline{-coverage,}] +[testenv:offline{-prefetch,}{-coverage,}] description = Run tests with pytest under {basepython}, preventing socket connections (except for unix sockets for async support) commands: - python -c "import xclim.testing.helpers; xclim.testing.helpers.populate_testing_data()" + prefetch: xclim prefetch_testing_data python -c 'print("Running offline tests with positional arguments \"not requires_internet\". This can be overridden with: tox -e offline -- -m \"some other marker statement\"")' pytest --disable-socket --allow-unix-socket {posargs:-m "not requires_internet"} +allowlist_externals = + xclim [testenv] description = Run tests with pytest under {basepython} diff --git a/xclim/cli.py b/xclim/cli.py index f3d9a6e26..42dcd1f63 100644 --- a/xclim/cli.py +++ b/xclim/cli.py @@ -15,7 +15,8 @@ import xclim as xc from xclim.core.dataflags import DataQualityException, data_flags, ecad_compliant from xclim.core.utils import InputKind -from xclim.testing.utils import publish_release_notes, show_versions +from xclim.testing.helpers import TESTDATA_BRANCH, populate_testing_data +from xclim.testing.utils import _default_cache_dir, publish_release_notes, show_versions try: from dask.distributed import Client, progress # pylint: disable=ungrouped-imports @@ -146,6 +147,29 @@ def show_version_info(ctx): ctx.exit() +@click.command(short_help="Prefetch xclim testing data for development purposes.") +@click.option( + "-b", + "--branch", + help="The xclim-testdata branch to be fetched and cached. If not specified, defaults to " + "`XCLIM_TESTING_DATA_BRANCH` (if set) or `main`.", +) +@click.pass_context +def prefetch_testing_data(ctx, branch): + """Prefetch xclim testing data for development purposes.""" + if branch: + testdata_branch = branch + else: + testdata_branch = TESTDATA_BRANCH + + click.echo( + f"Gathering testing data from xclim-testdata `{testdata_branch}` branch..." + ) + click.echo(populate_testing_data(branch=testdata_branch)) + click.echo(f"Testing data saved to `{_default_cache_dir}`.") + ctx.exit() + + @click.command(short_help="Print history for publishing purposes.") @click.option("-m", "--md", is_flag=True, help="Prints the history in Markdown format.") @click.option( @@ -325,14 +349,22 @@ class XclimCli(click.MultiCommand): def list_commands(self, ctx): """Return the available commands (other than the indicators).""" - return "indices", "info", "dataflags", "release_notes", "show_version_info" + return ( + "indices", + "info", + "dataflags", + "prefetch_testing_data", + "release_notes", + "show_version_info", + ) def get_command(self, ctx, name): """Return the requested command.""" command = { + "dataflags": dataflags, "indices": indices, "info": info, - "dataflags": dataflags, + "prefetch_testing_data": prefetch_testing_data, "release_notes": release_notes, "show_version_info": show_version_info, }.get(name) diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index d98de5e74..ecd202c51 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -251,7 +251,7 @@ def _get( msg = ( f"Unable to access {fullname.as_posix()} online. Testing suite is being run with `--disable-socket`. " f"If you intend to run tests with this option enabled, please download the file beforehand with the " - f"following command:\n`xclim.testing.helpers.populate_testing_data()`." + f"following console command: `xclim prefetch_testing_data`." ) raise FileNotFoundError(msg) from e try: From 76e96f7ae16df27db94d36f436ae544044b70f58 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:06:52 -0400 Subject: [PATCH 17/22] mark `test_open_dataset` as `requires_internet` --- tests/test_testing_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py index d4a56c9ac..1b718f10b 100644 --- a/tests/test_testing_utils.py +++ b/tests/test_testing_utils.py @@ -74,6 +74,7 @@ def test_open_dataset_with_bad_file(self, tmp_path): == Path(cmip3_folder, cmip3_md5).read_text() ) + @pytest.mark.requires_internet def test_open_testdata(self): ds = utilities.open_dataset( Path("cmip5/tas_Amon_CanESM2_rcp85_r1i1p1_200701-200712") From 2b2d27a10acb14148fa89a083d8ea007861b7432 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:11:56 -0400 Subject: [PATCH 18/22] update testdata tag and remove calls to `spatial-analogs-nb` branch see: https://github.com/Ouranosinc/xclim-testdata/releases/tag/v2023.9.12 --- .github/workflows/main.yml | 2 +- docs/notebooks/analogs.ipynb | 4 +--- xclim/testing/helpers.py | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7aed26f74..c4440af83 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ on: - submitted env: - XCLIM_TESTDATA_BRANCH: v2023.6.28 + XCLIM_TESTDATA_BRANCH: v2023.9.12 jobs: lint: diff --git a/docs/notebooks/analogs.ipynb b/docs/notebooks/analogs.ipynb index c789afcfc..4c292eea4 100644 --- a/docs/notebooks/analogs.ipynb +++ b/docs/notebooks/analogs.ipynb @@ -49,7 +49,6 @@ "source": [ "sim = open_dataset(\n", " \"SpatialAnalogs/CanESM2_ScenGen_Chibougamau_2041-2070.nc\",\n", - " branch=\"spatial-analogs-nb\",\n", " decode_timedelta=False,\n", ")\n", "sim" @@ -74,7 +73,6 @@ "source": [ "obs = open_dataset(\n", " \"SpatialAnalogs/NRCAN_SECan_1981-2010.nc\",\n", - " branch=\"spatial-analogs-nb\",\n", " decode_timedelta=False,\n", ")\n", "obs" @@ -88,7 +86,7 @@ "outputs": [], "source": [ "obs.tg_mean.isel(time=0).plot()\n", - "plt.plot(sim.lon, sim.lat, \"ro\"); # Plot a point over chibougamau" + "plt.plot(sim.lon, sim.lat, \"ro\"); # Plot a point over Chibougamau" ] }, { diff --git a/xclim/testing/helpers.py b/xclim/testing/helpers.py index faf99f1c5..e87894edf 100644 --- a/xclim/testing/helpers.py +++ b/xclim/testing/helpers.py @@ -129,6 +129,8 @@ def populate_testing_data( "NRCANdaily/nrcan_canada_daily_tasmax_1990.nc", "NRCANdaily/nrcan_canada_daily_tasmin_1990.nc", "Raven/q_sim.nc", + "SpatialAnalogs/CanESM2_ScenGen_Chibougamau_2041-2070.nc", + "SpatialAnalogs/NRCAN_SECan_1981-2010.nc", "SpatialAnalogs/dissimilarity.nc", "SpatialAnalogs/indicators.nc", "cmip3/tas.sresb1.giss_model_e_r.run1.atm.da.nc", From cd07798df36736327e8c0c6da62ee9525c8b7fa9 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:37:06 -0400 Subject: [PATCH 19/22] fix some hyperlink targets --- CONTRIBUTING.rst | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6d63dae45..e56b96ed5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -174,7 +174,7 @@ Ready to contribute? Here's how to set up `xclim` for local development. `offline`: this configuration runs by default with the `-m "not requires_internet"` test marker. Be aware that running `tox` and manually setting a `pytest` marker will override this default. .. note:: - `xclim` tests are organized to support the `pytest-xdist `_ plugin for distributed testing across workers or CPUs. In order to benefit from multiple processes, add the flag `--numprocesses=auto` or `-n auto` to your `pytest` calls. + `xclim` tests are organized to support the `pytest-xdist`_ plugin for distributed testing across workers or CPUs. In order to benefit from multiple processes, add the flag `--numprocesses=auto` or `-n auto` to your `pytest` calls. When running tests via `tox`, `numprocesses` is set to the number of logical cores available (`numprocesses=logical`), with a maximum amount of `8`. @@ -254,7 +254,7 @@ Before you submit a pull request, please follow these guidelines: Updating Testing Data ~~~~~~~~~~~~~~~~~~~~~ -If your code changes require changes to the testing data of `xclim` (i.e.: modifications to existing datasets or new datasets), these changes must be made via a Pull Request at https://github.com/Ouranosinc/xclim-testdata. +If your code changes require changes to the testing data of `xclim` (i.e.: modifications to existing datasets or new datasets), these changes must be made via a Pull Request at the `xclim-testdata repository`_. `xclim` allows for developers to test specific branches/versions of `xclim-testdata` via the `XCLIM_TESTDATA_BRANCH` environment variable, either through export, e.g.:: @@ -272,7 +272,7 @@ or by setting the variable at runtime:: This will ensure that tests load the testing data from this branch before running. -If you anticipate not having internet access, we suggest prefetching the testing data from `Ouranosinc/xclim-testdata` and storing it in your local cache. This can be done by running the following console command:: +If you anticipate not having internet access, we suggest prefetching the testing data from `xclim-testdata repository`_ and storing it in your local cache. This can be done by running the following console command:: $ xclim prefetch_testing_data @@ -285,7 +285,6 @@ or, alternatively, with the `--branch` option:: $ xclim prefetch_testing_data --branch my_new_branch_of_testing_data - If you wish to test a specific branch using GitHub CI, this can be set in `.github/workflows/main.yml`: .. code-block:: yaml @@ -294,7 +293,7 @@ If you wish to test a specific branch using GitHub CI, this can be set in `.gith XCLIM_TESTDATA_BRANCH: my_new_branch_of_testing_data .. warning:: - In order for a Pull Request to be allowed to merge to main development branch, this variable must match the latest tagged commit name on `Ouranosinc/xclim-testdata`. + In order for a Pull Request to be allowed to merge to main development branch, this variable must match the latest tagged commit name on `xclim-testdata repository`_. We suggest merging changed testing data first, tagging a new version of `xclim-testdata`, then re-running tests on your Pull Request at `Ouranosinc/xclim` with the newest tag. Running Tests in Offline Mode @@ -302,7 +301,7 @@ Running Tests in Offline Mode `xclim` testing is designed with the assumption that the machine running the tests has internet access. Many calls to `xclim` functions will attempt to download data or verify checksums from the `Ouranosinc/xclim-testdata` repository. This can be problematic for developers working on features where internet access is not reliably available. -If you wish to ensure that your feature or bugfix can be developed without internet access, the testing can be run in offline mode by running pytest with the following options:: +If you wish to ensure that your feature or bugfix can be developed without internet access, `xclim` leverages the `pytest-socket`_ plugin so that testing can be run in "offline" mode by invoking pytest with the following options:: $ pytest --disable-socket --allow-unix-socket -m "not requires_internet" @@ -310,7 +309,7 @@ or, alternatively, using `tox` :: $ tox -e offline -These options will disable all network calls and skip tests marked with the `requires_internet` marker. The `--allow-unix-socket` option is required to allow the `pytest-xdist` plugin to function properly. +These options will disable all network calls and skip tests marked with the `requires_internet` marker. The `--allow-unix-socket` option is required to allow the `pytest-xdist`_ plugin to function properly. Tips ---- @@ -399,7 +398,7 @@ When publishing on GitHub, maintainers will need to generate the release notes f The Manual Approach ~~~~~~~~~~~~~~~~~~~ -The manual approach to library packaging for general support (pip wheels) requires that the `flit `_ library is installed. +The manual approach to library packaging for general support (pip wheels) requires that the `flit`_ library is installed. From the command line on your Linux distribution, simply run the following from the clone's main dev branch:: @@ -433,8 +432,12 @@ Before updating the main conda-forge recipe, we *strongly* suggest performing th .. _`GitHub Repository`: https://github.com/Ouranosinc/xclim .. _`PEP8`: https://peps.python.org/pep-0008/ +.. _`flit`: https://flit.pypa.io/en/stable/index.html .. _`numpydoc`: https://numpydoc.readthedocs.io/en/latest/format.html#docstring-standard +.. _`pytest-socket`: https://github.com/miketheman/pytest-socket +.. _`pytest-xdist`: https://pytest-xdist.readthedocs.io/en/latest/ .. _`reStructuredText (ReST)`: https://www.jetbrains.com/help/pycharm/using-docstrings-to-specify-types.html .. _`reStructuredText Primer`: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html .. _`sphinxcontrib-bibtex`: https://sphinxcontrib-bibtex.readthedocs.io .. _`xclim on TestPyPI`: https://test.pypi.org/project/xclim/ +.. _`xclim-testdata repository`: https://github.com/Ouranosinc/xclim-testdata From 307fff28a2bbda5534600d4055ba8d02d2f85985 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 17:26:00 -0400 Subject: [PATCH 20/22] fix the print call for offline build --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c6599eb17..df68b578a 100644 --- a/tox.ini +++ b/tox.ini @@ -83,7 +83,7 @@ commands = description = Run tests with pytest under {basepython}, preventing socket connections (except for unix sockets for async support) commands: prefetch: xclim prefetch_testing_data - python -c 'print("Running offline tests with positional arguments \"not requires_internet\". This can be overridden with: tox -e offline -- -m \"some other marker statement\"")' + python -c 'print("Running offline tests with positional arguments: {posargs:-m \"not requires_internet\"}. This can be overwritten with: tox -e offline -- -m \"some other marker statement\"")' pytest --disable-socket --allow-unix-socket {posargs:-m "not requires_internet"} allowlist_externals = xclim From 269a269203ab14a7bf50c6659c23b5af26fef14e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 17:26:10 -0400 Subject: [PATCH 21/22] update CHANGES.rst --- CHANGES.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index dfc25b7ff..741a29299 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,21 +8,26 @@ Contributors to this version: Éric Dupuis (:user:`coxipi`), Trevor James Smith New features and enhancements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The testing suite now offers a means of running tests without internet access. This requires a local copy of `xclim-testdata` to be present in the user's home directory and for certain `pytest` options and markers to be set. For more information, see the contributing documentation section `Running Tests Offline`. (:issue:`1468`, :pull:`1473`). +* `xclim` now has a dedicated console command for prefetching testing data from `xclim-testdata` with branch options (e.g.: `$ xclim prefetch_testing_data --branch some_development_branch`). This command can be used to download the testing data to a local cache, which can then be used to run the testing suite without internet access or in "offline" mode. For more information, see the contributing documentation section for `Updating Testing Data`. (:issue:`1468`, :pull:`1473`). +* The testing suite now offers a means of running tests in "offline" mode (using `pytest-socket `_ to block external connections). This requires a local copy of `xclim-testdata` to be present in the user's home cache directory and for certain `pytest` options and markers to be set when invoked. For more information, see the contributing documentation section for `Running Tests in Offline Mode`. (:issue:`1468`, :pull:`1473`). Bug fixes ^^^^^^^^^ * Fixed an error in the `pytest` configuration that prevented copying of testing data to thread-safe caches of workers under certain conditions (this should always occur). (:pull:`1473`). + * Coincidentally, this also fixes an error that caused `pytest` to error-out when invoked without an active internet connection. Running `pytest` without network access is now supported (requires cached testing data). (:issue:`1468`). Breaking changes ^^^^^^^^^^^^^^^^ +* `pytest-socket` is now a required development dependency for running `"offline"` tests or the `"offline"` configuration of the `tox` testing suite. This has been added to the `dev` installation recipe. (:issue:`1468`, :pull:`1473`). * For better transparency and control in development, the `tox` configuration has been adapted to allow passing of markers directly to the `pytest` call. Positional arguments must be passed to tox after the `--` separator to select/deselect tests (e.g. ``'tox -e py38 -- -m "not slow"'``). (:pull:`1473`). -* `pytest-socket` is now a required development dependency for running the `"offline"` configurations of the `tox` testing suite. This has been added to the `dev` installation recipe. (:issue:`1468`, :pull:`1473`). +* For better accuracy, the `tox -e black` recipe has been renamed to `tox -e lint`, as this configuration already included several other linting checks. (:pull:`1473`). Internal changes ^^^^^^^^^^^^^^^^ * Changed "degK" to "K" (used to designate Kelvin units). (:pull:`1475`). +* Added a `pytest` marker (``pytest.mark.requires_internet``) to allow for skipping of tests that depend on remote network calls to function properly. (:pull:`1473`). * Added handling for `pytest-socket`'s ``SocketBlockedError`` in ``xclim.testing.open_dataset`` when attempting to fetch md5 validation files for cached testing data while explicitly disabling internet sockets. (:issue:`1468`, :pull:`1473`). +* Updated the testing data used in the `analogs.ipynb` notebook to use the testing data now found in `Ouranosinc/xclim-testdata`'s main branch. (`xclim-testdata PR/26 `_, :pull:`1473`). v0.45.0 (2023-09-05) -------------------- From d80ce92ba121b96394f80ae8d0445508709321d1 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 12 Sep 2023 18:41:51 -0400 Subject: [PATCH 22/22] fix markers --- .github/workflows/main.yml | 10 +++++----- tox.ini | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4440af83..ddd0f7d40 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,20 +88,20 @@ jobs: include: - tox-env: py38-coverage-eofs python-version: "3.8" - markers: '-- -m "not slow"' + markers: -m 'not slow' - tox-env: py39-coverage-sbck-eofs python-version: "3.9" - markers: '-- -m "not slow"' + markers: -m 'not slow' - tox-env: py310-coverage # No markers -- includes slow tests python-version: "3.10" - tox-env: py311-coverage-sbck python-version: "3.11" - markers: '-- -m "not slow"' + markers: -m 'not slow' - tox-env: notebooks_doctests python-version: "3.10" - tox-env: offline-prefetch python-version: "3.11" - markers: '-- -m "not slow and not requires_internet"' + markers: -m 'not slow and not requires_internet' steps: - uses: actions/checkout@v3.6.0 - name: Install Eigen3 @@ -116,7 +116,7 @@ jobs: - name: Install tox run: pip install tox~=4.0 - name: Test with tox - run: tox -e ${{ matrix.tox-env }} ${{ matrix.markers }} + run: tox -e ${{ matrix.tox-env }} -- ${{ matrix.markers }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-{{ matrix.tox-env }} diff --git a/tox.ini b/tox.ini index df68b578a..155efb572 100644 --- a/tox.ini +++ b/tox.ini @@ -83,8 +83,8 @@ commands = description = Run tests with pytest under {basepython}, preventing socket connections (except for unix sockets for async support) commands: prefetch: xclim prefetch_testing_data - python -c 'print("Running offline tests with positional arguments: {posargs:-m \"not requires_internet\"}. This can be overwritten with: tox -e offline -- -m \"some other marker statement\"")' - pytest --disable-socket --allow-unix-socket {posargs:-m "not requires_internet"} + python -c 'print("Running offline tests with positional arguments. These can be overwritten with: tox -e offline -- -m \"some other marker statement\"")' + pytest --disable-socket --allow-unix-socket {posargs:-m 'not requires_internet'} allowlist_externals = xclim