From 173a000f51216d3af9aab4581858e6a631205b9d Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:35:15 -0400 Subject: [PATCH 1/6] add pytest-xdist to development dependencies, enable in pyproject.toml and tox.ini --- environment-dev.yml | 3 ++- pyproject.toml | 7 ++++++- tox.ini | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 6a746e00..9d4434da 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -57,10 +57,11 @@ dependencies: - pre-commit >=3.5.0 - pytest >=8.3.2 - pytest-cov >=5.0.0 + - pytest-xdist >=3.2.0 - ruff >=0.5.7 - setuptools >=65.0.0 - setuptools-scm >=8.0.0 - - sphinx + - sphinx >=7.0.0 - sphinx-autoapi - sphinx-rtd-theme >=1.0 - sphinxcontrib-napoleon diff --git a/pyproject.toml b/pyproject.toml index 75eadeb1..af28f454 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ dev = [ "pre-commit >=3.3.2", "pytest-cov >=5.0.0", "pytest >=8.3.2", + "pytest-xdist[psutil] >=3.2.0", "ruff >=0.5.7", "tox >=4.18.0", "watchdog >=4.0.0", @@ -249,8 +250,12 @@ addopts = [ "--color=yes", "--cov=xscen", "--ignore-glob='*.ipynb_checkpoints'", + "--strict-config", "--strict-markers", - "--verbose" + "--verbose", + "--numprocesses=0", + "--maxprocesses=8", + "--dist=worksteal" ] filterwarnings = ["ignore::UserWarning"] testpaths = "tests" diff --git a/tox.ini b/tox.ini index 5ed966da..25f08bd6 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,7 @@ commands = description = Run tests with pytest under {basepython} (Anaconda distribution) setenv = COV_CORE_SOURCE = - PYTEST_ADDOPTS = --color=yes --cov=xscen --strict-markers --verbose + PYTEST_ADDOPTS = --numprocesses=logical --durations=10 PYTHONPATH = {toxinidir} passenv = COVERALLS_* From af507b49e62669375a2f83f25d314b19b8820dac Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:35:31 -0400 Subject: [PATCH 2/6] remove obsolete tox-conda configurations --- tox.ini | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tox.ini b/tox.ini index 25f08bd6..ad94cf5b 100644 --- a/tox.ini +++ b/tox.ini @@ -21,8 +21,6 @@ python = description = Check for Code Compliance and missing french translations skip_install = True download = true -conda_channels = -conda_env = deps = black[jupyter] ==24.8.0 blackdoc ==0.3.9 @@ -41,8 +39,6 @@ allowlist_externals = [testenv:docs{,-esmpy}] description = Run Build of xscen Documentation -conda_deps = -conda_env = environment-dev.yml extras = docs commands = @@ -52,8 +48,6 @@ allowlist_externals = [testenv:doctests{,-esmpy}] description = Run documentation linters and doctests with pytest under {basepython} -conda_deps = -conda_env = environment-dev.yml extras = dev docs @@ -68,10 +62,12 @@ setenv = PYTEST_ADDOPTS = --numprocesses=logical --durations=10 PYTHONPATH = {toxinidir} passenv = + CI COVERALLS_* ESMFMKFILE ESMF_VERSION GITHUB_* + XCLIM_* download = true deps = coveralls: coveralls @@ -79,14 +75,6 @@ deps = upstream: -rrequirements_upstream.txt extras = dev -conda_channels = - conda-forge - defaults -conda_deps = - pytest - pytest-cov - xdoctest -conda_env = environment.yml install_command = python -m pip install --no-user {opts} {packages} commands_pre = pip list From 3dddd7af47f48865131bf77423c65891e2bf851e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:06:41 -0400 Subject: [PATCH 3/6] add numprocesses=logical to anaconda build pytest calls --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b862c55..d291c51f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -161,7 +161,7 @@ jobs: python -m pip check || true - name: Test with pytest run: | - python -m pytest --cov xscen + python -m pytest --numprocesses=auto - name: Report Coverage run: | python -m coveralls From 709cb2d976a759677aba91be6c9338bee7b87f16 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:25:30 -0400 Subject: [PATCH 4/6] use tmpdir, use copies --- tests/test_biasadjust.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_biasadjust.py b/tests/test_biasadjust.py index c2fe765b..c4b5d8cf 100644 --- a/tests/test_biasadjust.py +++ b/tests/test_biasadjust.py @@ -129,7 +129,7 @@ def test_basic( self, periods, to_level, bias_adjust_institution, bias_adjust_project ): dtrain = xs.train( - self.dref, + self.dref.copy(), self.dsim.sel(time=slice("2001", "2003")), var="tas", period=["2001", "2003"], @@ -137,7 +137,7 @@ def test_basic( out = xs.adjust( dtrain, - self.dsim, + self.dsim.copy(), periods=periods, to_level=to_level, bias_adjust_institution=bias_adjust_institution, @@ -175,9 +175,9 @@ def test_basic( np.concatenate([np.ones(365 * 1) * 1, np.ones(365 * 1) * 3]), ) - def test_write_train(self): + def test_write_train(self, tmpdir): dtrain = xs.train( - self.dref, + self.dref.copy(), self.dsim.sel(time=slice("2001", "2003")), var="tas", period=["2001", "2003"], @@ -186,7 +186,7 @@ def test_write_train(self): jitter_under={"thresh": "2 K"}, ) - root = str(notebooks / "_data") + root = str(tmpdir / "_data") xs.save_to_zarr(dtrain, f"{root}/test.zarr", mode="o") dtrain2 = xr.open_dataset( f"{root}/test.zarr", chunks={"dayofyear": 365, "quantiles": 15} @@ -194,7 +194,7 @@ def test_write_train(self): out = xs.adjust( dtrain, - self.dsim, + self.dsim.copy(), periods=["2001", "2006"], xclim_adjust_args={ "detrend": { @@ -205,7 +205,7 @@ def test_write_train(self): out2 = xs.adjust( dtrain2, - self.dsim, + self.dsim.copy(), periods=["2001", "2006"], xclim_adjust_args={ "detrend": { From a80ab23779ef6d580230d0dfc0231adf54964691 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:46:50 -0400 Subject: [PATCH 5/6] disable pytest-xdist --- .github/workflows/main.yml | 2 +- tox.ini | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d291c51f..d78f1216 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -161,7 +161,7 @@ jobs: python -m pip check || true - name: Test with pytest run: | - python -m pytest --numprocesses=auto + python -m pytest - name: Report Coverage run: | python -m coveralls diff --git a/tox.ini b/tox.ini index ad94cf5b..6599f016 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,6 @@ commands = description = Run tests with pytest under {basepython} (Anaconda distribution) setenv = COV_CORE_SOURCE = - PYTEST_ADDOPTS = --numprocesses=logical --durations=10 PYTHONPATH = {toxinidir} passenv = CI From c61cdd86cb839f3f8935fb922b8c5a2604c63257 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:16:58 -0400 Subject: [PATCH 6/6] update CHANGELOG.rst --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 60f01b3a..baa71dcb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,7 @@ New features and enhancements * Convenience functions ``xs.io.zip_directory`` and ``xs.io.unzip_directory`` (for zarrs). (:pull:`431`). * New argument ``compute_indicators``: ``rechunk_input`` to rechunk the inputs to resample-appropriate chunks before calling xclim. (:pull:`431`). * New ``xs.indicators.get_indicator_outputs`` to retrieve what variable name(s) and frequency to expect from an xclim indicator. (:pull:`431`). +* `xscen` now supports launches tests from `pytest` with the `--numprocesses` option. See the `pytest-xdist documentation `_ for more information. (:pull:`464`). Bug fixes ^^^^^^^^^ @@ -44,6 +45,7 @@ Breaking changes * `convert_calendar` in ``clean_up`` now uses `xarray` instead of `xclim`. Keywords aren't compatible between the two, but given that `xclim` will abandon its function, no backwards compatibility was sought. (:pull:`450`). * `attrs_to_remove` and `remove_all_attrs_except` in ``clean_up`` now use real regex. It should not be too breaking since a `fullmatch()` is used, but `*` is now `.*`. (:pull:`450`). * Python 3.9 is no longer supported. (:pull:`456`). +* `pytest-xdist` is now a development dependency. (:pull:`464`). Internal changes ^^^^^^^^^^^^^^^^