From 4f47102a09bc421bc6347a4d8445a13293844d4e Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Mon, 16 Oct 2023 15:33:36 -0400 Subject: [PATCH 1/9] Better no translation error and add note to doc --- docs/goodtoknow.rst | 2 ++ xscen/utils.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/goodtoknow.rst b/docs/goodtoknow.rst index 6314b133..14682126 100644 --- a/docs/goodtoknow.rst +++ b/docs/goodtoknow.rst @@ -62,6 +62,8 @@ Which can also be activated in the code using :py:func:`xclim.core.options.set_o Translation is of course not automatic but relies on manually populated `gettext `_ catalogs. xscen ships with a catalog of french (fr) translations. See :ref:`translating-xscen` to learn how to add translations to xscen. xclim's documentation of the same subject is `here `_. +If your xscen is installed in "editable" mode in its source directory (``pip install -e .``), you should run ``make translate`` each time you pull changes from the upstream source. + Module-wide options ------------------- diff --git a/xscen/utils.py b/xscen/utils.py index c7d931a8..ff19854c 100644 --- a/xscen/utils.py +++ b/xscen/utils.py @@ -55,12 +55,16 @@ If a language is not defined or a message not translated, the function will return the raw message. """ - -for loc in (Path(__file__).parent / "data").iterdir(): - if loc.is_dir() and len(loc.name) == 2: - TRANSLATOR[loc.name] = gettext.translation( - "xscen", localedir=loc.parent, languages=[loc.name] - ).gettext +try: + for loc in (Path(__file__).parent / "data").iterdir(): + if loc.is_dir() and len(loc.name) == 2: + TRANSLATOR[loc.name] = gettext.translation( + "xscen", localedir=loc.parent, languages=[loc.name] + ).gettext +except FileNotFoundError as err: + raise ImportError( + "Your xscen installation doesn't have compiled translations. Run `make translate` from the source directory to fix." + ) from err def update_attr(ds, attr, new, others=None, **fmt): From ea0ab1ef39c65da521c418848d3a4a66a607e986 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 20:20:16 +0000 Subject: [PATCH 2/9] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black-pre-commit-mirror: 23.9.1 → 23.10.0](https://github.com/psf/black-pre-commit-mirror/compare/23.9.1...23.10.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f9de583..cec80d05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: rst-inline-touching-normal - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 23.10.0 hooks: - id: black exclude: ^docs/ From c91cb97283f54d7ae06e0c91595b3e97f4f52968 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:27:55 -0400 Subject: [PATCH 3/9] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cec80d05..f9f3c774 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: rev: v0.3.8 hooks: - id: blackdoc - additional_dependencies: [ 'black==23.9.1' ] + additional_dependencies: [ 'black==23.10.0' ] exclude: config.py - repo: https://github.com/adrienverge/yamllint.git rev: v1.32.0 @@ -65,10 +65,13 @@ repos: hooks: - id: nbqa-pyupgrade args: [ '--py39-plus' ] + additional_dependencies: [ 'pyupgrade==3.15.0' ] - id: nbqa-black args: [ '--target-version=py39' ] + additional_dependencies: [ 'black==23.10.0' ] - id: nbqa-isort args: [ "--settings-file=setup.cfg" ] + additional_dependencies: [ 'isort==5.12.0' ] - repo: https://github.com/kynan/nbstripout rev: 0.6.1 hooks: From cf564232a5c71e0bdc602bec562a2396253126d0 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Thu, 26 Oct 2023 11:57:47 -0400 Subject: [PATCH 4/9] Pin fsspec everywhere --- environment-dev.yml | 2 +- environment.yml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 048e5206..1d97332a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -11,7 +11,7 @@ dependencies: - clisops >=0.10 - dask - flox - - fsspec + - fsspec < 2023.10.0 - geopandas - h5netcdf - h5py diff --git a/environment.yml b/environment.yml index 9a70a030..4e3c9de5 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,7 @@ dependencies: - clisops >=0.10 - dask - flox - - fsspec + - fsspec < 2023.10.0 - geopandas - h5netcdf - h5py diff --git a/setup.py b/setup.py index 4952b2ab..f0924378 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ def run(self): "clisops>=0.10", "dask", "flox", - "fsspec", + "fsspec<2023.10.0", "geopandas", "h5netcdf", "h5py", From 9995e8f8f273c29d43b6fb4c07c9317e6423b250 Mon Sep 17 00:00:00 2001 From: "bumpversion[bot]" Date: Thu, 26 Oct 2023 19:34:23 +0000 Subject: [PATCH 5/9] =?UTF-8?q?Bump=20version:=200.7.15-beta=20=E2=86=92?= =?UTF-8?q?=200.7.16-beta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cruft.json | 2 +- setup.cfg | 2 +- setup.py | 2 +- tests/test_xscen.py | 2 +- xscen/__init__.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.cruft.json b/.cruft.json index a7dc6521..23d43b6a 100644 --- a/.cruft.json +++ b/.cruft.json @@ -11,7 +11,7 @@ "project_slug": "xscen", "project_short_description": "A climate change scenario-building analysis framework, built with xclim/xarray.", "pypi_username": "RondeauG", - "version": "0.7.15-beta", + "version": "0.7.16-beta", "use_pytest": "y", "use_black": "y", "add_pyup_badge": "n", diff --git a/setup.cfg b/setup.cfg index d52fe214..8746bde5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.7.15-beta +current_version = 0.7.16-beta commit = True tag = False parse = (?P\d+)\.(?P\d+).(?P\d+)(\-(?P[a-z]+))? diff --git a/setup.py b/setup.py index f0924378..55cb1dae 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,6 @@ def run(self): test_suite="tests", extras_require={"dev": dev_requirements}, url="https://github.com/Ouranosinc/xscen", - version="0.7.15-beta", + version="0.7.16-beta", zip_safe=False, ) diff --git a/tests/test_xscen.py b/tests/test_xscen.py index 30c400e2..d0186187 100644 --- a/tests/test_xscen.py +++ b/tests/test_xscen.py @@ -28,4 +28,4 @@ def test_package_metadata(self): contents = f.read() assert """Gabriel Rondeau-Genesse""" in contents assert '__email__ = "rondeau-genesse.gabriel@ouranos.ca"' in contents - assert '__version__ = "0.7.15-beta"' in contents + assert '__version__ = "0.7.16-beta"' in contents diff --git a/xscen/__init__.py b/xscen/__init__.py index 55280871..dc4ba6fe 100644 --- a/xscen/__init__.py +++ b/xscen/__init__.py @@ -52,7 +52,7 @@ __author__ = """Gabriel Rondeau-Genesse""" __email__ = "rondeau-genesse.gabriel@ouranos.ca" -__version__ = "0.7.15-beta" +__version__ = "0.7.16-beta" # monkeypatch so that warnings.warn() doesn't mention itself From ad3e1c1de102f1b3b722f8d1c275bedebe30b364 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 20:07:40 +0000 Subject: [PATCH 6/9] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black-pre-commit-mirror: 23.10.0 → 23.10.1](https://github.com/psf/black-pre-commit-mirror/compare/23.10.0...23.10.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f9f3c774..0fe20b60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: rst-inline-touching-normal - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.10.0 + rev: 23.10.1 hooks: - id: black exclude: ^docs/ From bc1a94f4fb7439277271d8d5dea9e7c8474293c7 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Tue, 31 Oct 2023 09:33:49 -0400 Subject: [PATCH 7/9] more intuitive exclusions --- tests/test_extract.py | 6 ++++-- xscen/extract.py | 15 +++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/test_extract.py b/tests/test_extract.py index cd36f34a..940bf3a9 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -28,9 +28,11 @@ def test_basic(self, variables_and_freqs, other_arg): other_search_criteria={"experiment": ["ssp585"]} if other_arg == "other" else None, - exclusions={"member": "r2.*"} if other_arg == "exclusion" else None, + exclusions={"member": "r2.*", "domain": ["gr2"]} + if other_arg == "exclusion" + else None, ) - assert len(out) == 13 if other_arg is None else 2 if other_arg == "other" else 6 + assert len(out) == 13 if other_arg is None else 2 if other_arg == "other" else 4 @pytest.mark.parametrize( "periods, coverage_kwargs", diff --git a/xscen/extract.py b/xscen/extract.py index 8edea223..96a9f6df 100644 --- a/xscen/extract.py +++ b/xscen/extract.py @@ -616,7 +616,7 @@ def search_data_catalogs( You can also pass 'require_all_on: list(columns_name)' in order to only return results that correspond to all other criteria across the listed columns. More details available at https://intake-esm.readthedocs.io/en/stable/how-to/enforce-search-query-criteria-via-require-all-on.html . exclusions : dict, optional - Same as other_search_criteria, but for eliminating results. + Same as other_search_criteria, but for eliminating results. Any result that matches any of the exclusions will be removed. match_hist_and_fut: bool, optional If True, historical and future simulations will be combined into the same line, and search results lacking one of them will be rejected. periods : list @@ -712,11 +712,14 @@ def search_data_catalogs( # Cut entries that do not match search criteria if exclusions: - ex = catalog.search(**exclusions) - catalog.esmcat._df = pd.concat([catalog.df, ex.df]).drop_duplicates(keep=False) - logger.info( - f"Removing {len(ex.df)} assets based on exclusion dict : {exclusions}." - ) + for k in exclusions.keys(): + ex = catalog.search(**{k: exclusions[k]}) + catalog.esmcat._df = pd.concat([catalog.df, ex.df]).drop_duplicates( + keep=False + ) + logger.info( + f"Removing {len(ex.df)} assets based on exclusion dict '{k}': {exclusions[k]}." + ) full_catalog = deepcopy(catalog) # Used for searching for fixed fields if other_search_criteria: catalog = catalog.search(**other_search_criteria) From 72bb7dacba61adcd2e46572e3cced1364ea2a337 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Tue, 31 Oct 2023 09:37:47 -0400 Subject: [PATCH 8/9] upd HISTORY --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 2849ba51..9281407b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ Bug fixes * Fixed a bug in ``xs.search_data_catalogs`` when searching for fixed fields and specific experiments/members. (:pull:`251`). * Fixed a bug in the documentation build configuration that prevented stable/latest and tagged documentation builds from resolving on ReadTheDocs. (:pull:`256`). * Fixed ``get_warming_level`` to avoid incomplete matches. (:pull:`269`). +* `search_data_catalogs` now eliminates anything that matches any entry in `exclusions`. (:issue:`275`, :pull:`280`). Internal changes ^^^^^^^^^^^^^^^^ From 9deacee87a21c183a324f64dbe31dcc6975c0c43 Mon Sep 17 00:00:00 2001 From: "bumpversion[bot]" Date: Tue, 31 Oct 2023 17:00:53 +0000 Subject: [PATCH 9/9] =?UTF-8?q?Bump=20version:=200.7.16-beta=20=E2=86=92?= =?UTF-8?q?=200.7.17-beta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cruft.json | 2 +- setup.cfg | 2 +- setup.py | 2 +- tests/test_xscen.py | 2 +- xscen/__init__.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.cruft.json b/.cruft.json index 23d43b6a..86c1c175 100644 --- a/.cruft.json +++ b/.cruft.json @@ -11,7 +11,7 @@ "project_slug": "xscen", "project_short_description": "A climate change scenario-building analysis framework, built with xclim/xarray.", "pypi_username": "RondeauG", - "version": "0.7.16-beta", + "version": "0.7.17-beta", "use_pytest": "y", "use_black": "y", "add_pyup_badge": "n", diff --git a/setup.cfg b/setup.cfg index 8746bde5..6e3eb34e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.7.16-beta +current_version = 0.7.17-beta commit = True tag = False parse = (?P\d+)\.(?P\d+).(?P\d+)(\-(?P[a-z]+))? diff --git a/setup.py b/setup.py index 55cb1dae..d3ce6c68 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,6 @@ def run(self): test_suite="tests", extras_require={"dev": dev_requirements}, url="https://github.com/Ouranosinc/xscen", - version="0.7.16-beta", + version="0.7.17-beta", zip_safe=False, ) diff --git a/tests/test_xscen.py b/tests/test_xscen.py index d0186187..1b2772b1 100644 --- a/tests/test_xscen.py +++ b/tests/test_xscen.py @@ -28,4 +28,4 @@ def test_package_metadata(self): contents = f.read() assert """Gabriel Rondeau-Genesse""" in contents assert '__email__ = "rondeau-genesse.gabriel@ouranos.ca"' in contents - assert '__version__ = "0.7.16-beta"' in contents + assert '__version__ = "0.7.17-beta"' in contents diff --git a/xscen/__init__.py b/xscen/__init__.py index dc4ba6fe..77e63129 100644 --- a/xscen/__init__.py +++ b/xscen/__init__.py @@ -52,7 +52,7 @@ __author__ = """Gabriel Rondeau-Genesse""" __email__ = "rondeau-genesse.gabriel@ouranos.ca" -__version__ = "0.7.16-beta" +__version__ = "0.7.17-beta" # monkeypatch so that warnings.warn() doesn't mention itself