diff --git a/.cruft.json b/.cruft.json index a7dc6521..86c1c175 100644 --- a/.cruft.json +++ b/.cruft.json @@ -11,7 +11,7 @@ "project_slug": "xscen", "project_short_description": "A climate change scenario-building analysis framework, built with xclim/xarray.", "pypi_username": "RondeauG", - "version": "0.7.15-beta", + "version": "0.7.17-beta", "use_pytest": "y", "use_black": "y", "add_pyup_badge": "n", diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f9de583..0fe20b60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: rst-inline-touching-normal - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 23.10.1 hooks: - id: black exclude: ^docs/ @@ -53,7 +53,7 @@ repos: rev: v0.3.8 hooks: - id: blackdoc - additional_dependencies: [ 'black==23.9.1' ] + additional_dependencies: [ 'black==23.10.0' ] exclude: config.py - repo: https://github.com/adrienverge/yamllint.git rev: v1.32.0 @@ -65,10 +65,13 @@ repos: hooks: - id: nbqa-pyupgrade args: [ '--py39-plus' ] + additional_dependencies: [ 'pyupgrade==3.15.0' ] - id: nbqa-black args: [ '--target-version=py39' ] + additional_dependencies: [ 'black==23.10.0' ] - id: nbqa-isort args: [ "--settings-file=setup.cfg" ] + additional_dependencies: [ 'isort==5.12.0' ] - repo: https://github.com/kynan/nbstripout rev: 0.6.1 hooks: diff --git a/HISTORY.rst b/HISTORY.rst index e7cf612f..1a00b585 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ Bug fixes * Fixed a bug in ``xs.search_data_catalogs`` when searching for fixed fields and specific experiments/members. (:pull:`251`). * Fixed a bug in the documentation build configuration that prevented stable/latest and tagged documentation builds from resolving on ReadTheDocs. (:pull:`256`). * Fixed ``get_warming_level`` to avoid incomplete matches. (:pull:`269`). +* `search_data_catalogs` now eliminates anything that matches any entry in `exclusions`. (:issue:`275`, :pull:`280`). Internal changes ^^^^^^^^^^^^^^^^ diff --git a/docs/goodtoknow.rst b/docs/goodtoknow.rst index 6314b133..14682126 100644 --- a/docs/goodtoknow.rst +++ b/docs/goodtoknow.rst @@ -62,6 +62,8 @@ Which can also be activated in the code using :py:func:`xclim.core.options.set_o Translation is of course not automatic but relies on manually populated `gettext `_ catalogs. xscen ships with a catalog of french (fr) translations. See :ref:`translating-xscen` to learn how to add translations to xscen. xclim's documentation of the same subject is `here `_. +If your xscen is installed in "editable" mode in its source directory (``pip install -e .``), you should run ``make translate`` each time you pull changes from the upstream source. + Module-wide options ------------------- diff --git a/environment-dev.yml b/environment-dev.yml index f538c159..9650fe0b 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -11,7 +11,7 @@ dependencies: - clisops >=0.10 - dask - flox - - fsspec + - fsspec < 2023.10.0 - geopandas - h5netcdf - h5py diff --git a/environment.yml b/environment.yml index 9a70a030..4e3c9de5 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,7 @@ dependencies: - clisops >=0.10 - dask - flox - - fsspec + - fsspec < 2023.10.0 - geopandas - h5netcdf - h5py diff --git a/setup.cfg b/setup.cfg index 5a93f94e..3e41fe0e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.7.15-beta +current_version = 0.7.17-beta commit = True tag = False parse = (?P\d+)\.(?P\d+).(?P\d+)(\-(?P[a-z]+))? diff --git a/setup.py b/setup.py index 22140424..4d9cd48d 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ def run(self): "clisops>=0.10", "dask", "flox", - "fsspec", + "fsspec<2023.10.0", "geopandas", "h5netcdf", "h5py", @@ -119,6 +119,6 @@ def run(self): test_suite="tests", extras_require={"dev": dev_requirements, "docs": docs_requirements}, url="https://github.com/Ouranosinc/xscen", - version="0.7.15-beta", + version="0.7.17-beta", zip_safe=False, ) diff --git a/tests/test_extract.py b/tests/test_extract.py index cd36f34a..940bf3a9 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -28,9 +28,11 @@ def test_basic(self, variables_and_freqs, other_arg): other_search_criteria={"experiment": ["ssp585"]} if other_arg == "other" else None, - exclusions={"member": "r2.*"} if other_arg == "exclusion" else None, + exclusions={"member": "r2.*", "domain": ["gr2"]} + if other_arg == "exclusion" + else None, ) - assert len(out) == 13 if other_arg is None else 2 if other_arg == "other" else 6 + assert len(out) == 13 if other_arg is None else 2 if other_arg == "other" else 4 @pytest.mark.parametrize( "periods, coverage_kwargs", diff --git a/tests/test_xscen.py b/tests/test_xscen.py index 30c400e2..1b2772b1 100644 --- a/tests/test_xscen.py +++ b/tests/test_xscen.py @@ -28,4 +28,4 @@ def test_package_metadata(self): contents = f.read() assert """Gabriel Rondeau-Genesse""" in contents assert '__email__ = "rondeau-genesse.gabriel@ouranos.ca"' in contents - assert '__version__ = "0.7.15-beta"' in contents + assert '__version__ = "0.7.17-beta"' in contents diff --git a/xscen/__init__.py b/xscen/__init__.py index 55280871..77e63129 100644 --- a/xscen/__init__.py +++ b/xscen/__init__.py @@ -52,7 +52,7 @@ __author__ = """Gabriel Rondeau-Genesse""" __email__ = "rondeau-genesse.gabriel@ouranos.ca" -__version__ = "0.7.15-beta" +__version__ = "0.7.17-beta" # monkeypatch so that warnings.warn() doesn't mention itself diff --git a/xscen/extract.py b/xscen/extract.py index 8edea223..96a9f6df 100644 --- a/xscen/extract.py +++ b/xscen/extract.py @@ -616,7 +616,7 @@ def search_data_catalogs( You can also pass 'require_all_on: list(columns_name)' in order to only return results that correspond to all other criteria across the listed columns. More details available at https://intake-esm.readthedocs.io/en/stable/how-to/enforce-search-query-criteria-via-require-all-on.html . exclusions : dict, optional - Same as other_search_criteria, but for eliminating results. + Same as other_search_criteria, but for eliminating results. Any result that matches any of the exclusions will be removed. match_hist_and_fut: bool, optional If True, historical and future simulations will be combined into the same line, and search results lacking one of them will be rejected. periods : list @@ -712,11 +712,14 @@ def search_data_catalogs( # Cut entries that do not match search criteria if exclusions: - ex = catalog.search(**exclusions) - catalog.esmcat._df = pd.concat([catalog.df, ex.df]).drop_duplicates(keep=False) - logger.info( - f"Removing {len(ex.df)} assets based on exclusion dict : {exclusions}." - ) + for k in exclusions.keys(): + ex = catalog.search(**{k: exclusions[k]}) + catalog.esmcat._df = pd.concat([catalog.df, ex.df]).drop_duplicates( + keep=False + ) + logger.info( + f"Removing {len(ex.df)} assets based on exclusion dict '{k}': {exclusions[k]}." + ) full_catalog = deepcopy(catalog) # Used for searching for fixed fields if other_search_criteria: catalog = catalog.search(**other_search_criteria) diff --git a/xscen/utils.py b/xscen/utils.py index c7d931a8..ff19854c 100644 --- a/xscen/utils.py +++ b/xscen/utils.py @@ -55,12 +55,16 @@ If a language is not defined or a message not translated, the function will return the raw message. """ - -for loc in (Path(__file__).parent / "data").iterdir(): - if loc.is_dir() and len(loc.name) == 2: - TRANSLATOR[loc.name] = gettext.translation( - "xscen", localedir=loc.parent, languages=[loc.name] - ).gettext +try: + for loc in (Path(__file__).parent / "data").iterdir(): + if loc.is_dir() and len(loc.name) == 2: + TRANSLATOR[loc.name] = gettext.translation( + "xscen", localedir=loc.parent, languages=[loc.name] + ).gettext +except FileNotFoundError as err: + raise ImportError( + "Your xscen installation doesn't have compiled translations. Run `make translate` from the source directory to fix." + ) from err def update_attr(ds, attr, new, others=None, **fmt):