From c175626473f28319b1646adac57c2f6bdd4c26d7 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 18 Sep 2024 10:54:42 -0400 Subject: [PATCH 01/10] drop python3.9 --- .github/workflows/main.yml | 4 ---- .pre-commit-config.yaml | 6 +++--- CONTRIBUTING.rst | 2 +- environment-dev.yml | 2 +- environment.yml | 2 +- pyproject.toml | 13 +++++-------- 6 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b40b165..5b862c55 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -148,10 +148,6 @@ jobs: environment-file: environment-dev.yml create-args: >- python=${{ matrix.python-version }} - - name: Downgrade intake-esm - if: matrix.python-version == '3.9' - run: | - micromamba install -y -c conda-forge intake-esm=2023.11.10 - name: Conda and Mamba versions run: | echo "micromamba $(micromamba --version)" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ef7e837d..6157ff99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: rev: v3.17.0 hooks: - id: pyupgrade - args: [ '--py39-plus' ] + args: [ '--py310-plus' ] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: @@ -77,10 +77,10 @@ repos: rev: 1.8.5 hooks: - id: nbqa-pyupgrade - args: [ '--py39-plus' ] + args: [ '--py310-plus' ] additional_dependencies: [ 'pyupgrade==3.17.0' ] - id: nbqa-black - args: [ '--target-version=py39' ] + args: [ '--target-version=py310' ] additional_dependencies: [ 'black==24.8.0' ] - id: nbqa-isort additional_dependencies: [ 'isort==5.13.2' ] diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 52c2d478..35f6a7de 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -257,7 +257,7 @@ Before you submit a pull request, check that it meets these guidelines: #. The pull request should not break the templates. -#. The pull request should work for all currently supported Python versions. Check the `pyproject.toml` or `tox.ini` files for the list of supported versions. +#. The pull request should work for all currently supported Python versions. Check the `pyproject.toml` or `tox.ini` files for the list of supported versions. We aim to follow the support and drop schedule of Python versions as recommended by the NumPy NEP calendar: https://numpy.org/neps/nep-0029-deprecation_policy.html Tips ---- diff --git a/environment-dev.yml b/environment-dev.yml index 7d266d10..2d62b72c 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -2,7 +2,7 @@ name: xscen-dev channels: - conda-forge dependencies: - - python >=3.9,<3.13 + - python >=3.10,<3.13 # Don't forget to sync changes between environment.yml, environment-dev.yml, and pyproject.toml! # Also consider updating the list in xs.utils.show_versions if you add a new package. # Main packages diff --git a/environment.yml b/environment.yml index 7192487c..7fb1a9d0 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: xscen channels: - conda-forge dependencies: - - python >=3.9,<3.13 + - python >=3.10,<3.13 # Don't forget to sync changes between environment.yml, environment-dev.yml, and pyproject.toml! # Also consider updating the list in xs.utils.show_versions if you add a new package. # Main packages diff --git a/pyproject.toml b/pyproject.toml index dc682289..73ba1818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ maintainers = [ ] description = "A climate change scenario-building analysis framework, built with xclim/xarray." readme = "README.rst" -requires-python = ">=3.9.0" +requires-python = ">=3.10.0" keywords = ["xscen"] classifiers = [ "Development Status :: 4 - Beta", @@ -30,7 +30,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -50,8 +49,7 @@ dependencies = [ "geopandas", "h5netcdf", "h5py", - "intake-esm >=2023.07.07,=2023.07.07; python_version >= '3.10'", + "intake-esm >=2023.07.07", "matplotlib", "netCDF4 <1.7", "numcodecs", @@ -127,7 +125,6 @@ all = ["xscen[dev]", "xscen[docs]", "xscen[extra]"] [tool.black] target-version = [ - "py39", "py310", "py311", "py312" @@ -205,11 +202,11 @@ source = ["xscen"] append_only = true known_first_party = "xscen" profile = "black" -py_version = 39 +py_version = 310 [tool.mypy] files = "." -python_version = 3.9 +python_version = 3.10 show_error_codes = true strict = true warn_no_return = true @@ -261,7 +258,7 @@ markers = ["requires_netcdf: marks tests that require netcdf files to run"] [tool.ruff] src = ["src/xscen"] line-length = 150 -target-version = "py39" +target-version = "py310" exclude = [ ".eggs", ".git", From b28a853b5cb815083d631c4251da8b176f94f78a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:56:16 +0000 Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/xscen/aggregate.py | 58 +++++++++++++------------- src/xscen/biasadjust.py | 24 +++++------ src/xscen/catalog.py | 54 ++++++++++++------------ src/xscen/catutils.py | 88 ++++++++++++++++++++-------------------- src/xscen/config.py | 2 +- src/xscen/diagnostics.py | 52 ++++++++++++------------ src/xscen/ensembles.py | 40 +++++++++--------- src/xscen/extract.py | 74 ++++++++++++++++----------------- src/xscen/indicators.py | 38 ++++++++--------- src/xscen/io.py | 76 +++++++++++++++++----------------- src/xscen/reduce.py | 14 +++---- src/xscen/regrid.py | 12 +++--- src/xscen/scripting.py | 34 ++++++++-------- src/xscen/spatial.py | 20 ++++----- src/xscen/testing.py | 4 +- src/xscen/utils.py | 84 ++++++++++++++++++-------------------- 16 files changed, 330 insertions(+), 344 deletions(-) diff --git a/src/xscen/aggregate.py b/src/xscen/aggregate.py index 3110e883..6b87a13d 100644 --- a/src/xscen/aggregate.py +++ b/src/xscen/aggregate.py @@ -51,11 +51,11 @@ def _(s): def climatological_mean( ds: xr.Dataset, *, - window: Optional[int] = None, - min_periods: Optional[int] = None, + window: int | None = None, + min_periods: int | None = None, interval: int = 1, - periods: Optional[Union[list[str], list[list[str]]]] = None, - to_level: Optional[str] = "climatology", + periods: list[str] | list[list[str]] | None = None, + to_level: str | None = "climatology", ) -> xr.Dataset: """Compute the mean over 'year' for given time periods, respecting the temporal resolution of ds. @@ -109,11 +109,11 @@ def climatological_mean( def climatological_op( # noqa: C901 ds: xr.Dataset, *, - op: Union[str, dict] = "mean", - window: Optional[int] = None, - min_periods: Optional[Union[int, float]] = None, + op: str | dict = "mean", + window: int | None = None, + min_periods: int | float | None = None, stride: int = 1, - periods: Optional[Union[list[str], list[list[str]]]] = None, + periods: list[str] | list[list[str]] | None = None, rename_variables: bool = True, to_level: str = "climatology", horizons_as_dim: bool = False, @@ -506,11 +506,11 @@ def _ulinregress(x, y, **kwargs): @parse_config def compute_deltas( # noqa: C901 ds: xr.Dataset, - reference_horizon: Union[str, xr.Dataset], + reference_horizon: str | xr.Dataset, *, - kind: Union[str, dict] = "+", + kind: str | dict = "+", rename_variables: bool = True, - to_level: Optional[str] = "deltas", + to_level: str | None = "deltas", ) -> xr.Dataset: """Compute deltas in comparison to a reference time period, respecting the temporal resolution of ds. @@ -702,13 +702,13 @@ def spatial_mean( # noqa: C901 ds: xr.Dataset, method: str, *, - spatial_subset: Optional[bool] = None, - call_clisops: Optional[bool] = False, - region: Optional[Union[dict, str]] = None, - kwargs: Optional[dict] = None, - simplify_tolerance: Optional[float] = None, - to_domain: Optional[str] = None, - to_level: Optional[str] = None, + spatial_subset: bool | None = None, + call_clisops: bool | None = False, + region: dict | str | None = None, + kwargs: dict | None = None, + simplify_tolerance: float | None = None, + to_domain: str | None = None, + to_level: str | None = None, ) -> xr.Dataset: """Compute the spatial mean using a variety of available methods. @@ -1034,18 +1034,18 @@ def spatial_mean( # noqa: C901 @parse_config def produce_horizon( # noqa: C901 ds: xr.Dataset, - indicators: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], + indicators: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), *, - periods: Optional[Union[list[str], list[list[str]]]] = None, - warminglevels: Optional[dict] = None, - to_level: Optional[str] = "horizons", - period: Optional[list] = None, + periods: list[str] | list[list[str]] | None = None, + warminglevels: dict | None = None, + to_level: str | None = "horizons", + period: list | None = None, ) -> xr.Dataset: """ Compute indicators, then the climatological mean, and finally unstack dates in order diff --git a/src/xscen/biasadjust.py b/src/xscen/biasadjust.py index 5a7362a5..8359c62b 100644 --- a/src/xscen/biasadjust.py +++ b/src/xscen/biasadjust.py @@ -58,17 +58,17 @@ def _add_preprocessing_attr(scen, train_kwargs): def train( dref: xr.Dataset, dhist: xr.Dataset, - var: Union[str, list[str]], + var: str | list[str], period: list[str], *, method: str = "DetrendedQuantileMapping", - group: Optional[Union[sdba.Grouper, str, dict]] = None, - xclim_train_args: Optional[dict] = None, + group: sdba.Grouper | str | dict | None = None, + xclim_train_args: dict | None = None, maximal_calendar: str = "noleap", - adapt_freq: Optional[dict] = None, - jitter_under: Optional[dict] = None, - jitter_over: Optional[dict] = None, - align_on: Optional[str] = "year", + adapt_freq: dict | None = None, + jitter_under: dict | None = None, + jitter_over: dict | None = None, + align_on: str | None = "year", ) -> xr.Dataset: """ Train a bias-adjustment. @@ -194,13 +194,13 @@ def train( def adjust( dtrain: xr.Dataset, dsim: xr.Dataset, - periods: Union[list[str], list[list[str]]], + periods: list[str] | list[list[str]], *, - xclim_adjust_args: Optional[dict] = None, + xclim_adjust_args: dict | None = None, to_level: str = "biasadjusted", - bias_adjust_institution: Optional[str] = None, - bias_adjust_project: Optional[str] = None, - align_on: Optional[str] = "year", + bias_adjust_institution: str | None = None, + bias_adjust_project: str | None = None, + align_on: str | None = "year", ) -> xr.Dataset: """ Adjust a simulation. diff --git a/src/xscen/catalog.py b/src/xscen/catalog.py index 788e4f79..adc77654 100644 --- a/src/xscen/catalog.py +++ b/src/xscen/catalog.py @@ -198,10 +198,10 @@ def __init__( @classmethod def from_df( cls, - data: Union[pd.DataFrame, os.PathLike, Sequence[os.PathLike]], - esmdata: Optional[Union[os.PathLike, dict]] = None, + data: pd.DataFrame | os.PathLike | Sequence[os.PathLike], + esmdata: os.PathLike | dict | None = None, *, - read_csv_kwargs: Optional[Mapping[str, Any]] = None, + read_csv_kwargs: Mapping[str, Any] | None = None, name: str = "virtual", **intake_kwargs, ): @@ -263,7 +263,7 @@ def _find_unique(series): else: return data.apply(_find_unique, result_type="reduce").to_dict() - def unique(self, columns: Optional[Union[str, Sequence[str]]] = None): + def unique(self, columns: str | Sequence[str] | None = None): """Return a series of unique values in the catalog. Parameters @@ -309,7 +309,7 @@ def search(self, **columns): ) return cat - def drop_duplicates(self, columns: Optional[list[str]] = None): + def drop_duplicates(self, columns: list[str] | None = None): """Drop duplicates in the catalog based on a subset of columns. Parameters @@ -404,10 +404,10 @@ def exists_in_cat(self, **columns) -> bool: def to_dataset( self, - concat_on: Optional[Union[list[str], str]] = None, - create_ensemble_on: Optional[Union[list[str], str]] = None, - ensemble_name: Optional[Union[list[str]]] = None, - calendar: Optional[str] = "standard", + concat_on: list[str] | str | None = None, + create_ensemble_on: list[str] | str | None = None, + ensemble_name: list[str] | None = None, + calendar: str | None = "standard", **kwargs, ) -> xr.Dataset: """ @@ -538,7 +538,7 @@ def preprocess(ds): def copy_files( self, - dest: Union[str, os.PathLike], + dest: str | os.PathLike, flat: bool = True, unzip: bool = False, zipzarr: bool = False, @@ -636,9 +636,9 @@ class ProjectCatalog(DataCatalog): @classmethod def create( cls, - filename: Union[os.PathLike, str], + filename: os.PathLike | str, *, - project: Optional[dict] = None, + project: dict | None = None, overwrite: bool = False, ): r"""Create a new project catalog from some project metadata. @@ -716,11 +716,11 @@ def create( def __init__( self, - df: Union[str, dict], + df: str | dict, *args, create: bool = False, overwrite: bool = False, - project: Optional[dict] = None, + project: dict | None = None, **kwargs, ): """ @@ -756,15 +756,13 @@ def __init__( # TODO: Implement a way to easily destroy part of the catalog to "reset" some steps def update( self, - df: Optional[ - Union[ - DataCatalog, - intake_esm.esm_datastore, - pd.DataFrame, - pd.Series, - Sequence[pd.Series], - ] - ] = None, + df: None | ( + DataCatalog + | intake_esm.esm_datastore + | pd.DataFrame + | pd.Series + | Sequence[pd.Series] + ) = None, ): """Update the catalog with new data and writes the new data to the csv file. @@ -846,8 +844,8 @@ def update( def update_from_ds( self, ds: xr.Dataset, - path: Union[os.PathLike, str], - info_dict: Optional[dict] = None, + path: os.PathLike | str, + info_dict: dict | None = None, **info_kwargs, ): """Update the catalog with new data and writes the new data to the csv file. @@ -965,7 +963,7 @@ def _build_id(element: pd.Series, columns: list[str]): def generate_id( - df: Union[pd.DataFrame, xr.Dataset], id_columns: Optional[list] = None + df: pd.DataFrame | xr.Dataset, id_columns: list | None = None ) -> pd.Series: """Create an ID from column entries. @@ -996,7 +994,7 @@ def generate_id( return df.apply(_build_id, axis=1, args=(id_columns,)) -def unstack_id(df: Union[pd.DataFrame, ProjectCatalog, DataCatalog]) -> dict: +def unstack_id(df: pd.DataFrame | ProjectCatalog | DataCatalog) -> dict: """Reverse-engineer an ID using catalog entries. Parameters @@ -1038,7 +1036,7 @@ def unstack_id(df: Union[pd.DataFrame, ProjectCatalog, DataCatalog]) -> dict: def subset_file_coverage( df: pd.DataFrame, - periods: Union[list[str], list[list[str]]], + periods: list[str] | list[list[str]], *, coverage: float = 0.99, duplicates_ok: bool = False, diff --git a/src/xscen/catutils.py b/src/xscen/catutils.py index 12e875be..d36b09e1 100644 --- a/src/xscen/catutils.py +++ b/src/xscen/catutils.py @@ -110,7 +110,7 @@ def _parse_level(text: str) -> str: ) def _parse_datebounds( text: str, -) -> Union[list[str], tuple[None, None], tuple[str, str]]: +) -> list[str] | tuple[None, None] | tuple[str, str]: """Parse helper to translate date bounds, used in the special DATES field.""" if "-" in text: return text.split("-") @@ -120,10 +120,10 @@ def _parse_datebounds( def _find_assets( - root: Union[str, os.PathLike], + root: str | os.PathLike, exts: set[str], lengths: set[int], - dirglob: Optional[str] = None, + dirglob: str | None = None, ): """Walk recursively over files in a directory, filtering according to a glob pattern, path depth and extensions. @@ -191,13 +191,13 @@ def _compile_pattern(pattern: str) -> parse.Parser: def _name_parser( - path: Union[os.PathLike, str], - root: Union[os.PathLike, str], - patterns: list[Union[str, parse.Parser]], - read_from_file: Optional[Union[list[str], dict]] = None, - attrs_map: Optional[dict] = None, - xr_open_kwargs: Optional[dict] = None, -) -> Optional[dict]: + path: os.PathLike | str, + root: os.PathLike | str, + patterns: list[str | parse.Parser], + read_from_file: list[str] | dict | None = None, + attrs_map: dict | None = None, + xr_open_kwargs: dict | None = None, +) -> dict | None: """Extract metadata information from the file path. Parameters @@ -267,13 +267,13 @@ def _name_parser( def _parse_dir( # noqa: C901 - root: Union[os.PathLike, str], + root: os.PathLike | str, patterns: list[str], - dirglob: Optional[str] = None, - checks: Optional[list[str]] = None, - read_from_file: Optional[Union[list[str], dict]] = None, - attrs_map: Optional[dict] = None, - xr_open_kwargs: Optional[dict] = None, + dirglob: str | None = None, + checks: list[str] | None = None, + read_from_file: list[str] | dict | None = None, + attrs_map: dict | None = None, + xr_open_kwargs: dict | None = None, progress: bool = False, ): """Iterate and parses files in a directory, filtering according to basic pattern properties and optional checks. @@ -452,24 +452,24 @@ def _parse_first_ds( @parse_config def parse_directory( # noqa: C901 - directories: Union[str, list[Union[str, os.PathLike]]], + directories: str | list[str | os.PathLike], patterns: list[str], *, - id_columns: Optional[list[str]] = None, - read_from_file: Union[ - bool, - Sequence[str], - tuple[Sequence[str], Sequence[str]], - Sequence[tuple[Sequence[str], Sequence[str]]], - ] = False, - homogenous_info: Optional[dict] = None, - cvs: Optional[Union[str, os.PathLike, dict]] = None, - dirglob: Optional[str] = None, - xr_open_kwargs: Optional[Mapping[str, Any]] = None, + id_columns: list[str] | None = None, + read_from_file: ( + bool + | Sequence[str] + | tuple[Sequence[str], Sequence[str]] + | Sequence[tuple[Sequence[str], Sequence[str]]] + ) = False, + homogenous_info: dict | None = None, + cvs: str | os.PathLike | dict | None = None, + dirglob: str | None = None, + xr_open_kwargs: Mapping[str, Any] | None = None, only_official_columns: bool = True, progress: bool = False, - parallel_dirs: Union[bool, int] = False, - file_checks: Optional[list[str]] = None, + parallel_dirs: bool | int = False, + file_checks: list[str] | None = None, ) -> pd.DataFrame: r"""Parse files in a directory and return them as a pd.DataFrame. @@ -726,9 +726,9 @@ def parse_directory( # noqa: C901 def parse_from_ds( # noqa: C901 - obj: Union[str, os.PathLike, xr.Dataset], + obj: str | os.PathLike | xr.Dataset, names: Sequence[str], - attrs_map: Optional[Mapping[str, str]] = None, + attrs_map: Mapping[str, str] | None = None, **xrkwargs, ): """Parse a list of catalog fields from the file/dataset itself. @@ -818,7 +818,7 @@ def parse_from_ds( # noqa: C901 def _parse_from_zarr( - path: Union[os.PathLike, str], get_vars: bool = True, get_time: bool = True + path: os.PathLike | str, get_vars: bool = True, get_time: bool = True ): """Obtain the list of variables, the time coordinate and the list of global attributes from a zarr dataset. @@ -881,7 +881,7 @@ def _parse_from_zarr( def _parse_from_nc( - path: Union[os.PathLike, str], get_vars: bool = True, get_time: bool = True + path: os.PathLike | str, get_vars: bool = True, get_time: bool = True ): """Obtain the list of variables, the time coordinate, and the list of global attributes from a netCDF dataset, using netCDF4. @@ -934,7 +934,7 @@ def _schema_option(option: dict, facets: dict): return answer -def _schema_level(schema: Union[dict, list[str], str], facets: dict): +def _schema_level(schema: dict | list[str] | str, facets: dict): if isinstance(schema, str): if schema.startswith("(") and schema.endswith(")"): optional = True @@ -1060,12 +1060,12 @@ def _read_schemas(schemas): def _build_path( - data: Union[dict, xr.Dataset, xr.DataArray, pd.Series], + data: dict | xr.Dataset | xr.DataArray | pd.Series, schemas: dict, - root: Union[str, os.PathLike], + root: str | os.PathLike, get_type: bool = False, **extra_facets, -) -> Union[Path, tuple[Path, str]]: +) -> Path | tuple[Path, str]: # Get all known metadata if isinstance(data, (xr.Dataset, xr.DataArray)): facets = ( @@ -1127,11 +1127,11 @@ def _build_path( @parse_config def build_path( - data: Union[dict, xr.Dataset, xr.DataArray, pd.Series, DataCatalog, pd.DataFrame], - schemas: Optional[Union[str, os.PathLike, dict]] = None, - root: Optional[Union[str, os.PathLike]] = None, + data: dict | xr.Dataset | xr.DataArray | pd.Series | DataCatalog | pd.DataFrame, + schemas: str | os.PathLike | dict | None = None, + root: str | os.PathLike | None = None, **extra_facets, -) -> Union[Path, DataCatalog, pd.DataFrame]: +) -> Path | DataCatalog | pd.DataFrame: r"""Parse the schema from a configuration and construct path using a dictionary of facets. Parameters @@ -1210,9 +1210,7 @@ def __missing__(self, key): return template.format_map(PartialFormatDict(**fmtargs)) -def patterns_from_schema( - schema: Union[str, dict], exts: Optional[Sequence[str]] = None -): +def patterns_from_schema(schema: str | dict, exts: Sequence[str] | None = None): """Generate all valid patterns for a given schema. Generated patterns are meant for use with :py:func:`parse_directory`. diff --git a/src/xscen/config.py b/src/xscen/config.py index 06a4eeb7..a2f556c3 100644 --- a/src/xscen/config.py +++ b/src/xscen/config.py @@ -133,7 +133,7 @@ def args_as_str(*args: tuple[Any, ...]) -> tuple[str, ...]: def load_config( *elements, reset: bool = False, - encoding: Optional[str] = None, + encoding: str | None = None, verbose: bool = False, ): """Load configuration from given files or key=value pairs. diff --git a/src/xscen/diagnostics.py b/src/xscen/diagnostics.py index 3af73970..ec7c2916 100644 --- a/src/xscen/diagnostics.py +++ b/src/xscen/diagnostics.py @@ -44,21 +44,21 @@ def _(s): @parse_config def health_checks( # noqa: C901 - ds: Union[xr.Dataset, xr.DataArray], + ds: xr.Dataset | xr.DataArray, *, - structure: Optional[dict] = None, - calendar: Optional[str] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - variables_and_units: Optional[dict] = None, - cfchecks: Optional[dict] = None, - freq: Optional[str] = None, - missing: Optional[Union[dict, str, list]] = None, - flags: Optional[dict] = None, - flags_kwargs: Optional[dict] = None, + structure: dict | None = None, + calendar: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + variables_and_units: dict | None = None, + cfchecks: dict | None = None, + freq: str | None = None, + missing: dict | str | list | None = None, + flags: dict | None = None, + flags_kwargs: dict | None = None, return_flags: bool = False, - raise_on: Optional[list] = None, -) -> Union[None, xr.Dataset]: + raise_on: list | None = None, +) -> None | xr.Dataset: """ Perform a series of health checks on the dataset. Be aware that missing data checks and flag checks can be slow. @@ -304,18 +304,18 @@ def _message(): @parse_config def properties_and_measures( # noqa: C901 ds: xr.Dataset, - properties: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], - period: Optional[list[str]] = None, + properties: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), + period: list[str] | None = None, unstack: bool = False, - rechunk: Optional[dict] = None, - dref_for_measure: Optional[xr.Dataset] = None, - change_units_arg: Optional[dict] = None, + rechunk: dict | None = None, + dref_for_measure: xr.Dataset | None = None, + change_units_arg: dict | None = None, to_level_prop: str = "diag-properties", to_level_meas: str = "diag-measures", ) -> tuple[xr.Dataset, xr.Dataset]: @@ -446,7 +446,7 @@ def properties_and_measures( # noqa: C901 def measures_heatmap( - meas_datasets: Union[list[xr.Dataset], dict], to_level: str = "diag-heatmap" + meas_datasets: list[xr.Dataset] | dict, to_level: str = "diag-heatmap" ) -> xr.Dataset: """Create a heatmap to compare the performance of the different datasets. @@ -528,7 +528,7 @@ def measures_heatmap( def measures_improvement( - meas_datasets: Union[list[xr.Dataset], dict], to_level: str = "diag-improved" + meas_datasets: list[xr.Dataset] | dict, to_level: str = "diag-improved" ) -> xr.Dataset: """ Calculate the fraction of improved grid points for each property between two datasets of measures. diff --git a/src/xscen/ensembles.py b/src/xscen/ensembles.py index 6eeddc0c..3d7cddd2 100644 --- a/src/xscen/ensembles.py +++ b/src/xscen/ensembles.py @@ -31,17 +31,17 @@ @parse_config def ensemble_stats( # noqa: C901 - datasets: Union[ - dict, - list[Union[str, os.PathLike]], - list[xr.Dataset], - list[xr.DataArray], - xr.Dataset, - ], + datasets: ( + dict + | list[str | os.PathLike] + | list[xr.Dataset] + | list[xr.DataArray] + | xr.Dataset + ), statistics: dict, *, - create_kwargs: Optional[dict] = None, - weights: Optional[xr.DataArray] = None, + create_kwargs: dict | None = None, + weights: xr.DataArray | None = None, common_attrs_only: bool = True, to_level: str = "ensemble", ) -> xr.Dataset: @@ -245,13 +245,13 @@ def ensemble_stats( # noqa: C901 def generate_weights( # noqa: C901 - datasets: Union[dict, list], + datasets: dict | list, *, independence_level: str = "model", balance_experiments: bool = False, - attribute_weights: Optional[dict] = None, + attribute_weights: dict | None = None, skipna: bool = True, - v_for_skipna: Optional[str] = None, + v_for_skipna: str | None = None, standardize: bool = False, experiment_weights: bool = False, ) -> xr.DataArray: @@ -673,12 +673,12 @@ def generate_weights( # noqa: C901 def build_partition_data( - datasets: Union[dict, list[xr.Dataset]], + datasets: dict | list[xr.Dataset], partition_dim: list[str] = ["source", "experiment", "bias_adjust_project"], - subset_kw: Optional[dict] = None, - regrid_kw: Optional[dict] = None, - indicators_kw: Optional[dict] = None, - rename_dict: Optional[dict] = None, + subset_kw: dict | None = None, + regrid_kw: dict | None = None, + indicators_kw: dict | None = None, + rename_dict: dict | None = None, ): """ Get the input for the xclim partition functions. @@ -765,11 +765,11 @@ def build_partition_data( @parse_config def reduce_ensemble( - data: Union[xr.DataArray, dict, list, xr.Dataset], + data: xr.DataArray | dict | list | xr.Dataset, method: str, *, - horizons: Optional[list[str]] = None, - create_kwargs: Optional[dict] = None, + horizons: list[str] | None = None, + create_kwargs: dict | None = None, **kwargs, ): r"""Reduce an ensemble of simulations using clustering algorithms from xclim.ensembles. diff --git a/src/xscen/extract.py b/src/xscen/extract.py index 322ea242..c036b058 100644 --- a/src/xscen/extract.py +++ b/src/xscen/extract.py @@ -6,10 +6,10 @@ import re import warnings from collections import defaultdict -from collections.abc import Sequence +from collections.abc import Callable, Sequence from copy import deepcopy from pathlib import Path -from typing import Callable, Optional, Union +from typing import Optional, Union import numpy as np import pandas as pd @@ -49,16 +49,16 @@ def extract_dataset( # noqa: C901 catalog: DataCatalog, *, - variables_and_freqs: Optional[dict] = None, - periods: Optional[Union[list[str], list[list[str]]]] = None, - region: Optional[dict] = None, + variables_and_freqs: dict | None = None, + periods: list[str] | list[list[str]] | None = None, + region: dict | None = None, to_level: str = "extracted", ensure_correct_time: bool = True, - xr_open_kwargs: Optional[dict] = None, - xr_combine_kwargs: Optional[dict] = None, - preprocess: Optional[Callable] = None, - resample_methods: Optional[dict] = None, - mask: Union[bool, xr.Dataset, xr.DataArray] = False, + xr_open_kwargs: dict | None = None, + xr_combine_kwargs: dict | None = None, + preprocess: Callable | None = None, + resample_methods: dict | None = None, + mask: bool | xr.Dataset | xr.DataArray = False, ) -> dict: """ Take one element of the output of `search_data_catalogs` and returns a dataset, @@ -322,9 +322,9 @@ def resample( # noqa: C901 da: xr.DataArray, target_frequency: str, *, - ds: Optional[xr.Dataset] = None, - method: Optional[str] = None, - missing: Optional[Union[str, dict]] = None, + ds: xr.Dataset | None = None, + method: str | None = None, + missing: str | dict | None = None, ) -> xr.DataArray: """Aggregate variable to the target frequency. @@ -542,23 +542,23 @@ def resample( # noqa: C901 @parse_config def search_data_catalogs( # noqa: C901 - data_catalogs: Union[ - str, os.PathLike, DataCatalog, list[Union[str, os.PathLike, DataCatalog]] - ], + data_catalogs: ( + str | os.PathLike | DataCatalog | list[str | os.PathLike | DataCatalog] + ), variables_and_freqs: dict, *, - other_search_criteria: Optional[dict] = None, - exclusions: Optional[dict] = None, + other_search_criteria: dict | None = None, + exclusions: dict | None = None, match_hist_and_fut: bool = False, - periods: Optional[Union[list[str], list[list[str]]]] = None, - coverage_kwargs: Optional[dict] = None, - id_columns: Optional[list[str]] = None, + periods: list[str] | list[list[str]] | None = None, + coverage_kwargs: dict | None = None, + id_columns: list[str] | None = None, allow_resampling: bool = False, allow_conversion: bool = False, - conversion_yaml: Optional[str] = None, - restrict_resolution: Optional[str] = None, - restrict_members: Optional[dict] = None, - restrict_warming_level: Optional[Union[dict, bool]] = None, + conversion_yaml: str | None = None, + restrict_resolution: str | None = None, + restrict_members: dict | None = None, + restrict_warming_level: dict | bool | None = None, ) -> dict: """Search through DataCatalogs. @@ -884,17 +884,17 @@ def search_data_catalogs( # noqa: C901 @parse_config def get_warming_level( # noqa: C901 - realization: Union[ - xr.Dataset, xr.DataArray, dict, pd.Series, pd.DataFrame, str, list - ], + realization: ( + xr.Dataset | xr.DataArray | dict | pd.Series | pd.DataFrame | str | list + ), wl: float, *, window: int = 20, - tas_baseline_period: Optional[Sequence[str]] = None, + tas_baseline_period: Sequence[str] | None = None, ignore_member: bool = False, - tas_src: Optional[Union[str, os.PathLike]] = None, + tas_src: str | os.PathLike | None = None, return_horizon: bool = True, -) -> Union[dict, list[str], str]: +) -> dict | list[str] | str: """ Use the IPCC Atlas method to return the window of time over which the requested level of global warming is first reached. @@ -1072,11 +1072,11 @@ def _get_warming_level(model): @parse_config def subset_warming_level( ds: xr.Dataset, - wl: Union[float, Sequence[float]], + wl: float | Sequence[float], to_level: str = "warminglevel-{wl}vs{period0}-{period1}", - wl_dim: Union[str, bool] = "+{wl}Cvs{period0}-{period1}", + wl_dim: str | bool = "+{wl}Cvs{period0}-{period1}", **kwargs, -) -> Optional[xr.Dataset]: +) -> xr.Dataset | None: r""" Subsets the input dataset with only the window of time over which the requested level of global warming is first reached, using the IPCC Atlas method. @@ -1273,7 +1273,7 @@ def subset_warming_level( def _dispatch_historical_to_future( - catalog: DataCatalog, id_columns: Optional[list[str]] = None + catalog: DataCatalog, id_columns: list[str] | None = None ) -> DataCatalog: """Update a DataCatalog by recopying each "historical" entry to its corresponding future experiments. @@ -1387,7 +1387,7 @@ def _dispatch_historical_to_future( def _restrict_by_resolution( - catalogs: dict, restrictions: str, id_columns: Optional[list[str]] = None + catalogs: dict, restrictions: str, id_columns: list[str] | None = None ) -> dict: """Update the results from search_data_catalogs by removing simulations with multiple resolutions available. @@ -1527,7 +1527,7 @@ def _restrict_by_resolution( def _restrict_multimembers( - catalogs: dict, restrictions: dict, id_columns: Optional[list[str]] = None + catalogs: dict, restrictions: dict, id_columns: list[str] | None = None ): """Update the results from search_data_catalogs by removing simulations with multiple members available. diff --git a/src/xscen/indicators.py b/src/xscen/indicators.py index da156beb..06fe8db0 100644 --- a/src/xscen/indicators.py +++ b/src/xscen/indicators.py @@ -26,9 +26,7 @@ __all__ = ["compute_indicators", "load_xclim_module", "registry_from_module"] -def load_xclim_module( - filename: Union[str, os.PathLike], reload: bool = False -) -> ModuleType: +def load_xclim_module(filename: str | os.PathLike, reload: bool = False) -> ModuleType: """Return the xclim module described by the yaml file (or group of yaml, jsons and py). Parameters @@ -103,17 +101,17 @@ def get_indicator_outputs(ind: xc.core.indicator.Indicator, in_freq: str): @parse_config def compute_indicators( # noqa: C901 ds: xr.Dataset, - indicators: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], + indicators: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), *, - periods: Optional[Union[list[str], list[list[str]]]] = None, + periods: list[str] | list[list[str]] | None = None, restrict_years: bool = True, - to_level: Optional[str] = "indicators", + to_level: str | None = "indicators", rechunk_input: bool = False, ) -> dict: """Calculate variables and indicators based on a YAML call to xclim. @@ -284,7 +282,7 @@ def compute_indicators( # noqa: C901 def registry_from_module( module: ModuleType, - registry: Optional[DerivedVariableRegistry] = None, + registry: DerivedVariableRegistry | None = None, variable_column: str = "variable", ) -> DerivedVariableRegistry: """Convert a xclim virtual indicators module to an intake_esm Derived Variable Registry. @@ -338,13 +336,13 @@ def func(ds, *, ind, nout): def select_inds_for_avail_vars( ds: xr.Dataset, - indicators: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], + indicators: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), ) -> ModuleType: """Filter the indicators for which the necessary variables are available. diff --git a/src/xscen/io.py b/src/xscen/io.py index a3ed9259..ae6198c1 100644 --- a/src/xscen/io.py +++ b/src/xscen/io.py @@ -49,7 +49,7 @@ ] -def get_engine(file: Union[str, os.PathLike]) -> str: +def get_engine(file: str | os.PathLike) -> str: """Determine which Xarray engine should be used to open the given file. The .zarr, .zarr.zip and .zip extensions are recognized as Zarr datasets, @@ -77,7 +77,7 @@ def get_engine(file: Union[str, os.PathLike]) -> str: def estimate_chunks( # noqa: C901 - ds: Union[str, os.PathLike, xr.Dataset], + ds: str | os.PathLike | xr.Dataset, dims: list, target_mb: float = 50, chunk_per_variable: bool = False, @@ -263,7 +263,7 @@ def subset_maxsize( ) -def clean_incomplete(path: Union[str, os.PathLike], complete: Sequence[str]) -> None: +def clean_incomplete(path: str | os.PathLike, complete: Sequence[str]) -> None: """Delete un-catalogued variables from a zarr folder. The goal of this function is to clean up an incomplete calculation. @@ -335,7 +335,7 @@ def round_bits(da: xr.DataArray, keepbits: int): return da -def _get_keepbits(bitround: Union[bool, int, dict], varname: str, vartype): +def _get_keepbits(bitround: bool | int | dict, varname: str, vartype): # Guess the number of bits to keep depending on how bitround was passed, the var dtype and the var name. if not np.issubdtype(vartype, np.floating) or bitround is False: if isinstance(bitround, dict) and varname in bitround: @@ -355,12 +355,12 @@ def _get_keepbits(bitround: Union[bool, int, dict], varname: str, vartype): @parse_config def save_to_netcdf( ds: xr.Dataset, - filename: Union[str, os.PathLike], + filename: str | os.PathLike, *, - rechunk: Optional[dict] = None, - bitround: Union[bool, int, dict] = False, + rechunk: dict | None = None, + bitround: bool | int | dict = False, compute: bool = True, - netcdf_kwargs: Optional[dict] = None, + netcdf_kwargs: dict | None = None, ): """Save a Dataset to NetCDF, rechunking or compressing if requested. @@ -422,13 +422,13 @@ def save_to_netcdf( @parse_config def save_to_zarr( # noqa: C901 ds: xr.Dataset, - filename: Union[str, os.PathLike], + filename: str | os.PathLike, *, - rechunk: Optional[dict] = None, - zarr_kwargs: Optional[dict] = None, + rechunk: dict | None = None, + zarr_kwargs: dict | None = None, compute: bool = True, - encoding: Optional[dict] = None, - bitround: Union[bool, int, dict] = False, + encoding: dict | None = None, + bitround: bool | int | dict = False, mode: str = "f", itervar: bool = False, timeout_cleanup: bool = True, @@ -638,13 +638,13 @@ def _to_dataframe( def to_table( - ds: Union[xr.Dataset, xr.DataArray], + ds: xr.Dataset | xr.DataArray, *, - row: Optional[Union[str, Sequence[str]]] = None, - column: Optional[Union[str, Sequence[str]]] = None, - sheet: Optional[Union[str, Sequence[str]]] = None, - coords: Union[bool, str, Sequence[str]] = True, -) -> Union[pd.DataFrame, dict]: + row: str | Sequence[str] | None = None, + column: str | Sequence[str] | None = None, + sheet: str | Sequence[str] | None = None, + coords: bool | str | Sequence[str] = True, +) -> pd.DataFrame | dict: """Convert a dataset to a pandas DataFrame with support for multicolumns and multisheet. This function will trigger a computation of the dataset. @@ -737,9 +737,7 @@ def _ensure_list(seq): return _to_dataframe(da, **table_kwargs) -def make_toc( - ds: Union[xr.Dataset, xr.DataArray], loc: Optional[str] = None -) -> pd.DataFrame: +def make_toc(ds: xr.Dataset | xr.DataArray, loc: str | None = None) -> pd.DataFrame: """Make a table of content describing a dataset's variables. This return a simple DataFrame with variable names as index, the long_name as "description" and units. @@ -785,17 +783,17 @@ def make_toc( def save_to_table( - ds: Union[xr.Dataset, xr.DataArray], - filename: Union[str, os.PathLike], - output_format: Optional[str] = None, + ds: xr.Dataset | xr.DataArray, + filename: str | os.PathLike, + output_format: str | None = None, *, - row: Optional[Union[str, Sequence[str]]] = None, - column: Union[None, str, Sequence[str]] = "variable", - sheet: Optional[Union[str, Sequence[str]]] = None, - coords: Union[bool, Sequence[str]] = True, + row: str | Sequence[str] | None = None, + column: None | str | Sequence[str] = "variable", + sheet: str | Sequence[str] | None = None, + coords: bool | Sequence[str] = True, col_sep: str = "_", - row_sep: Optional[str] = None, - add_toc: Union[bool, pd.DataFrame] = False, + row_sep: str | None = None, + add_toc: bool | pd.DataFrame = False, **kwargs, ): """Save the dataset to a tabular file (csv, excel, ...). @@ -933,13 +931,13 @@ def rechunk_for_saving(ds: xr.Dataset, rechunk: dict): @parse_config def rechunk( - path_in: Union[os.PathLike, str, xr.Dataset], - path_out: Union[os.PathLike, str], + path_in: os.PathLike | str | xr.Dataset, + path_out: os.PathLike | str, *, - chunks_over_var: Optional[dict] = None, - chunks_over_dim: Optional[dict] = None, + chunks_over_var: dict | None = None, + chunks_over_dim: dict | None = None, worker_mem: str, - temp_store: Optional[Union[os.PathLike, str]] = None, + temp_store: os.PathLike | str | None = None, overwrite: bool = False, ) -> None: """Rechunk a dataset into a new zarr. @@ -1007,8 +1005,8 @@ def rechunk( def zip_directory( - root: Union[str, os.PathLike], - zipfile: Union[str, os.PathLike], + root: str | os.PathLike, + zipfile: str | os.PathLike, delete: bool = False, **zip_args, ): @@ -1042,7 +1040,7 @@ def _add_to_zip(zf, path, root): sh.rmtree(root) -def unzip_directory(zipfile: Union[str, os.PathLike], root: Union[str, os.PathLike]): +def unzip_directory(zipfile: str | os.PathLike, root: str | os.PathLike): r"""Unzip an archive to a directory. This function is the exact opposite of :py:func:`xscen.io.zip_directory`. diff --git a/src/xscen/reduce.py b/src/xscen/reduce.py index b7310769..33a4c1a8 100644 --- a/src/xscen/reduce.py +++ b/src/xscen/reduce.py @@ -12,10 +12,10 @@ @parse_config def build_reduction_data( - datasets: Union[dict, list[xr.Dataset]], + datasets: dict | list[xr.Dataset], *, - xrfreqs: Optional[list[str]] = None, - horizons: Optional[list[str]] = None, + xrfreqs: list[str] | None = None, + horizons: list[str] | None = None, ) -> xr.DataArray: """Construct the input required for ensemble reduction. @@ -84,11 +84,11 @@ def build_reduction_data( @parse_config def reduce_ensemble( - data: Union[xr.DataArray, dict, list, xr.Dataset], + data: xr.DataArray | dict | list | xr.Dataset, method: str, *, - horizons: Optional[list[str]] = None, - create_kwargs: Optional[dict] = None, + horizons: list[str] | None = None, + create_kwargs: dict | None = None, **kwargs, ): r"""Reduce an ensemble of simulations using clustering algorithms from xclim.ensembles. @@ -140,7 +140,7 @@ def reduce_ensemble( ) -def _concat_criteria(criteria: Optional[xr.DataArray], ens: xr.Dataset): +def _concat_criteria(criteria: xr.DataArray | None, ens: xr.Dataset): """Combine all variables and dimensions excepting 'realization'.""" if criteria is None: i = 0 diff --git a/src/xscen/regrid.py b/src/xscen/regrid.py index 24c680cd..ece74918 100644 --- a/src/xscen/regrid.py +++ b/src/xscen/regrid.py @@ -34,10 +34,10 @@ def regrid_dataset( # noqa: C901 ds: xr.Dataset, ds_grid: xr.Dataset, - weights_location: Union[str, os.PathLike], + weights_location: str | os.PathLike, *, - regridder_kwargs: Optional[dict] = None, - intermediate_grids: Optional[dict] = None, + regridder_kwargs: dict | None = None, + intermediate_grids: dict | None = None, to_level: str = "regridded", ) -> xr.Dataset: """Regrid a dataset according to weights and a reference grid. @@ -228,7 +228,7 @@ def regrid_dataset( # noqa: C901 @parse_config -def create_mask(ds: Union[xr.Dataset, xr.DataArray], mask_args: dict) -> xr.DataArray: +def create_mask(ds: xr.Dataset | xr.DataArray, mask_args: dict) -> xr.DataArray: """Create a 0-1 mask based on incoming arguments. Parameters @@ -302,10 +302,10 @@ def cmp(arg1, op, arg2): def _regridder( ds_in: xr.Dataset, ds_grid: xr.Dataset, - filename: Union[str, os.PathLike], + filename: str | os.PathLike, *, method: str = "bilinear", - unmapped_to_nan: Optional[bool] = True, + unmapped_to_nan: bool | None = True, **kwargs, ) -> Regridder: """Call to xesmf Regridder with a few default arguments. diff --git a/src/xscen/scripting.py b/src/xscen/scripting.py index 532ca06b..d772c1af 100644 --- a/src/xscen/scripting.py +++ b/src/xscen/scripting.py @@ -43,12 +43,12 @@ def send_mail( *, subject: str, msg: str, - to: Optional[str] = None, + to: str | None = None, server: str = "127.0.0.1", port: int = 25, - attachments: Optional[ - list[Union[tuple[str, Union[Figure, os.PathLike]], Figure, os.PathLike]] - ] = None, + attachments: None | ( + list[tuple[str, Figure | os.PathLike] | Figure | os.PathLike] + ) = None, ) -> None: """Send email. @@ -160,9 +160,9 @@ def err_handler(self, *exc_info): @parse_config def send_mail_on_exit( *, - subject: Optional[str] = None, - msg_ok: Optional[str] = None, - msg_err: Optional[str] = None, + subject: str | None = None, + msg_ok: str | None = None, + msg_err: str | None = None, on_error_only: bool = False, skip_ctrlc: bool = True, **mail_kwargs, @@ -244,7 +244,7 @@ class measure_time: def __init__( self, - name: Optional[str] = None, + name: str | None = None, cpu: bool = False, logger: logging.Logger = logger, ): @@ -311,9 +311,7 @@ def _timeout_handler(signum, frame): @contextmanager -def skippable( - seconds: int = 2, task: str = "", logger: Optional[logging.Logger] = None -): +def skippable(seconds: int = 2, task: str = "", logger: logging.Logger | None = None): """Skippable context manager. When CTRL-C (SIGINT, KeyboardInterrupt) is sent within the context, @@ -355,11 +353,11 @@ def skippable( def save_and_update( ds: xr.Dataset, pcat: ProjectCatalog, - path: Optional[Union[str, os.PathLike]] = None, - file_format: Optional[str] = None, - build_path_kwargs: Optional[dict] = None, - save_kwargs: Optional[dict] = None, - update_kwargs: Optional[dict] = None, + path: str | os.PathLike | None = None, + file_format: str | None = None, + build_path_kwargs: dict | None = None, + save_kwargs: dict | None = None, + update_kwargs: dict | None = None, ): """ Construct the path, save and delete. @@ -431,9 +429,9 @@ def save_and_update( def move_and_delete( - moving: list[list[Union[str, os.PathLike]]], + moving: list[list[str | os.PathLike]], pcat: ProjectCatalog, - deleting: Optional[list[Union[str, os.PathLike]]] = None, + deleting: list[str | os.PathLike] | None = None, copy: bool = False, ): """ diff --git a/src/xscen/spatial.py b/src/xscen/spatial.py index 5725d050..40cd8a04 100644 --- a/src/xscen/spatial.py +++ b/src/xscen/spatial.py @@ -140,7 +140,7 @@ def subset( ds: xr.Dataset, method: str, *, - name: Optional[str] = None, + name: str | None = None, tile_buffer: float = 0, **kwargs, ) -> xr.Dataset: @@ -205,10 +205,10 @@ def subset( def _subset_gridpoint( ds: xr.Dataset, - lon: Union[float, Sequence[float], xr.DataArray], - lat: Union[float, Sequence[float], xr.DataArray], + lon: float | Sequence[float] | xr.DataArray, + lat: float | Sequence[float] | xr.DataArray, *, - name: Optional[str] = None, + name: str | None = None, **kwargs, ) -> xr.Dataset: r"""Subset the data to a gridpoint. @@ -254,10 +254,10 @@ def _subset_gridpoint( def _subset_bbox( ds: xr.Dataset, - lon_bnds: Union[tuple[float, float], list[float]], - lat_bnds: Union[tuple[float, float], list[float]], + lon_bnds: tuple[float, float] | list[float], + lat_bnds: tuple[float, float] | list[float], *, - name: Optional[str] = None, + name: str | None = None, tile_buffer: float = 0, **kwargs, ) -> xr.Dataset: @@ -317,9 +317,9 @@ def _subset_bbox( def _subset_shape( ds: xr.Dataset, - shape: Union[str, Path, gpd.GeoDataFrame], + shape: str | Path | gpd.GeoDataFrame, *, - name: Optional[str] = None, + name: str | None = None, tile_buffer: float = 0, **kwargs, ) -> xr.Dataset: @@ -392,7 +392,7 @@ def _subset_shape( return update_history_and_name(ds_subset, new_history, name) -def _subset_sel(ds: xr.Dataset, *, name: Optional[str] = None, **kwargs) -> xr.Dataset: +def _subset_sel(ds: xr.Dataset, *, name: str | None = None, **kwargs) -> xr.Dataset: r"""Subset the data using the .sel() method. Parameters diff --git a/src/xscen/testing.py b/src/xscen/testing.py index 866b0252..3aa519a7 100644 --- a/src/xscen/testing.py +++ b/src/xscen/testing.py @@ -22,9 +22,9 @@ def datablock_3d( y_step: float = 0.1, start: str = "7/1/2000", freq: str = "D", - units: Optional[str] = None, + units: str | None = None, as_dataset: bool = False, -) -> Union[xr.DataArray, xr.Dataset]: +) -> xr.DataArray | xr.Dataset: """Create a generic timeseries object based on pre-defined dictionaries of existing variables. Parameters diff --git a/src/xscen/utils.py b/src/xscen/utils.py index ab6e51b1..e7d2043b 100644 --- a/src/xscen/utils.py +++ b/src/xscen/utils.py @@ -78,12 +78,12 @@ def update_attr( - ds: Union[xr.Dataset, xr.DataArray], + ds: xr.Dataset | xr.DataArray, attr: str, new: str, - others: Optional[Sequence[Union[xr.Dataset, xr.DataArray]]] = None, + others: Sequence[xr.Dataset | xr.DataArray] | None = None, **fmt, -) -> Union[xr.Dataset, xr.DataArray]: +) -> xr.Dataset | xr.DataArray: r"""Format an attribute referencing itself in a translatable way. Parameters @@ -157,7 +157,7 @@ def update_attr( ) -def add_attr(ds: Union[xr.Dataset, xr.DataArray], attr: str, new: str, **fmt): +def add_attr(ds: xr.Dataset | xr.DataArray, attr: str, new: str, **fmt): """Add a formatted translatable attribute to a dataset.""" ds.attrs[attr] = new.format(**fmt) for loc in XC_OPTIONS[METADATA_LOCALES]: @@ -165,13 +165,13 @@ def add_attr(ds: Union[xr.Dataset, xr.DataArray], attr: str, new: str, **fmt): def date_parser( # noqa: C901 - date: Union[str, cftime.datetime, pd.Timestamp, datetime, pd.Period], + date: str | cftime.datetime | pd.Timestamp | datetime | pd.Period, *, - end_of_period: Union[bool, str] = False, + end_of_period: bool | str = False, out_dtype: str = "datetime", strtime_format: str = "%Y-%m-%d", freq: str = "H", -) -> Union[str, pd.Period, pd.Timestamp]: +) -> str | pd.Period | pd.Timestamp: """Return a datetime from a string. Parameters @@ -360,10 +360,10 @@ def translate_time_chunk(chunks: dict, calendar: str, timesize: int) -> dict: @parse_config def stack_drop_nans( ds: xr.Dataset, - mask: Union[xr.DataArray, list[str]], + mask: xr.DataArray | list[str], *, new_dim: str = "loc", - to_file: Optional[str] = None, + to_file: str | None = None, ) -> xr.Dataset: """Stack dimensions into a single axis and drops indexes where the mask is false. @@ -451,11 +451,9 @@ def unstack_fill_nan( ds: xr.Dataset, *, dim: str = "loc", - coords: Optional[ - Union[ - str, os.PathLike, Sequence[Union[str, os.PathLike]], dict[str, xr.DataArray] - ] - ] = None, + coords: None | ( + str | os.PathLike | Sequence[str | os.PathLike] | dict[str, xr.DataArray] + ) = None, ): """Unstack a Dataset that was stacked by :py:func:`stack_drop_nans`. @@ -602,7 +600,7 @@ def natural_sort(_list: list[str]): def get_cat_attrs( - ds: Union[xr.Dataset, xr.DataArray, dict], prefix: str = "cat:", var_as_str=False + ds: xr.Dataset | xr.DataArray | dict, prefix: str = "cat:", var_as_str=False ) -> dict: """Return the catalog-specific attributes from a dataset or dictionary. @@ -642,9 +640,9 @@ def get_cat_attrs( @parse_config def maybe_unstack( ds: xr.Dataset, - dim: Optional[str] = None, - coords: Optional[str] = None, - rechunk: Optional[dict] = None, + dim: str | None = None, + coords: str | None = None, + rechunk: dict | None = None, stack_drop_nans: bool = False, ) -> xr.Dataset: """If stack_drop_nans is True, unstack and rechunk. @@ -835,20 +833,18 @@ def change_units(ds: xr.Dataset, variables_and_units: dict) -> xr.Dataset: def clean_up( # noqa: C901 ds: xr.Dataset, *, - variables_and_units: Optional[dict] = None, - convert_calendar_kwargs: Optional[dict] = None, - missing_by_var: Optional[dict] = None, - maybe_unstack_dict: Optional[dict] = None, - round_var: Optional[dict] = None, - common_attrs_only: Optional[ - Union[dict, list[Union[xr.Dataset, str, os.PathLike]]] - ] = None, - common_attrs_open_kwargs: Optional[dict] = None, - attrs_to_remove: Optional[dict] = None, - remove_all_attrs_except: Optional[dict] = None, - add_attrs: Optional[dict] = None, - change_attr_prefix: Optional[Union[str, dict]] = None, - to_level: Optional[str] = None, + variables_and_units: dict | None = None, + convert_calendar_kwargs: dict | None = None, + missing_by_var: dict | None = None, + maybe_unstack_dict: dict | None = None, + round_var: dict | None = None, + common_attrs_only: None | (dict | list[xr.Dataset | str | os.PathLike]) = None, + common_attrs_open_kwargs: dict | None = None, + attrs_to_remove: dict | None = None, + remove_all_attrs_except: dict | None = None, + add_attrs: dict | None = None, + change_attr_prefix: str | dict | None = None, + to_level: str | None = None, ) -> xr.Dataset: """Clean up of the dataset. @@ -1089,9 +1085,9 @@ def clean_up( # noqa: C901 def publish_release_notes( style: str = "md", - file: Optional[Union[os.PathLike, StringIO, TextIO]] = None, - changes: Optional[Union[str, os.PathLike]] = None, -) -> Optional[str]: + file: os.PathLike | StringIO | TextIO | None = None, + changes: str | os.PathLike | None = None, +) -> str | None: """Format release history in Markdown or ReStructuredText. Parameters @@ -1171,8 +1167,8 @@ def publish_release_notes( def unstack_dates( # noqa: C901 ds: xr.Dataset, - seasons: Optional[dict[int, str]] = None, - new_dim: Optional[str] = None, + seasons: dict[int, str] | None = None, + new_dim: str | None = None, winter_starts_year: bool = False, ): """Unstack a multi-season timeseries into a yearly axis and a season one. @@ -1345,9 +1341,9 @@ def reshape_da(da): def show_versions( - file: Optional[Union[os.PathLike, StringIO, TextIO]] = None, - deps: Optional[list] = None, -) -> Optional[str]: + file: os.PathLike | StringIO | TextIO | None = None, + deps: list | None = None, +) -> str | None: """Print the versions of xscen and its dependencies. Parameters @@ -1467,8 +1463,8 @@ def ensure_correct_time(ds: xr.Dataset, xrfreq: str) -> xr.Dataset: def standardize_periods( - periods: Optional[Union[list[str], list[list[str]]]], multiple: bool = True -) -> Optional[Union[list[str], list[list[str]]]]: + periods: list[str] | list[list[str]] | None, multiple: bool = True +) -> list[str] | list[list[str]] | None: """Reformats the input to a list of strings, ['start', 'end'], or a list of such lists. Parameters @@ -1505,7 +1501,7 @@ def standardize_periods( return periods[0] -def season_sort_key(idx: pd.Index, name: Optional[str] = None): +def season_sort_key(idx: pd.Index, name: str | None = None): """Get a proper sort key for a "season" or "month" index to avoid alphabetical sorting. If any of the values in the index is not recognized as a 3-letter @@ -1608,7 +1604,7 @@ def _xarray_defaults(**kwargs): return kwargs -def rechunk_for_resample(obj: Union[xr.DataArray, xr.Dataset], **resample_kwargs): +def rechunk_for_resample(obj: xr.DataArray | xr.Dataset, **resample_kwargs): if not uses_dask(obj): return obj From 77d55200e8622719de2fb90598fccfd4159f6298 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 18 Sep 2024 10:56:46 -0400 Subject: [PATCH 03/10] upd changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f6f0768e..5c89e7eb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -41,6 +41,7 @@ Breaking changes ^^^^^^^^^^^^^^^^ * `convert_calendar` in ``clean_up`` now uses `xarray` instead of `xclim`. Keywords aren't compatible between the two, but given that `xclim` will abandon its function, no backwards compatibility was sought. (:pull:`450`). * `attrs_to_remove` and `remove_all_attrs_except` in ``clean_up`` now use real regex. It should not be too breaking since a `fullmatch()` is used, but `*` is now `.*`. (:pull:`450`). +* Python 3.9 is no longer supported. (:pull:`456`). Internal changes ^^^^^^^^^^^^^^^^ From 101837ca3d3ec7bdec76e84d7fd0f0ff9b2ad1cd Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 18 Sep 2024 11:04:47 -0400 Subject: [PATCH 04/10] help precommit --- src/xscen/aggregate.py | 60 ++++++++++++------------- src/xscen/biasadjust.py | 24 +++++----- src/xscen/catalog.py | 58 ++++++++++++------------ src/xscen/catutils.py | 96 ++++++++++++++++++++-------------------- src/xscen/config.py | 2 +- src/xscen/diagnostics.py | 54 +++++++++++----------- src/xscen/ensembles.py | 44 +++++++++--------- src/xscen/extract.py | 84 +++++++++++++++++------------------ src/xscen/indicators.py | 46 +++++++++---------- src/xscen/io.py | 84 +++++++++++++++++------------------ src/xscen/reduce.py | 14 +++--- src/xscen/regrid.py | 12 ++--- src/xscen/scripting.py | 34 +++++++------- src/xscen/spatial.py | 24 +++++----- src/xscen/testing.py | 4 +- src/xscen/utils.py | 94 +++++++++++++++++++-------------------- 16 files changed, 360 insertions(+), 374 deletions(-) diff --git a/src/xscen/aggregate.py b/src/xscen/aggregate.py index 3110e883..019b1618 100644 --- a/src/xscen/aggregate.py +++ b/src/xscen/aggregate.py @@ -51,11 +51,11 @@ def _(s): def climatological_mean( ds: xr.Dataset, *, - window: Optional[int] = None, - min_periods: Optional[int] = None, + window: int | None = None, + min_periods: int | None = None, interval: int = 1, - periods: Optional[Union[list[str], list[list[str]]]] = None, - to_level: Optional[str] = "climatology", + periods: list[str] | list[list[str]] | None = None, + to_level: str | None = "climatology", ) -> xr.Dataset: """Compute the mean over 'year' for given time periods, respecting the temporal resolution of ds. @@ -109,11 +109,11 @@ def climatological_mean( def climatological_op( # noqa: C901 ds: xr.Dataset, *, - op: Union[str, dict] = "mean", - window: Optional[int] = None, - min_periods: Optional[Union[int, float]] = None, + op: str | dict = "mean", + window: int | None = None, + min_periods: int | float | None = None, stride: int = 1, - periods: Optional[Union[list[str], list[list[str]]]] = None, + periods: list[str] | list[list[str]] | None = None, rename_variables: bool = True, to_level: str = "climatology", horizons_as_dim: bool = False, @@ -506,11 +506,11 @@ def _ulinregress(x, y, **kwargs): @parse_config def compute_deltas( # noqa: C901 ds: xr.Dataset, - reference_horizon: Union[str, xr.Dataset], + reference_horizon: str | xr.Dataset, *, - kind: Union[str, dict] = "+", + kind: str | dict = "+", rename_variables: bool = True, - to_level: Optional[str] = "deltas", + to_level: str | None = "deltas", ) -> xr.Dataset: """Compute deltas in comparison to a reference time period, respecting the temporal resolution of ds. @@ -702,13 +702,13 @@ def spatial_mean( # noqa: C901 ds: xr.Dataset, method: str, *, - spatial_subset: Optional[bool] = None, - call_clisops: Optional[bool] = False, - region: Optional[Union[dict, str]] = None, - kwargs: Optional[dict] = None, - simplify_tolerance: Optional[float] = None, - to_domain: Optional[str] = None, - to_level: Optional[str] = None, + spatial_subset: bool | None = None, + call_clisops: bool | None = False, + region: dict | str | None = None, + kwargs: dict | None = None, + simplify_tolerance: float | None = None, + to_domain: str | None = None, + to_level: str | None = None, ) -> xr.Dataset: """Compute the spatial mean using a variety of available methods. @@ -1034,18 +1034,18 @@ def spatial_mean( # noqa: C901 @parse_config def produce_horizon( # noqa: C901 ds: xr.Dataset, - indicators: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], + indicators: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), *, - periods: Optional[Union[list[str], list[list[str]]]] = None, - warminglevels: Optional[dict] = None, - to_level: Optional[str] = "horizons", - period: Optional[list] = None, + periods: list[str] | list[list[str]] | None = None, + warminglevels: dict | None = None, + to_level: str | None = "horizons", + period: list | None = None, ) -> xr.Dataset: """ Compute indicators, then the climatological mean, and finally unstack dates in order @@ -1095,7 +1095,7 @@ def produce_horizon( # noqa: C901 if periods is not None: all_periods.extend(standardize_periods(periods)) if warminglevels is not None: - if isinstance(warminglevels["wl"], (int, float)): + if isinstance(warminglevels["wl"], int | float): all_periods.append(warminglevels) elif isinstance(warminglevels["wl"], list): template = deepcopy(warminglevels) diff --git a/src/xscen/biasadjust.py b/src/xscen/biasadjust.py index 5a7362a5..8359c62b 100644 --- a/src/xscen/biasadjust.py +++ b/src/xscen/biasadjust.py @@ -58,17 +58,17 @@ def _add_preprocessing_attr(scen, train_kwargs): def train( dref: xr.Dataset, dhist: xr.Dataset, - var: Union[str, list[str]], + var: str | list[str], period: list[str], *, method: str = "DetrendedQuantileMapping", - group: Optional[Union[sdba.Grouper, str, dict]] = None, - xclim_train_args: Optional[dict] = None, + group: sdba.Grouper | str | dict | None = None, + xclim_train_args: dict | None = None, maximal_calendar: str = "noleap", - adapt_freq: Optional[dict] = None, - jitter_under: Optional[dict] = None, - jitter_over: Optional[dict] = None, - align_on: Optional[str] = "year", + adapt_freq: dict | None = None, + jitter_under: dict | None = None, + jitter_over: dict | None = None, + align_on: str | None = "year", ) -> xr.Dataset: """ Train a bias-adjustment. @@ -194,13 +194,13 @@ def train( def adjust( dtrain: xr.Dataset, dsim: xr.Dataset, - periods: Union[list[str], list[list[str]]], + periods: list[str] | list[list[str]], *, - xclim_adjust_args: Optional[dict] = None, + xclim_adjust_args: dict | None = None, to_level: str = "biasadjusted", - bias_adjust_institution: Optional[str] = None, - bias_adjust_project: Optional[str] = None, - align_on: Optional[str] = "year", + bias_adjust_institution: str | None = None, + bias_adjust_project: str | None = None, + align_on: str | None = "year", ) -> xr.Dataset: """ Adjust a simulation. diff --git a/src/xscen/catalog.py b/src/xscen/catalog.py index 788e4f79..f3bfa681 100644 --- a/src/xscen/catalog.py +++ b/src/xscen/catalog.py @@ -198,10 +198,10 @@ def __init__( @classmethod def from_df( cls, - data: Union[pd.DataFrame, os.PathLike, Sequence[os.PathLike]], - esmdata: Optional[Union[os.PathLike, dict]] = None, + data: pd.DataFrame | os.PathLike | Sequence[os.PathLike], + esmdata: os.PathLike | dict | None = None, *, - read_csv_kwargs: Optional[Mapping[str, Any]] = None, + read_csv_kwargs: Mapping[str, Any] | None = None, name: str = "virtual", **intake_kwargs, ): @@ -263,7 +263,7 @@ def _find_unique(series): else: return data.apply(_find_unique, result_type="reduce").to_dict() - def unique(self, columns: Optional[Union[str, Sequence[str]]] = None): + def unique(self, columns: str | Sequence[str] | None = None): """Return a series of unique values in the catalog. Parameters @@ -309,7 +309,7 @@ def search(self, **columns): ) return cat - def drop_duplicates(self, columns: Optional[list[str]] = None): + def drop_duplicates(self, columns: list[str] | None = None): """Drop duplicates in the catalog based on a subset of columns. Parameters @@ -404,10 +404,10 @@ def exists_in_cat(self, **columns) -> bool: def to_dataset( self, - concat_on: Optional[Union[list[str], str]] = None, - create_ensemble_on: Optional[Union[list[str], str]] = None, - ensemble_name: Optional[Union[list[str]]] = None, - calendar: Optional[str] = "standard", + concat_on: list[str] | str | None = None, + create_ensemble_on: list[str] | str | None = None, + ensemble_name: list[str] | None = None, + calendar: str | None = "standard", **kwargs, ) -> xr.Dataset: """ @@ -538,7 +538,7 @@ def preprocess(ds): def copy_files( self, - dest: Union[str, os.PathLike], + dest: str | os.PathLike, flat: bool = True, unzip: bool = False, zipzarr: bool = False, @@ -636,9 +636,9 @@ class ProjectCatalog(DataCatalog): @classmethod def create( cls, - filename: Union[os.PathLike, str], + filename: os.PathLike | str, *, - project: Optional[dict] = None, + project: dict | None = None, overwrite: bool = False, ): r"""Create a new project catalog from some project metadata. @@ -716,11 +716,11 @@ def create( def __init__( self, - df: Union[str, dict], + df: str | dict, *args, create: bool = False, overwrite: bool = False, - project: Optional[dict] = None, + project: dict | None = None, **kwargs, ): """ @@ -746,7 +746,7 @@ def __init__( The ‘df’ key must be a Pandas DataFrame containing content that would otherwise be in the CSV file. """ if create: - if isinstance(df, (str, Path)) and (not Path(df).is_file() or overwrite): + if isinstance(df, str | Path) and (not Path(df).is_file() or overwrite): self.create(df, project=project, overwrite=overwrite) super().__init__(df, *args, **kwargs) self.check_valid() @@ -756,15 +756,13 @@ def __init__( # TODO: Implement a way to easily destroy part of the catalog to "reset" some steps def update( self, - df: Optional[ - Union[ - DataCatalog, - intake_esm.esm_datastore, - pd.DataFrame, - pd.Series, - Sequence[pd.Series], - ] - ] = None, + df: None | ( + DataCatalog + | intake_esm.esm_datastore + | pd.DataFrame + | pd.Series + | Sequence[pd.Series] + ) = None, ): """Update the catalog with new data and writes the new data to the csv file. @@ -846,8 +844,8 @@ def update( def update_from_ds( self, ds: xr.Dataset, - path: Union[os.PathLike, str], - info_dict: Optional[dict] = None, + path: os.PathLike | str, + info_dict: dict | None = None, **info_kwargs, ): """Update the catalog with new data and writes the new data to the csv file. @@ -965,7 +963,7 @@ def _build_id(element: pd.Series, columns: list[str]): def generate_id( - df: Union[pd.DataFrame, xr.Dataset], id_columns: Optional[list] = None + df: pd.DataFrame | xr.Dataset, id_columns: list | None = None ) -> pd.Series: """Create an ID from column entries. @@ -996,7 +994,7 @@ def generate_id( return df.apply(_build_id, axis=1, args=(id_columns,)) -def unstack_id(df: Union[pd.DataFrame, ProjectCatalog, DataCatalog]) -> dict: +def unstack_id(df: pd.DataFrame | ProjectCatalog | DataCatalog) -> dict: """Reverse-engineer an ID using catalog entries. Parameters @@ -1009,7 +1007,7 @@ def unstack_id(df: Union[pd.DataFrame, ProjectCatalog, DataCatalog]) -> dict: dict Dictionary with one entry per unique ID, which are themselves dictionaries of all the individual parts of the ID. """ - if isinstance(df, (ProjectCatalog, DataCatalog)): + if isinstance(df, ProjectCatalog | DataCatalog): df = df.df out = {} @@ -1038,7 +1036,7 @@ def unstack_id(df: Union[pd.DataFrame, ProjectCatalog, DataCatalog]) -> dict: def subset_file_coverage( df: pd.DataFrame, - periods: Union[list[str], list[list[str]]], + periods: list[str] | list[list[str]], *, coverage: float = 0.99, duplicates_ok: bool = False, diff --git a/src/xscen/catutils.py b/src/xscen/catutils.py index 12e875be..31c74b24 100644 --- a/src/xscen/catutils.py +++ b/src/xscen/catutils.py @@ -110,7 +110,7 @@ def _parse_level(text: str) -> str: ) def _parse_datebounds( text: str, -) -> Union[list[str], tuple[None, None], tuple[str, str]]: +) -> list[str] | tuple[None, None] | tuple[str, str]: """Parse helper to translate date bounds, used in the special DATES field.""" if "-" in text: return text.split("-") @@ -120,10 +120,10 @@ def _parse_datebounds( def _find_assets( - root: Union[str, os.PathLike], + root: str | os.PathLike, exts: set[str], lengths: set[int], - dirglob: Optional[str] = None, + dirglob: str | None = None, ): """Walk recursively over files in a directory, filtering according to a glob pattern, path depth and extensions. @@ -191,13 +191,13 @@ def _compile_pattern(pattern: str) -> parse.Parser: def _name_parser( - path: Union[os.PathLike, str], - root: Union[os.PathLike, str], - patterns: list[Union[str, parse.Parser]], - read_from_file: Optional[Union[list[str], dict]] = None, - attrs_map: Optional[dict] = None, - xr_open_kwargs: Optional[dict] = None, -) -> Optional[dict]: + path: os.PathLike | str, + root: os.PathLike | str, + patterns: list[str | parse.Parser], + read_from_file: list[str] | dict | None = None, + attrs_map: dict | None = None, + xr_open_kwargs: dict | None = None, +) -> dict | None: """Extract metadata information from the file path. Parameters @@ -267,13 +267,13 @@ def _name_parser( def _parse_dir( # noqa: C901 - root: Union[os.PathLike, str], + root: os.PathLike | str, patterns: list[str], - dirglob: Optional[str] = None, - checks: Optional[list[str]] = None, - read_from_file: Optional[Union[list[str], dict]] = None, - attrs_map: Optional[dict] = None, - xr_open_kwargs: Optional[dict] = None, + dirglob: str | None = None, + checks: list[str] | None = None, + read_from_file: list[str] | dict | None = None, + attrs_map: dict | None = None, + xr_open_kwargs: dict | None = None, progress: bool = False, ): """Iterate and parses files in a directory, filtering according to basic pattern properties and optional checks. @@ -413,7 +413,7 @@ def _replace_in_row(oldrow: pd.Series, replacements: dict): List-like fields are handled. """ row = oldrow.copy() - list_cols = [col for col in oldrow.index if isinstance(oldrow[col], (tuple, list))] + list_cols = [col for col in oldrow.index if isinstance(oldrow[col], tuple | list)] for col, reps in replacements.items(): if col not in row: continue @@ -452,24 +452,24 @@ def _parse_first_ds( @parse_config def parse_directory( # noqa: C901 - directories: Union[str, list[Union[str, os.PathLike]]], + directories: str | list[str | os.PathLike], patterns: list[str], *, - id_columns: Optional[list[str]] = None, - read_from_file: Union[ - bool, - Sequence[str], - tuple[Sequence[str], Sequence[str]], - Sequence[tuple[Sequence[str], Sequence[str]]], - ] = False, - homogenous_info: Optional[dict] = None, - cvs: Optional[Union[str, os.PathLike, dict]] = None, - dirglob: Optional[str] = None, - xr_open_kwargs: Optional[Mapping[str, Any]] = None, + id_columns: list[str] | None = None, + read_from_file: ( + bool + | Sequence[str] + | tuple[Sequence[str], Sequence[str]] + | Sequence[tuple[Sequence[str], Sequence[str]]] + ) = False, + homogenous_info: dict | None = None, + cvs: str | os.PathLike | dict | None = None, + dirglob: str | None = None, + xr_open_kwargs: Mapping[str, Any] | None = None, only_official_columns: bool = True, progress: bool = False, - parallel_dirs: Union[bool, int] = False, - file_checks: Optional[list[str]] = None, + parallel_dirs: bool | int = False, + file_checks: list[str] | None = None, ) -> pd.DataFrame: r"""Parse files in a directory and return them as a pd.DataFrame. @@ -555,7 +555,7 @@ def parse_directory( # noqa: C901 pd.DataFrame Parsed directory files """ - if isinstance(directories, (str, Path)): + if isinstance(directories, str | Path): directories = [directories] homogenous_info = homogenous_info or {} xr_open_kwargs = xr_open_kwargs or {} @@ -726,9 +726,9 @@ def parse_directory( # noqa: C901 def parse_from_ds( # noqa: C901 - obj: Union[str, os.PathLike, xr.Dataset], + obj: str | os.PathLike | xr.Dataset, names: Sequence[str], - attrs_map: Optional[Mapping[str, str]] = None, + attrs_map: Mapping[str, str] | None = None, **xrkwargs, ): """Parse a list of catalog fields from the file/dataset itself. @@ -818,7 +818,7 @@ def parse_from_ds( # noqa: C901 def _parse_from_zarr( - path: Union[os.PathLike, str], get_vars: bool = True, get_time: bool = True + path: os.PathLike | str, get_vars: bool = True, get_time: bool = True ): """Obtain the list of variables, the time coordinate and the list of global attributes from a zarr dataset. @@ -881,7 +881,7 @@ def _parse_from_zarr( def _parse_from_nc( - path: Union[os.PathLike, str], get_vars: bool = True, get_time: bool = True + path: os.PathLike | str, get_vars: bool = True, get_time: bool = True ): """Obtain the list of variables, the time coordinate, and the list of global attributes from a netCDF dataset, using netCDF4. @@ -934,7 +934,7 @@ def _schema_option(option: dict, facets: dict): return answer -def _schema_level(schema: Union[dict, list[str], str], facets: dict): +def _schema_level(schema: dict | list[str] | str, facets: dict): if isinstance(schema, str): if schema.startswith("(") and schema.endswith(")"): optional = True @@ -1060,14 +1060,14 @@ def _read_schemas(schemas): def _build_path( - data: Union[dict, xr.Dataset, xr.DataArray, pd.Series], + data: dict | xr.Dataset | xr.DataArray | pd.Series, schemas: dict, - root: Union[str, os.PathLike], + root: str | os.PathLike, get_type: bool = False, **extra_facets, -) -> Union[Path, tuple[Path, str]]: +) -> Path | tuple[Path, str]: # Get all known metadata - if isinstance(data, (xr.Dataset, xr.DataArray)): + if isinstance(data, xr.Dataset | xr.DataArray): facets = ( # Get non-attribute metadata parse_from_ds( @@ -1127,11 +1127,11 @@ def _build_path( @parse_config def build_path( - data: Union[dict, xr.Dataset, xr.DataArray, pd.Series, DataCatalog, pd.DataFrame], - schemas: Optional[Union[str, os.PathLike, dict]] = None, - root: Optional[Union[str, os.PathLike]] = None, + data: dict | xr.Dataset | xr.DataArray | pd.Series | DataCatalog | pd.DataFrame, + schemas: str | os.PathLike | dict | None = None, + root: str | os.PathLike | None = None, **extra_facets, -) -> Union[Path, DataCatalog, pd.DataFrame]: +) -> Path | DataCatalog | pd.DataFrame: r"""Parse the schema from a configuration and construct path using a dictionary of facets. Parameters @@ -1172,7 +1172,7 @@ def build_path( if root: root = Path(root) schemas = _read_schemas(schemas) - if isinstance(data, (esm_datastore, pd.DataFrame)): + if isinstance(data, esm_datastore | pd.DataFrame): if isinstance(data, esm_datastore): df = data.df else: @@ -1210,9 +1210,7 @@ def __missing__(self, key): return template.format_map(PartialFormatDict(**fmtargs)) -def patterns_from_schema( - schema: Union[str, dict], exts: Optional[Sequence[str]] = None -): +def patterns_from_schema(schema: str | dict, exts: Sequence[str] | None = None): """Generate all valid patterns for a given schema. Generated patterns are meant for use with :py:func:`parse_directory`. diff --git a/src/xscen/config.py b/src/xscen/config.py index 06a4eeb7..a2f556c3 100644 --- a/src/xscen/config.py +++ b/src/xscen/config.py @@ -133,7 +133,7 @@ def args_as_str(*args: tuple[Any, ...]) -> tuple[str, ...]: def load_config( *elements, reset: bool = False, - encoding: Optional[str] = None, + encoding: str | None = None, verbose: bool = False, ): """Load configuration from given files or key=value pairs. diff --git a/src/xscen/diagnostics.py b/src/xscen/diagnostics.py index 3af73970..d2e4534b 100644 --- a/src/xscen/diagnostics.py +++ b/src/xscen/diagnostics.py @@ -44,21 +44,21 @@ def _(s): @parse_config def health_checks( # noqa: C901 - ds: Union[xr.Dataset, xr.DataArray], + ds: xr.Dataset | xr.DataArray, *, - structure: Optional[dict] = None, - calendar: Optional[str] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - variables_and_units: Optional[dict] = None, - cfchecks: Optional[dict] = None, - freq: Optional[str] = None, - missing: Optional[Union[dict, str, list]] = None, - flags: Optional[dict] = None, - flags_kwargs: Optional[dict] = None, + structure: dict | None = None, + calendar: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + variables_and_units: dict | None = None, + cfchecks: dict | None = None, + freq: str | None = None, + missing: dict | str | list | None = None, + flags: dict | None = None, + flags_kwargs: dict | None = None, return_flags: bool = False, - raise_on: Optional[list] = None, -) -> Union[None, xr.Dataset]: + raise_on: list | None = None, +) -> None | xr.Dataset: """ Perform a series of health checks on the dataset. Be aware that missing data checks and flag checks can be slow. @@ -304,18 +304,18 @@ def _message(): @parse_config def properties_and_measures( # noqa: C901 ds: xr.Dataset, - properties: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], - period: Optional[list[str]] = None, + properties: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), + period: list[str] | None = None, unstack: bool = False, - rechunk: Optional[dict] = None, - dref_for_measure: Optional[xr.Dataset] = None, - change_units_arg: Optional[dict] = None, + rechunk: dict | None = None, + dref_for_measure: xr.Dataset | None = None, + change_units_arg: dict | None = None, to_level_prop: str = "diag-properties", to_level_meas: str = "diag-measures", ) -> tuple[xr.Dataset, xr.Dataset]: @@ -362,7 +362,7 @@ def properties_and_measures( # noqa: C901 -------- xclim.sdba.properties, xclim.sdba.measures, xclim.core.indicator.build_indicator_module_from_yaml """ - if isinstance(properties, (str, Path)): + if isinstance(properties, str | Path): logger.debug("Loading properties module.") module = load_xclim_module(properties) properties = module.iter_indicators() @@ -446,7 +446,7 @@ def properties_and_measures( # noqa: C901 def measures_heatmap( - meas_datasets: Union[list[xr.Dataset], dict], to_level: str = "diag-heatmap" + meas_datasets: list[xr.Dataset] | dict, to_level: str = "diag-heatmap" ) -> xr.Dataset: """Create a heatmap to compare the performance of the different datasets. @@ -528,7 +528,7 @@ def measures_heatmap( def measures_improvement( - meas_datasets: Union[list[xr.Dataset], dict], to_level: str = "diag-improved" + meas_datasets: list[xr.Dataset] | dict, to_level: str = "diag-improved" ) -> xr.Dataset: """ Calculate the fraction of improved grid points for each property between two datasets of measures. diff --git a/src/xscen/ensembles.py b/src/xscen/ensembles.py index 6eeddc0c..7b19cf67 100644 --- a/src/xscen/ensembles.py +++ b/src/xscen/ensembles.py @@ -31,17 +31,17 @@ @parse_config def ensemble_stats( # noqa: C901 - datasets: Union[ - dict, - list[Union[str, os.PathLike]], - list[xr.Dataset], - list[xr.DataArray], - xr.Dataset, - ], + datasets: ( + dict + | list[str | os.PathLike] + | list[xr.Dataset] + | list[xr.DataArray] + | xr.Dataset + ), statistics: dict, *, - create_kwargs: Optional[dict] = None, - weights: Optional[xr.DataArray] = None, + create_kwargs: dict | None = None, + weights: xr.DataArray | None = None, common_attrs_only: bool = True, to_level: str = "ensemble", ) -> xr.Dataset: @@ -107,7 +107,7 @@ def ensemble_stats( # noqa: C901 statistics = deepcopy(statistics) # to avoid modifying the original dictionary # if input files are .zarr, change the engine automatically - if isinstance(datasets, list) and isinstance(datasets[0], (str, os.PathLike)): + if isinstance(datasets, list) and isinstance(datasets[0], str | os.PathLike): path = Path(datasets[0]) if path.suffix == ".zarr": create_kwargs.setdefault("engine", "zarr") @@ -245,13 +245,13 @@ def ensemble_stats( # noqa: C901 def generate_weights( # noqa: C901 - datasets: Union[dict, list], + datasets: dict | list, *, independence_level: str = "model", balance_experiments: bool = False, - attribute_weights: Optional[dict] = None, + attribute_weights: dict | None = None, skipna: bool = True, - v_for_skipna: Optional[str] = None, + v_for_skipna: str | None = None, standardize: bool = False, experiment_weights: bool = False, ) -> xr.DataArray: @@ -673,12 +673,12 @@ def generate_weights( # noqa: C901 def build_partition_data( - datasets: Union[dict, list[xr.Dataset]], + datasets: dict | list[xr.Dataset], partition_dim: list[str] = ["source", "experiment", "bias_adjust_project"], - subset_kw: Optional[dict] = None, - regrid_kw: Optional[dict] = None, - indicators_kw: Optional[dict] = None, - rename_dict: Optional[dict] = None, + subset_kw: dict | None = None, + regrid_kw: dict | None = None, + indicators_kw: dict | None = None, + rename_dict: dict | None = None, ): """ Get the input for the xclim partition functions. @@ -765,11 +765,11 @@ def build_partition_data( @parse_config def reduce_ensemble( - data: Union[xr.DataArray, dict, list, xr.Dataset], + data: xr.DataArray | dict | list | xr.Dataset, method: str, *, - horizons: Optional[list[str]] = None, - create_kwargs: Optional[dict] = None, + horizons: list[str] | None = None, + create_kwargs: dict | None = None, **kwargs, ): r"""Reduce an ensemble of simulations using clustering algorithms from xclim.ensembles. @@ -808,7 +808,7 @@ def reduce_ensemble( If the indicators are a mix of yearly, seasonal, and monthly, they should be stacked on the same time/horizon axis and put in the same dataset. You can use py:func:`xscen.utils.unstack_dates` on seasonal or monthly indicators to this end. """ - if isinstance(data, (list, dict)): + if isinstance(data, list | dict): data = ensembles.create_ensemble(datasets=data, **(create_kwargs or {})) if horizons: if "horizon" not in data.dims: diff --git a/src/xscen/extract.py b/src/xscen/extract.py index 322ea242..76197ce4 100644 --- a/src/xscen/extract.py +++ b/src/xscen/extract.py @@ -6,10 +6,10 @@ import re import warnings from collections import defaultdict -from collections.abc import Sequence +from collections.abc import Callable, Sequence from copy import deepcopy from pathlib import Path -from typing import Callable, Optional, Union +from typing import Optional, Union import numpy as np import pandas as pd @@ -49,16 +49,16 @@ def extract_dataset( # noqa: C901 catalog: DataCatalog, *, - variables_and_freqs: Optional[dict] = None, - periods: Optional[Union[list[str], list[list[str]]]] = None, - region: Optional[dict] = None, + variables_and_freqs: dict | None = None, + periods: list[str] | list[list[str]] | None = None, + region: dict | None = None, to_level: str = "extracted", ensure_correct_time: bool = True, - xr_open_kwargs: Optional[dict] = None, - xr_combine_kwargs: Optional[dict] = None, - preprocess: Optional[Callable] = None, - resample_methods: Optional[dict] = None, - mask: Union[bool, xr.Dataset, xr.DataArray] = False, + xr_open_kwargs: dict | None = None, + xr_combine_kwargs: dict | None = None, + preprocess: Callable | None = None, + resample_methods: dict | None = None, + mask: bool | xr.Dataset | xr.DataArray = False, ) -> dict: """ Take one element of the output of `search_data_catalogs` and returns a dataset, @@ -322,9 +322,9 @@ def resample( # noqa: C901 da: xr.DataArray, target_frequency: str, *, - ds: Optional[xr.Dataset] = None, - method: Optional[str] = None, - missing: Optional[Union[str, dict]] = None, + ds: xr.Dataset | None = None, + method: str | None = None, + missing: str | dict | None = None, ) -> xr.DataArray: """Aggregate variable to the target frequency. @@ -542,23 +542,23 @@ def resample( # noqa: C901 @parse_config def search_data_catalogs( # noqa: C901 - data_catalogs: Union[ - str, os.PathLike, DataCatalog, list[Union[str, os.PathLike, DataCatalog]] - ], + data_catalogs: ( + str | os.PathLike | DataCatalog | list[str | os.PathLike | DataCatalog] + ), variables_and_freqs: dict, *, - other_search_criteria: Optional[dict] = None, - exclusions: Optional[dict] = None, + other_search_criteria: dict | None = None, + exclusions: dict | None = None, match_hist_and_fut: bool = False, - periods: Optional[Union[list[str], list[list[str]]]] = None, - coverage_kwargs: Optional[dict] = None, - id_columns: Optional[list[str]] = None, + periods: list[str] | list[list[str]] | None = None, + coverage_kwargs: dict | None = None, + id_columns: list[str] | None = None, allow_resampling: bool = False, allow_conversion: bool = False, - conversion_yaml: Optional[str] = None, - restrict_resolution: Optional[str] = None, - restrict_members: Optional[dict] = None, - restrict_warming_level: Optional[Union[dict, bool]] = None, + conversion_yaml: str | None = None, + restrict_resolution: str | None = None, + restrict_members: dict | None = None, + restrict_warming_level: dict | bool | None = None, ) -> dict: """Search through DataCatalogs. @@ -634,14 +634,14 @@ def search_data_catalogs( # noqa: C901 intake_esm.core.esm_datastore.search """ # Cast single items to a list - if isinstance(data_catalogs, (str, os.PathLike, DataCatalog)): + if isinstance(data_catalogs, str | os.PathLike | DataCatalog): data_catalogs = [data_catalogs] # Open the catalogs given as paths data_catalogs = [ ( dc - if not isinstance(dc, (str, os.PathLike)) + if not isinstance(dc, str | os.PathLike) else ( DataCatalog(dc) if Path(dc).suffix == ".json" @@ -884,17 +884,17 @@ def search_data_catalogs( # noqa: C901 @parse_config def get_warming_level( # noqa: C901 - realization: Union[ - xr.Dataset, xr.DataArray, dict, pd.Series, pd.DataFrame, str, list - ], + realization: ( + xr.Dataset | xr.DataArray | dict | pd.Series | pd.DataFrame | str | list + ), wl: float, *, window: int = 20, - tas_baseline_period: Optional[Sequence[str]] = None, + tas_baseline_period: Sequence[str] | None = None, ignore_member: bool = False, - tas_src: Optional[Union[str, os.PathLike]] = None, + tas_src: str | os.PathLike | None = None, return_horizon: bool = True, -) -> Union[dict, list[str], str]: +) -> dict | list[str] | str: """ Use the IPCC Atlas method to return the window of time over which the requested level of global warming is first reached. @@ -947,7 +947,7 @@ def get_warming_level( # noqa: C901 FIELDS = ["mip_era", "source", "experiment", "member"] - if isinstance(realization, (xr.Dataset, str, dict, pd.Series)): + if isinstance(realization, xr.Dataset | str | dict | pd.Series): reals = [realization] elif isinstance(realization, pd.DataFrame): reals = (row for i, row in realization.iterrows()) @@ -1072,11 +1072,11 @@ def _get_warming_level(model): @parse_config def subset_warming_level( ds: xr.Dataset, - wl: Union[float, Sequence[float]], + wl: float | Sequence[float], to_level: str = "warminglevel-{wl}vs{period0}-{period1}", - wl_dim: Union[str, bool] = "+{wl}Cvs{period0}-{period1}", + wl_dim: str | bool = "+{wl}Cvs{period0}-{period1}", **kwargs, -) -> Optional[xr.Dataset]: +) -> xr.Dataset | None: r""" Subsets the input dataset with only the window of time over which the requested level of global warming is first reached, using the IPCC Atlas method. @@ -1128,7 +1128,7 @@ def subset_warming_level( # Fake time generation is needed : real is a dim or multiple levels if ( fake_time is None - and not isinstance(wl, (int, float)) + and not isinstance(wl, int | float) or "realization" in ds.dims ): freq = xr.infer_freq(ds.time) @@ -1142,7 +1142,7 @@ def subset_warming_level( ) # If we got a wl sequence, call ourself multiple times and concatenate - if not isinstance(wl, (int, float)): + if not isinstance(wl, int | float): if not wl_dim or (isinstance(wl_dim, str) and "{wl}" not in wl_dim): raise ValueError( "`wl_dim` must be True or a template string including '{wl}' if multiple levels are passed." @@ -1273,7 +1273,7 @@ def subset_warming_level( def _dispatch_historical_to_future( - catalog: DataCatalog, id_columns: Optional[list[str]] = None + catalog: DataCatalog, id_columns: list[str] | None = None ) -> DataCatalog: """Update a DataCatalog by recopying each "historical" entry to its corresponding future experiments. @@ -1387,7 +1387,7 @@ def _dispatch_historical_to_future( def _restrict_by_resolution( - catalogs: dict, restrictions: str, id_columns: Optional[list[str]] = None + catalogs: dict, restrictions: str, id_columns: list[str] | None = None ) -> dict: """Update the results from search_data_catalogs by removing simulations with multiple resolutions available. @@ -1527,7 +1527,7 @@ def _restrict_by_resolution( def _restrict_multimembers( - catalogs: dict, restrictions: dict, id_columns: Optional[list[str]] = None + catalogs: dict, restrictions: dict, id_columns: list[str] | None = None ): """Update the results from search_data_catalogs by removing simulations with multiple members available. diff --git a/src/xscen/indicators.py b/src/xscen/indicators.py index da156beb..f651f07e 100644 --- a/src/xscen/indicators.py +++ b/src/xscen/indicators.py @@ -26,9 +26,7 @@ __all__ = ["compute_indicators", "load_xclim_module", "registry_from_module"] -def load_xclim_module( - filename: Union[str, os.PathLike], reload: bool = False -) -> ModuleType: +def load_xclim_module(filename: str | os.PathLike, reload: bool = False) -> ModuleType: """Return the xclim module described by the yaml file (or group of yaml, jsons and py). Parameters @@ -103,17 +101,17 @@ def get_indicator_outputs(ind: xc.core.indicator.Indicator, in_freq: str): @parse_config def compute_indicators( # noqa: C901 ds: xr.Dataset, - indicators: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], + indicators: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), *, - periods: Optional[Union[list[str], list[list[str]]]] = None, + periods: list[str] | list[list[str]] | None = None, restrict_years: bool = True, - to_level: Optional[str] = "indicators", + to_level: str | None = "indicators", rechunk_input: bool = False, ) -> dict: """Calculate variables and indicators based on a YAML call to xclim. @@ -158,7 +156,7 @@ def compute_indicators( # noqa: C901 -------- xclim.indicators, xclim.core.indicator.build_indicator_module_from_yaml """ - if isinstance(indicators, (str, os.PathLike)): + if isinstance(indicators, str | os.PathLike): logger.debug("Loading indicator module.") module = load_xclim_module(indicators) indicators = module.iter_indicators() @@ -284,7 +282,7 @@ def compute_indicators( # noqa: C901 def registry_from_module( module: ModuleType, - registry: Optional[DerivedVariableRegistry] = None, + registry: DerivedVariableRegistry | None = None, variable_column: str = "variable", ) -> DerivedVariableRegistry: """Convert a xclim virtual indicators module to an intake_esm Derived Variable Registry. @@ -319,7 +317,7 @@ def registry_from_module( def _ensure_list(x): - if not isinstance(x, (list, tuple)): + if not isinstance(x, list | tuple): return [x] return x @@ -338,13 +336,13 @@ def func(ds, *, ind, nout): def select_inds_for_avail_vars( ds: xr.Dataset, - indicators: Union[ - str, - os.PathLike, - Sequence[Indicator], - Sequence[tuple[str, Indicator]], - ModuleType, - ], + indicators: ( + str + | os.PathLike + | Sequence[Indicator] + | Sequence[tuple[str, Indicator]] + | ModuleType + ), ) -> ModuleType: """Filter the indicators for which the necessary variables are available. @@ -370,12 +368,12 @@ def select_inds_for_avail_vars( is_list_of_tuples = isinstance(indicators, list) and all( isinstance(i, tuple) for i in indicators ) - if isinstance(indicators, (str, os.PathLike)): + if isinstance(indicators, str | os.PathLike): logger.debug("Loading indicator module.") indicators = load_xclim_module(indicators, reload=True) if hasattr(indicators, "iter_indicators"): indicators = [(name, ind) for name, ind in indicators.iter_indicators()] - elif isinstance(indicators, (list, tuple)) and not is_list_of_tuples: + elif isinstance(indicators, list | tuple) and not is_list_of_tuples: indicators = [(ind.base, ind) for ind in indicators] available_vars = { diff --git a/src/xscen/io.py b/src/xscen/io.py index a3ed9259..511d43be 100644 --- a/src/xscen/io.py +++ b/src/xscen/io.py @@ -49,7 +49,7 @@ ] -def get_engine(file: Union[str, os.PathLike]) -> str: +def get_engine(file: str | os.PathLike) -> str: """Determine which Xarray engine should be used to open the given file. The .zarr, .zarr.zip and .zip extensions are recognized as Zarr datasets, @@ -77,7 +77,7 @@ def get_engine(file: Union[str, os.PathLike]) -> str: def estimate_chunks( # noqa: C901 - ds: Union[str, os.PathLike, xr.Dataset], + ds: str | os.PathLike | xr.Dataset, dims: list, target_mb: float = 50, chunk_per_variable: bool = False, @@ -158,7 +158,7 @@ def _estimate_chunks(ds, target_mb, size_of_slice, rechunk_dims): out = {} # If ds is the path to a file, use NetCDF4 - if isinstance(ds, (str, os.PathLike)): + if isinstance(ds, str | os.PathLike): ds = netCDF4.Dataset(ds, "r") # Loop on variables @@ -263,7 +263,7 @@ def subset_maxsize( ) -def clean_incomplete(path: Union[str, os.PathLike], complete: Sequence[str]) -> None: +def clean_incomplete(path: str | os.PathLike, complete: Sequence[str]) -> None: """Delete un-catalogued variables from a zarr folder. The goal of this function is to clean up an incomplete calculation. @@ -296,9 +296,9 @@ def _coerce_attrs(attrs): """Ensure no funky objects in attrs.""" for k in list(attrs.keys()): if not ( - isinstance(attrs[k], (str, float, int, np.ndarray)) - or isinstance(attrs[k], (tuple, list)) - and isinstance(attrs[k][0], (str, float, int)) + isinstance(attrs[k], str | float | int | np.ndarray) + or isinstance(attrs[k], tuple | list) + and isinstance(attrs[k][0], str | float | int) ): attrs[k] = str(attrs[k]) @@ -335,7 +335,7 @@ def round_bits(da: xr.DataArray, keepbits: int): return da -def _get_keepbits(bitround: Union[bool, int, dict], varname: str, vartype): +def _get_keepbits(bitround: bool | int | dict, varname: str, vartype): # Guess the number of bits to keep depending on how bitround was passed, the var dtype and the var name. if not np.issubdtype(vartype, np.floating) or bitround is False: if isinstance(bitround, dict) and varname in bitround: @@ -355,12 +355,12 @@ def _get_keepbits(bitround: Union[bool, int, dict], varname: str, vartype): @parse_config def save_to_netcdf( ds: xr.Dataset, - filename: Union[str, os.PathLike], + filename: str | os.PathLike, *, - rechunk: Optional[dict] = None, - bitround: Union[bool, int, dict] = False, + rechunk: dict | None = None, + bitround: bool | int | dict = False, compute: bool = True, - netcdf_kwargs: Optional[dict] = None, + netcdf_kwargs: dict | None = None, ): """Save a Dataset to NetCDF, rechunking or compressing if requested. @@ -422,13 +422,13 @@ def save_to_netcdf( @parse_config def save_to_zarr( # noqa: C901 ds: xr.Dataset, - filename: Union[str, os.PathLike], + filename: str | os.PathLike, *, - rechunk: Optional[dict] = None, - zarr_kwargs: Optional[dict] = None, + rechunk: dict | None = None, + zarr_kwargs: dict | None = None, compute: bool = True, - encoding: Optional[dict] = None, - bitround: Union[bool, int, dict] = False, + encoding: dict | None = None, + bitround: bool | int | dict = False, mode: str = "f", itervar: bool = False, timeout_cleanup: bool = True, @@ -638,13 +638,13 @@ def _to_dataframe( def to_table( - ds: Union[xr.Dataset, xr.DataArray], + ds: xr.Dataset | xr.DataArray, *, - row: Optional[Union[str, Sequence[str]]] = None, - column: Optional[Union[str, Sequence[str]]] = None, - sheet: Optional[Union[str, Sequence[str]]] = None, - coords: Union[bool, str, Sequence[str]] = True, -) -> Union[pd.DataFrame, dict]: + row: str | Sequence[str] | None = None, + column: str | Sequence[str] | None = None, + sheet: str | Sequence[str] | None = None, + coords: bool | str | Sequence[str] = True, +) -> pd.DataFrame | dict: """Convert a dataset to a pandas DataFrame with support for multicolumns and multisheet. This function will trigger a computation of the dataset. @@ -737,9 +737,7 @@ def _ensure_list(seq): return _to_dataframe(da, **table_kwargs) -def make_toc( - ds: Union[xr.Dataset, xr.DataArray], loc: Optional[str] = None -) -> pd.DataFrame: +def make_toc(ds: xr.Dataset | xr.DataArray, loc: str | None = None) -> pd.DataFrame: """Make a table of content describing a dataset's variables. This return a simple DataFrame with variable names as index, the long_name as "description" and units. @@ -785,17 +783,17 @@ def make_toc( def save_to_table( - ds: Union[xr.Dataset, xr.DataArray], - filename: Union[str, os.PathLike], - output_format: Optional[str] = None, + ds: xr.Dataset | xr.DataArray, + filename: str | os.PathLike, + output_format: str | None = None, *, - row: Optional[Union[str, Sequence[str]]] = None, - column: Union[None, str, Sequence[str]] = "variable", - sheet: Optional[Union[str, Sequence[str]]] = None, - coords: Union[bool, Sequence[str]] = True, + row: str | Sequence[str] | None = None, + column: None | str | Sequence[str] = "variable", + sheet: str | Sequence[str] | None = None, + coords: bool | Sequence[str] = True, col_sep: str = "_", - row_sep: Optional[str] = None, - add_toc: Union[bool, pd.DataFrame] = False, + row_sep: str | None = None, + add_toc: bool | pd.DataFrame = False, **kwargs, ): """Save the dataset to a tabular file (csv, excel, ...). @@ -933,13 +931,13 @@ def rechunk_for_saving(ds: xr.Dataset, rechunk: dict): @parse_config def rechunk( - path_in: Union[os.PathLike, str, xr.Dataset], - path_out: Union[os.PathLike, str], + path_in: os.PathLike | str | xr.Dataset, + path_out: os.PathLike | str, *, - chunks_over_var: Optional[dict] = None, - chunks_over_dim: Optional[dict] = None, + chunks_over_var: dict | None = None, + chunks_over_dim: dict | None = None, worker_mem: str, - temp_store: Optional[Union[os.PathLike, str]] = None, + temp_store: os.PathLike | str | None = None, overwrite: bool = False, ) -> None: """Rechunk a dataset into a new zarr. @@ -1007,8 +1005,8 @@ def rechunk( def zip_directory( - root: Union[str, os.PathLike], - zipfile: Union[str, os.PathLike], + root: str | os.PathLike, + zipfile: str | os.PathLike, delete: bool = False, **zip_args, ): @@ -1042,7 +1040,7 @@ def _add_to_zip(zf, path, root): sh.rmtree(root) -def unzip_directory(zipfile: Union[str, os.PathLike], root: Union[str, os.PathLike]): +def unzip_directory(zipfile: str | os.PathLike, root: str | os.PathLike): r"""Unzip an archive to a directory. This function is the exact opposite of :py:func:`xscen.io.zip_directory`. diff --git a/src/xscen/reduce.py b/src/xscen/reduce.py index b7310769..33a4c1a8 100644 --- a/src/xscen/reduce.py +++ b/src/xscen/reduce.py @@ -12,10 +12,10 @@ @parse_config def build_reduction_data( - datasets: Union[dict, list[xr.Dataset]], + datasets: dict | list[xr.Dataset], *, - xrfreqs: Optional[list[str]] = None, - horizons: Optional[list[str]] = None, + xrfreqs: list[str] | None = None, + horizons: list[str] | None = None, ) -> xr.DataArray: """Construct the input required for ensemble reduction. @@ -84,11 +84,11 @@ def build_reduction_data( @parse_config def reduce_ensemble( - data: Union[xr.DataArray, dict, list, xr.Dataset], + data: xr.DataArray | dict | list | xr.Dataset, method: str, *, - horizons: Optional[list[str]] = None, - create_kwargs: Optional[dict] = None, + horizons: list[str] | None = None, + create_kwargs: dict | None = None, **kwargs, ): r"""Reduce an ensemble of simulations using clustering algorithms from xclim.ensembles. @@ -140,7 +140,7 @@ def reduce_ensemble( ) -def _concat_criteria(criteria: Optional[xr.DataArray], ens: xr.Dataset): +def _concat_criteria(criteria: xr.DataArray | None, ens: xr.Dataset): """Combine all variables and dimensions excepting 'realization'.""" if criteria is None: i = 0 diff --git a/src/xscen/regrid.py b/src/xscen/regrid.py index 24c680cd..ece74918 100644 --- a/src/xscen/regrid.py +++ b/src/xscen/regrid.py @@ -34,10 +34,10 @@ def regrid_dataset( # noqa: C901 ds: xr.Dataset, ds_grid: xr.Dataset, - weights_location: Union[str, os.PathLike], + weights_location: str | os.PathLike, *, - regridder_kwargs: Optional[dict] = None, - intermediate_grids: Optional[dict] = None, + regridder_kwargs: dict | None = None, + intermediate_grids: dict | None = None, to_level: str = "regridded", ) -> xr.Dataset: """Regrid a dataset according to weights and a reference grid. @@ -228,7 +228,7 @@ def regrid_dataset( # noqa: C901 @parse_config -def create_mask(ds: Union[xr.Dataset, xr.DataArray], mask_args: dict) -> xr.DataArray: +def create_mask(ds: xr.Dataset | xr.DataArray, mask_args: dict) -> xr.DataArray: """Create a 0-1 mask based on incoming arguments. Parameters @@ -302,10 +302,10 @@ def cmp(arg1, op, arg2): def _regridder( ds_in: xr.Dataset, ds_grid: xr.Dataset, - filename: Union[str, os.PathLike], + filename: str | os.PathLike, *, method: str = "bilinear", - unmapped_to_nan: Optional[bool] = True, + unmapped_to_nan: bool | None = True, **kwargs, ) -> Regridder: """Call to xesmf Regridder with a few default arguments. diff --git a/src/xscen/scripting.py b/src/xscen/scripting.py index 532ca06b..d772c1af 100644 --- a/src/xscen/scripting.py +++ b/src/xscen/scripting.py @@ -43,12 +43,12 @@ def send_mail( *, subject: str, msg: str, - to: Optional[str] = None, + to: str | None = None, server: str = "127.0.0.1", port: int = 25, - attachments: Optional[ - list[Union[tuple[str, Union[Figure, os.PathLike]], Figure, os.PathLike]] - ] = None, + attachments: None | ( + list[tuple[str, Figure | os.PathLike] | Figure | os.PathLike] + ) = None, ) -> None: """Send email. @@ -160,9 +160,9 @@ def err_handler(self, *exc_info): @parse_config def send_mail_on_exit( *, - subject: Optional[str] = None, - msg_ok: Optional[str] = None, - msg_err: Optional[str] = None, + subject: str | None = None, + msg_ok: str | None = None, + msg_err: str | None = None, on_error_only: bool = False, skip_ctrlc: bool = True, **mail_kwargs, @@ -244,7 +244,7 @@ class measure_time: def __init__( self, - name: Optional[str] = None, + name: str | None = None, cpu: bool = False, logger: logging.Logger = logger, ): @@ -311,9 +311,7 @@ def _timeout_handler(signum, frame): @contextmanager -def skippable( - seconds: int = 2, task: str = "", logger: Optional[logging.Logger] = None -): +def skippable(seconds: int = 2, task: str = "", logger: logging.Logger | None = None): """Skippable context manager. When CTRL-C (SIGINT, KeyboardInterrupt) is sent within the context, @@ -355,11 +353,11 @@ def skippable( def save_and_update( ds: xr.Dataset, pcat: ProjectCatalog, - path: Optional[Union[str, os.PathLike]] = None, - file_format: Optional[str] = None, - build_path_kwargs: Optional[dict] = None, - save_kwargs: Optional[dict] = None, - update_kwargs: Optional[dict] = None, + path: str | os.PathLike | None = None, + file_format: str | None = None, + build_path_kwargs: dict | None = None, + save_kwargs: dict | None = None, + update_kwargs: dict | None = None, ): """ Construct the path, save and delete. @@ -431,9 +429,9 @@ def save_and_update( def move_and_delete( - moving: list[list[Union[str, os.PathLike]]], + moving: list[list[str | os.PathLike]], pcat: ProjectCatalog, - deleting: Optional[list[Union[str, os.PathLike]]] = None, + deleting: list[str | os.PathLike] | None = None, copy: bool = False, ): """ diff --git a/src/xscen/spatial.py b/src/xscen/spatial.py index 5725d050..cde9c3e8 100644 --- a/src/xscen/spatial.py +++ b/src/xscen/spatial.py @@ -140,7 +140,7 @@ def subset( ds: xr.Dataset, method: str, *, - name: Optional[str] = None, + name: str | None = None, tile_buffer: float = 0, **kwargs, ) -> xr.Dataset: @@ -205,10 +205,10 @@ def subset( def _subset_gridpoint( ds: xr.Dataset, - lon: Union[float, Sequence[float], xr.DataArray], - lat: Union[float, Sequence[float], xr.DataArray], + lon: float | Sequence[float] | xr.DataArray, + lat: float | Sequence[float] | xr.DataArray, *, - name: Optional[str] = None, + name: str | None = None, **kwargs, ) -> xr.Dataset: r"""Subset the data to a gridpoint. @@ -254,10 +254,10 @@ def _subset_gridpoint( def _subset_bbox( ds: xr.Dataset, - lon_bnds: Union[tuple[float, float], list[float]], - lat_bnds: Union[tuple[float, float], list[float]], + lon_bnds: tuple[float, float] | list[float], + lat_bnds: tuple[float, float] | list[float], *, - name: Optional[str] = None, + name: str | None = None, tile_buffer: float = 0, **kwargs, ) -> xr.Dataset: @@ -317,9 +317,9 @@ def _subset_bbox( def _subset_shape( ds: xr.Dataset, - shape: Union[str, Path, gpd.GeoDataFrame], + shape: str | Path | gpd.GeoDataFrame, *, - name: Optional[str] = None, + name: str | None = None, tile_buffer: float = 0, **kwargs, ) -> xr.Dataset: @@ -360,7 +360,7 @@ def _subset_shape( lon_res, lat_res = _estimate_grid_resolution(ds) # The buffer argument needs to be in the same units as the shapefile, so it's simpler to always project the shapefile to WGS84. - if isinstance(shape, (str, Path)): + if isinstance(shape, str | Path): shape = gpd.read_file(shape) try: @@ -385,14 +385,14 @@ def _subset_shape( new_history = ( f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] " f"shape spatial subsetting with {'buffer=' + str(tile_buffer) if tile_buffer > 0 else 'no buffer'}" - f", shape={Path(shape).name if isinstance(shape, (str, Path)) else 'gpd.GeoDataFrame'}" + f", shape={Path(shape).name if isinstance(shape, str | Path) else 'gpd.GeoDataFrame'}" f" - clisops v{clisops.__version__}" ) return update_history_and_name(ds_subset, new_history, name) -def _subset_sel(ds: xr.Dataset, *, name: Optional[str] = None, **kwargs) -> xr.Dataset: +def _subset_sel(ds: xr.Dataset, *, name: str | None = None, **kwargs) -> xr.Dataset: r"""Subset the data using the .sel() method. Parameters diff --git a/src/xscen/testing.py b/src/xscen/testing.py index 866b0252..3aa519a7 100644 --- a/src/xscen/testing.py +++ b/src/xscen/testing.py @@ -22,9 +22,9 @@ def datablock_3d( y_step: float = 0.1, start: str = "7/1/2000", freq: str = "D", - units: Optional[str] = None, + units: str | None = None, as_dataset: bool = False, -) -> Union[xr.DataArray, xr.Dataset]: +) -> xr.DataArray | xr.Dataset: """Create a generic timeseries object based on pre-defined dictionaries of existing variables. Parameters diff --git a/src/xscen/utils.py b/src/xscen/utils.py index ab6e51b1..86538f59 100644 --- a/src/xscen/utils.py +++ b/src/xscen/utils.py @@ -78,12 +78,12 @@ def update_attr( - ds: Union[xr.Dataset, xr.DataArray], + ds: xr.Dataset | xr.DataArray, attr: str, new: str, - others: Optional[Sequence[Union[xr.Dataset, xr.DataArray]]] = None, + others: Sequence[xr.Dataset | xr.DataArray] | None = None, **fmt, -) -> Union[xr.Dataset, xr.DataArray]: +) -> xr.Dataset | xr.DataArray: r"""Format an attribute referencing itself in a translatable way. Parameters @@ -157,7 +157,7 @@ def update_attr( ) -def add_attr(ds: Union[xr.Dataset, xr.DataArray], attr: str, new: str, **fmt): +def add_attr(ds: xr.Dataset | xr.DataArray, attr: str, new: str, **fmt): """Add a formatted translatable attribute to a dataset.""" ds.attrs[attr] = new.format(**fmt) for loc in XC_OPTIONS[METADATA_LOCALES]: @@ -165,13 +165,13 @@ def add_attr(ds: Union[xr.Dataset, xr.DataArray], attr: str, new: str, **fmt): def date_parser( # noqa: C901 - date: Union[str, cftime.datetime, pd.Timestamp, datetime, pd.Period], + date: str | cftime.datetime | pd.Timestamp | datetime | pd.Period, *, - end_of_period: Union[bool, str] = False, + end_of_period: bool | str = False, out_dtype: str = "datetime", strtime_format: str = "%Y-%m-%d", freq: str = "H", -) -> Union[str, pd.Period, pd.Timestamp]: +) -> str | pd.Period | pd.Timestamp: """Return a datetime from a string. Parameters @@ -360,10 +360,10 @@ def translate_time_chunk(chunks: dict, calendar: str, timesize: int) -> dict: @parse_config def stack_drop_nans( ds: xr.Dataset, - mask: Union[xr.DataArray, list[str]], + mask: xr.DataArray | list[str], *, new_dim: str = "loc", - to_file: Optional[str] = None, + to_file: str | None = None, ) -> xr.Dataset: """Stack dimensions into a single axis and drops indexes where the mask is false. @@ -451,11 +451,9 @@ def unstack_fill_nan( ds: xr.Dataset, *, dim: str = "loc", - coords: Optional[ - Union[ - str, os.PathLike, Sequence[Union[str, os.PathLike]], dict[str, xr.DataArray] - ] - ] = None, + coords: None | ( + str | os.PathLike | Sequence[str | os.PathLike] | dict[str, xr.DataArray] + ) = None, ): """Unstack a Dataset that was stacked by :py:func:`stack_drop_nans`. @@ -504,7 +502,7 @@ def unstack_fill_nan( logger.info("Dataset unstacked using no coords argument.") coords = [d for d in ds.coords if ds[d].dims == (dim,)] - if isinstance(coords, (str, os.PathLike)): + if isinstance(coords, str | os.PathLike): # find original shape in the attrs of one of the dimension original_shape = "unknown" for c in ds.coords: @@ -602,7 +600,7 @@ def natural_sort(_list: list[str]): def get_cat_attrs( - ds: Union[xr.Dataset, xr.DataArray, dict], prefix: str = "cat:", var_as_str=False + ds: xr.Dataset | xr.DataArray | dict, prefix: str = "cat:", var_as_str=False ) -> dict: """Return the catalog-specific attributes from a dataset or dictionary. @@ -620,7 +618,7 @@ def get_cat_attrs( dict Compilation of all attributes in a dictionary. """ - if isinstance(ds, (xr.Dataset, xr.DataArray)): + if isinstance(ds, xr.Dataset | xr.DataArray): attrs = ds.attrs else: attrs = ds @@ -642,9 +640,9 @@ def get_cat_attrs( @parse_config def maybe_unstack( ds: xr.Dataset, - dim: Optional[str] = None, - coords: Optional[str] = None, - rechunk: Optional[dict] = None, + dim: str | None = None, + coords: str | None = None, + rechunk: dict | None = None, stack_drop_nans: bool = False, ) -> xr.Dataset: """If stack_drop_nans is True, unstack and rechunk. @@ -835,20 +833,18 @@ def change_units(ds: xr.Dataset, variables_and_units: dict) -> xr.Dataset: def clean_up( # noqa: C901 ds: xr.Dataset, *, - variables_and_units: Optional[dict] = None, - convert_calendar_kwargs: Optional[dict] = None, - missing_by_var: Optional[dict] = None, - maybe_unstack_dict: Optional[dict] = None, - round_var: Optional[dict] = None, - common_attrs_only: Optional[ - Union[dict, list[Union[xr.Dataset, str, os.PathLike]]] - ] = None, - common_attrs_open_kwargs: Optional[dict] = None, - attrs_to_remove: Optional[dict] = None, - remove_all_attrs_except: Optional[dict] = None, - add_attrs: Optional[dict] = None, - change_attr_prefix: Optional[Union[str, dict]] = None, - to_level: Optional[str] = None, + variables_and_units: dict | None = None, + convert_calendar_kwargs: dict | None = None, + missing_by_var: dict | None = None, + maybe_unstack_dict: dict | None = None, + round_var: dict | None = None, + common_attrs_only: None | (dict | list[xr.Dataset | str | os.PathLike]) = None, + common_attrs_open_kwargs: dict | None = None, + attrs_to_remove: dict | None = None, + remove_all_attrs_except: dict | None = None, + add_attrs: dict | None = None, + change_attr_prefix: str | dict | None = None, + to_level: str | None = None, ) -> xr.Dataset: """Clean up of the dataset. @@ -997,7 +993,7 @@ def clean_up( # noqa: C901 common_attrs_only = list(common_attrs_only.values()) for i in range(len(common_attrs_only)): - if isinstance(common_attrs_only[i], (str, os.PathLike)): + if isinstance(common_attrs_only[i], str | os.PathLike): dataset = xr.open_dataset( common_attrs_only[i], **common_attrs_open_kwargs ) @@ -1089,9 +1085,9 @@ def clean_up( # noqa: C901 def publish_release_notes( style: str = "md", - file: Optional[Union[os.PathLike, StringIO, TextIO]] = None, - changes: Optional[Union[str, os.PathLike]] = None, -) -> Optional[str]: + file: os.PathLike | StringIO | TextIO | None = None, + changes: str | os.PathLike | None = None, +) -> str | None: """Format release history in Markdown or ReStructuredText. Parameters @@ -1112,7 +1108,7 @@ def publish_release_notes( ----- This function exists solely for development purposes. Adapted from xclim.testing.utils.publish_release_notes. """ - if isinstance(changes, (str, Path)): + if isinstance(changes, str | Path): changes_file = Path(changes).absolute() else: changes_file = Path(__file__).absolute().parents[2].joinpath("CHANGELOG.rst") @@ -1164,15 +1160,15 @@ def publish_release_notes( if not file: return changes - if isinstance(file, (Path, os.PathLike)): + if isinstance(file, Path | os.PathLike): file = Path(file).open("w") print(changes, file=file) def unstack_dates( # noqa: C901 ds: xr.Dataset, - seasons: Optional[dict[int, str]] = None, - new_dim: Optional[str] = None, + seasons: dict[int, str] | None = None, + new_dim: str | None = None, winter_starts_year: bool = False, ): """Unstack a multi-season timeseries into a yearly axis and a season one. @@ -1345,9 +1341,9 @@ def reshape_da(da): def show_versions( - file: Optional[Union[os.PathLike, StringIO, TextIO]] = None, - deps: Optional[list] = None, -) -> Optional[str]: + file: os.PathLike | StringIO | TextIO | None = None, + deps: list | None = None, +) -> str | None: """Print the versions of xscen and its dependencies. Parameters @@ -1467,8 +1463,8 @@ def ensure_correct_time(ds: xr.Dataset, xrfreq: str) -> xr.Dataset: def standardize_periods( - periods: Optional[Union[list[str], list[list[str]]]], multiple: bool = True -) -> Optional[Union[list[str], list[list[str]]]]: + periods: list[str] | list[list[str]] | None, multiple: bool = True +) -> list[str] | list[list[str]] | None: """Reformats the input to a list of strings, ['start', 'end'], or a list of such lists. Parameters @@ -1505,7 +1501,7 @@ def standardize_periods( return periods[0] -def season_sort_key(idx: pd.Index, name: Optional[str] = None): +def season_sort_key(idx: pd.Index, name: str | None = None): """Get a proper sort key for a "season" or "month" index to avoid alphabetical sorting. If any of the values in the index is not recognized as a 3-letter @@ -1608,7 +1604,7 @@ def _xarray_defaults(**kwargs): return kwargs -def rechunk_for_resample(obj: Union[xr.DataArray, xr.Dataset], **resample_kwargs): +def rechunk_for_resample(obj: xr.DataArray | xr.Dataset, **resample_kwargs): if not uses_dask(obj): return obj From 2423aedc8c2d7cf05980a3a06b134f99e3ca4311 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 18 Sep 2024 11:41:33 -0400 Subject: [PATCH 05/10] remove deprecated folders --- .github/deactivated/conda-build.yml | 60 ----------------------- conda/xscen/meta.yaml | 76 ----------------------------- conda/xscen/recipe.yaml | 75 ---------------------------- 3 files changed, 211 deletions(-) delete mode 100644 .github/deactivated/conda-build.yml delete mode 100644 conda/xscen/meta.yaml delete mode 100644 conda/xscen/recipe.yaml diff --git a/.github/deactivated/conda-build.yml b/.github/deactivated/conda-build.yml deleted file mode 100644 index d8d7bc22..00000000 --- a/.github/deactivated/conda-build.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Build and upload conda packages - -on: - release: - types: - - released - - prereleased - workflow_dispatch: - inputs: - tag: - description: 'Tag to be built and uploaded' - required: true - type: string - label: - description: 'The type of release' - default: 'dev' - type: choice - options: - - dev - - main - -jobs: - conda_deployment_with_tag: - name: Build conda package with Python${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ "3.9" ] - steps: - - uses: actions/checkout@v4.1.1 - if: ${{ github.event.inputs.tag == '' }} - - uses: actions/checkout@v4.1.1 - if: ${{ github.event.inputs.tag != '' }} - with: - fetch-depth: 0 - ref: ${{ inputs.tag }} - - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} - uses: mamba-org/provision-with-micromamba@v16 - with: - cache-downloads: true - channels: conda-forge,defaults - extra-specs: | - python=${{ matrix.python-version }} - anaconda-client - conda-build - - name: Conditionally set label - uses: haya14busa/action-cond@v1.1.1 - id: label - with: - cond: ${{ github.event_name == 'workflow_dispatch' }} - if_true: ${{ github.event.inputs.label }} - if_false: "auto" - - name: Build and upload the conda packages - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - with: - meta_yaml_dir: conda/xscen - python-version: ${{ matrix.python-version }} - user: Ouranosinc - label: ${{ steps.label.outputs.value }} - token: ${{ secrets.ANACONDA_TOKEN }} diff --git a/conda/xscen/meta.yaml b/conda/xscen/meta.yaml deleted file mode 100644 index f1bfff33..00000000 --- a/conda/xscen/meta.yaml +++ /dev/null @@ -1,76 +0,0 @@ -{% set name = "xscen" %} - -package: - name: {{ name|lower }} - version: {{ environ['GIT_DESCRIBE_TAG'] }} - -source: - path: ../../ - -channels: - - conda-forge - - defaults - -build: - noarch: python - script: {{ PYTHON }} -m pip install . -vv - number: 0 - -requirements: - host: - - python >=3.9 - - pip - run: - - cartopy - - cftime - - cf_xarray >=0.7.6 - - clisops >=0.10 - - dask-core - - flox - - fsspec <2023.10.0 - - geopandas - - h5netcdf - - h5py - - intake-esm >=2023.07.07 - - matplotlib - - netCDF4 - - numcodecs - - numpy - - pandas >= 2 - - parse - - pyyaml - - rechunker - - scipy - - shapely >= 2 - - sparse <=0.14 - - toolz - - xarray <2023.11.0 # FIXME: Remove when pandas 2.2 is released and xclim is fixed. - - xclim >=0.46.0 - - xesmf >=0.7 - - zarr - # Opt - - nc-time-axis >=1.3.1 - - pyarrow >=1.0.0 - -test: - imports: - - xscen - commands: - - pip check - requires: - - pip - - pytest-json-report # Added due to a bug in esmpy. See: https://github.com/esmf-org/esmf/issues/115 - -about: - home: https://github.com/Ouranosinc/xscen - summary: A climate change scenario-building analysis framework, built with xclim/xarray. - license: Apache-2.0 - license_file: LICENSE - -extra: - recipe-maintainers: - # GitHub.com - - Zeitsperre - - RondeauG - - aulemahal - - juliettelavoie diff --git a/conda/xscen/recipe.yaml b/conda/xscen/recipe.yaml deleted file mode 100644 index 81494f77..00000000 --- a/conda/xscen/recipe.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Build recipe using `boa` build standard. Not suitable for conda-forge. See: https://github.com/mamba-org/boa - -context: - name: xscen - version: 0.5.0 - -package: - name: '{{ name|lower }}' - version: '{{ version }}' - -source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/xscen-{{ version }}.tar.gz - sha256: f31df2cb52e87dd82d2fc7d340788e4edf14abccf04685a9249a2067594b721a - -build: - noarch: python - script: '{{ PYTHON }} -m pip install . -vv' - number: 1 - -requirements: - host: - - python >=3.9 - - pip - run: - - cartopy - - cftime - - cf_xarray >=0.7.6 - - clisops >=0.10 - - dask-core - - flox - - fsspec <2023.10.0 - - geopandas - - h5netcdf - - h5py - - intake-esm >=2023.07.07 - - matplotlib - - netCDF4 - - numcodecs - - numpy - - pandas >= 2 - - parse - - pyyaml - - rechunker - - scipy - - shapely >= 2 - - sparse <=0.14 - - toolz - - xarray <2023.11.0 # FIXME: Remove when pandas 2.2 is released and xclim is fixed. - - xclim >=0.46.0 - - xesmf >=0.7 - - zarr - # Opt - - nc-time-axis >=1.3.1 - - pyarrow >=1.0.0 - -test: - imports: - - xscen - commands: - - pip check - requires: - - pip - - pytest-json-report # Added due to a bug in esmpy. See: https://github.com/esmf-org/esmf/issues/115 - -about: - home: https://github.com/Ouranosinc/xscen - summary: A climate change scenario-building analysis framework, built with xclim/xarray. - license: Apache-2.0 - license_file: LICENSE - -extra: - recipe-maintainers: - # Anaconda.org - - Zeitsperre - - aule From 738be6c7fb2067cc62ed92aae8a99c0f4c1e1503 Mon Sep 17 00:00:00 2001 From: RondeauG <38501935+RondeauG@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:46:17 -0400 Subject: [PATCH 06/10] Update pyproject.toml Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 73ba1818..da70739a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -258,7 +258,6 @@ markers = ["requires_netcdf: marks tests that require netcdf files to run"] [tool.ruff] src = ["src/xscen"] line-length = 150 -target-version = "py310" exclude = [ ".eggs", ".git", From ca081dd99ab5895a29b961ba57ca527deeb8ce17 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 18 Sep 2024 11:51:08 -0400 Subject: [PATCH 07/10] adress review comments --- .yamllint.yaml | 3 --- pyproject.toml | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.yamllint.yaml b/.yamllint.yaml index 92c0cacb..fc95e170 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -37,6 +37,3 @@ rules: trailing-spaces: {} truthy: disable - -ignore: | - conda/xscen/meta.yaml diff --git a/pyproject.toml b/pyproject.toml index da70739a..a8a3122a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,7 +127,8 @@ all = ["xscen[dev]", "xscen[docs]", "xscen[extra]"] target-version = [ "py310", "py311", - "py312" + "py312", + "py313" ] [tool.bumpversion] From ffe6777b854d599d1f2038aaa2560fa8377d26b0 Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 18 Sep 2024 13:29:30 -0400 Subject: [PATCH 08/10] remove deprecated link --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6157ff99..7dc1163b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,6 @@ repos: - id: check-toml - id: check-yaml args: [ '--allow-multiple-documents' ] - exclude: conda/xscen/meta.yaml - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: From 0d36dc8b672cafeb0854de46884f0a5fdab40cca Mon Sep 17 00:00:00 2001 From: RondeauG Date: Wed, 18 Sep 2024 13:42:12 -0400 Subject: [PATCH 09/10] SPEC0 pins --- environment-dev.yml | 12 ++++++------ environment.yml | 12 ++++++------ pyproject.toml | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 2d62b72c..0f014a48 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -17,22 +17,22 @@ dependencies: - h5netcdf - h5py <3.11 # writting and reading with engine h5netcdf was broken - intake-esm >=2023.07.07 - - matplotlib + - matplotlib >=3.6 - netCDF4 <1.7 # writting and reading with engine h5netcdf was broken - numcodecs - - numpy + - numpy >=1.24 - pandas >=2.2 - parse - pyyaml - rechunker - - scipy + - scipy >=1.10 - shapely >=2.0 - sparse - toolz - - xarray >=2023.11.0, !=2024.6.0, <2024.09.0 - - xclim >=0.50, <0.51 + - xarray >=2023.11.0, !=2024.6.0 + - xclim >=0.52.2, <0.53 - xesmf >=0.7 - - zarr + - zarr >=2.13 # Opt - nc-time-axis >=1.3.1 - pyarrow >=10.0.1 diff --git a/environment.yml b/environment.yml index 7fb1a9d0..3a4bea73 100644 --- a/environment.yml +++ b/environment.yml @@ -17,22 +17,22 @@ dependencies: - h5netcdf - h5py <3.11 - intake-esm >=2023.07.07 - - matplotlib + - matplotlib >=3.6 - netCDF4 <1.7 - numcodecs - - numpy + - numpy >=1.24 - pandas >=2.2 - parse - pyyaml - rechunker - - scipy + - scipy >=1.10 - shapely >=2.0 - sparse - toolz - - xarray >=2023.11.0, !=2024.6.0, <2024.09.0 - - xclim >=0.50, <0.51 + - xarray >=2023.11.0, !=2024.6.0 + - xclim >=0.52.2, <0.53 - xesmf >=0.7 - - zarr + - zarr >=2.13 # To install from source - setuptools >=65.0.0 - setuptools-scm >=8.0.0 diff --git a/pyproject.toml b/pyproject.toml index 05bc9177..88a92406 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,23 +50,23 @@ dependencies = [ "h5netcdf", "h5py", "intake-esm >=2023.07.07", - "matplotlib", + "matplotlib >=3.6", "netCDF4 <1.7", "numcodecs", - "numpy", + "numpy >=1.24", "pandas >=2.2", "parse", # Used when opening catalogs. "pyarrow>=10.0.1", "pyyaml", "rechunker", - "scipy", + "scipy >=1.10", "shapely >=2.0", "sparse", "toolz", - "xarray >=2023.11.0, !=2024.6.0, <2024.09.0", - "xclim >=0.50, <0.51", - "zarr" + "xarray >=2023.11.0, !=2024.6.0", + "xclim >=0.52.2, <0.53", + "zarr >=2.13" ] [project.optional-dependencies] From 5c6428bd6938c50812324080056c28d26f755a69 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:53:22 -0400 Subject: [PATCH 10/10] pin numpy below v2.0 for netcdf4 support --- environment-dev.yml | 6 +++--- environment.yml | 6 +++--- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 0f014a48..8e3e949e 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -15,12 +15,12 @@ dependencies: - fsspec - geopandas - h5netcdf - - h5py <3.11 # writting and reading with engine h5netcdf was broken + - h5py <3.11 # writing and reading with engine h5netcdf was broken - intake-esm >=2023.07.07 - matplotlib >=3.6 - - netCDF4 <1.7 # writting and reading with engine h5netcdf was broken + - netCDF4 <1.7 # writing and reading with engine h5netcdf was broken - numcodecs - - numpy >=1.24 + - numpy >=1.24,<2.0 # v2.0 is not supported by python-netcdf4 - pandas >=2.2 - parse - pyyaml diff --git a/environment.yml b/environment.yml index 3a4bea73..9f0cb1bf 100644 --- a/environment.yml +++ b/environment.yml @@ -15,12 +15,12 @@ dependencies: - fsspec - geopandas - h5netcdf - - h5py <3.11 + - h5py <3.11 # writing and reading with engine h5netcdf was broken - intake-esm >=2023.07.07 - matplotlib >=3.6 - - netCDF4 <1.7 + - netCDF4 <1.7 # writing and reading with engine h5netcdf was broken - numcodecs - - numpy >=1.24 + - numpy >=1.24,<2.0 # v2.0 is not supported by python-netcdf4 - pandas >=2.2 - parse - pyyaml diff --git a/pyproject.toml b/pyproject.toml index 88a92406..1b4e7062 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "matplotlib >=3.6", "netCDF4 <1.7", "numcodecs", - "numpy >=1.24", + "numpy >=1.24,<2.0", # v2.0 is not supported by python-netcdf4 "pandas >=2.2", "parse", # Used when opening catalogs.