Merge branch 'master' into add-keywords

.github/workflows/main.yml

@@ -70,9 +70,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install tox
-        run: pip install tox~=4.0
+        run: |
+          pip install tox~=4.0
       - name: Test with tox
-        run: tox -e ${{ matrix.tox-env }}
+        run: |
+          tox -e ${{ matrix.tox-env }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_FLAG_NAME: run-{{ matrix.tox-env }}
@@ -117,9 +119,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install tox
-        run: pip install tox~=4.0
+        run: |
+          pip install tox~=4.0
       - name: Test with tox
-        run: tox -e ${{ matrix.tox-env }} -- ${{ matrix.markers }}
+        run: |
+          tox -e ${{ matrix.tox-env }} -- ${{ matrix.markers }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_FLAG_NAME: run-{{ matrix.tox-env }}

CHANGES.rst

@@ -4,7 +4,7 @@ Changelog
 
 v0.46.0 (unreleased)
 --------------------
-Contributors to this version: Éric Dupuis (:user:`coxipi`), Trevor James Smith (:user:`Zeitsperre`), David Huard (:user:`huard`) and Pascal Bourgault (:user:`aulemahal`).
+Contributors to this version: Éric Dupuis (:user:`coxipi`), Trevor James Smith (:user:`Zeitsperre`), David Huard (:user:`huard`), Pascal Bourgault (:user:`aulemahal`).
 
 New features and enhancements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -13,6 +13,8 @@ New features and enhancements
 * The testing suite now offers a means of running tests in "offline" mode (using `pytest-socket <https://github.com/miketheman/pytest-socket>`_ to block external connections). This requires a local copy of `xclim-testdata` to be present in the user's home cache directory and for certain `pytest` options and markers to be set when invoked. For more information, see the contributing documentation section for `Running Tests in Offline Mode`. (:issue:`1468`, :pull:`1473`).
 * The `SKIP_NOTEBOOKS` flag to speed up docs builds is now documented. See the contributing documentation section `Get Started!` for details. (:issue:`1470`, :pull:`1476`).
 * Refactored the indicators page with the addition of a search bar.
+* Indicator ``generic.stats`` now accepts any frequency (previously only daily). (:pull:`1498`).
+* Added argument ``out_units`` to ``select_resample_op`` to bypass limitations of ``to_agg_units`` in custom indicators. Add "var" to supported operations in ``to_agg_units``. (:pull:`1498`).
 
 Bug fixes
 ^^^^^^^^^
@@ -43,6 +45,7 @@ Internal changes
 * GitHub testing workflows now use `Concurrency` instead of the styfle/cancel-workflow-action to cancel redundant workflows. (:pull:`1487`).
 * The `pkg_resources` library has been replaced for the `packaging` library when version comparisons have been performed, and a few warning messages have been silenced in the testing suite. (:issue:`1489`, :pull:`1490`).
 * New ``xclim.testing.helpers.assert_lazy`` context manager to assert the laziness of code blocks. (:pull:`1484`).
+* Added a fix for the deprecation warnings that `importlib.resources` throws, made backwards-compatible for Python3.8 with `importlib_resources` backport. (:pull:`1485`).
 
 v0.45.0 (2023-09-05)
 --------------------

environment.yml

@@ -11,6 +11,7 @@ dependencies:
     - cftime>=1.4.1
     - Click >=8.1
     - dask>=2.6.0
+    - importlib-resources  # For Python3.8
     - jsonpickle
     - lmoments3
     - numba
@@ -42,6 +43,7 @@ dependencies:
     - ipython
     - matplotlib
     - mypy
+    - nbqa
     - nbsphinx
     - nbval
     - nc-time-axis

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
   "cftime>=1.4.1",
   "Click>=8.1",
   "dask[array]>=2.6",
+  "importlib-resources; python_version == '3.8'",
   "jsonpickle",
   "lmoments3>=1.0.5",
   "numba",

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.45.13-beta
+current_version = 0.45.15-beta
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+).(?P<patch>\d+)(\-(?P<release>[a-z]+))?

tests/test_generic_indicators.py

@@ -102,3 +102,9 @@ def test_missing(self, ndq_series):
 
         np.testing.assert_array_equal(out.sel(time="1900").isnull(), False)
         np.testing.assert_array_equal(out.sel(time="1902").isnull(), True)
+
+    def test_3hourly(self, pr_hr_series, random):
+        pr = pr_hr_series(random.random(366 * 24)).resample(time="3H").mean()
+        out = generic.stats(pr, freq="MS", op="var")
+        assert out.units == "kg^2 m-4 s-2"
+        assert out.long_name == "Variance of variable"

tests/test_indicators.py

@@ -592,7 +592,7 @@ def test_AttrFormatter():
     # Missing mod:
     assert fmt.format("{adj}", adj="evil") == "méchant"
     # Mod with unknown value
-    with pytest.raises(ValueError):
+    with pytest.warns(match="Requested formatting `m` for unknown string `funny`."):
         fmt.format("{adj:m}", adj="funny")
 
 

tests/test_missing.py

@@ -14,6 +14,15 @@
 class TestMissingBase:
     """The base class is well tested for daily input through the subclasses."""
 
+    def test_3hourly_input(self, random):
+        """Creating array with 21 days of 3h"""
+        n = 21 * 8
+        time = xr.cftime_range(start="2002-01-01", periods=n, freq="3H")
+        ts = xr.DataArray(random.random(n), dims="time", coords={"time": time})
+        mb = missing.MissingBase(ts, freq="MS", src_timestep="3H")
+        # Make sure count is 31  * 8, because we're requesting a MS freq.
+        assert mb.count == 31 * 8
+
     def test_monthly_input(self, random):
         """Creating array with 11 months."""
         n = 11

xclim/__init__.py

@@ -1,7 +1,10 @@
 """Climate indices computation package based on Xarray."""
 from __future__ import annotations
 
-from importlib.resources import contents, path
+try:
+    from importlib.resources import files as _files
+except ImportError:
+    from importlib_resources import files as _files
 
 from xclim.core import units  # noqa
 from xclim.core.indicator import build_indicator_module_from_yaml
@@ -11,22 +14,18 @@
 
 __author__ = """Travis Logan"""
 __email__ = "[email protected]"
-__version__ = "0.45.13-beta"
+__version__ = "0.45.15-beta"
 
 
+_module_data = _files("xclim.data")
+
 # Load official locales
-for filename in contents("xclim.data"):
+for filename in _module_data.glob("??.json"):
     # Only select <locale>.json and not <module>.<locale>.json
-    if filename.endswith(".json") and filename.count(".") == 1:
-        locale = filename.split(".")[0]
-        with path("xclim.data", filename) as f:
-            load_locale(f, locale)
+    load_locale(filename, filename.stem)
 
 
 # Virtual modules creation:
-with path("xclim.data", "icclim.yml") as f:
-    build_indicator_module_from_yaml(f.with_suffix(""), mode="raise")
-with path("xclim.data", "anuclim.yml") as f:
-    build_indicator_module_from_yaml(f.with_suffix(""), mode="raise")
-with path("xclim.data", "cf.yml") as f:
-    build_indicator_module_from_yaml(f.with_suffix(""), mode="raise")
+build_indicator_module_from_yaml(_module_data / "icclim", mode="raise")
+build_indicator_module_from_yaml(_module_data / "anuclim", mode="raise")
+build_indicator_module_from_yaml(_module_data / "cf", mode="raise")

xclim/core/formatting.py

@@ -6,9 +6,9 @@
 
 import datetime as dt
 import itertools
-import logging
 import re
 import string
+import warnings
 from ast import literal_eval
 from fnmatch import fnmatch
 from inspect import _empty, signature  # noqa
@@ -124,18 +124,30 @@ def format_field(self, value, format_spec):
         'La moyenne annuelle est faite sur un échantillon mensuel'
         """
         baseval = self._match_value(value)
-        if baseval is not None and not format_spec:
+        if baseval is None:  # Not something we know how to translate
+            if format_spec in self.modifiers + [
+                "r"
+            ]:  # Woops, however a known format spec was asked
+                warnings.warn(
+                    f"Requested formatting `{format_spec}` for unknown string `{value}`."
+                )
+                format_spec = ""
+            return super().format_field(value, format_spec)
+        # Thus, known value
+
+        if not format_spec:  # (None or '') No modifiers, return first
             return self.mapping[baseval][0]
 
-        if format_spec in self.modifiers:
-            if baseval is not None:
-                return self.mapping[baseval][self.modifiers.index(format_spec)]
-            raise ValueError(
-                f"No known mapping for string '{value}' with modifier '{format_spec}'"
-            )
-        if format_spec == "r":
+        if format_spec == "r":  # Raw modifier
             return super().format_field(value, "")
-        return super().format_field(value, format_spec)
+
+        if format_spec in self.modifiers:  # Known modifier
+            if len(self.mapping[baseval]) == 1:  # But unmodifiable entry
+                return self.mapping[baseval][0]
+            # Known modifier, modifiable entry
+            return self.mapping[baseval][self.modifiers.index(format_spec)]
+        # Known value but unknown modifier, must be a built-in one, only works for the default val...
+        return super().format_field(self.mapping[baseval][0], format_spec)
 
     def _match_value(self, value):
         if isinstance(value, str):
@@ -178,8 +190,11 @@ def _match_value(self, value):
         "min": ["minimal", "minimum"],
         "sum": ["total", "sum"],
         "std": ["standard deviation"],
+        "var": ["variance"],
         "absamp": ["absolute amplitude"],
         "relamp": ["relative amplitude"],
+        # For when we are formatting indicator classes with empty options
+        "<class 'inspect._empty'>": ["<empty>"],
     },
     ["adj", "noun"],
 )

xclim/core/missing.py

@@ -173,7 +173,10 @@ def prepare(self, da, freq, src_timestep, **indexer):
 
         else:
             delta = end_time - start_time
-            n = delta.values.astype(_np_timedelta64[src_timestep]).astype(float)
+            n = (
+                delta.values.astype(_np_timedelta64[offset[1]]).astype(float)
+                / offset[0]
+            )
 
             if freq:
                 count = xr.DataArray(n, coords={"time": c.time}, dims="time")

xclim/core/units.py

@@ -11,9 +11,14 @@
 import logging
 import re
 import warnings
-from importlib.resources import open_text
+
+try:
+    from importlib.resources import files
+except ImportError:
+    from importlib_resources import files
+
 from inspect import _empty, signature  # noqa
-from typing import Any, Callable, Tuple
+from typing import Any, Callable
 
 import numpy as np
 import pint
@@ -112,7 +117,8 @@
 units.add_context(hydro)
 
 
-CF_CONVERSIONS = safe_load(open_text("xclim.data", "variables.yml"))["conversions"]
+with (files("xclim.data") / "variables.yml").open() as f:
+    CF_CONVERSIONS = safe_load(f)["conversions"]
 _CONVERSIONS = {}
 
 
@@ -520,7 +526,7 @@ def to_agg_units(
     orig : xr.DataArray
         The original array before the aggregation operation,
         used to infer the sampling units and get the variable units.
-    op : {'min', 'max', 'mean', 'std', 'doymin', 'doymax',  'count', 'integral'}
+    op : {'min', 'max', 'mean', 'std', 'var', 'doymin', 'doymax',  'count', 'integral', 'sum'}
         The type of aggregation operation performed. The special "delta_*" ops are used
         with temperature units needing conversion to their "delta" counterparts (e.g. degree days)
     dim : str
@@ -574,6 +580,9 @@ def to_agg_units(
     if op in ["amin", "min", "amax", "max", "mean", "std"]:
         out.attrs["units"] = orig.attrs["units"]
 
+    elif op in ["var"]:
+        out.attrs["units"] = pint2cfunits(str2pint(orig.units) ** 2)
+
     elif op in ["doymin", "doymax"]:
         out.attrs.update(
             units="", is_dayofyear=np.int32(1), calendar=get_calendar(orig)
@@ -591,7 +600,7 @@ def to_agg_units(
             out.attrs["units"] = pint2cfunits(orig_u * freq_u)
     else:
         raise ValueError(
-            f"Aggregation op {op} not in [min, max, mean, std, doymin, doymax, count, integral]."
+            f"Unknown aggregation op {op}. Known ops are [min, max, mean, std, var, doymin, doymax, count, integral, sum]."
         )
 
     return out

xclim/core/utils.py

@@ -13,11 +13,16 @@
 from collections import defaultdict
 from enum import IntEnum
 from functools import partial
-from importlib.resources import open_text
+
+try:
+    from importlib.resources import files
+except ImportError:
+    from importlib_resources import files
+
 from inspect import Parameter, _empty  # noqa
 from io import StringIO
 from pathlib import Path
-from typing import Callable, Mapping, NewType, Sequence, TypeVar
+from typing import Callable, NewType, Sequence, TypeVar
 
 import numpy as np
 import xarray as xr
@@ -37,8 +42,9 @@
 #: Type annotation for thresholds and other not-exactly-a-variable quantities
 Quantified = TypeVar("Quantified", xr.DataArray, str, Quantity)
 
-VARIABLES = safe_load(open_text("xclim.data", "variables.yml"))["variables"]
-"""Official variables definitions.
+with (files("xclim.data") / "variables.yml").open() as f:
+    VARIABLES = safe_load(f)["variables"]
+    """Official variables definitions.
 
 A mapping from variable name to a dict with the following keys:
 

xclim/data/fr.json

@@ -143,6 +143,9 @@
     "std": [
       "écart-type"
     ],
+    "var": [
+      "variance"
+    ],
     "absamp": [
       "amplitude absolue"
     ],
@@ -661,24 +664,12 @@
     "title": "Valeur de retour",
     "abstract": "Analyse fréquentielle selon un mode et une distribution."
   },
-  "DISCHARGE_STATS": {
-    "long_name": "Statistique des débits quotidiens",
-    "description": "{op} {freq:f} des débits quotidiens ({indexer}).",
-    "title": "Calcul de statistiques sur des sous-périodes.",
-    "abstract": ""
-  },
   "STATS": {
-    "long_name": "Statistique des valeurs quotidiennes",
-    "description": "{op} {freq:f} des valeurs quotidiennes ({indexer}).",
+    "long_name": "{op:nom} de la variable.",
+    "description": "{op:nom} {freq:f} da la variable ({indexer}).",
     "title": "Calcul de statistiques sur des sous-périodes.",
     "abstract": ""
   },
-  "DISCHARGE_DISTRIBUTION_FIT": {
-    "long_name": "Paramètres d'une distribution {dist:f} d'une série de débits",
-    "description": "Paramètres d'une distribution {dist:f} d'une série de débits.",
-    "title": "Calcul les paramètres d'une distribution univariée pour une série de débits",
-    "abstract": ""
-  },
   "FIT": {
     "long_name": "Paramètres d'une distribution {dist:f}",
     "description": "Paramètres d'une distribution {dist:f}.",
@@ -931,8 +922,8 @@
     "abstract": "Calcule le premier jour d'une période où la température maximale quotidienne est plus élevée qu'un certain seuil durant un nombre de jours donné, limité par une date minimale."
   },
   "DEGREE_DAYS_EXCEEDANCE_DATE": {
-    "long_name": "Premier jour de l'année où l'intégral de la température moyenne quotidienne {op} {thresh} est au-dessus de {sum_thresh}, avec la somme cumulative à partir de {after_date}",
-    "description": "Premier jour de l'année où l'intégral des degrés-jours (ou température moyenne quotidienne {op} {thresh}) est au-dessus de {sum_thresh}, avec la somme cumulative à partir de {after_date}.",
+    "long_name": "Premier jour de l'année où l'intégrale de la température moyenne quotidienne {op} {thresh} est au-dessus de {sum_thresh}, avec la somme cumulative à partir de {after_date}",
+    "description": "Premier jour de l'année où l'intégrale des degrés-jours (ou température moyenne quotidienne {op} {thresh}) est au-dessus de {sum_thresh}, avec la somme cumulative à partir de {after_date}.",
     "title": "Jour du dépassement des degrés-jours",
     "abstract": "Jour de l'année où la somme des degrés-jours est au-dessus d'un seuil donné, survenant après une date donnée. Les degrés-jours sont calculés au-dessus ou en dessous d'un seuil de température donné."
   },

xclim/indicators/generic/_stats.py

@@ -44,11 +44,11 @@ class GenericResampling(ResamplingIndicator):
 
 
 stats = GenericResampling(
-    title="Statistic of the daily values for a given period.",
+    title="Simple resampled statistic of the values.",
     identifier="stats",
     var_name="stat_{indexer}{op:r}",
-    long_name="Daily statistics",
-    description="{freq} {op} of daily values ({indexer}).",
+    long_name="{op:noun} of variable",
+    description="{freq} {op:noun} of variable ({indexer}).",
     compute=select_resample_op,
-    src_freq="D",
+    parameters=dict(out_units=None),
 )