xclim 0.48 compatibility (#351)

<!-- Please ensure the PR fulfills the following requirements! -->
<!-- If this is your first PR, make sure to add your details to the
AUTHORS.rst! -->
### Pull Request Checklist:
- [x] This PR addresses an already opened issue (for bug fixes /
features)
    - No
- [x] (If applicable) Documentation has been added / updated (for bug
fixes / features).
- [x] (If applicable) Tests have been added.
- [x] This PR does not seem to break the templates.
- [x] CHANGES.rst has been updated (with summary of main changes).
- [x] Link to issue (:issue:`number`) and pull request (:pull:`number`)
has been added.
- [x] Maybe... Add a workaround to convert the old xrfreq nomenclature
to the new one upon loading an existing catalog or parsing a directory.

### What kind of change does this PR introduce?

* Changes to be compatible with recent versions of `pandas/xarray/xclim`

### Does this PR introduce a breaking change?

- `xscen` now requires `pandas >= 2.2`
- Functions that output a dict with keys as xrfreq (`extract_dataset,
compute_indicators`) will now return the new nomenclature.
- The `frequency --> xrfreq/timedelta` will now only result in the new
nomenclature for hours (lower case `h`).

### Other information:

.github/workflows/main.yml

@@ -52,7 +52,7 @@ jobs:
           python -m tox -e lint
 
   test-pypi:
-    name: Test with Python${{ matrix.python-version }} (PyPI/tox)
+    name: ${{ matrix.tox-build }} (Python${{ matrix.python-version }})
     needs: lint
     runs-on: ubuntu-latest
     env:
@@ -67,9 +67,9 @@ jobs:
           - python-version: "3.10"
             tox-build: "py310-coveralls"
           - python-version: "3.11"
-            tox-build: "py311-coveralls"
-#          - python-version: "3.12"
-#            tox-build: "py312-esmpy-coveralls"
+            tox-build: "py311-xclim-coveralls"
+          - python-version: "3.12"
+            tox-build: "py312-esmpy-coveralls"
     defaults:
       run:
         shell: bash -l {0}
@@ -124,12 +124,20 @@ jobs:
 #          COVERALLS_SERVICE_NAME: github
 
   test-conda:
-    name: Test with Python${{ matrix.python-version }} (Anaconda)
+    name: Python${{ matrix.python-version }} (conda, upstream=${{ matrix.xclim-upstream }})
     needs: lint
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        include:
+          - python-version: "3.9"
+            xclim-upstream: false
+          - python-version: "3.10"
+            xclim-upstream: false
+          - python-version: "3.11"
+            xclim-upstream: false
+          - python-version: "3.12"
+            xclim-upstream: true
     defaults:
       run:
         shell: bash -l {0}
@@ -151,6 +159,10 @@ jobs:
         if: matrix.python-version == '3.9'
         run: |
           micromamba install -y -c conda-forge intake-esm=2023.11.10
+      - name: Install xclim main branch
+        if: matrix.xclim-upstream
+        run: |
+          python -m pip install git+https://github.com/Ouranosinc/xclim.git@main
       - name: Conda and Mamba versions
         run: |
           mamba --version

CHANGES.rst

@@ -4,15 +4,24 @@ Changelog
 
 v0.8.3 (unreleased)
 -------------------
-Contributors to this version: Juliette Lavoie (:user:`juliettelavoie`), Trevor James Smith (:user:`Zeitsperre`).
+Contributors to this version: Juliette Lavoie (:user:`juliettelavoie`), Trevor James Smith (:user:`Zeitsperre`), Gabriel Rondeau-Genesse (:user:`RondeauG`), Pascal Bourgault (:user:`aulemahal`).
 
 Announcements
 ^^^^^^^^^^^^^
-* `xscen` is now has a `security disclosure policy <https://github.com/Ouranosinc/xscen/tree/main?tab=security-ov-file#security-ov-file>`_. (:pull:`353`).
+* `xscen` now has a `security disclosure policy <https://github.com/Ouranosinc/xscen/tree/main?tab=security-ov-file#security-ov-file>`_. (:pull:`353`).
+* Various frequency-related changes to match the new `pandas` naming conventions. (:pull:`351`).
 
 Internal changes
 ^^^^^^^^^^^^^^^^
 * Added tests for diagnostics. (:pull:`352`).
+* Added a `SECURITY.md` file to the repository and the documentation. (:pull:`353`).
+* Added `tox` modifier for testing builds against the `main` development branch of `xclim`. (:pull:`351`).
+
+Breaking changes
+^^^^^^^^^^^^^^^^
+* `xscen` now requires `pandas` >= 2.2 and `xclim` >= 0.48.2. (:pull:`351`).
+* Functions that output a dict with keys as xrfreq (such as ``extract_dataset``, ``compute_indicators``) will now return the new nomenclature (e.g. "YS-JAN" instead of "AS-JAN"). (:pull:`351`).
+* Going from `xrfreq` to frequencies or timedeltas will still work, but the opposite (frequency --> xrfreq/timedelta) will now only result in the new pandas nomenclature. (:pull:`351`).
 
 v0.8.2 (2024-02-12)
 -------------------

docs/notebooks/1_catalog.ipynb

@@ -324,7 +324,7 @@
    "source": [
     "import xscen as xs\n",
     "\n",
-    "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3H\", \"sftlf\": \"fx\"}\n",
+    "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3h\", \"sftlf\": \"fx\"}\n",
     "other_search_criteria = {\"institution\": [\"NOAA-GFDL\"]}\n",
     "\n",
     "cat_sim = xs.search_data_catalogs(\n",
@@ -376,7 +376,7 @@
    },
    "outputs": [],
    "source": [
-    "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3H\", \"sftlf\": \"fx\"}\n",
+    "variables_and_freqs = {\"tasmin\": \"D\", \"pr\": \"3h\", \"sftlf\": \"fx\"}\n",
     "other_search_criteria = {\"institution\": [\"NOAA-GFDL\"], \"experiment\": [\"ssp585\"]}\n",
     "\n",
     "cat_sim = xs.search_data_catalogs(\n",
@@ -1129,6 +1129,14 @@
     "PC.df[\"path\"] = newdf[\"new_path\"]\n",
     "PC.update()"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c7acfa82-e2e7-42ce-b24e-ab5d4acc6c57",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -1146,7 +1154,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,

environment-dev.yml

@@ -2,7 +2,7 @@ name: xscen-dev
 channels:
   - conda-forge
 dependencies:
-  - python >=3.9
+  - python >=3.9,<3.13
   # Don't forget to sync changes between environment.yml, environment-dev.yml, and setup.py!
   # Also consider updating the list in xs.utils.show_versions if you add a new package.
   # Main packages
@@ -21,16 +21,16 @@ dependencies:
   - netCDF4
   - numcodecs
   - numpy
-  - pandas >=2.0,<2.2
+  - pandas >=2.2
   - parse
   - pyyaml
   - rechunker
   - scipy
   - shapely >=2.0
   - sparse
   - toolz
-  - xarray <2023.11.0  # FIXME: Remove when pandas 2.2 is released and xclim is fixed.
-  - xclim >=0.46.0
+  - xarray >=2023.11.0
+  - xclim >=0.48.2
   - xesmf >=0.7
   - zarr
   # Opt
@@ -41,8 +41,8 @@ dependencies:
   - black ==24.1.1
   - blackdoc ==0.3.9
   - bump-my-version >=0.17.1
-  - coverage>=6.2.2,<7.0.0
-  - coveralls>=3.3.1
+#  - coverage>=6.2.2,<7.0.0
+#  - coveralls>=3.3.1
   - flake8 >=6.1.0
   - flake8-rst-docstrings>=0.3.0
   - ipykernel
@@ -55,7 +55,7 @@ dependencies:
   - pooch
   - pre-commit >=3.3.2
   - pytest >=7.3.1
-  - pytest-cov >=4.0.0
+#  - pytest-cov >=4.0.0
   - ruff >=0.1.0
   - sphinx
   - sphinx-autoapi
@@ -66,9 +66,14 @@ dependencies:
   - sphinx-mdinclude
   - watchdog >=3.0.0
   - xdoctest
-  - pip
   # Testing
   - tox >=4.5.1
   # packaging
   - build
   - wheel
+  - pip
+  - pip:
+     # coverage is not available in conda-forge for Python3.12
+     - coverage>=6.2.2,<7.0.0
+     - coveralls>=3.3.1
+     - pytest-cov >=4.0.0

environment.yml

@@ -2,7 +2,7 @@ name: xscen
 channels:
   - conda-forge
 dependencies:
-  - python >=3.9
+  - python >=3.9,<3.13
   # Don't forget to sync changes between environment.yml, environment-dev.yml, and setup.py!
   # Also consider updating the list in xs.utils.show_versions if you add a new package.
   # Main packages
@@ -21,16 +21,16 @@ dependencies:
   - netCDF4
   - numcodecs
   - numpy
-  - pandas >=2.0,<2.2
+  - pandas >=2.2
   - parse
   - pyyaml
   - rechunker
   - scipy
   - shapely >=2.0
   - sparse
   - toolz
-  - xarray <2023.11.0  # FIXME: Remove when pandas 2.2 is released and xclim is fixed.
-  - xclim >=0.46.0
+  - xarray >=2023.11.0
+  - xclim >=0.48.2
   - xesmf >=0.7
   - zarr
   # To install from source and get translations

pyproject.toml

@@ -1,7 +1,7 @@
 [build-system]
 requires = [
-  "setuptools>=60",
-  "setuptools-scm>=8.0",
+  "setuptools >=60",
+  "setuptools-scm >=8.0",
   "babel",
   "wheel"
 ]
@@ -29,7 +29,7 @@ classifiers = [
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
-  # "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: Implementation :: CPython",
   "Topic :: Scientific/Engineering :: Atmospheric Science"
 ]
@@ -38,8 +38,8 @@ dependencies = [
   "babel",
   "cartopy",
   "cftime",
-  "cf_xarray>=0.7.6",
-  "clisops>=0.10",
+  "cf_xarray >=0.7.6",
+  "clisops >=0.10",
   "dask",
   "flox",
   "fsspec",
@@ -52,41 +52,41 @@ dependencies = [
   "netCDF4",
   "numcodecs",
   "numpy",
-  "pandas>=2.0,<2.2",
+  "pandas >=2.2",
   "parse",
   # Used when opening catalogs.
   "pyarrow",
   "pyyaml",
   "rechunker",
   "scipy",
-  "shapely>=2.0",
+  "shapely >=2.0",
   "sparse",
   "toolz",
-  "xarray<2023.11.0", # FIXME: Remove when pandas 2.2 is released and xclim is fixed.
-  "xclim>=0.46.0",
+  "xarray >=2023.11.0",
+  "xclim >=0.48.2",
   "zarr"
 ]
 
 [project.optional-dependencies]
 dev = [
   # Dev tools and testing
-  "pip>=23.3.0",
-  "black==24.2.0",
-  "blackdoc==0.3.9",
-  "bump-my-version>=0.17.1",
-  "coverage>=6.2.2,<8.0.0",
-  "coveralls>=3.3.1",
-  "flake8-alphabetize>=0.0.21",
-  "flake8-rst-docstrings>=0.3.0",
-  "flake8>=6.1.0",
-  "isort==5.13.2",
+  "pip >=23.3.0",
+  "black ==24.2.0",
+  "blackdoc ==0.3.9",
+  "bump-my-version >=0.17.1",
+  "coverage >=6.2.2,<8.0.0",
+  "coveralls >=3.3.1",
+  "flake8-alphabetize >=0.0.21",
+  "flake8-rst-docstrings >=0.3.0",
+  "flake8 >=6.1.0",
+  "isort ==5.13.2",
   "pooch",
-  "pre-commit>=3.3.2",
-  "pytest-cov>=4.0.0",
-  "pytest>=7.3.1",
-  "ruff>=0.1.0",
-  "tox>=4.5.1",
-  "watchdog>=3.0.0",
+  "pre-commit >=3.3.2",
+  "pytest-cov >=4.0.0",
+  "pytest >=7.3.1",
+  "ruff >=0.1.0",
+  "tox >=4.5.1",
+  "watchdog >=3.0.0",
   "xdoctest"
 ]
 docs = [
@@ -102,7 +102,7 @@ docs = [
   "sphinx-copybutton",
   "sphinx-intl",
   "sphinx-mdinclude",
-  "sphinx-rtd-theme>=1.0",
+  "sphinx-rtd-theme >=1.0",
   "sphinxcontrib-napoleon"
 ]
 extra = [

tests/test_catalog.py

@@ -1,4 +1,5 @@
 import pandas as pd
+from conftest import SAMPLES_DIR
 
 from xscen import catalog
 
@@ -29,3 +30,8 @@ def test_subset_file_coverage():
         catalog.subset_file_coverage(df, [1951, 1976], coverage=0.8),
         pd.DataFrame(columns=df.columns),
     )
+
+
+def test_xrfreq_fix():
+    cat = catalog.DataCatalog(SAMPLES_DIR.parent / "pangeo-cmip6.json")
+    assert set(cat.df.xrfreq) == {"3h", "D", "fx"}

tests/test_catutils.py

@@ -230,7 +230,7 @@ def test_build_path(samplecat):
 
 def test_build_path_ds():
     ds = xr.tutorial.open_dataset("air_temperature")
-    ds = ds.assign(time=xr.cftime_range("0001-01-01", freq="6H", periods=ds.time.size))
+    ds = ds.assign(time=xr.cftime_range("0001-01-01", freq="6h", periods=ds.time.size))
     ds.attrs.update(source="source", institution="institution")
     new_path = cu.build_path(
         ds,
@@ -240,7 +240,7 @@ def test_build_path_ds():
         },
     )
     assert new_path == Path(
-        "source/institution/air_6H/source_institution_air_6hr_0001-0002"
+        "source/institution/air_6h/source_institution_air_6hr_0001-0002"
     )
 
 

tests/test_diagnostics.py

@@ -127,7 +127,7 @@ def test_variables(self):
             xs.diagnostics.health_checks(ds, variables_and_units={"tas": "degC"})
         with pytest.warns(
             UserWarning,
-            match="Data units kelvin are not compatible with requested mm.",
+            match="are not compatible with requested mm.",
         ):
             xs.diagnostics.health_checks(ds, variables_and_units={"tas": "mm"})
 
@@ -164,7 +164,7 @@ def test_cfchecks(self):
             xs.diagnostics.health_checks(ds, cfchecks=bad_cfcheck)
 
     @pytest.mark.parametrize(
-        "freq, gap", [("D", False), ("MS", False), ("3H", False), ("D", True)]
+        "freq, gap", [("D", False), ("MS", False), ("3h", False), ("D", True)]
     )
     def test_freq(self, freq, gap):
         ds = timeseries(