Unpin netcdf4 and numpy (#459)

<!-- Please ensure the PR fulfills the following requirements! -->
<!-- If this is your first PR, make sure to add your details to the
AUTHORS.rst! -->
### Pull Request Checklist:
- [ ] This PR addresses an already opened issue (for bug fixes /
features)
    - This PR fixes #xyz
- [ ] (If applicable) Documentation has been added / updated (for bug
fixes / features).
- [ ] (If applicable) Tests have been added.
- [x] This PR does not seem to break the templates.
- [x] CHANGELOG.rst has been updated (with summary of main changes).
- [x] Link to issue (:issue:`number`) and pull request (:pull:`number`)
has been added.

### What kind of change does this PR introduce?

* Circumvents incompatibility issues between versions of numpy, netcdf4
and hypy.

### Does this PR introduce a breaking change?


### Other information:

CHANGELOG.rst

@@ -37,6 +37,7 @@ Bug fixes
 * Fix ``xs.catalog.concat_data_catalogs`` for catalogs that have not been search yet. (:pull:`431`).
 * Fix indicator computation using ``freq=2Q*`` by assuming this means a semiannual frequency anchored at the given month (pandas assumes 2 quarter steps, any of them anchored at the given month). (:pull:`431`).
 * ``create_bounds_rotated_pole`` now uses the default value if the dataset has no `north_pole_grid_longitude` attribute, instead of crashing. (:pull:`455`).
+* Rewrote the global tas data file with latest HDF5/h5py to avoid errors when using h5py 3.11 and hdf5 1.14.2. (:pull:`1861`).
 
 Breaking changes
 ^^^^^^^^^^^^^^^^

environment-dev.yml

@@ -15,12 +15,12 @@ dependencies:
   - fsspec
   - geopandas
   - h5netcdf
-  - h5py <3.11 # writing and reading with engine h5netcdf was broken
+  - h5py
   - intake-esm >=2023.07.07
   - matplotlib >=3.6
-  - netCDF4 <1.7 # writing and reading with engine h5netcdf was broken
+  - netCDF4
   - numcodecs
-  - numpy >=1.24,<2.0  # v2.0 is not supported by python-netcdf4
+  - numpy >=1.24
   - pandas >=2.2
   - parse
   - pyyaml

environment.yml

@@ -15,12 +15,12 @@ dependencies:
   - fsspec
   - geopandas
   - h5netcdf
-  - h5py <3.11 # writing and reading with engine h5netcdf was broken
+  - h5py
   - intake-esm >=2023.07.07
   - matplotlib >=3.6
-  - netCDF4 <1.7 # writing and reading with engine h5netcdf was broken
+  - netCDF4
   - numcodecs
-  - numpy >=1.24,<2.0  # v2.0 is not supported by python-netcdf4
+  - numpy >=1.24
   - pandas >=2.2
   - parse
   - pyyaml

pyproject.toml

@@ -48,12 +48,12 @@ dependencies = [
   "fsspec",
   "geopandas",
   "h5netcdf",
-  "h5py",
+  "h5py", # <3.11", # writing and reading with engine h5netcdf was broken
   "intake-esm >=2023.07.07",
   "matplotlib >=3.6",
-  "netCDF4 <1.7",
+  "netCDF4",
   "numcodecs",
-  "numpy >=1.24,<2.0", # v2.0 is not supported by python-netcdf4
+  "numpy >=1.24",
   "pandas >=2.2",
   "parse",
   # Used when opening catalogs.

src/xscen/extract.py

@@ -994,7 +994,7 @@ def get_warming_level(  # noqa: C901
         info_models.append(info)
 
     # open nc
-    tas = xr.open_dataset(tas_src, engine="h5netcdf").tas
+    tas = xr.open_dataset(tas_src).tas
 
     def _get_warming_level(model):
         # choose colum based in ds cat attrs, +'$' to ensure a full match (matches end-of-string)
@@ -1236,8 +1236,8 @@ def subset_warming_level(
         else:
             # WL not reached, not in ds, or not fully contained in ds.time
             if wl_not_reached:
-                ds_wl = ds.isel(time=slice(0, fake_time.size)) * np.NaN
-                wlbnds = (("warminglevel", "wl_bounds"), [[np.NaN, np.NaN]])
+                ds_wl = ds.isel(time=slice(0, fake_time.size)) * np.nan
+                wlbnds = (("warminglevel", "wl_bounds"), [[np.nan, np.nan]])
             else:
                 wlbnds = (
                     ("warminglevel", "wl_bounds"),
@@ -1337,9 +1337,9 @@ def _dispatch_historical_to_future(
                 "For example, xscen expects experiment `historical` to have `CMIP` activity "
                 "and experiments `sspXYZ` to have `ScenarioMIP` activity. "
             )
-        for activity_id in set(sdf.activity) - {"HighResMip", np.NaN}:
+        for activity_id in set(sdf.activity) - {"HighResMip", np.nan}:
             sub_sdf = sdf[sdf.activity == activity_id]
-            for exp_id in set(sub_sdf.experiment) - {"historical", "piControl", np.NaN}:
+            for exp_id in set(sub_sdf.experiment) - {"historical", "piControl", np.nan}:
                 exp_hist = hist.copy()
                 exp_hist["experiment"] = exp_id
                 exp_hist["activity"] = activity_id

tests/test_scripting.py

@@ -45,6 +45,8 @@ def test_save_and_update(self):
             TestScripting.ds,
             cat,
             file_format="nc",
+            # To fix hdf5 issues with h5py 3.11 on pip
+            save_kwargs=dict(netcdf_kwargs={"engine": "netcdf4"}),
             build_path_kwargs={"root": root},
         )