Merge branch 'main' into namedarray_dtype_type

.github/workflows/ci-additional.yaml

@@ -6,12 +6,6 @@ on:
   pull_request:
     branches:
       - "main"
-    paths:
-      - 'ci/**'
-      - '.github/**'
-      - '/*'  # covers files such as `pyproject.toml`
-      - 'properties/**'
-      - 'xarray/**'
   workflow_dispatch: # allows you to trigger manually
 
 concurrency:
@@ -136,7 +130,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report xarray/
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.4.1
+        uses: codecov/codecov-action@v4.5.0
         with:
           file: mypy_report/cobertura.xml
           flags: mypy
@@ -190,7 +184,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report xarray/
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.4.1
+        uses: codecov/codecov-action@v4.5.0
         with:
           file: mypy_report/cobertura.xml
           flags: mypy39
@@ -251,7 +245,7 @@ jobs:
           python -m pyright xarray/
 
       - name: Upload pyright coverage to Codecov
-        uses: codecov/codecov-action@v4.4.1
+        uses: codecov/codecov-action@v4.5.0
         with:
           file: pyright_report/cobertura.xml
           flags: pyright
@@ -310,7 +304,7 @@ jobs:
           python -m pyright xarray/
 
       - name: Upload pyright coverage to Codecov
-        uses: codecov/codecov-action@v4.4.1
+        uses: codecov/codecov-action@v4.5.0
         with:
           file: pyright_report/cobertura.xml
           flags: pyright39

.github/workflows/ci.yaml

@@ -6,13 +6,6 @@ on:
   pull_request:
     branches:
       - "main"
-    paths:
-      - "ci/**"
-      - ".github/**"
-      - "/*" # covers files such as `pyproject.toml`
-      - "/.*" # ...and dotfiles in the root such as `.pre-commit-config.yaml`
-      - "properties/**"
-      - "xarray/**"
   workflow_dispatch: # allows you to trigger manually
 
 concurrency:
@@ -54,7 +47,7 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         # Bookend python versions
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.9",  "3.12"]
         env: [""]
         include:
           # Minimum python version:
@@ -166,7 +159,7 @@ jobs:
           path: pytest.xml
 
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v4.4.1
+        uses: codecov/codecov-action@v4.5.0
         with:
           file: ./coverage.xml
           flags: unittests

.github/workflows/hypothesis.yaml

@@ -39,9 +39,9 @@ jobs:
     if: |
         always()
         && (
-            (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
-            || needs.detect-ci-trigger.outputs.triggered == 'true'
-            || contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis')
+            needs.detect-ci-trigger.outputs.triggered == 'false'
+            && ( (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+                || contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis'))
         )
     defaults:
       run:

.github/workflows/pypi-release.yaml

@@ -88,7 +88,7 @@ jobs:
           path: dist
       - name: Publish package to TestPyPI
         if: github.event_name == 'push'
-        uses: pypa/gh-action-pypi-publish@v1.8.14
+        uses: pypa/gh-action-pypi-publish@v1.9.0
         with:
           repository_url: https://test.pypi.org/legacy/
           verbose: true
@@ -111,6 +111,6 @@ jobs:
           name: releases
           path: dist
       - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.8.14
+        uses: pypa/gh-action-pypi-publish@v1.9.0
         with:
           verbose: true

.github/workflows/upstream-dev-ci.yaml

@@ -146,7 +146,7 @@ jobs:
         run: |
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.4.1
+        uses: codecov/codecov-action@v4.5.0
         with:
           file: mypy_report/cobertura.xml
           flags: mypy

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
       - id: mixed-line-ending
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.3'
+    rev: 'v0.4.7'
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]

CITATION.cff

@@ -11,6 +11,7 @@ authors:
   orcid: "https://orcid.org/0000-0001-7479-8439"
 - family-names: "Magin"
   given-names: "Justus"
+  orcid: "https://orcid.org/0000-0002-4254-8002"
 - family-names: "Cherian"
   given-names: "Deepak"
   orcid: "https://orcid.org/0000-0002-6861-8734"
@@ -75,6 +76,19 @@ authors:
 - family-names: "Wolfram"
   given-names: "Phillip J."
   orcid: "https://orcid.org/0000-0001-5971-4241"
+- family-names: "Henderson"
+  given-names: "Scott"
+  orcid: "https://orcid.org/0000-0003-0624-4965"
+- family-names: "Awowale"
+  given-names: "Eniola Olufunke"
+- family-names: "Scheick"
+  given-names: "Jessica"
+  orcid: "https://orcid.org/0000-0002-3421-4459"
+- family-names: "Savoie"
+  given-names: "Matthew"
+  orcid: "https://orcid.org/0000-0002-8881-2550"
+- family-names: "Littlejohns"
+  given-names: "Owen"
 title: "xarray"
 abstract: "N-D labeled arrays and datasets in Python."
 license: Apache-2.0

README.md

@@ -7,7 +7,7 @@
 [![Available on pypi](https://img.shields.io/pypi/v/xarray.svg)](https://pypi.python.org/pypi/xarray/)
 [![Formatted with black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black)
 [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/)
-[![Mirror on zendoo](https://zenodo.org/badge/DOI/10.5281/zenodo.598201.svg)](https://doi.org/10.5281/zenodo.598201)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.11183201.svg)](https://doi.org/10.5281/zenodo.11183201)
 [![Examples on binder](https://img.shields.io/badge/launch-binder-579ACA.svg?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAFkAAABZCAMAAABi1XidAAAB8lBMVEX///9XmsrmZYH1olJXmsr1olJXmsrmZYH1olJXmsr1olJXmsrmZYH1olL1olJXmsr1olJXmsrmZYH1olL1olJXmsrmZYH1olJXmsr1olL1olJXmsrmZYH1olL1olJXmsrmZYH1olL1olL0nFf1olJXmsrmZYH1olJXmsq8dZb1olJXmsrmZYH1olJXmspXmspXmsr1olL1olJXmsrmZYH1olJXmsr1olL1olJXmsrmZYH1olL1olLeaIVXmsrmZYH1olL1olL1olJXmsrmZYH1olLna31Xmsr1olJXmsr1olJXmsrmZYH1olLqoVr1olJXmsr1olJXmsrmZYH1olL1olKkfaPobXvviGabgadXmsqThKuofKHmZ4Dobnr1olJXmsr1olJXmspXmsr1olJXmsrfZ4TuhWn1olL1olJXmsqBi7X1olJXmspZmslbmMhbmsdemsVfl8ZgmsNim8Jpk8F0m7R4m7F5nLB6jbh7jbiDirOEibOGnKaMhq+PnaCVg6qWg6qegKaff6WhnpKofKGtnomxeZy3noG6dZi+n3vCcpPDcpPGn3bLb4/Mb47UbIrVa4rYoGjdaIbeaIXhoWHmZYHobXvpcHjqdHXreHLroVrsfG/uhGnuh2bwj2Hxk17yl1vzmljzm1j0nlX1olL3AJXWAAAAbXRSTlMAEBAQHx8gICAuLjAwMDw9PUBAQEpQUFBXV1hgYGBkcHBwcXl8gICAgoiIkJCQlJicnJ2goKCmqK+wsLC4usDAwMjP0NDQ1NbW3Nzg4ODi5+3v8PDw8/T09PX29vb39/f5+fr7+/z8/Pz9/v7+zczCxgAABC5JREFUeAHN1ul3k0UUBvCb1CTVpmpaitAGSLSpSuKCLWpbTKNJFGlcSMAFF63iUmRccNG6gLbuxkXU66JAUef/9LSpmXnyLr3T5AO/rzl5zj137p136BISy44fKJXuGN/d19PUfYeO67Znqtf2KH33Id1psXoFdW30sPZ1sMvs2D060AHqws4FHeJojLZqnw53cmfvg+XR8mC0OEjuxrXEkX5ydeVJLVIlV0e10PXk5k7dYeHu7Cj1j+49uKg7uLU61tGLw1lq27ugQYlclHC4bgv7VQ+TAyj5Zc/UjsPvs1sd5cWryWObtvWT2EPa4rtnWW3JkpjggEpbOsPr7F7EyNewtpBIslA7p43HCsnwooXTEc3UmPmCNn5lrqTJxy6nRmcavGZVt/3Da2pD5NHvsOHJCrdc1G2r3DITpU7yic7w/7Rxnjc0kt5GC4djiv2Sz3Fb2iEZg41/ddsFDoyuYrIkmFehz0HR2thPgQqMyQYb2OtB0WxsZ3BeG3+wpRb1vzl2UYBog8FfGhttFKjtAclnZYrRo9ryG9uG/FZQU4AEg8ZE9LjGMzTmqKXPLnlWVnIlQQTvxJf8ip7VgjZjyVPrjw1te5otM7RmP7xm+sK2Gv9I8Gi++BRbEkR9EBw8zRUcKxwp73xkaLiqQb+kGduJTNHG72zcW9LoJgqQxpP3/Tj//c3yB0tqzaml05/+orHLksVO+95kX7/7qgJvnjlrfr2Ggsyx0eoy9uPzN5SPd86aXggOsEKW2Prz7du3VID3/tzs/sSRs2w7ovVHKtjrX2pd7ZMlTxAYfBAL9jiDwfLkq55Tm7ifhMlTGPyCAs7RFRhn47JnlcB9RM5T97ASuZXIcVNuUDIndpDbdsfrqsOppeXl5Y+XVKdjFCTh+zGaVuj0d9zy05PPK3QzBamxdwtTCrzyg/2Rvf2EstUjordGwa/kx9mSJLr8mLLtCW8HHGJc2R5hS219IiF6PnTusOqcMl57gm0Z8kanKMAQg0qSyuZfn7zItsbGyO9QlnxY0eCuD1XL2ys/MsrQhltE7Ug0uFOzufJFE2PxBo/YAx8XPPdDwWN0MrDRYIZF0mSMKCNHgaIVFoBbNoLJ7tEQDKxGF0kcLQimojCZopv0OkNOyWCCg9XMVAi7ARJzQdM2QUh0gmBozjc3Skg6dSBRqDGYSUOu66Zg+I2fNZs/M3/f/Grl/XnyF1Gw3VKCez0PN5IUfFLqvgUN4C0qNqYs5YhPL+aVZYDE4IpUk57oSFnJm4FyCqqOE0jhY2SMyLFoo56zyo6becOS5UVDdj7Vih0zp+tcMhwRpBeLyqtIjlJKAIZSbI8SGSF3k0pA3mR5tHuwPFoa7N7reoq2bqCsAk1HqCu5uvI1n6JuRXI+S1Mco54YmYTwcn6Aeic+kssXi8XpXC4V3t7/ADuTNKaQJdScAAAAAElFTkSuQmCC)](https://mybinder.org/v2/gh/pydata/xarray/main?urlpath=lab/tree/doc/examples/weather-data.ipynb)
 [![Twitter](https://img.shields.io/twitter/follow/xarray_dev?style=social)](https://twitter.com/xarray_dev)
 
@@ -46,15 +46,15 @@ provide a powerful and concise interface. For example:
 
 - Apply operations over dimensions by name: `x.sum('time')`.
 - Select values by label instead of integer location:
-    `x.loc['2014-01-01']` or `x.sel(time='2014-01-01')`.
+  `x.loc['2014-01-01']` or `x.sel(time='2014-01-01')`.
 - Mathematical operations (e.g., `x - y`) vectorize across multiple
-    dimensions (array broadcasting) based on dimension names, not shape.
+  dimensions (array broadcasting) based on dimension names, not shape.
 - Flexible split-apply-combine operations with groupby:
-    `x.groupby('time.dayofyear').mean()`.
+  `x.groupby('time.dayofyear').mean()`.
 - Database like alignment based on coordinate labels that smoothly
-    handles missing values: `x, y = xr.align(x, y, join='outer')`.
+  handles missing values: `x, y = xr.align(x, y, join='outer')`.
 - Keep track of arbitrary metadata in the form of a Python dictionary:
-    `x.attrs`.
+  `x.attrs`.
 
 ## Documentation
 
@@ -73,12 +73,12 @@ page](https://docs.xarray.dev/en/stable/contributing.html).
 ## Get in touch
 
 - Ask usage questions ("How do I?") on
-    [GitHub Discussions](https://github.com/pydata/xarray/discussions).
+  [GitHub Discussions](https://github.com/pydata/xarray/discussions).
 - Report bugs, suggest features or view the source code [on
-    GitHub](https://github.com/pydata/xarray).
+  GitHub](https://github.com/pydata/xarray).
 - For less well defined questions or ideas, or to announce other
-    projects of interest to xarray users, use the [mailing
-    list](https://groups.google.com/forum/#!forum/xarray).
+  projects of interest to xarray users, use the [mailing
+  list](https://groups.google.com/forum/#!forum/xarray).
 
 ## NumFOCUS
 
@@ -114,7 +114,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License. You may
 obtain a copy of the License at
 
- <https://www.apache.org/licenses/LICENSE-2.0>
+<https://www.apache.org/licenses/LICENSE-2.0>
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

asv_bench/benchmarks/dataarray_missing.py

@@ -66,7 +66,7 @@ def time_ffill(self, shape, chunks, limit):
         ),
     )
     def time_bfill(self, shape, chunks, limit):
-        actual = self.da.ffill(dim="time", limit=limit)
+        actual = self.da.bfill(dim="time", limit=limit)
 
         if chunks is not None:
             actual = actual.compute()

asv_bench/benchmarks/dataset_io.py

@@ -7,6 +7,8 @@
 import pandas as pd
 
 import xarray as xr
+from xarray.backends.api import open_datatree
+from xarray.core.datatree import DataTree
 
 from . import _skip_slow, parameterized, randint, randn, requires_dask
 
@@ -16,7 +18,6 @@
 except ImportError:
     pass
 
-
 os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
 
 _ENGINES = tuple(xr.backends.list_engines().keys() - {"store"})
@@ -469,6 +470,116 @@ def create_delayed_write():
     return ds.to_netcdf("file.nc", engine="netcdf4", compute=False)
 
 
+class IONestedDataTree:
+    """
+    A few examples that benchmark reading/writing a heavily nested netCDF datatree with
+    xarray
+    """
+
+    timeout = 300.0
+    repeat = 1
+    number = 5
+
+    def make_datatree(self, nchildren=10):
+        # multiple Dataset
+        self.ds = xr.Dataset()
+        self.nt = 1000
+        self.nx = 90
+        self.ny = 45
+        self.nchildren = nchildren
+
+        self.block_chunks = {
+            "time": self.nt / 4,
+            "lon": self.nx / 3,
+            "lat": self.ny / 3,
+        }
+
+        self.time_chunks = {"time": int(self.nt / 36)}
+
+        times = pd.date_range("1970-01-01", periods=self.nt, freq="D")
+        lons = xr.DataArray(
+            np.linspace(0, 360, self.nx),
+            dims=("lon",),
+            attrs={"units": "degrees east", "long_name": "longitude"},
+        )
+        lats = xr.DataArray(
+            np.linspace(-90, 90, self.ny),
+            dims=("lat",),
+            attrs={"units": "degrees north", "long_name": "latitude"},
+        )
+        self.ds["foo"] = xr.DataArray(
+            randn((self.nt, self.nx, self.ny), frac_nan=0.2),
+            coords={"lon": lons, "lat": lats, "time": times},
+            dims=("time", "lon", "lat"),
+            name="foo",
+            attrs={"units": "foo units", "description": "a description"},
+        )
+        self.ds["bar"] = xr.DataArray(
+            randn((self.nt, self.nx, self.ny), frac_nan=0.2),
+            coords={"lon": lons, "lat": lats, "time": times},
+            dims=("time", "lon", "lat"),
+            name="bar",
+            attrs={"units": "bar units", "description": "a description"},
+        )
+        self.ds["baz"] = xr.DataArray(
+            randn((self.nx, self.ny), frac_nan=0.2).astype(np.float32),
+            coords={"lon": lons, "lat": lats},
+            dims=("lon", "lat"),
+            name="baz",
+            attrs={"units": "baz units", "description": "a description"},
+        )
+
+        self.ds.attrs = {"history": "created for xarray benchmarking"}
+
+        self.oinds = {
+            "time": randint(0, self.nt, 120),
+            "lon": randint(0, self.nx, 20),
+            "lat": randint(0, self.ny, 10),
+        }
+        self.vinds = {
+            "time": xr.DataArray(randint(0, self.nt, 120), dims="x"),
+            "lon": xr.DataArray(randint(0, self.nx, 120), dims="x"),
+            "lat": slice(3, 20),
+        }
+        root = {f"group_{group}": self.ds for group in range(self.nchildren)}
+        nested_tree1 = {
+            f"group_{group}/subgroup_1": xr.Dataset() for group in range(self.nchildren)
+        }
+        nested_tree2 = {
+            f"group_{group}/subgroup_2": xr.DataArray(np.arange(1, 10)).to_dataset(
+                name="a"
+            )
+            for group in range(self.nchildren)
+        }
+        nested_tree3 = {
+            f"group_{group}/subgroup_2/sub-subgroup_1": self.ds
+            for group in range(self.nchildren)
+        }
+        dtree = root | nested_tree1 | nested_tree2 | nested_tree3
+        self.dtree = DataTree.from_dict(dtree)
+
+
+class IOReadDataTreeNetCDF4(IONestedDataTree):
+    def setup(self):
+        # TODO: Lazily skipped in CI as it is very demanding and slow.
+        # Improve times and remove errors.
+        _skip_slow()
+
+        requires_dask()
+
+        self.make_datatree()
+        self.format = "NETCDF4"
+        self.filepath = "datatree.nc4.nc"
+        dtree = self.dtree
+        dtree.to_netcdf(filepath=self.filepath)
+
+    def time_load_datatree_netcdf4(self):
+        open_datatree(self.filepath, engine="netcdf4").load()
+
+    def time_open_datatree_netcdf4(self):
+        open_datatree(self.filepath, engine="netcdf4")
+
+
 class IOWriteNetCDFDask:
     timeout = 60
     repeat = 1

ci/install-upstream-wheels.sh

@@ -48,6 +48,8 @@ python -m pip install \
     --pre \
     --upgrade \
     pyarrow
+# manually install `pint` to pull in new dependencies
+python -m pip install --upgrade pint
 python -m pip install \
     --no-deps \
     --upgrade \

ci/requirements/all-but-dask.yml

@@ -22,12 +22,12 @@ dependencies:
   - netcdf4
   - numba
   - numbagg
-  - numpy
+  - numpy<2
   - packaging
   - pandas
   - pint>=0.22
   - pip
-  - pydap
+  # - pydap
   - pytest
   - pytest-cov
   - pytest-env

ci/requirements/doc.yml

@@ -21,7 +21,7 @@ dependencies:
   - nbsphinx
   - netcdf4>=1.5
   - numba
-  - numpy>=1.21
+  - numpy>=1.21,<2
   - packaging>=21.3
   - pandas>=1.4,!=2.1.0
   - pooch
@@ -42,5 +42,6 @@ dependencies:
   - sphinxext-rediraffe
   - zarr>=2.10
   - pip:
+      - sphinxcontrib-mermaid
       # relative to this file. Needs to be editable to be accepted.
       - -e ../..