Merge branch 'main' into add-set-dims

pydata · Jan 23, 2024 · 3fba906 · 3fba906
2 parents ea24613 + 4b5c87b
commit 3fba906
Show file tree

Hide file tree

Showing 76 changed files with 2,715 additions and 804 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -5,3 +5,7 @@ updates:
     schedule:
       # Check for updates once a week
       interval: 'weekly'
+    groups:
+      actions:
+        patterns:
+          - "*"
diff --git a/.github/workflows/benchmarks-last-release.yml b/.github/workflows/benchmarks-last-release.yml
@@ -72,7 +72,7 @@ jobs:
           cp benchmarks/README_CI.md benchmarks.log .asv/results/
         working-directory: ${{ env.ASV_DIR }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: asv-benchmark-results-${{ runner.os }}

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -67,7 +67,7 @@ jobs:
           cp benchmarks/README_CI.md benchmarks.log .asv/results/
         working-directory: ${{ env.ASV_DIR }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: asv-benchmark-results-${{ runner.os }}

diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml
@@ -76,6 +76,9 @@ jobs:
           # Raise an error if there are warnings in the doctests, with `-Werror`.
           # This is a trial; if it presents an problem, feel free to remove.
           # See https://github.com/pydata/xarray/issues/7164 for more info.
+          #
+          # If dependencies emit warnings we can't do anything about, add ignores to
+          # `xarray/tests/__init__.py`.
           python -m pytest --doctest-modules xarray --ignore xarray/tests -Werror
 
   mypy:
@@ -117,7 +120,7 @@ jobs:
           python xarray/util/print_versions.py
       - name: Install mypy
         run: |
-          python -m pip install "mypy<1.8" --force-reinstall
+          python -m pip install "mypy<1.9" --force-reinstall
 
       - name: Run mypy
         run: |
@@ -171,7 +174,7 @@ jobs:
           python xarray/util/print_versions.py
       - name: Install mypy
         run: |
-          python -m pip install "mypy<1.8" --force-reinstall
+          python -m pip install "mypy<1.9" --force-reinstall
 
       - name: Run mypy
         run: |
@@ -317,11 +320,6 @@ jobs:
       run:
         shell: bash -l {0}
 
-    strategy:
-      matrix:
-        environment-file: ["bare-minimum", "min-all-deps"]
-      fail-fast: false
-
     steps:
       - uses: actions/checkout@v4
         with:
@@ -337,6 +335,10 @@ jobs:
             conda
             python-dateutil
 
-      - name: minimum versions policy
+      - name: All-deps minimum versions policy
+        run: |
+          python ci/min_deps_check.py ci/requirements/min-all-deps.yml
+
+      - name: Bare minimum versions policy
         run: |
-          python ci/min_deps_check.py ci/requirements/${{ matrix.environment-file }}.yml
+          python ci/min_deps_check.py ci/requirements/bare-minimum.yml
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -34,6 +34,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     needs: detect-ci-trigger
     if: needs.detect-ci-trigger.outputs.triggered == 'false'
+    env:
+      ZARR_V3_EXPERIMENTAL_API: 1
     defaults:
       run:
         shell: bash -l {0}
@@ -42,7 +44,7 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         # Bookend python versions
-        python-version: ["3.9", "3.11"]
+        python-version: ["3.9", "3.11", "3.12"]
         env: [""]
         include:
           # Minimum python version:
@@ -69,7 +71,11 @@ jobs:
 
           if [[ ${{ matrix.os }} == windows* ]] ;
           then
-            echo "CONDA_ENV_FILE=ci/requirements/environment-windows.yml" >> $GITHUB_ENV
+            if [[ ${{ matrix.python-version }} != "3.12" ]]; then
+              echo "CONDA_ENV_FILE=ci/requirements/environment-windows.yml" >> $GITHUB_ENV
+            else
+              echo "CONDA_ENV_FILE=ci/requirements/environment-windows-3.12.yml" >> $GITHUB_ENV
+            fi
           elif [[ "${{ matrix.env }}" != "" ]] ;
           then
             if [[ "${{ matrix.env }}" == "flaky" ]] ;
@@ -80,7 +86,11 @@ jobs:
               echo "CONDA_ENV_FILE=ci/requirements/${{ matrix.env }}.yml" >> $GITHUB_ENV
             fi
           else
-            echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV
+            if [[ ${{ matrix.python-version }} != "3.12" ]]; then
+              echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV
+            else
+              echo "CONDA_ENV_FILE=ci/requirements/environment-3.12.yml" >> $GITHUB_ENV
+            fi
           fi
 
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
@@ -127,9 +137,9 @@ jobs:
 
       - name: Upload test results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: Test results for ${{ runner.os }}-${{ matrix.python-version }}
+          name: Test results for ${{ runner.os }}-${{ matrix.python-version }} ${{ matrix.env }}
           path: pytest.xml
 
       - name: Upload code coverage to Codecov
@@ -147,7 +157,7 @@ jobs:
     if: github.repository == 'pydata/xarray'
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}
diff --git a/.github/workflows/nightly-wheels.yml b/.github/workflows/nightly-wheels.yml
@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 

diff --git a/.github/workflows/pypi-release.yaml b/.github/workflows/pypi-release.yaml
@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: "3.11"
@@ -41,7 +41,7 @@ jobs:
           else
             echo "✅ Looks good"
           fi
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: releases
           path: dist
@@ -50,11 +50,11 @@ jobs:
     needs: build-artifacts
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: "3.11"
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: releases
           path: dist
@@ -82,7 +82,7 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: releases
           path: dist
@@ -106,7 +106,7 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: releases
           path: dist

diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml
@@ -37,6 +37,8 @@ jobs:
     name: upstream-dev
     runs-on: ubuntu-latest
     needs: detect-ci-trigger
+    env:
+      ZARR_V3_EXPERIMENTAL_API: 1
     if: |
         always()
         && (
@@ -82,7 +84,6 @@ jobs:
         if: success()
         id: status
         run: |
-          export ZARR_V3_EXPERIMENTAL_API=1
           python -m pytest --timeout=60 -rf \
             --report-log output-${{ matrix.python-version }}-log.jsonl
       - name: Generate and publish the report

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,24 +18,24 @@ repos:
         files: ^xarray/
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.1.6'
+    rev: 'v0.1.9'
     hooks:
       - id: ruff
-        args: ["--fix"]
+        args: ["--fix", "--show-fixes"]
   # https://github.com/python/black#version-control-integration
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.12.1
     hooks:
       - id: black-jupyter
   - repo: https://github.com/keewis/blackdoc
     rev: v0.3.9
     hooks:
       - id: blackdoc
         exclude: "generate_aggregations.py"
-        additional_dependencies: ["black==23.11.0"]
+        additional_dependencies: ["black==23.12.1"]
       - id: blackdoc-autoupdate-black
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
+    rev: v1.8.0
     hooks:
       - id: mypy
         # Copied from setup.cfg

diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh
@@ -1,26 +1,25 @@
 #!/usr/bin/env bash
 
+# install cython for building cftime without build isolation
+micromamba install "cython>=0.29.20" py-cpuinfo
 # temporarily (?) remove numbagg and numba
-pip uninstall -y numbagg
-conda uninstall -y numba
+micromamba remove -y numba numbagg
+# temporarily remove backends
+micromamba remove -y cf_units h5py hdf5 netcdf4
 # forcibly remove packages to avoid artifacts
 conda uninstall -y --force \
     numpy \
     scipy \
     pandas \
-    matplotlib \
-    dask \
     distributed \
     fsspec \
     zarr \
     cftime \
     packaging \
     pint \
     bottleneck \
-    sparse \
     flox \
-    h5netcdf \
-    xarray
+    numcodecs
 # to limit the runtime of Upstream CI
 python -m pip install \
     -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
@@ -31,19 +30,34 @@ python -m pip install \
     scipy \
     matplotlib \
     pandas
+# without build isolation for packages compiling against numpy
+# TODO: remove once there are `numpy>=2.0` builds for numcodecs and cftime
+python -m pip install \
+    --no-deps \
+    --upgrade \
+    --no-build-isolation \
+    git+https://github.com/Unidata/cftime
+python -m pip install \
+    --no-deps \
+    --upgrade \
+    --no-build-isolation \
+    git+https://github.com/zarr-developers/numcodecs
+python -m pip install \
+    --no-deps \
+    --upgrade \
+    --no-build-isolation \
+    git+https://github.com/pydata/bottleneck
 python -m pip install \
     --no-deps \
     --upgrade \
     git+https://github.com/dask/dask \
     git+https://github.com/dask/distributed \
     git+https://github.com/zarr-developers/zarr \
-    git+https://github.com/Unidata/cftime \
     git+https://github.com/pypa/packaging \
     git+https://github.com/hgrecco/pint \
-    git+https://github.com/pydata/bottleneck \
     git+https://github.com/pydata/sparse \
     git+https://github.com/intake/filesystem_spec \
     git+https://github.com/SciTools/nc-time-axis \
     git+https://github.com/xarray-contrib/flox \
-    git+https://github.com/h5netcdf/h5netcdf \
     git+https://github.com/dgasmith/opt_einsum
+    # git+https://github.com/h5netcdf/h5netcdf
diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py
@@ -3,6 +3,8 @@
 publication date. Compare it against requirements/min-all-deps.yml to verify the
 policy on obsolete dependencies is being followed. Print a pretty report :)
 """
+from __future__ import annotations
+
 import itertools
 import sys
 from collections.abc import Iterator

diff --git a/ci/requirements/bare-minimum.yml b/ci/requirements/bare-minimum.yml
@@ -11,6 +11,6 @@ dependencies:
   - pytest-env
   - pytest-xdist
   - pytest-timeout
-  - numpy=1.22
-  - packaging=21.3
-  - pandas=1.4
+  - numpy=1.23
+  - packaging=22.0
+  - pandas=1.5
diff --git a/ci/requirements/environment-3.12.yml b/ci/requirements/environment-3.12.yml
@@ -0,0 +1,47 @@
+name: xarray-tests
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - aiobotocore
+  - boto3
+  - bottleneck
+  - cartopy
+  - cftime
+  - dask-core
+  - distributed
+  - flox
+  - fsspec!=2021.7.0
+  - h5netcdf
+  - h5py
+  - hdf5
+  - hypothesis
+  - iris
+  - lxml  # Optional dep of pydap
+  - matplotlib-base
+  - nc-time-axis
+  - netcdf4
+  # - numba
+  # - numbagg
+  - numexpr
+  - numpy
+  - opt_einsum
+  - packaging
+  - pandas
+  # - pint>=0.22
+  - pip
+  - pooch
+  - pre-commit
+  - pydap
+  - pytest
+  - pytest-cov
+  - pytest-env
+  - pytest-xdist
+  - pytest-timeout
+  - rasterio
+  - scipy
+  - seaborn
+  # - sparse
+  - toolz
+  - typing_extensions
+  - zarr