Merge branch 'main' into v2-tests-fsspec2024

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -27,7 +27,7 @@ body:
     attributes:
       label: Python Version
       description: Version of Python interpreter
-      placeholder: 3.9, 3.10, 3.11, etc.
+      placeholder: 3.10, 3.11, 3.12 etc.
     validations:
       required: true
   - type: input

.github/workflows/python-package.yml

@@ -15,15 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
-        numpy_version: ['>=1.24.0', '==1.23.*']
+        python-version: ['3.10', '3.11', '3.12']
+        numpy_version: ['>=2.1', '==1.24.*']
         exclude:
-          - python-version: '3.10'
-            numpy_version: '==1.23.*'
-          - python-version: '3.11'
-            numpy_version: '==1.23.*'
           - python-version: '3.12'
-            numpy_version: '==1.23.*'
+            numpy_version: '==1.24.*'
     services:
       redis:
         image: redis
@@ -54,16 +50,15 @@ jobs:
     - name: Create Conda environment with the rights deps
       shell: "bash -l {0}"
       run: |
-        conda create -n zarr-env python==${{matrix.python-version}} bsddb3 numcodecs lmdb pip nodejs flake8 mypy
+        conda create -n zarr-env python==${{matrix.python-version}} bsddb3 pip nodejs
         conda activate zarr-env
         npm install -g azurite
     - name: Install dependencies
       shell: "bash -l {0}"
       run: |
         conda activate zarr-env
         python -m pip install --upgrade pip
-        python -m pip install -U pip setuptools wheel line_profiler
-        python -m pip install -rrequirements_dev_minimal.txt numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis
+        python -m pip install -r requirements_dev_minimal.txt numpy${{matrix.numpy_version}} -r requirements_dev_optional.txt line_profiler pymongo redis
         python -m pip install -e .
         python -m pip freeze
     - name: Tests

.github/workflows/releases.yml

@@ -16,10 +16,10 @@ jobs:
           submodules: true
           fetch-depth: 0
 
-      - uses: actions/setup-python@v5.1.0
+      - uses: actions/setup-python@v5.2.0
         name: Install Python
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - name: Install PyBuild
         run: |
@@ -64,7 +64,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/gh-action-pypi-publish@v1.8.14
+      - uses: pypa/gh-action-pypi-publish@v1.10.0
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.github/workflows/windows-testing.yml

@@ -31,13 +31,12 @@ jobs:
       - name: Create Conda environment with the rights deps
         shell: bash -l {0}
         run: |
-          conda create -n zarr-env python==${{matrix.python-version}} numcodecs pip nodejs
+          conda create -n zarr-env python==${{matrix.python-version}} pip nodejs
       - name: Install dependencies
         shell: bash -l {0}
         run: |
           conda activate zarr-env
           python -m pip install --upgrade pip
-          python -m pip install -U pip setuptools wheel
           python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt
           python -m pip install .
           python -m pip freeze

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.9"
+    python: "3.11"
 
 sphinx:
   configuration: docs/conf.py

TEAM.md

@@ -7,6 +7,7 @@
 - @jakirkham (jakirkham)
 - @martindurant (Martin Durant)
 - @normanrz (Norman Rzepka)
+- @dstansby (David Stansby)
 
 ## Emeritus core-developers
 - @alimanfoo (Alistair Miles)

docs/conf.py

@@ -16,11 +16,8 @@
 import os
 import sys
 
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
+from importlib.metadata import version as get_version
+
 import zarr
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -75,9 +72,8 @@
 copyright = "2024, Zarr Developers"
 author = "Zarr Developers"
 
-version = zarr.__version__
-# The full version, including alpha/beta/rc tags.
-release = zarr.__version__
+version = get_version("zarr")
+release = get_version("zarr")
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/release.rst

@@ -18,13 +18,63 @@ Release notes
    See `GH1777 <https://github.com/zarr-developers/zarr-python/issues/1777>`_ for more details on the upcoming
    3.0 release.
 
+.. _release_2.18.4:
+
+2.18.4 (unreleased)
+-------------------
+
+Enhancements
+~~~~~~~~~~~~
+
+Maintenance
+~~~~~~~~~~~
+
+Deprecations
+~~~~~~~~~~~~
+
+.. _release_2.18.3:
+
+2.18.3
+------
+
+Enhancements
+~~~~~~~~~~~~
+* Added support for creating a copy of data when converting a `zarr.Array`
+  to a numpy array.
+  By :user:`David Stansby <dstansby>` (:issue:`2106`) and 
+  :user:`Joe Hamman <jhamman>` (:issue:`2123`).
+
+Maintenance
+~~~~~~~~~~~
+* Removed support for Python 3.9.
+  By :user:`David Stansby <dstansby>` (:issue:`2074`).
+
+* Fix a regression when using orthogonal indexing with a scalar.
+  By :user:`Deepak Cherian <dcherian>` :issue:`1931`
+
+* Added compatibility with NumPy 2.1.
+  By :user:`David Stansby <dstansby>`
+
+* Bump minimum NumPy version to 1.24.
+  :user:`Joe Hamman <jhamman>` (:issue:`2127`).
+
+Deprecations
+~~~~~~~~~~~~
+
+* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These
+  stores are slated to be removed in Zarr Python 3.0.
+  By :user:`Joe Hamman <jhamman>` :issue:`2085`.
+
 .. _release_2.18.2:
 
+2.18.2
+------
+
 Enhancements
 ~~~~~~~~~~~~
 
 * Add Zstd codec to old V3 code path.
-  By :user:`Ryan Abernathey <rabernat>` 
+  By :user:`Ryan Abernathey <rabernat>`
 
 .. _release_2.18.1:
 
@@ -51,7 +101,7 @@ Maintenance
 * Enable ruff/bugbear rules (B) and fix issues.
   By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1702`.
 
-* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. 
+* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release.
   By :user:`Joe Hamman <jhamman>` :issue:`1842`.
 
 Deprecations

pyproject.toml

@@ -10,10 +10,10 @@ readme = { file = "README.md", content-type = "text/markdown" }
 maintainers = [
     { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     'asciitree',
-    'numpy>=1.23',
+    'numpy>=1.24',
     'fasteners; sys_platform != "emscripten"',
     'numcodecs>=0.10.0',
 ]
@@ -30,9 +30,9 @@ classifiers = [
     'Topic :: Software Development :: Libraries :: Python Modules',
     'Operating System :: Unix',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
 ]
 license = { text = "MIT" }
 
@@ -107,6 +107,7 @@ exclude = [
 extend-select = [
     "B"
 ]
+ignore = ["B905"] # zip-without-explicit-strict
 
 [tool.black]
 line-length = 100
@@ -145,6 +146,10 @@ filterwarnings = [
     "ignore:The .* is deprecated and will be removed in a Zarr-Python version 3*:FutureWarning",
     "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning",
 ]
+doctest_subpackage_requires =[
+    "zarr/core.py = numpy>=2",
+    "zarr/creation.py = numpy>=2"
+]
 
 
 [tool.codespell]

requirements_dev_minimal.txt

@@ -1,8 +1,8 @@
 # library requirements
 asciitree==0.3.3
 fasteners==0.19
-numcodecs==0.12.1
+numcodecs==0.13.0
 msgpack-python==0.5.6
-setuptools-scm==8.0.4
+setuptools-scm==8.1.0
 # test requirements
-pytest==8.1.1
+pytest==8.3.2

requirements_dev_numpy.txt

@@ -1,4 +1,4 @@
 # Break this out into a separate file to allow testing against
 # different versions of numpy. This file should pin to the latest
 # numpy version.
-numpy==1.26.4
+numpy==2.1.1

requirements_dev_optional.txt

@@ -1,17 +1,17 @@
 # optional library requirements
 # bsddb3==6.2.6; sys_platform != 'win32'
-lmdb==1.4.1; sys_platform != 'win32'
+lmdb==1.5.1; sys_platform != 'win32'
 # optional library requirements for Jupyter
 ipytree==0.2.2
-ipywidgets==8.1.2
+ipywidgets==8.1.5
 # optional library requirements for services
 # don't let pyup change pinning for azure-storage-blob, need to pin to older
 # version to get compatibility with azure storage emulator on appveyor (FIXME)
-azure-storage-blob==12.16.0 # pyup: ignore
-redis==5.0.4
+azure-storage-blob==12.21.0 # pyup: ignore
+redis==5.0.8
 types-redis
 types-setuptools
-pymongo==4.6.3
+pymongo==4.8.0
 # optional test requirements
 coverage
 pytest-cov==5.0.0

zarr/core.py

@@ -575,11 +575,8 @@ def __eq__(self, other):
             # store comparison
         )
 
-    def __array__(self, *args):
-        a = self[...]
-        if args:
-            a = a.astype(args[0])
-        return a
+    def __array__(self, *args, **kwargs):
+        return np.array(self[...], *args, **kwargs)
 
     def islice(self, start=None, end=None):
         """
@@ -609,11 +606,11 @@ def islice(self, start=None, end=None):
 
         Iterate over part of the array:
             >>> for value in z.islice(25, 30): value;
-            25
-            26
-            27
-            28
-            29
+            np.int64(25)
+            np.int64(26)
+            np.int64(27)
+            np.int64(28)
+            np.int64(29)
         """
 
         if len(self.shape) == 0:
@@ -679,7 +676,7 @@ def __getitem__(self, selection):
         Retrieve a single item::
 
             >>> z[5]
-            5
+            np.int64(5)
 
         Retrieve a region via slicing::
 
@@ -706,7 +703,7 @@ def __getitem__(self, selection):
         Retrieve an item::
 
             >>> z[2, 2]
-            22
+            np.int64(22)
 
         Retrieve a region via slicing::
 
@@ -830,7 +827,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None):
         Retrieve a single item::
 
             >>> z.get_basic_selection(5)
-            5
+            np.int64(5)
 
         Retrieve a region via slicing::
 
@@ -852,7 +849,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None):
         Retrieve an item::
 
             >>> z.get_basic_selection((2, 2))
-            22
+            np.int64(22)
 
         Retrieve a region via slicing::
 
@@ -2053,7 +2050,12 @@ def _process_chunk(
                     if isinstance(cdata, UncompressedPartialReadBufferV3):
                         cdata = cdata.read_full()
                     chunk = ensure_ndarray_like(cdata).view(self._dtype)
-                    chunk = chunk.reshape(self._chunks, order=self._order)
+                    # dest.shape is not self._chunks when a dimensions is squeezed out
+                    # For example, assume self._chunks = (5, 5, 1)
+                    # and the selection is [:, :, 0]
+                    # Then out_selection is (slice(5), slice(5))
+                    # See https://github.com/zarr-developers/zarr-python/issues/1931
+                    chunk = chunk.reshape(dest.shape, order=self._order)
                     np.copyto(dest, chunk)
                 return
 
@@ -2819,7 +2821,7 @@ def view(
             >>> v[:]
             array([False, False,  True, ...,  True, False, False])
             >>> np.all(a[:].view(dtype=bool) == v[:])
-            True
+            np.True_
 
         An array can be viewed with a dtype with a different item size, however
         some care is needed to adjust the shape and chunk shape so that chunk
@@ -2833,7 +2835,7 @@ def view(
             >>> v[:10]
             array([0, 0, 1, 0, 2, 0, 3, 0, 4, 0], dtype=uint8)
             >>> np.all(a[:].view('u1') == v[:])
-            True
+            np.True_
 
         Change fill value for uninitialized chunks:
 

zarr/creation.py

@@ -569,7 +569,7 @@ def open_array(
     >>> z2
     <zarr.core.Array (10000, 10000) float64 read-only>
     >>> np.all(z1[:] == z2[:])
-    True
+    np.True_
 
     Notes
     -----