diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ec98af029e..7b0c4dcfc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.9, 3.10, 3.11, etc. + placeholder: 3.10, 3.11, 3.12 etc. validations: required: true - type: input diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d8e8d4d57a..5a0befe9b5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,10 @@ updates: directory: "/" schedule: interval: "daily" + groups: + requirements: + patterns: + - "*" - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index dba6918514..b5b2f48d62 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.3 + uses: conda-incubator/setup-miniconda@v3.0.4 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fd2603ff95..b1be7e425d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,13 +15,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11'] - numpy_version: ['>=1.22.0', '==1.21.*'] + python-version: ['3.10', '3.11', '3.12'] + numpy_version: ['>=2.1', '==1.24.*'] exclude: - - python-version: '3.10' - numpy_version: '==1.21.*' - - python-version: '3.11' - numpy_version: '==1.21.*' + - python-version: '3.12' + numpy_version: '==1.24.*' services: redis: image: redis @@ -42,7 +40,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.3 + uses: conda-incubator/setup-miniconda@v3.0.4 with: channels: conda-forge python-version: ${{ matrix.python-version }} @@ -52,7 +50,7 @@ jobs: - name: Create Conda environment with the rights deps shell: "bash -l {0}" run: | - conda create -n zarr-env python==${{matrix.python-version}} bsddb3 numcodecs lmdb pip nodejs flake8 mypy + conda create -n zarr-env python==${{matrix.python-version}} bsddb3 pip nodejs conda activate zarr-env npm install -g azurite - name: Install dependencies @@ -60,9 +58,8 @@ jobs: run: | conda activate zarr-env python -m pip install --upgrade pip - python -m pip install -U pip setuptools wheel line_profiler - python -m pip install -rrequirements_dev_minimal.txt numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis - python -m pip install . + python -m pip install -r requirements_dev_minimal.txt numpy${{matrix.numpy_version}} -r requirements_dev_optional.txt line_profiler pymongo redis + python -m pip install -e . python -m pip freeze - name: Tests shell: "bash -l {0}" @@ -78,11 +75,8 @@ jobs: mkdir ~/blob_emulator azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log & pytest --cov=zarr --cov-config=pyproject.toml --doctest-plus --cov-report xml --cov=./ --timeout=300 - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - #files: ./coverage1.xml,./coverage2.xml # optional - #flags: unittests # optional - #name: codecov-umbrella # optional - #fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index fe168d2862..dccfbc208f 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,10 +16,10 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.0.0 + - uses: actions/setup-python@v5.2.0 name: Install Python with: - python-version: '3.9' + python-version: '3.11' - name: Install PyBuild run: | @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.14 + - uses: pypa/gh-action-pypi-publish@v1.10.2 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index d580ef3f0e..1e22fec6d1 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.3 + - uses: conda-incubator/setup-miniconda@v3.0.4 with: auto-update-conda: true python-version: ${{ matrix.python-version }} @@ -31,13 +31,12 @@ jobs: - name: Create Conda environment with the rights deps shell: bash -l {0} run: | - conda create -n zarr-env python==${{matrix.python-version}} numcodecs pip nodejs + conda create -n zarr-env python==${{matrix.python-version}} pip nodejs - name: Install dependencies shell: bash -l {0} run: | conda activate zarr-env python -m pip install --upgrade pip - python -m pip install -U pip setuptools wheel python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt python -m pip install . python -m pip freeze diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41b65f1d02..be57770200 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.2.2' + rev: 'v0.4.4' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -20,11 +20,11 @@ repos: hooks: - id: codespell - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.10.0 hooks: - id: mypy files: zarr diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e45cae1b45..d7190b4771 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py diff --git a/TEAM.md b/TEAM.md index 6a22d83d1f..e6975d7c04 100644 --- a/TEAM.md +++ b/TEAM.md @@ -7,6 +7,9 @@ - @jakirkham (jakirkham) - @martindurant (Martin Durant) - @normanrz (Norman Rzepka) +- @dstansby (David Stansby) +- @dcherian (Deepak Cherian) +- @TomAugspurger (Tom Augspurger) ## Emeritus core-developers - @alimanfoo (Alistair Miles) diff --git a/docs/api/v3.rst b/docs/api/v3.rst index 7665b2ddd1..3503e3fe81 100644 --- a/docs/api/v3.rst +++ b/docs/api/v3.rst @@ -1,13 +1,12 @@ V3 Specification Implementation(``zarr._storage.v3``) ===================================================== -This module contains the implementation of the `Zarr V3 Specification `_. +This module contains an experimental implementation of the `Zarr V3 Specification `_. .. warning:: - Since Zarr Python 2.12 release, this module provides experimental infrastructure for reading and - writing the upcoming V3 spec of the Zarr format. Users wishing to prepare for the migration can set - the environment variable ``ZARR_V3_EXPERIMENTAL_API=1`` to begin experimenting, however data - written with this API should be expected to become stale, as the implementation will still change. + The experimental v3 implementation included in Zarr Python >2.12,<3 is not aligned with the final + V3 specification. This version is deprecated and will be removed in Zarr Python 3.0 in favor of a + spec compliant version. The new ``zarr._store.v3`` package has the necessary classes and functions for evaluating Zarr V3. Since the design is not finalised, the classes and functions are not automatically imported into diff --git a/docs/conf.py b/docs/conf.py index 048e77f51d..136fcf32d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,11 +16,8 @@ import os import sys -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. +from importlib.metadata import version as get_version + import zarr # If extensions (or modules to document with autodoc) are in another directory, @@ -75,9 +72,8 @@ copyright = "2024, Zarr Developers" author = "Zarr Developers" -version = zarr.__version__ -# The full version, including alpha/beta/rc tags. -release = zarr.__version__ +version = get_version("zarr") +release = get_version("zarr") # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -199,7 +195,7 @@ def setup(app): # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -# html_sidebars = {} +html_sidebars = {"tutorial": []} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/docs/release.rst b/docs/release.rst index 037432ca58..a62d6a653c 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,10 +13,162 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. -.. _unreleased: +.. note:: + Zarr-Python 2.18.* is expected be the final release in the 2.* series. Work on Zarr-Python 3.0 is underway. + See `GH1777 `_ for more details on the upcoming + 3.0 release. -Unreleased ----------- +.. _release_2.18.4: + +2.18.4 (unreleased) +------------------- + +Enhancements +~~~~~~~~~~~~ + +Maintenance +~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ + +.. _release_2.18.3: + +2.18.3 +------ + +Enhancements +~~~~~~~~~~~~ +* Added support for creating a copy of data when converting a `zarr.Array` + to a numpy array. + By :user:`David Stansby ` (:issue:`2106`) and + :user:`Joe Hamman ` (:issue:`2123`). + +Maintenance +~~~~~~~~~~~ +* Removed support for Python 3.9. + By :user:`David Stansby ` (:issue:`2074`). + +* Fix a regression when using orthogonal indexing with a scalar. + By :user:`Deepak Cherian ` :issue:`1931` + +* Added compatibility with NumPy 2.1. + By :user:`David Stansby ` + +* Bump minimum NumPy version to 1.24. + :user:`Joe Hamman ` (:issue:`2127`). + +Deprecations +~~~~~~~~~~~~ + +* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These + stores are slated to be removed in Zarr Python 3.0. + By :user:`Joe Hamman ` :issue:`2085`. + +.. _release_2.18.2: + +2.18.2 +------ + +Enhancements +~~~~~~~~~~~~ + +* Add Zstd codec to old V3 code path. + By :user:`Ryan Abernathey ` + +.. _release_2.18.1: + +2.18.1 +------ + +Maintenance +~~~~~~~~~~~ +* Fix a regression when getting or setting a single value from arrays with size-1 chunks. + By :user:`Deepak Cherian ` :issue:`1874` + +.. _release_2.18.0: + +2.18.0 +------ + +Enhancements +~~~~~~~~~~~~ +* Performance improvement for reading and writing chunks if any of the dimensions is size 1. + By :user:`Deepak Cherian ` :issue:`1730`. + +Maintenance +~~~~~~~~~~~ +* Enable ruff/bugbear rules (B) and fix issues. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. + +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. + By :user:`Joe Hamman ` :issue:`1842`. + +Deprecations +~~~~~~~~~~~~ + +* Deprecate experimental v3 support by issuing a `FutureWarning`. + Also updated docs to warn about using the experimental v3 version. + By :user:`Joe Hamman ` :issue:`1802` and :issue:`1807`. + +* Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, + :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, + and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. + By :user:`Joe Hamman ` :issue:`1801`. + +.. _release_2.17.2: + +2.17.2 +------ + +Enhancements +~~~~~~~~~~~~ + +* [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` + and handling any error that may arise. + By :user:`Deepak Cherian ` :issue:`1741`. + +* [v3] Reuse the downloaded array metadata when creating an ``Array``. + By :user:`Deepak Cherian ` :issue:`1734`. + +* Optimize ``Array.info`` so that it calls `getsize` only once. + By :user:`Deepak Cherian ` :issue:`1733`. + +* Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. + By :user:`Deepak Cherian ` :issue:`1716`. + +* FSStore now raises rather than return bad data. + By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. + +* Avoid redundant ``__contains__``. + By :user:`Deepak Cherian ` :issue:`1739`. + +Docs +~~~~ + +* Fix link to GCSMap in ``tutorial.rst``. + By :user:`Daniel Jahn ` :issue:`1689`. + +* Endorse `SPEC0000 `_ and state version support policy in ``installation.rst``. + By :user:`Sanket Verma ` :issue:`1665`. + +* Migrate v1 and v2 specification to `Zarr-Specs `_. + By :user:`Sanket Verma ` :issue:`1582`. + +Maintenance +~~~~~~~~~~~ + +* Add CI test environment for Python 3.12 + By :user:`Joe Hamman ` :issue:`1719`. + +* Bump minimum supported NumPy version to 1.23 (per spec 0000) + By :user:`Joe Hamman ` :issue:`1719`. + +* Minor fixes: Using ``is`` instead of ``type`` and removing unnecessary ``None``. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1737`. + +* Fix tests failure related to Pytest 8. + By :user:`David Stansby ` :issue:`1714`. .. _release_2.17.1: @@ -1610,11 +1762,11 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -* Some changes have been made to the :ref:`spec_v2` document to clarify +* Some changes have been made to the Zarr Specification v2 document to clarify ambiguities and add some missing information. These changes do not break compatibility with any of the material as previously implemented, and so the changes have been made in-place in the document without incrementing the document version number. See the - section on :ref:`spec_v2_changes` in the specification document for more information. + section on changes in the specification document for more information. * A new :ref:`tutorial_indexing` section has been added to the tutorial. * A new :ref:`tutorial_strings` section has been added to the tutorial (:issue:`135`, :issue:`175`). diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst index 13f68ef36e..27a0490e0a 100644 --- a/docs/spec/v1.rst +++ b/docs/spec/v1.rst @@ -3,268 +3,5 @@ Zarr Storage Specification Version 1 ==================================== -This document provides a technical specification of the protocol and -format used for storing a Zarr array. The key words "MUST", "MUST -NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", -"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is deprecated. See :ref:`spec` for the latest version. - -Storage -------- - -A Zarr array can be stored in any storage system that provides a -key/value interface, where a key is an ASCII string and a value is an -arbitrary sequence of bytes, and the supported operations are read -(get the sequence of bytes associated with a given key), write (set -the sequence of bytes associated with a given key) and delete (remove -a key/value pair). - -For example, a directory in a file system can provide this interface, -where keys are file names, values are file contents, and files can be -read, written or deleted via the operating system. Equally, an S3 -bucket can provide this interface, where keys are resource names, -values are resource contents, and resources can be read, written or -deleted via HTTP. - -Below an "array store" refers to any system implementing this -interface. - -Metadata --------- - -Each array requires essential configuration metadata to be stored, -enabling correct interpretation of the stored data. This metadata is -encoded using JSON and stored as the value of the 'meta' key within an -array store. - -The metadata resource is a JSON object. The following keys MUST be -present within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compression - A string identifying the primary compression library used to compress - each chunk of the array. -compression_opts - An integer, string or dictionary providing options to the primary - compression library. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array. -order - Either 'C' or 'F', defining the layout of bytes within each chunk of the - array. 'C' means row-major order, i.e., the last dimension varies fastest; - 'F' means column-major order, i.e., the first dimension varies fastest. - -Other keys MAY be present within the metadata object however they MUST -NOT alter the interpretation of the required fields defined above. - -For example, the JSON object below defines a 2-dimensional array of -64-bit little-endian floating point numbers with 10000 rows and 10000 -columns, divided into chunks of 1000 rows and 1000 columns (so there -will be 100 chunks in total arranged in a 10 by 10 grid). Within each -chunk the data are laid out in C contiguous order, and each chunk is -compressed using the Blosc compression library:: - - { - "chunks": [ - 1000, - 1000 - ], - "compression": "blosc", - "compression_opts": { - "clevel": 5, - "cname": "lz4", - "shuffle": 1 - }, - "dtype": "`_. The -format consists of 3 parts: a character describing the byteorder of -the data (``<``: little-endian, ``>``: big-endian, ``|``: -not-relevant), a character code giving the basic type of the array, -and an integer providing the number of bytes the type uses. The byte -order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data types. - -Structure data types (i.e., with multiple named fields) are encoded as -a list of two-element lists, following `NumPy array protocol type -descriptions (descr) -`_. -For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", -"|u1"]]`` defines a data type composed of three single-byte unsigned -integers labelled 'r', 'g' and 'b'. - -Chunks ------- - -Each chunk of the array is compressed by passing the raw bytes for the -chunk through the primary compression library to obtain a new sequence -of bytes comprising the compressed chunk data. No header is added to -the compressed bytes or any other modification made. The internal -structure of the compressed bytes will depend on which primary -compressor was used. For example, the `Blosc compressor -`_ -produces a sequence of bytes that begins with a 16-byte header -followed by compressed data. - -The compressed sequence of bytes for each chunk is stored under a key -formed from the index of the chunk within the grid of chunks -representing the array. To form a string key for a chunk, the indices -are converted to strings and concatenated with the period character -('.') separating each index. For example, given an array with shape -(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks -laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides -data for rows 0-999 and columns 0-999 and is stored under the key -'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999 -and columns 4000-4999 and is stored under the key '2.4'; etc. - -There is no need for all chunks to be present within an array -store. If a chunk is not present then it is considered to be in an -uninitialized state. An uninitialized chunk MUST be treated as if it -was uniformly filled with the value of the 'fill_value' field in the -array metadata. If the 'fill_value' field is ``null`` then the -contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of -any array dimension is not exactly divisible by the length of the -corresponding chunk dimension then some chunks will overhang the edge -of the array. The contents of any chunk region falling outside the -array are undefined. - -Attributes ----------- - -Each array can also be associated with custom attributes, which are -simple key/value items with application-specific meaning. Custom -attributes are encoded as a JSON object and stored under the 'attrs' -key within an array store. Even if the attributes are empty, the -'attrs' key MUST be present within an array store. - -For example, the JSON object below encodes three attributes named -'foo', 'bar' and 'baz':: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -Example -------- - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Initialize the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('example.zarr') - >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), - ... dtype='i4', fill_value=42, compression='zlib', - ... compression_opts=1, overwrite=True) - -No chunks are initialized yet, so only the 'meta' and 'attrs' keys -have been set:: - - >>> import os - >>> sorted(os.listdir('example.zarr')) - ['attrs', 'meta'] - -Inspect the array metadata:: - - >>> print(open('example.zarr/meta').read()) - { - "chunks": [ - 10, - 10 - ], - "compression": "zlib", - "compression_opts": 1, - "dtype": ">> print(open('example.zarr/attrs').read()) - {} - -Set some data:: - - >>> z = zarr.Array(store) - >>> z[0:10, 0:10] = 1 - >>> sorted(os.listdir('example.zarr')) - ['0.0', 'attrs', 'meta'] - -Set some more data:: - - >>> z[0:10, 10:20] = 2 - >>> z[10:20, :] = 3 - >>> sorted(os.listdir('example.zarr')) - ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> a = np.frombuffer(b, dtype='>> a - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> z.attrs['foo'] = 42 - >>> z.attrs['bar'] = 'apples' - >>> z.attrs['baz'] = [1, 2, 3, 4] - >>> print(open('example.zarr/attrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } +The V1 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index c1e12e1218..deb6d46ce6 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -3,563 +3,5 @@ Zarr Storage Specification Version 2 ==================================== -This document provides a technical specification of the protocol and format -used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", -"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and -"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is the latest version. See :ref:`spec` for previous -versions. - -.. _spec_v2_storage: - -Storage -------- - -A Zarr array can be stored in any storage system that provides a key/value -interface, where a key is an ASCII string and a value is an arbitrary sequence -of bytes, and the supported operations are read (get the sequence of bytes -associated with a given key), write (set the sequence of bytes associated with -a given key) and delete (remove a key/value pair). - -For example, a directory in a file system can provide this interface, where -keys are file names, values are file contents, and files can be read, written -or deleted via the operating system. Equally, an S3 bucket can provide this -interface, where keys are resource names, values are resource contents, and -resources can be read, written or deleted via HTTP. - -Below an "array store" refers to any system implementing this interface. - -.. _spec_v2_array: - -Arrays ------- - -.. _spec_v2_array_metadata: - -Metadata -~~~~~~~~ - -Each array requires essential configuration metadata to be stored, enabling -correct interpretation of the stored data. This metadata is encoded using JSON -and stored as the value of the ".zarray" key within an array store. - -The metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compressor - A JSON object identifying the primary compression codec and providing - configuration parameters, or ``null`` if no compressor is to be used. - The object MUST contain an ``"id"`` key identifying the codec to be used. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array, or ``null`` if no fill_value is to be used. -order - Either "C" or "F", defining the layout of bytes within each chunk of the - array. "C" means row-major order, i.e., the last dimension varies fastest; - "F" means column-major order, i.e., the first dimension varies fastest. -filters - A list of JSON objects providing codec configurations, or ``null`` if no - filters are to be applied. Each codec configuration object MUST contain a - ``"id"`` key identifying the codec to be used. - -The following keys MAY be present within the object: - -dimension_separator - If present, either the string ``"."`` or ``"/"`` defining the separator placed - between the dimensions of a chunk. If the value is not set, then the - default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". - Arrays defined with ``"/"`` as the dimension separator can be considered to have - nested, or hierarchical, keys of the form "0/0" that SHOULD where possible - produce a directory-like structure. - -Other keys SHOULD NOT be present within the metadata object and SHOULD be -ignored by implementations. - -For example, the JSON object below defines a 2-dimensional array of 64-bit -little-endian floating point numbers with 10000 rows and 10000 columns, divided -into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total -arranged in a 10 by 10 grid). Within each chunk the data are laid out in C -contiguous order. Each chunk is encoded using a delta filter and compressed -using the Blosc compression library prior to storage:: - - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "id": "blosc", - "cname": "lz4", - "clevel": 5, - "shuffle": 1 - }, - "dtype": "`. The format -consists of 3 parts: - -* One character describing the byteorder of the data (``"<"``: little-endian; - ``">"``: big-endian; ``"|"``: not-relevant) -* One character code giving the basic type of the array (``"b"``: Boolean (integer - type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; - ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode - (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a - fixed-size chunk of memory)) -* An integer specifying the number of bytes the type uses. - -The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data type encodings. - -For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the -units within square brackets. A list of valid units and their definitions are given in -the :ref:`NumPy documentation on Datetimes and Timedeltas -`. -For example, ``"`. Each -sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` -is optional. ``fieldname`` is a string, ``datatype`` is a string -specifying a simple data type (see above), and ``shape`` is a list of -integers specifying subarray shape. For example, the JSON list below -defines a data type composed of three single-byte unsigned integer -fields named "r", "g" and "b":: - - [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] - -For example, the JSON list below defines a data type composed of three -fields named "x", "y" and "z", where "x" and "y" each contain 32-bit -floats, and each item in "z" is a 2 by 2 array of floats:: - - [["x", "`_ -produces a sequence of bytes that begins with a 16-byte header followed by -compressed data. - -The compressed sequence of bytes for each chunk is stored under a key formed -from the index of the chunk within the grid of chunks representing the array. -To form a string key for a chunk, the indices are converted to strings and -concatenated with the period character (".") separating each index. For -example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) -there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices -(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the -key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and -columns 4000-4999 and is stored under the key "2.4"; etc. - -There is no need for all chunks to be present within an array store. If a chunk -is not present then it is considered to be in an uninitialized state. An -uninitialized chunk MUST be treated as if it was uniformly filled with the value -of the "fill_value" field in the array metadata. If the "fill_value" field is -``null`` then the contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of any -array dimension is not exactly divisible by the length of the corresponding -chunk dimension then some chunks will overhang the edge of the array. The -contents of any chunk region falling outside the array are undefined. - -.. _spec_v2_array_filters: - -Filters -~~~~~~~ - -Optionally a sequence of one or more filters can be used to transform chunk -data prior to compression. When storing data, filters are applied in the order -specified in array metadata to encode data, then the encoded data are passed to -the primary compressor. When retrieving data, stored chunk data are -decompressed by the primary compressor then decoded using filters in the -reverse order. - -.. _spec_v2_hierarchy: - -Hierarchies ------------ - -.. _spec_v2_hierarchy_paths: - -Logical storage paths -~~~~~~~~~~~~~~~~~~~~~ - -Multiple arrays can be stored in the same array store by associating each array -with a different logical path. A logical path is simply an ASCII string. The -logical path is used to form a prefix for keys used by the array. For example, -if an array is stored at logical path "foo/bar" then the array metadata will be -stored under the key "foo/bar/.zarray", the user-defined attributes will be -stored under the key "foo/bar/.zattrs", and the chunks will be stored under -keys like "foo/bar/0.0", "foo/bar/0.1", etc. - -To ensure consistent behaviour across different storage systems, logical paths -MUST be normalized as follows: - -* Replace all backward slash characters ("\\\\") with forward slash characters - ("/") -* Strip any leading "/" characters -* Strip any trailing "/" characters -* Collapse any sequence of more than one "/" character into a single "/" - character - -The key prefix is then obtained by appending a single "/" character to the -normalized logical path. - -After normalization, if splitting a logical path by the "/" character results -in any path segment equal to the string "." or the string ".." then an error -MUST be raised. - -N.B., how the underlying array store processes requests to store values under -keys containing the "/" character is entirely up to the store implementation -and is not constrained by this specification. E.g., an array store could simply -treat all keys as opaque ASCII strings; equally, an array store could map -logical paths onto some kind of hierarchical storage (e.g., directories on a -file system). - -.. _spec_v2_hierarchy_groups: - -Groups -~~~~~~ - -Arrays can be organized into groups which can also contain other groups. A -group is created by storing group metadata under the ".zgroup" key under some -logical path. E.g., a group exists at the root of an array store if the -".zgroup" key exists in the store, and a group exists at logical path "foo/bar" -if the "foo/bar/.zgroup" key exists in the store. - -If the user requests a group to be created under some logical path, then groups -MUST also be created at all ancestor paths. E.g., if the user requests group -creation at path "foo/bar" then groups MUST be created at path "foo" and the -root of the store, if they don't already exist. - -If the user requests an array to be created under some logical path, then -groups MUST also be created at all ancestor paths. E.g., if the user requests -array creation at path "foo/bar/baz" then groups must be created at path -"foo/bar", path "foo", and the root of the store, if they don't already exist. - -The group metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. - -Other keys MUST NOT be present within the metadata object. - -The members of a group are arrays and groups stored under logical paths that -are direct children of the parent group's logical path. E.g., if groups exist -under the logical paths "foo" and "foo/bar" and an array exists at logical path -"foo/baz" then the members of the group at path "foo" are the group at path -"foo/bar" and the array at path "foo/baz". - -.. _spec_v2_attrs: - -Attributes ----------- - -An array or group can be associated with custom attributes, which are arbitrary -key/value pairs with application-specific meaning. Custom attributes are encoded -as a JSON object and stored under the ".zattrs" key within an array store. The -".zattrs" key does not have to be present, and if it is absent the attributes -should be treated as empty. - -For example, the JSON object below encodes three attributes named -"foo", "bar" and "baz":: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -.. _spec_v2_examples: - -Examples --------- - -Storing a single array -~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Create an array:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', - ... fill_value=42, compressor=zarr.Zlib(level=1), - ... store=store, overwrite=True) - -No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys -have been set in the store:: - - >>> import os - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray'] - -Inspect the array metadata:: - - >>> print(open('data/example.zarr/.zarray').read()) - { - "chunks": [ - 10, - 10 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": ">> a[0:10, 0:10] = 1 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0'] - -Set some more data:: - - >>> a[0:10, 10:20] = 2 - >>> a[10:20, :] = 3 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> chunk = np.frombuffer(buf, dtype='>> chunk - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> a.attrs['foo'] = 42 - >>> a.attrs['bar'] = 'apples' - >>> a.attrs['baz'] = [1, 2, 3, 4] - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - >>> print(open('data/example.zarr/.zattrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } - -Storing multiple arrays in a hierarchy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing multiple Zarr arrays organized into a group -hierarchy, using a directory on the local file system as storage. This storage -implementation maps logical paths onto directory paths on the file system, -however this is an implementation choice and is not required. - -Setup the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/group.zarr') - -Create the root group:: - - >>> root_grp = zarr.group(store, overwrite=True) - -The metadata resource for the root group has been created:: - - >>> import os - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup'] - -Inspect the group metadata:: - - >>> print(open('data/group.zarr/.zgroup').read()) - { - "zarr_format": 2 - } - -Create a sub-group:: - - >>> sub_grp = root_grp.create_group('foo') - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup'] - -Create an array within the sub-group:: - - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - -Set a custom attributes:: - - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - -Here is the same example using a Zip file as storage:: - - >>> store = zarr.ZipStore('data/group.zip', mode='w') - >>> root_grp = zarr.group(store) - >>> sub_grp = root_grp.create_group('foo') - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - >>> store.close() - -What has been stored:: - - >>> import zipfile - >>> zf = zipfile.ZipFile('data/group.zip', mode='r') - >>> for name in sorted(zf.namelist()): - ... print(name) - .zgroup - foo/.zgroup - foo/bar/.zarray - foo/bar/.zattrs - foo/bar/0.0 - foo/bar/0.1 - foo/bar/1.0 - foo/bar/1.1 - -.. _spec_v2_changes: - -Changes -------- - -Version 2 clarifications -~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes have been made to the version 2 specification since it was -initially published to clarify ambiguities and add some missing information. - -* The specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). - -* The specification now clarifies that units must be specified for datetime64 and - timedelta64 data types (:issue:`85`, :issue:`215`). - -* The specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. - -* The specification now describes how structured datatypes with - subarray shapes and/or with nested structured data types are encoded - in array metadata (:issue:`111`, :issue:`296`). - -* Clarified the key/value pairs of custom attributes as "arbitrary" rather than - "simple". - -Changes from version 1 to version 2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes were made between version 1 and version 2 of this specification: - -* Added support for storing multiple arrays in the same store and organising - arrays into hierarchies using groups. -* Array metadata is now stored under the ".zarray" key instead of the "meta" - key. -* Custom attributes are now stored under the ".zattrs" key instead of the - "attrs" key. -* Added support for filters. -* Changed encoding of "fill_value" field within array metadata. -* Changed encoding of compressor information within array metadata to be - consistent with representation of filter information. +The V2 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v3.rst b/docs/spec/v3.rst index bd8852707b..3d39f35ba6 100644 --- a/docs/spec/v3.rst +++ b/docs/spec/v3.rst @@ -1,7 +1,7 @@ .. _spec_v3: Zarr Storage Specification Version 3 -======================================================= +==================================== The V3 Specification has been migrated to its website → https://zarr-specs.readthedocs.io/. diff --git a/pyproject.toml b/pyproject.toml index 4da3079808..ec06b63a96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,10 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ 'asciitree', - 'numpy>=1.21.1', + 'numpy>=1.24', 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] @@ -30,9 +30,9 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ] license = { text = "MIT" } @@ -103,6 +103,12 @@ exclude = [ "docs" ] +[tool.ruff.lint] +extend-select = [ + "B" +] +ignore = ["B905"] # zip-without-explicit-strict + [tool.black] line-length = 100 exclude = ''' @@ -137,6 +143,12 @@ filterwarnings = [ "error:::zarr.*", "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", + "ignore:The .* is deprecated and will be removed in a Zarr-Python version 3*:FutureWarning", + "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", +] +doctest_subpackage_requires =[ + "zarr/core.py = numpy>=2", + "zarr/creation.py = numpy>=2" ] diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 94d3fff8a6..1d29404d6c 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -1,8 +1,8 @@ # library requirements asciitree==0.3.3 fasteners==0.19 -numcodecs==0.12.1 +numcodecs==0.13.0 msgpack-python==0.5.6 -setuptools-scm==8.0.4 +setuptools-scm==8.1.0 # test requirements -pytest==7.4.4 +pytest==8.3.3 diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index d8d6c3d097..190610d3d3 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.26.4 +numpy==2.1.1 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 7ff673cebd..52c4a737a1 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -1,23 +1,23 @@ # optional library requirements # bsddb3==6.2.6; sys_platform != 'win32' -lmdb==1.4.1; sys_platform != 'win32' +lmdb==1.5.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.2 +ipywidgets==8.1.5 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) -azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.3 +azure-storage-blob==12.21.0 # pyup: ignore +redis==5.1.0 types-redis types-setuptools -pymongo==4.6.2 +pymongo==4.10.1 # optional test requirements coverage -pytest-cov==4.1.0 +pytest-cov==5.0.0 pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 -h5py==3.10.0 +h5py==3.12.1 fsspec==2023.12.2 s3fs==2023.12.2 moto[server]>=5.0.1 diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index 217b2a29e0..1e49754f38 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -5,7 +5,14 @@ from numcodecs.compat import ensure_bytes from zarr.util import normalize_storage_path -from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, Store, StoreV3 +from zarr._storage.store import ( + _get_metadata_suffix, + data_root, + meta_root, + Store, + StoreV3, + V3_DEPRECATION_MESSAGE, +) from zarr.types import DIMENSION_SEPARATOR __doctest_requires__ = { @@ -73,6 +80,12 @@ def __init__( dimension_separator: Optional[DIMENSION_SEPARATOR] = None, client=None, ): + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=3, + ) + self._dimension_separator = dimension_separator self.prefix = normalize_storage_path(prefix) if client is None: @@ -143,8 +156,8 @@ def __getitem__(self, key): blob_name = self._append_path_to_prefix(key) try: return self.client.download_blob(blob_name).readall() - except ResourceNotFoundError: - raise KeyError(f"Blob {blob_name} not found") + except ResourceNotFoundError as e: + raise KeyError(f"Blob {blob_name} not found") from e def __setitem__(self, key, value): value = ensure_bytes(value) @@ -156,8 +169,8 @@ def __delitem__(self, key): try: self.client.delete_blob(self._append_path_to_prefix(key)) - except ResourceNotFoundError: - raise KeyError(f"Blob {key} not found") + except ResourceNotFoundError as e: + raise KeyError(f"Blob {key} not found") from e def __eq__(self, other): return ( diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 209f118534..dba29d13c0 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -1,5 +1,6 @@ import abc import os +import warnings from collections import defaultdict from collections.abc import MutableMapping from copy import copy @@ -23,9 +24,26 @@ DEFAULT_ZARR_VERSION: ZARR_VERSION = 2 v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"] +_has_warned_about_v3 = False # to avoid printing the warning multiple times + +V3_DEPRECATION_MESSAGE = ( + "The {store} is deprecated and will be removed in a Zarr-Python version 3, see " + "https://github.com/zarr-developers/zarr-python/issues/1274 for more information." +) def assert_zarr_v3_api_available(): + # we issue a warning about the experimental v3 implementation when it is first used + global _has_warned_about_v3 + if v3_api_available and not _has_warned_about_v3: + warnings.warn( + "The experimental Zarr V3 implementation in this version of Zarr-Python is not " + "in alignment with the final V3 specification. This version will be removed in " + "Zarr-Python 3 in favor of a spec compliant version.", + FutureWarning, + stacklevel=1, + ) + _has_warned_about_v3 = True if not v3_api_available: raise NotImplementedError( "# V3 reading and writing is experimental! To enable support, set:\n" @@ -462,7 +480,7 @@ def inner_store(self) -> Union["StorageTransformer", StoreV3]: def __eq__(self, other): return ( - type(self) == type(other) + type(self) is type(other) and self._inner_store == other._inner_store and self.get_config() == other.get_config() ) diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index 37e56f8ecd..00467d44f9 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -183,8 +183,8 @@ def __getitem__(self, key): shard_key, chunk_subkey = self._key_to_shard(key) try: full_shard_value = self.inner_store[shard_key] - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e index = self._get_index_from_buffer(full_shard_value) chunk_slice = index.get_chunk_slice(chunk_subkey) if chunk_slice is not None: @@ -265,8 +265,8 @@ def __delitem__(self, key): shard_key, chunk_subkey = self._key_to_shard(key) try: index = self._get_index_from_store(shard_key) - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e index.set_chunk_slice(chunk_subkey, None) diff --git a/zarr/attrs.py b/zarr/attrs.py index e967c5b853..af9a5f1d30 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -25,14 +25,16 @@ class Attributes(MutableMapping): """ - def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None): + def __init__( + self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None, cached_dict=None + ): self._version = getattr(store, "_store_version", 2) _Store = Store if self._version == 2 else StoreV3 self.store = _Store._ensure_store(store) self.key = key self.read_only = read_only self.cache = cache - self._cached_asdict = None + self._cached_asdict = cached_dict if cache else None self.synchronizer = synchronizer def _get_nosync(self): @@ -149,19 +151,20 @@ def _put_nosync(self, d): if self.cache: self._cached_asdict = d else: - if self.key in self.store: + try: + meta_unparsed = self.store[self.key] # Cannot write the attributes directly to JSON, but have to # store it within the pre-existing attributes key of the v3 # metadata. # Note: this changes the store.counter result in test_caching_on! - meta = self.store._metadata_class.parse_metadata(self.store[self.key]) + meta = self.store._metadata_class.parse_metadata(meta_unparsed) if "attributes" in meta and "filters" in meta["attributes"]: # need to preserve any existing "filters" attribute d["attributes"]["filters"] = meta["attributes"]["filters"] meta["attributes"] = d["attributes"] - else: + except KeyError: meta = d self.store[self.key] = json_dumps(meta) if self.cache: diff --git a/zarr/codecs.py b/zarr/codecs.py index 4ad68b8627..6fd5e20401 100644 --- a/zarr/codecs.py +++ b/zarr/codecs.py @@ -1,4 +1,4 @@ # flake8: noqa from numcodecs import * -from numcodecs import get_codec, Blosc, Pickle, Zlib, Delta, AsType, BZ2 +from numcodecs import get_codec, Blosc, Pickle, Zlib, Zstd, Delta, AsType, BZ2 from numcodecs.registry import codec_registry diff --git a/zarr/convenience.py b/zarr/convenience.py index 7ca5d426f0..bd284e0844 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -55,6 +55,11 @@ def open(store: StoreLike = None, mode: str = "a", *, zarr_version=None, path=No The zarr protocol version to use. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store to open. **kwargs @@ -150,6 +155,11 @@ def save_array(store: StoreLike, arr, *, zarr_version=None, path=None, **kwargs) The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store where the array will be saved. kwargs @@ -200,6 +210,11 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional Path within the store where the group will be saved. kwargs @@ -282,6 +297,11 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the group where the arrays will be saved. kwargs @@ -395,6 +415,11 @@ def load(store: StoreLike, zarr_version=None, path=None): The zarr protocol version to use when loading. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store from which to load. diff --git a/zarr/core.py b/zarr/core.py index c3184c6652..d13da27bc6 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -172,11 +172,15 @@ def __init__( # initialize attributes akey = _prefix_to_attrs_key(self._store, self._key_prefix) self._attrs = Attributes( - store, key=akey, read_only=read_only, synchronizer=synchronizer, cache=cache_attrs + store, + key=akey, + read_only=read_only, + synchronizer=synchronizer, + cache=cache_attrs, + cached_dict=self._meta["attributes"] if self._version == 3 else None, ) # initialize info reporter - self._info_reporter = InfoReporter(self) # initialize indexing helpers self._oindex = OIndex(self) @@ -196,8 +200,8 @@ def _load_metadata_nosync(self): try: mkey = _prefix_to_array_key(self._store, self._key_prefix) meta_bytes = self._store[mkey] - except KeyError: - raise ArrayNotFoundError(self._path) + except KeyError as e: + raise ArrayNotFoundError(self._path) from e else: # decode and store metadata as instance members meta = self._store._metadata_class.decode_array_metadata(meta_bytes) @@ -571,11 +575,8 @@ def __eq__(self, other): # store comparison ) - def __array__(self, *args): - a = self[...] - if args: - a = a.astype(args[0]) - return a + def __array__(self, *args, **kwargs): + return np.array(self[...], *args, **kwargs) def islice(self, start=None, end=None): """ @@ -605,11 +606,11 @@ def islice(self, start=None, end=None): Iterate over part of the array: >>> for value in z.islice(25, 30): value; - 25 - 26 - 27 - 28 - 29 + np.int64(25) + np.int64(26) + np.int64(27) + np.int64(28) + np.int64(29) """ if len(self.shape) == 0: @@ -675,7 +676,7 @@ def __getitem__(self, selection): Retrieve a single item:: >>> z[5] - 5 + np.int64(5) Retrieve a region via slicing:: @@ -702,7 +703,7 @@ def __getitem__(self, selection): Retrieve an item:: >>> z[2, 2] - 22 + np.int64(22) Retrieve a region via slicing:: @@ -826,7 +827,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): Retrieve a single item:: >>> z.get_basic_selection(5) - 5 + np.int64(5) Retrieve a region via slicing:: @@ -848,7 +849,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): Retrieve an item:: >>> z.get_basic_selection((2, 2)) - 22 + np.int64(22) Retrieve a region via slicing:: @@ -2026,7 +2027,9 @@ def _process_chunk( and not self._filters and self._dtype != object ): - dest = out[out_selection] + # For 0D arrays out_selection = () and out[out_selection] is a scalar + # Avoid that + dest = out[out_selection] if out_selection else out # Assume that array-like objects that doesn't have a # `writeable` flag is writable. dest_is_writable = getattr(dest, "writeable", True) @@ -2047,7 +2050,12 @@ def _process_chunk( if isinstance(cdata, UncompressedPartialReadBufferV3): cdata = cdata.read_full() chunk = ensure_ndarray_like(cdata).view(self._dtype) - chunk = chunk.reshape(self._chunks, order=self._order) + # dest.shape is not self._chunks when a dimensions is squeezed out + # For example, assume self._chunks = (5, 5, 1) + # and the selection is [:, :, 0] + # Then out_selection is (slice(5), slice(5)) + # See https://github.com/zarr-developers/zarr-python/issues/1931 + chunk = chunk.reshape(dest.shape, order=self._order) np.copyto(dest, chunk) return @@ -2429,7 +2437,7 @@ def info(self): Chunks initialized : 0/10 """ - return self._info_reporter + return InfoReporter(self) def info_items(self): return self._synchronized_op(self._info_items_nosync) @@ -2471,14 +2479,16 @@ def bytestr(n): items += [("Synchronizer type", typestr(self._synchronizer))] # storage info + nbytes = self.nbytes + nbytes_stored = self.nbytes_stored items += [("Store type", typestr(self._store))] if self._chunk_store is not None: items += [("Chunk store type", typestr(self._chunk_store))] - items += [("No. bytes", bytestr(self.nbytes))] - if self.nbytes_stored > 0: + items += [("No. bytes", bytestr(nbytes))] + if nbytes_stored > 0: items += [ - ("No. bytes stored", bytestr(self.nbytes_stored)), - ("Storage ratio", f"{self.nbytes / self.nbytes_stored:.1f}"), + ("No. bytes stored", bytestr(nbytes_stored)), + ("Storage ratio", f"{nbytes / nbytes_stored:.1f}"), ] items += [("Chunks initialized", f"{self.nchunks_initialized}/{self.nchunks}")] @@ -2811,7 +2821,7 @@ def view( >>> v[:] array([False, False, True, ..., True, False, False]) >>> np.all(a[:].view(dtype=bool) == v[:]) - True + np.True_ An array can be viewed with a dtype with a different item size, however some care is needed to adjust the shape and chunk shape so that chunk @@ -2825,7 +2835,7 @@ def view( >>> v[:10] array([0, 0, 1, 0, 2, 0, 3, 0, 4, 0], dtype=uint8) >>> np.all(a[:].view('u1') == v[:]) - True + np.True_ Change fill value for uninitialized chunks: diff --git a/zarr/creation.py b/zarr/creation.py index c541531d54..f7f3d5a094 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -297,7 +297,7 @@ def _kwargs_compat(compressor, fill_value, kwargs): # ignore other keyword arguments for k in kwargs: - warn(f"ignoring keyword argument {k!r}") + warn(f"ignoring keyword argument {k!r}", stacklevel=2) return compressor, fill_value @@ -569,7 +569,7 @@ def open_array( >>> z2 >>> np.all(z1[:] == z2[:]) - True + np.True_ Notes ----- diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 44af1d63d1..8894a5ed57 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -27,6 +27,7 @@ from zarr.errors import ( ContainsArrayError, ContainsGroupError, + ArrayNotFoundError, GroupNotFoundError, ReadOnlyError, ) @@ -186,16 +187,16 @@ def __init__( mkey = _prefix_to_group_key(self._store, self._key_prefix) assert not mkey.endswith("root/.group") meta_bytes = store[mkey] - except KeyError: + except KeyError as e: if self._version == 2: - raise GroupNotFoundError(path) + raise GroupNotFoundError(path) from e else: implicit_prefix = meta_root + self._key_prefix if self._store.list_prefix(implicit_prefix): # implicit group does not have any metadata self._meta = None else: - raise GroupNotFoundError(path) + raise GroupNotFoundError(path) from e else: self._meta = self._store._metadata_class.decode_group_metadata(meta_bytes) @@ -207,11 +208,15 @@ def __init__( # object can still be created. akey = mkey self._attrs = Attributes( - store, key=akey, read_only=read_only, cache=cache_attrs, synchronizer=synchronizer + store, + key=akey, + read_only=read_only, + cache=cache_attrs, + synchronizer=synchronizer, + cached_dict=self._meta["attributes"] if self._version == 3 and self._meta else None, ) # setup info - self._info = InfoReporter(self) @property def store(self): @@ -266,7 +271,7 @@ def attrs(self): @property def info(self): """Return diagnostic information about the group.""" - return self._info + return InfoReporter(self) @property def meta_array(self): @@ -458,7 +463,7 @@ def __getitem__(self, item): """ path = self._item_path(item) - if contains_array(self._store, path): + try: return Array( self._store, read_only=self._read_only, @@ -469,7 +474,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif contains_group(self._store, path, explicit_only=True): + except ArrayNotFoundError: + pass + + try: return Group( self._store, read_only=self._read_only, @@ -480,7 +488,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif self._version == 3: + except GroupNotFoundError: + pass + + if self._version == 3: implicit_group = meta_root + path + "/" # non-empty folder in the metadata path implies an implicit group if self._store.list_prefix(implicit_group): @@ -515,11 +526,18 @@ def _delitem_nosync(self, item): raise KeyError(item) def __getattr__(self, item): + # https://github.com/jupyter/notebook/issues/2014 + # Save a possibly expensive lookup (for e.g. against cloud stores) + # Note: The _ipython_display_ method is required to display the right info as a side-effect. + # It is simpler to pretend it doesn't exist. + if item in ["_ipython_canary_method_should_not_exist_", "_ipython_display_"]: + raise AttributeError + # allow access to group members via dot notation try: return self.__getitem__(item) - except KeyError: - raise AttributeError + except KeyError as e: + raise AttributeError from e def __dir__(self): # noinspection PyUnresolvedReferences @@ -1331,6 +1349,40 @@ def move(self, source, dest): self._write_op(self._move_nosync, source, dest) + # Override ipython repr methods, GH1716 + # https://ipython.readthedocs.io/en/stable/config/integrating.html#custom-methods + # " If the methods don’t exist, the standard repr() is used. If a method exists and + # returns None, it is treated the same as if it does not exist." + def _repr_html_(self): + return None + + def _repr_latex_(self): + return None + + def _repr_mimebundle_(self, **kwargs): + return None + + def _repr_svg_(self): + return None + + def _repr_png_(self): + return None + + def _repr_jpeg_(self): + return None + + def _repr_markdown_(self): + return None + + def _repr_javascript_(self): + return None + + def _repr_pdf_(self): + return None + + def _repr_json_(self): + return None + def _normalize_store_arg(store, *, storage_options=None, mode="r", zarr_version=None): if zarr_version is None: diff --git a/zarr/indexing.py b/zarr/indexing.py index 9889fcadad..35c1e813b1 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -52,6 +52,8 @@ def is_scalar(value, dtype): return True if isinstance(value, tuple) and dtype.names and len(value) == len(dtype.names): return True + if dtype.kind == "O" and not isinstance(value, np.ndarray): + return True return False @@ -932,7 +934,7 @@ def check_fields(fields, dtype): # multiple field selection out_dtype = np.dtype([(f, dtype[f]) for f in fields]) except KeyError as e: - raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") + raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") from e else: return out_dtype else: diff --git a/zarr/meta.py b/zarr/meta.py index 4b360270de..44a2b7ebec 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -227,9 +227,9 @@ def decode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> if v == "NaN": return np.nan elif v == "Infinity": - return np.PINF + return np.inf elif v == "-Infinity": - return np.NINF + return -np.inf else: return np.array(v, dtype=dtype)[()] elif dtype.kind in "c": @@ -310,8 +310,8 @@ def decode_dtype(cls, d, validate=True): # extract the type from the extension info try: d = d["type"] - except KeyError: - raise KeyError("Extended dtype info must provide a key named 'type'.") + except KeyError as e: + raise KeyError("Extended dtype info must provide a key named 'type'.") from e d = cls._decode_dtype_descr(d) dtype = np.dtype(d) if validate: @@ -414,6 +414,8 @@ def _encode_codec_metadata(cls, codec: Codec) -> Optional[Mapping]: uri = uri + "lz4/1.0" elif isinstance(codec, numcodecs.LZMA): uri = uri + "lzma/1.0" + elif isinstance(codec, numcodecs.Zstd): + uri = uri + "zstd/1.0" meta = { "codec": uri, "configuration": config, @@ -439,6 +441,8 @@ def _decode_codec_metadata(cls, meta: Optional[Mapping]) -> Optional[Codec]: conf["id"] = "lz4" elif meta["codec"].startswith(uri + "lzma/"): conf["id"] = "lzma" + elif meta["codec"].startswith(uri + "zstd/"): + conf["id"] = "zstd" else: raise NotImplementedError @@ -518,7 +522,7 @@ def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType meta["storage_transformers"] = storage_transformers except Exception as e: - raise MetadataError(f"error decoding metadata: {e}") + raise MetadataError(f"error decoding metadata: {e}") from e else: return meta diff --git a/zarr/meta_v1.py b/zarr/meta_v1.py index 65bfd3488e..714f55f477 100644 --- a/zarr/meta_v1.py +++ b/zarr/meta_v1.py @@ -23,7 +23,7 @@ def decode_metadata(b): order=meta["order"], ) except Exception as e: - raise MetadataError(f"error decoding metadata: {e}") + raise MetadataError(f"error decoding metadata: {e}") from e else: return meta diff --git a/zarr/n5.py b/zarr/n5.py index fdd3d5babf..3bb7093128 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -69,8 +69,20 @@ class N5Store(NestedDirectoryStore): Safe to write in multiple threads or processes. + .. deprecated:: 2.18.3 + `N5Store` will be removed in Zarr 3.0.0. """ + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + warnings.warn( + "The N5Store is deprecated and will be removed in a Zarr-Python version 3, " + "see https://github.com/zarr-developers/zarr-python/issues/1274 and " + "https://github.com/zarr-developers/n5py for more information.", + FutureWarning, + stacklevel=2, + ) + def __getitem__(self, key: str) -> bytes: if key.endswith(zarr_group_meta_key): key_new = key.replace(zarr_group_meta_key, n5_attrs_key) @@ -125,7 +137,11 @@ def __setitem__(self, key: str, value: Any): for k in n5_keywords: if k in zarr_attrs: - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) + warnings.warn( + f"Attribute {k} is a reserved N5 keyword", + UserWarning, + stacklevel=2, + ) # remove previous user attributes for k in list(n5_attrs.keys()): @@ -318,6 +334,9 @@ class N5FSStore(FSStore): storage, and this procedure requires chunk keys with "." separated dimensions, hence the Zarr arrays targeting N5 have the deceptive "." dimension separator. + + .. deprecated:: 2.18.3 + `N5FSStore` will be removed in Zarr 3.0.0. """ _array_meta_key = "attributes.json" @@ -325,9 +344,19 @@ class N5FSStore(FSStore): _attrs_key = "attributes.json" def __init__(self, *args, **kwargs): + warnings.warn( + "The N5FSStore is deprecated and will be removed in a Zarr-Python version 3, " + "see https://github.com/zarr-developers/zarr-python/issues/1274 and " + "https://github.com/zarr-developers/n5py for more information.", + FutureWarning, + stacklevel=2, + ) if "dimension_separator" in kwargs: kwargs.pop("dimension_separator") - warnings.warn("Keyword argument `dimension_separator` will be ignored") + warnings.warn( + "Keyword argument `dimension_separator` will be ignored", + stacklevel=2, + ) dimension_separator = "." super().__init__(*args, dimension_separator=dimension_separator, **kwargs) @@ -411,7 +440,11 @@ def __setitem__(self, key: str, value: Any): for k in n5_keywords: if k in zarr_attrs.keys(): - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) + warnings.warn( + f"Attribute {k} is a reserved N5 keyword", + UserWarning, + stacklevel=2, + ) # replace previous user attributes for k in list(n5_attrs.keys()): @@ -597,8 +630,8 @@ def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dic array_metadata["n5"] = N5_FORMAT try: dtype = np.dtype(array_metadata["dataType"]) - except TypeError: - raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5") + except TypeError as e: + raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5") from e array_metadata["dataType"] = dtype.name array_metadata["dimensions"] = array_metadata["dimensions"][::-1] @@ -711,6 +744,7 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict "Not all N5 implementations support lzma compression (yet). You " "might not be able to open the dataset with another N5 library.", RuntimeWarning, + stacklevel=2, ) n5_config["format"] = _compressor_config["format"] n5_config["check"] = _compressor_config["check"] diff --git a/zarr/storage.py b/zarr/storage.py index f6903d29b2..f412870f75 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -88,6 +88,7 @@ DEFAULT_ZARR_VERSION, BaseStore, Store, + V3_DEPRECATION_MESSAGE, ) __doctest_requires__ = { @@ -588,11 +589,15 @@ def _init_array_metadata( "missing object_codec for object array; this will raise a " "ValueError in version 3.0", FutureWarning, + stacklevel=2, ) else: filters_config.insert(0, object_codec.get_config()) elif object_codec is not None: - warnings.warn("an object_codec is only needed for object arrays") + warnings.warn( + "an object_codec is only needed for object arrays", + stacklevel=2, + ) # use null to indicate no filters if not filters_config: @@ -868,8 +873,8 @@ def __getitem__(self, item: str): parent, key = self._get_parent(item) try: value = parent[key] - except KeyError: - raise KeyError(item) + except KeyError as e: + raise KeyError(item) from e else: if isinstance(value, self.cls): raise KeyError(item) @@ -887,8 +892,8 @@ def __delitem__(self, item: str): parent, key = self._get_parent(item) try: del parent[key] - except KeyError: - raise KeyError(item) + except KeyError as e: + raise KeyError(item) from e def __contains__(self, item: str): # type: ignore[override] try: @@ -1136,7 +1141,7 @@ def __setitem__(self, key, value): os.makedirs(dir_path) except OSError as e: if e.errno != errno.EEXIST: - raise KeyError(key) + raise KeyError(key) from e # write to temporary file # note we're not using tempfile.NamedTemporaryFile to avoid restrictive file permissions @@ -1417,11 +1422,23 @@ def _normalize_key(self, key): def getitems( self, keys: Sequence[str], *, contexts: Mapping[str, Context] ) -> Mapping[str, Any]: - keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") - # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back into the original key space. - return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} + keys_transformed = {self._normalize_key(key): key for key in keys} + results_transformed = self.map.getitems(list(keys_transformed), on_error="return") + results = {} + for k, v in results_transformed.items(): + if isinstance(v, self.exceptions): + # Cause recognized exceptions to prompt a KeyError in the + # function calling this method + continue + elif isinstance(v, Exception): + # Raise any other exception + raise v + else: + # The function calling this method may not recognize the transformed + # keys, so we send the values returned by self.map.getitems back into + # the original key space. + results[keys_transformed[k]] = v + return results def __getitem__(self, key): key = self._normalize_key(key) @@ -1592,6 +1609,12 @@ class NestedDirectoryStore(DirectoryStore): special handling for chunk keys so that chunk files for multidimensional arrays are stored in a nested directory tree. + .. deprecated:: 2.18.0 + NestedDirectoryStore will be removed in Zarr-Python 3.0 where controlling + the chunk key encoding will be supported as part of the array metadata. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -1663,6 +1686,13 @@ class NestedDirectoryStore(DirectoryStore): def __init__( self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = "/" ): + + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + super().__init__(path, normalize_keys=normalize_keys) if dimension_separator is None: dimension_separator = "/" @@ -1983,6 +2013,11 @@ def migrate_1to2(store): class DBMStore(Store): """Storage class using a DBM-style database. + .. deprecated:: 2.18.0 + DBMStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -2071,6 +2106,12 @@ def __init__( dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **open_kwargs, ): + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + if open is None: import dbm @@ -2188,6 +2229,10 @@ class LMDBStore(Store): """Storage class using LMDB. Requires the `lmdb `_ package to be installed. + .. deprecated:: 2.18.0 + LMDBStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. Parameters ---------- @@ -2249,6 +2294,12 @@ def __init__( ): import lmdb + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + # set default memory map size to something larger than the lmdb default, which is # very likely to be too small for any moderate array (logic copied from zict) map_size = 2**40 if sys.maxsize >= 2**32 else 2**28 @@ -2568,6 +2619,11 @@ def __delitem__(self, key): class SQLiteStore(Store): """Storage class using SQLite. + .. deprecated:: 2.18.0 + SQLiteStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -2600,6 +2656,12 @@ class SQLiteStore(Store): def __init__(self, path, dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs): import sqlite3 + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._dimension_separator = dimension_separator # normalize path @@ -2766,6 +2828,11 @@ class MongoDBStore(Store): .. note:: This is an experimental feature. + .. deprecated:: 2.18.0 + MongoDBStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Requires the `pymongo `_ package to be installed. @@ -2798,6 +2865,12 @@ def __init__( ): import pymongo + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._database = database self._collection = collection self._dimension_separator = dimension_separator @@ -2854,6 +2927,11 @@ class RedisStore(Store): .. note:: This is an experimental feature. + .. deprecated:: 2.18.0 + RedisStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Requires the `redis `_ package to be installed. @@ -2873,6 +2951,12 @@ def __init__( ): import redis + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._prefix = prefix self._kwargs = kwargs self._dimension_separator = dimension_separator diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index d9447c0832..4729dc01b6 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -123,7 +123,7 @@ def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): "compressor": kwargs.pop("compressor", self.compressor), "chunk_store": chunk_store, "storage_transformers": self.create_storage_transformers(shape), - "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype", None))), + "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype"))), } # keyword arguments for array instantiation @@ -3157,3 +3157,69 @@ def test_issue_1279(tmpdir): written_data = ds_reopened[:] assert_array_equal(data, written_data) + + +def test_scalar_indexing(): + store = zarr.KVStore({}) + + store["a"] = zarr.create((3,), chunks=(1,), store=store) + store["a"][:] = [1, 2, 3] + + assert store["a"][1] == np.array(2.0) + assert store["a"][(1,)] == np.array(2.0) + + store["a"][slice(1)] = [-1] + assert store["a"][0] == np.array(-1) + + store["a"][0] = -2 + assert store["a"][0] == np.array(-2) + + store["a"][slice(1)] = (-3,) + assert store["a"][0] == np.array(-3) + + +def test_object_array_indexing(): + # regression test for #1874 + from numcodecs import MsgPack + + root = zarr.group() + arr = root.create_dataset( + name="my_dataset", + shape=0, + dtype=object, + object_codec=MsgPack(), + ) + new_items = [ + ["A", 1], + ["B", 2, "hello"], + ] + arr_add = np.empty(len(new_items), dtype=object) + arr_add[:] = new_items + arr.append(arr_add) + + # heterogeneous elements + elem = ["C", 3] + arr[0] = elem + assert arr[0] == elem + + # homogeneous elements + elem = [1, 3] + arr[1] = elem + assert arr[1] == elem + + +@pytest.mark.parametrize("shape", ((1, 1, 1), (5, 5, 1), (1, 5, 5))) +def test_scalar_orthogonal_indexing(shape): + # regression test for https://github.com/zarr-developers/zarr-python/issues/1931 + store = zarr.MemoryStore({}) + data = np.random.randint(0, 255, shape) + arr = zarr.zeros( + shape=shape, chunks=shape[:-1] + (1,), compressor=None, store=store, dtype="u1" + ) + arr[:, :, :] = data + store.close() + + zf = zarr.open(store, "r") + assert_array_equal(zf[0, :, :], data[0, :, :]) + assert_array_equal(zf[:, 0, :], data[:, 0, :]) + assert_array_equal(zf[:, :, 0], data[:, :, 0]) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 6c08d7b88a..161e1eb813 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1,4 +1,5 @@ import atexit +import operator import os import sys import pickle @@ -87,6 +88,26 @@ def create_group( ) return g + def test_ipython_repr_methods(self): + g = self.create_group() + for method in [ + "html", + "json", + "javascript", + "markdown", + "svg", + "png", + "jpeg", + "latex", + "pdf", + "mimebundle", + ]: + assert operator.methodcaller(f"_repr_{method}_")(g) is None + with pytest.raises(AttributeError): + g._ipython_display_() + with pytest.raises(AttributeError): + g._ipython_canary_method_should_not_exist_() + def test_group_init_1(self): store, chunk_store = self.create_store() g = self.create_group(store, chunk_store=chunk_store) diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index c85d3f923f..57ab9a0781 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from zarr.codecs import Blosc, Delta, Pickle, Zlib +from zarr.codecs import Blosc, Delta, Pickle, Zlib, Zstd from zarr.errors import MetadataError from zarr.meta import ( ZARR_FORMAT, @@ -268,39 +268,42 @@ def test_encode_decode_array_dtype_shape(): assert meta_dec["filters"] is None -def test_encode_decode_array_dtype_shape_v3(): +@pytest.mark.parametrize("cname", ["zlib", "zstd"]) +def test_encode_decode_array_dtype_shape_v3(cname): + if cname == "zlib": + compressor = Zlib(1) + elif cname == "zstd": + compressor = Zstd(1) meta = dict( shape=(100,), chunk_grid=dict(type="regular", chunk_shape=(10,), separator=("/")), data_type=np.dtype("(10, 10) Tuple[np.dtype args = [] try: object_codec = codec_registry[codec_id](*args) - except KeyError: # pragma: no cover + except KeyError as e: # pragma: no cover raise ValueError( f"codec {codec_id!r} for object type {key!r} is not " f"available; please provide an object_codec manually" - ) + ) from e return dtype, object_codec dtype = np.dtype(dtype) @@ -234,8 +234,17 @@ def is_total_slice(item, shape: Tuple[int]) -> bool: if isinstance(item, tuple): return all( ( - isinstance(it, slice) - and ((it == slice(None)) or ((it.stop - it.start == sh) and (it.step in [1, None]))) + ( + isinstance(it, slice) + and ( + (it == slice(None)) + or ((it.stop - it.start == sh) and (it.step in [1, None])) + ) + ) + # The only scalar edge case, indexing with int 0 along a size-1 dimension + # is identical to a total slice + # https://github.com/zarr-developers/zarr-python/issues/1730 + or (isinstance(it, int) and it == 0 and sh == 1) ) for it, sh in zip(item, shape) ) @@ -323,7 +332,7 @@ def normalize_fill_value(fill_value, dtype: np.dtype): raise ValueError( f"fill_value {fill_value!r} is not valid for dtype {dtype}; " f"nested exception: {e}" - ) + ) from e return fill_value @@ -408,14 +417,13 @@ def info_html_report(items) -> str: class InfoReporter: def __init__(self, obj): self.obj = obj + self.items = self.obj.info_items() def __repr__(self): - items = self.obj.info_items() - return info_text_report(items) + return info_text_report(self.items) def _repr_html_(self): - items = self.obj.info_items() - return info_html_report(items) + return info_html_report(self.items) class TreeNode: @@ -484,13 +492,13 @@ def tree_widget_sublist(node, root=False, expand=False): def tree_widget(group, expand, level): try: import ipytree - except ImportError as error: + except ImportError as e: raise ImportError( - f"{error}: Run `pip install zarr[jupyter]` or `conda install ipytree`" + f"{e}: Run `pip install zarr[jupyter]` or `conda install ipytree`" f"to get the required ipytree dependency for displaying the tree " f"widget. If using jupyterlab<3, you also need to run " f"`jupyter labextension install ipytree`" - ) + ) from e result = ipytree.Tree() root = TreeNode(group, level=level)