Merge branch 'main' into v2-codec

zarr-developers · Oct 24, 2024 · 08c924d · 08c924d
2 parents c608fef + bc588a7
commit 08c924d
Show file tree

Hide file tree

Showing 32 changed files with 789 additions and 329 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -43,3 +43,35 @@ jobs:
     - name: Run Tests
       run: |
         hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run
+
+  test-upstream-and-min-deps:
+    name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }}
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11', "3.13"]
+        dependency-set: ["upstream", "min_deps"]
+        exclude:
+          - python-version: "3.13"
+            dependency-set: min_deps
+          - python-version: "3.11"
+            dependency-set: upstream
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install Hatch
+      run: |
+        python -m pip install --upgrade pip 
+        pip install hatch
+    - name: Set Up Hatch Env
+      run: |
+        hatch env create ${{ matrix.dependency-set }}
+        hatch env run -e ${{ matrix.dependency-set }} list-env
+    - name: Run Tests
+      run: |
+        hatch env run --env ${{ matrix.dependency-set }} run
diff --git a/.pep8speaks.yml b/.pep8speaks.yml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -7,7 +7,7 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.9
+    rev: v0.7.0
     hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
@@ -22,7 +22,7 @@ repos:
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.12.1
     hooks:
       - id: mypy
         files: src|tests

diff --git a/docs/guide/storage.rst b/docs/guide/storage.rst
@@ -72,7 +72,7 @@ that implements the `AbstractFileSystem` API,
 .. code-block:: python
 
    >>> import zarr
-   >>> store = zarr.storage.RemoteStore("gs://foo/bar", mode="r")
+   >>> store = zarr.storage.RemoteStore.from_url("gs://foo/bar", mode="r")
    >>> zarr.open(store=store)
    <Array <RemoteStore(GCSFileSystem, foo/bar)> shape=(10, 20) dtype=float32>
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -28,12 +28,13 @@ requires-python = ">=3.11"
 dependencies = [
     'asciitree',
     'numpy>=1.25',
-    'numcodecs>=0.10.2',
-    'fsspec>2024',
-    'crc32c',
-    'typing_extensions',
-    'donfig',
+    'numcodecs>=0.13',
+    'fsspec>=2022.10.0',
+    'crc32c>=2.3',
+    'typing_extensions>=4.6',
+    'donfig>=0.8',
 ]
+
 dynamic = [
   "version",
 ]
@@ -98,7 +99,7 @@ extra = [
 ]
 optional = [
     'lmdb',
-    'universal-pathlib',
+    'universal-pathlib>=0.0.22',
 ]
 
 [project.urls]
@@ -183,6 +184,65 @@ features = ['docs']
 build = "cd docs && make html"
 serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0"
 
+[tool.hatch.envs.upstream]
+python = "3.13"
+dependencies = [
+    'numpy',  # from scientific-python-nightly-wheels
+    'numcodecs @ git+https://github.com/zarr-developers/numcodecs',
+    'fsspec @ git+https://github.com/fsspec/filesystem_spec',
+    's3fs @ git+https://github.com/fsspec/s3fs',
+    'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib',
+    'crc32c @ git+https://github.com/ICRAR/crc32c',
+    'typing_extensions @ git+https://github.com/python/typing_extensions',
+    'donfig @ git+https://github.com/pytroll/donfig',
+    # test deps
+    'hypothesis',
+    'pytest',
+    'pytest-cov',
+    'pytest-asyncio',
+    'moto[s3]',
+]
+
+[tool.hatch.envs.upstream.env-vars]
+PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/"
+PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/"
+PIP_PRE = "1"
+
+[tool.hatch.envs.upstream.scripts]
+run = "pytest --verbose"
+run-mypy = "mypy src"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
+list-env = "pip list"
+
+[tool.hatch.envs.min_deps]
+description = """Test environment for minimum supported dependencies
+
+See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/
+"""
+python = "3.11"
+dependencies = [
+    'numpy==1.25.*',
+    'numcodecs==0.13.*',  # 0.13 needed for? (should be 0.11)
+    'fsspec==2022.10.0',
+    's3fs==2022.10.0',
+    'universal_pathlib==0.0.22',
+    'crc32c==2.3.*',
+    'typing_extensions==4.6.*',  # 4.5 needed for @deprecated, 4.6 for Buffer
+    'donfig==0.8.*',
+    # test deps
+    'hypothesis',
+    'pytest',
+    'pytest-cov',
+    'pytest-asyncio',
+    'moto[s3]',
+]
+
+[tool.hatch.envs.min_deps.scripts]
+run = "pytest --verbose"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
+list-env = "pip list"
+
+
 [tool.ruff]
 line-length = 100
 force-exclude = true
@@ -214,6 +274,7 @@ extend-select = [
     "B",    # flake8-bugbear
     "C4",   # flake8-comprehensions
     "FLY",  # flynt
+    "FURB", # refurb
     "G",    # flake8-logging-format
     "I",    # isort
     "ISC",  # flake8-implicit-str-concat
@@ -329,4 +390,15 @@ ignore = [
 
 [tool.numpydoc_validation]
 # See https://numpydoc.readthedocs.io/en/latest/validation.html#built-in-validation-checks for list of checks
-checks = ["GL06", "GL07", "GL10", "PR03", "PR05", "PR06"]
+checks = [
+    "GL06",
+    "GL07",
+    "GL09",
+    "GL10",
+    "SS02",
+    "SS04",
+    "PR02",
+    "PR03",
+    "PR05",
+    "PR06",
+]
diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py
@@ -85,7 +85,7 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
 
         Parameters
         ----------
-        chunk_spec : ArraySpec
+        array_spec : ArraySpec
 
         Returns
         -------
@@ -99,11 +99,11 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun
 
         Parameters
         ----------
-        shape: ChunkCoords
+        shape : ChunkCoords
             The array shape
-        dtype: np.dtype[Any]
+        dtype : np.dtype[Any]
             The array data type
-        chunk_grid: ChunkGrid
+        chunk_grid : ChunkGrid
             The array chunk grid
         """
         ...
@@ -292,11 +292,11 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun
 
         Parameters
         ----------
-        shape: ChunkCoords
+        shape : ChunkCoords
             The array shape
-        dtype: np.dtype[Any]
+        dtype : np.dtype[Any]
             The array data type
-        chunk_grid: ChunkGrid
+        chunk_grid : ChunkGrid
             The array chunk grid
         """
         ...
@@ -308,7 +308,7 @@ def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int:
 
         Parameters
         ----------
-        input_byte_length : int
+        byte_length : int
         array_spec : ArraySpec
 
         Returns
@@ -327,7 +327,7 @@ async def decode(
 
         Parameters
         ----------
-        chunks_and_specs : Iterable[tuple[Buffer | None, ArraySpec]]
+        chunk_bytes_and_specs : Iterable[tuple[Buffer | None, ArraySpec]]
             Ordered set of encoded chunks with their accompanying chunk spec.
 
         Returns
@@ -346,7 +346,7 @@ async def encode(
 
         Parameters
         ----------
-        chunks_and_specs : Iterable[tuple[NDBuffer | None, ArraySpec]]
+        chunk_arrays_and_specs : Iterable[tuple[NDBuffer | None, ArraySpec]]
             Ordered set of to-be-encoded chunks with their accompanying chunk spec.
 
         Returns

diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py
@@ -2,6 +2,7 @@
 
 from abc import ABC, abstractmethod
 from asyncio import gather
+from itertools import starmap
 from typing import TYPE_CHECKING, NamedTuple, Protocol, runtime_checkable
 
 if TYPE_CHECKING:
@@ -162,12 +163,12 @@ def with_mode(self, mode: AccessModeLiteral) -> Self:
 
         Parameters
         ----------
-        mode: AccessModeLiteral
+        mode : AccessModeLiteral
             The new mode to use.
 
         Returns
         -------
-        store:
+        store
             A new store of the same type with the new mode.
 
         Examples
@@ -282,7 +283,7 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
         """
         Insert multiple (key, value) pairs into storage.
         """
-        await gather(*(self.set(key, value) for key, value in values))
+        await gather(*starmap(self.set, values))
         return
 
     @property