Merge branch 'main' into string-arguments-for-codecs

Author: brokkoli71

.pre-commit-config.yaml

@@ -38,6 +38,7 @@ repos:
           # Tests
           - pytest
           - hypothesis
+          - s3fs
   - repo: https://github.com/scientific-python/cookie
     rev: 2025.01.22
     hooks:

changes/2862.bugfix.rst

@@ -0,0 +1 @@
+Fix a bug that prevented the number of initialized chunks being counted properly.

docs/user-guide/groups.rst

@@ -140,7 +140,7 @@ property. E.g.::
    No. bytes          : 8000000 (7.6M)
    No. bytes stored   : 1614
    Storage ratio      : 4956.6
-   Chunks Initialized : 0
+   Chunks Initialized : 10
    >>> baz.info
    Type               : Array
    Zarr format        : 3

pyproject.toml

@@ -358,7 +358,11 @@ module = [
     "tests.package_with_entrypoint.*",
     "zarr.testing.stateful",
     "tests.test_codecs.test_transpose",
-    "tests.test_config"
+    "tests.test_config",
+    "tests.test_store.test_zip",
+    "tests.test_store.test_local",
+    "tests.test_store.test_fsspec",
+    "tests.test_store.test_memory",
 ]
 strict = false
 
@@ -368,7 +372,11 @@ strict = false
 module = [
     "tests.test_codecs.test_codecs",
     "tests.test_metadata.*",
-    "tests.test_store.*",
+    "tests.test_store.test_core",
+    "tests.test_store.test_logging",
+    "tests.test_store.test_object",
+    "tests.test_store.test_stateful",
+    "tests.test_store.test_wrapper",
     "tests.test_group",
     "tests.test_indexing",
     "tests.test_properties",

src/zarr/core/array.py

@@ -117,6 +117,7 @@
     get_pipeline_class,
 )
 from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path
+from zarr.storage._utils import _relativize_path
 
 if TYPE_CHECKING:
     from collections.abc import Iterator, Sequence
@@ -3737,7 +3738,12 @@ async def chunks_initialized(
     store_contents = [
         x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path)
     ]
-    return tuple(chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents)
+    store_contents_relative = [
+        _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents
+    ]
+    return tuple(
+        chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents_relative
+    )
 
 
 def _build_parents(

src/zarr/storage/_utils.py

@@ -74,11 +74,62 @@ def _join_paths(paths: Iterable[str]) -> str:
     """
     Filter out instances of '' and join the remaining strings with '/'.
 
-    Because the root node of a zarr hierarchy is represented by an empty string,
+    Parameters
+    ----------
+    paths : Iterable[str]
+
+    Returns
+    -------
+    str
+
+    Examples
+    --------
+    >>> _join_paths(["", "a", "b"])
+    'a/b'
+    >>> _join_paths(["a", "b", "c"])
+    'a/b/c'
     """
     return "/".join(filter(lambda v: v != "", paths))
 
 
+def _relativize_path(*, path: str, prefix: str) -> str:
+    """
+    Make a "/"-delimited path relative to some prefix. If the prefix is '', then the path is
+    returned as-is. Otherwise, the prefix is removed from the path as well as the separator
+    string "/".
+
+    If ``prefix`` is not the empty string and ``path`` does not start with ``prefix``
+    followed by a "/" character, then an error is raised.
+
+    This function assumes that the prefix does not end with "/".
+
+    Parameters
+    ----------
+    path : str
+        The path to make relative to the prefix.
+    prefix : str
+        The prefix to make the path relative to.
+
+    Returns
+    -------
+    str
+
+    Examples
+    --------
+    >>> _relativize_path(path="", prefix="a/b")
+    'a/b'
+    >>> _relativize_path(path="a/b", prefix="a/b/c")
+    'c'
+    """
+    if prefix == "":
+        return path
+    else:
+        _prefix = prefix + "/"
+        if not path.startswith(_prefix):
+            raise ValueError(f"The first component of {path} does not start with {prefix}.")
+        return path.removeprefix(f"{prefix}/")
+
+
 def _normalize_paths(paths: Iterable[str]) -> tuple[str, ...]:
     """
     Normalize the input paths according to the normalization scheme used for zarr node paths.

src/zarr/testing/store.py

@@ -58,7 +58,7 @@ async def get(self, store: S, key: str) -> Buffer:
 
     @abstractmethod
     @pytest.fixture
-    def store_kwargs(self) -> dict[str, Any]:
+    def store_kwargs(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
         """Kwargs for instantiating a store"""
         ...
 

src/zarr/testing/utils.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import Callable, Coroutine
-from typing import TYPE_CHECKING, Any, TypeVar, cast
+from typing import TYPE_CHECKING, TypeVar, cast
 
 import pytest
 
@@ -38,13 +37,13 @@ def has_cupy() -> bool:
         return False
 
 
-T_Callable = TypeVar("T_Callable", bound=Callable[..., Coroutine[Any, Any, None] | None])
+T = TypeVar("T")
 
 
 # Decorator for GPU tests
-def gpu_test(func: T_Callable) -> T_Callable:
+def gpu_test(func: T) -> T:
     return cast(
-        "T_Callable",
+        "T",
         pytest.mark.gpu(
             pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")(
                 func

tests/test_array.py

@@ -388,12 +388,13 @@ async def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]
         assert observed == expected
 
 
-async def test_chunks_initialized() -> None:
+@pytest.mark.parametrize("path", ["", "foo"])
+async def test_chunks_initialized(path: str) -> None:
     """
     Test that chunks_initialized accurately returns the keys of stored chunks.
     """
     store = MemoryStore()
-    arr = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="i4")
+    arr = zarr.create_array(store, name=path, shape=(100,), chunks=(10,), dtype="i4")
 
     chunks_accumulated = tuple(
         accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_chunk_keys()))

tests/test_store/test_core.py

@@ -8,7 +8,13 @@
 from zarr.core.common import AccessModeLiteral, ZarrFormat
 from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath
 from zarr.storage._common import contains_array, contains_group, make_store_path
-from zarr.storage._utils import _join_paths, _normalize_path_keys, _normalize_paths, normalize_path
+from zarr.storage._utils import (
+    _join_paths,
+    _normalize_path_keys,
+    _normalize_paths,
+    _relativize_path,
+    normalize_path,
+)
 
 
 @pytest.mark.parametrize("path", ["foo", "foo/bar"])
@@ -221,3 +227,27 @@ def test_normalize_path_keys():
     """
     data = {"a": 10, "//b": 10}
     assert _normalize_path_keys(data) == {normalize_path(k): v for k, v in data.items()}
+
+
+@pytest.mark.parametrize(
+    ("path", "prefix", "expected"),
+    [
+        ("a", "", "a"),
+        ("a/b/c", "a/b", "c"),
+        ("a/b/c", "a", "b/c"),
+    ],
+)
+def test_relativize_path_valid(path: str, prefix: str, expected: str) -> None:
+    """
+    Test the normal behavior of the _relativize_path function. Prefixes should be removed from the
+    path argument.
+    """
+    assert _relativize_path(path=path, prefix=prefix) == expected
+
+
+def test_relativize_path_invalid() -> None:
+    path = "a/b/c"
+    prefix = "b"
+    msg = f"The first component of {path} does not start with {prefix}."
+    with pytest.raises(ValueError, match=msg):
+        _relativize_path(path="a/b/c", prefix="b")

tests/test_store/test_fsspec.py

@@ -3,7 +3,7 @@
 import json
 import os
 import re
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import pytest
 from packaging.version import parse as parse_version
@@ -17,8 +17,13 @@
 
 if TYPE_CHECKING:
     from collections.abc import Generator
+    from pathlib import Path
 
     import botocore.client
+    import s3fs
+
+    from zarr.core.common import JSON
+
 
 # Warning filter due to https://github.com/boto/boto3/issues/3889
 pytestmark = [
@@ -109,10 +114,13 @@ async def test_basic() -> None:
     data = b"hello"
     await store.set("foo", cpu.Buffer.from_bytes(data))
     assert await store.exists("foo")
-    assert (await store.get("foo", prototype=default_buffer_prototype())).to_bytes() == data
+    buf = await store.get("foo", prototype=default_buffer_prototype())
+    assert buf is not None
+    assert buf.to_bytes() == data
     out = await store.get_partial_values(
         prototype=default_buffer_prototype(), key_ranges=[("foo", OffsetByteRequest(1))]
     )
+    assert out[0] is not None
     assert out[0].to_bytes() == data[1:]
 
 
@@ -121,7 +129,7 @@ class TestFsspecStoreS3(StoreTests[FsspecStore, cpu.Buffer]):
     buffer_cls = cpu.Buffer
 
     @pytest.fixture
-    def store_kwargs(self, request) -> dict[str, str | bool]:
+    def store_kwargs(self) -> dict[str, str | bool]:
         try:
             from fsspec import url_to_fs
         except ImportError:
@@ -133,7 +141,7 @@ def store_kwargs(self, request) -> dict[str, str | bool]:
         return {"fs": fs, "path": path}
 
     @pytest.fixture
-    def store(self, store_kwargs: dict[str, str | bool]) -> FsspecStore:
+    async def store(self, store_kwargs: dict[str, Any]) -> FsspecStore:
         return self.store_cls(**store_kwargs)
 
     async def get(self, store: FsspecStore, key: str) -> Buffer:
@@ -168,7 +176,11 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None:
             "anon": False,
         }
 
-        meta = {"attributes": {"key": "value"}, "zarr_format": 3, "node_type": "group"}
+        meta: dict[str, JSON] = {
+            "attributes": {"key": "value"},
+            "zarr_format": 3,
+            "node_type": "group",
+        }
 
         await store.set(
             "zarr.json",
@@ -179,7 +191,7 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None:
         )
         assert dict(group.attrs) == {"key": "value"}
 
-        meta["attributes"]["key"] = "value-2"
+        meta["attributes"]["key"] = "value-2"  # type: ignore[index]
         await store.set(
             "directory-2/zarr.json",
             self.buffer_cls.from_bytes(json.dumps(meta).encode()),
@@ -189,7 +201,7 @@ async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None:
         )
         assert dict(group.attrs) == {"key": "value-2"}
 
-        meta["attributes"]["key"] = "value-3"
+        meta["attributes"]["key"] = "value-3"  # type: ignore[index]
         await store.set(
             "directory-3/zarr.json",
             self.buffer_cls.from_bytes(json.dumps(meta).encode()),
@@ -216,7 +228,7 @@ def test_from_upath(self) -> None:
         assert result.fs.asynchronous
         assert result.path == f"{test_bucket_name}/foo/bar"
 
-    def test_init_raises_if_path_has_scheme(self, store_kwargs) -> None:
+    def test_init_raises_if_path_has_scheme(self, store_kwargs: dict[str, Any]) -> None:
         # regression test for https://github.com/zarr-developers/zarr-python/issues/2342
         store_kwargs["path"] = "s3://" + store_kwargs["path"]
         with pytest.raises(
@@ -237,7 +249,7 @@ def test_init_warns_if_fs_asynchronous_is_false(self) -> None:
         with pytest.warns(UserWarning, match=r".* was not created with `asynchronous=True`.*"):
             self.store_cls(**store_kwargs)
 
-    async def test_empty_nonexistent_path(self, store_kwargs) -> None:
+    async def test_empty_nonexistent_path(self, store_kwargs: dict[str, Any]) -> None:
         # regression test for https://github.com/zarr-developers/zarr-python/pull/2343
         store_kwargs["path"] += "/abc"
         store = await self.store_cls.open(**store_kwargs)
@@ -256,7 +268,7 @@ async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None:
     parse_version(fsspec.__version__) < parse_version("2024.12.0"),
     reason="No AsyncFileSystemWrapper",
 )
-def test_wrap_sync_filesystem():
+def test_wrap_sync_filesystem() -> None:
     """The local fs is not async so we should expect it to be wrapped automatically"""
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 
@@ -270,7 +282,7 @@ def test_wrap_sync_filesystem():
     parse_version(fsspec.__version__) < parse_version("2024.12.0"),
     reason="No AsyncFileSystemWrapper",
 )
-def test_no_wrap_async_filesystem():
+def test_no_wrap_async_filesystem() -> None:
     """An async fs should not be wrapped automatically; fsspec's https filesystem is such an fs"""
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 
@@ -284,12 +296,12 @@ def test_no_wrap_async_filesystem():
     parse_version(fsspec.__version__) < parse_version("2024.12.0"),
     reason="No AsyncFileSystemWrapper",
 )
-async def test_delete_dir_wrapped_filesystem(tmpdir) -> None:
+async def test_delete_dir_wrapped_filesystem(tmp_path: Path) -> None:
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
     from fsspec.implementations.local import LocalFileSystem
 
     wrapped_fs = AsyncFileSystemWrapper(LocalFileSystem(auto_mkdir=True))
-    store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmpdir}/test/path")
+    store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmp_path}/test/path")
 
     assert isinstance(store.fs, AsyncFileSystemWrapper)
     assert store.fs.asynchronous