Merge branch 'main' into string-arguments-for-codecs

Author: brokkoli71

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -57,7 +57,22 @@ body:
     id: reproduce
     attributes:
       label: Steps to reproduce
-      description: Minimal, reproducible code sample, a copy-pastable example if possible.
+      description: Minimal, reproducible code sample. Must list dependencies in [inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/#example). When put in a file named `issue.py` calling `uv run issue.py` should show the issue.
+      value: |
+        ```python
+        # /// script
+        # requires-python = ">=3.11"
+        # dependencies = [
+        #   "zarr@git+https://github.com/zarr-developers/zarr-python.git@main",
+        # ]
+        # ///
+        #
+        # This script automatically imports the development branch of zarr to check for issues
+
+        import zarr
+        # your reproducer code
+        # zarr.print_debug_info()
+        ```
     validations:
       required: true
   - type: textarea

changes/2950.bufgix.rst

@@ -0,0 +1 @@
+Implemented ``move`` for ``LocalStore`` and ``ZipStore``. This allows users to move the store to a different root path.

changes/2962.fix.rst

@@ -0,0 +1 @@
+Trying to open an array with ``mode='r'`` when the store is not read-only now raises an error.

changes/3021.feature.rst

@@ -0,0 +1 @@
+Adds ``fill_value`` to the list of attributes displayed in the output of the ``AsyncArray.info()`` method.

changes/3068.bugfix.rst

@@ -0,0 +1 @@
+Use :py:func:`numpy.zeros` instead of :py:func:`np.full` for a performance speedup when creating a `zarr.core.buffer.NDBuffer` with `fill_value=0`.

changes/3081.feature.rst

@@ -31,6 +31,8 @@ Bugfixes
   To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. (:issue:`3039`)
 - Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. (:issue:`3045`)
 - Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. (:issue:`3062`)
+- Internally use `typesize` constructor parameter for :class:`numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. (:issue:`2962`)
+- Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed. (:issue:`2950`)
 
 
 Misc

changes/3082.feature.rst

@@ -183,6 +183,7 @@ which can be used to print useful diagnostics, e.g.::
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.int32
+   Fill value         : 0
    Shape              : (10000, 10000)
    Chunk shape        : (1000, 1000)
    Order              : C
@@ -200,6 +201,7 @@ prints additional diagnostics, e.g.::
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.int32
+   Fill value         : 0
    Shape              : (10000, 10000)
    Chunk shape        : (1000, 1000)
    Order              : C
@@ -287,6 +289,7 @@ Here is an example using a delta filter with the Blosc compressor::
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.int32
+   Fill value         : 0
    Shape              : (10000, 10000)
    Chunk shape        : (1000, 1000)
    Order              : C
@@ -601,6 +604,7 @@ Sharded arrays can be created by providing the ``shards`` parameter to :func:`za
   Type               : Array
   Zarr format        : 3
   Data type          : DataType.uint8
+  Fill value         : 0
   Shape              : (10000, 10000)
   Shard shape        : (1000, 1000)
   Chunk shape        : (100, 100)

docs/release-notes.rst

@@ -129,6 +129,7 @@ property. E.g.::
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.int64
+   Fill value         : 0
    Shape              : (1000000,)
    Chunk shape        : (100000,)
    Order              : C
@@ -145,6 +146,7 @@ property. E.g.::
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.float32
+   Fill value         : 0.0
    Shape              : (1000, 1000)
    Chunk shape        : (100, 100)
    Order              : C

docs/user-guide/arrays.rst

@@ -92,6 +92,7 @@ To use sharding, you need to specify the ``shards`` parameter when creating the
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.uint8
+   Fill value         : 0
    Shape              : (10000, 10000, 1000)
    Shard shape        : (1000, 1000, 1000)
    Chunk shape        : (100, 100, 100)
@@ -122,6 +123,7 @@ ratios, depending on the correlation structure within the data. E.g.::
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.int32
+   Fill value         : 0
    Shape              : (10000, 10000)
    Chunk shape        : (1000, 1000)
    Order              : C
@@ -141,6 +143,7 @@ ratios, depending on the correlation structure within the data. E.g.::
    Type               : Array
    Zarr format        : 3
    Data type          : DataType.int32
+   Fill value         : 0
    Shape              : (10000, 10000)
    Chunk shape        : (1000, 1000)
    Order              : F

docs/user-guide/groups.rst

@@ -291,8 +291,8 @@ extend-exclude = [
 extend-select = [
     "ANN",  # flake8-annotations
     "B",    # flake8-bugbear
-    "EXE",  # flake8-executable
     "C4",   # flake8-comprehensions
+    "EXE",  # flake8-executable
     "FA",   # flake8-future-annotations
     "FLY",  # flynt
     "FURB", # refurb
@@ -364,14 +364,14 @@ module = [
     "tests.test_store.test_local",
     "tests.test_store.test_fsspec",
     "tests.test_store.test_memory",
+    "tests.test_codecs.test_codecs",
 ]
 strict = false
 
 # TODO: Move the next modules up to the strict = false section
 # and fix the errors
 [[tool.mypy.overrides]]
 module = [
-    "tests.test_codecs.test_codecs",
     "tests.test_metadata.*",
     "tests.test_store.test_core",
     "tests.test_store.test_logging",

docs/user-guide/performance.rst

@@ -329,7 +329,7 @@ async def open(
         try:
             metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format)
             # TODO: remove this cast when we fix typing for array metadata dicts
-            _metadata_dict = cast(ArrayMetadataDict, metadata_dict)
+            _metadata_dict = cast("ArrayMetadataDict", metadata_dict)
             # for v2, the above would already have raised an exception if not an array
             zarr_format = _metadata_dict["zarr_format"]
             is_v3_array = zarr_format == 3 and _metadata_dict.get("node_type") == "array"

pyproject.toml

@@ -40,7 +40,9 @@ async def _decode_single(
         inner_bytes = data[:-4]
 
         # Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved
-        computed_checksum = np.uint32(crc32c(cast(typing_extensions.Buffer, inner_bytes))).tobytes()
+        computed_checksum = np.uint32(
+            crc32c(cast("typing_extensions.Buffer", inner_bytes))
+        ).tobytes()
         stored_checksum = bytes(crc32_bytes)
         if computed_checksum != stored_checksum:
             raise ValueError(
@@ -55,7 +57,7 @@ async def _encode_single(
     ) -> Buffer | None:
         data = chunk_bytes.as_numpy_array()
         # Calculate the checksum and "cast" it to a numpy array
-        checksum = np.array([crc32c(cast(typing_extensions.Buffer, data))], dtype=np.uint32)
+        checksum = np.array([crc32c(cast("typing_extensions.Buffer", data))], dtype=np.uint32)
         # Append the checksum (as bytes) to the data
         return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B")))
 

src/zarr/api/asynchronous.py

@@ -115,7 +115,7 @@ class _ShardIndex(NamedTuple):
     def chunks_per_shard(self) -> ChunkCoords:
         result = tuple(self.offsets_and_lengths.shape[0:-1])
         # The cast is required until https://github.com/numpy/numpy/pull/27211 is merged
-        return cast(ChunkCoords, result)
+        return cast("ChunkCoords", result)
 
     def _localize_chunk(self, chunk_coords: ChunkCoords) -> ChunkCoords:
         return tuple(

src/zarr/codecs/crc32c_.py

@@ -23,7 +23,7 @@ def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]:
         raise TypeError(f"Expected an iterable. Got {data} instead.")
     if not all(isinstance(a, int) for a in data):
         raise TypeError(f"Expected an iterable of integers. Got {data} instead.")
-    return tuple(cast(Iterable[int], data))
+    return tuple(cast("Iterable[int]", data))
 
 
 @dataclass(frozen=True)

src/zarr/codecs/sharding.py

@@ -67,7 +67,7 @@ def byte_info(size: int) -> str:
         return f"{size} ({human_readable_size(size)})"
 
 
-@dataclasses.dataclass(kw_only=True)
+@dataclasses.dataclass(kw_only=True, frozen=True, slots=True)
 class ArrayInfo:
     """
     Visual summary for an Array.
@@ -79,6 +79,7 @@ class ArrayInfo:
     _type: Literal["Array"] = "Array"
     _zarr_format: ZarrFormat
     _data_type: np.dtype[Any] | DataType
+    _fill_value: object
     _shape: tuple[int, ...]
     _shard_shape: tuple[int, ...] | None = None
     _chunk_shape: tuple[int, ...] | None = None
@@ -97,6 +98,7 @@ def __repr__(self) -> str:
         Type               : {_type}
         Zarr format        : {_zarr_format}
         Data type          : {_data_type}
+        Fill value         : {_fill_value}
         Shape              : {_shape}""")
 
         if self._shard_shape is not None:

src/zarr/codecs/transpose.py

@@ -903,7 +903,7 @@ async def open(
         store_path = await make_store_path(store)
         metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format)
         # TODO: remove this cast when we have better type hints
-        _metadata_dict = cast(ArrayV3MetadataDict, metadata_dict)
+        _metadata_dict = cast("ArrayV3MetadataDict", metadata_dict)
         return cls(store_path=store_path, metadata=_metadata_dict)
 
     @property
@@ -1399,7 +1399,7 @@ async def _set_selection(
             if isinstance(array_like, np._typing._SupportsArrayFunc):
                 # TODO: need to handle array types that don't support __array_function__
                 # like PyTorch and JAX
-                array_like_ = cast(np._typing._SupportsArrayFunc, array_like)
+                array_like_ = cast("np._typing._SupportsArrayFunc", array_like)
             value = np.asanyarray(value, dtype=self.metadata.dtype, like=array_like_)
         else:
             if not hasattr(value, "shape"):
@@ -1413,7 +1413,7 @@ async def _set_selection(
                     value = value.astype(dtype=self.metadata.dtype, order="A")
                 else:
                     value = np.array(value, dtype=self.metadata.dtype, order="A")
-        value = cast(NDArrayLike, value)
+        value = cast("NDArrayLike", value)
         # We accept any ndarray like object from the user and convert it
         # to a NDBuffer (or subclass). From this point onwards, we only pass
         # Buffer and NDBuffer between components.
@@ -1702,6 +1702,7 @@ def _info(
         return ArrayInfo(
             _zarr_format=self.metadata.zarr_format,
             _data_type=_data_type,
+            _fill_value=self.metadata.fill_value,
             _shape=self.shape,
             _order=self.order,
             _shard_shape=self.shards,
@@ -2436,11 +2437,11 @@ def __getitem__(self, selection: Selection) -> NDArrayLikeOrScalar:
         """
         fields, pure_selection = pop_fields(selection)
         if is_pure_fancy_indexing(pure_selection, self.ndim):
-            return self.vindex[cast(CoordinateSelection | MaskSelection, selection)]
+            return self.vindex[cast("CoordinateSelection | MaskSelection", selection)]
         elif is_pure_orthogonal_indexing(pure_selection, self.ndim):
             return self.get_orthogonal_selection(pure_selection, fields=fields)
         else:
-            return self.get_basic_selection(cast(BasicSelection, pure_selection), fields=fields)
+            return self.get_basic_selection(cast("BasicSelection", pure_selection), fields=fields)
 
     def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None:
         """Modify data for an item or region of the array.
@@ -2535,11 +2536,11 @@ def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None:
         """
         fields, pure_selection = pop_fields(selection)
         if is_pure_fancy_indexing(pure_selection, self.ndim):
-            self.vindex[cast(CoordinateSelection | MaskSelection, selection)] = value
+            self.vindex[cast("CoordinateSelection | MaskSelection", selection)] = value
         elif is_pure_orthogonal_indexing(pure_selection, self.ndim):
             self.set_orthogonal_selection(pure_selection, value, fields=fields)
         else:
-            self.set_basic_selection(cast(BasicSelection, pure_selection), value, fields=fields)
+            self.set_basic_selection(cast("BasicSelection", pure_selection), value, fields=fields)
 
     @_deprecate_positional_args
     def get_basic_selection(
@@ -3657,7 +3658,7 @@ def update_attributes(self, new_attributes: dict[str, JSON]) -> Array:
         # TODO: remove this cast when type inference improves
         new_array = sync(self._async_array.update_attributes(new_attributes))
         # TODO: remove this cast when type inference improves
-        _new_array = cast(AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], new_array)
+        _new_array = cast("AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]", new_array)
         return type(self)(_new_array)
 
     def __repr__(self) -> str:
@@ -4252,7 +4253,7 @@ async def init_array(
             serializer=serializer,
             dtype=dtype_parsed,
         )
-        sub_codecs = cast(tuple[Codec, ...], (*array_array, array_bytes, *bytes_bytes))
+        sub_codecs = cast("tuple[Codec, ...]", (*array_array, array_bytes, *bytes_bytes))
         codecs_out: tuple[Codec, ...]
         if shard_shape_parsed is not None:
             index_location = None
@@ -4523,7 +4524,7 @@ def _parse_keep_array_attr(
                 compressors = "auto"
         if serializer == "keep":
             if zarr_format == 3 and data.metadata.zarr_format == 3:
-                serializer = cast(SerializerLike, data.serializer)
+                serializer = cast("SerializerLike", data.serializer)
             else:
                 serializer = "auto"
         if fill_value is None:
@@ -4692,7 +4693,7 @@ def _parse_chunk_encoding_v3(
         if isinstance(filters, dict | Codec):
             maybe_array_array = (filters,)
         else:
-            maybe_array_array = cast(Iterable[Codec | dict[str, JSON]], filters)
+            maybe_array_array = cast("Iterable[Codec | dict[str, JSON]]", filters)
         out_array_array = tuple(_parse_array_array_codec(c) for c in maybe_array_array)
 
     if serializer == "auto":
@@ -4711,7 +4712,7 @@ def _parse_chunk_encoding_v3(
         if isinstance(compressors, dict | Codec):
             maybe_bytes_bytes = (compressors,)
         else:
-            maybe_bytes_bytes = cast(Iterable[Codec | dict[str, JSON]], compressors)
+            maybe_bytes_bytes = cast("Iterable[Codec | dict[str, JSON]]", compressors)
 
         out_bytes_bytes = tuple(_parse_bytes_bytes_codec(c) for c in maybe_bytes_bytes)
 

src/zarr/core/_info.py

@@ -64,7 +64,7 @@ def from_dict(cls, data: ArrayConfigParams) -> Self:
         """
         kwargs_out: ArrayConfigParams = {}
         for f in fields(ArrayConfig):
-            field_name = cast(Literal["order", "write_empty_chunks"], f.name)
+            field_name = cast("Literal['order', 'write_empty_chunks']", f.name)
             if field_name not in data:
                 kwargs_out[field_name] = zarr_config.get(f"array.{field_name}")
             else:

src/zarr/core/array.py

@@ -159,7 +159,7 @@ def create_zero_length(cls) -> Self:
         if cls is Buffer:
             raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'")
         return cls(
-            cast(ArrayLike, None)
+            cast("ArrayLike", None)
         )  # This line will never be reached, but it satisfies the type checker
 
     @classmethod
@@ -207,7 +207,7 @@ def from_buffer(cls, buffer: Buffer) -> Self:
         if cls is Buffer:
             raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'")
         return cls(
-            cast(ArrayLike, None)
+            cast("ArrayLike", None)
         )  # This line will never be reached, but it satisfies the type checker
 
     @classmethod
@@ -227,7 +227,7 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self:
         if cls is Buffer:
             raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'")
         return cls(
-            cast(ArrayLike, None)
+            cast("ArrayLike", None)
         )  # This line will never be reached, but it satisfies the type checker
 
     def as_array_like(self) -> ArrayLike:
@@ -371,7 +371,7 @@ def create(
                 "Cannot call abstract method on the abstract class 'NDBuffer'"
             )
         return cls(
-            cast(NDArrayLike, None)
+            cast("NDArrayLike", None)
         )  # This line will never be reached, but it satisfies the type checker
 
     @classmethod
@@ -408,7 +408,7 @@ def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self:
                 "Cannot call abstract method on the abstract class 'NDBuffer'"
             )
         return cls(
-            cast(NDArrayLike, None)
+            cast("NDArrayLike", None)
         )  # This line will never be reached, but it satisfies the type checker
 
     def as_ndarray_like(self) -> NDArrayLike:
@@ -440,7 +440,7 @@ def as_scalar(self) -> ScalarType:
         """Returns the buffer as a scalar value"""
         if self._data.size != 1:
             raise ValueError("Buffer does not contain a single scalar value")
-        return cast(ScalarType, self.as_numpy_array()[()])
+        return cast("ScalarType", self.as_numpy_array()[()])
 
     @property
     def dtype(self) -> np.dtype[Any]:

src/zarr/core/array_spec.py

@@ -154,7 +154,8 @@ def create(
         order: Literal["C", "F"] = "C",
         fill_value: Any | None = None,
     ) -> Self:
-        if fill_value is None:
+        # np.zeros is much faster than np.full, and therefore using it when possible is better.
+        if fill_value is None or (isinstance(fill_value, int) and fill_value == 0):
             return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order))
         else:
             return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order))