Merge branch 'doc/tutorial' of github.com:jhamman/zarr-python into do…

…c/tutorial

.github/workflows/test.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.11', '3.12']
+        python-version: ['3.11', '3.12', '3.13']
         numpy-version: ['1.25', '1.26', '2.0']
         dependency-set: ["minimal", "optional"]
 

pyproject.toml

@@ -49,6 +49,7 @@ classifiers = [
     'Programming Language :: Python :: 3',
     'Programming Language :: Python :: 3.11',
     'Programming Language :: Python :: 3.12',
+    'Programming Language :: Python :: 3.13',
 ]
 license = {text = "MIT License"}
 keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"]
@@ -132,17 +133,17 @@ dependencies = [
 features = ["test", "extra"]
 
 [[tool.hatch.envs.test.matrix]]
-python = ["3.11", "3.12"]
+python = ["3.11", "3.12", "3.13"]
 numpy = ["1.25", "1.26", "2.0"]
 version = ["minimal"]
 
 [[tool.hatch.envs.test.matrix]]
-python = ["3.11", "3.12"]
+python = ["3.11", "3.12", "3.13"]
 numpy = ["1.25", "1.26", "2.0"]
 features = ["optional"]
 
 [[tool.hatch.envs.test.matrix]]
-python = ["3.11", "3.12"]
+python = ["3.11", "3.12", "3.13"]
 numpy = ["1.25", "1.26", "2.0"]
 features = ["gpu"]
 
@@ -152,7 +153,7 @@ run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pypro
 run = "run-coverage --no-cov"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
-run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
 list-env = "pip list"
 
 [tool.hatch.envs.gputest]
@@ -163,7 +164,7 @@ dependencies = [
 features = ["test", "extra", "gpu"]
 
 [[tool.hatch.envs.gputest.matrix]]
-python = ["3.11", "3.12"]
+python = ["3.11", "3.12", "3.13"]
 numpy = ["1.25", "1.26", "2.0"]
 version = ["minimal"]
 
@@ -172,7 +173,7 @@ run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests"
 run = "run-coverage --no-cov"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
-run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*"
+run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*"
 list-env = "pip list"
 
 [tool.hatch.envs.docs]
@@ -281,18 +282,17 @@ ignore_errors = true
 
 [[tool.mypy.overrides]]
 module = [
-    "tests.v2.*",
-    "tests.v3.package_with_entrypoint.*",
-    "tests.v3.test_codecs.test_codecs",
-    "tests.v3.test_codecs.test_transpose",
-    "tests.v3.test_metadata.*",
-    "tests.v3.test_store.*",
-    "tests.v3.test_config",
-    "tests.v3.test_group",
-    "tests.v3.test_indexing",
-    "tests.v3.test_properties",
-    "tests.v3.test_sync",
-    "tests.v3.test_v2",
+    "tests.package_with_entrypoint.*",
+    "tests.test_codecs.test_codecs",
+    "tests.test_codecs.test_transpose",
+    "tests.test_metadata.*",
+    "tests.test_store.*",
+    "tests.test_config",
+    "tests.test_group",
+    "tests.test_indexing",
+    "tests.test_properties",
+    "tests.test_sync",
+    "tests.test_v2",
 ]
 ignore_errors = true
 

src/zarr/codecs/pipeline.py

@@ -162,7 +162,6 @@ async def decode_batch(
     ) -> Iterable[NDBuffer | None]:
         chunk_bytes_batch: Iterable[Buffer | None]
         chunk_bytes_batch, chunk_specs = _unzip2(chunk_bytes_and_specs)
-
         (
             aa_codecs_with_spec,
             ab_codec_with_spec,

src/zarr/codecs/sharding.py

@@ -129,7 +129,7 @@ def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None:
         if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64):
             return None
         else:
-            return (int(chunk_start), int(chunk_start) + int(chunk_len))
+            return (int(chunk_start), int(chunk_len))
 
     def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: slice | None) -> None:
         localized_chunk = self._localize_chunk(chunk_coords)
@@ -203,7 +203,7 @@ def create_empty(
     def __getitem__(self, chunk_coords: ChunkCoords) -> Buffer:
         chunk_byte_slice = self.index.get_chunk_slice(chunk_coords)
         if chunk_byte_slice:
-            return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]]
+            return self.buf[chunk_byte_slice[0] : (chunk_byte_slice[0] + chunk_byte_slice[1])]
         raise KeyError
 
     def __len__(self) -> int:
@@ -265,7 +265,8 @@ async def finalize(
     ) -> Buffer:
         index_bytes = await index_encoder(self.index)
         if index_location == ShardingCodecIndexLocation.start:
-            self.index.offsets_and_lengths[..., 0] += len(index_bytes)
+            empty_chunks_mask = self.index.offsets_and_lengths[..., 0] == MAX_UINT_64
+            self.index.offsets_and_lengths[~empty_chunks_mask, 0] += len(index_bytes)
             index_bytes = await index_encoder(self.index)  # encode again with corrected offsets
             out_buf = index_bytes + self.buf
         else:

src/zarr/testing/utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, cast
+from collections.abc import Callable, Coroutine
+from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 import pytest
 
@@ -37,8 +38,16 @@ def has_cupy() -> bool:
         return False
 
 
+T_Callable = TypeVar("T_Callable", bound=Callable[[], Coroutine[Any, Any, None]])
+
+
 # Decorator for GPU tests
-def gpu_test(func: Any) -> Any:
-    return pytest.mark.gpu(
-        pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")(func)
+def gpu_test(func: T_Callable) -> T_Callable:
+    return cast(
+        T_Callable,
+        pytest.mark.gpu(
+            pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")(
+                func
+            )
+        ),
     )

tests/v3/test_codecs/test_sharding.py → tests/test_codecs/test_sharding.py

@@ -118,6 +118,42 @@ def test_sharding_partial(
     assert np.array_equal(data, read_data)
 
 
+@pytest.mark.parametrize("index_location", ["start", "end"])
+@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"])
+@pytest.mark.parametrize(
+    "array_fixture",
+    [
+        ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"),
+    ],
+    indirect=["array_fixture"],
+)
+def test_sharding_partial_readwrite(
+    store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation
+) -> None:
+    data = array_fixture
+    spath = StorePath(store)
+    a = Array.create(
+        spath,
+        shape=data.shape,
+        chunk_shape=data.shape,
+        dtype=data.dtype,
+        fill_value=0,
+        codecs=[
+            ShardingCodec(
+                chunk_shape=(1, data.shape[1], data.shape[2]),
+                codecs=[BytesCodec()],
+                index_location=index_location,
+            )
+        ],
+    )
+
+    a[:] = data
+
+    for x in range(data.shape[0]):
+        read_data = a[x, :, :]
+        assert np.array_equal(data[x], read_data)
+
+
 @pytest.mark.parametrize(
     "array_fixture",
     [
@@ -330,3 +366,30 @@ async def test_delete_empty_shards(store: Store) -> None:
 def test_pickle() -> None:
     codec = ShardingCodec(chunk_shape=(8, 8))
     assert pickle.loads(pickle.dumps(codec)) == codec
+
+
+@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"])
+@pytest.mark.parametrize(
+    "index_location", [ShardingCodecIndexLocation.start, ShardingCodecIndexLocation.end]
+)
+async def test_sharding_with_empty_inner_chunk(
+    store: Store, index_location: ShardingCodecIndexLocation
+) -> None:
+    data = np.arange(0, 16 * 16, dtype="uint32").reshape((16, 16))
+    fill_value = 1
+
+    path = f"sharding_with_empty_inner_chunk_{index_location}"
+    spath = StorePath(store, path)
+    a = await AsyncArray.create(
+        spath,
+        shape=(16, 16),
+        chunk_shape=(8, 8),
+        dtype="uint32",
+        fill_value=fill_value,
+        codecs=[ShardingCodec(chunk_shape=(4, 4), index_location=index_location)],
+    )
+    data[:4, :4] = fill_value
+    await a.setitem(..., data)
+    print("read data")
+    data_read = await a.getitem(...)
+    assert np.array_equal(data_read, data)

tests/v3/test_config.py → tests/test_config.py

@@ -87,7 +87,7 @@ class MockClass:
 
     assert (
         fully_qualified_name(MockClass)
-        == "tests.v3.test_config.test_fully_qualified_name.<locals>.MockClass"
+        == "tests.test_config.test_fully_qualified_name.<locals>.MockClass"
     )