Merge branch 'v3_group_tests' of github.com:d-v-b/zarr-python into fe…

…ature/store-list-methods-async-generators

.git-blame-ignore-revs

@@ -1,2 +1,4 @@
 # lint codebase with black and ruff
 4e348d6b80c96da461fd866576c971b8a659ba15
+# migrate from black to ruff format
+22cea005629913208a85799372e045f353744add

.github/workflows/test-v3.yml

@@ -10,15 +10,22 @@ on:
     branches: [ v3 ]
 
 jobs:
-  run-tests:
+  test:
+    name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
 
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11']
+        numpy-version: ['1.24', '1.26']
+        dependency-set: ["minimal", "optional"]
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: ${{ matrix.python-version }}
         cache: 'pip'
     - name: Install Hatch
       run: |
@@ -29,8 +36,8 @@ jobs:
         hatch env create
     - name: Run Tests
       run: |
-        hatch run test:run
+        hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run
     - name: Run mypy
       continue-on-error: true
       run: |
-        hatch run test:run-mypy
+        hatch run test:run-mypy

.pre-commit-config.yaml

@@ -7,31 +7,25 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # Ruff version.
-    rev: 'v0.0.224'
+    rev: 'v0.2.1'
     hooks:
     - id: ruff
-      # Respect `exclude` and `extend-exclude` settings.
-      args: ["--force-exclude"]
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-    - id: black
+      args: ["--fix", "--show-fixes"]
+    - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
       - id: codespell
         args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
+    rev: v1.8.0
     hooks:
       - id: mypy
         files: src
-        exclude: ^src/zarr/v3
         args: []
         additional_dependencies:
           - types-redis

bench/compress_normal.py

@@ -8,7 +8,6 @@
 from zarr import blosc
 
 if __name__ == "__main__":
-
     sys.path.insert(0, "..")
 
     # setup

docs/installation.rst

@@ -19,13 +19,4 @@ latest GitHub main::
 
     $ pip install git+https://github.com/zarr-developers/zarr-python.git
 
-To work with Zarr source code in development, install from GitHub::
-
-    $ git clone --recursive https://github.com/zarr-developers/zarr-python.git
-    $ cd zarr-python
-    $ python -m pip install -e .
-
-To verify that Zarr has been fully installed, run the test suite::
-
-    $ pip install pytest
-    $ python -m pytest -v --pyargs zarr
+To work with Zarr source code in development, see `Contributing <contributing.html>`_.

pyproject.toml

@@ -127,7 +127,8 @@ serve = "sphinx-autobuild docs docs/_build --ignore 'docs/_autoapi/**/*' --host
 
 [tool.ruff]
 line-length = 100
-exclude = [
+force-exclude = true
+extend-exclude = [
     ".bzr",
     ".direnv",
     ".eggs",
@@ -146,25 +147,33 @@ exclude = [
     "docs"
 ]
 
-[tool.black]
-line-length = 100
-exclude = '''
-/(
-    \.git
-  | \.mypy_cache
-  | \.venv
-  | _build
-  | buck-out
-  | build
-  | dist
-  | docs
-)/
-'''
-
 [tool.mypy]
-python_version = "3.8"
+python_version = "3.10"
 ignore_missing_imports = true
-follow_imports = "silent"
+namespace_packages = false
+
+warn_unused_configs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+
+
+check_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = [
+    "zarr._storage.store",
+    "zarr._storage.v3_storage_transformers",
+    "zarr.v3.group",
+    "zarr.core",
+    "zarr.hierarchy",
+    "zarr.indexing",
+    "zarr.storage",
+    "zarr.sync",
+    "zarr.util",
+    "tests.*",
+]
+check_untyped_defs = false
+
 
 [tool.pytest.ini_options]
 doctest_optionflags = [

src/zarr/_storage/absstore.py

@@ -87,7 +87,7 @@ def __init__(
                 "https://{}.blob.core.windows.net/".format(account_name),
                 container,
                 credential=account_key,
-                **blob_service_kwargs
+                **blob_service_kwargs,
             )
 
         self.client = client
@@ -240,7 +240,6 @@ def __setitem__(self, key, value):
         super().__setitem__(key, value)
 
     def rmdir(self, path=None):
-
         if not path:
             # Currently allowing clear to delete everything as in v2
 

src/zarr/_storage/store.py

@@ -629,7 +629,6 @@ def _rmdir_from_keys(store: StoreLike, path: Optional[str] = None) -> None:
 
 
 def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None:
-
     meta_dir = meta_root + path
     meta_dir = meta_dir.rstrip("/")
     _rmdir_from_keys(store, meta_dir)
@@ -643,10 +642,10 @@ def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None:
     sfx = _get_metadata_suffix(store)
     array_meta_file = meta_dir + ".array" + sfx
     if array_meta_file in store:
-        store.erase(array_meta_file)  # type: ignore
+        store.erase(array_meta_file)
     group_meta_file = meta_dir + ".group" + sfx
     if group_meta_file in store:
-        store.erase(group_meta_file)  # type: ignore
+        store.erase(group_meta_file)
 
 
 def _listdir_from_keys(store: BaseStore, path: Optional[str] = None) -> List[str]:

src/zarr/_storage/v3.py

@@ -118,7 +118,6 @@ def _get_files_and_dirs_from_path(store, path):
 
 
 class FSStoreV3(FSStore, StoreV3):
-
     # FSStoreV3 doesn't use this (FSStore uses it within _normalize_key)
     _META_KEYS = ()
 

src/zarr/_storage/v3_storage_transformers.py

@@ -367,7 +367,7 @@ def erase_prefix(self, prefix):
 
     def rmdir(self, path=None):
         path = normalize_storage_path(path)
-        _rmdir_from_keys_v3(self, path)  # type: ignore
+        _rmdir_from_keys_v3(self, path)
 
     def __contains__(self, key):
         if self._is_data_key(key):

src/zarr/attrs.py

@@ -1,3 +1,4 @@
+from typing import Any
 import warnings
 from collections.abc import MutableMapping
 
@@ -26,7 +27,6 @@ class Attributes(MutableMapping):
     """
 
     def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None):
-
         self._version = getattr(store, "_store_version", 2)
         _Store = Store if self._version == 2 else StoreV3
         self.store = _Store._ensure_store(store)
@@ -40,7 +40,7 @@ def _get_nosync(self):
         try:
             data = self.store[self.key]
         except KeyError:
-            d = dict()
+            d: dict[str, Any] = dict()
             if self._version > 2:
                 d["attributes"] = {}
         else:
@@ -73,7 +73,6 @@ def __getitem__(self, item):
         return self.asdict()[item]
 
     def _write_op(self, f, *args, **kwargs):
-
         # guard condition
         if self.read_only:
             raise PermissionError("attributes are read-only")
@@ -89,7 +88,6 @@ def __setitem__(self, item, value):
         self._write_op(self._setitem_nosync, item, value)
 
     def _setitem_nosync(self, item, value):
-
         # load existing data
         d = self._get_nosync()
 
@@ -106,7 +104,6 @@ def __delitem__(self, item):
         self._write_op(self._delitem_nosync, item)
 
     def _delitem_nosync(self, key):
-
         # load existing data
         d = self._get_nosync()
 
@@ -128,7 +125,6 @@ def put(self, d):
             self._write_op(self._put_nosync, dict(attributes=d))
 
     def _put_nosync(self, d):
-
         d_to_check = d if self._version == 2 else d["attributes"]
         if not all(isinstance(item, str) for item in d_to_check):
             # TODO: Raise an error for non-string keys
@@ -178,7 +174,6 @@ def update(self, *args, **kwargs):
         self._write_op(self._update_nosync, *args, **kwargs)
 
     def _update_nosync(self, *args, **kwargs):
-
         # load existing data
         d = self._get_nosync()
 

src/zarr/convenience.py

@@ -675,10 +675,8 @@ def copy_store(
 
     # setup logging
     with _LogWriter(log) as log:
-
         # iterate over source keys
         for source_key in sorted(source.keys()):
-
             # filter to keys under source path
             if source_store_version == 2:
                 if not source_key.startswith(source_path):
@@ -757,7 +755,7 @@ def copy(
     log=None,
     if_exists="raise",
     dry_run=False,
-    **create_kws
+    **create_kws,
 ):
     """Copy the `source` array or group into the `dest` group.
 
@@ -878,7 +876,6 @@ def copy(
 
     # setup logging
     with _LogWriter(log) as log:
-
         # do the copying
         n_copied, n_skipped, n_bytes_copied = _copy(
             log,
@@ -890,7 +887,7 @@ def copy(
             without_attrs=without_attrs,
             if_exists=if_exists,
             dry_run=dry_run,
-            **create_kws
+            **create_kws,
         )
 
         # log a final message with a summary of what happened
@@ -948,12 +945,10 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_
 
         # take action
         if do_copy:
-
             # log a message about what we're going to do
             log("copy {} {} {}".format(source.name, source.shape, source.dtype))
 
             if not dry_run:
-
                 # clear the way
                 if exists:
                     del dest[name]
@@ -1038,12 +1033,10 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_
 
         # take action
         if do_copy:
-
             # log action
             log("copy {}".format(source.name))
 
             if not dry_run:
-
                 # clear the way
                 if exists_array:
                     del dest[name]
@@ -1056,7 +1049,6 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_
                     grp.attrs.update(source.attrs)
 
             else:
-
                 # setup for dry run without creating any groups in the
                 # destination
                 if dest is not None:
@@ -1076,7 +1068,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_
                     without_attrs=without_attrs,
                     if_exists=if_exists,
                     dry_run=dry_run,
-                    **create_kws
+                    **create_kws,
                 )
                 n_copied += c
                 n_skipped += s
@@ -1099,7 +1091,7 @@ def copy_all(
     log=None,
     if_exists="raise",
     dry_run=False,
-    **create_kws
+    **create_kws,
 ):
     """Copy all children of the `source` group into the `dest` group.
 
@@ -1189,7 +1181,6 @@ def copy_all(
 
     # setup logging
     with _LogWriter(log) as log:
-
         for k in source.keys():
             c, s, b = _copy(
                 log,
@@ -1201,7 +1192,7 @@ def copy_all(
                 without_attrs=without_attrs,
                 if_exists=if_exists,
                 dry_run=dry_run,
-                **create_kws
+                **create_kws,
             )
             n_copied += c
             n_skipped += s
@@ -1262,7 +1253,6 @@ def is_zarr_key(key):
             return key.endswith(".zarray") or key.endswith(".zgroup") or key.endswith(".zattrs")
 
     else:
-
         assert_zarr_v3_api_available()
 
         sfx = _get_metadata_suffix(store)  # type: ignore