Skip to content

Commit

Permalink
store -> readonly
Browse files Browse the repository at this point in the history
  • Loading branch information
@jhamman
jhamman committed Nov 9, 2024
1 parent 89e8539 commit cb470d4
Show file tree
Hide file tree
Showing 30 changed files with 244 additions and 381 deletions.
12 changes: 6 additions & 6 deletions docs/guide/storage.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Storage
=======

Zarr-Python supports multiple storage backends, including: local file systems,
Zip files, remote stores via ``fspec`` (S3, HTTP, etc.), and in-memory stores. In
Zip files, remote stores via ``fsspec`` (S3, HTTP, etc.), and in-memory stores. In
Zarr-Python 3, stores must implement the abstract store API from
:class:`zarr.abc.store.Store`.

Expand All @@ -19,9 +19,9 @@ to Zarr's top level API will result in the store being created automatically.
.. code-block:: python
>>> import zarr
>>> zarr.open("data/foo/bar", mode="r") # implicitly creates a LocalStore
>>> zarr.open("data/foo/bar", mode="r") # implicitly creates a read-only LocalStore
<Group file://data/foo/bar>
>>> zarr.open("s3://foo/bar", mode="r") # implicitly creates a RemoteStore
>>> zarr.open("s3://foo/bar", mode="r") # implicitly creates a read-only RemoteStore
<Group s3://foo/bar>
>>> data = {}
>>> zarr.open(data, mode="w") # implicitly creates a MemoryStore
Expand All @@ -43,7 +43,7 @@ filesystem.
.. code-block:: python
>>> import zarr
>>> store = zarr.storage.LocalStore("data/foo/bar", mode="r")
>>> store = zarr.storage.LocalStore("data/foo/bar", readonly=True)
>>> zarr.open(store=store)
<Group file://data/foo/bar>
Expand Down Expand Up @@ -72,7 +72,7 @@ that implements the `AbstractFileSystem` API,
.. code-block:: python
>>> import zarr
>>> store = zarr.storage.RemoteStore.from_url("gs://foo/bar", mode="r")
>>> store = zarr.storage.RemoteStore.from_url("gs://foo/bar", readonly=True)
>>> zarr.open(store=store)
<Array <RemoteStore(GCSFileSystem, foo/bar)> shape=(10, 20) dtype=float32>
Expand All @@ -86,7 +86,7 @@ Zarr data (metadata and chunks) to a dictionary.
>>> import zarr
>>> data = {}
>>> store = zarr.storage.MemoryStore(data, mode="w")
>>> store = zarr.storage.MemoryStore(data)
>>> zarr.open(store=store, shape=(2, ))
<Array memory://4943638848 shape=(2,) dtype=float64>
Expand Down
64 changes: 9 additions & 55 deletions src/zarr/abc/store.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from abc import ABC, abstractmethod
from asyncio import gather
from itertools import starmap
from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable
from typing import TYPE_CHECKING, Protocol, runtime_checkable

if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Iterable
Expand All @@ -13,24 +13,22 @@
from zarr.core.buffer import Buffer, BufferPrototype
from zarr.core.common import BytesLike

__all__ = ["StoreAccessMode", "ByteGetter", "ByteSetter", "Store", "set_or_delete"]
__all__ = ["ByteGetter", "ByteSetter", "Store", "set_or_delete"]

ByteRangeRequest: TypeAlias = tuple[int | None, int | None]
StoreAccessMode = Literal["r", "w"]


class Store(ABC):
"""
Abstract base class for Zarr stores.
"""

_mode: StoreAccessMode
_readonly: bool
_is_open: bool

def __init__(self, *args: Any, mode: StoreAccessMode = "r", **kwargs: Any) -> None:
def __init__(self, *, readonly: bool = False) -> None:
self._is_open = False
assert mode in ("r", "w")
self._mode = mode
self._readonly = readonly

@classmethod
async def open(cls, *args: Any, **kwargs: Any) -> Self:
Expand Down Expand Up @@ -99,7 +97,7 @@ async def empty(self, prefix: str = "") -> bool:
True if the store is empty, False otherwise.
"""

if not prefix.endswith("/"):
if prefix and not prefix.endswith("/"):
prefix += "/"
async for _ in self.list_prefix(prefix):
return False
Expand All @@ -114,45 +112,15 @@ async def clear(self) -> None:
async for key in self.list():
await self.delete(key)

@abstractmethod
def with_mode(self, mode: StoreAccessMode) -> Self:
"""
Return a new store of the same type pointing to the same location with a new mode.
The returned Store is not automatically opened. Call :meth:`Store.open` before
using.
Parameters
----------
mode : StoreAccessMode
The new mode to use.
Returns
-------
store
A new store of the same type with the new mode.
Examples
--------
>>> writer = zarr.store.MemoryStore(mode="w")
>>> reader = writer.with_mode("r")
"""
...

@property
def mode(self) -> StoreAccessMode:
"""Access mode of the store."""
return self._mode

@property
def readonly(self) -> bool:
"""Is the store read-only?"""
return self.mode == "r"
return self._readonly

def _check_writable(self) -> None:
"""Raise an exception if the store is not writable."""
if self.mode != "w":
raise ValueError(f"store mode ({self.mode}) does not support writing")
if self.readonly:
raise ValueError("store was opened in read-only mode and does not support writing")

@abstractmethod
def __eq__(self, value: object) -> bool:
Expand Down Expand Up @@ -347,20 +315,6 @@ async def delete_dir(self, prefix: str) -> None:
async for key in self.list_prefix(prefix):
await self.delete(key)

async def delete_dir(self, prefix: str) -> None:
"""
Remove all keys and prefixes in the store that begin with a given prefix.
"""
if not self.supports_deletes:
raise NotImplementedError
if not self.supports_listing:
raise NotImplementedError
self._check_writable()
if not prefix.endswith("/"):
prefix += "/"
async for key in self.list_prefix(prefix):
await self.delete(key)

def close(self) -> None:
"""Close the store."""
self._is_open = False
Expand Down
60 changes: 7 additions & 53 deletions src/zarr/api/asynchronous.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@
from collections.abc import Iterable

from zarr.abc.codec import Codec
from zarr.abc.store import StoreAccessMode
from zarr.core.buffer import NDArrayLike
from zarr.core.chunk_key_encodings import ChunkKeyEncoding

# TODO: this type could use some more thought
Expand Down Expand Up @@ -66,30 +64,9 @@
"zeros_like",
]

# Persistence mode: 'r' means read only (must exist); 'r+' means
# read/write (must exist); 'a' means read/write (create if doesn't
# exist); 'w' means create (overwrite if exists); 'w-' means create
# (fail if exists).
persistence_to_store_modes: dict[AccessModeLiteral, StoreAccessMode] = {
"r": "r",
"r+": "w",
"a": "w",
"w": "w",
"w-": "w",
}


def _handle_store_mode(mode: AccessModeLiteral | None) -> StoreAccessMode:
if mode is None:
return "r"
else:
return persistence_to_store_modes[mode]


def _infer_exists_ok(mode: AccessModeLiteral) -> bool:
if mode in ("a", "r+", "w"):
return True
return False
return mode in ("a", "r+", "w")


def _get_shape_chunks(a: ArrayLike | Any) -> tuple[ChunkCoords | None, ChunkCoords | None]:
Expand Down Expand Up @@ -313,11 +290,7 @@ async def open(
"""
zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)

store_mode = _handle_store_mode(mode)
store_path = await make_store_path(
store, mode=store_mode, path=path, storage_options=storage_options
)
await store_path._init(mode=mode)
store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options)

# TODO: the mode check below seems wrong!
if "shape" not in kwargs and mode in {"a", "r", "r+"}:
Expand Down Expand Up @@ -427,11 +400,7 @@ async def save_array(
raise TypeError("arr argument must be numpy or other NDArrayLike array")

mode = kwargs.pop("mode", "a")
store_mode = _handle_store_mode(mode)
store_path = await make_store_path(
store, path=path, mode=store_mode, storage_options=storage_options
)
await store_path._init(mode=mode)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
if np.isscalar(arr):
arr = np.array(arr)
shape = arr.shape
Expand Down Expand Up @@ -626,11 +595,7 @@ async def group(
mode = "w"
else:
mode = "r+"
store_mode = _handle_store_mode(mode)
store_path = await make_store_path(
store, path=path, mode=store_mode, storage_options=storage_options
)
await store_path._init(mode=mode)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)

if chunk_store is not None:
warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)
Expand Down Expand Up @@ -743,12 +708,8 @@ async def open_group(
if chunk_store is not None:
warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)

store_mode = _handle_store_mode(mode)
exists_ok = _infer_exists_ok(mode)
store_path = await make_store_path(
store, mode=store_mode, storage_options=storage_options, path=path
)
await store_path._init(mode=mode)
store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)

if attributes is None:
attributes = {}
Expand Down Expand Up @@ -931,11 +892,7 @@ async def create(
mode = kwargs.pop("mode", None)
if mode is None:
mode = "a"
store_mode = _handle_store_mode(mode)
store_path = await make_store_path(
store, path=path, mode=store_mode, storage_options=storage_options
)
await store_path._init(mode)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)
return await AsyncArray.create(
store_path,
shape=shape,
Expand Down Expand Up @@ -1122,10 +1079,7 @@ async def open_array(
"""

mode = kwargs.pop("mode", None)
store_mode = _handle_store_mode(mode)
store_path = await make_store_path(
store, path=path, mode=store_mode, storage_options=storage_options
)
store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options)

zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)

Expand Down
Loading

0 comments on commit cb470d4

Please sign in to comment.