Async object store support

README.md

@@ -41,20 +41,20 @@ implementation, with some slight adjustments for ease of use in python programs.
 ### `ObjectStore` api
 
 ```py
-from object_store import ObjectStore, ObjectMeta
+from object_store import ObjectStore, ObjectMeta, Path
 
 # we use an in-memory store for demonstration purposes.
 # data will not be persisted and is not shared across store instances
 store = ObjectStore("memory://")
 
-store.put("data", b"some data")
+store.put(Path("data"), b"some data")
 
 data = store.get("data")
 assert data == b"some data"
 
 blobs = store.list()
 
-meta: ObjectMeta = store.head("data")
+meta = store.head("data")
 
 range = store.get_range("data", start=0, length=4)
 assert range == b"some"
@@ -64,6 +64,33 @@ copied = store.get("copied")
 assert copied == data
 ```
 
+#### Async api
+
+```py
+from object_store import ObjectStore, ObjectMeta, Path
+
+# we use an in-memory store for demonstration purposes.
+# data will not be persisted and is not shared across store instances
+store = ObjectStore("memory://")
+
+path = Path("data")
+await store.put_async(path, b"some data")
+
+data = await store.get_async(path)
+assert data == b"some data"
+
+blobs = await store.list_async()
+
+meta = await store.head_async(path)
+
+range = await store.get_range_async(path, start=0, length=4)
+assert range == b"some"
+
+await store.copy_async(Path("data"), Path("copied"))
+copied = await store.get_async(Path("copied"))
+assert copied == data
+```
+
 ### Configuration
 
 As much as possible we aim to make access to various storage backends dependent

object-store/Cargo.toml

@@ -13,6 +13,7 @@ once_cell = "1.12.0"
 object_store = { version = "0.9", features = ["azure", "aws", "gcp"] }
 percent-encoding = "2"
 pyo3 = { version = "0.20", features = ["abi3", "abi3-py38"] }
+pyo3-asyncio = { version = "0.20", features = ["tokio-runtime"] }
 thiserror = "1.0.34"
 tokio = { version = "1.0", features = [
   "macros",

object-store/python/object_store/_internal.pyi

@@ -131,36 +131,70 @@ class ObjectStore:
     ) -> None: ...
     def get(self, location: Path) -> bytes:
         """Return the bytes that are stored at the specified location."""
+    async def get_async(self, location: Path) -> bytes:
+        """Return the bytes that are stored at the specified location."""
     def get_range(self, location: Path, start: int, length: int) -> bytes:
         """Return the bytes that are stored at the specified location in the given byte range."""
+    async def get_range_async(self, location: Path, start: int, length: int) -> bytes:
+        """Return the bytes that are stored at the specified location in the given byte range."""
     def put(self, location: Path, bytes: bytes) -> None:
         """Save the provided bytes to the specified location."""
+    async def put_async(self, location: Path, bytes: bytes) -> None:
+        """Save the provided bytes to the specified location."""
     def list(self, prefix: Path | None) -> list[ObjectMeta]:
         """List all the objects with the given prefix.
 
+        Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix
+        of `foo/bar/x` but not of `foo/bar_baz/x`.
+        """
+    async def list_async(self, prefix: Path | None) -> list[ObjectMeta]:
+        """List all the objects with the given prefix.
+
         Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix
         of `foo/bar/x` but not of `foo/bar_baz/x`.
         """
     def head(self, location: Path) -> ObjectMeta:
         """Return the metadata for the specified location"""
+    async def head_async(self, location: Path) -> ObjectMeta:
+        """Return the metadata for the specified location"""
     def list_with_delimiter(self, prefix: Path | None) -> ListResult:
         """List objects with the given prefix and an implementation specific
         delimiter. Returns common prefixes (directories) in addition to object
         metadata.
 
+        Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix
+        of `foo/bar/x` but not of `foo/bar_baz/x`.
+        """
+    async def list_with_delimiter_async(self, prefix: Path | None) -> ListResult:
+        """List objects with the given prefix and an implementation specific
+        delimiter. Returns common prefixes (directories) in addition to object
+        metadata.
+
         Prefixes are evaluated on a path segment basis, i.e. `foo/bar/` is a prefix
         of `foo/bar/x` but not of `foo/bar_baz/x`.
         """
     def delete(self, location: Path) -> None:
         """Delete the object at the specified location."""
+    async def delete_async(self, location: Path) -> None:
+        """Delete the object at the specified location."""
     def copy(self, src: Path, dst: Path) -> None:
         """Copy an object from one path to another in the same object store.
 
+        If there exists an object at the destination, it will be overwritten.
+        """
+    async def copy_async(self, src: Path, dst: Path) -> None:
+        """Copy an object from one path to another in the same object store.
+
         If there exists an object at the destination, it will be overwritten.
         """
     def copy_if_not_exists(self, src: Path, dst: Path) -> None:
         """Copy an object from one path to another, only if destination is empty.
 
+        Will return an error if the destination already has an object.
+        """
+    async def copy_if_not_exists_async(self, src: Path, dst: Path) -> None:
+        """Copy an object from one path to another, only if destination is empty.
+
         Will return an error if the destination already has an object.
         """
     def rename(self, src: Path, dst: Path) -> None:
@@ -169,11 +203,24 @@ class ObjectStore:
         By default, this is implemented as a copy and then delete source. It may not
         check when deleting source that it was the same object that was originally copied.
 
+        If there exists an object at the destination, it will be overwritten.
+        """
+    async def rename_async(self, src: Path, dst: Path) -> None:
+        """Move an object from one path to another in the same object store.
+
+        By default, this is implemented as a copy and then delete source. It may not
+        check when deleting source that it was the same object that was originally copied.
+
         If there exists an object at the destination, it will be overwritten.
         """
     def rename_if_not_exists(self, src: Path, dst: Path) -> None:
         """Move an object from one path to another in the same object store.
 
+        Will return an error if the destination already has an object.
+        """
+    async def rename_if_not_exists_async(self, src: Path, dst: Path) -> None:
+        """Move an object from one path to another in the same object store.
+
         Will return an error if the destination already has an object.
         """