diff --git a/Cargo.lock b/Cargo.lock index 5ae65d2c..4464f4bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,6 +124,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3419eecc9f5967e6f0f29a0c3fefe22bda6ea34b15798f3c452cb81f2c3fa7" +[[package]] +name = "assert-json-diff" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4259cbe96513d2f1073027a259fc2ca917feb3026a5a8d984e3628e490255cc0" +dependencies = [ + "extend", + "serde", + "serde_json", +] + [[package]] name = "async-channel" version = "1.8.0" @@ -371,6 +382,41 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-s3" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1533be023eeac69668eb718b1c48af7bd5e26305ed770553d2877ab1f7507b68" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes 1.4.0", + "bytes-utils", + "fastrand", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tokio-stream", + "tower", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sso" version = "0.24.0" @@ -429,6 +475,7 @@ checksum = "660a02a98ab1af83bd8d714afbab2d502ba9b18c49e7e4cddd6bf8837ff778cb" dependencies = [ "aws-credential-types", "aws-sigv4", + "aws-smithy-eventstream", "aws-smithy-http", "aws-types", "http", @@ -441,7 +488,9 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdaf11005b7444e6cd66f600d09861a3aeb6eb89a0f003c7c9820dbab2d15297" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-http", + "bytes 1.4.0", "form_urlencoded", "hex", "hmac 0.12.1", @@ -466,6 +515,27 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3875fb4b28606a5368a048016a28c15707f2b21238d5b2e4a23198f590e92c4" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes 1.4.0", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1 0.10.5", + "sha2 0.10.6", + "tracing", +] + [[package]] name = "aws-smithy-client" version = "0.54.4" @@ -475,6 +545,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-tower", + "aws-smithy-protocol-test", "aws-smithy-types", "bytes 1.4.0", "fastrand", @@ -484,17 +555,30 @@ dependencies = [ "hyper-rustls", "lazy_static", "pin-project-lite", + "serde", "tokio", "tower", "tracing", ] +[[package]] +name = "aws-smithy-eventstream" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac250d8c0e42af0097a6837ffc5a6fb9f8ba4107bb53124c047c91bc2a58878f" +dependencies = [ + "aws-smithy-types", + "bytes 1.4.0", + "crc32fast", +] + [[package]] name = "aws-smithy-http" version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "873f316f1833add0d3aa54ed1b0cd252ddd88c792a0cf839886400099971e844" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-types", "bytes 1.4.0", "bytes-utils", @@ -536,6 +620,21 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-protocol-test" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d1c9bcb35ce11055ec128dab2c66a7ed47e2dfff99883e32c21a1ab6d6bee6" +dependencies = [ + "assert-json-diff", + "http", + "pretty_assertions", + "regex", + "roxmltree", + "serde_json", + "thiserror", +] + [[package]] name = "aws-smithy-query" version = "0.54.4" @@ -675,13 +774,40 @@ dependencies = [ "uuid", ] +[[package]] +name = "azure_core" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32568c56fda7f2f1173430298bddeb507ed44e99bd989ba1156a25534bff5d98" +dependencies = [ + "async-trait", + "base64 0.21.0", + "bytes 1.4.0", + "dyn-clone", + "futures", + "getrandom 0.2.8", + "http-types", + "log", + "paste", + "pin-project", + "quick-xml", + "rand 0.8.5", + "reqwest", + "rustc_version 0.4.0", + "serde", + "serde_json", + "time 0.3.20", + "url", + "uuid", +] + [[package]] name = "azure_messaging_servicebus" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57a56b96f2d5dcd60299a61e285aba88329d9bdd259b1ced98e31ae33523052d" dependencies = [ - "azure_core", + "azure_core 0.10.0", "base64 0.13.1", "bytes 1.4.0", "hmac 0.12.1", @@ -700,7 +826,7 @@ checksum = "4b5940f60d0cdfe6312e9feb6528c975ba9de061026843b0ded723585b540338" dependencies = [ "RustyXML", "async-trait", - "azure_core", + "azure_core 0.10.0", "base64 0.13.1", "bytes 1.4.0", "futures", @@ -717,6 +843,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "azure_storage" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29747ca7da0f81ea199d1c957bae737bc5bc54502e0f7c00ca7be1894306944a" +dependencies = [ + "RustyXML", + "async-trait", + "azure_core 0.11.0", + "bytes 1.4.0", + "futures", + "hmac 0.12.1", + "log", + "once_cell", + "serde", + "serde_derive", + "serde_json", + "sha2 0.10.6", + "time 0.3.20", + "url", + "uuid", +] + [[package]] name = "azure_storage_blobs" version = "0.10.0" @@ -724,8 +873,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf7e8d719a89c6079ba6298fc3969acf53a24d590e1f5c1faab6f732e3ddf56b" dependencies = [ "RustyXML", - "azure_core", - "azure_storage", + "azure_core 0.10.0", + "azure_storage 0.10.0", "base64 0.13.1", "bytes 1.4.0", "futures", @@ -740,6 +889,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "azure_storage_blobs" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b56de7a67648a7a434db81f11264df2a07e33468e751da379bddbfd7c25d5117" +dependencies = [ + "RustyXML", + "azure_core 0.11.0", + "azure_storage 0.11.0", + "bytes 1.4.0", + "futures", + "log", + "md5", + "serde", + "serde_derive", + "serde_json", + "time 0.3.20", + "url", + "uuid", +] + [[package]] name = "base-x" version = "0.2.11" @@ -1224,6 +1394,15 @@ dependencies = [ "debug-helper", ] +[[package]] +name = "crc32c" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e" +dependencies = [ + "rustc_version 0.4.0", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -1424,6 +1603,12 @@ dependencies = [ "const-oid", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.9.0" @@ -1589,6 +1774,18 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "extend" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47da3a72ec598d9c8937a7ebca8962a5c7a1f28444e38c2b33c771ba3f55f05" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -2780,6 +2977,15 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +[[package]] +name = "output_vt100" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" +dependencies = [ + "winapi", +] + [[package]] name = "outref" version = "0.5.1" @@ -2996,6 +3202,18 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_assertions" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" +dependencies = [ + "ctor", + "diff", + "output_vt100", + "yansi", +] + [[package]] name = "prettyplease" version = "0.1.25" @@ -3129,6 +3347,16 @@ dependencies = [ "unicase", ] +[[package]] +name = "quick-xml" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.26" @@ -3458,6 +3686,15 @@ dependencies = [ "routerify", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -3766,6 +4003,17 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.6", +] + [[package]] name = "sha1_smol" version = "1.0.0" @@ -3891,6 +4139,7 @@ dependencies = [ "log", "rand 0.8.5", "reqwest", + "slight-blob-store", "slight-common", "slight-core", "slight-distributed-locking", @@ -3911,6 +4160,26 @@ dependencies = [ "wit-bindgen-wasmtime 0.2.0 (git+https://github.com/fermyon/wit-bindgen-backport)", ] +[[package]] +name = "slight-blob-store" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-sdk-s3", + "azure_storage 0.11.0", + "azure_storage_blobs 0.11.0", + "bytes 1.4.0", + "futures", + "slight-common", + "slight-runtime-configs", + "tokio", + "tracing", + "wit-bindgen-wasmtime 0.2.0 (git+https://github.com/fermyon/wit-bindgen-backport)", + "wit-error-rs", +] + [[package]] name = "slight-common" version = "0.1.0" @@ -4042,8 +4311,8 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-dynamodb", - "azure_storage", - "azure_storage_blobs", + "azure_storage 0.10.0", + "azure_storage_blobs 0.10.0", "bytes 1.4.0", "futures", "redis", @@ -4063,7 +4332,7 @@ dependencies = [ "anyhow", "async-channel", "async-trait", - "azure_core", + "azure_core 0.10.0", "azure_messaging_servicebus", "crossbeam-channel", "filesystem-pubsub", @@ -4242,7 +4511,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1", + "sha1 0.6.1", "syn", ] @@ -5824,6 +6093,12 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zeroize" version = "1.5.7" diff --git a/Cargo.toml b/Cargo.toml index d2058350..d6324d81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,13 +8,13 @@ repository = { workspace = true } [[bin]] name = "slight" -test = false [lib] name = "slight_lib" path = "src/lib.rs" [dependencies] +slight-blob-store = { workspace = true, features = ["aws_s3"], optional = true } slight-core = { workspace = true } slight-runtime = { workspace = true } slight-keyvalue = { workspace = true, features = ["filesystem", "awsdynamodb", "redis", "azblob"], optional = true} @@ -44,7 +44,8 @@ tempfile = { workspace = true } rand = { worspace = true } [features] -default = ["keyvalue", "distributed-locking", "messaging", "runtime-configs", "sql", "http-server", "http-client"] +default = ["blob-store", "keyvalue", "distributed-locking", "messaging", "runtime-configs", "sql", "http-server", "http-client"] +blob-store = ["dep:slight-blob-store"] keyvalue = ["dep:slight-keyvalue"] distributed-locking = ["dep:slight-distributed-locking"] messaging = ["dep:slight-messaging"] @@ -61,6 +62,7 @@ license = "MIT" repository = "https://github.com/deislabs/spiderlightning" [workspace.dependencies] +slight-blob-store = { path = "./crates/blob-store" } slight-core = { path = "./crates/core" } slight-runtime = { path = "./crates/runtime" } slight-keyvalue = { path = "./crates/keyvalue" } diff --git a/crates/blob-store/Cargo.toml b/crates/blob-store/Cargo.toml new file mode 100644 index 00000000..1ce6f2ae --- /dev/null +++ b/crates/blob-store/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "slight-blob-store" +version = "0.1.0" +edition = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +repository = { workspace = true } + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +wit-bindgen-wasmtime = { workspace = true } +wit-error-rs = { workspace = true } +slight-common = { workspace = true } +slight-runtime-configs = { workspace = true } +anyhow = { workspace = true } +tracing = { workspace = true } +tokio = { workspace = true } +async-trait = { workspace = true } +# blobstore.s3 deps +aws-config = { version = "0.54", optional = true } +aws-sdk-s3 = { version = "0.24" , optional = true } +futures = { version = "0.3", optional = true } +# kv.azblob deps +azure_storage_blobs = { version = "0.11", optional = true } +azure_storage = { version = "0.11", optional = true } +bytes = { version = "1", optional = true } + +[features] +default = ["aws_s3", "azblob"] +aws_s3 = ["aws-config", "aws-sdk-s3", "futures"] +azblob = ["azure_storage_blobs", "azure_storage", "bytes", "futures"] diff --git a/crates/blob-store/src/container.rs b/crates/blob-store/src/container.rs new file mode 100644 index 00000000..65de3bd5 --- /dev/null +++ b/crates/blob-store/src/container.rs @@ -0,0 +1,66 @@ +use anyhow::Result; + +use std::sync::Arc; + +use async_trait::async_trait; +use slight_common::BasicState; + +use crate::{ + blob_store::{ContainerMetadata, ObjectMetadata, ObjectNameParam, ObjectNameResult}, + implementors::{aws_s3::S3Container, azblob::AzBlobContainer}, + read_stream::{ReadStreamImplementor, ReadStreamInner}, + write_stream::{WriteStreamImplementor, WriteStreamInner}, + BlobStoreImplementors, +}; + +pub(crate) type DynW = dyn WriteStreamImplementor + Send + Sync; +pub(crate) type DynR = dyn ReadStreamImplementor + Send + Sync; +pub(crate) type DynContainer = dyn ContainerImplementor + Send + Sync; + +#[async_trait] +pub trait ContainerImplementor { + async fn name(&self) -> Result; + async fn info(&self) -> Result; + async fn list_objects(&self) -> Result>; + async fn delete_object(&self, name: ObjectNameParam<'_>) -> Result<()>; + async fn delete_objects(&self, names: Vec>) -> Result<()>; + async fn has_object(&self, name: ObjectNameParam<'_>) -> Result; + async fn object_info(&self, name: ObjectNameParam<'_>) -> Result; + async fn read_object(&self, name: ObjectNameParam<'_>) -> Result; + async fn write_object(&self, name: ObjectNameParam<'_>) -> Result; +} + +impl std::fmt::Debug for DynContainer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ContainerImplementor") + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub struct ContainerInner { + pub implementor: Arc, +} + +impl ContainerInner { + pub(crate) async fn new( + blobstore_implementor: BlobStoreImplementors, + slight_state: &BasicState, + name: &str, + ) -> Result { + let container = Self { + implementor: match blobstore_implementor { + #[cfg(feature = "aws_s3")] + BlobStoreImplementors::S3 => Arc::new(S3Container::new(slight_state, name).await?), + #[cfg(feature = "azblob")] + BlobStoreImplementors::AzBlob => { + Arc::new(AzBlobContainer::new(slight_state, name).await?) + } + BlobStoreImplementors::None => { + panic!("No implementor specified") + } + }, + }; + Ok(container) + } +} diff --git a/crates/blob-store/src/implementors/aws_s3.rs b/crates/blob-store/src/implementors/aws_s3.rs new file mode 100644 index 00000000..768490b4 --- /dev/null +++ b/crates/blob-store/src/implementors/aws_s3.rs @@ -0,0 +1,275 @@ +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use aws_config::{from_env, meta::region::RegionProviderChain}; +use aws_sdk_s3::{ + client::fluent_builders::GetObject, + error::{GetObjectError, GetObjectErrorKind}, + model::{Bucket, Delete, ObjectAttributes::ObjectSize, ObjectIdentifier}, + types::ByteStream, + Client, +}; +use slight_common::BasicState; +use slight_runtime_configs::get_from_state; + +use tracing::info; + +use crate::{ + blob_store::{ContainerMetadata, ObjectMetadata, ObjectNameParam, ObjectNameResult}, + container::ContainerImplementor, + read_stream::{ReadStreamImplementor, ReadStreamInner}, + write_stream::{WriteStreamImplementor, WriteStreamInner}, +}; + +pub const S3_CAPABILITY_NAME: &str = "blobstore.aws_s3"; + +/// A container maps to a bucket in aws S3 +#[derive(Debug, Clone)] +pub struct S3Container { + client: Arc, + bucket: Bucket, +} + +/// A read stream maps to a GetObject request +/// +/// To use this stream, you must call `send` on it. +#[derive(Debug)] +pub struct S3ReadStream { + req: GetObject, +} + +/// A write stream contains a S3 client, the bucket name and a key +#[derive(Debug, Clone)] +pub struct S3WriteStream { + client: Arc, + bucket: String, + key: String, +} + +impl S3Container { + pub async fn new(slight_state: &BasicState, name: &str) -> Result { + let access_id = get_from_state("AWS_ACCESS_KEY_ID", slight_state) + .await + .unwrap(); + std::env::set_var("AWS_ACCESS_KEY_ID", access_id); + + let access_key = get_from_state("AWS_SECRET_ACCESS_KEY", slight_state) + .await + .unwrap(); + std::env::set_var("AWS_SECRET_ACCESS_KEY", access_key); + + let region = get_from_state("AWS_REGION", slight_state).await; + let default_region = get_from_state("AWS_DEFAULT_REGION", slight_state).await; + if region.is_err() && default_region.is_err() { + panic!("AWS_REGION or AWS_DEFAULT_REGION must be set"); + } else if region.is_err() { + std::env::set_var("AWS_DEFAULT_REGION", default_region.unwrap()); + } else { + std::env::set_var("AWS_REGION", region.unwrap()); + } + + let region = RegionProviderChain::default_provider(); + let config = from_env().region(region).load().await; + let client = Arc::new(Client::new(&config)); + + // perform list buckets, too costly? + let resp = client.list_buckets().send().await?; + let buckets = resp.buckets().unwrap_or_default(); + let bucket = buckets + .iter() + .find(|b| b.name().unwrap_or_default() == name) + .ok_or(anyhow::anyhow!(format!("container {name} not found")))? + .clone(); + + Ok(Self { client, bucket }) + } +} + +#[async_trait] +impl ContainerImplementor for S3Container { + async fn name(&self) -> Result { + Ok(self.bucket.name().unwrap_or_default().to_string()) + } + async fn info(&self) -> Result { + Ok(ContainerMetadata::from(&self.bucket)) + } + async fn list_objects(&self) -> Result> { + let resp = self + .client + .list_objects_v2() + .bucket(self.name().await?) + .send() + .await?; + info!("{}", "received list objects response"); + let res = resp + .contents() + .unwrap_or_default() + .iter() + .map(|object| object.key().unwrap_or_default().to_string()) + .collect(); + Ok(res) + } + async fn delete_object(&self, name: ObjectNameParam<'_>) -> Result<()> { + let _ = self + .client + .delete_object() + .bucket(self.name().await?) + .key(name) + .send() + .await?; + info!("{}", format!("object {name} deleted")); + Ok(()) + } + async fn delete_objects(&self, names: Vec>) -> Result<()> { + let mut delete_objects: Vec = vec![]; + for name in names { + let obj_id = ObjectIdentifier::builder() + .set_key(Some(name.to_owned())) + .build(); + delete_objects.push(obj_id); + } + let delete = Delete::builder().set_objects(Some(delete_objects)).build(); + let _ = self + .client + .delete_objects() + .bucket(self.name().await?) + .delete(delete) + .send() + .await?; + info!("{}", "objects deleted"); + Ok(()) + } + async fn has_object(&self, name: ObjectNameParam<'_>) -> Result { + let res = self + .client + .get_object() + .bucket(self.name().await?) + .key(name) + .send() + .await; + let mut key_exists: bool = true; + if let Err(err) = res { + match err.into_service_error() { + GetObjectError { + kind: GetObjectErrorKind::NoSuchKey(_), + .. + } => key_exists = false, + err => return Err(err.into()), + } + } + Ok(key_exists) + } + async fn object_info(&self, name: ObjectNameParam<'_>) -> Result { + let container = self.name().await?; + let metadata = self + .client + .get_object_attributes() + .bucket(container.clone()) + .key(name) + .object_attributes(ObjectSize) + .send() + .await?; + let res = ObjectMetadata { + name: name.to_owned(), + container, + created_at: metadata.last_modified().unwrap().as_secs_f64() as u64, + size: metadata.object_size() as u64, + }; + Ok(res) + } + async fn read_object(&self, name: ObjectNameParam<'_>) -> Result { + let resp = self + .client + .get_object() + .bucket(self.name().await?) + .key(name); + let read_stream_inner = ReadStreamInner::new(Box::new(S3ReadStream::new(resp).await)).await; + Ok(read_stream_inner) + } + async fn write_object(&self, name: ObjectNameParam<'_>) -> Result { + let write_stream_inner = WriteStreamInner::new(Box::new( + S3WriteStream::new(self.client.clone(), self.bucket.name().unwrap(), name).await, + )) + .await; + Ok(write_stream_inner) + } +} + +impl S3ReadStream { + pub async fn new(req: GetObject) -> Self { + Self { req } + } +} + +impl S3WriteStream { + pub async fn new(client: Arc, bucket: &str, key: &str) -> Self { + Self { + client, + bucket: bucket.into(), + key: key.into(), + } + } +} + +#[async_trait] +impl ReadStreamImplementor for S3ReadStream { + async fn read(&self, size: u64) -> Result>> { + // In wasi-blob-store, `read` takes a mutable buffer as an argument. + // I changed it to return a vector of bytes instead because as of right now, + // wit-bindgen does not support generating mutable buffers. + // + // This is something we might want to go back and change in the future + // when we transform wit-bindgen v0.2.0 to the newest component model syntax. + // + // TODO: change `read` to take a mutable buffer as an argument + let resp = self.req.clone().send().await?; + let content_length = resp.content_length() as u64; + let stream: ByteStream = resp.body; + if size == 0 { + Ok(Some(vec![])) + } else if size > content_length { + Ok(Some(stream.collect().await?.to_vec())) + } else { + let mut res = stream.collect().await?.to_vec(); + res.truncate(size as usize); + Ok(Some(res)) + } + } + async fn available(&self) -> Result { + todo!() + } +} + +#[async_trait] +impl WriteStreamImplementor for S3WriteStream { + async fn write(&self, data: &[u8]) -> Result<()> { + // TODO: same comment from `read` applies here + let _ = self + .client + .put_object() + .bucket(self.bucket.clone()) + .key(self.key.clone()) + .body(ByteStream::from(data.to_vec())) + .send() + .await?; + Ok(()) + } + async fn close(&self) -> Result<()> { + todo!() + } +} + +impl From<&Bucket> for ContainerMetadata { + fn from(bucket: &Bucket) -> Self { + let created_at = if let Some(creation_date) = bucket.creation_date() { + creation_date.secs() as u64 + } else { + 0_u64 + }; + Self { + name: bucket.name().unwrap_or_default().into(), + created_at, + } + } +} diff --git a/crates/blob-store/src/implementors/azblob.rs b/crates/blob-store/src/implementors/azblob.rs new file mode 100644 index 00000000..38b4228e --- /dev/null +++ b/crates/blob-store/src/implementors/azblob.rs @@ -0,0 +1,216 @@ +use anyhow::{bail, Result}; +use async_trait::async_trait; +use azure_storage::prelude::*; +use azure_storage_blobs::{ + container::{operations::BlobItem, Container}, + prelude::*, +}; +use futures::StreamExt; +use slight_common::BasicState; + +use slight_runtime_configs::get_from_state; +use tracing::info; + +use crate::{ + blob_store::{ContainerMetadata, ObjectMetadata, ObjectNameParam, ObjectNameResult}, + container::ContainerImplementor, + read_stream::{ReadStreamImplementor, ReadStreamInner}, + write_stream::{WriteStreamImplementor, WriteStreamInner}, +}; + +pub const AZBLOB_CAPABILITY_NAME: &str = "blobstore.azblob"; + +/// A container maps to a bucket in azure blob storage +#[derive(Debug, Clone)] +pub struct AzBlobContainer { + client: ContainerClient, +} + +#[derive(Debug)] +pub struct AzBlobReadStream { + blob_client: BlobClient, +} + +#[derive(Debug, Clone)] +pub struct AzBlobWriteStream { + client: BlobClient, +} + +impl AzBlobContainer { + pub async fn new(slight_state: &BasicState, name: &str) -> Result { + let storage_account_name = get_from_state("AZURE_STORAGE_ACCOUNT", slight_state) + .await + .unwrap(); + let storage_account_key = get_from_state("AZURE_STORAGE_KEY", slight_state) + .await + .unwrap(); + + let storage_credentials = + StorageCredentials::Key(storage_account_name.clone(), storage_account_key); + let service_client = BlobServiceClient::new(storage_account_name, storage_credentials); + + let container_client = service_client.container_client(name); + if container_client.exists().await? { + Ok(Self { + client: container_client, + }) + } else { + bail!(format!("container {name} not found")) + } + } +} + +#[async_trait] +impl ContainerImplementor for AzBlobContainer { + async fn name(&self) -> Result { + Ok(self.client.container_name().to_owned()) + } + async fn info(&self) -> Result { + let properties = self.client.get_properties().await?; + Ok(properties.container.into()) + } + async fn list_objects(&self) -> Result> { + let mut stream = self.client.list_blobs().into_stream(); + let mut results = vec![]; + while let Some(value) = stream.next().await { + let value = value?; + results.push(value); + } + + let mut result = vec![]; + for list_blob in results { + for blob in list_blob.blobs.items { + let blob_name = match blob { + BlobItem::Blob(b) => b.name.clone(), + BlobItem::BlobPrefix(b) => b.name.clone(), + }; + result.push(blob_name) + } + } + Ok(result) + } + async fn delete_object(&self, name: ObjectNameParam<'_>) -> Result<()> { + self.client + .blob_client(name) + .delete() + .delete_snapshots_method(DeleteSnapshotsMethod::Include) + .into_future() + .await?; + Ok(()) + } + async fn delete_objects(&self, _names: Vec>) -> Result<()> { + // TODO: there isn't an API in azure blob storage to do this directly + // if we are going to delete a lot of objects, we should use a batch delete + // otherwise, we run into issues with deleting half the objects and then + // failing + // + // + // followed up on https://github.com/Azure/azure-sdk-for-rust/issues/1249 + todo!() + } + async fn has_object(&self, name: ObjectNameParam<'_>) -> Result { + let res = self.client.blob_client(name).exists().await?; + Ok(res) + } + async fn object_info(&self, name: ObjectNameParam<'_>) -> Result { + let blob = self.client.blob_client(name).get_properties().await?.blob; + Ok(ObjectMetadata { + name: blob.name, + container: self.name().await?, + created_at: blob.properties.creation_time.unix_timestamp() as u64, + size: blob.properties.content_length, + }) + } + async fn read_object(&self, name: ObjectNameParam<'_>) -> Result { + let client = self.client.blob_client(name); + if client.exists().await? { + info!("found blob {name}"); + let read_stream_inner = + ReadStreamInner::new(Box::new(AzBlobReadStream::new(client.clone()).await)).await; + Ok(read_stream_inner) + } else { + bail!(format!("blob {name} not found")) + } + } + async fn write_object(&self, name: ObjectNameParam<'_>) -> Result { + // unlike read-object, there is no need for write-object to check if the object exists + // this is because the write-stream will create the object if it doesn't exist or + // overwrite it if it does + let write_stream_inner = WriteStreamInner::new(Box::new( + AzBlobWriteStream::new(self.client.blob_client(name).clone()).await, + )) + .await; + Ok(write_stream_inner) + } +} + +impl AzBlobReadStream { + pub async fn new(blob_client: BlobClient) -> Self { + Self { blob_client } + } +} + +impl AzBlobWriteStream { + pub async fn new(client: BlobClient) -> Self { + Self { client } + } +} + +#[async_trait] +impl ReadStreamImplementor for AzBlobReadStream { + async fn read(&self, size: u64) -> Result>> { + let mut size = size as usize; + let mut stream = self.blob_client.get().chunk_size(128u64).into_stream(); + let mut result = vec![]; + // The stream is composed of individual calls to the get blob endpoint + while let Some(value) = stream.next().await { + let mut body = value?.data; + // For each response, we stream the body instead of collecting it all into one large allocation. + // We use take to limit the number of bytes read from the body + while let Some(value) = (&mut body).take(size).next().await { + let value = value?; + result.extend(&value); + // reduce the size by the number of bytes read + size -= value.len(); + // if size is zero, we break out of the loop + if size == 0 { + break; + } + } + } + Ok(Some(result)) + } + async fn available(&self) -> Result { + todo!() + } +} + +#[async_trait] +impl WriteStreamImplementor for AzBlobWriteStream { + async fn write(&self, data: &[u8]) -> Result<()> { + let exists = self.client.exists().await?; + if !exists { + self.client.put_append_blob().into_future().await?; + } + self.client + .append_block(data.to_vec()) + .into_future() + .await?; + Ok(()) + } + async fn close(&self) -> Result<()> { + todo!() + } +} + +impl From for ContainerMetadata { + fn from(container: Container) -> Self { + // TODO: the last-modified timestamp is not the same as created_at + // there is no other APIs exposed by azure blob storage to get + // the container's creation time + Self { + name: container.name, + created_at: container.last_modified.unix_timestamp() as u64, + } + } +} diff --git a/crates/blob-store/src/implementors/mod.rs b/crates/blob-store/src/implementors/mod.rs new file mode 100644 index 00000000..e5cc5abc --- /dev/null +++ b/crates/blob-store/src/implementors/mod.rs @@ -0,0 +1,4 @@ +#[cfg(feature = "aws_s3")] +pub mod aws_s3; +#[cfg(feature = "azblob")] +pub mod azblob; diff --git a/crates/blob-store/src/lib.rs b/crates/blob-store/src/lib.rs new file mode 100644 index 00000000..4b37c25d --- /dev/null +++ b/crates/blob-store/src/lib.rs @@ -0,0 +1,207 @@ +mod container; +mod implementors; +mod read_stream; +mod write_stream; +use std::{ + collections::HashMap, + fmt::{Debug, Display}, +}; + +use async_trait::async_trait; + +use container::ContainerInner; +use read_stream::ReadStreamInner; +use slight_common::{impl_resource, BasicState}; + +use blob_store::*; +use write_stream::WriteStreamInner; +wit_bindgen_wasmtime::export!({paths: ["../../wit/blob-store.wit"], async: *}); +wit_error_rs::impl_error!(blob_store::Error); +wit_error_rs::impl_from!(anyhow::Error, blob_store::Error::UnexpectedError); + +pub const BLOB_STORE_SCHEME_NAME: &str = "blob-store"; + +#[cfg(feature = "aws_s3")] +pub use implementors::aws_s3::S3_CAPABILITY_NAME; +#[cfg(feature = "azblob")] +pub use implementors::azblob::AZBLOB_CAPABILITY_NAME; + +/// A BlobStore is a container for storing and retrieving arbitrary data. +/// +/// The implementation of the blobstore roughtly follows the +/// [wasi-blob-store](https://github.com/WebAssembly/wasi-blob-store) interfaces, +#[derive(Clone, Default)] +pub struct BlobStore { + implementor: BlobStoreImplementors, + capability_store: HashMap, +} + +impl BlobStore { + pub fn new(implementor: String, keyvalue_store: HashMap) -> Self { + Self { + implementor: implementor.as_str().into(), + capability_store: keyvalue_store, + } + } +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, Default)] +pub enum BlobStoreImplementors { + #[cfg(feature = "aws_s3")] + S3, + #[cfg(feature = "azblob")] + AzBlob, + #[default] + None, +} + +impl From<&str> for BlobStoreImplementors { + fn from(s: &str) -> Self { + match s { + #[cfg(feature = "aws_s3")] + S3_CAPABILITY_NAME => Self::S3, + #[cfg(feature = "azblob")] + AZBLOB_CAPABILITY_NAME => Self::AzBlob, + p => panic!( + "failed to match provided name (i.e., '{p}') to any known host implementations" + ), + } + } +} + +impl Display for BlobStoreImplementors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + #[cfg(feature = "aws_s3")] + Self::S3 => write!(f, "{S3_CAPABILITY_NAME}"), + #[cfg(feature = "azblob")] + Self::AzBlob => write!(f, "{AZBLOB_CAPABILITY_NAME}"), + Self::None => panic!("No implementor specified"), + } + } +} + +impl_resource!( + BlobStore, + blob_store::BlobStoreTables, + blob_store::add_to_linker, + BLOB_STORE_SCHEME_NAME.to_string() +); + +impl BlobStore { + fn fetch_state(&mut self, name: &str) -> BasicState { + let s: String = self.implementor.to_string(); + let state = if let Some(r) = self.capability_store.get(name) { + r.clone() + } else if let Some(r) = self.capability_store.get(&s) { + r.clone() + } else { + panic!( + "could not find capability under name '{}' for implementor '{}'", + name, &s + ); + }; + + state + } +} + +/// This is the implementation of the wit-generated BlobStore trait for the BlobStore struct. +#[async_trait] +impl blob_store::BlobStore for BlobStore { + type Container = ContainerInner; + type ReadStream = ReadStreamInner; + type WriteStream = WriteStreamInner; + + async fn container_open(&mut self, name: &str) -> Result { + let state = self.fetch_state(name); + tracing::log::info!("Opening implementor {}", &state.implementor); + let inner = Self::Container::new(state.implementor.as_str().into(), &state, name).await?; + + Ok(inner) + } + + async fn container_name(&mut self, self_: &Self::Container) -> Result { + Ok(self_.implementor.name().await?) + } + async fn container_info( + &mut self, + self_: &Self::Container, + ) -> Result { + Ok(self_.implementor.info().await?) + } + async fn container_read_object( + &mut self, + self_: &Self::Container, + name: ObjectNameParam<'_>, + ) -> Result { + let read_stream = self_.implementor.read_object(name).await?; + Ok(read_stream) + } + async fn container_write_object( + &mut self, + self_: &Self::Container, + name: ObjectNameParam<'_>, + ) -> Result { + let write_stream = self_.implementor.write_object(name).await?; + Ok(write_stream) + } + async fn container_list_objects( + &mut self, + self_: &Self::Container, + ) -> Result, Error> { + Ok(self_.implementor.list_objects().await?) + } + async fn container_delete_object( + &mut self, + self_: &Self::Container, + name: ObjectNameParam<'_>, + ) -> Result<(), Error> { + Ok(self_.implementor.delete_object(name).await?) + } + async fn container_delete_objects( + &mut self, + self_: &Self::Container, + names: Vec>, + ) -> Result<(), Error> { + Ok(self_.implementor.delete_objects(names).await?) + } + async fn container_has_object( + &mut self, + self_: &Self::Container, + name: ObjectNameParam<'_>, + ) -> Result { + Ok(self_.implementor.has_object(name).await?) + } + async fn container_object_info( + &mut self, + self_: &Self::Container, + name: ObjectNameParam<'_>, + ) -> Result { + Ok(self_.implementor.object_info(name).await?) + } + async fn container_clear(&mut self, _self_: &Self::Container) -> Result<(), Error> { + todo!() + } + async fn write_stream_write( + &mut self, + self_: &Self::WriteStream, + data: &[u8], + ) -> Result<(), Error> { + Ok(self_.implementor.write(data).await?) + } + async fn write_stream_close(&mut self, self_: &Self::WriteStream) -> Result<(), Error> { + Ok(self_.implementor.close().await?) + } + async fn read_stream_read( + &mut self, + self_: &Self::ReadStream, + size: u64, + ) -> Result>, Error> { + Ok(self_.implementor.read(size).await?) + } + async fn read_stream_available(&mut self, self_: &Self::ReadStream) -> Result { + Ok(self_.implementor.available().await?) + } +} diff --git a/crates/blob-store/src/read_stream.rs b/crates/blob-store/src/read_stream.rs new file mode 100644 index 00000000..27def24b --- /dev/null +++ b/crates/blob-store/src/read_stream.rs @@ -0,0 +1,38 @@ +use anyhow::Result; + +use crate::container::DynR; +use async_trait::async_trait; +use std::fmt::Debug; + +/// A stream of bytes that can be read from +#[async_trait] +pub trait ReadStreamImplementor { + /// Read a number of bytes from the stream + /// + /// Returns `None` if the stream has reached the end + /// Otherwise returns a `Vec` of the bytes read + async fn read(&self, size: u64) -> Result>>; + + /// Returns the number of bytes available to read + /// + /// TODO: This is not implemented for all implementors + async fn available(&self) -> Result; +} + +impl Debug for dyn ReadStreamImplementor + Send + Sync { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReadStreamImplementor") + .finish_non_exhaustive() + } +} + +#[derive(Debug)] +pub struct ReadStreamInner { + pub implementor: Box, +} + +impl ReadStreamInner { + pub async fn new(implementor: Box) -> Self { + Self { implementor } + } +} diff --git a/crates/blob-store/src/write_stream.rs b/crates/blob-store/src/write_stream.rs new file mode 100644 index 00000000..99004bcb --- /dev/null +++ b/crates/blob-store/src/write_stream.rs @@ -0,0 +1,38 @@ +use anyhow::Result; + +use async_trait::async_trait; + +use crate::container::DynW; + +/// A stream of bytes that can be written to +#[async_trait] +pub trait WriteStreamImplementor { + /// Write a number of bytes to the stream + /// + /// This is a blocking operation that write the data byte array + /// to the blob. + async fn write(&self, data: &[u8]) -> Result<()>; + + /// Close the stream + /// + /// TODO: Not used + async fn close(&self) -> Result<()>; +} + +impl std::fmt::Debug for dyn WriteStreamImplementor + Send + Sync { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WriteStreamImplementor") + .finish_non_exhaustive() + } +} + +#[derive(Debug)] +pub struct WriteStreamInner { + pub implementor: Box, +} + +impl WriteStreamInner { + pub async fn new(implementor: Box) -> Self { + Self { implementor } + } +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 17d71d7f..316408ae 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -11,6 +11,7 @@ use async_trait::async_trait; use ctx::SlightCtxBuilder; use resource::{get_host_state, HttpData, HttpServerExportData}; use slight_common::{CapabilityBuilder, WasmtimeBuildable, WasmtimeLinkable}; +use tracing::info; use wasi_cap_std_sync::{ambient_authority, Dir, WasiCtxBuilder}; use wasi_common::pipe::{ReadPipe, WritePipe}; use wasi_common::WasiCtx; @@ -168,10 +169,11 @@ impl WasmtimeBuildable for Builder { fn build_wasi_context(io_redirects: IORedirects) -> Result { let mut ctx: WasiCtxBuilder = WasiCtxBuilder::new(); ctx = add_io_redirects_to_wasi_context(ctx, io_redirects)?; - Ok(ctx - .inherit_args()? - .preopened_dir(Dir::open_ambient_dir(".", ambient_authority())?, ".")? - .build()) + let dir = Dir::open_ambient_dir(".", ambient_authority())?; + // get pwd + let path = std::env::current_dir()?; + info!("Currnet dir: {:?}", path); + Ok(ctx.inherit_args()?.preopened_dir(dir, ".")?.build()) } /// add_io_redirects_to_wasi_context inherits existing stdio and overrides stdio as available. diff --git a/examples/blob-store-demo/Cargo.lock b/examples/blob-store-demo/Cargo.lock new file mode 100644 index 00000000..2f0b2480 --- /dev/null +++ b/examples/blob-store-demo/Cargo.lock @@ -0,0 +1,264 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.9", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blob-store-demo" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde_json", + "wit-bindgen-rust", + "wit-error-rs", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "proc-macro2" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "serde" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" + +[[package]] +name = "serde_json" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da4a3c17e109f700685ec577c0f85efd9b19bcf15c913985f14dc1ac01775aa" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wit-bindgen-gen-core" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "anyhow", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-gen-rust" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "heck", + "wit-bindgen-gen-core", +] + +[[package]] +name = "wit-bindgen-gen-rust-wasm" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "heck", + "wit-bindgen-gen-core", + "wit-bindgen-gen-rust", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "async-trait", + "bitflags", + "wit-bindgen-rust-impl", +] + +[[package]] +name = "wit-bindgen-rust-impl" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "proc-macro2", + "syn 1.0.109", + "wit-bindgen-gen-core", + "wit-bindgen-gen-rust-wasm", +] + +[[package]] +name = "wit-error-rs" +version = "0.1.0" +source = "git+https://github.com/danbugs/wit-error-rs?rev=05362f1a4a3a9dc6a1de39195e06d2d5d6491a5e#05362f1a4a3a9dc6a1de39195e06d2d5d6491a5e" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wit-parser" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "anyhow", + "id-arena", + "pulldown-cmark", + "unicode-normalization", + "unicode-xid", +] diff --git a/examples/blob-store-demo/Cargo.toml b/examples/blob-store-demo/Cargo.toml new file mode 100644 index 00000000..6147949f --- /dev/null +++ b/examples/blob-store-demo/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "blob-store-demo" +version = "0.1.0" +edition = "2021" +authors = ["DeisLabs Engineering Team"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "blob-store-demo" +test = false + +[dependencies] +wit-bindgen-rust = { git = "https://github.com/fermyon/wit-bindgen-backport" } +anyhow = "1" +wit-error-rs = { git = "https://github.com/danbugs/wit-error-rs", rev = "05362f1a4a3a9dc6a1de39195e06d2d5d6491a5e" } +serde_json = "1" + +[workspace] diff --git a/examples/blob-store-demo/az_blob.toml b/examples/blob-store-demo/az_blob.toml new file mode 100644 index 00000000..2560378a --- /dev/null +++ b/examples/blob-store-demo/az_blob.toml @@ -0,0 +1,8 @@ +specversion = "0.2" + +[[capability]] +resource = "blobstore.azblob" +name = "slight-example-bucket" + [capability.configs] + AZURE_STORAGE_ACCOUNT = "${azapp.AZURE_STORAGE_ACCOUNT}" + AZURE_STORAGE_KEY = "${azapp.AZURE_STORAGE_KEY}" \ No newline at end of file diff --git a/examples/blob-store-demo/blob_s3.toml b/examples/blob-store-demo/blob_s3.toml new file mode 100644 index 00000000..043b3307 --- /dev/null +++ b/examples/blob-store-demo/blob_s3.toml @@ -0,0 +1,10 @@ +specversion = "0.2" + +[[capability]] +resource = "blobstore.aws_s3" +name = "slight-example-bucket" + # This capability does not require any configs + [capability.configs] + AWS_ACCESS_KEY_ID = "${envvars.AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY = "${envvars.AWS_SECRET_ACCESS_KEY}" + AWS_REGION = "${envvars.AWS_REGION}" \ No newline at end of file diff --git a/examples/blob-store-demo/src/main.rs b/examples/blob-store-demo/src/main.rs new file mode 100644 index 00000000..d4c6bd00 --- /dev/null +++ b/examples/blob-store-demo/src/main.rs @@ -0,0 +1,19 @@ +use anyhow::Result; + +use blob_store::*; +wit_bindgen_rust::import!("../../wit/blob-store.wit"); +wit_error_rs::impl_error!(Error); + +fn main() -> Result<()> { + let bucket = blob_store::Container::open("slight-example-bucket")?; + if bucket.has_object("file.txt")? { + let read_stream = bucket.read_object("file.txt")?; + let contents = read_stream.read(1024)?.unwrap(); + + println!("The contents of file.txt are: {}", std::str::from_utf8(&contents)?); + } else { + let writer = bucket.write_object("file.txt")?; + writer.write(b"Hello, world!")?; + } + Ok(()) +} diff --git a/src/commands/add.rs b/src/commands/add.rs index 511485fd..10daa791 100644 --- a/src/commands/add.rs +++ b/src/commands/add.rs @@ -20,6 +20,7 @@ const HTTP_CLIENT_DOWNLOADS: [&str; 2] = ["http-types", "http-client"]; const DISTRIBUTED_LOCKING_DOWNLOADS: [&str; 1] = ["distributed-locking"]; const MESSAGING_DOWNLOADS: [&str; 1] = ["messaging"]; const SQL_DOWNLOADS: [&str; 1] = ["sql"]; +const BLOBSTORE_DOWNLOADS: [&str; 2] = ["blob-store", "blob-types"]; const ERROR_MSG: &str = "invalid interface name (2): currently, slight only supports the download of 'configs', 'keyvalue', 'distributed_locking', 'messaging', 'sql', and 'http'."; @@ -86,6 +87,7 @@ pub fn get_interface_downloads_by_name(name: &str) -> Vec<&str> { _ if name.eq("distributed-locking") => DISTRIBUTED_LOCKING_DOWNLOADS.to_vec(), _ if name.eq("messaging") => MESSAGING_DOWNLOADS.to_vec(), _ if name.eq("sql") => SQL_DOWNLOADS.to_vec(), + _ if name.eq("blobstore") | name.eq("blob-store") => BLOBSTORE_DOWNLOADS.to_vec(), _ => { vec![] } diff --git a/src/commands/run.rs b/src/commands/run.rs index f0996c5d..ea141616 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -5,6 +5,10 @@ use std::{ use anyhow::{bail, Result}; use as_any::Downcast; +#[cfg(feature = "blob-store")] +use slight_blob_store::{ + BlobStore, AZBLOB_CAPABILITY_NAME, BLOB_STORE_SCHEME_NAME, S3_CAPABILITY_NAME, +}; use slight_common::{BasicState, Capability, Ctx as _, WasmtimeBuildable}; use slight_core::slightfile::{Capability as TomlCapability, TomlFile}; #[cfg(feature = "distributed-locking")] @@ -24,6 +28,9 @@ use slight_runtime_configs::Configs; use slight_sql::Sql; use wit_bindgen_wasmtime::wasmtime::Store; +#[cfg(feature = "blob-store")] +const BLOB_STORE_HOST_IMPLEMENTORS: [&str; 2] = [S3_CAPABILITY_NAME, AZBLOB_CAPABILITY_NAME]; + #[cfg(feature = "keyvalue")] const KEYVALUE_HOST_IMPLEMENTORS: [&str; 8] = [ "kv.filesystem", @@ -228,6 +235,18 @@ async fn build_store_instance( } match resource_type { + #[cfg(feature = "blob-store")] + _ if BLOB_STORE_HOST_IMPLEMENTORS.contains(&resource_type) => { + if !linked_capabilities.contains(BLOB_STORE_SCHEME_NAME) { + builder.link_capability::()?; + linked_capabilities.insert(BLOB_STORE_SCHEME_NAME.to_string()); + } + let resource = slight_blob_store::BlobStore::new( + resource_type.to_string(), + capability_store.clone(), + ); + builder.add_to_builder(BLOB_STORE_SCHEME_NAME.to_string(), resource); + } #[cfg(feature = "keyvalue")] _ if KEYVALUE_HOST_IMPLEMENTORS.contains(&resource_type) => { if !linked_capabilities.contains("keyvalue") { @@ -331,6 +350,9 @@ async fn build_store_instance( #[cfg(feature = "sql")] allowed_schemes.extend(&SQL_HOST_IMPLEMENTORS); + #[cfg(feature = "blob-store")] + allowed_schemes.extend(&BLOB_STORE_HOST_IMPLEMENTORS); + let allowed_schemes_str = allowed_schemes.join(", "); bail!("invalid url: currently slight only supports http, and the '{}' capability schemes", allowed_schemes_str); diff --git a/tests/blob-store-test/Cargo.lock b/tests/blob-store-test/Cargo.lock new file mode 100644 index 00000000..35685fad --- /dev/null +++ b/tests/blob-store-test/Cargo.lock @@ -0,0 +1,234 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.9", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blob-store-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "wit-bindgen-rust", + "wit-error-rs", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "proc-macro2" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da4a3c17e109f700685ec577c0f85efd9b19bcf15c913985f14dc1ac01775aa" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wit-bindgen-gen-core" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "anyhow", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-gen-rust" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "heck", + "wit-bindgen-gen-core", +] + +[[package]] +name = "wit-bindgen-gen-rust-wasm" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "heck", + "wit-bindgen-gen-core", + "wit-bindgen-gen-rust", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "async-trait", + "bitflags", + "wit-bindgen-rust-impl", +] + +[[package]] +name = "wit-bindgen-rust-impl" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "proc-macro2", + "syn 1.0.109", + "wit-bindgen-gen-core", + "wit-bindgen-gen-rust-wasm", +] + +[[package]] +name = "wit-error-rs" +version = "0.1.0" +source = "git+https://github.com/danbugs/wit-error-rs?rev=05362f1a4a3a9dc6a1de39195e06d2d5d6491a5e#05362f1a4a3a9dc6a1de39195e06d2d5d6491a5e" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wit-parser" +version = "0.2.0" +source = "git+https://github.com/fermyon/wit-bindgen-backport#e1e2525bbbc8430c4ebe57e9f4b3f63b6facff98" +dependencies = [ + "anyhow", + "id-arena", + "pulldown-cmark", + "unicode-normalization", + "unicode-xid", +] diff --git a/tests/blob-store-test/Cargo.toml b/tests/blob-store-test/Cargo.toml new file mode 100644 index 00000000..3454b6bd --- /dev/null +++ b/tests/blob-store-test/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "blob-store-test" +version = "0.1.0" +edition = "2021" +authors = [ "DeisLabs Engineering Team" ] + +[[bin]] +name = "blob-store-test" +test = false + +[dependencies] +wit-bindgen-rust = { git = "https://github.com/fermyon/wit-bindgen-backport" } +anyhow = "1" +wit-error-rs = { git = "https://github.com/danbugs/wit-error-rs", rev = "05362f1a4a3a9dc6a1de39195e06d2d5d6491a5e" } + +[workspace] \ No newline at end of file diff --git a/tests/blob-store-test/az_blob.toml b/tests/blob-store-test/az_blob.toml new file mode 100644 index 00000000..4ed37591 --- /dev/null +++ b/tests/blob-store-test/az_blob.toml @@ -0,0 +1,8 @@ +specversion = "0.2" + +[[capability]] +resource = "blobstore.azblob" +name = "slight-test-bucket" + [capability.configs] + AZURE_STORAGE_ACCOUNT = "${azapp.AZURE_STORAGE_ACCOUNT}" + AZURE_STORAGE_KEY = "${azapp.AZURE_STORAGE_KEY}" diff --git a/tests/blob-store-test/blob_s3.toml b/tests/blob-store-test/blob_s3.toml new file mode 100644 index 00000000..43daa24a --- /dev/null +++ b/tests/blob-store-test/blob_s3.toml @@ -0,0 +1,9 @@ +specversion = "0.2" + +[[capability]] +resource = "blobstore.aws_s3" +name = "slight-test-bucket" + [capability.configs] + AWS_ACCESS_KEY_ID = "${envvars.AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY = "${envvars.AWS_SECRET_ACCESS_KEY}" + AWS_REGION = "${envvars.AWS_REGION}" \ No newline at end of file diff --git a/tests/blob-store-test/src/main.rs b/tests/blob-store-test/src/main.rs new file mode 100644 index 00000000..3de9a1b5 --- /dev/null +++ b/tests/blob-store-test/src/main.rs @@ -0,0 +1,66 @@ +use anyhow::{bail, Result}; + +use blob_store::*; +wit_bindgen_rust::import!("../../wit/blob-store.wit"); +wit_error_rs::impl_error!(Error); + +fn main() -> Result<()> { + let bucket = blob_store::Container::open("slight-test-bucket")?; + + // verify container name + assert_eq!(bucket.name()?, "slight-test-bucket"); + + for name in bucket.list_objects()? { + println!("Found object: {name}"); + bucket.delete_object(&name)?; + } + + // upload 3 files + for i in 0..3 { + let body = std::fs::read(format!("testfile{i}.txt")) + .expect("should have been able to read the file"); + bucket + .write_object(&format!("testfile{i}.txt"))? + .write(body.as_ref())?; + } + + // read 3 files + for name in bucket.list_objects()? { + println!("Found object: {name}"); + let read_stream = bucket.read_object(&name)?; + let contents = read_stream.read(1024 * 4)?.unwrap(); + + // compare the contents with content on filesystem + let body = std::fs::read(&name).expect("should have been able to read the file"); + assert_eq!(body, contents); + } + + // read one file + if bucket.has_object("testfile1.txt")? { + let read_stream = bucket.read_object("testfile1.txt")?; + let contents = read_stream.read(1024 * 4)?.unwrap(); + + // compare the content with content on filesystem + let body = std::fs::read("testfile1.txt").expect("should have been able to read the file"); + assert_eq!(body, contents); + } else { + bail!("testfile1.txt not found") + } + + // return metadata for the testfile1.txt + let body = std::fs::read("testfile1.txt").expect("should have been able to read the file"); + let metadata = bucket.object_info("testfile1.txt")?; + assert_eq!(metadata.name, "testfile1.txt"); + assert_eq!(metadata.size, body.len() as u64); + assert_eq!(metadata.container, "slight-test-bucket"); + println!("metadata created-at: {:?}", metadata.created_at); + + // delete all three files + // TODO: re-enable this once the delete_objects() method is implemented in azblob + // bucket.delete_objects(&["testfile0.txt", "testfile1.txt", "testfile2.txt"])?; + + for name in bucket.list_objects()? { + bucket.delete_object(&name)?; + } + Ok(()) +} diff --git a/tests/blob-store-test/testfile0.txt b/tests/blob-store-test/testfile0.txt new file mode 100644 index 00000000..29182e49 --- /dev/null +++ b/tests/blob-store-test/testfile0.txt @@ -0,0 +1,6 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris nulla dolor, varius vel vehicula vitae, dapibus id tellus. Vivamus nisl risus, pretium in nisi non, venenatis rhoncus ligula. Sed pharetra nibh nulla. Integer vitae mollis nisi. Nunc ante nulla, cursus id dolor sit amet, suscipit molestie est. Vestibulum sollicitudin fermentum arcu ut dictum. Sed finibus feugiat libero, sit amet fermentum ex consequat auctor. Donec varius fringilla sagittis. Sed gravida efficitur erat et viverra. Nulla malesuada metus vitae lacus malesuada, at faucibus velit tincidunt. Cras fermentum blandit purus. Integer rutrum semper lorem, rutrum aliquet elit egestas ac. Integer tincidunt sem nec turpis aliquet consectetur. Nunc accumsan est leo, ut tincidunt turpis interdum in. Ut finibus nibh et ullamcorper pharetra. +Vivamus quis dictum ipsum. Proin congue vulputate elit rhoncus congue. Mauris elementum libero mollis, mattis nibh eu, commodo erat. Duis sodales feugiat diam, ut tempor purus viverra eu. Vestibulum sodales sit amet eros quis placerat. Phasellus venenatis condimentum nisl eget tristique. Suspendisse potenti. Cras a orci nec elit pulvinar condimentum at et diam. Mauris faucibus aliquam posuere. Donec accumsan, metus ac scelerisque iaculis, felis enim ullamcorper ante, quis tristique elit justo et ex. Donec et velit pharetra, viverra lacus vitae, laoreet elit. +Duis sodales odio velit, nec ornare leo venenatis ut. Pellentesque in libero sit amet libero vestibulum lobortis. Vivamus laoreet ex eget mi suscipit, vitae volutpat felis iaculis. Etiam rhoncus ac arcu nec commodo. Phasellus posuere, mi eget egestas tincidunt, orci tellus placerat turpis, sit amet blandit nibh magna eu est. Nunc ultrices tincidunt rhoncus. Praesent faucibus id augue quis laoreet. Maecenas ac pellentesque eros, id pellentesque magna. Integer faucibus auctor ante. Mauris vitae pharetra erat. Nulla facilisi. Fusce ac ex libero. Duis posuere lacus lectus, ut varius sapien efficitur eget. Nunc enim urna, congue non tristique id, consequat eget nisi. Nulla in massa sit amet nisi rhoncus tempor sagittis id erat. +Donec in accumsan odio. Integer faucibus velit posuere sem commodo tincidunt. Fusce facilisis ex eget nisl feugiat pulvinar. Curabitur dolor diam, tempus in ligula ut, feugiat cursus elit. Sed et neque molestie, vehicula diam sed, ultricies justo. Maecenas rutrum pharetra sapien eu interdum. Donec vulputate, massa quis malesuada eleifend, ex arcu viverra magna, sit amet volutpat sem risus sit amet dui. Sed ultricies at eros at consequat. Morbi fringilla tristique mauris vel iaculis. Pellentesque ornare dictum finibus. Fusce aliquet odio a blandit interdum. +Donec mollis finibus metus in cursus. In blandit ornare purus. Vestibulum a ipsum diam. Curabitur vel iaculis diam, id egestas ligula. Morbi condimentum imperdiet tellus. Aenean ut ligula nulla. Nam quis auctor odio. Sed eget blandit magna, sit amet consectetur ex. Quisque fermentum nunc at nisi dictum molestie. Ut tempus fermentum ipsum, vel aliquet sem rutrum quis. Nunc nec tristique diam. +Touch test. diff --git a/tests/blob-store-test/testfile1.txt b/tests/blob-store-test/testfile1.txt new file mode 100644 index 00000000..29182e49 --- /dev/null +++ b/tests/blob-store-test/testfile1.txt @@ -0,0 +1,6 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris nulla dolor, varius vel vehicula vitae, dapibus id tellus. Vivamus nisl risus, pretium in nisi non, venenatis rhoncus ligula. Sed pharetra nibh nulla. Integer vitae mollis nisi. Nunc ante nulla, cursus id dolor sit amet, suscipit molestie est. Vestibulum sollicitudin fermentum arcu ut dictum. Sed finibus feugiat libero, sit amet fermentum ex consequat auctor. Donec varius fringilla sagittis. Sed gravida efficitur erat et viverra. Nulla malesuada metus vitae lacus malesuada, at faucibus velit tincidunt. Cras fermentum blandit purus. Integer rutrum semper lorem, rutrum aliquet elit egestas ac. Integer tincidunt sem nec turpis aliquet consectetur. Nunc accumsan est leo, ut tincidunt turpis interdum in. Ut finibus nibh et ullamcorper pharetra. +Vivamus quis dictum ipsum. Proin congue vulputate elit rhoncus congue. Mauris elementum libero mollis, mattis nibh eu, commodo erat. Duis sodales feugiat diam, ut tempor purus viverra eu. Vestibulum sodales sit amet eros quis placerat. Phasellus venenatis condimentum nisl eget tristique. Suspendisse potenti. Cras a orci nec elit pulvinar condimentum at et diam. Mauris faucibus aliquam posuere. Donec accumsan, metus ac scelerisque iaculis, felis enim ullamcorper ante, quis tristique elit justo et ex. Donec et velit pharetra, viverra lacus vitae, laoreet elit. +Duis sodales odio velit, nec ornare leo venenatis ut. Pellentesque in libero sit amet libero vestibulum lobortis. Vivamus laoreet ex eget mi suscipit, vitae volutpat felis iaculis. Etiam rhoncus ac arcu nec commodo. Phasellus posuere, mi eget egestas tincidunt, orci tellus placerat turpis, sit amet blandit nibh magna eu est. Nunc ultrices tincidunt rhoncus. Praesent faucibus id augue quis laoreet. Maecenas ac pellentesque eros, id pellentesque magna. Integer faucibus auctor ante. Mauris vitae pharetra erat. Nulla facilisi. Fusce ac ex libero. Duis posuere lacus lectus, ut varius sapien efficitur eget. Nunc enim urna, congue non tristique id, consequat eget nisi. Nulla in massa sit amet nisi rhoncus tempor sagittis id erat. +Donec in accumsan odio. Integer faucibus velit posuere sem commodo tincidunt. Fusce facilisis ex eget nisl feugiat pulvinar. Curabitur dolor diam, tempus in ligula ut, feugiat cursus elit. Sed et neque molestie, vehicula diam sed, ultricies justo. Maecenas rutrum pharetra sapien eu interdum. Donec vulputate, massa quis malesuada eleifend, ex arcu viverra magna, sit amet volutpat sem risus sit amet dui. Sed ultricies at eros at consequat. Morbi fringilla tristique mauris vel iaculis. Pellentesque ornare dictum finibus. Fusce aliquet odio a blandit interdum. +Donec mollis finibus metus in cursus. In blandit ornare purus. Vestibulum a ipsum diam. Curabitur vel iaculis diam, id egestas ligula. Morbi condimentum imperdiet tellus. Aenean ut ligula nulla. Nam quis auctor odio. Sed eget blandit magna, sit amet consectetur ex. Quisque fermentum nunc at nisi dictum molestie. Ut tempus fermentum ipsum, vel aliquet sem rutrum quis. Nunc nec tristique diam. +Touch test. diff --git a/tests/blob-store-test/testfile2.txt b/tests/blob-store-test/testfile2.txt new file mode 100644 index 00000000..29182e49 --- /dev/null +++ b/tests/blob-store-test/testfile2.txt @@ -0,0 +1,6 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris nulla dolor, varius vel vehicula vitae, dapibus id tellus. Vivamus nisl risus, pretium in nisi non, venenatis rhoncus ligula. Sed pharetra nibh nulla. Integer vitae mollis nisi. Nunc ante nulla, cursus id dolor sit amet, suscipit molestie est. Vestibulum sollicitudin fermentum arcu ut dictum. Sed finibus feugiat libero, sit amet fermentum ex consequat auctor. Donec varius fringilla sagittis. Sed gravida efficitur erat et viverra. Nulla malesuada metus vitae lacus malesuada, at faucibus velit tincidunt. Cras fermentum blandit purus. Integer rutrum semper lorem, rutrum aliquet elit egestas ac. Integer tincidunt sem nec turpis aliquet consectetur. Nunc accumsan est leo, ut tincidunt turpis interdum in. Ut finibus nibh et ullamcorper pharetra. +Vivamus quis dictum ipsum. Proin congue vulputate elit rhoncus congue. Mauris elementum libero mollis, mattis nibh eu, commodo erat. Duis sodales feugiat diam, ut tempor purus viverra eu. Vestibulum sodales sit amet eros quis placerat. Phasellus venenatis condimentum nisl eget tristique. Suspendisse potenti. Cras a orci nec elit pulvinar condimentum at et diam. Mauris faucibus aliquam posuere. Donec accumsan, metus ac scelerisque iaculis, felis enim ullamcorper ante, quis tristique elit justo et ex. Donec et velit pharetra, viverra lacus vitae, laoreet elit. +Duis sodales odio velit, nec ornare leo venenatis ut. Pellentesque in libero sit amet libero vestibulum lobortis. Vivamus laoreet ex eget mi suscipit, vitae volutpat felis iaculis. Etiam rhoncus ac arcu nec commodo. Phasellus posuere, mi eget egestas tincidunt, orci tellus placerat turpis, sit amet blandit nibh magna eu est. Nunc ultrices tincidunt rhoncus. Praesent faucibus id augue quis laoreet. Maecenas ac pellentesque eros, id pellentesque magna. Integer faucibus auctor ante. Mauris vitae pharetra erat. Nulla facilisi. Fusce ac ex libero. Duis posuere lacus lectus, ut varius sapien efficitur eget. Nunc enim urna, congue non tristique id, consequat eget nisi. Nulla in massa sit amet nisi rhoncus tempor sagittis id erat. +Donec in accumsan odio. Integer faucibus velit posuere sem commodo tincidunt. Fusce facilisis ex eget nisl feugiat pulvinar. Curabitur dolor diam, tempus in ligula ut, feugiat cursus elit. Sed et neque molestie, vehicula diam sed, ultricies justo. Maecenas rutrum pharetra sapien eu interdum. Donec vulputate, massa quis malesuada eleifend, ex arcu viverra magna, sit amet volutpat sem risus sit amet dui. Sed ultricies at eros at consequat. Morbi fringilla tristique mauris vel iaculis. Pellentesque ornare dictum finibus. Fusce aliquet odio a blandit interdum. +Donec mollis finibus metus in cursus. In blandit ornare purus. Vestibulum a ipsum diam. Curabitur vel iaculis diam, id egestas ligula. Morbi condimentum imperdiet tellus. Aenean ut ligula nulla. Nam quis auctor odio. Sed eget blandit magna, sit amet consectetur ex. Quisque fermentum nunc at nisi dictum molestie. Ut tempus fermentum ipsum, vel aliquet sem rutrum quis. Nunc nec tristique diam. +Touch test. diff --git a/tests/build.rs b/tests/build.rs index bb60b316..53ad3b0c 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -9,6 +9,7 @@ const HTTP_TEST_PATH: &str = "./http-test"; const CONFIGS_TEST_PATH: &str = "./configs-test"; const FILESYSTEM_ACCESS_TEST_PATH: &str = "./filesystem-access-test"; const IO_TEST_PATH: &str = "./io-test"; +const BLOB_STORE_TEST_PATH: &str = "./blob-store-test"; fn main() { println!("cargo:rerun-if-changed=build.rs"); @@ -17,6 +18,8 @@ fn main() { println!("cargo:rerun-if-changed={HTTP_TEST_PATH}/src/main.rs"); println!("cargo:rerun-if-changed={CONFIGS_TEST_PATH}/src/main.rs"); println!("cargo:rerun-if-changed={FILESYSTEM_ACCESS_TEST_PATH}/src/main.rs"); + println!("cargo:rerun-if-changed={IO_TEST_PATH}/src/main.rs"); + println!("cargo:rerun-if-changed={BLOB_STORE_TEST_PATH}/src/main.rs"); // Check if wasm32-wasi target is installed @@ -36,6 +39,7 @@ fn main() { cargo_wasi_build(HTTP_TEST_PATH); cargo_wasi_build(CONFIGS_TEST_PATH); cargo_wasi_build(FILESYSTEM_ACCESS_TEST_PATH); + cargo_wasi_build(BLOB_STORE_TEST_PATH); } } diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 52c4488b..0680b03d 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -8,12 +8,15 @@ fn slight_path() -> String { format!("{}/../target/release/slight", env!("CARGO_MANIFEST_DIR")) } -pub fn run(executable: &str, args: Vec<&str>) { +pub fn run(executable: &str, args: Vec<&str>, current_dir: Option<&str>) { println!("Running {executable} with args: {args:?}"); let mut cmd = Command::new(executable); for arg in args { cmd.arg(arg); } + if let Some(dir) = current_dir { + cmd.current_dir(dir); + } let output = cmd .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) @@ -47,6 +50,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); Ok(()) } @@ -62,6 +66,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); Ok(()) } @@ -77,6 +82,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); Ok(()) } @@ -92,6 +98,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); Ok(()) } @@ -121,6 +128,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); Ok(()) } @@ -136,6 +144,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); Ok(()) } @@ -148,6 +157,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); Ok(()) } @@ -193,6 +203,7 @@ mod integration_tests { run( &slight_path(), vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + None, ); // kill the server @@ -465,4 +476,46 @@ mod integration_tests { Ok(()) } } + + #[cfg(test)] + mod blob_store_tests { + #[cfg(unix)] + use std::env; + use std::path::PathBuf; + + use crate::{run, slight_path}; + use anyhow::Result; + + #[test] + fn s3_test() -> Result<()> { + let out_dir = PathBuf::from(format!("{}/target/wasms", env!("CARGO_MANIFEST_DIR"))); + let out_dir = out_dir.join("wasm32-wasi/debug/blob-store-test.wasm"); + let file_config = &format!( + "{}/blob-store-test/blob_s3.toml", + env!("CARGO_MANIFEST_DIR") + ); + run( + &slight_path(), + vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + Some(&format!("{}/blob-store-test/", env!("CARGO_MANIFEST_DIR"))), + ); + Ok(()) + } + + #[test] + fn az_blob_test() -> Result<()> { + let out_dir = PathBuf::from(format!("{}/target/wasms", env!("CARGO_MANIFEST_DIR"))); + let out_dir = out_dir.join("wasm32-wasi/debug/blob-store-test.wasm"); + let file_config = &format!( + "{}/blob-store-test/az_blob.toml", + env!("CARGO_MANIFEST_DIR") + ); + run( + &slight_path(), + vec!["-c", file_config, "run", out_dir.to_str().unwrap()], + Some(&format!("{}/blob-store-test/", env!("CARGO_MANIFEST_DIR"))), + ); + Ok(()) + } + } } diff --git a/wit/blob-store.wit b/wit/blob-store.wit new file mode 100644 index 00000000..b502abac --- /dev/null +++ b/wit/blob-store.wit @@ -0,0 +1,103 @@ +// wasi-blob-store based on https://github.com/WebAssembly/wasi-blob-store + +use { container-metadata, object-name, object-metadata, object-id } from blob-types + +// a Container is a collection of objects +resource container { + static open: func(name: string) -> expected + + // returns container name + name: func() -> expected + + // returns container metadata + info: func() -> expected + + // begins reading an object + read-object: func(name: object-name) -> expected + + // creates or replaces an object. + write-object: func(name: object-name) -> expected + + // retrieves an object or portion of an object, as a resource. + // Start and end offsets are inclusive. + // Once a data-blob resource has been created, the underlying bytes are held by the blobstore service for the lifetime + // of the data-blob resource, even if the object they came from is later deleted. + // get-data: func(name: object-name, start: u64, end: u64) -> expected + + // creates or replaces an object with the data blob. + // write-data: func(name: object-name, data: data-blob) -> expected + + // returns list of objects in the container. Order is undefined. + list-objects: func() -> expected, error> + + // deletes object. + // does not return error if object did not exist. + delete-object: func(name: object-name) -> expected + + // deletes multiple objects in the container + delete-objects: func(names: list) -> expected + + // returns true if the object exists in this container + has-object: func(name: object-name) -> expected + + // returns metadata for the object + object-info: func(name: object-name) -> expected + + // removes all objects within the container, leaving the container empty. + clear: func() -> expected + } + +// A write stream for saving an object to a blobstore. +resource write-stream { + + // writes (appends) bytes to the object. + write: func(data: list) -> expected + + // closes the write stream + close: func() -> expected + } + + +// A read stream for retrieving an object (or object region) from blob store +resource read-stream { + + // reads bytes from the object into an existing array, + // until the buffer is full or the end of the stream. + // Returns number of bytes written, or none if the stream has ended. + // read-into: func(ref: list) -> expected, error> + + read: func(size: u64) -> expected>, error> + + // returns the number of bytes remaining that could be read until the end of the stream. + available: func() -> expected +} + +/* + +// A data-blob resource references a byte array. It is intended to be lightweight +// and can be passed to other components, without the overhead of copying the underlying bytes. +// A data-blob can be created with object::get-data(), or with the create() +resource data-blob { + // creates a new data blob + create: func() -> data-blob-writer + // begins reading this data-blob + read: func() -> expected + // returns the total size of this data-blob + size: func() -> expected +} + +// A data-blob-writer is a writable stream that creates a transient data-blob. +// The data-blob can later be saved to an object with container::write-data() +resource data-blob-writer { + // append bytes to a data-blob + write: func(data: list) -> expected + // finish writing blob + finalize: func() -> expected +} + +*/ + +/// common keyvalue errors +variant error { + unexpected-error(string) +} \ No newline at end of file diff --git a/wit/blob-types.wit b/wit/blob-types.wit new file mode 100644 index 00000000..f19a2fa3 --- /dev/null +++ b/wit/blob-types.wit @@ -0,0 +1,35 @@ +// name of a container, a collection of objects. +// The container name may be any valid UTF-8 string. +type container-name = string + +// name of an object within a container +// The object name may be any valid UTF-8 string. +type object-name = string + +type timestamp = u64 + +// information about a container +record container-metadata { + // the container's name + name: container-name, + // date and time container was created + created-at: timestamp, +} + +// information about an object +record object-metadata { + // the object's name + name: object-name, + // the object's parent container + container: container-name, + // date and time the object was created + created-at: u64, + // size of the object, in bytes + size: u64, +} + +// identifier for an object that includes its container name +record object-id { + container: container-name, + object: object-name +} \ No newline at end of file