From a8f191b306b2a83448683bb61395c10adf97627f Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 20 May 2024 10:02:50 -0400 Subject: [PATCH 001/195] Add Spin Factors crates Signed-off-by: Lann Martin --- Cargo.lock | 135 +++++++++++++------ crates/factor-wasi/Cargo.toml | 13 ++ crates/factor-wasi/src/lib.rs | 80 +++++++++++ crates/factor-wasi/src/preview1.rs | 32 +++++ crates/factors-derive/Cargo.toml | 20 +++ crates/factors-derive/src/lib.rs | 155 +++++++++++++++++++++ crates/factors/Cargo.toml | 17 +++ crates/factors/src/lib.rs | 209 +++++++++++++++++++++++++++++ crates/factors/tests/smoke.rs | 24 ++++ 9 files changed, 641 insertions(+), 44 deletions(-) create mode 100644 crates/factor-wasi/Cargo.toml create mode 100644 crates/factor-wasi/src/lib.rs create mode 100644 crates/factor-wasi/src/preview1.rs create mode 100644 crates/factors-derive/Cargo.toml create mode 100644 crates/factors-derive/src/lib.rs create mode 100644 crates/factors/Cargo.toml create mode 100644 crates/factors/src/lib.rs create mode 100644 crates/factors/tests/smoke.rs diff --git a/Cargo.lock b/Cargo.lock index ac7a0a983..bd451ad02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,18 +23,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "aes" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" -dependencies = [ - "cfg-if", - "cipher 0.3.0", - "cpufeatures", - "opaque-debug", -] - [[package]] name = "aes" version = "0.8.4" @@ -42,7 +30,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher 0.4.4", + "cipher", "cpufeatures", ] @@ -770,29 +758,31 @@ dependencies = [ ] [[package]] -name = "block-buffer" -version = "0.10.4" +name = "blake2" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "generic-array", + "digest", ] [[package]] -name = "block-modes" -version = "0.8.1" +name = "block-buffer" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "block-padding", - "cipher 0.3.0", + "generic-array", ] [[package]] name = "block-padding" -version = "0.2.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] [[package]] name = "blocking" @@ -1220,6 +1210,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.0.90" @@ -1266,15 +1265,6 @@ dependencies = [ "windows-targets 0.52.4", ] -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", -] - [[package]] name = "cipher" version = "0.4.4" @@ -2417,6 +2407,20 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "expander" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00e83c02035136f1592a47964ea60c05a50e4ed8b5892cfac197063850898d4d" +dependencies = [ + "blake2", + "fs-err", + "prettier-please", + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -2557,6 +2561,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] + [[package]] name = "fs-set-times" version = "0.20.1" @@ -3657,6 +3670,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ + "block-padding", "generic-array", ] @@ -5067,12 +5081,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "openssl" version = "0.10.64" @@ -5811,6 +5819,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettier-please" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22020dfcf177fcc7bf5deaf7440af371400c67c0de14c399938d8ed4fb4645d3" +dependencies = [ + "proc-macro2", + "syn 2.0.58", +] + [[package]] name = "prettyplease" version = "0.2.17" @@ -6827,12 +6845,12 @@ dependencies = [ [[package]] name = "secret-service" -version = "3.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da1a5ad4d28c03536f82f77d9f36603f5e37d8869ac98f0a750d5b5686d8d95" +checksum = "b5204d39df37f06d1944935232fd2dfe05008def7ca599bf28c0800366c8a8f9" dependencies = [ - "aes 0.7.5", - "block-modes", + "aes", + "cbc", "futures-util", "generic-array", "hkdf", @@ -7540,6 +7558,35 @@ dependencies = [ "toml 0.5.11", ] +[[package]] +name = "spin-factor-wasi" +version = "2.6.0-pre0" +dependencies = [ + "anyhow", + "spin-factors", + "wasmtime-wasi", +] + +[[package]] +name = "spin-factors" +version = "2.6.0-pre0" +dependencies = [ + "anyhow", + "spin-app", + "spin-factors-derive", + "wasmtime", +] + +[[package]] +name = "spin-factors-derive" +version = "2.6.0-pre0" +dependencies = [ + "expander", + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "spin-http" version = "2.7.0-pre0" @@ -10838,7 +10885,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes 0.8.4", + "aes", "byteorder", "bzip2", "constant_time_eq", diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml new file mode 100644 index 000000000..c50f2e629 --- /dev/null +++ b/crates/factor-wasi/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "spin-factor-wasi" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +spin-factors = { path = "../factors" } +wasmtime-wasi = { workspace = true } + +[lints] +workspace = true diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs new file mode 100644 index 000000000..bfc3aae0d --- /dev/null +++ b/crates/factor-wasi/src/lib.rs @@ -0,0 +1,80 @@ +pub mod preview1; + +use spin_factors::{Factor, FactorBuilder, InitContext, PrepareContext, Result, SpinFactors}; +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; + +pub struct WasiFactor; + +impl Factor for WasiFactor { + type Builder = Builder; + type Data = Data; + + fn init(&mut self, mut ctx: InitContext) -> Result<()> { + use wasmtime_wasi::bindings; + ctx.link_bindings(bindings::clocks::wall_clock::add_to_linker_get_host)?; + ctx.link_bindings(bindings::clocks::monotonic_clock::add_to_linker_get_host)?; + ctx.link_bindings(bindings::filesystem::types::add_to_linker_get_host)?; + ctx.link_bindings(bindings::filesystem::preopens::add_to_linker_get_host)?; + ctx.link_bindings(bindings::io::error::add_to_linker_get_host)?; + ctx.link_bindings(bindings::io::poll::add_to_linker_get_host)?; + ctx.link_bindings(bindings::io::streams::add_to_linker_get_host)?; + ctx.link_bindings(bindings::random::random::add_to_linker_get_host)?; + ctx.link_bindings(bindings::random::insecure::add_to_linker_get_host)?; + ctx.link_bindings(bindings::random::insecure_seed::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::exit::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::environment::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::stdin::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::stdout::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::stderr::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::terminal_input::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::terminal_output::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::terminal_stdin::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::terminal_stdout::add_to_linker_get_host)?; + ctx.link_bindings(bindings::cli::terminal_stderr::add_to_linker_get_host)?; + ctx.link_bindings(bindings::sockets::tcp::add_to_linker_get_host)?; + ctx.link_bindings(bindings::sockets::tcp_create_socket::add_to_linker_get_host)?; + ctx.link_bindings(bindings::sockets::udp::add_to_linker_get_host)?; + ctx.link_bindings(bindings::sockets::udp_create_socket::add_to_linker_get_host)?; + ctx.link_bindings(bindings::sockets::instance_network::add_to_linker_get_host)?; + ctx.link_bindings(bindings::sockets::network::add_to_linker_get_host)?; + ctx.link_bindings(bindings::sockets::ip_name_lookup::add_to_linker_get_host)?; + Ok(()) + } +} + +pub struct Builder { + wasi_ctx: WasiCtxBuilder, +} + +impl FactorBuilder for Builder { + fn prepare( + _factor: &WasiFactor, + _ctx: PrepareContext, + ) -> Result { + Ok(Self { + wasi_ctx: WasiCtxBuilder::new(), + }) + } + + fn build(mut self) -> Result { + Ok(Data { + ctx: self.wasi_ctx.build(), + table: Default::default(), + }) + } +} + +pub struct Data { + ctx: WasiCtx, + table: ResourceTable, +} + +impl WasiView for Data { + fn ctx(&mut self) -> &mut WasiCtx { + &mut self.ctx + } + + fn table(&mut self) -> &mut ResourceTable { + &mut self.table + } +} diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs new file mode 100644 index 000000000..360b713ca --- /dev/null +++ b/crates/factor-wasi/src/preview1.rs @@ -0,0 +1,32 @@ +use spin_factors::{Factor, FactorBuilder, ModuleInitContext, PrepareContext, Result, SpinFactors}; +use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; + +pub struct WasiPreview1Factor; + +impl Factor for WasiPreview1Factor { + type Builder = Builder; + type Data = WasiP1Ctx; + + fn module_init(&mut self, mut ctx: ModuleInitContext) -> Result<()> { + ctx.link_bindings(wasmtime_wasi::preview1::add_to_linker_async) + } +} + +pub struct Builder { + wasi_ctx: WasiCtxBuilder, +} + +impl FactorBuilder for Builder { + fn prepare( + _factor: &WasiPreview1Factor, + _ctx: PrepareContext, + ) -> Result { + Ok(Self { + wasi_ctx: WasiCtxBuilder::new(), + }) + } + + fn build(mut self) -> Result<::Data> { + Ok(self.wasi_ctx.build_p1()) + } +} diff --git a/crates/factors-derive/Cargo.toml b/crates/factors-derive/Cargo.toml new file mode 100644 index 000000000..859a3c329 --- /dev/null +++ b/crates/factors-derive/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "spin-factors-derive" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[lib] +proc-macro = true + +[features] +expander = ["dep:expander"] + +[dependencies] +expander = { version = "2.1.0", optional = true } +proc-macro2 = "1.0.79" +quote = "1.0.35" +syn = "2.0.52" + +[lints] +workspace = true diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs new file mode 100644 index 000000000..a0aa43b9d --- /dev/null +++ b/crates/factors-derive/src/lib.rs @@ -0,0 +1,155 @@ +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; +use syn::{parse_macro_input, Data, DeriveInput, Error}; + +#[proc_macro_derive(SpinFactors)] +pub fn derive_factors(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let expanded = expand_factors(&input).unwrap_or_else(|err| err.into_compile_error()); + + #[cfg(feature = "expander")] + let expanded = expander::Expander::new("factors") + .write_to_out_dir(expanded) + .unwrap(); + + expanded.into() +} + +#[allow(non_snake_case)] +fn expand_factors(input: &DeriveInput) -> syn::Result { + let name = &input.ident; + let vis = &input.vis; + + let builders_name = format_ident!("{name}Builders"); + let data_name = format_ident!("{name}Data"); + + if !input.generics.params.is_empty() { + return Err(Error::new_spanned( + input, + "cannot derive Factors for generic structs", + )); + } + + // Get struct fields + let fields = match &input.data { + Data::Struct(struct_data) => &struct_data.fields, + _ => { + return Err(Error::new_spanned( + input, + "can only derive Factors for structs", + )) + } + }; + let mut factor_names = Vec::with_capacity(fields.len()); + let mut factor_types = Vec::with_capacity(fields.len()); + for field in fields.iter() { + factor_names.push( + field + .ident + .as_ref() + .ok_or_else(|| Error::new_spanned(input, "tuple structs are not supported"))?, + ); + factor_types.push(&field.ty); + } + + let factors_crate = format_ident!("spin_factors"); + let factors_path = quote!(::#factors_crate); + let Factor = quote!(#factors_path::Factor); + let Result = quote!(#factors_path::Result); + let wasmtime = quote!(#factors_path::wasmtime); + let TypeId = quote!(::std::any::TypeId); + + Ok(quote! { + impl #name { + pub fn init( + &mut self, + linker: &mut #wasmtime::component::Linker<#data_name> + ) -> #Result<()> { + #( + self.#factor_names.init( + #factors_path::InitContext::::new( + linker, + |data| &mut data.#factor_names, + ) + )?; + )* + Ok(()) + } + + pub fn module_init( + &mut self, + linker: &mut #wasmtime::Linker<#data_name> + ) -> #Result<()> { + #( + self.#factor_names.module_init::( + #factors_path::ModuleInitContext::::new( + linker, + |data| &mut data.#factor_names, + ) + )?; + )* + Ok(()) + } + + pub fn build_data(&self) -> #Result<#data_name> { + let mut builders = #builders_name { + #( #factor_names: None, )* + }; + #( + builders.#factor_names = Some( + #factors_path::FactorBuilder::<#factor_types>::prepare::<#name>( + &self.#factor_names, + #factors_path::PrepareContext::new(&mut builders), + )? + ); + )* + Ok(#data_name { + #( + #factor_names: #factors_path::FactorBuilder::<#factor_types>::build( + builders.#factor_names.unwrap() + )?, + )* + }) + } + + } + + impl #factors_path::SpinFactors for #name { + type Builders = #builders_name; + type Data = #data_name; + + unsafe fn factor_builder_offset() -> Option { + let type_id = #TypeId::of::(); + #( + if type_id == #TypeId::of::<#factor_types>() { + return Some(std::mem::offset_of!(Self::Builders, #factor_names)); + } + )* + None + } + + unsafe fn factor_data_offset() -> Option { + let type_id = #TypeId::of::(); + #( + if type_id == #TypeId::of::<#factor_types>() { + return Some(std::mem::offset_of!(Self::Data, #factor_names)); + } + )* + None + + } + } + + #vis struct #builders_name { + #( + pub #factor_names: Option<<#factor_types as #Factor>::Builder>, + )* + } + + #vis struct #data_name { + #( + pub #factor_names: <#factor_types as #Factor>::Data, + )* + } + }) +} diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml new file mode 100644 index 000000000..55fbd82de --- /dev/null +++ b/crates/factors/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "spin-factors" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +spin-app = { path = "../app" } +spin-factors-derive = { path = "../factors-derive" } +wasmtime = { workspace = true } + +[dev-dependencies] +spin-factors-derive = { path = "../factors-derive", features = ["expander"] } + +[lints] +workspace = true diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs new file mode 100644 index 000000000..f65e65e15 --- /dev/null +++ b/crates/factors/src/lib.rs @@ -0,0 +1,209 @@ +use std::{any::Any, marker::PhantomData}; + +use spin_app::App; +pub use spin_factors_derive::SpinFactors; + +pub use wasmtime; + +pub type Error = wasmtime::Error; +pub type Result = std::result::Result; + +pub type Linker = wasmtime::component::Linker<::Data>; +pub type ModuleLinker = wasmtime::Linker<::Data>; + +pub trait Factor: Any + Sized { + type Builder: FactorBuilder; + type Data; + + /// Initializes this Factor for a runtime. This will be called exactly once + fn init(&mut self, mut ctx: InitContext) -> Result<()> { + _ = &mut ctx; + Ok(()) + } + + fn module_init( + &mut self, + mut ctx: ModuleInitContext, + ) -> Result<()> { + _ = &mut ctx; + Ok(()) + } + + fn validate_app(&self, app: &App) -> Result<()> { + _ = app; + Ok(()) + } +} + +pub struct FactorInitContext<'a, Factors: SpinFactors, Fact: Factor, Linker> { + linker: &'a mut Linker, + get_data: fn(&mut Factors::Data) -> &mut Fact::Data, +} + +pub type InitContext<'a, Factors, Fact> = FactorInitContext<'a, Factors, Fact, Linker>; + +pub type ModuleInitContext<'a, Factors, Fact> = + FactorInitContext<'a, Factors, Fact, ModuleLinker>; + +impl<'a, Factors: SpinFactors, Fact: Factor, Linker> FactorInitContext<'a, Factors, Fact, Linker> { + #[doc(hidden)] + pub fn new( + linker: &'a mut Linker, + get_data: fn(&mut Factors::Data) -> &mut Fact::Data, + ) -> Self { + Self { linker, get_data } + } + + pub fn linker(&mut self) -> &mut Linker { + self.linker + } + + pub fn link_bindings( + &mut self, + add_to_linker: impl Fn(&mut Linker, fn(&mut Factors::Data) -> &mut Fact::Data) -> Result<()>, + ) -> Result<()> +where { + add_to_linker(self.linker, self.get_data) + } +} + +impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { + pub fn builder_mut(&mut self) -> Result<&mut T::Builder> { + let err_msg = match Factors::builder_mut::(self.builders) { + Some(Some(builder)) => return Ok(builder), + Some(None) => "builder not yet prepared", + None => "no such factor", + }; + Err(Error::msg(format!( + "could not get builder for {ty}: {err_msg}", + ty = std::any::type_name::() + ))) + } +} + +/// Implemented by `#[derive(SpinFactors)]` +pub trait SpinFactors: Sized { + type Builders; + type Data: Send + 'static; + + #[doc(hidden)] + unsafe fn factor_builder_offset() -> Option; + + #[doc(hidden)] + unsafe fn factor_data_offset() -> Option; + + fn data_getter() -> Option> { + let offset = unsafe { Self::factor_data_offset::()? }; + Some(Getter { + offset, + _phantom: PhantomData, + }) + } + + fn data_getter2() -> Option> { + let offset1 = unsafe { Self::factor_data_offset::()? }; + let offset2 = unsafe { Self::factor_data_offset::()? }; + assert_ne!( + offset1, offset2, + "data_getter2 with same factor twice would alias" + ); + Some(Getter2 { + offset1, + offset2, + _phantom: PhantomData, + }) + } + + fn builder_mut(builders: &mut Self::Builders) -> Option> { + unsafe { + let offset = Self::factor_builder_offset::()?; + let ptr = builders as *mut Self::Builders; + let opt = &mut *ptr.add(offset).cast::>(); + Some(opt.as_mut()) + } + } +} + +pub struct Getter { + offset: usize, + _phantom: PhantomData &mut U>, +} + +impl Getter { + pub fn get_mut<'a>(&self, container: &'a mut T) -> &'a mut U { + let ptr = container as *mut T; + unsafe { &mut *ptr.add(self.offset).cast::() } + } +} + +impl Clone for Getter { + fn clone(&self) -> Self { + *self + } +} +impl Copy for Getter {} + +pub struct Getter2 { + offset1: usize, + offset2: usize, + #[allow(clippy::type_complexity)] + _phantom: PhantomData (&mut U, &mut V)>, +} + +impl Getter2 { + pub fn get_mut<'a>(&self, container: &'a mut T) -> (&'a mut U, &'a mut V) + where + T: 'static, + U: 'static, + V: 'static, + { + let ptr = container as *mut T; + unsafe { + ( + &mut *ptr.add(self.offset1).cast::(), + &mut *ptr.add(self.offset2).cast::(), + ) + } + } +} + +impl Clone for Getter2 { + fn clone(&self) -> Self { + *self + } +} +impl Copy for Getter2 {} + +pub trait FactorBuilder: Sized { + fn prepare(_factor: &T, _ctx: PrepareContext) -> Result; + + fn build(self) -> Result; +} + +pub struct PrepareContext<'a, Factors: SpinFactors> { + builders: &'a mut Factors::Builders, + // TODO: component: &'a AppComponent, +} + +impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { + #[doc(hidden)] + pub fn new(builders: &'a mut Factors::Builders) -> Self { + Self { builders } + } +} + +pub type DefaultBuilder = (); + +impl FactorBuilder for DefaultBuilder +where + T::Data: Default, +{ + fn prepare(factor: &T, ctx: PrepareContext) -> Result { + (_, _) = (factor, ctx); + Ok(()) + } + + fn build(self) -> Result { + Ok(Default::default()) + } +} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs new file mode 100644 index 000000000..2deb8cc24 --- /dev/null +++ b/crates/factors/tests/smoke.rs @@ -0,0 +1,24 @@ +use spin_factors::SpinFactors; + +#[derive(SpinFactors)] +struct Factors {} + +fn main() -> anyhow::Result<()> { + let mut factors = Factors {}; + + let engine = wasmtime::Engine::default(); + let mut linker = wasmtime::component::Linker::new(&engine); + factors.init(&mut linker).unwrap(); + + let factors = Factors { + // wasi: WasiFactor, + // outbound_networking_factor: OutboundNetworkingFactor, + // outbound_http_factor: OutboundHttpFactor, + }; + let data = factors.build_data().unwrap(); + + let mut store = wasmtime::Store::new(&engine, data); + let component = wasmtime::component::Component::new(&engine, b"(component)").unwrap(); + let _instance = linker.instantiate(&mut store, &component).unwrap(); + Ok(()) +} From 84b500760891b5c2a032a424d41095eaea8b6d74 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 21 May 2024 14:55:21 -0400 Subject: [PATCH 002/195] factors: Naming updates Signed-off-by: Lann Martin --- crates/factor-wasi/src/lib.rs | 20 +++++++------ crates/factor-wasi/src/preview1.rs | 19 +++++++----- crates/factors-derive/src/lib.rs | 8 +++--- crates/factors/src/lib.rs | 46 +++++++++++++++++------------- 4 files changed, 53 insertions(+), 40 deletions(-) diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index bfc3aae0d..fe08d8f35 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,13 +1,15 @@ pub mod preview1; -use spin_factors::{Factor, FactorBuilder, InitContext, PrepareContext, Result, SpinFactors}; +use spin_factors::{ + Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, +}; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; pub struct WasiFactor; impl Factor for WasiFactor { - type Builder = Builder; - type Data = Data; + type InstancePreparer = Builder; + type InstanceState = InstanceState; fn init(&mut self, mut ctx: InitContext) -> Result<()> { use wasmtime_wasi::bindings; @@ -46,8 +48,8 @@ pub struct Builder { wasi_ctx: WasiCtxBuilder, } -impl FactorBuilder for Builder { - fn prepare( +impl FactorInstancePreparer for Builder { + fn new( _factor: &WasiFactor, _ctx: PrepareContext, ) -> Result { @@ -56,20 +58,20 @@ impl FactorBuilder for Builder { }) } - fn build(mut self) -> Result { - Ok(Data { + fn prepare(mut self) -> Result { + Ok(InstanceState { ctx: self.wasi_ctx.build(), table: Default::default(), }) } } -pub struct Data { +pub struct InstanceState { ctx: WasiCtx, table: ResourceTable, } -impl WasiView for Data { +impl WasiView for InstanceState { fn ctx(&mut self) -> &mut WasiCtx { &mut self.ctx } diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 360b713ca..5773a0a62 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,13 +1,18 @@ -use spin_factors::{Factor, FactorBuilder, ModuleInitContext, PrepareContext, Result, SpinFactors}; +use spin_factors::{ + Factor, FactorInstancePreparer, ModuleInitContext, PrepareContext, Result, SpinFactors, +}; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; pub struct WasiPreview1Factor; impl Factor for WasiPreview1Factor { - type Builder = Builder; - type Data = WasiP1Ctx; + type InstancePreparer = Builder; + type InstanceState = WasiP1Ctx; - fn module_init(&mut self, mut ctx: ModuleInitContext) -> Result<()> { + fn module_init( + &mut self, + mut ctx: ModuleInitContext, + ) -> Result<()> { ctx.link_bindings(wasmtime_wasi::preview1::add_to_linker_async) } } @@ -16,8 +21,8 @@ pub struct Builder { wasi_ctx: WasiCtxBuilder, } -impl FactorBuilder for Builder { - fn prepare( +impl FactorInstancePreparer for Builder { + fn new( _factor: &WasiPreview1Factor, _ctx: PrepareContext, ) -> Result { @@ -26,7 +31,7 @@ impl FactorBuilder for Builder { }) } - fn build(mut self) -> Result<::Data> { + fn prepare(mut self) -> Result<::InstanceState> { Ok(self.wasi_ctx.build_p1()) } } diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index a0aa43b9d..0cb8381b0 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -21,7 +21,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let vis = &input.vis; let builders_name = format_ident!("{name}Builders"); - let data_name = format_ident!("{name}Data"); + let data_name = format_ident!("{name}InstanceState"); if !input.generics.params.is_empty() { return Err(Error::new_spanned( @@ -116,7 +116,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { impl #factors_path::SpinFactors for #name { type Builders = #builders_name; - type Data = #data_name; + type InstanceState = #data_name; unsafe fn factor_builder_offset() -> Option { let type_id = #TypeId::of::(); @@ -132,7 +132,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { - return Some(std::mem::offset_of!(Self::Data, #factor_names)); + return Some(std::mem::offset_of!(Self::InstanceState, #factor_names)); } )* None @@ -148,7 +148,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #vis struct #data_name { #( - pub #factor_names: <#factor_types as #Factor>::Data, + pub #factor_names: <#factor_types as #Factor>::InstanceState, )* } }) diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index f65e65e15..af86ebb3d 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -8,12 +8,12 @@ pub use wasmtime; pub type Error = wasmtime::Error; pub type Result = std::result::Result; -pub type Linker = wasmtime::component::Linker<::Data>; -pub type ModuleLinker = wasmtime::Linker<::Data>; +pub type Linker = wasmtime::component::Linker<::InstanceState>; +pub type ModuleLinker = wasmtime::Linker<::InstanceState>; pub trait Factor: Any + Sized { - type Builder: FactorBuilder; - type Data; + type InstancePreparer: FactorInstancePreparer; + type InstanceState; /// Initializes this Factor for a runtime. This will be called exactly once fn init(&mut self, mut ctx: InitContext) -> Result<()> { @@ -37,7 +37,7 @@ pub trait Factor: Any + Sized { pub struct FactorInitContext<'a, Factors: SpinFactors, Fact: Factor, Linker> { linker: &'a mut Linker, - get_data: fn(&mut Factors::Data) -> &mut Fact::Data, + get_data: fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, } pub type InitContext<'a, Factors, Fact> = FactorInitContext<'a, Factors, Fact, Linker>; @@ -49,7 +49,7 @@ impl<'a, Factors: SpinFactors, Fact: Factor, Linker> FactorInitContext<'a, Facto #[doc(hidden)] pub fn new( linker: &'a mut Linker, - get_data: fn(&mut Factors::Data) -> &mut Fact::Data, + get_data: fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, ) -> Self { Self { linker, get_data } } @@ -60,7 +60,10 @@ impl<'a, Factors: SpinFactors, Fact: Factor, Linker> FactorInitContext<'a, Facto pub fn link_bindings( &mut self, - add_to_linker: impl Fn(&mut Linker, fn(&mut Factors::Data) -> &mut Fact::Data) -> Result<()>, + add_to_linker: impl Fn( + &mut Linker, + fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, + ) -> Result<()>, ) -> Result<()> where { add_to_linker(self.linker, self.get_data) @@ -68,7 +71,7 @@ where { } impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { - pub fn builder_mut(&mut self) -> Result<&mut T::Builder> { + pub fn builder_mut(&mut self) -> Result<&mut T::InstancePreparer> { let err_msg = match Factors::builder_mut::(self.builders) { Some(Some(builder)) => return Ok(builder), Some(None) => "builder not yet prepared", @@ -84,7 +87,7 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { /// Implemented by `#[derive(SpinFactors)]` pub trait SpinFactors: Sized { type Builders; - type Data: Send + 'static; + type InstanceState: Send + 'static; #[doc(hidden)] unsafe fn factor_builder_offset() -> Option; @@ -92,7 +95,7 @@ pub trait SpinFactors: Sized { #[doc(hidden)] unsafe fn factor_data_offset() -> Option; - fn data_getter() -> Option> { + fn data_getter() -> Option> { let offset = unsafe { Self::factor_data_offset::()? }; Some(Getter { offset, @@ -100,7 +103,8 @@ pub trait SpinFactors: Sized { }) } - fn data_getter2() -> Option> { + fn data_getter2( + ) -> Option> { let offset1 = unsafe { Self::factor_data_offset::()? }; let offset2 = unsafe { Self::factor_data_offset::()? }; assert_ne!( @@ -114,11 +118,13 @@ pub trait SpinFactors: Sized { }) } - fn builder_mut(builders: &mut Self::Builders) -> Option> { + fn builder_mut( + builders: &mut Self::Builders, + ) -> Option> { unsafe { let offset = Self::factor_builder_offset::()?; let ptr = builders as *mut Self::Builders; - let opt = &mut *ptr.add(offset).cast::>(); + let opt = &mut *ptr.add(offset).cast::>(); Some(opt.as_mut()) } } @@ -174,10 +180,10 @@ impl Clone for Getter2 { } impl Copy for Getter2 {} -pub trait FactorBuilder: Sized { - fn prepare(_factor: &T, _ctx: PrepareContext) -> Result; +pub trait FactorInstancePreparer: Sized { + fn new(_factor: &T, _ctx: PrepareContext) -> Result; - fn build(self) -> Result; + fn prepare(self) -> Result; } pub struct PrepareContext<'a, Factors: SpinFactors> { @@ -194,16 +200,16 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { pub type DefaultBuilder = (); -impl FactorBuilder for DefaultBuilder +impl FactorInstancePreparer for DefaultBuilder where - T::Data: Default, + T::InstanceState: Default, { - fn prepare(factor: &T, ctx: PrepareContext) -> Result { + fn new(factor: &T, ctx: PrepareContext) -> Result { (_, _) = (factor, ctx); Ok(()) } - fn build(self) -> Result { + fn prepare(self) -> Result { Ok(Default::default()) } } From c5c2e61fc7ab44a641f6a1927df1d40e2d1956fa Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 21 May 2024 17:00:53 -0400 Subject: [PATCH 003/195] factors: Fix derive macro Signed-off-by: Lann Martin --- Cargo.lock | 1 + crates/factor-wasi/src/lib.rs | 6 ++-- crates/factor-wasi/src/preview1.rs | 4 +-- crates/factors-derive/src/lib.rs | 51 ++++++++++++++++-------------- crates/factors/Cargo.toml | 1 + crates/factors/src/lib.rs | 50 ++++++++++++++--------------- crates/factors/tests/smoke.rs | 15 +++++---- 7 files changed, 66 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd451ad02..4159011a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7573,6 +7573,7 @@ version = "2.6.0-pre0" dependencies = [ "anyhow", "spin-app", + "spin-factor-wasi", "spin-factors-derive", "wasmtime", ] diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index fe08d8f35..02a135e03 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,8 +1,6 @@ pub mod preview1; -use spin_factors::{ - Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, -}; +use spin_factors::{Factor, InitContext, InstancePreparer, PrepareContext, Result, SpinFactors}; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; pub struct WasiFactor; @@ -48,7 +46,7 @@ pub struct Builder { wasi_ctx: WasiCtxBuilder, } -impl FactorInstancePreparer for Builder { +impl InstancePreparer for Builder { fn new( _factor: &WasiFactor, _ctx: PrepareContext, diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 5773a0a62..4c4cef5a3 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,5 +1,5 @@ use spin_factors::{ - Factor, FactorInstancePreparer, ModuleInitContext, PrepareContext, Result, SpinFactors, + Factor, InstancePreparer, ModuleInitContext, PrepareContext, Result, SpinFactors, }; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; @@ -21,7 +21,7 @@ pub struct Builder { wasi_ctx: WasiCtxBuilder, } -impl FactorInstancePreparer for Builder { +impl InstancePreparer for Builder { fn new( _factor: &WasiPreview1Factor, _ctx: PrepareContext, diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 0cb8381b0..18aed6d1c 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -20,8 +20,8 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let name = &input.ident; let vis = &input.vis; - let builders_name = format_ident!("{name}Builders"); - let data_name = format_ident!("{name}InstanceState"); + let preparers_name = format_ident!("{name}InstancePreparers"); + let state_name = format_ident!("{name}InstanceState"); if !input.generics.params.is_empty() { return Err(Error::new_spanned( @@ -63,50 +63,53 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { impl #name { pub fn init( &mut self, - linker: &mut #wasmtime::component::Linker<#data_name> + linker: &mut #wasmtime::component::Linker<#state_name> ) -> #Result<()> { #( - self.#factor_names.init( + #Factor::init::( + &mut self.#factor_names, #factors_path::InitContext::::new( linker, - |data| &mut data.#factor_names, + |state| &mut state.#factor_names, ) )?; )* Ok(()) } + #[allow(dead_code)] pub fn module_init( &mut self, - linker: &mut #wasmtime::Linker<#data_name> + linker: &mut #wasmtime::Linker<#state_name> ) -> #Result<()> { #( - self.#factor_names.module_init::( + #Factor::module_init::( + &mut self.#factor_names, #factors_path::ModuleInitContext::::new( linker, - |data| &mut data.#factor_names, + |state| &mut state.#factor_names, ) )?; )* Ok(()) } - pub fn build_data(&self) -> #Result<#data_name> { - let mut builders = #builders_name { + pub fn build_store_data(&self) -> #Result<#state_name> { + let mut preparers = #preparers_name { #( #factor_names: None, )* }; #( - builders.#factor_names = Some( - #factors_path::FactorBuilder::<#factor_types>::prepare::<#name>( + preparers.#factor_names = Some( + #factors_path::InstancePreparer::<#factor_types>::new::<#name>( &self.#factor_names, - #factors_path::PrepareContext::new(&mut builders), + #factors_path::PrepareContext::new(&mut preparers), )? ); )* - Ok(#data_name { + Ok(#state_name { #( - #factor_names: #factors_path::FactorBuilder::<#factor_types>::build( - builders.#factor_names.unwrap() + #factor_names: #factors_path::InstancePreparer::<#factor_types>::prepare( + preparers.#factor_names.unwrap() )?, )* }) @@ -115,20 +118,20 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } impl #factors_path::SpinFactors for #name { - type Builders = #builders_name; - type InstanceState = #data_name; + type InstancePreparers = #preparers_name; + type InstanceState = #state_name; - unsafe fn factor_builder_offset() -> Option { + unsafe fn instance_preparer_offset() -> Option { let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { - return Some(std::mem::offset_of!(Self::Builders, #factor_names)); + return Some(std::mem::offset_of!(Self::InstancePreparers, #factor_names)); } )* None } - unsafe fn factor_data_offset() -> Option { + unsafe fn instance_state_offset() -> Option { let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { @@ -140,13 +143,13 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } } - #vis struct #builders_name { + #vis struct #preparers_name { #( - pub #factor_names: Option<<#factor_types as #Factor>::Builder>, + pub #factor_names: Option<<#factor_types as #Factor>::InstancePreparer>, )* } - #vis struct #data_name { + #vis struct #state_name { #( pub #factor_names: <#factor_types as #Factor>::InstanceState, )* diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 55fbd82de..6e3e75d81 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -12,6 +12,7 @@ wasmtime = { workspace = true } [dev-dependencies] spin-factors-derive = { path = "../factors-derive", features = ["expander"] } +spin-factor-wasi = { path = "../factor-wasi" } [lints] workspace = true diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index af86ebb3d..317200622 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -12,7 +12,7 @@ pub type Linker = wasmtime::component::Linker<: pub type ModuleLinker = wasmtime::Linker<::InstanceState>; pub trait Factor: Any + Sized { - type InstancePreparer: FactorInstancePreparer; + type InstancePreparer: InstancePreparer; type InstanceState; /// Initializes this Factor for a runtime. This will be called exactly once @@ -71,14 +71,14 @@ where { } impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { - pub fn builder_mut(&mut self) -> Result<&mut T::InstancePreparer> { - let err_msg = match Factors::builder_mut::(self.builders) { - Some(Some(builder)) => return Ok(builder), - Some(None) => "builder not yet prepared", + pub fn instance_preparer_mut(&mut self) -> Result<&mut T::InstancePreparer> { + let err_msg = match Factors::instance_preparer_mut::(self.instance_preparers) { + Some(Some(preparer)) => return Ok(preparer), + Some(None) => "preparer not yet initialized", None => "no such factor", }; Err(Error::msg(format!( - "could not get builder for {ty}: {err_msg}", + "could not get instance preparer for {ty}: {err_msg}", ty = std::any::type_name::() ))) } @@ -86,30 +86,30 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { /// Implemented by `#[derive(SpinFactors)]` pub trait SpinFactors: Sized { - type Builders; + type InstancePreparers; type InstanceState: Send + 'static; #[doc(hidden)] - unsafe fn factor_builder_offset() -> Option; + unsafe fn instance_preparer_offset() -> Option; #[doc(hidden)] - unsafe fn factor_data_offset() -> Option; + unsafe fn instance_state_offset() -> Option; - fn data_getter() -> Option> { - let offset = unsafe { Self::factor_data_offset::()? }; + fn instance_state_getter() -> Option> { + let offset = unsafe { Self::instance_state_offset::()? }; Some(Getter { offset, _phantom: PhantomData, }) } - fn data_getter2( + fn instance_state_getter2( ) -> Option> { - let offset1 = unsafe { Self::factor_data_offset::()? }; - let offset2 = unsafe { Self::factor_data_offset::()? }; + let offset1 = unsafe { Self::instance_state_offset::()? }; + let offset2 = unsafe { Self::instance_state_offset::()? }; assert_ne!( offset1, offset2, - "data_getter2 with same factor twice would alias" + "instance_state_getter2 with same factor twice would alias" ); Some(Getter2 { offset1, @@ -118,12 +118,12 @@ pub trait SpinFactors: Sized { }) } - fn builder_mut( - builders: &mut Self::Builders, + fn instance_preparer_mut( + preparers: &mut Self::InstancePreparers, ) -> Option> { unsafe { - let offset = Self::factor_builder_offset::()?; - let ptr = builders as *mut Self::Builders; + let offset = Self::instance_preparer_offset::()?; + let ptr = preparers as *mut Self::InstancePreparers; let opt = &mut *ptr.add(offset).cast::>(); Some(opt.as_mut()) } @@ -180,27 +180,27 @@ impl Clone for Getter2 { } impl Copy for Getter2 {} -pub trait FactorInstancePreparer: Sized { +pub trait InstancePreparer: Sized { fn new(_factor: &T, _ctx: PrepareContext) -> Result; fn prepare(self) -> Result; } pub struct PrepareContext<'a, Factors: SpinFactors> { - builders: &'a mut Factors::Builders, + instance_preparers: &'a mut Factors::InstancePreparers, // TODO: component: &'a AppComponent, } impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { #[doc(hidden)] - pub fn new(builders: &'a mut Factors::Builders) -> Self { - Self { builders } + pub fn new(instance_preparers: &'a mut Factors::InstancePreparers) -> Self { + Self { instance_preparers } } } -pub type DefaultBuilder = (); +pub type DefaultInstancePreparer = (); -impl FactorInstancePreparer for DefaultBuilder +impl InstancePreparer for DefaultInstancePreparer where T::InstanceState: Default, { diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 2deb8cc24..2c8117306 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,21 +1,22 @@ +use spin_factor_wasi::WasiFactor; use spin_factors::SpinFactors; #[derive(SpinFactors)] -struct Factors {} +struct Factors { + wasi: WasiFactor, +} fn main() -> anyhow::Result<()> { - let mut factors = Factors {}; - let engine = wasmtime::Engine::default(); let mut linker = wasmtime::component::Linker::new(&engine); - factors.init(&mut linker).unwrap(); - let factors = Factors { - // wasi: WasiFactor, + let mut factors = Factors { + wasi: WasiFactor, // outbound_networking_factor: OutboundNetworkingFactor, // outbound_http_factor: OutboundHttpFactor, }; - let data = factors.build_data().unwrap(); + factors.init(&mut linker).unwrap(); + let data = factors.build_store_data().unwrap(); let mut store = wasmtime::Store::new(&engine, data); let component = wasmtime::component::Component::new(&engine, b"(component)").unwrap(); From 8cf5e54fc87a6d5db895d5a9b54681edbf165210 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 21 May 2024 17:19:05 -0400 Subject: [PATCH 004/195] factors: trait InstancePreparer -> FactorInstancePreparer Signed-off-by: Lann Martin --- crates/factor-wasi/src/lib.rs | 10 ++-- crates/factor-wasi/src/preview1.rs | 8 +-- crates/factors-derive/src/lib.rs | 4 +- crates/factors/src/lib.rs | 82 +++++++++++++++--------------- 4 files changed, 54 insertions(+), 50 deletions(-) diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 02a135e03..c6a90f177 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,12 +1,14 @@ pub mod preview1; -use spin_factors::{Factor, InitContext, InstancePreparer, PrepareContext, Result, SpinFactors}; +use spin_factors::{ + Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, +}; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; pub struct WasiFactor; impl Factor for WasiFactor { - type InstancePreparer = Builder; + type InstancePreparer = InstancePreparer; type InstanceState = InstanceState; fn init(&mut self, mut ctx: InitContext) -> Result<()> { @@ -42,11 +44,11 @@ impl Factor for WasiFactor { } } -pub struct Builder { +pub struct InstancePreparer { wasi_ctx: WasiCtxBuilder, } -impl InstancePreparer for Builder { +impl FactorInstancePreparer for InstancePreparer { fn new( _factor: &WasiFactor, _ctx: PrepareContext, diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 4c4cef5a3..1909cf35c 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,12 +1,12 @@ use spin_factors::{ - Factor, InstancePreparer, ModuleInitContext, PrepareContext, Result, SpinFactors, + Factor, FactorInstancePreparer, ModuleInitContext, PrepareContext, Result, SpinFactors, }; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; pub struct WasiPreview1Factor; impl Factor for WasiPreview1Factor { - type InstancePreparer = Builder; + type InstancePreparer = InstancePreparer; type InstanceState = WasiP1Ctx; fn module_init( @@ -17,11 +17,11 @@ impl Factor for WasiPreview1Factor { } } -pub struct Builder { +pub struct InstancePreparer { wasi_ctx: WasiCtxBuilder, } -impl InstancePreparer for Builder { +impl FactorInstancePreparer for InstancePreparer { fn new( _factor: &WasiPreview1Factor, _ctx: PrepareContext, diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 18aed6d1c..d3126050b 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -100,7 +100,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { }; #( preparers.#factor_names = Some( - #factors_path::InstancePreparer::<#factor_types>::new::<#name>( + #factors_path::FactorInstancePreparer::<#factor_types>::new::<#name>( &self.#factor_names, #factors_path::PrepareContext::new(&mut preparers), )? @@ -108,7 +108,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { )* Ok(#state_name { #( - #factor_names: #factors_path::InstancePreparer::<#factor_types>::prepare( + #factor_names: #factors_path::FactorInstancePreparer::<#factor_types>::prepare( preparers.#factor_names.unwrap() )?, )* diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 317200622..d42de76f8 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -12,7 +12,7 @@ pub type Linker = wasmtime::component::Linker<: pub type ModuleLinker = wasmtime::Linker<::InstanceState>; pub trait Factor: Any + Sized { - type InstancePreparer: InstancePreparer; + type InstancePreparer: FactorInstancePreparer; type InstanceState; /// Initializes this Factor for a runtime. This will be called exactly once @@ -35,9 +35,12 @@ pub trait Factor: Any + Sized { } } +type GetDataFn = + fn(&mut ::InstanceState) -> &mut ::InstanceState; + pub struct FactorInitContext<'a, Factors: SpinFactors, Fact: Factor, Linker> { linker: &'a mut Linker, - get_data: fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, + get_data: GetDataFn, } pub type InitContext<'a, Factors, Fact> = FactorInitContext<'a, Factors, Fact, Linker>; @@ -47,10 +50,7 @@ pub type ModuleInitContext<'a, Factors, Fact> = impl<'a, Factors: SpinFactors, Fact: Factor, Linker> FactorInitContext<'a, Factors, Fact, Linker> { #[doc(hidden)] - pub fn new( - linker: &'a mut Linker, - get_data: fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, - ) -> Self { + pub fn new(linker: &'a mut Linker, get_data: GetDataFn) -> Self { Self { linker, get_data } } @@ -58,6 +58,10 @@ impl<'a, Factors: SpinFactors, Fact: Factor, Linker> FactorInitContext<'a, Facto self.linker } + pub fn get_data_fn(&self) -> GetDataFn { + self.get_data + } + pub fn link_bindings( &mut self, add_to_linker: impl Fn( @@ -70,7 +74,23 @@ where { } } +pub trait FactorInstancePreparer: Sized { + fn new(factor: &T, _ctx: PrepareContext) -> Result; + + fn prepare(self) -> Result; +} + +pub struct PrepareContext<'a, Factors: SpinFactors> { + instance_preparers: &'a mut Factors::InstancePreparers, + // TODO: component: &'a AppComponent, +} + impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { + #[doc(hidden)] + pub fn new(instance_preparers: &'a mut Factors::InstancePreparers) -> Self { + Self { instance_preparers } + } + pub fn instance_preparer_mut(&mut self) -> Result<&mut T::InstancePreparer> { let err_msg = match Factors::instance_preparer_mut::(self.instance_preparers) { Some(Some(preparer)) => return Ok(preparer), @@ -84,6 +104,22 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { } } +pub type DefaultInstancePreparer = (); + +impl FactorInstancePreparer for DefaultInstancePreparer +where + T::InstanceState: Default, +{ + fn new(factor: &T, ctx: PrepareContext) -> Result { + (_, _) = (factor, ctx); + Ok(()) + } + + fn prepare(self) -> Result { + Ok(Default::default()) + } +} + /// Implemented by `#[derive(SpinFactors)]` pub trait SpinFactors: Sized { type InstancePreparers; @@ -179,37 +215,3 @@ impl Clone for Getter2 { } } impl Copy for Getter2 {} - -pub trait InstancePreparer: Sized { - fn new(_factor: &T, _ctx: PrepareContext) -> Result; - - fn prepare(self) -> Result; -} - -pub struct PrepareContext<'a, Factors: SpinFactors> { - instance_preparers: &'a mut Factors::InstancePreparers, - // TODO: component: &'a AppComponent, -} - -impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { - #[doc(hidden)] - pub fn new(instance_preparers: &'a mut Factors::InstancePreparers) -> Self { - Self { instance_preparers } - } -} - -pub type DefaultInstancePreparer = (); - -impl InstancePreparer for DefaultInstancePreparer -where - T::InstanceState: Default, -{ - fn new(factor: &T, ctx: PrepareContext) -> Result { - (_, _) = (factor, ctx); - Ok(()) - } - - fn prepare(self) -> Result { - Ok(Default::default()) - } -} From 69305076d4790dceeb2e873b0e0092b158442afc Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 23 May 2024 08:55:16 -0400 Subject: [PATCH 005/195] factors: Merge InitContext and ModuleInitContext Signed-off-by: Lann Martin --- crates/factor-wasi/src/preview1.rs | 9 ++--- crates/factors-derive/src/lib.rs | 23 ++--------- crates/factors/src/lib.rs | 63 +++++++++++++++++++----------- crates/factors/tests/smoke.rs | 14 ++++++- 4 files changed, 60 insertions(+), 49 deletions(-) diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 1909cf35c..c1b61e1f1 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,5 +1,5 @@ use spin_factors::{ - Factor, FactorInstancePreparer, ModuleInitContext, PrepareContext, Result, SpinFactors, + Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, }; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; @@ -9,11 +9,8 @@ impl Factor for WasiPreview1Factor { type InstancePreparer = InstancePreparer; type InstanceState = WasiP1Ctx; - fn module_init( - &mut self, - mut ctx: ModuleInitContext, - ) -> Result<()> { - ctx.link_bindings(wasmtime_wasi::preview1::add_to_linker_async) + fn init(&mut self, mut ctx: InitContext) -> Result<()> { + ctx.link_module_bindings(wasmtime_wasi::preview1::add_to_linker_async) } } diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index d3126050b..d4939da2b 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -63,30 +63,15 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { impl #name { pub fn init( &mut self, - linker: &mut #wasmtime::component::Linker<#state_name> + mut linker: Option<&mut #wasmtime::component::Linker<#state_name>>, + mut module_linker: Option<&mut #wasmtime::Linker<#state_name>>, ) -> #Result<()> { #( #Factor::init::( &mut self.#factor_names, #factors_path::InitContext::::new( - linker, - |state| &mut state.#factor_names, - ) - )?; - )* - Ok(()) - } - - #[allow(dead_code)] - pub fn module_init( - &mut self, - linker: &mut #wasmtime::Linker<#state_name> - ) -> #Result<()> { - #( - #Factor::module_init::( - &mut self.#factor_names, - #factors_path::ModuleInitContext::::new( - linker, + linker.as_deref_mut(), + module_linker.as_deref_mut(), |state| &mut state.#factor_names, ) )?; diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index d42de76f8..9f9b9ea7b 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -21,14 +21,6 @@ pub trait Factor: Any + Sized { Ok(()) } - fn module_init( - &mut self, - mut ctx: ModuleInitContext, - ) -> Result<()> { - _ = &mut ctx; - Ok(()) - } - fn validate_app(&self, app: &App) -> Result<()> { _ = app; Ok(()) @@ -38,24 +30,32 @@ pub trait Factor: Any + Sized { type GetDataFn = fn(&mut ::InstanceState) -> &mut ::InstanceState; -pub struct FactorInitContext<'a, Factors: SpinFactors, Fact: Factor, Linker> { - linker: &'a mut Linker, +pub struct InitContext<'a, Factors: SpinFactors, Fact: Factor> { + linker: Option<&'a mut Linker>, + module_linker: Option<&'a mut ModuleLinker>, get_data: GetDataFn, } -pub type InitContext<'a, Factors, Fact> = FactorInitContext<'a, Factors, Fact, Linker>; - -pub type ModuleInitContext<'a, Factors, Fact> = - FactorInitContext<'a, Factors, Fact, ModuleLinker>; - -impl<'a, Factors: SpinFactors, Fact: Factor, Linker> FactorInitContext<'a, Factors, Fact, Linker> { +impl<'a, Factors: SpinFactors, Fact: Factor> InitContext<'a, Factors, Fact> { #[doc(hidden)] - pub fn new(linker: &'a mut Linker, get_data: GetDataFn) -> Self { - Self { linker, get_data } + pub fn new( + linker: Option<&'a mut Linker>, + module_linker: Option<&'a mut ModuleLinker>, + get_data: GetDataFn, + ) -> Self { + Self { + linker, + module_linker, + get_data, + } + } + + pub fn linker(&mut self) -> Option<&mut Linker> { + self.linker.as_deref_mut() } - pub fn linker(&mut self) -> &mut Linker { - self.linker + pub fn module_linker(&mut self) -> Option<&mut ModuleLinker> { + self.module_linker.as_deref_mut() } pub fn get_data_fn(&self) -> GetDataFn { @@ -65,12 +65,31 @@ impl<'a, Factors: SpinFactors, Fact: Factor, Linker> FactorInitContext<'a, Facto pub fn link_bindings( &mut self, add_to_linker: impl Fn( - &mut Linker, + &mut Linker, fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, ) -> Result<()>, ) -> Result<()> where { - add_to_linker(self.linker, self.get_data) + if let Some(linker) = self.linker.as_deref_mut() { + add_to_linker(linker, self.get_data) + } else { + Ok(()) + } + } + + pub fn link_module_bindings( + &mut self, + add_to_linker: impl Fn( + &mut ModuleLinker, + fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, + ) -> Result<()>, + ) -> Result<()> +where { + if let Some(linker) = self.module_linker.as_deref_mut() { + add_to_linker(linker, self.get_data) + } else { + Ok(()) + } } } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 2c8117306..12e67fba0 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,25 +1,35 @@ -use spin_factor_wasi::WasiFactor; +use spin_factor_wasi::{preview1::WasiPreview1Factor, WasiFactor}; use spin_factors::SpinFactors; #[derive(SpinFactors)] struct Factors { wasi: WasiFactor, + wasip1: WasiPreview1Factor, } fn main() -> anyhow::Result<()> { let engine = wasmtime::Engine::default(); let mut linker = wasmtime::component::Linker::new(&engine); + let mut module_linker = wasmtime::Linker::new(&engine); let mut factors = Factors { wasi: WasiFactor, + wasip1: WasiPreview1Factor, // outbound_networking_factor: OutboundNetworkingFactor, // outbound_http_factor: OutboundHttpFactor, }; - factors.init(&mut linker).unwrap(); + factors + .init(Some(&mut linker), Some(&mut module_linker)) + .unwrap(); let data = factors.build_store_data().unwrap(); let mut store = wasmtime::Store::new(&engine, data); + let component = wasmtime::component::Component::new(&engine, b"(component)").unwrap(); let _instance = linker.instantiate(&mut store, &component).unwrap(); + + let module = wasmtime::Module::new(&engine, b"(module)").unwrap(); + let _module_instance = module_linker.instantiate(&mut store, &module).unwrap(); + Ok(()) } From 2f580a9776f80c9a4f93cc4c6ce6759b00c09575 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 23 May 2024 15:28:48 -0400 Subject: [PATCH 006/195] factors: Add comments Signed-off-by: Lann Martin --- crates/factors/src/lib.rs | 23 ++++++++++++++++++++++- crates/factors/tests/smoke.rs | 9 +++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 9f9b9ea7b..33a031e81 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -12,15 +12,22 @@ pub type Linker = wasmtime::component::Linker<: pub type ModuleLinker = wasmtime::Linker<::InstanceState>; pub trait Factor: Any + Sized { + /// The [`FactorInstancePreparer`] for this factor. type InstancePreparer: FactorInstancePreparer; + + /// The per-instance state for this factor, constructed by a + /// [`FactorInstancePreparer`] and available to any host-provided imports + /// defined by this factor. type InstanceState; - /// Initializes this Factor for a runtime. This will be called exactly once + /// Initializes this Factor for a runtime. This should be called at most once. fn init(&mut self, mut ctx: InitContext) -> Result<()> { _ = &mut ctx; Ok(()) } + /// Performs factor-specific validation of the given [`App`]`. This may be + /// called before, after, or instead of `init`. fn validate_app(&self, app: &App) -> Result<()> { _ = app; Ok(()) @@ -30,6 +37,8 @@ pub trait Factor: Any + Sized { type GetDataFn = fn(&mut ::InstanceState) -> &mut ::InstanceState; +/// An InitContext is passed to [`Factor::init`], giving access to the global +/// common [`wasmtime::component::Linker`]. pub struct InitContext<'a, Factors: SpinFactors, Fact: Factor> { linker: Option<&'a mut Linker>, module_linker: Option<&'a mut ModuleLinker>, @@ -94,11 +103,16 @@ where { } pub trait FactorInstancePreparer: Sized { + /// Returns a new instance of this preparer for the given [`Factor`]. fn new(factor: &T, _ctx: PrepareContext) -> Result; + /// Returns a new instance of the associated [`Factor::InstanceState`]. fn prepare(self) -> Result; } +/// A PrepareContext is passed to [`FactorInstancePreparer::new`], giving access +/// to any already-initialized [`FactorInstancePreparer`]s, allowing for +/// inter-[`Factor`] dependencies. pub struct PrepareContext<'a, Factors: SpinFactors> { instance_preparers: &'a mut Factors::InstancePreparers, // TODO: component: &'a AppComponent, @@ -110,6 +124,11 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { Self { instance_preparers } } + /// Returns a already-initialized preparer for the given [`Factor`]. + /// + /// Fails if the current [`SpinFactors`] does not include the given + /// [`Factor`] or if the given [`Factor`]'s preparer has not been + /// initialized yet (because it is sequenced after this factor). pub fn instance_preparer_mut(&mut self) -> Result<&mut T::InstancePreparer> { let err_msg = match Factors::instance_preparer_mut::(self.instance_preparers) { Some(Some(preparer)) => return Ok(preparer), @@ -123,6 +142,8 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { } } +/// DefaultInstancePreparer can be used as a [`FactorInstancePreparer`] to +/// produce a [`Default`] [`Factor::InstanceState`]. pub type DefaultInstancePreparer = (); impl FactorInstancePreparer for DefaultInstancePreparer diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 12e67fba0..5cb72dd1d 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -8,16 +8,17 @@ struct Factors { } fn main() -> anyhow::Result<()> { - let engine = wasmtime::Engine::default(); - let mut linker = wasmtime::component::Linker::new(&engine); - let mut module_linker = wasmtime::Linker::new(&engine); - let mut factors = Factors { wasi: WasiFactor, wasip1: WasiPreview1Factor, // outbound_networking_factor: OutboundNetworkingFactor, // outbound_http_factor: OutboundHttpFactor, }; + + let engine = wasmtime::Engine::default(); + let mut linker = wasmtime::component::Linker::new(&engine); + let mut module_linker = wasmtime::Linker::new(&engine); + factors .init(Some(&mut linker), Some(&mut module_linker)) .unwrap(); From d0ea626b7f5e1f643db898e47a4f522c668f5b04 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 24 May 2024 16:34:45 -0400 Subject: [PATCH 007/195] factors: Add configure_app hook Signed-off-by: Lann Martin --- Cargo.lock | 14 +++ crates/app/src/lib.rs | 8 ++ crates/factor-outbound-networking/Cargo.toml | 16 +++ crates/factor-outbound-networking/src/lib.rs | 86 +++++++++++++ crates/factor-wasi/Cargo.toml | 3 +- crates/factor-wasi/src/lib.rs | 106 +++++++++++++++- crates/factor-wasi/src/preview1.rs | 4 +- crates/factors-derive/src/lib.rs | 45 ++++++- crates/factors/Cargo.toml | 2 + crates/factors/src/lib.rs | 126 ++++++++++++++----- crates/factors/tests/smoke.rs | 27 +++- 11 files changed, 393 insertions(+), 44 deletions(-) create mode 100644 crates/factor-outbound-networking/Cargo.toml create mode 100644 crates/factor-outbound-networking/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 4159011a4..b5a3d5c01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7558,11 +7558,23 @@ dependencies = [ "toml 0.5.11", ] +[[package]] +name = "spin-factor-outbound-networking" +version = "2.6.0-pre0" +dependencies = [ + "anyhow", + "ipnet", + "spin-factor-wasi", + "spin-factors", + "spin-outbound-networking", +] + [[package]] name = "spin-factor-wasi" version = "2.6.0-pre0" dependencies = [ "anyhow", + "cap-primitives 3.0.0", "spin-factors", "wasmtime-wasi", ] @@ -7572,7 +7584,9 @@ name = "spin-factors" version = "2.6.0-pre0" dependencies = [ "anyhow", + "serde_json", "spin-app", + "spin-factor-outbound-networking", "spin-factor-wasi", "spin-factors-derive", "wasmtime", diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 70d6a59b6..97077032d 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -296,6 +296,14 @@ impl<'a, L> AppComponent<'a, L> { &self.locked.source } + /// Returns an iterator of environment variable (key, value) pairs. + pub fn environment(&self) -> impl IntoIterator { + self.locked + .env + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + } + /// Returns an iterator of [`ContentPath`]s for this component's configured /// "directory mounts". pub fn files(&self) -> std::slice::Iter { diff --git a/crates/factor-outbound-networking/Cargo.toml b/crates/factor-outbound-networking/Cargo.toml new file mode 100644 index 000000000..d4be54172 --- /dev/null +++ b/crates/factor-outbound-networking/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "spin-factor-outbound-networking" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1" +ipnet = "2.9.0" +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors = { path = "../factors" } +# TODO: merge with this crate +spin-outbound-networking = { path = "../outbound-networking" } + +[lints] +workspace = true diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs new file mode 100644 index 000000000..d994296a8 --- /dev/null +++ b/crates/factor-outbound-networking/src/lib.rs @@ -0,0 +1,86 @@ +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Context; +use spin_factor_wasi::WasiFactor; +use spin_factors::{Factor, FactorInstancePreparer, Result, SpinFactors}; +use spin_outbound_networking::{AllowedHostsConfig, HostConfig, PortConfig, ALLOWED_HOSTS_KEY}; + +pub struct OutboundNetworkingFactor; + +impl Factor for OutboundNetworkingFactor { + type AppConfig = AppConfig; + type InstancePreparer = InstancePreparer; + type InstanceState = (); + + fn configure_app( + &self, + app: &spin_factors::App, + _ctx: spin_factors::ConfigureAppContext, + ) -> Result { + let mut cfg = AppConfig::default(); + // TODO: resolve resolver resolution + let resolver = Default::default(); + for component in app.components() { + if let Some(hosts) = component.get_metadata(ALLOWED_HOSTS_KEY)? { + let allowed_hosts = AllowedHostsConfig::parse(&hosts, &resolver)?; + cfg.component_allowed_hosts + .insert(component.id().to_string(), Arc::new(allowed_hosts)); + } + } + Ok(cfg) + } +} + +#[derive(Default)] +pub struct AppConfig { + component_allowed_hosts: HashMap>, +} + +pub struct InstancePreparer { + allowed_hosts: Arc, +} + +impl InstancePreparer { + pub fn allowed_hosts(&self) -> &Arc { + &self.allowed_hosts + } +} + +impl FactorInstancePreparer for InstancePreparer { + fn new( + _factor: &OutboundNetworkingFactor, + app_component: &spin_factors::AppComponent, + mut ctx: spin_factors::PrepareContext, + ) -> Result { + let allowed_hosts = ctx + .app_config::()? + .component_allowed_hosts + .get(app_component.id()) + .context("missing component")? + .clone(); + + // Update Wasi socket allowed ports + let wasi_preparer = ctx.instance_preparer_mut::()?; + match &*allowed_hosts { + AllowedHostsConfig::All => wasi_preparer.inherit_network(), + AllowedHostsConfig::SpecificHosts(configs) => { + for config in configs { + if config.scheme().allows_any() { + match (config.host(), config.port()) { + (HostConfig::Cidr(ip_net), PortConfig::Any) => { + wasi_preparer.socket_allow_ports(*ip_net, 0, None) + } + _ => todo!(), // TODO: complete and validate against existing Network TriggerHooks + } + } + } + } + } + + Ok(Self { allowed_hosts }) + } + + fn prepare(self) -> Result<::InstanceState> { + Ok(()) + } +} diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index c50f2e629..58d4b5ff4 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -5,7 +5,8 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] -anyhow = "1.0" +anyhow = "1" +cap-primitives = "3.0.0" spin-factors = { path = "../factors" } wasmtime-wasi = { workspace = true } diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index c6a90f177..03ca3c14e 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,13 +1,28 @@ pub mod preview1; +use std::path::Path; + +use anyhow::ensure; +use cap_primitives::{ipnet::IpNet, net::Pool}; use spin_factors::{ - Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, + AppComponent, Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, }; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; -pub struct WasiFactor; +pub struct WasiFactor { + files_mounter: Box, +} + +impl WasiFactor { + pub fn new(files_mounter: impl FilesMounter + 'static) -> Self { + Self { + files_mounter: Box::new(files_mounter), + } + } +} impl Factor for WasiFactor { + type AppConfig = (); type InstancePreparer = InstancePreparer; type InstanceState = InstanceState; @@ -44,28 +59,107 @@ impl Factor for WasiFactor { } } +pub trait FilesMounter { + fn mount_files(&self, app_component: &AppComponent, ctx: MountFilesContext) -> Result<()>; +} + +pub struct DummyFilesMounter; + +impl FilesMounter for DummyFilesMounter { + fn mount_files(&self, app_component: &AppComponent, _ctx: MountFilesContext) -> Result<()> { + ensure!( + app_component.files().next().is_none(), + "DummyFilesMounter can't actually mount files" + ); + Ok(()) + } +} + +pub struct MountFilesContext<'a> { + wasi_ctx: &'a mut WasiCtxBuilder, +} + +impl<'a> MountFilesContext<'a> { + pub fn preopened_dir( + &mut self, + host_path: impl AsRef, + guest_path: impl AsRef, + writable: bool, + ) -> Result<()> { + use wasmtime_wasi::{DirPerms, FilePerms}; + let (dir_perms, file_perms) = if writable { + (DirPerms::all(), FilePerms::all()) + } else { + (DirPerms::READ, FilePerms::READ) + }; + self.wasi_ctx + .preopened_dir(host_path, guest_path, dir_perms, file_perms)?; + Ok(()) + } +} + pub struct InstancePreparer { wasi_ctx: WasiCtxBuilder, + socket_allow_ports: Pool, } impl FactorInstancePreparer for InstancePreparer { + // NOTE: Replaces WASI parts of AppComponent::apply_store_config fn new( - _factor: &WasiFactor, + factor: &WasiFactor, + app_component: &AppComponent, _ctx: PrepareContext, ) -> Result { + let mut wasi_ctx = WasiCtxBuilder::new(); + + // Apply environment variables + for (key, val) in app_component.environment() { + wasi_ctx.env(key, val); + } + + // Mount files + let mount_ctx = MountFilesContext { + wasi_ctx: &mut wasi_ctx, + }; + factor.files_mounter.mount_files(app_component, mount_ctx)?; + Ok(Self { - wasi_ctx: WasiCtxBuilder::new(), + wasi_ctx, + socket_allow_ports: Default::default(), }) } - fn prepare(mut self) -> Result { + fn prepare(self) -> Result { + let Self { + mut wasi_ctx, + socket_allow_ports, + } = self; + + // Enforce socket_allow_ports + wasi_ctx.socket_addr_check(move |addr, _| socket_allow_ports.check_addr(addr).is_ok()); + Ok(InstanceState { - ctx: self.wasi_ctx.build(), + ctx: wasi_ctx.build(), table: Default::default(), }) } } +impl InstancePreparer { + pub fn inherit_network(&mut self) { + self.wasi_ctx.inherit_network(); + } + + pub fn socket_allow_ports(&mut self, ip_net: IpNet, ports_start: u16, ports_end: Option) { + self.socket_allow_ports.insert_ip_net_port_range( + ip_net, + ports_start, + ports_end, + cap_primitives::ambient_authority(), + ); + } +} + pub struct InstanceState { ctx: WasiCtx, table: ResourceTable, diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index c1b61e1f1..a4441a018 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,11 +1,12 @@ use spin_factors::{ - Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, + AppComponent, Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, }; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; pub struct WasiPreview1Factor; impl Factor for WasiPreview1Factor { + type AppConfig = (); type InstancePreparer = InstancePreparer; type InstanceState = WasiP1Ctx; @@ -21,6 +22,7 @@ pub struct InstancePreparer { impl FactorInstancePreparer for InstancePreparer { fn new( _factor: &WasiPreview1Factor, + _app_component: &AppComponent, _ctx: PrepareContext, ) -> Result { Ok(Self { diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index d4939da2b..456068c2c 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -20,6 +20,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let name = &input.ident; let vis = &input.vis; + let app_configs_name = format_ident!("{name}AppConfigs"); let preparers_name = format_ident!("{name}InstancePreparers"); let state_name = format_ident!("{name}InstanceState"); @@ -57,6 +58,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let Factor = quote!(#factors_path::Factor); let Result = quote!(#factors_path::Result); let wasmtime = quote!(#factors_path::wasmtime); + let ConfiguredApp = quote!(#factors_path::ConfiguredApp); let TypeId = quote!(::std::any::TypeId); Ok(quote! { @@ -79,7 +81,26 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { Ok(()) } - pub fn build_store_data(&self) -> #Result<#state_name> { + pub fn configure_app(&self, app: #factors_path::App) -> #Result<#ConfiguredApp> { + let mut app_configs = #app_configs_name { + #( #factor_names: None, )* + }; + #( + app_configs.#factor_names = Some( + #Factor::configure_app( + &self.#factor_names, + &app, + #factors_path::ConfigureAppContext::::new(&app_configs), + )? + ); + )* + Ok(#ConfiguredApp::new(app, app_configs)) + } + + pub fn build_store_data(&self, configured_app: &#ConfiguredApp, component_id: &str) -> #Result<#state_name> { + let app_component = configured_app.app().get_component(component_id).ok_or_else(|| { + #factors_path::Error::msg(format!("unknown component {component_id:?}")) + })?; let mut preparers = #preparers_name { #( #factor_names: None, )* }; @@ -87,14 +108,15 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { preparers.#factor_names = Some( #factors_path::FactorInstancePreparer::<#factor_types>::new::<#name>( &self.#factor_names, - #factors_path::PrepareContext::new(&mut preparers), + &app_component, + #factors_path::PrepareContext::new(configured_app, &mut preparers), )? ); )* Ok(#state_name { #( #factor_names: #factors_path::FactorInstancePreparer::<#factor_types>::prepare( - preparers.#factor_names.unwrap() + preparers.#factor_names.unwrap(), )?, )* }) @@ -103,6 +125,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } impl #factors_path::SpinFactors for #name { + type AppConfigs = #app_configs_name; type InstancePreparers = #preparers_name; type InstanceState = #state_name; @@ -126,6 +149,22 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { None } + + fn app_config(app_configs: &Self::AppConfigs) -> Option<&T::AppConfig> { + let type_id = #TypeId::of::(); + #( + if type_id == #TypeId::of::<#factor_types>() { + return Some(unsafe { std::mem::transmute(&app_configs.#factor_names) }); + } + )* + None + } + } + + #vis struct #app_configs_name { + #( + pub #factor_names: Option<<#factor_types as #Factor>::AppConfig>, + )* } #vis struct #preparers_name { diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 6e3e75d81..779b95ba0 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -11,7 +11,9 @@ spin-factors-derive = { path = "../factors-derive" } wasmtime = { workspace = true } [dev-dependencies] +serde_json = "1.0" spin-factors-derive = { path = "../factors-derive", features = ["expander"] } +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-wasi = { path = "../factor-wasi" } [lints] diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 33a031e81..d917c392e 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -1,6 +1,6 @@ use std::{any::Any, marker::PhantomData}; -use spin_app::App; +use anyhow::Context; pub use spin_factors_derive::SpinFactors; pub use wasmtime; @@ -11,7 +11,16 @@ pub type Result = std::result::Result; pub type Linker = wasmtime::component::Linker<::InstanceState>; pub type ModuleLinker = wasmtime::Linker<::InstanceState>; +// Temporary wrappers while refactoring +pub type App = spin_app::App<'static, spin_app::InertLoader>; +pub type AppComponent<'a> = spin_app::AppComponent<'a, spin_app::InertLoader>; + pub trait Factor: Any + Sized { + /// App configuration for this factor. + /// + /// See [`Factor::configure_app`]. + type AppConfig: Default; + /// The [`FactorInstancePreparer`] for this factor. type InstancePreparer: FactorInstancePreparer; @@ -20,17 +29,25 @@ pub trait Factor: Any + Sized { /// defined by this factor. type InstanceState; - /// Initializes this Factor for a runtime. This should be called at most once. + /// Initializes this Factor for a runtime. This will be called at most once, + /// before any call to [`FactorInstancePreparer::new`] fn init(&mut self, mut ctx: InitContext) -> Result<()> { _ = &mut ctx; Ok(()) } - /// Performs factor-specific validation of the given [`App`]`. This may be - /// called before, after, or instead of `init`. - fn validate_app(&self, app: &App) -> Result<()> { + /// Performs factor-specific validation and configuration for the given + /// [`App`] and [`RuntimeConfig`]. A runtime may - but is not required to - + /// reuse the returned config across multiple instances. Note that this may + /// be called without any call to `init` in cases where only validation is + /// needed. + fn configure_app( + &self, + app: &App, + _ctx: ConfigureAppContext, + ) -> Result { _ = app; - Ok(()) + Ok(Default::default()) } } @@ -102,9 +119,28 @@ where { } } +pub struct ConfigureAppContext<'a, Factors: SpinFactors> { + app_configs: &'a Factors::AppConfigs, +} + +impl<'a, Factors: SpinFactors> ConfigureAppContext<'a, Factors> { + #[doc(hidden)] + pub fn new(app_configs: &'a Factors::AppConfigs) -> Self { + Self { app_configs } + } + + pub fn app_config(&self) -> Result<&T::AppConfig> { + Factors::app_config::(self.app_configs).context("no such factor") + } +} + pub trait FactorInstancePreparer: Sized { /// Returns a new instance of this preparer for the given [`Factor`]. - fn new(factor: &T, _ctx: PrepareContext) -> Result; + fn new( + factor: &T, + app_component: &AppComponent, + _ctx: PrepareContext, + ) -> Result; /// Returns a new instance of the associated [`Factor::InstanceState`]. fn prepare(self) -> Result; @@ -114,14 +150,24 @@ pub trait FactorInstancePreparer: Sized { /// to any already-initialized [`FactorInstancePreparer`]s, allowing for /// inter-[`Factor`] dependencies. pub struct PrepareContext<'a, Factors: SpinFactors> { + configured_app: &'a ConfiguredApp, instance_preparers: &'a mut Factors::InstancePreparers, - // TODO: component: &'a AppComponent, } impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { #[doc(hidden)] - pub fn new(instance_preparers: &'a mut Factors::InstancePreparers) -> Self { - Self { instance_preparers } + pub fn new( + configured_app: &'a ConfiguredApp, + instance_preparers: &'a mut Factors::InstancePreparers, + ) -> Self { + Self { + configured_app, + instance_preparers, + } + } + + pub fn app_config(&self) -> Result<&T::AppConfig> { + self.configured_app.app_config::() } /// Returns a already-initialized preparer for the given [`Factor`]. @@ -130,28 +176,27 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { /// [`Factor`] or if the given [`Factor`]'s preparer has not been /// initialized yet (because it is sequenced after this factor). pub fn instance_preparer_mut(&mut self) -> Result<&mut T::InstancePreparer> { - let err_msg = match Factors::instance_preparer_mut::(self.instance_preparers) { - Some(Some(preparer)) => return Ok(preparer), - Some(None) => "preparer not yet initialized", - None => "no such factor", - }; - Err(Error::msg(format!( - "could not get instance preparer for {ty}: {err_msg}", - ty = std::any::type_name::() - ))) + Factors::instance_preparer_mut::(self.instance_preparers) + .and_then(|maybe_preparer| maybe_preparer.context("preparer not yet initialized")) + .with_context(|| { + format!( + "could not get instance preparer for {}", + std::any::type_name::() + ) + }) } } -/// DefaultInstancePreparer can be used as a [`FactorInstancePreparer`] to -/// produce a [`Default`] [`Factor::InstanceState`]. -pub type DefaultInstancePreparer = (); - -impl FactorInstancePreparer for DefaultInstancePreparer +impl FactorInstancePreparer for () where T::InstanceState: Default, { - fn new(factor: &T, ctx: PrepareContext) -> Result { - (_, _) = (factor, ctx); + fn new( + factor: &T, + app_component: &AppComponent, + _ctx: PrepareContext, + ) -> Result { + (_, _) = (factor, app_component); Ok(()) } @@ -160,8 +205,29 @@ where } } +pub struct ConfiguredApp { + app: App, + app_configs: Factors::AppConfigs, +} + +impl ConfiguredApp { + #[doc(hidden)] + pub fn new(app: App, app_configs: Factors::AppConfigs) -> Self { + Self { app, app_configs } + } + + pub fn app(&self) -> &App { + &self.app + } + + pub fn app_config(&self) -> Result<&T::AppConfig> { + Factors::app_config::(&self.app_configs).context("no such factor") + } +} + /// Implemented by `#[derive(SpinFactors)]` pub trait SpinFactors: Sized { + type AppConfigs; type InstancePreparers; type InstanceState: Send + 'static; @@ -171,6 +237,8 @@ pub trait SpinFactors: Sized { #[doc(hidden)] unsafe fn instance_state_offset() -> Option; + fn app_config(app_configs: &Self::AppConfigs) -> Option<&T::AppConfig>; + fn instance_state_getter() -> Option> { let offset = unsafe { Self::instance_state_offset::()? }; Some(Getter { @@ -196,12 +264,12 @@ pub trait SpinFactors: Sized { fn instance_preparer_mut( preparers: &mut Self::InstancePreparers, - ) -> Option> { + ) -> Result> { unsafe { - let offset = Self::instance_preparer_offset::()?; + let offset = Self::instance_preparer_offset::().context("no such factor")?; let ptr = preparers as *mut Self::InstancePreparers; let opt = &mut *ptr.add(offset).cast::>(); - Some(opt.as_mut()) + Ok(opt.as_mut()) } } } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 5cb72dd1d..e1f39f2c5 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,20 +1,37 @@ -use spin_factor_wasi::{preview1::WasiPreview1Factor, WasiFactor}; +use spin_app::App; +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_wasi::{preview1::WasiPreview1Factor, DummyFilesMounter, WasiFactor}; use spin_factors::SpinFactors; #[derive(SpinFactors)] struct Factors { wasi: WasiFactor, wasip1: WasiPreview1Factor, + outbound_networking_factor: OutboundNetworkingFactor, } fn main() -> anyhow::Result<()> { let mut factors = Factors { - wasi: WasiFactor, + wasi: WasiFactor::new(DummyFilesMounter), wasip1: WasiPreview1Factor, - // outbound_networking_factor: OutboundNetworkingFactor, + outbound_networking_factor: OutboundNetworkingFactor, // outbound_http_factor: OutboundHttpFactor, }; + let locked = serde_json::from_value(serde_json::json!({ + "spin_locked_version": 1, + "triggers": [], + "components": [{ + "id": "test", + "source": { + "content_type": "application/wasm", + "content": {"inline": "KGNvbXBvbmVudCk="} + } + }] + })) + .unwrap(); + let app = App::inert(locked); + let engine = wasmtime::Engine::default(); let mut linker = wasmtime::component::Linker::new(&engine); let mut module_linker = wasmtime::Linker::new(&engine); @@ -22,7 +39,9 @@ fn main() -> anyhow::Result<()> { factors .init(Some(&mut linker), Some(&mut module_linker)) .unwrap(); - let data = factors.build_store_data().unwrap(); + + let configured_app = factors.configure_app(app).unwrap(); + let data = factors.build_store_data(&configured_app, "test").unwrap(); let mut store = wasmtime::Store::new(&engine, data); From a081459f70911514d2625ef94749ee955ae3722a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 31 May 2024 10:43:52 -0400 Subject: [PATCH 008/195] VariablesFactor wip Signed-off-by: Lann Martin --- Cargo.lock | 1 + crates/factor-outbound-networking/Cargo.toml | 3 + crates/factor-outbound-networking/src/lib.rs | 118 ++++++++++------- crates/factor-variables/Cargo.toml | 14 +++ crates/factor-variables/src/lib.rs | 125 ++++++++++++++++++ crates/factor-wasi/src/lib.rs | 126 ++++++++++--------- crates/factor-wasi/src/preview1.rs | 7 +- crates/factors-derive/src/lib.rs | 9 +- crates/factors/Cargo.toml | 1 + crates/factors/src/lib.rs | 106 +++++++++------- crates/factors/tests/smoke.rs | 3 + crates/world/Cargo.toml | 1 + crates/world/src/lib.rs | 2 + 13 files changed, 361 insertions(+), 155 deletions(-) create mode 100644 crates/factor-variables/Cargo.toml create mode 100644 crates/factor-variables/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index b5a3d5c01..032f8dcc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7567,6 +7567,7 @@ dependencies = [ "spin-factor-wasi", "spin-factors", "spin-outbound-networking", + "tracing", ] [[package]] diff --git a/crates/factor-outbound-networking/Cargo.toml b/crates/factor-outbound-networking/Cargo.toml index d4be54172..8eddb55eb 100644 --- a/crates/factor-outbound-networking/Cargo.toml +++ b/crates/factor-outbound-networking/Cargo.toml @@ -6,11 +6,14 @@ edition = { workspace = true } [dependencies] anyhow = "1" +futures-util = "0.3" ipnet = "2.9.0" +spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } spin-factors = { path = "../factors" } # TODO: merge with this crate spin-outbound-networking = { path = "../outbound-networking" } +tracing = { workspace = true } [lints] workspace = true diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index d994296a8..39013b806 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -1,9 +1,16 @@ use std::{collections::HashMap, sync::Arc}; use anyhow::Context; +use futures_util::{ + future::{BoxFuture, Shared}, + FutureExt, +}; +use spin_factor_variables::VariablesFactor; use spin_factor_wasi::WasiFactor; -use spin_factors::{Factor, FactorInstancePreparer, Result, SpinFactors}; -use spin_outbound_networking::{AllowedHostsConfig, HostConfig, PortConfig, ALLOWED_HOSTS_KEY}; +use spin_factors::{ + Factor, FactorInstancePreparer, InstancePreparers, PrepareContext, Result, SpinFactors, +}; +use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; pub struct OutboundNetworkingFactor; @@ -17,70 +24,93 @@ impl Factor for OutboundNetworkingFactor { app: &spin_factors::App, _ctx: spin_factors::ConfigureAppContext, ) -> Result { - let mut cfg = AppConfig::default(); - // TODO: resolve resolver resolution - let resolver = Default::default(); - for component in app.components() { - if let Some(hosts) = component.get_metadata(ALLOWED_HOSTS_KEY)? { - let allowed_hosts = AllowedHostsConfig::parse(&hosts, &resolver)?; - cfg.component_allowed_hosts - .insert(component.id().to_string(), Arc::new(allowed_hosts)); - } - } - Ok(cfg) + // Extract allowed_outbound_hosts for all components + let component_allowed_hosts = app + .components() + .map(|component| { + Ok(( + component.id().to_string(), + component + .get_metadata(ALLOWED_HOSTS_KEY)? + .unwrap_or_default() + .into_boxed_slice() + .into(), + )) + }) + .collect::>()?; + Ok(AppConfig { + component_allowed_hosts, + }) } } #[derive(Default)] pub struct AppConfig { - component_allowed_hosts: HashMap>, + component_allowed_hosts: HashMap>, } -pub struct InstancePreparer { - allowed_hosts: Arc, -} +type AllowedHostsFuture = Shared>>>; -impl InstancePreparer { - pub fn allowed_hosts(&self) -> &Arc { - &self.allowed_hosts - } +pub struct InstancePreparer { + allowed_hosts_future: AllowedHostsFuture, } impl FactorInstancePreparer for InstancePreparer { fn new( - _factor: &OutboundNetworkingFactor, - app_component: &spin_factors::AppComponent, - mut ctx: spin_factors::PrepareContext, + ctx: PrepareContext, + mut preparers: InstancePreparers, ) -> Result { - let allowed_hosts = ctx - .app_config::()? + let hosts = ctx + .app_config() .component_allowed_hosts - .get(app_component.id()) - .context("missing component")? - .clone(); + .get(ctx.app_component().id()) + .cloned() + .context("missing component allowed hosts")?; + let resolver = preparers.get_mut::()?.resolver().clone(); + let allowed_hosts_future = async move { + let prepared = resolver.prepare().await?; + AllowedHostsConfig::parse(&hosts, &prepared) + } + .map(Arc::new) + .boxed() + .shared(); + // let prepared_resolver = resolver.prepare().await?; + // let allowed_hosts = AllowedHostsConfig::parse( + // .context("missing component allowed hosts")?, + // &prepared_resolver, + // )?; // Update Wasi socket allowed ports - let wasi_preparer = ctx.instance_preparer_mut::()?; - match &*allowed_hosts { - AllowedHostsConfig::All => wasi_preparer.inherit_network(), - AllowedHostsConfig::SpecificHosts(configs) => { - for config in configs { - if config.scheme().allows_any() { - match (config.host(), config.port()) { - (HostConfig::Cidr(ip_net), PortConfig::Any) => { - wasi_preparer.socket_allow_ports(*ip_net, 0, None) - } - _ => todo!(), // TODO: complete and validate against existing Network TriggerHooks - } + let wasi_preparer = preparers.get_mut::()?; + let hosts_future = allowed_hosts_future.clone(); + wasi_preparer.outbound_socket_addr_check(move |addr| { + let hosts_future = hosts_future.clone(); + async move { + match &*hosts_future.await { + Ok(allowed_hosts) => { + // TODO: verify this actually works... + spin_outbound_networking::check_url(&addr.to_string(), "*", allowed_hosts) + } + Err(err) => { + // TODO: should this trap (somehow)? + tracing::error!(%err, "allowed_outbound_hosts variable resolution failed"); + false } } } - } - - Ok(Self { allowed_hosts }) + }); + Ok(Self { + allowed_hosts_future, + }) } fn prepare(self) -> Result<::InstanceState> { Ok(()) } } + +impl InstancePreparer { + pub async fn resolve_allowed_hosts(&self) -> Arc> { + self.allowed_hosts_future.clone().await + } +} diff --git a/crates/factor-variables/Cargo.toml b/crates/factor-variables/Cargo.toml new file mode 100644 index 000000000..27ab8498b --- /dev/null +++ b/crates/factor-variables/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "spin-factor-variables" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1" +spin-expressions = { path = "../expressions" } +spin-factors = { path = "../factors" } +spin-world = { path = "../world" } + +[lints] +workspace = true diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs new file mode 100644 index 000000000..360956aad --- /dev/null +++ b/crates/factor-variables/src/lib.rs @@ -0,0 +1,125 @@ +use std::sync::Arc; + +use spin_expressions::ProviderResolver; +use spin_factors::{ + Factor, FactorInstancePreparer, InstancePreparers, PrepareContext, Result, SpinFactors, +}; +use spin_world::{async_trait, v1::config as v1_config, v2::variables}; + +pub struct VariablesFactor; + +impl Factor for VariablesFactor { + type AppConfig = AppConfig; + type InstancePreparer = InstancePreparer; + type InstanceState = InstanceState; + + fn init( + &mut self, + mut ctx: spin_factors::InitContext, + ) -> Result<()> { + ctx.link_bindings(v1_config::add_to_linker)?; + ctx.link_bindings(variables::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + app: &spin_factors::App, + _ctx: spin_factors::ConfigureAppContext, + ) -> Result { + let mut resolver = + ProviderResolver::new(app.variables().map(|(key, val)| (key.clone(), val.clone())))?; + for component in app.components() { + resolver.add_component_variables( + component.id(), + component.config().map(|(k, v)| (k.into(), v.into())), + )?; + } + // TODO: add providers from runtime config + Ok(AppConfig { + resolver: Arc::new(resolver), + }) + } +} + +#[derive(Default)] +pub struct AppConfig { + resolver: Arc, +} + +pub struct InstancePreparer { + state: InstanceState, +} + +impl InstancePreparer { + pub fn resolver(&self) -> &Arc { + &self.state.resolver + } +} + +impl FactorInstancePreparer for InstancePreparer { + fn new( + ctx: PrepareContext, + _preparers: InstancePreparers, + ) -> Result { + let component_id = ctx.app_component().id().to_string(); + let resolver = ctx.app_config().resolver.clone(); + Ok(Self { + state: InstanceState { + component_id, + resolver, + }, + }) + } + + fn prepare(self) -> Result<::InstanceState> { + Ok(self.state) + } +} + +pub struct InstanceState { + component_id: String, + resolver: Arc, +} + +#[async_trait] +impl variables::Host for InstanceState { + async fn get(&mut self, key: String) -> Result { + let key = spin_expressions::Key::new(&key).map_err(expressions_to_variables_err)?; + self.resolver + .resolve(&self.component_id, key) + .await + .map_err(expressions_to_variables_err) + } + + fn convert_error(&mut self, error: variables::Error) -> Result { + Ok(error) + } +} + +#[async_trait] +impl v1_config::Host for InstanceState { + async fn get_config(&mut self, key: String) -> Result { + ::get(self, key) + .await + .map_err(|err| match err { + variables::Error::InvalidName(msg) => v1_config::Error::InvalidKey(msg), + variables::Error::Undefined(msg) => v1_config::Error::Provider(msg), + other => v1_config::Error::Other(format!("{other}")), + }) + } + + fn convert_error(&mut self, err: v1_config::Error) -> anyhow::Result { + Ok(err) + } +} + +fn expressions_to_variables_err(err: spin_expressions::Error) -> variables::Error { + use spin_expressions::Error; + match err { + Error::InvalidName(msg) => variables::Error::InvalidName(msg), + Error::Undefined(msg) => variables::Error::Undefined(msg), + Error::Provider(err) => variables::Error::Provider(err.to_string()), + other => variables::Error::Other(format!("{other}")), + } +} diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 03ca3c14e..1bc01f376 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,11 +1,11 @@ pub mod preview1; -use std::path::Path; +use std::{future::Future, net::SocketAddr, path::Path}; use anyhow::ensure; -use cap_primitives::{ipnet::IpNet, net::Pool}; use spin_factors::{ - AppComponent, Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, + AppComponent, Factor, FactorInstancePreparer, InitContext, InstancePreparers, PrepareContext, + Result, SpinFactors, }; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; @@ -27,34 +27,44 @@ impl Factor for WasiFactor { type InstanceState = InstanceState; fn init(&mut self, mut ctx: InitContext) -> Result<()> { - use wasmtime_wasi::bindings; - ctx.link_bindings(bindings::clocks::wall_clock::add_to_linker_get_host)?; - ctx.link_bindings(bindings::clocks::monotonic_clock::add_to_linker_get_host)?; - ctx.link_bindings(bindings::filesystem::types::add_to_linker_get_host)?; - ctx.link_bindings(bindings::filesystem::preopens::add_to_linker_get_host)?; - ctx.link_bindings(bindings::io::error::add_to_linker_get_host)?; - ctx.link_bindings(bindings::io::poll::add_to_linker_get_host)?; - ctx.link_bindings(bindings::io::streams::add_to_linker_get_host)?; - ctx.link_bindings(bindings::random::random::add_to_linker_get_host)?; - ctx.link_bindings(bindings::random::insecure::add_to_linker_get_host)?; - ctx.link_bindings(bindings::random::insecure_seed::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::exit::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::environment::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::stdin::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::stdout::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::stderr::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::terminal_input::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::terminal_output::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::terminal_stdin::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::terminal_stdout::add_to_linker_get_host)?; - ctx.link_bindings(bindings::cli::terminal_stderr::add_to_linker_get_host)?; - ctx.link_bindings(bindings::sockets::tcp::add_to_linker_get_host)?; - ctx.link_bindings(bindings::sockets::tcp_create_socket::add_to_linker_get_host)?; - ctx.link_bindings(bindings::sockets::udp::add_to_linker_get_host)?; - ctx.link_bindings(bindings::sockets::udp_create_socket::add_to_linker_get_host)?; - ctx.link_bindings(bindings::sockets::instance_network::add_to_linker_get_host)?; - ctx.link_bindings(bindings::sockets::network::add_to_linker_get_host)?; - ctx.link_bindings(bindings::sockets::ip_name_lookup::add_to_linker_get_host)?; + fn type_annotate(f: F) -> F + where + F: Fn(&mut T) -> &mut dyn WasiView, + { + f + } + let get_data = ctx.get_data_fn(); + let closure = type_annotate(move |data| get_data(data) as &mut dyn WasiView); + if let Some(linker) = ctx.linker() { + use wasmtime_wasi::bindings; + bindings::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; + bindings::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; + bindings::filesystem::types::add_to_linker_get_host(linker, closure)?; + bindings::filesystem::preopens::add_to_linker_get_host(linker, closure)?; + bindings::io::error::add_to_linker_get_host(linker, closure)?; + bindings::io::poll::add_to_linker_get_host(linker, closure)?; + bindings::io::streams::add_to_linker_get_host(linker, closure)?; + bindings::random::random::add_to_linker_get_host(linker, closure)?; + bindings::random::insecure::add_to_linker_get_host(linker, closure)?; + bindings::random::insecure_seed::add_to_linker_get_host(linker, closure)?; + bindings::cli::exit::add_to_linker_get_host(linker, closure)?; + bindings::cli::environment::add_to_linker_get_host(linker, closure)?; + bindings::cli::stdin::add_to_linker_get_host(linker, closure)?; + bindings::cli::stdout::add_to_linker_get_host(linker, closure)?; + bindings::cli::stderr::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_input::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_output::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_stdin::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_stdout::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_stderr::add_to_linker_get_host(linker, closure)?; + bindings::sockets::tcp::add_to_linker_get_host(linker, closure)?; + bindings::sockets::tcp_create_socket::add_to_linker_get_host(linker, closure)?; + bindings::sockets::udp::add_to_linker_get_host(linker, closure)?; + bindings::sockets::udp_create_socket::add_to_linker_get_host(linker, closure)?; + bindings::sockets::instance_network::add_to_linker_get_host(linker, closure)?; + bindings::sockets::network::add_to_linker_get_host(linker, closure)?; + bindings::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; + } Ok(()) } } @@ -100,20 +110,18 @@ impl<'a> MountFilesContext<'a> { pub struct InstancePreparer { wasi_ctx: WasiCtxBuilder, - socket_allow_ports: Pool, } impl FactorInstancePreparer for InstancePreparer { // NOTE: Replaces WASI parts of AppComponent::apply_store_config fn new( - factor: &WasiFactor, - app_component: &AppComponent, - _ctx: PrepareContext, + ctx: PrepareContext, + _preparers: InstancePreparers, ) -> Result { let mut wasi_ctx = WasiCtxBuilder::new(); // Apply environment variables - for (key, val) in app_component.environment() { + for (key, val) in ctx.app_component().environment() { wasi_ctx.env(key, val); } @@ -121,23 +129,15 @@ impl FactorInstancePreparer for InstancePreparer { let mount_ctx = MountFilesContext { wasi_ctx: &mut wasi_ctx, }; - factor.files_mounter.mount_files(app_component, mount_ctx)?; + ctx.factor() + .files_mounter + .mount_files(ctx.app_component(), mount_ctx)?; - Ok(Self { - wasi_ctx, - socket_allow_ports: Default::default(), - }) + Ok(Self { wasi_ctx }) } fn prepare(self) -> Result { - let Self { - mut wasi_ctx, - socket_allow_ports, - } = self; - - // Enforce socket_allow_ports - wasi_ctx.socket_addr_check(move |addr, _| socket_allow_ports.check_addr(addr).is_ok()); - + let Self { mut wasi_ctx } = self; Ok(InstanceState { ctx: wasi_ctx.build(), table: Default::default(), @@ -146,17 +146,23 @@ impl FactorInstancePreparer for InstancePreparer { } impl InstancePreparer { - pub fn inherit_network(&mut self) { - self.wasi_ctx.inherit_network(); - } - - pub fn socket_allow_ports(&mut self, ip_net: IpNet, ports_start: u16, ports_end: Option) { - self.socket_allow_ports.insert_ip_net_port_range( - ip_net, - ports_start, - ports_end, - cap_primitives::ambient_authority(), - ); + pub fn outbound_socket_addr_check(&mut self, check: F) + where + F: Fn(SocketAddr) -> Fut + Send + Sync + Clone + 'static, + Fut: Future + Send + Sync, + { + self.wasi_ctx.socket_addr_check(move |addr, addr_use| { + let check = check.clone(); + Box::pin(async move { + match addr_use { + wasmtime_wasi::SocketAddrUse::TcpBind => false, + wasmtime_wasi::SocketAddrUse::TcpConnect + | wasmtime_wasi::SocketAddrUse::UdpBind + | wasmtime_wasi::SocketAddrUse::UdpConnect + | wasmtime_wasi::SocketAddrUse::UdpOutgoingDatagram => check(addr).await, + } + }) + }); } } diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index a4441a018..bc8f0813b 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,5 +1,5 @@ use spin_factors::{ - AppComponent, Factor, FactorInstancePreparer, InitContext, PrepareContext, Result, SpinFactors, + Factor, FactorInstancePreparer, InitContext, InstancePreparers, PrepareContext, Result, SpinFactors }; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; @@ -21,9 +21,8 @@ pub struct InstancePreparer { impl FactorInstancePreparer for InstancePreparer { fn new( - _factor: &WasiPreview1Factor, - _app_component: &AppComponent, - _ctx: PrepareContext, + _ctx: PrepareContext, + _preparers: InstancePreparers, ) -> Result { Ok(Self { wasi_ctx: WasiCtxBuilder::new(), diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 456068c2c..85d712ea0 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -107,9 +107,12 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #( preparers.#factor_names = Some( #factors_path::FactorInstancePreparer::<#factor_types>::new::<#name>( - &self.#factor_names, - &app_component, - #factors_path::PrepareContext::new(configured_app, &mut preparers), + #factors_path::PrepareContext::new( + &self.#factor_names, + configured_app.app_config::<#factor_types>().unwrap(), + &app_component, + ), + #factors_path::InstancePreparers::new(&mut preparers), )? ); )* diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 779b95ba0..6c54eb427 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -14,6 +14,7 @@ wasmtime = { workspace = true } serde_json = "1.0" spin-factors-derive = { path = "../factors-derive", features = ["expander"] } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } [lints] diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index d917c392e..5d36498d5 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -37,10 +37,9 @@ pub trait Factor: Any + Sized { } /// Performs factor-specific validation and configuration for the given - /// [`App`] and [`RuntimeConfig`]. A runtime may - but is not required to - - /// reuse the returned config across multiple instances. Note that this may - /// be called without any call to `init` in cases where only validation is - /// needed. + /// [`App`]. A runtime may - but is not required to - reuse the returned + /// config across multiple instances. Note that this may be called without + /// any call to `init` in cases where only validation is needed. fn configure_app( &self, app: &App, @@ -56,18 +55,18 @@ type GetDataFn = /// An InitContext is passed to [`Factor::init`], giving access to the global /// common [`wasmtime::component::Linker`]. -pub struct InitContext<'a, Factors: SpinFactors, Fact: Factor> { +pub struct InitContext<'a, Factors: SpinFactors, T: Factor> { linker: Option<&'a mut Linker>, module_linker: Option<&'a mut ModuleLinker>, - get_data: GetDataFn, + get_data: GetDataFn, } -impl<'a, Factors: SpinFactors, Fact: Factor> InitContext<'a, Factors, Fact> { +impl<'a, Factors: SpinFactors, T: Factor> InitContext<'a, Factors, T> { #[doc(hidden)] pub fn new( linker: Option<&'a mut Linker>, module_linker: Option<&'a mut ModuleLinker>, - get_data: GetDataFn, + get_data: GetDataFn, ) -> Self { Self { linker, @@ -84,7 +83,7 @@ impl<'a, Factors: SpinFactors, Fact: Factor> InitContext<'a, Factors, Fact> { self.module_linker.as_deref_mut() } - pub fn get_data_fn(&self) -> GetDataFn { + pub fn get_data_fn(&self) -> GetDataFn { self.get_data } @@ -92,7 +91,7 @@ impl<'a, Factors: SpinFactors, Fact: Factor> InitContext<'a, Factors, Fact> { &mut self, add_to_linker: impl Fn( &mut Linker, - fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, + fn(&mut Factors::InstanceState) -> &mut T::InstanceState, ) -> Result<()>, ) -> Result<()> where { @@ -107,7 +106,7 @@ where { &mut self, add_to_linker: impl Fn( &mut ModuleLinker, - fn(&mut Factors::InstanceState) -> &mut Fact::InstanceState, + fn(&mut Factors::InstanceState) -> &mut T::InstanceState, ) -> Result<()>, ) -> Result<()> where { @@ -137,37 +136,74 @@ impl<'a, Factors: SpinFactors> ConfigureAppContext<'a, Factors> { pub trait FactorInstancePreparer: Sized { /// Returns a new instance of this preparer for the given [`Factor`]. fn new( - factor: &T, - app_component: &AppComponent, - _ctx: PrepareContext, + ctx: PrepareContext, + _preparers: InstancePreparers, ) -> Result; /// Returns a new instance of the associated [`Factor::InstanceState`]. fn prepare(self) -> Result; } +impl FactorInstancePreparer for () +where + T::InstanceState: Default, +{ + fn new( + _ctx: PrepareContext, + _preparers: InstancePreparers, + ) -> Result { + Ok(()) + } + + fn prepare(self) -> Result { + Ok(Default::default()) + } +} + /// A PrepareContext is passed to [`FactorInstancePreparer::new`], giving access /// to any already-initialized [`FactorInstancePreparer`]s, allowing for /// inter-[`Factor`] dependencies. -pub struct PrepareContext<'a, Factors: SpinFactors> { - configured_app: &'a ConfiguredApp, - instance_preparers: &'a mut Factors::InstancePreparers, +pub struct PrepareContext<'a, T: Factor> { + factor: &'a T, + app_config: &'a T::AppConfig, + app_component: &'a AppComponent<'a>, } -impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { +impl<'a, T: Factor> PrepareContext<'a, T> { #[doc(hidden)] pub fn new( - configured_app: &'a ConfiguredApp, - instance_preparers: &'a mut Factors::InstancePreparers, + factor: &'a T, + app_config: &'a T::AppConfig, + app_component: &'a AppComponent, ) -> Self { Self { - configured_app, - instance_preparers, + factor, + app_config, + app_component, } } - pub fn app_config(&self) -> Result<&T::AppConfig> { - self.configured_app.app_config::() + pub fn factor(&self) -> &T { + self.factor + } + + pub fn app_config(&self) -> &T::AppConfig { + self.app_config + } + + pub fn app_component(&self) -> &AppComponent { + self.app_component + } +} + +pub struct InstancePreparers<'a, Factors: SpinFactors> { + inner: &'a mut Factors::InstancePreparers, +} + +impl<'a, Factors: SpinFactors> InstancePreparers<'a, Factors> { + #[doc(hidden)] + pub fn new(inner: &'a mut Factors::InstancePreparers) -> Self { + Self { inner } } /// Returns a already-initialized preparer for the given [`Factor`]. @@ -175,8 +211,8 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { /// Fails if the current [`SpinFactors`] does not include the given /// [`Factor`] or if the given [`Factor`]'s preparer has not been /// initialized yet (because it is sequenced after this factor). - pub fn instance_preparer_mut(&mut self) -> Result<&mut T::InstancePreparer> { - Factors::instance_preparer_mut::(self.instance_preparers) + pub fn get_mut(&mut self) -> Result<&mut T::InstancePreparer> { + Factors::instance_preparer_mut::(self.inner) .and_then(|maybe_preparer| maybe_preparer.context("preparer not yet initialized")) .with_context(|| { format!( @@ -187,24 +223,6 @@ impl<'a, Factors: SpinFactors> PrepareContext<'a, Factors> { } } -impl FactorInstancePreparer for () -where - T::InstanceState: Default, -{ - fn new( - factor: &T, - app_component: &AppComponent, - _ctx: PrepareContext, - ) -> Result { - (_, _) = (factor, app_component); - Ok(()) - } - - fn prepare(self) -> Result { - Ok(Default::default()) - } -} - pub struct ConfiguredApp { app: App, app_configs: Factors::AppConfigs, diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index e1f39f2c5..a4157bc6a 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,5 +1,6 @@ use spin_app::App; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{preview1::WasiPreview1Factor, DummyFilesMounter, WasiFactor}; use spin_factors::SpinFactors; @@ -7,6 +8,7 @@ use spin_factors::SpinFactors; struct Factors { wasi: WasiFactor, wasip1: WasiPreview1Factor, + variables: VariablesFactor, outbound_networking_factor: OutboundNetworkingFactor, } @@ -14,6 +16,7 @@ fn main() -> anyhow::Result<()> { let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), wasip1: WasiPreview1Factor, + variables: VariablesFactor, outbound_networking_factor: OutboundNetworkingFactor, // outbound_http_factor: OutboundHttpFactor, }; diff --git a/crates/world/Cargo.toml b/crates/world/Cargo.toml index 3c51d5b24..270b46a17 100644 --- a/crates/world/Cargo.toml +++ b/crates/world/Cargo.toml @@ -5,4 +5,5 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] +async-trait = "0.1" wasmtime = { workspace = true } diff --git a/crates/world/src/lib.rs b/crates/world/src/lib.rs index 42c4d3545..d7adeb19f 100644 --- a/crates/world/src/lib.rs +++ b/crates/world/src/lib.rs @@ -1,5 +1,7 @@ #![allow(missing_docs)] +pub use async_trait::async_trait; + wasmtime::component::bindgen!({ inline: r#" package fermyon:runtime; From e539cf6e5cd731e0423fc2d50bab4b43ac170db6 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 4 Jun 2024 09:53:06 -0400 Subject: [PATCH 009/195] factors: Reorganize spin-factors Signed-off-by: Lann Martin --- Cargo.lock | 18 +- crates/factor-outbound-networking/Cargo.toml | 1 - crates/factor-outbound-networking/src/lib.rs | 24 +- crates/factor-variables/Cargo.toml | 1 - crates/factor-variables/src/lib.rs | 20 +- crates/factor-wasi/Cargo.toml | 1 - crates/factor-wasi/src/lib.rs | 30 +- crates/factor-wasi/src/preview1.rs | 12 +- crates/factors-derive/src/lib.rs | 20 +- crates/factors/Cargo.toml | 3 + crates/factors/src/factor.rs | 152 ++++++++ crates/factors/src/instance_preparer.rs | 86 +++++ crates/factors/src/lib.rs | 348 +------------------ crates/factors/src/runtime_config.rs | 94 +++++ crates/factors/src/spin_factors.rs | 109 ++++++ crates/factors/tests/smoke.rs | 2 +- 16 files changed, 544 insertions(+), 377 deletions(-) create mode 100644 crates/factors/src/factor.rs create mode 100644 crates/factors/src/instance_preparer.rs create mode 100644 crates/factors/src/runtime_config.rs create mode 100644 crates/factors/src/spin_factors.rs diff --git a/Cargo.lock b/Cargo.lock index 032f8dcc5..9c575470f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7562,19 +7562,28 @@ dependencies = [ name = "spin-factor-outbound-networking" version = "2.6.0-pre0" dependencies = [ - "anyhow", + "futures-util", "ipnet", + "spin-factor-variables", "spin-factor-wasi", "spin-factors", "spin-outbound-networking", "tracing", ] +[[package]] +name = "spin-factor-variables" +version = "2.6.0-pre0" +dependencies = [ + "spin-expressions", + "spin-factors", + "spin-world", +] + [[package]] name = "spin-factor-wasi" version = "2.6.0-pre0" dependencies = [ - "anyhow", "cap-primitives 3.0.0", "spin-factors", "wasmtime-wasi", @@ -7585,11 +7594,15 @@ name = "spin-factors" version = "2.6.0-pre0" dependencies = [ "anyhow", + "serde 1.0.197", "serde_json", "spin-app", "spin-factor-outbound-networking", + "spin-factor-variables", "spin-factor-wasi", "spin-factors-derive", + "thiserror", + "tracing", "wasmtime", ] @@ -8138,6 +8151,7 @@ dependencies = [ name = "spin-world" version = "2.7.0-pre0" dependencies = [ + "async-trait", "wasmtime", ] diff --git a/crates/factor-outbound-networking/Cargo.toml b/crates/factor-outbound-networking/Cargo.toml index 8eddb55eb..13dd49c9e 100644 --- a/crates/factor-outbound-networking/Cargo.toml +++ b/crates/factor-outbound-networking/Cargo.toml @@ -5,7 +5,6 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] -anyhow = "1" futures-util = "0.3" ipnet = "2.9.0" spin-factor-variables = { path = "../factor-variables" } diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index 39013b806..7387d69a2 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -1,6 +1,5 @@ use std::{collections::HashMap, sync::Arc}; -use anyhow::Context; use futures_util::{ future::{BoxFuture, Shared}, FutureExt, @@ -8,7 +7,9 @@ use futures_util::{ use spin_factor_variables::VariablesFactor; use spin_factor_wasi::WasiFactor; use spin_factors::{ - Factor, FactorInstancePreparer, InstancePreparers, PrepareContext, Result, SpinFactors, + anyhow::{self, Context}, + ConfigureAppContext, Factor, FactorInstancePreparer, InstancePreparers, PrepareContext, + RuntimeConfig, SpinFactors, }; use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; @@ -21,11 +22,12 @@ impl Factor for OutboundNetworkingFactor { fn configure_app( &self, - app: &spin_factors::App, - _ctx: spin_factors::ConfigureAppContext, - ) -> Result { + ctx: ConfigureAppContext, + _runtime_config: &mut impl RuntimeConfig, + ) -> anyhow::Result { // Extract allowed_outbound_hosts for all components - let component_allowed_hosts = app + let component_allowed_hosts = ctx + .app() .components() .map(|component| { Ok(( @@ -37,7 +39,7 @@ impl Factor for OutboundNetworkingFactor { .into(), )) }) - .collect::>()?; + .collect::>()?; Ok(AppConfig { component_allowed_hosts, }) @@ -49,17 +51,17 @@ pub struct AppConfig { component_allowed_hosts: HashMap>, } -type AllowedHostsFuture = Shared>>>; +type SharedFutureResult = Shared>>>; pub struct InstancePreparer { - allowed_hosts_future: AllowedHostsFuture, + allowed_hosts_future: SharedFutureResult, } impl FactorInstancePreparer for InstancePreparer { fn new( ctx: PrepareContext, mut preparers: InstancePreparers, - ) -> Result { + ) -> anyhow::Result { let hosts = ctx .app_config() .component_allowed_hosts @@ -104,7 +106,7 @@ impl FactorInstancePreparer for InstancePreparer { }) } - fn prepare(self) -> Result<::InstanceState> { + fn prepare(self) -> anyhow::Result<::InstanceState> { Ok(()) } } diff --git a/crates/factor-variables/Cargo.toml b/crates/factor-variables/Cargo.toml index 27ab8498b..7621fb266 100644 --- a/crates/factor-variables/Cargo.toml +++ b/crates/factor-variables/Cargo.toml @@ -5,7 +5,6 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] -anyhow = "1" spin-expressions = { path = "../expressions" } spin-factors = { path = "../factors" } spin-world = { path = "../world" } diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 360956aad..423c8bba8 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -2,7 +2,8 @@ use std::sync::Arc; use spin_expressions::ProviderResolver; use spin_factors::{ - Factor, FactorInstancePreparer, InstancePreparers, PrepareContext, Result, SpinFactors, + anyhow, ConfigureAppContext, Factor, FactorInstancePreparer, InitContext, InstancePreparers, + PrepareContext, RuntimeConfig, SpinFactors, }; use spin_world::{async_trait, v1::config as v1_config, v2::variables}; @@ -15,8 +16,8 @@ impl Factor for VariablesFactor { fn init( &mut self, - mut ctx: spin_factors::InitContext, - ) -> Result<()> { + mut ctx: InitContext, + ) -> anyhow::Result<()> { ctx.link_bindings(v1_config::add_to_linker)?; ctx.link_bindings(variables::add_to_linker)?; Ok(()) @@ -24,9 +25,10 @@ impl Factor for VariablesFactor { fn configure_app( &self, - app: &spin_factors::App, - _ctx: spin_factors::ConfigureAppContext, - ) -> Result { + ctx: ConfigureAppContext, + _runtime_config: &mut impl RuntimeConfig, + ) -> anyhow::Result { + let app = ctx.app(); let mut resolver = ProviderResolver::new(app.variables().map(|(key, val)| (key.clone(), val.clone())))?; for component in app.components() { @@ -61,7 +63,7 @@ impl FactorInstancePreparer for InstancePreparer { fn new( ctx: PrepareContext, _preparers: InstancePreparers, - ) -> Result { + ) -> anyhow::Result { let component_id = ctx.app_component().id().to_string(); let resolver = ctx.app_config().resolver.clone(); Ok(Self { @@ -72,7 +74,7 @@ impl FactorInstancePreparer for InstancePreparer { }) } - fn prepare(self) -> Result<::InstanceState> { + fn prepare(self) -> anyhow::Result { Ok(self.state) } } @@ -92,7 +94,7 @@ impl variables::Host for InstanceState { .map_err(expressions_to_variables_err) } - fn convert_error(&mut self, error: variables::Error) -> Result { + fn convert_error(&mut self, error: variables::Error) -> anyhow::Result { Ok(error) } } diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index 58d4b5ff4..d1572e555 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -5,7 +5,6 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] -anyhow = "1" cap-primitives = "3.0.0" spin-factors = { path = "../factors" } wasmtime-wasi = { workspace = true } diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 1bc01f376..6320682da 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -2,10 +2,9 @@ pub mod preview1; use std::{future::Future, net::SocketAddr, path::Path}; -use anyhow::ensure; use spin_factors::{ - AppComponent, Factor, FactorInstancePreparer, InitContext, InstancePreparers, PrepareContext, - Result, SpinFactors, + anyhow, AppComponent, Factor, FactorInstancePreparer, InitContext, InstancePreparers, + PrepareContext, SpinFactors, }; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; @@ -26,7 +25,10 @@ impl Factor for WasiFactor { type InstancePreparer = InstancePreparer; type InstanceState = InstanceState; - fn init(&mut self, mut ctx: InitContext) -> Result<()> { + fn init( + &mut self, + mut ctx: InitContext, + ) -> anyhow::Result<()> { fn type_annotate(f: F) -> F where F: Fn(&mut T) -> &mut dyn WasiView, @@ -70,14 +72,22 @@ impl Factor for WasiFactor { } pub trait FilesMounter { - fn mount_files(&self, app_component: &AppComponent, ctx: MountFilesContext) -> Result<()>; + fn mount_files( + &self, + app_component: &AppComponent, + ctx: MountFilesContext, + ) -> anyhow::Result<()>; } pub struct DummyFilesMounter; impl FilesMounter for DummyFilesMounter { - fn mount_files(&self, app_component: &AppComponent, _ctx: MountFilesContext) -> Result<()> { - ensure!( + fn mount_files( + &self, + app_component: &AppComponent, + _ctx: MountFilesContext, + ) -> anyhow::Result<()> { + anyhow::ensure!( app_component.files().next().is_none(), "DummyFilesMounter can't actually mount files" ); @@ -95,7 +105,7 @@ impl<'a> MountFilesContext<'a> { host_path: impl AsRef, guest_path: impl AsRef, writable: bool, - ) -> Result<()> { + ) -> anyhow::Result<()> { use wasmtime_wasi::{DirPerms, FilePerms}; let (dir_perms, file_perms) = if writable { (DirPerms::all(), FilePerms::all()) @@ -117,7 +127,7 @@ impl FactorInstancePreparer for InstancePreparer { fn new( ctx: PrepareContext, _preparers: InstancePreparers, - ) -> Result { + ) -> anyhow::Result { let mut wasi_ctx = WasiCtxBuilder::new(); // Apply environment variables @@ -136,7 +146,7 @@ impl FactorInstancePreparer for InstancePreparer { Ok(Self { wasi_ctx }) } - fn prepare(self) -> Result { + fn prepare(self) -> anyhow::Result { let Self { mut wasi_ctx } = self; Ok(InstanceState { ctx: wasi_ctx.build(), diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index bc8f0813b..426ebff6d 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,5 +1,6 @@ use spin_factors::{ - Factor, FactorInstancePreparer, InitContext, InstancePreparers, PrepareContext, Result, SpinFactors + anyhow, Factor, FactorInstancePreparer, InitContext, InstancePreparers, PrepareContext, + SpinFactors, }; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; @@ -10,7 +11,10 @@ impl Factor for WasiPreview1Factor { type InstancePreparer = InstancePreparer; type InstanceState = WasiP1Ctx; - fn init(&mut self, mut ctx: InitContext) -> Result<()> { + fn init( + &mut self, + mut ctx: InitContext, + ) -> anyhow::Result<()> { ctx.link_module_bindings(wasmtime_wasi::preview1::add_to_linker_async) } } @@ -23,13 +27,13 @@ impl FactorInstancePreparer for InstancePreparer { fn new( _ctx: PrepareContext, _preparers: InstancePreparers, - ) -> Result { + ) -> anyhow::Result { Ok(Self { wasi_ctx: WasiCtxBuilder::new(), }) } - fn prepare(mut self) -> Result<::InstanceState> { + fn prepare(mut self) -> anyhow::Result { Ok(self.wasi_ctx.build_p1()) } } diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 85d712ea0..399606708 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -53,13 +53,14 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { factor_types.push(&field.ty); } + let TypeId = quote!(::std::any::TypeId); let factors_crate = format_ident!("spin_factors"); let factors_path = quote!(::#factors_crate); - let Factor = quote!(#factors_path::Factor); - let Result = quote!(#factors_path::Result); let wasmtime = quote!(#factors_path::wasmtime); + let Result = quote!(#factors_path::Result); + let Factor = quote!(#factors_path::Factor); let ConfiguredApp = quote!(#factors_path::ConfiguredApp); - let TypeId = quote!(::std::any::TypeId); + let RuntimeConfigTracker = quote!(#factors_path::__internal::RuntimeConfigTracker); Ok(quote! { impl #name { @@ -81,16 +82,21 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { Ok(()) } - pub fn configure_app(&self, app: #factors_path::App) -> #Result<#ConfiguredApp> { + pub fn configure_app( + &self, + app: #factors_path::App, + runtime_config: impl #factors_path::RuntimeConfigSource + ) -> #Result<#ConfiguredApp> { let mut app_configs = #app_configs_name { #( #factor_names: None, )* }; + let mut runtime_config = #RuntimeConfigTracker::new(runtime_config); #( app_configs.#factor_names = Some( #Factor::configure_app( &self.#factor_names, - &app, - #factors_path::ConfigureAppContext::::new(&app_configs), + #factors_path::ConfigureAppContext::::new(&app, &app_configs), + &mut runtime_config, )? ); )* @@ -99,7 +105,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { pub fn build_store_data(&self, configured_app: &#ConfiguredApp, component_id: &str) -> #Result<#state_name> { let app_component = configured_app.app().get_component(component_id).ok_or_else(|| { - #factors_path::Error::msg(format!("unknown component {component_id:?}")) + #wasmtime::Error::msg("unknown component") })?; let mut preparers = #preparers_name { #( #factor_names: None, )* diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 6c54eb427..2d9347dbe 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -6,8 +6,11 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" +serde = "1.0" spin-app = { path = "../app" } spin-factors-derive = { path = "../factors-derive" } +thiserror = "1.0" +tracing = { workspace = true } wasmtime = { workspace = true } [dev-dependencies] diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs new file mode 100644 index 000000000..552e84417 --- /dev/null +++ b/crates/factors/src/factor.rs @@ -0,0 +1,152 @@ +use std::any::Any; + +use anyhow::Context; + +use crate::{App, FactorInstancePreparer, Linker, ModuleLinker, RuntimeConfig, SpinFactors}; + +pub trait Factor: Any + Sized { + /// Per-app configuration for this factor. + /// + /// See [`Factor::configure_app`]. + type AppConfig: Default; + + /// The [`FactorInstancePreparer`] for this factor. + type InstancePreparer: FactorInstancePreparer; + + /// The per-instance state for this factor, constructed by a + /// [`FactorInstancePreparer`] and available to any host-provided imports + /// defined by this factor. + type InstanceState; + + /// Initializes this Factor for a runtime. This will be called at most once, + /// before any call to [`FactorInstancePreparer::new`] + fn init( + &mut self, + mut ctx: InitContext, + ) -> anyhow::Result<()> { + // TODO: Should `ctx` always be immut? Rename this param/type? + _ = &mut ctx; + Ok(()) + } + + /// Performs factor-specific validation and configuration for the given + /// [`App`]. A runtime may - but is not required to - reuse the returned + /// config across multiple instances. Note that this may be called without + /// any call to `init` in cases where only validation is needed. + fn configure_app( + &self, + ctx: ConfigureAppContext, + _runtime_config: &mut impl RuntimeConfig, + ) -> anyhow::Result { + _ = ctx; + Ok(Default::default()) + } +} + +pub(crate) type GetDataFn = + fn(&mut ::InstanceState) -> &mut ::InstanceState; + +/// An InitContext is passed to [`Factor::init`], giving access to the global +/// common [`wasmtime::component::Linker`]. +pub struct InitContext<'a, Factors: SpinFactors, T: Factor> { + pub(crate) linker: Option<&'a mut Linker>, + pub(crate) module_linker: Option<&'a mut ModuleLinker>, + pub(crate) get_data: GetDataFn, +} + +impl<'a, Factors: SpinFactors, T: Factor> InitContext<'a, Factors, T> { + #[doc(hidden)] + pub fn new( + linker: Option<&'a mut Linker>, + module_linker: Option<&'a mut ModuleLinker>, + get_data: GetDataFn, + ) -> Self { + Self { + linker, + module_linker, + get_data, + } + } + + pub fn linker(&mut self) -> Option<&mut Linker> { + self.linker.as_deref_mut() + } + + pub fn module_linker(&mut self) -> Option<&mut ModuleLinker> { + self.module_linker.as_deref_mut() + } + + pub fn get_data_fn(&self) -> GetDataFn { + self.get_data + } + + pub fn link_bindings( + &mut self, + add_to_linker: impl Fn( + &mut Linker, + fn(&mut Factors::InstanceState) -> &mut T::InstanceState, + ) -> anyhow::Result<()>, + ) -> anyhow::Result<()> +where { + if let Some(linker) = self.linker.as_deref_mut() { + add_to_linker(linker, self.get_data) + } else { + Ok(()) + } + } + + pub fn link_module_bindings( + &mut self, + add_to_linker: impl Fn( + &mut ModuleLinker, + fn(&mut Factors::InstanceState) -> &mut T::InstanceState, + ) -> anyhow::Result<()>, + ) -> anyhow::Result<()> +where { + if let Some(linker) = self.module_linker.as_deref_mut() { + add_to_linker(linker, self.get_data) + } else { + Ok(()) + } + } +} + +pub struct ConfigureAppContext<'a, Factors: SpinFactors> { + pub(crate) app: &'a App, + pub(crate) app_configs: &'a Factors::AppConfigs, +} + +impl<'a, Factors: SpinFactors> ConfigureAppContext<'a, Factors> { + #[doc(hidden)] + pub fn new(app: &'a App, app_configs: &'a Factors::AppConfigs) -> Self { + Self { app, app_configs } + } + + pub fn app(&self) -> &App { + self.app + } + + pub fn app_config(&self) -> crate::Result<&T::AppConfig> { + Factors::app_config::(self.app_configs).context("no such factor") + } +} + +pub struct ConfiguredApp { + app: App, + app_configs: Factors::AppConfigs, +} + +impl ConfiguredApp { + #[doc(hidden)] + pub fn new(app: App, app_configs: Factors::AppConfigs) -> Self { + Self { app, app_configs } + } + + pub fn app(&self) -> &App { + &self.app + } + + pub fn app_config(&self) -> crate::Result<&T::AppConfig> { + Factors::app_config::(&self.app_configs).context("no such factor") + } +} diff --git a/crates/factors/src/instance_preparer.rs b/crates/factors/src/instance_preparer.rs new file mode 100644 index 000000000..43fb01fce --- /dev/null +++ b/crates/factors/src/instance_preparer.rs @@ -0,0 +1,86 @@ +use anyhow::Context; + +use crate::{AppComponent, Factor, SpinFactors}; + +pub trait FactorInstancePreparer: Sized { + /// Returns a new instance of this preparer for the given [`Factor`]. + fn new( + ctx: PrepareContext, + _preparers: InstancePreparers, + ) -> anyhow::Result; + + /// Returns a new instance of the associated [`Factor::InstanceState`]. + fn prepare(self) -> anyhow::Result; +} + +impl FactorInstancePreparer for () +where + T::InstanceState: Default, +{ + fn new( + _ctx: PrepareContext, + _preparers: InstancePreparers, + ) -> anyhow::Result { + Ok(()) + } + + fn prepare(self) -> anyhow::Result { + Ok(Default::default()) + } +} + +/// A PrepareContext is passed to [`FactorInstancePreparer::new`], giving access +/// to any already-initialized [`FactorInstancePreparer`]s, allowing for +/// inter-[`Factor`] dependencies. +pub struct PrepareContext<'a, T: Factor> { + pub(crate) factor: &'a T, + pub(crate) app_config: &'a T::AppConfig, + pub(crate) app_component: &'a AppComponent<'a>, +} + +impl<'a, T: Factor> PrepareContext<'a, T> { + #[doc(hidden)] + pub fn new( + factor: &'a T, + app_config: &'a T::AppConfig, + app_component: &'a AppComponent, + ) -> Self { + Self { + factor, + app_config, + app_component, + } + } + + pub fn factor(&self) -> &T { + self.factor + } + + pub fn app_config(&self) -> &T::AppConfig { + self.app_config + } + + pub fn app_component(&self) -> &AppComponent { + self.app_component + } +} + +pub struct InstancePreparers<'a, Factors: SpinFactors> { + pub(crate) inner: &'a mut Factors::InstancePreparers, +} + +impl<'a, Factors: SpinFactors> InstancePreparers<'a, Factors> { + #[doc(hidden)] + pub fn new(inner: &'a mut Factors::InstancePreparers) -> Self { + Self { inner } + } + + /// Returns a already-initialized preparer for the given [`Factor`]. + /// + /// Fails if the current [`SpinFactors`] does not include the given + /// [`Factor`] or if the given [`Factor`]'s preparer has not been + /// initialized yet (because it is sequenced after this factor). + pub fn get_mut(&mut self) -> crate::Result<&mut T::InstancePreparer> { + Factors::instance_preparer_mut::(self.inner)?.context("preparer not initialized") + } +} diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 5d36498d5..0d18109cf 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -1,12 +1,19 @@ -use std::{any::Any, marker::PhantomData}; - -use anyhow::Context; -pub use spin_factors_derive::SpinFactors; +mod factor; +mod instance_preparer; +mod runtime_config; +mod spin_factors; +pub use anyhow; pub use wasmtime; -pub type Error = wasmtime::Error; -pub type Result = std::result::Result; +pub use spin_factors_derive::SpinFactors; + +pub use crate::{ + factor::{ConfigureAppContext, ConfiguredApp, Factor, InitContext}, + instance_preparer::{FactorInstancePreparer, InstancePreparers, PrepareContext}, + runtime_config::{RuntimeConfig, RuntimeConfigSource}, + spin_factors::SpinFactors, +}; pub type Linker = wasmtime::component::Linker<::InstanceState>; pub type ModuleLinker = wasmtime::Linker<::InstanceState>; @@ -15,329 +22,10 @@ pub type ModuleLinker = wasmtime::Linker<::Inst pub type App = spin_app::App<'static, spin_app::InertLoader>; pub type AppComponent<'a> = spin_app::AppComponent<'a, spin_app::InertLoader>; -pub trait Factor: Any + Sized { - /// App configuration for this factor. - /// - /// See [`Factor::configure_app`]. - type AppConfig: Default; - - /// The [`FactorInstancePreparer`] for this factor. - type InstancePreparer: FactorInstancePreparer; - - /// The per-instance state for this factor, constructed by a - /// [`FactorInstancePreparer`] and available to any host-provided imports - /// defined by this factor. - type InstanceState; - - /// Initializes this Factor for a runtime. This will be called at most once, - /// before any call to [`FactorInstancePreparer::new`] - fn init(&mut self, mut ctx: InitContext) -> Result<()> { - _ = &mut ctx; - Ok(()) - } - - /// Performs factor-specific validation and configuration for the given - /// [`App`]. A runtime may - but is not required to - reuse the returned - /// config across multiple instances. Note that this may be called without - /// any call to `init` in cases where only validation is needed. - fn configure_app( - &self, - app: &App, - _ctx: ConfigureAppContext, - ) -> Result { - _ = app; - Ok(Default::default()) - } -} - -type GetDataFn = - fn(&mut ::InstanceState) -> &mut ::InstanceState; - -/// An InitContext is passed to [`Factor::init`], giving access to the global -/// common [`wasmtime::component::Linker`]. -pub struct InitContext<'a, Factors: SpinFactors, T: Factor> { - linker: Option<&'a mut Linker>, - module_linker: Option<&'a mut ModuleLinker>, - get_data: GetDataFn, -} - -impl<'a, Factors: SpinFactors, T: Factor> InitContext<'a, Factors, T> { - #[doc(hidden)] - pub fn new( - linker: Option<&'a mut Linker>, - module_linker: Option<&'a mut ModuleLinker>, - get_data: GetDataFn, - ) -> Self { - Self { - linker, - module_linker, - get_data, - } - } - - pub fn linker(&mut self) -> Option<&mut Linker> { - self.linker.as_deref_mut() - } - - pub fn module_linker(&mut self) -> Option<&mut ModuleLinker> { - self.module_linker.as_deref_mut() - } - - pub fn get_data_fn(&self) -> GetDataFn { - self.get_data - } - - pub fn link_bindings( - &mut self, - add_to_linker: impl Fn( - &mut Linker, - fn(&mut Factors::InstanceState) -> &mut T::InstanceState, - ) -> Result<()>, - ) -> Result<()> -where { - if let Some(linker) = self.linker.as_deref_mut() { - add_to_linker(linker, self.get_data) - } else { - Ok(()) - } - } - - pub fn link_module_bindings( - &mut self, - add_to_linker: impl Fn( - &mut ModuleLinker, - fn(&mut Factors::InstanceState) -> &mut T::InstanceState, - ) -> Result<()>, - ) -> Result<()> -where { - if let Some(linker) = self.module_linker.as_deref_mut() { - add_to_linker(linker, self.get_data) - } else { - Ok(()) - } - } -} - -pub struct ConfigureAppContext<'a, Factors: SpinFactors> { - app_configs: &'a Factors::AppConfigs, -} - -impl<'a, Factors: SpinFactors> ConfigureAppContext<'a, Factors> { - #[doc(hidden)] - pub fn new(app_configs: &'a Factors::AppConfigs) -> Self { - Self { app_configs } - } - - pub fn app_config(&self) -> Result<&T::AppConfig> { - Factors::app_config::(self.app_configs).context("no such factor") - } -} - -pub trait FactorInstancePreparer: Sized { - /// Returns a new instance of this preparer for the given [`Factor`]. - fn new( - ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> Result; - - /// Returns a new instance of the associated [`Factor::InstanceState`]. - fn prepare(self) -> Result; -} - -impl FactorInstancePreparer for () -where - T::InstanceState: Default, -{ - fn new( - _ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> Result { - Ok(()) - } - - fn prepare(self) -> Result { - Ok(Default::default()) - } -} - -/// A PrepareContext is passed to [`FactorInstancePreparer::new`], giving access -/// to any already-initialized [`FactorInstancePreparer`]s, allowing for -/// inter-[`Factor`] dependencies. -pub struct PrepareContext<'a, T: Factor> { - factor: &'a T, - app_config: &'a T::AppConfig, - app_component: &'a AppComponent<'a>, -} - -impl<'a, T: Factor> PrepareContext<'a, T> { - #[doc(hidden)] - pub fn new( - factor: &'a T, - app_config: &'a T::AppConfig, - app_component: &'a AppComponent, - ) -> Self { - Self { - factor, - app_config, - app_component, - } - } - - pub fn factor(&self) -> &T { - self.factor - } - - pub fn app_config(&self) -> &T::AppConfig { - self.app_config - } - - pub fn app_component(&self) -> &AppComponent { - self.app_component - } -} - -pub struct InstancePreparers<'a, Factors: SpinFactors> { - inner: &'a mut Factors::InstancePreparers, -} - -impl<'a, Factors: SpinFactors> InstancePreparers<'a, Factors> { - #[doc(hidden)] - pub fn new(inner: &'a mut Factors::InstancePreparers) -> Self { - Self { inner } - } - - /// Returns a already-initialized preparer for the given [`Factor`]. - /// - /// Fails if the current [`SpinFactors`] does not include the given - /// [`Factor`] or if the given [`Factor`]'s preparer has not been - /// initialized yet (because it is sequenced after this factor). - pub fn get_mut(&mut self) -> Result<&mut T::InstancePreparer> { - Factors::instance_preparer_mut::(self.inner) - .and_then(|maybe_preparer| maybe_preparer.context("preparer not yet initialized")) - .with_context(|| { - format!( - "could not get instance preparer for {}", - std::any::type_name::() - ) - }) - } -} - -pub struct ConfiguredApp { - app: App, - app_configs: Factors::AppConfigs, -} - -impl ConfiguredApp { - #[doc(hidden)] - pub fn new(app: App, app_configs: Factors::AppConfigs) -> Self { - Self { app, app_configs } - } - - pub fn app(&self) -> &App { - &self.app - } - - pub fn app_config(&self) -> Result<&T::AppConfig> { - Factors::app_config::(&self.app_configs).context("no such factor") - } -} - -/// Implemented by `#[derive(SpinFactors)]` -pub trait SpinFactors: Sized { - type AppConfigs; - type InstancePreparers; - type InstanceState: Send + 'static; - - #[doc(hidden)] - unsafe fn instance_preparer_offset() -> Option; - - #[doc(hidden)] - unsafe fn instance_state_offset() -> Option; - - fn app_config(app_configs: &Self::AppConfigs) -> Option<&T::AppConfig>; - - fn instance_state_getter() -> Option> { - let offset = unsafe { Self::instance_state_offset::()? }; - Some(Getter { - offset, - _phantom: PhantomData, - }) - } - - fn instance_state_getter2( - ) -> Option> { - let offset1 = unsafe { Self::instance_state_offset::()? }; - let offset2 = unsafe { Self::instance_state_offset::()? }; - assert_ne!( - offset1, offset2, - "instance_state_getter2 with same factor twice would alias" - ); - Some(Getter2 { - offset1, - offset2, - _phantom: PhantomData, - }) - } - - fn instance_preparer_mut( - preparers: &mut Self::InstancePreparers, - ) -> Result> { - unsafe { - let offset = Self::instance_preparer_offset::().context("no such factor")?; - let ptr = preparers as *mut Self::InstancePreparers; - let opt = &mut *ptr.add(offset).cast::>(); - Ok(opt.as_mut()) - } - } -} - -pub struct Getter { - offset: usize, - _phantom: PhantomData &mut U>, -} - -impl Getter { - pub fn get_mut<'a>(&self, container: &'a mut T) -> &'a mut U { - let ptr = container as *mut T; - unsafe { &mut *ptr.add(self.offset).cast::() } - } -} - -impl Clone for Getter { - fn clone(&self) -> Self { - *self - } -} -impl Copy for Getter {} - -pub struct Getter2 { - offset1: usize, - offset2: usize, - #[allow(clippy::type_complexity)] - _phantom: PhantomData (&mut U, &mut V)>, -} - -impl Getter2 { - pub fn get_mut<'a>(&self, container: &'a mut T) -> (&'a mut U, &'a mut V) - where - T: 'static, - U: 'static, - V: 'static, - { - let ptr = container as *mut T; - unsafe { - ( - &mut *ptr.add(self.offset1).cast::(), - &mut *ptr.add(self.offset2).cast::(), - ) - } - } -} +// TODO: Add a real Error type +pub type Result = wasmtime::Result; -impl Clone for Getter2 { - fn clone(&self) -> Self { - *self - } +#[doc(hidden)] +pub mod __internal { + pub use crate::runtime_config::RuntimeConfigTracker; } -impl Copy for Getter2 {} diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs new file mode 100644 index 000000000..020707395 --- /dev/null +++ b/crates/factors/src/runtime_config.rs @@ -0,0 +1,94 @@ +use std::collections::HashSet; + +use anyhow::bail; +use serde::de::DeserializeOwned; + +/// RuntimeConfig represents an application's runtime configuration. +/// +/// Runtime configuration is partitioned, with each partition being the +/// responsibility of exactly one [`crate::Factor`]. If configuration needs to +/// be shared between Factors, one Factor can be selected as the owner and the +/// others will have a dependency relationship with that owner. +pub trait RuntimeConfig { + /// Returns deserialized runtime config of the given type for the given + /// factor config key. + /// + /// Returns Ok(None) if no configuration is available for the given key. + /// Returns Err if configuration is available but deserialization fails, + /// or if the given config key has already been retrieved. + fn get_config( + &mut self, + factor_config_key: &str, + ) -> anyhow::Result>; +} + +pub struct RuntimeConfigTracker { + source: Source, + used_keys: HashSet, + unused_keys: HashSet, +} + +impl RuntimeConfigTracker { + #[doc(hidden)] + pub fn new(source: Source) -> Self { + let unused_keys = source.factor_config_keys().map(ToOwned::to_owned).collect(); + Self { + source, + used_keys: Default::default(), + unused_keys, + } + } + + #[doc(hidden)] + pub fn validate_all_keys_used(self) -> Result<(), impl IntoIterator> { + if self.unused_keys.is_empty() { + Ok(()) + } else { + Err(self.unused_keys) + } + } +} + +impl RuntimeConfig for RuntimeConfigTracker { + fn get_config( + &mut self, + factor_config_key: &str, + ) -> anyhow::Result> { + if !self.used_keys.insert(factor_config_key.to_owned()) { + bail!("already got runtime config key {factor_config_key:?}"); + } + self.unused_keys.remove(factor_config_key); + self.source.get_config::(factor_config_key) + } +} + +pub trait RuntimeConfigSource { + /// Returns deserialized runtime config of the given type for the given + /// factor config key. + /// + /// Returns Ok(None) if no configuration is available for the given key. + /// Returns Err if configuration is available but deserialization fails. + fn get_config( + &self, + factor_config_key: &str, + ) -> anyhow::Result>; + + /// Returns an iterator of factor config keys available in this source. + /// + /// Should only include keys that have been positively provided. A runtime + /// may treat unrecognized keys as a warning or error. + fn factor_config_keys(&self) -> impl Iterator; +} + +impl RuntimeConfigSource for () { + fn get_config( + &self, + _factor_config_key: &str, + ) -> anyhow::Result> { + Ok(None) + } + + fn factor_config_keys(&self) -> impl Iterator { + std::iter::empty() + } +} diff --git a/crates/factors/src/spin_factors.rs b/crates/factors/src/spin_factors.rs new file mode 100644 index 000000000..43887aaa5 --- /dev/null +++ b/crates/factors/src/spin_factors.rs @@ -0,0 +1,109 @@ +use std::marker::PhantomData; + +use anyhow::Context; + +use crate::Factor; + +// TODO(lann): Most of the unsafe shenanigans here probably aren't worth it; +// consider replacing with e.g. `Any::downcast`. + +/// Implemented by `#[derive(SpinFactors)]` +pub trait SpinFactors: Sized { + type AppConfigs; + type InstancePreparers; + type InstanceState: Send + 'static; + + #[doc(hidden)] + unsafe fn instance_preparer_offset() -> Option; + + #[doc(hidden)] + unsafe fn instance_state_offset() -> Option; + + fn app_config(app_configs: &Self::AppConfigs) -> Option<&T::AppConfig>; + + fn instance_state_getter() -> Option> { + let offset = unsafe { Self::instance_state_offset::()? }; + Some(Getter { + offset, + _phantom: PhantomData, + }) + } + + fn instance_state_getter2( + ) -> Option> { + let offset1 = unsafe { Self::instance_state_offset::()? }; + let offset2 = unsafe { Self::instance_state_offset::()? }; + assert_ne!( + offset1, offset2, + "instance_state_getter2 with same factor twice would alias" + ); + Some(Getter2 { + offset1, + offset2, + _phantom: PhantomData, + }) + } + + fn instance_preparer_mut( + preparers: &mut Self::InstancePreparers, + ) -> crate::Result> { + unsafe { + let offset = Self::instance_preparer_offset::().context("no such factor")?; + let ptr = preparers as *mut Self::InstancePreparers; + let opt = &mut *ptr.add(offset).cast::>(); + Ok(opt.as_mut()) + } + } +} + +pub struct Getter { + pub(crate) offset: usize, + pub(crate) _phantom: PhantomData &mut U>, +} + +impl Getter { + pub fn get_mut<'a>(&self, container: &'a mut T) -> &'a mut U { + let ptr = container as *mut T; + unsafe { &mut *ptr.add(self.offset).cast::() } + } +} + +impl Clone for Getter { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Getter {} + +pub struct Getter2 { + pub(crate) offset1: usize, + pub(crate) offset2: usize, + #[allow(clippy::type_complexity)] + pub(crate) _phantom: PhantomData (&mut U, &mut V)>, +} + +impl Getter2 { + pub fn get_mut<'a>(&self, container: &'a mut T) -> (&'a mut U, &'a mut V) + where + T: 'static, + U: 'static, + V: 'static, + { + let ptr = container as *mut T; + unsafe { + ( + &mut *ptr.add(self.offset1).cast::(), + &mut *ptr.add(self.offset2).cast::(), + ) + } + } +} + +impl Clone for Getter2 { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Getter2 {} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index a4157bc6a..cf341e466 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -43,7 +43,7 @@ fn main() -> anyhow::Result<()> { .init(Some(&mut linker), Some(&mut module_linker)) .unwrap(); - let configured_app = factors.configure_app(app).unwrap(); + let configured_app = factors.configure_app(app, ()).unwrap(); let data = factors.build_store_data(&configured_app, "test").unwrap(); let mut store = wasmtime::Store::new(&engine, data); From 5985dcb9ab084a75598f37d346bfed2ae1db9a7e Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 6 Jun 2024 13:47:49 -0400 Subject: [PATCH 010/195] factors: Rename SpinFactors->RuntimeFactors And shorten some generic params. Signed-off-by: Lann Martin --- crates/factor-outbound-networking/src/lib.rs | 14 ++-- crates/factor-variables/src/lib.rs | 16 ++--- crates/factor-wasi/src/lib.rs | 8 +-- crates/factor-wasi/src/preview1.rs | 8 +-- crates/factors-derive/src/lib.rs | 12 ++-- crates/factors/src/factor.rs | 75 ++++++++++---------- crates/factors/src/instance_preparer.rs | 54 +++++++------- crates/factors/src/lib.rs | 8 +-- crates/factors/src/spin_factors.rs | 32 ++++----- crates/factors/tests/smoke.rs | 4 +- 10 files changed, 114 insertions(+), 117 deletions(-) diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index 7387d69a2..5013725ec 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -9,22 +9,22 @@ use spin_factor_wasi::WasiFactor; use spin_factors::{ anyhow::{self, Context}, ConfigureAppContext, Factor, FactorInstancePreparer, InstancePreparers, PrepareContext, - RuntimeConfig, SpinFactors, + RuntimeConfig, RuntimeFactors, }; use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; pub struct OutboundNetworkingFactor; impl Factor for OutboundNetworkingFactor { - type AppConfig = AppConfig; + type AppState = AppState; type InstancePreparer = InstancePreparer; type InstanceState = (); - fn configure_app( + fn configure_app( &self, ctx: ConfigureAppContext, _runtime_config: &mut impl RuntimeConfig, - ) -> anyhow::Result { + ) -> anyhow::Result { // Extract allowed_outbound_hosts for all components let component_allowed_hosts = ctx .app() @@ -40,14 +40,14 @@ impl Factor for OutboundNetworkingFactor { )) }) .collect::>()?; - Ok(AppConfig { + Ok(AppState { component_allowed_hosts, }) } } #[derive(Default)] -pub struct AppConfig { +pub struct AppState { component_allowed_hosts: HashMap>, } @@ -58,7 +58,7 @@ pub struct InstancePreparer { } impl FactorInstancePreparer for InstancePreparer { - fn new( + fn new( ctx: PrepareContext, mut preparers: InstancePreparers, ) -> anyhow::Result { diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 423c8bba8..68b1da814 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -3,18 +3,18 @@ use std::sync::Arc; use spin_expressions::ProviderResolver; use spin_factors::{ anyhow, ConfigureAppContext, Factor, FactorInstancePreparer, InitContext, InstancePreparers, - PrepareContext, RuntimeConfig, SpinFactors, + PrepareContext, RuntimeConfig, RuntimeFactors, }; use spin_world::{async_trait, v1::config as v1_config, v2::variables}; pub struct VariablesFactor; impl Factor for VariablesFactor { - type AppConfig = AppConfig; + type AppState = AppState; type InstancePreparer = InstancePreparer; type InstanceState = InstanceState; - fn init( + fn init( &mut self, mut ctx: InitContext, ) -> anyhow::Result<()> { @@ -23,11 +23,11 @@ impl Factor for VariablesFactor { Ok(()) } - fn configure_app( + fn configure_app( &self, ctx: ConfigureAppContext, _runtime_config: &mut impl RuntimeConfig, - ) -> anyhow::Result { + ) -> anyhow::Result { let app = ctx.app(); let mut resolver = ProviderResolver::new(app.variables().map(|(key, val)| (key.clone(), val.clone())))?; @@ -38,14 +38,14 @@ impl Factor for VariablesFactor { )?; } // TODO: add providers from runtime config - Ok(AppConfig { + Ok(AppState { resolver: Arc::new(resolver), }) } } #[derive(Default)] -pub struct AppConfig { +pub struct AppState { resolver: Arc, } @@ -60,7 +60,7 @@ impl InstancePreparer { } impl FactorInstancePreparer for InstancePreparer { - fn new( + fn new( ctx: PrepareContext, _preparers: InstancePreparers, ) -> anyhow::Result { diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 6320682da..7af726ff7 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -4,7 +4,7 @@ use std::{future::Future, net::SocketAddr, path::Path}; use spin_factors::{ anyhow, AppComponent, Factor, FactorInstancePreparer, InitContext, InstancePreparers, - PrepareContext, SpinFactors, + PrepareContext, RuntimeFactors, }; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; @@ -21,11 +21,11 @@ impl WasiFactor { } impl Factor for WasiFactor { - type AppConfig = (); + type AppState = (); type InstancePreparer = InstancePreparer; type InstanceState = InstanceState; - fn init( + fn init( &mut self, mut ctx: InitContext, ) -> anyhow::Result<()> { @@ -124,7 +124,7 @@ pub struct InstancePreparer { impl FactorInstancePreparer for InstancePreparer { // NOTE: Replaces WASI parts of AppComponent::apply_store_config - fn new( + fn new( ctx: PrepareContext, _preparers: InstancePreparers, ) -> anyhow::Result { diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 426ebff6d..88a6bc5b6 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,17 +1,17 @@ use spin_factors::{ anyhow, Factor, FactorInstancePreparer, InitContext, InstancePreparers, PrepareContext, - SpinFactors, + RuntimeFactors, }; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; pub struct WasiPreview1Factor; impl Factor for WasiPreview1Factor { - type AppConfig = (); + type AppState = (); type InstancePreparer = InstancePreparer; type InstanceState = WasiP1Ctx; - fn init( + fn init( &mut self, mut ctx: InitContext, ) -> anyhow::Result<()> { @@ -24,7 +24,7 @@ pub struct InstancePreparer { } impl FactorInstancePreparer for InstancePreparer { - fn new( + fn new( _ctx: PrepareContext, _preparers: InstancePreparers, ) -> anyhow::Result { diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 399606708..62439bcca 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -2,7 +2,7 @@ use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::{parse_macro_input, Data, DeriveInput, Error}; -#[proc_macro_derive(SpinFactors)] +#[proc_macro_derive(RuntimeFactors)] pub fn derive_factors(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let input = parse_macro_input!(input as DeriveInput); let expanded = expand_factors(&input).unwrap_or_else(|err| err.into_compile_error()); @@ -20,7 +20,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let name = &input.ident; let vis = &input.vis; - let app_configs_name = format_ident!("{name}AppConfigs"); + let app_configs_name = format_ident!("{name}AppState"); let preparers_name = format_ident!("{name}InstancePreparers"); let state_name = format_ident!("{name}InstanceState"); @@ -133,8 +133,8 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } - impl #factors_path::SpinFactors for #name { - type AppConfigs = #app_configs_name; + impl #factors_path::RuntimeFactors for #name { + type AppState = #app_configs_name; type InstancePreparers = #preparers_name; type InstanceState = #state_name; @@ -159,7 +159,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } - fn app_config(app_configs: &Self::AppConfigs) -> Option<&T::AppConfig> { + fn app_config(app_configs: &Self::AppState) -> Option<&T::AppState> { let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { @@ -172,7 +172,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #vis struct #app_configs_name { #( - pub #factor_names: Option<<#factor_types as #Factor>::AppConfig>, + pub #factor_names: Option<<#factor_types as #Factor>::AppState>, )* } diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 552e84417..162ddb8d6 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -2,13 +2,13 @@ use std::any::Any; use anyhow::Context; -use crate::{App, FactorInstancePreparer, Linker, ModuleLinker, RuntimeConfig, SpinFactors}; +use crate::{App, FactorInstancePreparer, Linker, ModuleLinker, RuntimeConfig, RuntimeFactors}; pub trait Factor: Any + Sized { - /// Per-app configuration for this factor. + /// Per-app state for this factor. /// /// See [`Factor::configure_app`]. - type AppConfig: Default; + type AppState: Default; /// The [`FactorInstancePreparer`] for this factor. type InstancePreparer: FactorInstancePreparer; @@ -20,10 +20,7 @@ pub trait Factor: Any + Sized { /// Initializes this Factor for a runtime. This will be called at most once, /// before any call to [`FactorInstancePreparer::new`] - fn init( - &mut self, - mut ctx: InitContext, - ) -> anyhow::Result<()> { + fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { // TODO: Should `ctx` always be immut? Rename this param/type? _ = &mut ctx; Ok(()) @@ -33,33 +30,33 @@ pub trait Factor: Any + Sized { /// [`App`]. A runtime may - but is not required to - reuse the returned /// config across multiple instances. Note that this may be called without /// any call to `init` in cases where only validation is needed. - fn configure_app( + fn configure_app( &self, - ctx: ConfigureAppContext, + ctx: ConfigureAppContext, _runtime_config: &mut impl RuntimeConfig, - ) -> anyhow::Result { + ) -> anyhow::Result { _ = ctx; Ok(Default::default()) } } -pub(crate) type GetDataFn = - fn(&mut ::InstanceState) -> &mut ::InstanceState; +pub(crate) type GetDataFn = + fn(&mut ::InstanceState) -> &mut ::InstanceState; /// An InitContext is passed to [`Factor::init`], giving access to the global /// common [`wasmtime::component::Linker`]. -pub struct InitContext<'a, Factors: SpinFactors, T: Factor> { - pub(crate) linker: Option<&'a mut Linker>, - pub(crate) module_linker: Option<&'a mut ModuleLinker>, - pub(crate) get_data: GetDataFn, +pub struct InitContext<'a, T: RuntimeFactors, F: Factor> { + pub(crate) linker: Option<&'a mut Linker>, + pub(crate) module_linker: Option<&'a mut ModuleLinker>, + pub(crate) get_data: GetDataFn, } -impl<'a, Factors: SpinFactors, T: Factor> InitContext<'a, Factors, T> { +impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { #[doc(hidden)] pub fn new( - linker: Option<&'a mut Linker>, - module_linker: Option<&'a mut ModuleLinker>, - get_data: GetDataFn, + linker: Option<&'a mut Linker>, + module_linker: Option<&'a mut ModuleLinker>, + get_data: GetDataFn, ) -> Self { Self { linker, @@ -68,23 +65,23 @@ impl<'a, Factors: SpinFactors, T: Factor> InitContext<'a, Factors, T> { } } - pub fn linker(&mut self) -> Option<&mut Linker> { + pub fn linker(&mut self) -> Option<&mut Linker> { self.linker.as_deref_mut() } - pub fn module_linker(&mut self) -> Option<&mut ModuleLinker> { + pub fn module_linker(&mut self) -> Option<&mut ModuleLinker> { self.module_linker.as_deref_mut() } - pub fn get_data_fn(&self) -> GetDataFn { + pub fn get_data_fn(&self) -> GetDataFn { self.get_data } pub fn link_bindings( &mut self, add_to_linker: impl Fn( - &mut Linker, - fn(&mut Factors::InstanceState) -> &mut T::InstanceState, + &mut Linker, + fn(&mut T::InstanceState) -> &mut F::InstanceState, ) -> anyhow::Result<()>, ) -> anyhow::Result<()> where { @@ -98,8 +95,8 @@ where { pub fn link_module_bindings( &mut self, add_to_linker: impl Fn( - &mut ModuleLinker, - fn(&mut Factors::InstanceState) -> &mut T::InstanceState, + &mut ModuleLinker, + fn(&mut T::InstanceState) -> &mut F::InstanceState, ) -> anyhow::Result<()>, ) -> anyhow::Result<()> where { @@ -111,14 +108,14 @@ where { } } -pub struct ConfigureAppContext<'a, Factors: SpinFactors> { +pub struct ConfigureAppContext<'a, T: RuntimeFactors> { pub(crate) app: &'a App, - pub(crate) app_configs: &'a Factors::AppConfigs, + pub(crate) app_configs: &'a T::AppState, } -impl<'a, Factors: SpinFactors> ConfigureAppContext<'a, Factors> { +impl<'a, T: RuntimeFactors> ConfigureAppContext<'a, T> { #[doc(hidden)] - pub fn new(app: &'a App, app_configs: &'a Factors::AppConfigs) -> Self { + pub fn new(app: &'a App, app_configs: &'a T::AppState) -> Self { Self { app, app_configs } } @@ -126,19 +123,19 @@ impl<'a, Factors: SpinFactors> ConfigureAppContext<'a, Factors> { self.app } - pub fn app_config(&self) -> crate::Result<&T::AppConfig> { - Factors::app_config::(self.app_configs).context("no such factor") + pub fn app_config(&self) -> crate::Result<&F::AppState> { + T::app_config::(self.app_configs).context("no such factor") } } -pub struct ConfiguredApp { +pub struct ConfiguredApp { app: App, - app_configs: Factors::AppConfigs, + app_configs: T::AppState, } -impl ConfiguredApp { +impl ConfiguredApp { #[doc(hidden)] - pub fn new(app: App, app_configs: Factors::AppConfigs) -> Self { + pub fn new(app: App, app_configs: T::AppState) -> Self { Self { app, app_configs } } @@ -146,7 +143,7 @@ impl ConfiguredApp { &self.app } - pub fn app_config(&self) -> crate::Result<&T::AppConfig> { - Factors::app_config::(&self.app_configs).context("no such factor") + pub fn app_config(&self) -> crate::Result<&F::AppState> { + T::app_config::(&self.app_configs).context("no such factor") } } diff --git a/crates/factors/src/instance_preparer.rs b/crates/factors/src/instance_preparer.rs index 43fb01fce..8cec89f35 100644 --- a/crates/factors/src/instance_preparer.rs +++ b/crates/factors/src/instance_preparer.rs @@ -1,30 +1,30 @@ use anyhow::Context; -use crate::{AppComponent, Factor, SpinFactors}; +use crate::{AppComponent, Factor, RuntimeFactors}; -pub trait FactorInstancePreparer: Sized { +pub trait FactorInstancePreparer: Sized { /// Returns a new instance of this preparer for the given [`Factor`]. - fn new( - ctx: PrepareContext, - _preparers: InstancePreparers, + fn new( + ctx: PrepareContext, + _preparers: InstancePreparers, ) -> anyhow::Result; /// Returns a new instance of the associated [`Factor::InstanceState`]. - fn prepare(self) -> anyhow::Result; + fn prepare(self) -> anyhow::Result; } -impl FactorInstancePreparer for () +impl FactorInstancePreparer for () where - T::InstanceState: Default, + F::InstanceState: Default, { - fn new( - _ctx: PrepareContext, - _preparers: InstancePreparers, + fn new( + _ctx: PrepareContext, + _preparers: InstancePreparers, ) -> anyhow::Result { Ok(()) } - fn prepare(self) -> anyhow::Result { + fn prepare(self) -> anyhow::Result { Ok(Default::default()) } } @@ -32,17 +32,17 @@ where /// A PrepareContext is passed to [`FactorInstancePreparer::new`], giving access /// to any already-initialized [`FactorInstancePreparer`]s, allowing for /// inter-[`Factor`] dependencies. -pub struct PrepareContext<'a, T: Factor> { - pub(crate) factor: &'a T, - pub(crate) app_config: &'a T::AppConfig, +pub struct PrepareContext<'a, F: Factor> { + pub(crate) factor: &'a F, + pub(crate) app_config: &'a F::AppState, pub(crate) app_component: &'a AppComponent<'a>, } -impl<'a, T: Factor> PrepareContext<'a, T> { +impl<'a, F: Factor> PrepareContext<'a, F> { #[doc(hidden)] pub fn new( - factor: &'a T, - app_config: &'a T::AppConfig, + factor: &'a F, + app_config: &'a F::AppState, app_component: &'a AppComponent, ) -> Self { Self { @@ -52,11 +52,11 @@ impl<'a, T: Factor> PrepareContext<'a, T> { } } - pub fn factor(&self) -> &T { + pub fn factor(&self) -> &F { self.factor } - pub fn app_config(&self) -> &T::AppConfig { + pub fn app_config(&self) -> &F::AppState { self.app_config } @@ -65,22 +65,22 @@ impl<'a, T: Factor> PrepareContext<'a, T> { } } -pub struct InstancePreparers<'a, Factors: SpinFactors> { - pub(crate) inner: &'a mut Factors::InstancePreparers, +pub struct InstancePreparers<'a, T: RuntimeFactors> { + pub(crate) inner: &'a mut T::InstancePreparers, } -impl<'a, Factors: SpinFactors> InstancePreparers<'a, Factors> { +impl<'a, T: RuntimeFactors> InstancePreparers<'a, T> { #[doc(hidden)] - pub fn new(inner: &'a mut Factors::InstancePreparers) -> Self { + pub fn new(inner: &'a mut T::InstancePreparers) -> Self { Self { inner } } /// Returns a already-initialized preparer for the given [`Factor`]. /// - /// Fails if the current [`SpinFactors`] does not include the given + /// Fails if the current [`RuntimeFactors`] does not include the given /// [`Factor`] or if the given [`Factor`]'s preparer has not been /// initialized yet (because it is sequenced after this factor). - pub fn get_mut(&mut self) -> crate::Result<&mut T::InstancePreparer> { - Factors::instance_preparer_mut::(self.inner)?.context("preparer not initialized") + pub fn get_mut(&mut self) -> crate::Result<&mut F::InstancePreparer> { + T::instance_preparer_mut::(self.inner)?.context("preparer not initialized") } } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 0d18109cf..a23644e2b 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -6,17 +6,17 @@ mod spin_factors; pub use anyhow; pub use wasmtime; -pub use spin_factors_derive::SpinFactors; +pub use spin_factors_derive::RuntimeFactors; pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, InitContext}, instance_preparer::{FactorInstancePreparer, InstancePreparers, PrepareContext}, runtime_config::{RuntimeConfig, RuntimeConfigSource}, - spin_factors::SpinFactors, + spin_factors::RuntimeFactors, }; -pub type Linker = wasmtime::component::Linker<::InstanceState>; -pub type ModuleLinker = wasmtime::Linker<::InstanceState>; +pub type Linker = wasmtime::component::Linker<::InstanceState>; +pub type ModuleLinker = wasmtime::Linker<::InstanceState>; // Temporary wrappers while refactoring pub type App = spin_app::App<'static, spin_app::InertLoader>; diff --git a/crates/factors/src/spin_factors.rs b/crates/factors/src/spin_factors.rs index 43887aaa5..fd961e679 100644 --- a/crates/factors/src/spin_factors.rs +++ b/crates/factors/src/spin_factors.rs @@ -7,32 +7,32 @@ use crate::Factor; // TODO(lann): Most of the unsafe shenanigans here probably aren't worth it; // consider replacing with e.g. `Any::downcast`. -/// Implemented by `#[derive(SpinFactors)]` -pub trait SpinFactors: Sized { - type AppConfigs; +/// Implemented by `#[derive(RuntimeFactors)]` +pub trait RuntimeFactors: Sized { + type AppState; type InstancePreparers; type InstanceState: Send + 'static; #[doc(hidden)] - unsafe fn instance_preparer_offset() -> Option; + unsafe fn instance_preparer_offset() -> Option; #[doc(hidden)] - unsafe fn instance_state_offset() -> Option; + unsafe fn instance_state_offset() -> Option; - fn app_config(app_configs: &Self::AppConfigs) -> Option<&T::AppConfig>; + fn app_config(app_configs: &Self::AppState) -> Option<&F::AppState>; - fn instance_state_getter() -> Option> { - let offset = unsafe { Self::instance_state_offset::()? }; + fn instance_state_getter() -> Option> { + let offset = unsafe { Self::instance_state_offset::()? }; Some(Getter { offset, _phantom: PhantomData, }) } - fn instance_state_getter2( - ) -> Option> { - let offset1 = unsafe { Self::instance_state_offset::()? }; - let offset2 = unsafe { Self::instance_state_offset::()? }; + fn instance_state_getter2( + ) -> Option> { + let offset1 = unsafe { Self::instance_state_offset::()? }; + let offset2 = unsafe { Self::instance_state_offset::()? }; assert_ne!( offset1, offset2, "instance_state_getter2 with same factor twice would alias" @@ -44,13 +44,13 @@ pub trait SpinFactors: Sized { }) } - fn instance_preparer_mut( + fn instance_preparer_mut( preparers: &mut Self::InstancePreparers, - ) -> crate::Result> { + ) -> crate::Result> { unsafe { - let offset = Self::instance_preparer_offset::().context("no such factor")?; + let offset = Self::instance_preparer_offset::().context("no such factor")?; let ptr = preparers as *mut Self::InstancePreparers; - let opt = &mut *ptr.add(offset).cast::>(); + let opt = &mut *ptr.add(offset).cast::>(); Ok(opt.as_mut()) } } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index cf341e466..c848be746 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -2,9 +2,9 @@ use spin_app::App; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{preview1::WasiPreview1Factor, DummyFilesMounter, WasiFactor}; -use spin_factors::SpinFactors; +use spin_factors::RuntimeFactors; -#[derive(SpinFactors)] +#[derive(RuntimeFactors)] struct Factors { wasi: WasiFactor, wasip1: WasiPreview1Factor, From b774ac0d4c5331554e4868ca9af8bdf657888dc1 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 6 Jun 2024 14:39:43 -0400 Subject: [PATCH 011/195] factors: Merge FactorInstancePreparer trait into Factor Signed-off-by: Lann Martin --- crates/factor-outbound-networking/src/lib.rs | 56 +++++++++------- crates/factor-variables/src/lib.rs | 44 ++++++------- crates/factor-wasi/src/lib.rs | 66 ++++++++++--------- crates/factor-wasi/src/preview1.rs | 24 +++---- crates/factors-derive/src/lib.rs | 5 +- crates/factors/src/factor.rs | 18 ++++- crates/factors/src/lib.rs | 4 +- .../src/{instance_preparer.rs => prepare.rs} | 27 -------- 8 files changed, 119 insertions(+), 125 deletions(-) rename crates/factors/src/{instance_preparer.rs => prepare.rs} (69%) diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index 5013725ec..fb48ba1c8 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -8,8 +8,7 @@ use spin_factor_variables::VariablesFactor; use spin_factor_wasi::WasiFactor; use spin_factors::{ anyhow::{self, Context}, - ConfigureAppContext, Factor, FactorInstancePreparer, InstancePreparers, PrepareContext, - RuntimeConfig, RuntimeFactors, + ConfigureAppContext, Factor, InstancePreparers, PrepareContext, RuntimeConfig, RuntimeFactors, }; use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; @@ -44,24 +43,11 @@ impl Factor for OutboundNetworkingFactor { component_allowed_hosts, }) } -} - -#[derive(Default)] -pub struct AppState { - component_allowed_hosts: HashMap>, -} - -type SharedFutureResult = Shared>>>; - -pub struct InstancePreparer { - allowed_hosts_future: SharedFutureResult, -} -impl FactorInstancePreparer for InstancePreparer { - fn new( - ctx: PrepareContext, - mut preparers: InstancePreparers, - ) -> anyhow::Result { + fn create_preparer( + ctx: PrepareContext, + mut preparers: InstancePreparers, + ) -> anyhow::Result { let hosts = ctx .app_config() .component_allowed_hosts @@ -101,18 +87,40 @@ impl FactorInstancePreparer for InstancePreparer { } } }); - Ok(Self { - allowed_hosts_future, - }) + Ok(InstancePreparer::new(allowed_hosts_future)) } - fn prepare(self) -> anyhow::Result<::InstanceState> { + fn prepare( + &self, + _preparer: InstancePreparer, + ) -> anyhow::Result<::InstanceState> { Ok(()) } } +#[derive(Default)] +pub struct AppState { + component_allowed_hosts: HashMap>, +} + +type SharedFutureResult = Shared>>>; + +#[derive(Default)] +pub struct InstancePreparer { + allowed_hosts_future: Option>, +} + impl InstancePreparer { + fn new(allowed_hosts_future: SharedFutureResult) -> Self { + Self { + allowed_hosts_future: Some(allowed_hosts_future), + } + } + pub async fn resolve_allowed_hosts(&self) -> Arc> { - self.allowed_hosts_future.clone().await + self.allowed_hosts_future + .clone() + .expect("allowed_hosts_future not set") + .await } } diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 68b1da814..ca6eb4253 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use spin_expressions::ProviderResolver; use spin_factors::{ - anyhow, ConfigureAppContext, Factor, FactorInstancePreparer, InitContext, InstancePreparers, - PrepareContext, RuntimeConfig, RuntimeFactors, + anyhow, ConfigureAppContext, Factor, InitContext, InstancePreparers, PrepareContext, + RuntimeConfig, RuntimeFactors, }; use spin_world::{async_trait, v1::config as v1_config, v2::variables}; @@ -42,6 +42,24 @@ impl Factor for VariablesFactor { resolver: Arc::new(resolver), }) } + + fn create_preparer( + ctx: PrepareContext, + _preparers: InstancePreparers, + ) -> anyhow::Result { + let component_id = ctx.app_component().id().to_string(); + let resolver = ctx.app_config().resolver.clone(); + Ok(InstancePreparer { + state: InstanceState { + component_id, + resolver, + }, + }) + } + + fn prepare(&self, preparer: InstancePreparer) -> anyhow::Result { + Ok(preparer.state) + } } #[derive(Default)] @@ -49,6 +67,7 @@ pub struct AppState { resolver: Arc, } +#[derive(Default)] pub struct InstancePreparer { state: InstanceState, } @@ -59,26 +78,7 @@ impl InstancePreparer { } } -impl FactorInstancePreparer for InstancePreparer { - fn new( - ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result { - let component_id = ctx.app_component().id().to_string(); - let resolver = ctx.app_config().resolver.clone(); - Ok(Self { - state: InstanceState { - component_id, - resolver, - }, - }) - } - - fn prepare(self) -> anyhow::Result { - Ok(self.state) - } -} - +#[derive(Default)] pub struct InstanceState { component_id: String, resolver: Arc, diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 7af726ff7..1e97ab55f 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -3,8 +3,7 @@ pub mod preview1; use std::{future::Future, net::SocketAddr, path::Path}; use spin_factors::{ - anyhow, AppComponent, Factor, FactorInstancePreparer, InitContext, InstancePreparers, - PrepareContext, RuntimeFactors, + anyhow, AppComponent, Factor, InitContext, InstancePreparers, PrepareContext, RuntimeFactors, }; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; @@ -69,6 +68,36 @@ impl Factor for WasiFactor { } Ok(()) } + + fn create_preparer( + ctx: PrepareContext, + _preparers: InstancePreparers, + ) -> anyhow::Result { + let mut wasi_ctx = WasiCtxBuilder::new(); + + // Apply environment variables + for (key, val) in ctx.app_component().environment() { + wasi_ctx.env(key, val); + } + + // Mount files + let mount_ctx = MountFilesContext { + wasi_ctx: &mut wasi_ctx, + }; + ctx.factor() + .files_mounter + .mount_files(ctx.app_component(), mount_ctx)?; + + Ok(InstancePreparer { wasi_ctx }) + } + + fn prepare(&self, preparer: InstancePreparer) -> anyhow::Result { + let InstancePreparer { mut wasi_ctx } = preparer; + Ok(InstanceState { + ctx: wasi_ctx.build(), + table: Default::default(), + }) + } } pub trait FilesMounter { @@ -122,36 +151,11 @@ pub struct InstancePreparer { wasi_ctx: WasiCtxBuilder, } -impl FactorInstancePreparer for InstancePreparer { - // NOTE: Replaces WASI parts of AppComponent::apply_store_config - fn new( - ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result { - let mut wasi_ctx = WasiCtxBuilder::new(); - - // Apply environment variables - for (key, val) in ctx.app_component().environment() { - wasi_ctx.env(key, val); +impl Default for InstancePreparer { + fn default() -> Self { + Self { + wasi_ctx: WasiCtxBuilder::new(), } - - // Mount files - let mount_ctx = MountFilesContext { - wasi_ctx: &mut wasi_ctx, - }; - ctx.factor() - .files_mounter - .mount_files(ctx.app_component(), mount_ctx)?; - - Ok(Self { wasi_ctx }) - } - - fn prepare(self) -> anyhow::Result { - let Self { mut wasi_ctx } = self; - Ok(InstanceState { - ctx: wasi_ctx.build(), - table: Default::default(), - }) } } diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 88a6bc5b6..ab3a7a76e 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,7 +1,4 @@ -use spin_factors::{ - anyhow, Factor, FactorInstancePreparer, InitContext, InstancePreparers, PrepareContext, - RuntimeFactors, -}; +use spin_factors::{anyhow, Factor, InitContext, RuntimeFactors}; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; pub struct WasiPreview1Factor; @@ -17,23 +14,20 @@ impl Factor for WasiPreview1Factor { ) -> anyhow::Result<()> { ctx.link_module_bindings(wasmtime_wasi::preview1::add_to_linker_async) } + + fn prepare(&self, mut preparer: InstancePreparer) -> anyhow::Result { + Ok(preparer.wasi_ctx.build_p1()) + } } pub struct InstancePreparer { wasi_ctx: WasiCtxBuilder, } -impl FactorInstancePreparer for InstancePreparer { - fn new( - _ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result { - Ok(Self { +impl Default for InstancePreparer { + fn default() -> Self { + Self { wasi_ctx: WasiCtxBuilder::new(), - }) - } - - fn prepare(mut self) -> anyhow::Result { - Ok(self.wasi_ctx.build_p1()) + } } } diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 62439bcca..f6dede3f2 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -112,7 +112,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { }; #( preparers.#factor_names = Some( - #factors_path::FactorInstancePreparer::<#factor_types>::new::<#name>( + #Factor::create_preparer::( #factors_path::PrepareContext::new( &self.#factor_names, configured_app.app_config::<#factor_types>().unwrap(), @@ -124,7 +124,8 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { )* Ok(#state_name { #( - #factor_names: #factors_path::FactorInstancePreparer::<#factor_types>::prepare( + #factor_names: #Factor::prepare( + &self.#factor_names, preparers.#factor_names.unwrap(), )?, )* diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 162ddb8d6..79b1f86f1 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -2,7 +2,9 @@ use std::any::Any; use anyhow::Context; -use crate::{App, FactorInstancePreparer, Linker, ModuleLinker, RuntimeConfig, RuntimeFactors}; +use crate::{ + App, InstancePreparers, Linker, ModuleLinker, PrepareContext, RuntimeConfig, RuntimeFactors, +}; pub trait Factor: Any + Sized { /// Per-app state for this factor. @@ -11,7 +13,7 @@ pub trait Factor: Any + Sized { type AppState: Default; /// The [`FactorInstancePreparer`] for this factor. - type InstancePreparer: FactorInstancePreparer; + type InstancePreparer: Default; /// The per-instance state for this factor, constructed by a /// [`FactorInstancePreparer`] and available to any host-provided imports @@ -38,6 +40,18 @@ pub trait Factor: Any + Sized { _ = ctx; Ok(Default::default()) } + + /// Returns a new instance of this preparer for the given [`Factor`]. + fn create_preparer( + ctx: PrepareContext, + _preparers: InstancePreparers, + ) -> anyhow::Result { + _ = ctx; + Ok(Default::default()) + } + + /// Returns a new instance of the associated [`Factor::InstanceState`]. + fn prepare(&self, preparer: Self::InstancePreparer) -> anyhow::Result; } pub(crate) type GetDataFn = diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index a23644e2b..eb34e6b37 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -1,5 +1,5 @@ mod factor; -mod instance_preparer; +mod prepare; mod runtime_config; mod spin_factors; @@ -10,7 +10,7 @@ pub use spin_factors_derive::RuntimeFactors; pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, InitContext}, - instance_preparer::{FactorInstancePreparer, InstancePreparers, PrepareContext}, + prepare::{InstancePreparers, PrepareContext}, runtime_config::{RuntimeConfig, RuntimeConfigSource}, spin_factors::RuntimeFactors, }; diff --git a/crates/factors/src/instance_preparer.rs b/crates/factors/src/prepare.rs similarity index 69% rename from crates/factors/src/instance_preparer.rs rename to crates/factors/src/prepare.rs index 8cec89f35..ccf975e8f 100644 --- a/crates/factors/src/instance_preparer.rs +++ b/crates/factors/src/prepare.rs @@ -2,33 +2,6 @@ use anyhow::Context; use crate::{AppComponent, Factor, RuntimeFactors}; -pub trait FactorInstancePreparer: Sized { - /// Returns a new instance of this preparer for the given [`Factor`]. - fn new( - ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result; - - /// Returns a new instance of the associated [`Factor::InstanceState`]. - fn prepare(self) -> anyhow::Result; -} - -impl FactorInstancePreparer for () -where - F::InstanceState: Default, -{ - fn new( - _ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result { - Ok(()) - } - - fn prepare(self) -> anyhow::Result { - Ok(Default::default()) - } -} - /// A PrepareContext is passed to [`FactorInstancePreparer::new`], giving access /// to any already-initialized [`FactorInstancePreparer`]s, allowing for /// inter-[`Factor`] dependencies. From 14f32411b898a8091afa9fd44d3d9cd1b039e4bd Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 7 Jun 2024 10:06:55 -0400 Subject: [PATCH 012/195] factors: Add RuntimeConfig Also address a bunch of feedback. Signed-off-by: Lann Martin --- crates/factor-outbound-networking/src/lib.rs | 45 ++++---- crates/factor-variables/src/lib.rs | 40 +++---- crates/factor-wasi/src/lib.rs | 41 ++++--- crates/factor-wasi/src/preview1.rs | 28 +++-- crates/factors-derive/src/lib.rs | 55 +++++----- crates/factors/src/factor.rs | 87 ++++++++------- crates/factors/src/lib.rs | 8 +- crates/factors/src/prepare.rs | 54 +++++---- crates/factors/src/runtime_config.rs | 103 ++++++++---------- .../{spin_factors.rs => runtime_factors.rs} | 26 +++-- 10 files changed, 255 insertions(+), 232 deletions(-) rename crates/factors/src/{spin_factors.rs => runtime_factors.rs} (75%) diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index fb48ba1c8..e19f6d607 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -8,21 +8,21 @@ use spin_factor_variables::VariablesFactor; use spin_factor_wasi::WasiFactor; use spin_factors::{ anyhow::{self, Context}, - ConfigureAppContext, Factor, InstancePreparers, PrepareContext, RuntimeConfig, RuntimeFactors, + ConfigureAppContext, Factor, FactorInstanceBuilder, InstanceBuilders, PrepareContext, + RuntimeFactors, }; use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; pub struct OutboundNetworkingFactor; impl Factor for OutboundNetworkingFactor { + type RuntimeConfig = (); type AppState = AppState; - type InstancePreparer = InstancePreparer; - type InstanceState = (); + type InstanceBuilder = InstanceBuilder; - fn configure_app( + fn configure_app( &self, - ctx: ConfigureAppContext, - _runtime_config: &mut impl RuntimeConfig, + ctx: ConfigureAppContext, ) -> anyhow::Result { // Extract allowed_outbound_hosts for all components let component_allowed_hosts = ctx @@ -44,17 +44,17 @@ impl Factor for OutboundNetworkingFactor { }) } - fn create_preparer( + fn prepare( ctx: PrepareContext, - mut preparers: InstancePreparers, - ) -> anyhow::Result { + builders: &mut InstanceBuilders, + ) -> anyhow::Result { let hosts = ctx - .app_config() + .app_state() .component_allowed_hosts .get(ctx.app_component().id()) .cloned() .context("missing component allowed hosts")?; - let resolver = preparers.get_mut::()?.resolver().clone(); + let resolver = builders.get_mut::()?.resolver().clone(); let allowed_hosts_future = async move { let prepared = resolver.prepare().await?; AllowedHostsConfig::parse(&hosts, &prepared) @@ -69,7 +69,7 @@ impl Factor for OutboundNetworkingFactor { // )?; // Update Wasi socket allowed ports - let wasi_preparer = preparers.get_mut::()?; + let wasi_preparer = builders.get_mut::()?; let hosts_future = allowed_hosts_future.clone(); wasi_preparer.outbound_socket_addr_check(move |addr| { let hosts_future = hosts_future.clone(); @@ -87,14 +87,7 @@ impl Factor for OutboundNetworkingFactor { } } }); - Ok(InstancePreparer::new(allowed_hosts_future)) - } - - fn prepare( - &self, - _preparer: InstancePreparer, - ) -> anyhow::Result<::InstanceState> { - Ok(()) + Ok(InstanceBuilder::new(allowed_hosts_future)) } } @@ -106,11 +99,11 @@ pub struct AppState { type SharedFutureResult = Shared>>>; #[derive(Default)] -pub struct InstancePreparer { +pub struct InstanceBuilder { allowed_hosts_future: Option>, } -impl InstancePreparer { +impl InstanceBuilder { fn new(allowed_hosts_future: SharedFutureResult) -> Self { Self { allowed_hosts_future: Some(allowed_hosts_future), @@ -124,3 +117,11 @@ impl InstancePreparer { .await } } + +impl FactorInstanceBuilder for InstanceBuilder { + type InstanceState = (); + + fn build(self) -> anyhow::Result { + Ok(()) + } +} diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index ca6eb4253..438e01679 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -2,17 +2,17 @@ use std::sync::Arc; use spin_expressions::ProviderResolver; use spin_factors::{ - anyhow, ConfigureAppContext, Factor, InitContext, InstancePreparers, PrepareContext, - RuntimeConfig, RuntimeFactors, + anyhow, ConfigureAppContext, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, + PrepareContext, RuntimeFactors, }; use spin_world::{async_trait, v1::config as v1_config, v2::variables}; pub struct VariablesFactor; impl Factor for VariablesFactor { + type RuntimeConfig = (); type AppState = AppState; - type InstancePreparer = InstancePreparer; - type InstanceState = InstanceState; + type InstanceBuilder = InstanceBuilder; fn init( &mut self, @@ -23,10 +23,9 @@ impl Factor for VariablesFactor { Ok(()) } - fn configure_app( + fn configure_app( &self, - ctx: ConfigureAppContext, - _runtime_config: &mut impl RuntimeConfig, + ctx: ConfigureAppContext, ) -> anyhow::Result { let app = ctx.app(); let mut resolver = @@ -43,23 +42,19 @@ impl Factor for VariablesFactor { }) } - fn create_preparer( + fn prepare( ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result { + _builders: &mut InstanceBuilders, + ) -> anyhow::Result { let component_id = ctx.app_component().id().to_string(); - let resolver = ctx.app_config().resolver.clone(); - Ok(InstancePreparer { + let resolver = ctx.app_state().resolver.clone(); + Ok(InstanceBuilder { state: InstanceState { component_id, resolver, }, }) } - - fn prepare(&self, preparer: InstancePreparer) -> anyhow::Result { - Ok(preparer.state) - } } #[derive(Default)] @@ -67,17 +62,24 @@ pub struct AppState { resolver: Arc, } -#[derive(Default)] -pub struct InstancePreparer { +pub struct InstanceBuilder { state: InstanceState, } -impl InstancePreparer { +impl InstanceBuilder { pub fn resolver(&self) -> &Arc { &self.state.resolver } } +impl FactorInstanceBuilder for InstanceBuilder { + type InstanceState = InstanceState; + + fn build(self) -> anyhow::Result { + Ok(self.state) + } +} + #[derive(Default)] pub struct InstanceState { component_id: String, diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 1e97ab55f..30597e107 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -3,7 +3,8 @@ pub mod preview1; use std::{future::Future, net::SocketAddr, path::Path}; use spin_factors::{ - anyhow, AppComponent, Factor, InitContext, InstancePreparers, PrepareContext, RuntimeFactors, + anyhow, AppComponent, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, + PrepareContext, RuntimeFactors, }; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; @@ -20,9 +21,9 @@ impl WasiFactor { } impl Factor for WasiFactor { + type RuntimeConfig = (); type AppState = (); - type InstancePreparer = InstancePreparer; - type InstanceState = InstanceState; + type InstanceBuilder = InstanceBuilder; fn init( &mut self, @@ -69,10 +70,10 @@ impl Factor for WasiFactor { Ok(()) } - fn create_preparer( + fn prepare( ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result { + _builders: &mut InstanceBuilders, + ) -> anyhow::Result { let mut wasi_ctx = WasiCtxBuilder::new(); // Apply environment variables @@ -88,15 +89,7 @@ impl Factor for WasiFactor { .files_mounter .mount_files(ctx.app_component(), mount_ctx)?; - Ok(InstancePreparer { wasi_ctx }) - } - - fn prepare(&self, preparer: InstancePreparer) -> anyhow::Result { - let InstancePreparer { mut wasi_ctx } = preparer; - Ok(InstanceState { - ctx: wasi_ctx.build(), - table: Default::default(), - }) + Ok(InstanceBuilder { wasi_ctx }) } } @@ -147,19 +140,23 @@ impl<'a> MountFilesContext<'a> { } } -pub struct InstancePreparer { +pub struct InstanceBuilder { wasi_ctx: WasiCtxBuilder, } -impl Default for InstancePreparer { - fn default() -> Self { - Self { - wasi_ctx: WasiCtxBuilder::new(), - } +impl FactorInstanceBuilder for InstanceBuilder { + type InstanceState = InstanceState; + + fn build(self) -> anyhow::Result { + let InstanceBuilder { mut wasi_ctx } = self; + Ok(InstanceState { + ctx: wasi_ctx.build(), + table: Default::default(), + }) } } -impl InstancePreparer { +impl InstanceBuilder { pub fn outbound_socket_addr_check(&mut self, check: F) where F: Fn(SocketAddr) -> Fut + Send + Sync + Clone + 'static, diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index ab3a7a76e..1a25260be 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -1,12 +1,12 @@ -use spin_factors::{anyhow, Factor, InitContext, RuntimeFactors}; +use spin_factors::{anyhow, Factor, FactorInstanceBuilder, InitContext, RuntimeFactors}; use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; pub struct WasiPreview1Factor; impl Factor for WasiPreview1Factor { + type RuntimeConfig = (); type AppState = (); - type InstancePreparer = InstancePreparer; - type InstanceState = WasiP1Ctx; + type InstanceBuilder = InstanceBuilder; fn init( &mut self, @@ -15,19 +15,25 @@ impl Factor for WasiPreview1Factor { ctx.link_module_bindings(wasmtime_wasi::preview1::add_to_linker_async) } - fn prepare(&self, mut preparer: InstancePreparer) -> anyhow::Result { - Ok(preparer.wasi_ctx.build_p1()) + fn prepare( + _ctx: spin_factors::PrepareContext, + _builders: &mut spin_factors::InstanceBuilders, + ) -> anyhow::Result { + Ok(InstanceBuilder { + wasi_ctx: WasiCtxBuilder::new(), + }) } } -pub struct InstancePreparer { +pub struct InstanceBuilder { wasi_ctx: WasiCtxBuilder, } -impl Default for InstancePreparer { - fn default() -> Self { - Self { - wasi_ctx: WasiCtxBuilder::new(), - } +impl FactorInstanceBuilder for InstanceBuilder { + type InstanceState = WasiP1Ctx; + + fn build(self) -> anyhow::Result { + let Self { mut wasi_ctx } = self; + Ok(wasi_ctx.build_p1()) } } diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index f6dede3f2..deb03e756 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -20,8 +20,8 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let name = &input.ident; let vis = &input.vis; - let app_configs_name = format_ident!("{name}AppState"); - let preparers_name = format_ident!("{name}InstancePreparers"); + let app_state_name = format_ident!("{name}AppState"); + let builders_name = format_ident!("{name}InstanceBuilders"); let state_name = format_ident!("{name}InstanceState"); if !input.generics.params.is_empty() { @@ -61,6 +61,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let Factor = quote!(#factors_path::Factor); let ConfiguredApp = quote!(#factors_path::ConfiguredApp); let RuntimeConfigTracker = quote!(#factors_path::__internal::RuntimeConfigTracker); + let FactorInstanceBuilder = quote!(#factors_path::FactorInstanceBuilder); Ok(quote! { impl #name { @@ -87,46 +88,48 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { app: #factors_path::App, runtime_config: impl #factors_path::RuntimeConfigSource ) -> #Result<#ConfiguredApp> { - let mut app_configs = #app_configs_name { + let mut app_state = #app_state_name { #( #factor_names: None, )* }; - let mut runtime_config = #RuntimeConfigTracker::new(runtime_config); + let mut runtime_config_tracker = #RuntimeConfigTracker::new(runtime_config); #( - app_configs.#factor_names = Some( + app_state.#factor_names = Some( #Factor::configure_app( &self.#factor_names, - #factors_path::ConfigureAppContext::::new(&app, &app_configs), - &mut runtime_config, + #factors_path::ConfigureAppContext::::new( + &app, + &app_state, + &mut runtime_config_tracker, + )?, )? ); )* - Ok(#ConfiguredApp::new(app, app_configs)) + Ok(#ConfiguredApp::new(app, app_state)) } pub fn build_store_data(&self, configured_app: &#ConfiguredApp, component_id: &str) -> #Result<#state_name> { let app_component = configured_app.app().get_component(component_id).ok_or_else(|| { #wasmtime::Error::msg("unknown component") })?; - let mut preparers = #preparers_name { + let mut builders = #builders_name { #( #factor_names: None, )* }; #( - preparers.#factor_names = Some( - #Factor::create_preparer::( + builders.#factor_names = Some( + #Factor::prepare::( #factors_path::PrepareContext::new( &self.#factor_names, - configured_app.app_config::<#factor_types>().unwrap(), + configured_app.app_state::<#factor_types>().unwrap(), &app_component, ), - #factors_path::InstancePreparers::new(&mut preparers), + &mut #factors_path::InstanceBuilders::new(&mut builders), )? ); )* Ok(#state_name { #( - #factor_names: #Factor::prepare( - &self.#factor_names, - preparers.#factor_names.unwrap(), + #factor_names: #FactorInstanceBuilder::build( + builders.#factor_names.unwrap(), )?, )* }) @@ -135,15 +138,15 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } impl #factors_path::RuntimeFactors for #name { - type AppState = #app_configs_name; - type InstancePreparers = #preparers_name; + type AppState = #app_state_name; + type InstanceBuilders = #builders_name; type InstanceState = #state_name; - unsafe fn instance_preparer_offset() -> Option { + unsafe fn instance_builder_offset() -> Option { let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { - return Some(std::mem::offset_of!(Self::InstancePreparers, #factor_names)); + return Some(std::mem::offset_of!(Self::InstanceBuilders, #factor_names)); } )* None @@ -160,32 +163,32 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } - fn app_config(app_configs: &Self::AppState) -> Option<&T::AppState> { + fn app_state(app_state: &Self::AppState) -> Option<&T::AppState> { let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { - return Some(unsafe { std::mem::transmute(&app_configs.#factor_names) }); + return Some(unsafe { std::mem::transmute(&app_state.#factor_names) }); } )* None } } - #vis struct #app_configs_name { + #vis struct #app_state_name { #( pub #factor_names: Option<<#factor_types as #Factor>::AppState>, )* } - #vis struct #preparers_name { + #vis struct #builders_name { #( - pub #factor_names: Option<<#factor_types as #Factor>::InstancePreparer>, + pub #factor_names: Option<<#factor_types as #Factor>::InstanceBuilder>, )* } #vis struct #state_name { #( - pub #factor_names: <#factor_types as #Factor>::InstanceState, + pub #factor_names: <<#factor_types as #Factor>::InstanceBuilder as #FactorInstanceBuilder>::InstanceState, )* } }) diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 79b1f86f1..c567bd974 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -3,22 +3,16 @@ use std::any::Any; use anyhow::Context; use crate::{ - App, InstancePreparers, Linker, ModuleLinker, PrepareContext, RuntimeConfig, RuntimeFactors, + prepare::FactorInstanceBuilder, runtime_config::RuntimeConfigTracker, App, FactorRuntimeConfig, + InstanceBuilders, Linker, ModuleLinker, PrepareContext, RuntimeConfigSource, RuntimeFactors, }; pub trait Factor: Any + Sized { - /// Per-app state for this factor. - /// - /// See [`Factor::configure_app`]. - type AppState: Default; + type RuntimeConfig: FactorRuntimeConfig; - /// The [`FactorInstancePreparer`] for this factor. - type InstancePreparer: Default; + type AppState: Default; - /// The per-instance state for this factor, constructed by a - /// [`FactorInstancePreparer`] and available to any host-provided imports - /// defined by this factor. - type InstanceState; + type InstanceBuilder: FactorInstanceBuilder; /// Initializes this Factor for a runtime. This will be called at most once, /// before any call to [`FactorInstancePreparer::new`] @@ -34,28 +28,23 @@ pub trait Factor: Any + Sized { /// any call to `init` in cases where only validation is needed. fn configure_app( &self, - ctx: ConfigureAppContext, - _runtime_config: &mut impl RuntimeConfig, + ctx: ConfigureAppContext, ) -> anyhow::Result { _ = ctx; Ok(Default::default()) } - /// Returns a new instance of this preparer for the given [`Factor`]. - fn create_preparer( + fn prepare( ctx: PrepareContext, - _preparers: InstancePreparers, - ) -> anyhow::Result { - _ = ctx; - Ok(Default::default()) - } - - /// Returns a new instance of the associated [`Factor::InstanceState`]. - fn prepare(&self, preparer: Self::InstancePreparer) -> anyhow::Result; + _builders: &mut InstanceBuilders, + ) -> anyhow::Result; } -pub(crate) type GetDataFn = - fn(&mut ::InstanceState) -> &mut ::InstanceState; +pub(crate) type FactorInstanceState = + <::InstanceBuilder as FactorInstanceBuilder>::InstanceState; + +pub(crate) type GetDataFn = + fn(&mut ::InstanceState) -> &mut FactorInstanceState; /// An InitContext is passed to [`Factor::init`], giving access to the global /// common [`wasmtime::component::Linker`]. @@ -95,7 +84,7 @@ impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { &mut self, add_to_linker: impl Fn( &mut Linker, - fn(&mut T::InstanceState) -> &mut F::InstanceState, + fn(&mut T::InstanceState) -> &mut FactorInstanceState, ) -> anyhow::Result<()>, ) -> anyhow::Result<()> where { @@ -110,7 +99,7 @@ where { &mut self, add_to_linker: impl Fn( &mut ModuleLinker, - fn(&mut T::InstanceState) -> &mut F::InstanceState, + fn(&mut T::InstanceState) -> &mut FactorInstanceState, ) -> anyhow::Result<()>, ) -> anyhow::Result<()> where { @@ -122,42 +111,60 @@ where { } } -pub struct ConfigureAppContext<'a, T: RuntimeFactors> { - pub(crate) app: &'a App, - pub(crate) app_configs: &'a T::AppState, +pub struct ConfigureAppContext<'a, T: RuntimeFactors, F: Factor> { + app: &'a App, + app_state: &'a T::AppState, + runtime_config: Option, } -impl<'a, T: RuntimeFactors> ConfigureAppContext<'a, T> { +impl<'a, T: RuntimeFactors, F: Factor> ConfigureAppContext<'a, T, F> { #[doc(hidden)] - pub fn new(app: &'a App, app_configs: &'a T::AppState) -> Self { - Self { app, app_configs } + pub fn new( + app: &'a App, + app_state: &'a T::AppState, + runtime_config_tracker: &mut RuntimeConfigTracker, + ) -> anyhow::Result { + let runtime_config = runtime_config_tracker.get_config::()?; + Ok(Self { + app, + app_state, + runtime_config, + }) } pub fn app(&self) -> &App { self.app } - pub fn app_config(&self) -> crate::Result<&F::AppState> { - T::app_config::(self.app_configs).context("no such factor") + pub fn app_state(&self) -> crate::Result<&U::AppState> { + T::app_state::(self.app_state).context("no such factor") + } + + pub fn runtime_config(&self) -> Option<&F::RuntimeConfig> { + self.runtime_config.as_ref() + } + + pub fn take_runtime_config(&mut self) -> Option { + self.runtime_config.take() } } pub struct ConfiguredApp { app: App, - app_configs: T::AppState, + app_state: T::AppState, } impl ConfiguredApp { #[doc(hidden)] - pub fn new(app: App, app_configs: T::AppState) -> Self { - Self { app, app_configs } + pub fn new(app: App, app_state: T::AppState) -> Self { + Self { app, app_state } } pub fn app(&self) -> &App { &self.app } - pub fn app_config(&self) -> crate::Result<&F::AppState> { - T::app_config::(&self.app_configs).context("no such factor") + pub fn app_state(&self) -> crate::Result<&F::AppState> { + T::app_state::(&self.app_state).context("no such factor") } } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index eb34e6b37..e72892760 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -1,7 +1,7 @@ mod factor; mod prepare; mod runtime_config; -mod spin_factors; +mod runtime_factors; pub use anyhow; pub use wasmtime; @@ -10,9 +10,9 @@ pub use spin_factors_derive::RuntimeFactors; pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, InitContext}, - prepare::{InstancePreparers, PrepareContext}, - runtime_config::{RuntimeConfig, RuntimeConfigSource}, - spin_factors::RuntimeFactors, + prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext}, + runtime_config::{FactorRuntimeConfig, RuntimeConfigSource}, + runtime_factors::RuntimeFactors, }; pub type Linker = wasmtime::component::Linker<::InstanceState>; diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index ccf975e8f..2c17f8be2 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -1,26 +1,40 @@ +use std::marker::PhantomData; + use anyhow::Context; use crate::{AppComponent, Factor, RuntimeFactors}; -/// A PrepareContext is passed to [`FactorInstancePreparer::new`], giving access -/// to any already-initialized [`FactorInstancePreparer`]s, allowing for +pub trait FactorInstanceBuilder { + type InstanceState; + + fn build(self) -> anyhow::Result; +} + +pub struct DefaultInstanceBuilder(PhantomData T>); + +impl FactorInstanceBuilder for DefaultInstanceBuilder { + type InstanceState = T; + + fn build(self) -> anyhow::Result { + Ok(Default::default()) + } +} + +/// A PrepareContext is passed to [`Factor::prepare`], giving access to any +/// already-initialized [`FactorInstanceBuilder`]s, allowing for /// inter-[`Factor`] dependencies. pub struct PrepareContext<'a, F: Factor> { pub(crate) factor: &'a F, - pub(crate) app_config: &'a F::AppState, + pub(crate) app_state: &'a F::AppState, pub(crate) app_component: &'a AppComponent<'a>, } impl<'a, F: Factor> PrepareContext<'a, F> { #[doc(hidden)] - pub fn new( - factor: &'a F, - app_config: &'a F::AppState, - app_component: &'a AppComponent, - ) -> Self { + pub fn new(factor: &'a F, app_state: &'a F::AppState, app_component: &'a AppComponent) -> Self { Self { factor, - app_config, + app_state, app_component, } } @@ -29,8 +43,8 @@ impl<'a, F: Factor> PrepareContext<'a, F> { self.factor } - pub fn app_config(&self) -> &F::AppState { - self.app_config + pub fn app_state(&self) -> &F::AppState { + self.app_state } pub fn app_component(&self) -> &AppComponent { @@ -38,22 +52,22 @@ impl<'a, F: Factor> PrepareContext<'a, F> { } } -pub struct InstancePreparers<'a, T: RuntimeFactors> { - pub(crate) inner: &'a mut T::InstancePreparers, +pub struct InstanceBuilders<'a, T: RuntimeFactors> { + pub(crate) inner: &'a mut T::InstanceBuilders, } -impl<'a, T: RuntimeFactors> InstancePreparers<'a, T> { +impl<'a, T: RuntimeFactors> InstanceBuilders<'a, T> { #[doc(hidden)] - pub fn new(inner: &'a mut T::InstancePreparers) -> Self { + pub fn new(inner: &'a mut T::InstanceBuilders) -> Self { Self { inner } } - /// Returns a already-initialized preparer for the given [`Factor`]. + /// Returns the prepared [`FactorInstanceBuilder`] for the given [`Factor`]. /// /// Fails if the current [`RuntimeFactors`] does not include the given - /// [`Factor`] or if the given [`Factor`]'s preparer has not been - /// initialized yet (because it is sequenced after this factor). - pub fn get_mut(&mut self) -> crate::Result<&mut F::InstancePreparer> { - T::instance_preparer_mut::(self.inner)?.context("preparer not initialized") + /// [`Factor`] or if the given [`Factor`]'s builder has not been prepared + /// yet (because it is sequenced after this factor). + pub fn get_mut(&mut self) -> crate::Result<&mut F::InstanceBuilder> { + T::instance_builder_mut::(self.inner)?.context("builder not prepared") } } diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index 020707395..6b161d217 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -3,34 +3,60 @@ use std::collections::HashSet; use anyhow::bail; use serde::de::DeserializeOwned; -/// RuntimeConfig represents an application's runtime configuration. +use crate::Factor; + +/// FactorRuntimeConfig represents an application's runtime configuration. /// /// Runtime configuration is partitioned, with each partition being the -/// responsibility of exactly one [`crate::Factor`]. If configuration needs to -/// be shared between Factors, one Factor can be selected as the owner and the -/// others will have a dependency relationship with that owner. -pub trait RuntimeConfig { +/// responsibility of exactly one [`crate::Factor`]. If configuration needs +/// to be shared between Factors, one Factor can be selected as the owner +/// and the others will have a dependency relationship with that owner. +pub trait FactorRuntimeConfig: DeserializeOwned { + const KEY: &'static str; +} + +impl FactorRuntimeConfig for () { + const KEY: &'static str = ""; +} + +pub trait RuntimeConfigSource { + /// Returns an iterator of factor config keys available in this source. + /// + /// Should only include keys that have been positively provided. A runtime + /// may treat unrecognized keys as a warning or error. + fn factor_config_keys(&self) -> impl Iterator; + /// Returns deserialized runtime config of the given type for the given /// factor config key. /// /// Returns Ok(None) if no configuration is available for the given key. - /// Returns Err if configuration is available but deserialization fails, - /// or if the given config key has already been retrieved. + /// Returns Err if configuration is available but deserialization fails. + fn get_config(&self, factor_config_key: &str) + -> anyhow::Result>; +} + +impl RuntimeConfigSource for () { fn get_config( - &mut self, - factor_config_key: &str, - ) -> anyhow::Result>; + &self, + _factor_config_key: &str, + ) -> anyhow::Result> { + Ok(None) + } + + fn factor_config_keys(&self) -> impl Iterator { + std::iter::empty() + } } -pub struct RuntimeConfigTracker { - source: Source, - used_keys: HashSet, +pub struct RuntimeConfigTracker { + source: S, + used_keys: HashSet<&'static str>, unused_keys: HashSet, } -impl RuntimeConfigTracker { +impl RuntimeConfigTracker { #[doc(hidden)] - pub fn new(source: Source) -> Self { + pub fn new(source: S) -> Self { let unused_keys = source.factor_config_keys().map(ToOwned::to_owned).collect(); Self { source, @@ -47,48 +73,13 @@ impl RuntimeConfigTracker { Err(self.unused_keys) } } -} -impl RuntimeConfig for RuntimeConfigTracker { - fn get_config( - &mut self, - factor_config_key: &str, - ) -> anyhow::Result> { - if !self.used_keys.insert(factor_config_key.to_owned()) { - bail!("already got runtime config key {factor_config_key:?}"); + pub fn get_config(&mut self) -> anyhow::Result> { + let key = F::RuntimeConfig::KEY; + if !self.used_keys.insert(key) { + bail!("already got runtime config key {key:?}"); } - self.unused_keys.remove(factor_config_key); - self.source.get_config::(factor_config_key) - } -} - -pub trait RuntimeConfigSource { - /// Returns deserialized runtime config of the given type for the given - /// factor config key. - /// - /// Returns Ok(None) if no configuration is available for the given key. - /// Returns Err if configuration is available but deserialization fails. - fn get_config( - &self, - factor_config_key: &str, - ) -> anyhow::Result>; - - /// Returns an iterator of factor config keys available in this source. - /// - /// Should only include keys that have been positively provided. A runtime - /// may treat unrecognized keys as a warning or error. - fn factor_config_keys(&self) -> impl Iterator; -} - -impl RuntimeConfigSource for () { - fn get_config( - &self, - _factor_config_key: &str, - ) -> anyhow::Result> { - Ok(None) - } - - fn factor_config_keys(&self) -> impl Iterator { - std::iter::empty() + self.unused_keys.remove(key); + self.source.get_config::(key) } } diff --git a/crates/factors/src/spin_factors.rs b/crates/factors/src/runtime_factors.rs similarity index 75% rename from crates/factors/src/spin_factors.rs rename to crates/factors/src/runtime_factors.rs index fd961e679..8d11e9c30 100644 --- a/crates/factors/src/spin_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -2,7 +2,7 @@ use std::marker::PhantomData; use anyhow::Context; -use crate::Factor; +use crate::{factor::FactorInstanceState, Factor}; // TODO(lann): Most of the unsafe shenanigans here probably aren't worth it; // consider replacing with e.g. `Any::downcast`. @@ -10,18 +10,19 @@ use crate::Factor; /// Implemented by `#[derive(RuntimeFactors)]` pub trait RuntimeFactors: Sized { type AppState; - type InstancePreparers; + type InstanceBuilders; type InstanceState: Send + 'static; #[doc(hidden)] - unsafe fn instance_preparer_offset() -> Option; + unsafe fn instance_builder_offset() -> Option; #[doc(hidden)] unsafe fn instance_state_offset() -> Option; - fn app_config(app_configs: &Self::AppState) -> Option<&F::AppState>; + fn app_state(app_state: &Self::AppState) -> Option<&F::AppState>; - fn instance_state_getter() -> Option> { + fn instance_state_getter( + ) -> Option>> { let offset = unsafe { Self::instance_state_offset::()? }; Some(Getter { offset, @@ -30,7 +31,8 @@ pub trait RuntimeFactors: Sized { } fn instance_state_getter2( - ) -> Option> { + ) -> Option, FactorInstanceState>> + { let offset1 = unsafe { Self::instance_state_offset::()? }; let offset2 = unsafe { Self::instance_state_offset::()? }; assert_ne!( @@ -44,13 +46,13 @@ pub trait RuntimeFactors: Sized { }) } - fn instance_preparer_mut( - preparers: &mut Self::InstancePreparers, - ) -> crate::Result> { + fn instance_builder_mut( + builders: &mut Self::InstanceBuilders, + ) -> crate::Result> { unsafe { - let offset = Self::instance_preparer_offset::().context("no such factor")?; - let ptr = preparers as *mut Self::InstancePreparers; - let opt = &mut *ptr.add(offset).cast::>(); + let offset = Self::instance_builder_offset::().context("no such factor")?; + let ptr = builders as *mut Self::InstanceBuilders; + let opt = &mut *ptr.add(offset).cast::>(); Ok(opt.as_mut()) } } From 9014ec6769aae36967dc4b6e80a11d65abddf2a3 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 7 Jun 2024 10:15:10 -0400 Subject: [PATCH 013/195] factors: Make Factor::configure_app required Signed-off-by: Lann Martin --- crates/factor-wasi/src/lib.rs | 7 +++++++ crates/factor-wasi/src/preview1.rs | 7 +++++++ crates/factors/src/factor.rs | 7 ++----- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 30597e107..c6fdb1279 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -70,6 +70,13 @@ impl Factor for WasiFactor { Ok(()) } + fn configure_app( + &self, + _ctx: spin_factors::ConfigureAppContext, + ) -> anyhow::Result { + Ok(()) + } + fn prepare( ctx: PrepareContext, _builders: &mut InstanceBuilders, diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 1a25260be..743708668 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -15,6 +15,13 @@ impl Factor for WasiPreview1Factor { ctx.link_module_bindings(wasmtime_wasi::preview1::add_to_linker_async) } + fn configure_app( + &self, + _ctx: spin_factors::ConfigureAppContext, + ) -> anyhow::Result { + Ok(()) + } + fn prepare( _ctx: spin_factors::PrepareContext, _builders: &mut spin_factors::InstanceBuilders, diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index c567bd974..a8c308b83 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -10,7 +10,7 @@ use crate::{ pub trait Factor: Any + Sized { type RuntimeConfig: FactorRuntimeConfig; - type AppState: Default; + type AppState; type InstanceBuilder: FactorInstanceBuilder; @@ -29,10 +29,7 @@ pub trait Factor: Any + Sized { fn configure_app( &self, ctx: ConfigureAppContext, - ) -> anyhow::Result { - _ = ctx; - Ok(Default::default()) - } + ) -> anyhow::Result; fn prepare( ctx: PrepareContext, From f43b2f4f1ae9ae36c9cd6dd1f2b580a82e1b673d Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 7 Jun 2024 17:00:51 -0400 Subject: [PATCH 014/195] factors: Add VariablesFactor runtime config Signed-off-by: Lann Martin --- Cargo.lock | 4 + crates/expressions/src/lib.rs | 10 ++ crates/factor-outbound-networking/src/lib.rs | 2 - crates/factor-variables/Cargo.toml | 2 + crates/factor-variables/src/lib.rs | 102 +++++++++++++------ crates/factor-variables/src/provider_type.rs | 49 +++++++++ crates/factors-derive/src/lib.rs | 32 +++--- crates/factors/Cargo.toml | 2 + crates/factors/src/lib.rs | 2 +- crates/factors/src/prepare.rs | 20 +++- crates/factors/src/runtime_config.rs | 39 ++++--- crates/factors/src/runtime_factors.rs | 98 +----------------- crates/factors/tests/smoke.rs | 69 ++++++++++--- 13 files changed, 255 insertions(+), 176 deletions(-) create mode 100644 crates/factor-variables/src/provider_type.rs diff --git a/Cargo.lock b/Cargo.lock index 9c575470f..4eaa2e597 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7575,9 +7575,11 @@ dependencies = [ name = "spin-factor-variables" version = "2.6.0-pre0" dependencies = [ + "serde 1.0.197", "spin-expressions", "spin-factors", "spin-world", + "toml 0.8.12", ] [[package]] @@ -7602,6 +7604,8 @@ dependencies = [ "spin-factor-wasi", "spin-factors-derive", "thiserror", + "tokio", + "toml 0.8.12", "tracing", "wasmtime", ] diff --git a/crates/expressions/src/lib.rs b/crates/expressions/src/lib.rs index 612c0696b..811c0e864 100644 --- a/crates/expressions/src/lib.rs +++ b/crates/expressions/src/lib.rs @@ -5,6 +5,8 @@ use std::{borrow::Cow, collections::HashMap, fmt::Debug}; use spin_locked_app::Variable; +pub use async_trait; + pub use provider::Provider; use template::Part; pub use template::Template; @@ -251,6 +253,14 @@ impl<'a> Key<'a> { } } +impl<'a> TryFrom<&'a str> for Key<'a> { + type Error = Error; + + fn try_from(value: &'a str) -> std::prelude::v1::Result { + Self::new(value) + } +} + impl<'a> AsRef for Key<'a> { fn as_ref(&self) -> &str { self.0 diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index e19f6d607..89a6d1e8c 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -91,14 +91,12 @@ impl Factor for OutboundNetworkingFactor { } } -#[derive(Default)] pub struct AppState { component_allowed_hosts: HashMap>, } type SharedFutureResult = Shared>>>; -#[derive(Default)] pub struct InstanceBuilder { allowed_hosts_future: Option>, } diff --git a/crates/factor-variables/Cargo.toml b/crates/factor-variables/Cargo.toml index 7621fb266..5e5cf5543 100644 --- a/crates/factor-variables/Cargo.toml +++ b/crates/factor-variables/Cargo.toml @@ -5,9 +5,11 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] +serde = { version = "1.0", features = ["rc"] } spin-expressions = { path = "../expressions" } spin-factors = { path = "../factors" } spin-world = { path = "../world" } +toml = "0.8" [lints] workspace = true diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 438e01679..6f0c310e7 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -1,18 +1,44 @@ -use std::sync::Arc; +mod provider_type; +use std::{collections::HashMap, sync::Arc}; + +use provider_type::{provider_maker, ProviderMaker}; +use serde::Deserialize; use spin_expressions::ProviderResolver; use spin_factors::{ - anyhow, ConfigureAppContext, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, - PrepareContext, RuntimeFactors, + anyhow::{self, bail, Context}, + ConfigureAppContext, Factor, FactorRuntimeConfig, InitContext, InstanceBuilders, + PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; use spin_world::{async_trait, v1::config as v1_config, v2::variables}; -pub struct VariablesFactor; +pub use provider_type::{StaticVariables, VariablesProviderType}; + +#[derive(Default)] +pub struct VariablesFactor { + provider_types: HashMap<&'static str, ProviderMaker>, +} + +impl VariablesFactor { + pub fn add_provider_type( + &mut self, + provider_type: T, + ) -> anyhow::Result<()> { + if self + .provider_types + .insert(T::TYPE, provider_maker(provider_type)) + .is_some() + { + bail!("duplicate provider type {:?}", T::TYPE); + } + Ok(()) + } +} impl Factor for VariablesFactor { - type RuntimeConfig = (); + type RuntimeConfig = RuntimeConfig; type AppState = AppState; - type InstanceBuilder = InstanceBuilder; + type InstanceBuilder = InstanceState; fn init( &mut self, @@ -25,18 +51,30 @@ impl Factor for VariablesFactor { fn configure_app( &self, - ctx: ConfigureAppContext, + mut ctx: ConfigureAppContext, ) -> anyhow::Result { let app = ctx.app(); let mut resolver = ProviderResolver::new(app.variables().map(|(key, val)| (key.clone(), val.clone())))?; + for component in app.components() { resolver.add_component_variables( component.id(), component.config().map(|(k, v)| (k.into(), v.into())), )?; } - // TODO: add providers from runtime config + + if let Some(runtime_config) = ctx.take_runtime_config() { + for ProviderConfig { type_, config } in runtime_config.provider_configs { + let provider_maker = self + .provider_types + .get(type_.as_str()) + .with_context(|| format!("unknown variables provider type {type_}"))?; + let provider = provider_maker(config)?; + resolver.add_provider(provider); + } + } + Ok(AppState { resolver: Arc::new(resolver), }) @@ -45,47 +83,51 @@ impl Factor for VariablesFactor { fn prepare( ctx: PrepareContext, _builders: &mut InstanceBuilders, - ) -> anyhow::Result { + ) -> anyhow::Result { let component_id = ctx.app_component().id().to_string(); let resolver = ctx.app_state().resolver.clone(); - Ok(InstanceBuilder { - state: InstanceState { - component_id, - resolver, - }, + Ok(InstanceState { + component_id, + resolver, }) } } -#[derive(Default)] -pub struct AppState { - resolver: Arc, +#[derive(Deserialize)] +#[serde(transparent)] +pub struct RuntimeConfig { + provider_configs: Vec, } -pub struct InstanceBuilder { - state: InstanceState, +impl FactorRuntimeConfig for RuntimeConfig { + const KEY: &'static str = "variable_provider"; } -impl InstanceBuilder { - pub fn resolver(&self) -> &Arc { - &self.state.resolver - } +#[derive(Deserialize)] +struct ProviderConfig { + #[serde(rename = "type")] + type_: String, + #[serde(flatten)] + config: toml::Table, } -impl FactorInstanceBuilder for InstanceBuilder { - type InstanceState = InstanceState; - - fn build(self) -> anyhow::Result { - Ok(self.state) - } +pub struct AppState { + resolver: Arc, } -#[derive(Default)] pub struct InstanceState { component_id: String, resolver: Arc, } +impl InstanceState { + pub fn resolver(&self) -> &Arc { + &self.resolver + } +} + +impl SelfInstanceBuilder for InstanceState {} + #[async_trait] impl variables::Host for InstanceState { async fn get(&mut self, key: String) -> Result { diff --git a/crates/factor-variables/src/provider_type.rs b/crates/factor-variables/src/provider_type.rs new file mode 100644 index 000000000..bd6a07c07 --- /dev/null +++ b/crates/factor-variables/src/provider_type.rs @@ -0,0 +1,49 @@ +use std::{collections::HashMap, sync::Arc}; + +use serde::{de::DeserializeOwned, Deserialize}; +use spin_expressions::{async_trait::async_trait, Key, Provider}; +use spin_factors::anyhow; + +pub trait VariablesProviderType: 'static { + const TYPE: &'static str; + + type RuntimeConfig: DeserializeOwned; + type Provider: Provider; + + fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result; +} + +pub(crate) type ProviderMaker = Box anyhow::Result>>; + +pub(crate) fn provider_maker(provider_type: T) -> ProviderMaker { + Box::new(move |table| { + let runtime_config: T::RuntimeConfig = table.try_into()?; + let provider = provider_type.make_provider(runtime_config)?; + Ok(Box::new(provider)) + }) +} + +pub struct StaticVariables; + +impl VariablesProviderType for StaticVariables { + const TYPE: &'static str = "static"; + + type RuntimeConfig = StaticVariablesProvider; + type Provider = StaticVariablesProvider; + + fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result { + Ok(runtime_config) + } +} + +#[derive(Debug, Deserialize)] +pub struct StaticVariablesProvider { + values: Arc>, +} + +#[async_trait] +impl Provider for StaticVariablesProvider { + async fn get(&self, key: &Key) -> anyhow::Result> { + Ok(self.values.get(key.as_str()).cloned()) + } +} diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index deb03e756..d787797b8 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -104,6 +104,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { )? ); )* + runtime_config_tracker.validate_all_keys_used()?; Ok(#ConfiguredApp::new(app, app_state)) } @@ -142,32 +143,29 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { type InstanceBuilders = #builders_name; type InstanceState = #state_name; - unsafe fn instance_builder_offset() -> Option { - let type_id = #TypeId::of::(); + fn app_state(app_state: &Self::AppState) -> Option<&F::AppState> { + let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { - return Some(std::mem::offset_of!(Self::InstanceBuilders, #factor_names)); + unsafe { + return Some(::std::mem::transmute(&app_state.#factor_names)); + } } )* None } - unsafe fn instance_state_offset() -> Option { - let type_id = #TypeId::of::(); + fn instance_builder_mut( + builders: &mut Self::InstanceBuilders, + ) -> Option> { + let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { - return Some(std::mem::offset_of!(Self::InstanceState, #factor_names)); - } - )* - None - - } - - fn app_state(app_state: &Self::AppState) -> Option<&T::AppState> { - let type_id = #TypeId::of::(); - #( - if type_id == #TypeId::of::<#factor_types>() { - return Some(unsafe { std::mem::transmute(&app_state.#factor_names) }); + return Some( + builders.#factor_names.as_mut().map(|builder| { + unsafe { ::std::mem::transmute(builder) } + }) + ); } )* None diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 2d9347dbe..62255f747 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -19,6 +19,8 @@ spin-factors-derive = { path = "../factors-derive", features = ["expander"] } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } +tokio = { version = "1", features = ["macros", "rt"] } +toml = "0.8" [lints] workspace = true diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index e72892760..208adb84f 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -10,7 +10,7 @@ pub use spin_factors_derive::RuntimeFactors; pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, InitContext}, - prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext}, + prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext, SelfInstanceBuilder}, runtime_config::{FactorRuntimeConfig, RuntimeConfigSource}, runtime_factors::RuntimeFactors, }; diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index 2c17f8be2..81359c5c5 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -1,10 +1,10 @@ -use std::marker::PhantomData; +use std::{any::Any, marker::PhantomData}; use anyhow::Context; use crate::{AppComponent, Factor, RuntimeFactors}; -pub trait FactorInstanceBuilder { +pub trait FactorInstanceBuilder: Any { type InstanceState; fn build(self) -> anyhow::Result; @@ -12,7 +12,7 @@ pub trait FactorInstanceBuilder { pub struct DefaultInstanceBuilder(PhantomData T>); -impl FactorInstanceBuilder for DefaultInstanceBuilder { +impl FactorInstanceBuilder for DefaultInstanceBuilder { type InstanceState = T; fn build(self) -> anyhow::Result { @@ -20,6 +20,16 @@ impl FactorInstanceBuilder for DefaultInstanceBuilder { } } +pub trait SelfInstanceBuilder: 'static {} + +impl FactorInstanceBuilder for T { + type InstanceState = Self; + + fn build(self) -> anyhow::Result { + Ok(self) + } +} + /// A PrepareContext is passed to [`Factor::prepare`], giving access to any /// already-initialized [`FactorInstanceBuilder`]s, allowing for /// inter-[`Factor`] dependencies. @@ -68,6 +78,8 @@ impl<'a, T: RuntimeFactors> InstanceBuilders<'a, T> { /// [`Factor`] or if the given [`Factor`]'s builder has not been prepared /// yet (because it is sequenced after this factor). pub fn get_mut(&mut self) -> crate::Result<&mut F::InstanceBuilder> { - T::instance_builder_mut::(self.inner)?.context("builder not prepared") + T::instance_builder_mut::(self.inner) + .context("no such factor")? + .context("builder not prepared") } } diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index 6b161d217..3f313f0f1 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -5,6 +5,8 @@ use serde::de::DeserializeOwned; use crate::Factor; +pub const NO_RUNTIME_CONFIG: &str = ""; + /// FactorRuntimeConfig represents an application's runtime configuration. /// /// Runtime configuration is partitioned, with each partition being the @@ -16,7 +18,7 @@ pub trait FactorRuntimeConfig: DeserializeOwned { } impl FactorRuntimeConfig for () { - const KEY: &'static str = ""; + const KEY: &'static str = NO_RUNTIME_CONFIG; } pub trait RuntimeConfigSource { @@ -24,15 +26,14 @@ pub trait RuntimeConfigSource { /// /// Should only include keys that have been positively provided. A runtime /// may treat unrecognized keys as a warning or error. - fn factor_config_keys(&self) -> impl Iterator; + fn config_keys(&self) -> impl IntoIterator; /// Returns deserialized runtime config of the given type for the given /// factor config key. /// /// Returns Ok(None) if no configuration is available for the given key. /// Returns Err if configuration is available but deserialization fails. - fn get_config(&self, factor_config_key: &str) - -> anyhow::Result>; + fn get_config(&self, key: &str) -> anyhow::Result>; } impl RuntimeConfigSource for () { @@ -43,7 +44,7 @@ impl RuntimeConfigSource for () { Ok(None) } - fn factor_config_keys(&self) -> impl Iterator { + fn config_keys(&self) -> impl IntoIterator { std::iter::empty() } } @@ -57,7 +58,11 @@ pub struct RuntimeConfigTracker { impl RuntimeConfigTracker { #[doc(hidden)] pub fn new(source: S) -> Self { - let unused_keys = source.factor_config_keys().map(ToOwned::to_owned).collect(); + let unused_keys = source + .config_keys() + .into_iter() + .map(ToOwned::to_owned) + .collect(); Self { source, used_keys: Default::default(), @@ -66,18 +71,28 @@ impl RuntimeConfigTracker { } #[doc(hidden)] - pub fn validate_all_keys_used(self) -> Result<(), impl IntoIterator> { - if self.unused_keys.is_empty() { - Ok(()) - } else { - Err(self.unused_keys) + pub fn validate_all_keys_used(self) -> anyhow::Result<()> { + if !self.unused_keys.is_empty() { + bail!( + "unused runtime config key(s): {keys}", + keys = self + .unused_keys + .iter() + .map(|key| format!("{key:?}")) + .collect::>() + .join(", ") + ); } + Ok(()) } pub fn get_config(&mut self) -> anyhow::Result> { let key = F::RuntimeConfig::KEY; + if key == NO_RUNTIME_CONFIG { + return Ok(None); + } if !self.used_keys.insert(key) { - bail!("already got runtime config key {key:?}"); + bail!("already used runtime config key {key:?}"); } self.unused_keys.remove(key); self.source.get_config::(key) diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 8d11e9c30..8e44b78ef 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,8 +1,4 @@ -use std::marker::PhantomData; - -use anyhow::Context; - -use crate::{factor::FactorInstanceState, Factor}; +use crate::Factor; // TODO(lann): Most of the unsafe shenanigans here probably aren't worth it; // consider replacing with e.g. `Any::downcast`. @@ -13,99 +9,9 @@ pub trait RuntimeFactors: Sized { type InstanceBuilders; type InstanceState: Send + 'static; - #[doc(hidden)] - unsafe fn instance_builder_offset() -> Option; - - #[doc(hidden)] - unsafe fn instance_state_offset() -> Option; - fn app_state(app_state: &Self::AppState) -> Option<&F::AppState>; - fn instance_state_getter( - ) -> Option>> { - let offset = unsafe { Self::instance_state_offset::()? }; - Some(Getter { - offset, - _phantom: PhantomData, - }) - } - - fn instance_state_getter2( - ) -> Option, FactorInstanceState>> - { - let offset1 = unsafe { Self::instance_state_offset::()? }; - let offset2 = unsafe { Self::instance_state_offset::()? }; - assert_ne!( - offset1, offset2, - "instance_state_getter2 with same factor twice would alias" - ); - Some(Getter2 { - offset1, - offset2, - _phantom: PhantomData, - }) - } - fn instance_builder_mut( builders: &mut Self::InstanceBuilders, - ) -> crate::Result> { - unsafe { - let offset = Self::instance_builder_offset::().context("no such factor")?; - let ptr = builders as *mut Self::InstanceBuilders; - let opt = &mut *ptr.add(offset).cast::>(); - Ok(opt.as_mut()) - } - } -} - -pub struct Getter { - pub(crate) offset: usize, - pub(crate) _phantom: PhantomData &mut U>, -} - -impl Getter { - pub fn get_mut<'a>(&self, container: &'a mut T) -> &'a mut U { - let ptr = container as *mut T; - unsafe { &mut *ptr.add(self.offset).cast::() } - } + ) -> Option>; } - -impl Clone for Getter { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for Getter {} - -pub struct Getter2 { - pub(crate) offset1: usize, - pub(crate) offset2: usize, - #[allow(clippy::type_complexity)] - pub(crate) _phantom: PhantomData (&mut U, &mut V)>, -} - -impl Getter2 { - pub fn get_mut<'a>(&self, container: &'a mut T) -> (&'a mut U, &'a mut V) - where - T: 'static, - U: 'static, - V: 'static, - { - let ptr = container as *mut T; - unsafe { - ( - &mut *ptr.add(self.offset1).cast::(), - &mut *ptr.add(self.offset2).cast::(), - ) - } - } -} - -impl Clone for Getter2 { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for Getter2 {} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index c848be746..a88f9ff5c 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,8 +1,8 @@ use spin_app::App; use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::VariablesFactor; +use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{preview1::WasiPreview1Factor, DummyFilesMounter, WasiFactor}; -use spin_factors::RuntimeFactors; +use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; #[derive(RuntimeFactors)] struct Factors { @@ -12,30 +12,40 @@ struct Factors { outbound_networking_factor: OutboundNetworkingFactor, } -fn main() -> anyhow::Result<()> { +#[tokio::test(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), wasip1: WasiPreview1Factor, - variables: VariablesFactor, + variables: VariablesFactor::default(), outbound_networking_factor: OutboundNetworkingFactor, // outbound_http_factor: OutboundHttpFactor, }; + factors.variables.add_provider_type(StaticVariables)?; let locked = serde_json::from_value(serde_json::json!({ - "spin_locked_version": 1, + "spin_lock_version": 1, + "variables": { + "foo": {} + }, "triggers": [], "components": [{ "id": "test", + "metadata": { + "allowed_outbound_hosts": ["http://{{ foo }}"] + }, "source": { "content_type": "application/wasm", "content": {"inline": "KGNvbXBvbmVudCk="} + }, + "config": { + "test_var": "{{foo}}" } }] - })) - .unwrap(); + }))?; let app = App::inert(locked); - let engine = wasmtime::Engine::default(); + let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; let mut linker = wasmtime::component::Linker::new(&engine); let mut module_linker = wasmtime::Linker::new(&engine); @@ -43,16 +53,47 @@ fn main() -> anyhow::Result<()> { .init(Some(&mut linker), Some(&mut module_linker)) .unwrap(); - let configured_app = factors.configure_app(app, ()).unwrap(); - let data = factors.build_store_data(&configured_app, "test").unwrap(); + let configured_app = factors.configure_app(app, TestSource)?; + let data = factors.build_store_data(&configured_app, "test")?; + + assert_eq!( + data.variables + .resolver() + .resolve("test", "test_var".try_into().unwrap()) + .await + .unwrap(), + "bar" + ); let mut store = wasmtime::Store::new(&engine, data); - let component = wasmtime::component::Component::new(&engine, b"(component)").unwrap(); - let _instance = linker.instantiate(&mut store, &component).unwrap(); + let component = wasmtime::component::Component::new(&engine, b"(component)")?; + let _instance = linker.instantiate_async(&mut store, &component).await?; - let module = wasmtime::Module::new(&engine, b"(module)").unwrap(); - let _module_instance = module_linker.instantiate(&mut store, &module).unwrap(); + let module = wasmtime::Module::new(&engine, b"(module)")?; + let _module_instance = module_linker.instantiate_async(&mut store, &module).await?; Ok(()) } + +struct TestSource; + +impl RuntimeConfigSource for TestSource { + fn config_keys(&self) -> impl IntoIterator { + [spin_factor_variables::RuntimeConfig::KEY] + } + + fn get_config(&self, key: &str) -> anyhow::Result> { + let Some(table) = toml::toml! { + [[variable_provider]] + type = "static" + [variable_provider.values] + foo = "bar" + } + .remove(key) else { + return Ok(None); + }; + let config = table.try_into()?; + Ok(Some(config)) + } +} From b5ec51cb3ff1df5b6ad8c95fdb6a061393a5b794 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 10 Jun 2024 10:33:49 -0400 Subject: [PATCH 015/195] factors: Rename VariablesProviderType -> MakeVariablesProvider Also sneak in Factor::runtime_config_json_schema Signed-off-by: Lann Martin --- crates/factor-variables/src/lib.rs | 8 +++--- .../src/{provider_type.rs => provider.rs} | 6 ++--- crates/factors/src/factor.rs | 26 ++++++++++++++++--- crates/factors/src/runtime_factors.rs | 3 --- 4 files changed, 29 insertions(+), 14 deletions(-) rename crates/factor-variables/src/{provider_type.rs => provider.rs} (89%) diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 6f0c310e7..c5c4ec099 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -1,8 +1,8 @@ -mod provider_type; +mod provider; use std::{collections::HashMap, sync::Arc}; -use provider_type::{provider_maker, ProviderMaker}; +use provider::{provider_maker, ProviderMaker}; use serde::Deserialize; use spin_expressions::ProviderResolver; use spin_factors::{ @@ -12,7 +12,7 @@ use spin_factors::{ }; use spin_world::{async_trait, v1::config as v1_config, v2::variables}; -pub use provider_type::{StaticVariables, VariablesProviderType}; +pub use provider::{MakeVariablesProvider, StaticVariables}; #[derive(Default)] pub struct VariablesFactor { @@ -20,7 +20,7 @@ pub struct VariablesFactor { } impl VariablesFactor { - pub fn add_provider_type( + pub fn add_provider_type( &mut self, provider_type: T, ) -> anyhow::Result<()> { diff --git a/crates/factor-variables/src/provider_type.rs b/crates/factor-variables/src/provider.rs similarity index 89% rename from crates/factor-variables/src/provider_type.rs rename to crates/factor-variables/src/provider.rs index bd6a07c07..a8ceeccaa 100644 --- a/crates/factor-variables/src/provider_type.rs +++ b/crates/factor-variables/src/provider.rs @@ -4,7 +4,7 @@ use serde::{de::DeserializeOwned, Deserialize}; use spin_expressions::{async_trait::async_trait, Key, Provider}; use spin_factors::anyhow; -pub trait VariablesProviderType: 'static { +pub trait MakeVariablesProvider: 'static { const TYPE: &'static str; type RuntimeConfig: DeserializeOwned; @@ -15,7 +15,7 @@ pub trait VariablesProviderType: 'static { pub(crate) type ProviderMaker = Box anyhow::Result>>; -pub(crate) fn provider_maker(provider_type: T) -> ProviderMaker { +pub(crate) fn provider_maker(provider_type: T) -> ProviderMaker { Box::new(move |table| { let runtime_config: T::RuntimeConfig = table.try_into()?; let provider = provider_type.make_provider(runtime_config)?; @@ -25,7 +25,7 @@ pub(crate) fn provider_maker(provider_type: T) -> Prov pub struct StaticVariables; -impl VariablesProviderType for StaticVariables { +impl MakeVariablesProvider for StaticVariables { const TYPE: &'static str = "static"; type RuntimeConfig = StaticVariablesProvider; diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index a8c308b83..2f3b25838 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -1,4 +1,4 @@ -use std::any::Any; +use std::{any::Any, collections::HashMap}; use anyhow::Context; @@ -23,18 +23,36 @@ pub trait Factor: Any + Sized { } /// Performs factor-specific validation and configuration for the given - /// [`App`]. A runtime may - but is not required to - reuse the returned - /// config across multiple instances. Note that this may be called without - /// any call to `init` in cases where only validation is needed. + /// [`App`]. + /// + /// A runtime may - but is not required to - reuse the returned config + /// across multiple instances. Note that this may be called without any call + /// to `init` in cases where only validation is needed. fn configure_app( &self, ctx: ConfigureAppContext, ) -> anyhow::Result; + /// Prepares an instance builder for this factor. + /// + /// This method is given access to the app component being instantiated and + /// to any other factors' instance builders that have already been prepared. fn prepare( ctx: PrepareContext, _builders: &mut InstanceBuilders, ) -> anyhow::Result; + + /// Returns [JSON Schema](https://json-schema.org/) for this factor's + /// runtime config. + /// + /// Note that this represents only a fragment of an entire JSON document (a + /// "child instance" in JSON Schema terms), so `$schema` isn't needed. + /// + /// The default implementation returns an empty schema, which accepts any + /// configuration. + fn runtime_config_json_schema(&self) -> impl serde::Serialize { + HashMap::<(), ()>::new() + } } pub(crate) type FactorInstanceState = diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 8e44b78ef..3d6163847 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,8 +1,5 @@ use crate::Factor; -// TODO(lann): Most of the unsafe shenanigans here probably aren't worth it; -// consider replacing with e.g. `Any::downcast`. - /// Implemented by `#[derive(RuntimeFactors)]` pub trait RuntimeFactors: Sized { type AppState; From a33c964314c36677b3bb7e0e1c90f1756fee8418 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 10 Jun 2024 11:15:10 -0400 Subject: [PATCH 016/195] factors: Rename MakeVariablesProvider::TYPE -> RUNTIME_CONFIG_TYPE Signed-off-by: Lann Martin --- crates/factor-variables/src/lib.rs | 4 ++-- crates/factor-variables/src/provider.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index c5c4ec099..82903787d 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -26,10 +26,10 @@ impl VariablesFactor { ) -> anyhow::Result<()> { if self .provider_types - .insert(T::TYPE, provider_maker(provider_type)) + .insert(T::RUNTIME_CONFIG_TYPE, provider_maker(provider_type)) .is_some() { - bail!("duplicate provider type {:?}", T::TYPE); + bail!("duplicate provider type {:?}", T::RUNTIME_CONFIG_TYPE); } Ok(()) } diff --git a/crates/factor-variables/src/provider.rs b/crates/factor-variables/src/provider.rs index a8ceeccaa..5b8530b51 100644 --- a/crates/factor-variables/src/provider.rs +++ b/crates/factor-variables/src/provider.rs @@ -5,7 +5,7 @@ use spin_expressions::{async_trait::async_trait, Key, Provider}; use spin_factors::anyhow; pub trait MakeVariablesProvider: 'static { - const TYPE: &'static str; + const RUNTIME_CONFIG_TYPE: &'static str; type RuntimeConfig: DeserializeOwned; type Provider: Provider; @@ -26,7 +26,7 @@ pub(crate) fn provider_maker(provider_type: T) -> Prov pub struct StaticVariables; impl MakeVariablesProvider for StaticVariables { - const TYPE: &'static str = "static"; + const RUNTIME_CONFIG_TYPE: &'static str = "static"; type RuntimeConfig = StaticVariablesProvider; type Provider = StaticVariablesProvider; From a3c48ae98fa015dcaef8ec5fa44180b913c86d96 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 10 Jun 2024 11:17:39 -0400 Subject: [PATCH 017/195] factors: ProviderMaker -> ProviderFromToml Signed-off-by: Lann Martin --- crates/factor-variables/src/lib.rs | 6 +++--- crates/factor-variables/src/provider.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 82903787d..29c36a975 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -2,7 +2,7 @@ mod provider; use std::{collections::HashMap, sync::Arc}; -use provider::{provider_maker, ProviderMaker}; +use provider::{provider_from_toml, ProviderFromToml}; use serde::Deserialize; use spin_expressions::ProviderResolver; use spin_factors::{ @@ -16,7 +16,7 @@ pub use provider::{MakeVariablesProvider, StaticVariables}; #[derive(Default)] pub struct VariablesFactor { - provider_types: HashMap<&'static str, ProviderMaker>, + provider_types: HashMap<&'static str, ProviderFromToml>, } impl VariablesFactor { @@ -26,7 +26,7 @@ impl VariablesFactor { ) -> anyhow::Result<()> { if self .provider_types - .insert(T::RUNTIME_CONFIG_TYPE, provider_maker(provider_type)) + .insert(T::RUNTIME_CONFIG_TYPE, provider_from_toml(provider_type)) .is_some() { bail!("duplicate provider type {:?}", T::RUNTIME_CONFIG_TYPE); diff --git a/crates/factor-variables/src/provider.rs b/crates/factor-variables/src/provider.rs index 5b8530b51..83d634bf9 100644 --- a/crates/factor-variables/src/provider.rs +++ b/crates/factor-variables/src/provider.rs @@ -13,9 +13,9 @@ pub trait MakeVariablesProvider: 'static { fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result; } -pub(crate) type ProviderMaker = Box anyhow::Result>>; +pub(crate) type ProviderFromToml = Box anyhow::Result>>; -pub(crate) fn provider_maker(provider_type: T) -> ProviderMaker { +pub(crate) fn provider_from_toml(provider_type: T) -> ProviderFromToml { Box::new(move |table| { let runtime_config: T::RuntimeConfig = table.try_into()?; let provider = provider_type.make_provider(runtime_config)?; From 2cd9a67a2fee7e1085165d14125f02f8d557389d Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 11 Jun 2024 12:48:59 -0400 Subject: [PATCH 018/195] factors: Add OutboundHttpFactor Signed-off-by: Lann Martin --- Cargo.lock | 111 ++++++++----------- crates/factor-outbound-http/Cargo.toml | 18 +++ crates/factor-outbound-http/src/lib.rs | 54 +++++++++ crates/factor-outbound-http/src/spin.rs | 31 ++++++ crates/factor-outbound-http/src/wasi.rs | 40 +++++++ crates/factor-outbound-networking/src/lib.rs | 58 +++++++--- crates/factor-wasi/src/lib.rs | 14 ++- crates/factors-derive/src/lib.rs | 12 ++ crates/factors/Cargo.toml | 1 + crates/factors/src/prepare.rs | 10 +- crates/factors/src/runtime_config.rs | 17 +-- crates/factors/src/runtime_factors.rs | 91 ++++++++++++++- crates/factors/tests/smoke.rs | 17 ++- 13 files changed, 367 insertions(+), 107 deletions(-) create mode 100644 crates/factor-outbound-http/Cargo.toml create mode 100644 crates/factor-outbound-http/src/lib.rs create mode 100644 crates/factor-outbound-http/src/spin.rs create mode 100644 crates/factor-outbound-http/src/wasi.rs diff --git a/Cargo.lock b/Cargo.lock index 4eaa2e597..2801b1fed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1566,8 +1566,7 @@ dependencies = [ [[package]] name = "cranelift-bforest" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6b33d7e757a887989eb18b35712b2a67d96171ec3149d1bfb657b29b7b367c" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "cranelift-entity", ] @@ -1575,8 +1574,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9acf15cb22be42d07c3b57d7856329cb228b7315d385346149df2566ad5e4aa" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "bumpalo", "cranelift-bforest", @@ -1597,8 +1595,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e934d301392b73b3f8b0540391fb82465a0f179a3cee7c726482ac4727efcc97" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "cranelift-codegen-shared", ] @@ -1606,14 +1603,12 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb2a2566b3d54b854dfb288b3b187f6d3d17d6f762c92898207eba302931da" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" [[package]] name = "cranelift-control" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0100f33b704cdacd01ad66ff41f8c5030d57cbff078e2a4e49ab1822591299fa" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "arbitrary", ] @@ -1621,8 +1616,7 @@ dependencies = [ [[package]] name = "cranelift-entity" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8cfdc315e5d18997093e040a8d234bea1ac1e118a716d3e30f40d449e78207b" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "serde 1.0.197", "serde_derive", @@ -1631,8 +1625,7 @@ dependencies = [ [[package]] name = "cranelift-frontend" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f74b84f16af2e982b0c0c72233503d9d55cbfe3865dbe807ca28dc6642a28b5" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "cranelift-codegen", "log", @@ -1643,14 +1636,12 @@ dependencies = [ [[package]] name = "cranelift-isle" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adf306d3dde705fb94bd48082f01d38c4ededc74293a4c007805f610bf08bc6e" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" [[package]] name = "cranelift-native" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ea0ebdef7aff4a79bcbc8b6495f31315f16b3bf311152f472eaa8d679352581" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "cranelift-codegen", "libc", @@ -1660,8 +1651,7 @@ dependencies = [ [[package]] name = "cranelift-wasm" version = "0.109.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d549108a1942065cdbac3bb96c2952afa0e1b9a3beff4b08c4308ac72257576d" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -7558,6 +7548,20 @@ dependencies = [ "toml 0.5.11", ] +[[package]] +name = "spin-factor-outbound-http" +version = "2.6.0-pre0" +dependencies = [ + "anyhow", + "http 1.1.0", + "spin-factor-outbound-networking", + "spin-factor-wasi", + "spin-factors", + "spin-world", + "tracing", + "wasmtime-wasi-http", +] + [[package]] name = "spin-factor-outbound-networking" version = "2.6.0-pre0" @@ -7599,6 +7603,7 @@ dependencies = [ "serde 1.0.197", "serde_json", "spin-app", + "spin-factor-outbound-http", "spin-factor-outbound-networking", "spin-factor-variables", "spin-factor-wasi", @@ -9488,8 +9493,7 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi-common" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86fd41e1e26ff6af9451c6a332a5ce5f5283ca51e87d875cdd9a05305598ee3" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "bitflags 2.5.0", @@ -9822,8 +9826,7 @@ dependencies = [ [[package]] name = "wasmtime" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d8b5e7a4d54917c5ebe555b9667337e5f93383f49bddaaeec2eba68093b45" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "addr2line", "anyhow", @@ -9878,8 +9881,7 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d697d99c341d4a9ffb72f3af7a02124d233eeb59aee010f36d88e97cca553d5e" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "cfg-if", ] @@ -9887,8 +9889,7 @@ dependencies = [ [[package]] name = "wasmtime-cache" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916610f9ae9a6c22deb25bba2e6247ba9f00b093d30620875203b91328a1adfa" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "base64 0.21.7", @@ -9907,8 +9908,7 @@ dependencies = [ [[package]] name = "wasmtime-component-macro" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b29b462b068e73b5b27fae092a27f47e5937cabf6b26be2779c978698a52feca" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "proc-macro2", @@ -9922,14 +9922,12 @@ dependencies = [ [[package]] name = "wasmtime-component-util" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d2912c53d9054984b380dfbd7579f9c3681b2a73b903a56bd71a1c4f175f1e" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" [[package]] name = "wasmtime-cranelift" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3975deafea000457ba84355c7c0fce0372937204f77026510b7b454f28a3a65" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "cfg-if", @@ -9952,8 +9950,7 @@ dependencies = [ [[package]] name = "wasmtime-environ" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f444e900e848b884d8a8a2949b6f5b92af642a3e663ff8fbe78731143a55be61" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "cpp_demangle", @@ -9977,8 +9974,7 @@ dependencies = [ [[package]] name = "wasmtime-fiber" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ded58eb2d1bf0dcd2182d0ccd7055c4b10b50d711514f1d73f61515d0fa829d" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "cc", @@ -9992,8 +9988,7 @@ dependencies = [ [[package]] name = "wasmtime-jit-debug" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc54198c6720f098210a85efb3ba8c078d1de4d373cdb6778850a66ae088d11" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "object 0.36.0", "once_cell", @@ -10004,8 +9999,7 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5afe2f0499542f9a4bcfa1b55bfdda803b6ade4e7c93c6b99e0f39dba44b0a91" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "cfg-if", @@ -10016,14 +10010,12 @@ dependencies = [ [[package]] name = "wasmtime-slab" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7de1f2bec5bbb35d532e61c85c049dc84ae671df60492f90b954ecf21169e7" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" [[package]] name = "wasmtime-types" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "412463e9000e14cf6856be48628d2213c20c153e29ffc22b036980c892ea6964" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "cranelift-entity", "serde 1.0.197", @@ -10035,8 +10027,7 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de5a9bc4f44ceeb168e9e8e3be4e0b4beb9095b468479663a9e24c667e36826f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "proc-macro2", "quote", @@ -10046,8 +10037,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8abb1301089ed8e0b4840f539cba316a73ac382090f1b25d22d8c8eed8df49c7" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "async-trait", @@ -10077,8 +10067,7 @@ dependencies = [ [[package]] name = "wasmtime-wasi-http" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "315cadc284b808cfbd6be9295da4009144c106723f09b421ce6c6d89275cfdb7" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "async-trait", @@ -10100,8 +10089,7 @@ dependencies = [ [[package]] name = "wasmtime-winch" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed4db238a0241df2d15f79ad17b3a37a27f2ea6cb885894d81b42ae107544466" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "cranelift-codegen", @@ -10117,8 +10105,7 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc077306b38288262e5ba01d4b21532a6987416cdc0aedf04bb06c22a68fdc" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "heck 0.4.1", @@ -10297,8 +10284,7 @@ dependencies = [ [[package]] name = "wiggle" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29830e5d01c182d24b94092c697aa7ab0ee97d22e78a2bf40ca91eae6ebca5c2" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "async-trait", @@ -10312,8 +10298,7 @@ dependencies = [ [[package]] name = "wiggle-generate" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "557567f2793508760cd855f7659b7a0b9dc4dbc451f53f1415d6943a15311ade" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "heck 0.4.1", @@ -10327,8 +10312,7 @@ dependencies = [ [[package]] name = "wiggle-macro" version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc26129a8aea20b62c961d1b9ab4a3c3b56b10042ed85d004f8678af0f21ba6e" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "proc-macro2", "quote", @@ -10370,8 +10354,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c6915884e731b2db0d8cf08cb64474cb69221a161675fd3c135f91febc3daa" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" dependencies = [ "anyhow", "cranelift-codegen", diff --git a/crates/factor-outbound-http/Cargo.toml b/crates/factor-outbound-http/Cargo.toml new file mode 100644 index 000000000..e922255e3 --- /dev/null +++ b/crates/factor-outbound-http/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "spin-factor-outbound-http" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +http = "1.1.0" +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors = { path = "../factors" } +spin-world = { path = "../world" } +tracing = { workspace = true } +wasmtime-wasi-http = { workspace = true } + +[lints] +workspace = true diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs new file mode 100644 index 000000000..026e52413 --- /dev/null +++ b/crates/factor-outbound-http/src/lib.rs @@ -0,0 +1,54 @@ +mod spin; +mod wasi; + +use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; +use spin_factors::{ + anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, + SelfInstanceBuilder, +}; +use wasmtime_wasi_http::WasiHttpCtx; +pub struct OutboundHttpFactor; + +impl Factor for OutboundHttpFactor { + type RuntimeConfig = (); + type AppState = (); + type InstanceBuilder = InstanceState; + + fn init( + &mut self, + mut ctx: spin_factors::InitContext, + ) -> anyhow::Result<()> { + ctx.link_bindings(spin_world::v1::http::add_to_linker)?; + if let Some(linker) = ctx.linker() { + wasi::add_to_linker::(linker)?; + } + Ok(()) + } + + fn configure_app( + &self, + _ctx: ConfigureAppContext, + ) -> anyhow::Result { + Ok(()) + } + + fn prepare( + _ctx: PrepareContext, + builders: &mut InstanceBuilders, + ) -> anyhow::Result { + let allowed_hosts = builders + .get_mut::()? + .allowed_hosts(); + Ok(InstanceState { + allowed_hosts, + wasi_http_ctx: WasiHttpCtx::new(), + }) + } +} + +pub struct InstanceState { + allowed_hosts: OutboundAllowedHosts, + wasi_http_ctx: WasiHttpCtx, +} + +impl SelfInstanceBuilder for InstanceState {} diff --git a/crates/factor-outbound-http/src/spin.rs b/crates/factor-outbound-http/src/spin.rs new file mode 100644 index 000000000..22ea723f2 --- /dev/null +++ b/crates/factor-outbound-http/src/spin.rs @@ -0,0 +1,31 @@ +use spin_factor_outbound_networking::OutboundUrl; +use spin_world::{ + async_trait, + v1::http, + v1::http_types::{self, HttpError, Request, Response}, +}; + +#[async_trait] +impl http::Host for crate::InstanceState { + async fn send_request(&mut self, req: Request) -> Result { + // FIXME(lann): This is all just a stub to test allowed_outbound_hosts + let outbound_url = OutboundUrl::parse(&req.uri, "https").or(Err(HttpError::InvalidUrl))?; + match self.allowed_hosts.allows(&outbound_url).await { + Ok(true) => (), + _ => { + return Err(HttpError::DestinationNotAllowed); + } + } + Ok(Response { + status: 200, + headers: None, + body: Some(b"test response".into()), + }) + } +} + +impl http_types::Host for crate::InstanceState { + fn convert_http_error(&mut self, err: HttpError) -> anyhow::Result { + Ok(err) + } +} diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs new file mode 100644 index 000000000..3de0d8437 --- /dev/null +++ b/crates/factor-outbound-http/src/wasi.rs @@ -0,0 +1,40 @@ +use anyhow::Context; +use spin_factors::{Linker, RuntimeFactors}; +use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; + +pub(crate) fn add_to_linker(linker: &mut Linker) -> anyhow::Result<()> { + fn type_annotate(f: F) -> F + where + F: Fn(&mut T) -> WasiHttpImpl, + { + f + } + let wasi_and_http_getter = + T::instance_state_getter2::() + .context("failed to get WasiFactor")?; + let host_getter = type_annotate(move |data| { + let (wasi, http) = wasi_and_http_getter.get_states(data); + WasiHttpImpl(MutStates { http, wasi }) + }); + wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker_get_host( + linker, + host_getter, + )?; + wasmtime_wasi_http::bindings::http::types::add_to_linker_get_host(linker, host_getter)?; + Ok(()) +} + +struct MutStates<'a> { + http: &'a mut crate::InstanceState, + wasi: &'a mut spin_factor_wasi::InstanceState, +} + +impl<'a> WasiHttpView for MutStates<'a> { + fn ctx(&mut self) -> &mut wasmtime_wasi_http::WasiHttpCtx { + &mut self.http.wasi_http_ctx + } + + fn table(&mut self) -> &mut spin_factors::wasmtime::component::ResourceTable { + self.wasi.table() + } +} diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index 89a6d1e8c..b4f88c714 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -13,6 +13,10 @@ use spin_factors::{ }; use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; +pub use spin_outbound_networking::OutboundUrl; + +pub type SharedFutureResult = Shared, Arc>>>; + pub struct OutboundNetworkingFactor; impl Factor for OutboundNetworkingFactor { @@ -59,7 +63,7 @@ impl Factor for OutboundNetworkingFactor { let prepared = resolver.prepare().await?; AllowedHostsConfig::parse(&hosts, &prepared) } - .map(Arc::new) + .map(|res| res.map(Arc::new).map_err(Arc::new)) .boxed() .shared(); // let prepared_resolver = resolver.prepare().await?; @@ -74,10 +78,10 @@ impl Factor for OutboundNetworkingFactor { wasi_preparer.outbound_socket_addr_check(move |addr| { let hosts_future = hosts_future.clone(); async move { - match &*hosts_future.await { + match hosts_future.await { Ok(allowed_hosts) => { // TODO: verify this actually works... - spin_outbound_networking::check_url(&addr.to_string(), "*", allowed_hosts) + spin_outbound_networking::check_url(&addr.to_string(), "*", &allowed_hosts) } Err(err) => { // TODO: should this trap (somehow)? @@ -87,7 +91,9 @@ impl Factor for OutboundNetworkingFactor { } } }); - Ok(InstanceBuilder::new(allowed_hosts_future)) + Ok(InstanceBuilder { + allowed_hosts_future, + }) } } @@ -95,25 +101,16 @@ pub struct AppState { component_allowed_hosts: HashMap>, } -type SharedFutureResult = Shared>>>; - pub struct InstanceBuilder { - allowed_hosts_future: Option>, + allowed_hosts_future: SharedFutureResult, } impl InstanceBuilder { - fn new(allowed_hosts_future: SharedFutureResult) -> Self { - Self { - allowed_hosts_future: Some(allowed_hosts_future), + pub fn allowed_hosts(&self) -> OutboundAllowedHosts { + OutboundAllowedHosts { + allowed_hosts_future: self.allowed_hosts_future.clone(), } } - - pub async fn resolve_allowed_hosts(&self) -> Arc> { - self.allowed_hosts_future - .clone() - .expect("allowed_hosts_future not set") - .await - } } impl FactorInstanceBuilder for InstanceBuilder { @@ -123,3 +120,30 @@ impl FactorInstanceBuilder for InstanceBuilder { Ok(()) } } + +// TODO: Refactor w/ spin-outbound-networking crate to simplify +pub struct OutboundAllowedHosts { + allowed_hosts_future: SharedFutureResult, +} + +impl OutboundAllowedHosts { + pub async fn allows(&self, url: &OutboundUrl) -> anyhow::Result { + Ok(self.resolve().await?.allows(url)) + } + + pub async fn check_url(&self, url: &str, scheme: &str) -> anyhow::Result { + let allowed_hosts = self.resolve().await?; + Ok(spin_outbound_networking::check_url( + url, + scheme, + &allowed_hosts, + )) + } + + async fn resolve(&self) -> anyhow::Result> { + self.allowed_hosts_future.clone().await.map_err(|err| { + // TODO: better way to handle this? + anyhow::Error::msg(err) + }) + } +} diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index c6fdb1279..226477bb3 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -6,7 +6,7 @@ use spin_factors::{ anyhow, AppComponent, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, }; -use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiImpl, WasiView}; pub struct WasiFactor { files_mounter: Box, @@ -29,14 +29,14 @@ impl Factor for WasiFactor { &mut self, mut ctx: InitContext, ) -> anyhow::Result<()> { - fn type_annotate(f: F) -> F + fn type_annotate(f: F) -> F where - F: Fn(&mut T) -> &mut dyn WasiView, + F: Fn(&mut T) -> WasiImpl<&mut U>, { f } let get_data = ctx.get_data_fn(); - let closure = type_annotate(move |data| get_data(data) as &mut dyn WasiView); + let closure = type_annotate(move |data| WasiImpl(get_data(data))); if let Some(linker) = ctx.linker() { use wasmtime_wasi::bindings; bindings::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; @@ -189,6 +189,12 @@ pub struct InstanceState { table: ResourceTable, } +impl InstanceState { + pub fn table(&mut self) -> &mut ResourceTable { + &mut self.table + } +} + impl WasiView for InstanceState { fn ctx(&mut self) -> &mut WasiCtx { &mut self.ctx diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index d787797b8..b7e30734c 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -65,6 +65,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { Ok(quote! { impl #name { + #[allow(clippy::needless_option_as_deref)] pub fn init( &mut self, mut linker: Option<&mut #wasmtime::component::Linker<#state_name>>, @@ -170,6 +171,17 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { )* None } + + fn instance_state_offset() -> Option { + let type_id = #TypeId::of::(); + #( + if type_id == #TypeId::of::<#factor_types>() { + return Some(std::mem::offset_of!(Self::InstanceState, #factor_names)); + } + )* + None + } + } #vis struct #app_state_name { diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 62255f747..117e89472 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -16,6 +16,7 @@ wasmtime = { workspace = true } [dev-dependencies] serde_json = "1.0" spin-factors-derive = { path = "../factors-derive", features = ["expander"] } +spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index 81359c5c5..525f77404 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -1,4 +1,4 @@ -use std::{any::Any, marker::PhantomData}; +use std::any::Any; use anyhow::Context; @@ -10,13 +10,11 @@ pub trait FactorInstanceBuilder: Any { fn build(self) -> anyhow::Result; } -pub struct DefaultInstanceBuilder(PhantomData T>); - -impl FactorInstanceBuilder for DefaultInstanceBuilder { - type InstanceState = T; +impl FactorInstanceBuilder for () { + type InstanceState = (); fn build(self) -> anyhow::Result { - Ok(Default::default()) + Ok(()) } } diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index 3f313f0f1..372c9ffe6 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -24,27 +24,28 @@ impl FactorRuntimeConfig for () { pub trait RuntimeConfigSource { /// Returns an iterator of factor config keys available in this source. /// - /// Should only include keys that have been positively provided. A runtime - /// may treat unrecognized keys as a warning or error. - fn config_keys(&self) -> impl IntoIterator; + /// Should only include keys that have been positively provided and that + /// haven't already been parsed by the runtime. A runtime may treat + /// unrecognized keys as a warning or error. + fn factor_config_keys(&self) -> impl IntoIterator; /// Returns deserialized runtime config of the given type for the given /// factor config key. /// /// Returns Ok(None) if no configuration is available for the given key. /// Returns Err if configuration is available but deserialization fails. - fn get_config(&self, key: &str) -> anyhow::Result>; + fn get_factor_config(&self, key: &str) -> anyhow::Result>; } impl RuntimeConfigSource for () { - fn get_config( + fn get_factor_config( &self, _factor_config_key: &str, ) -> anyhow::Result> { Ok(None) } - fn config_keys(&self) -> impl IntoIterator { + fn factor_config_keys(&self) -> impl IntoIterator { std::iter::empty() } } @@ -59,7 +60,7 @@ impl RuntimeConfigTracker { #[doc(hidden)] pub fn new(source: S) -> Self { let unused_keys = source - .config_keys() + .factor_config_keys() .into_iter() .map(ToOwned::to_owned) .collect(); @@ -95,6 +96,6 @@ impl RuntimeConfigTracker { bail!("already used runtime config key {key:?}"); } self.unused_keys.remove(key); - self.source.get_config::(key) + self.source.get_factor_config::(key) } } diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 3d6163847..4df65d475 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,7 +1,9 @@ -use crate::Factor; +use std::{any::TypeId, marker::PhantomData}; + +use crate::{factor::FactorInstanceState, Factor}; /// Implemented by `#[derive(RuntimeFactors)]` -pub trait RuntimeFactors: Sized { +pub trait RuntimeFactors: Sized + 'static { type AppState; type InstanceBuilders; type InstanceState: Send + 'static; @@ -11,4 +13,89 @@ pub trait RuntimeFactors: Sized { fn instance_builder_mut( builders: &mut Self::InstanceBuilders, ) -> Option>; + + #[doc(hidden)] + fn instance_state_offset() -> Option; + + fn instance_state_getter() -> Option> { + StateGetter::new() + } + + fn instance_state_getter2() -> Option> { + StateGetter2::new() + } +} + +pub struct StateGetter { + offset: isize, + _phantom: PhantomData F>, +} + +impl StateGetter { + fn new() -> Option { + Some(Self { + offset: T::instance_state_offset::()?.try_into().unwrap(), + _phantom: PhantomData, + }) + } + + pub fn get_state<'a>( + &self, + instance_state: &'a mut T::InstanceState, + ) -> &'a mut FactorInstanceState { + let ptr = instance_state as *mut T::InstanceState; + unsafe { &mut *(ptr.offset(self.offset) as *mut FactorInstanceState) } + } +} + +impl Clone for StateGetter { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for StateGetter {} + +pub struct StateGetter2 { + offset1: isize, + offset2: isize, + _phantom: PhantomData (F1, F2)>, +} + +impl StateGetter2 { + fn new() -> Option { + // Only safe if F1 and F2 are different (and so do not alias) + if TypeId::of::() == TypeId::of::() { + return None; + } + Some(StateGetter2 { + offset1: T::instance_state_offset::()?.try_into().unwrap(), + offset2: T::instance_state_offset::()?.try_into().unwrap(), + _phantom: PhantomData, + }) + } + + pub fn get_states<'a>( + &self, + instance_state: &'a mut T::InstanceState, + ) -> ( + &'a mut FactorInstanceState, + &'a mut FactorInstanceState, + ) { + let ptr = instance_state as *mut T::InstanceState; + unsafe { + ( + &mut *(ptr.offset(self.offset1) as *mut FactorInstanceState), + &mut *(ptr.offset(self.offset2) as *mut FactorInstanceState), + ) + } + } } + +impl Clone for StateGetter2 { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for StateGetter2 {} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index a88f9ff5c..6dfdf4899 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,4 +1,5 @@ use spin_app::App; +use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{preview1::WasiPreview1Factor, DummyFilesMounter, WasiFactor}; @@ -7,19 +8,20 @@ use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; #[derive(RuntimeFactors)] struct Factors { wasi: WasiFactor, - wasip1: WasiPreview1Factor, + wasi_p1: WasiPreview1Factor, variables: VariablesFactor, outbound_networking_factor: OutboundNetworkingFactor, + outbound_http_factor: OutboundHttpFactor, } -#[tokio::test(flavor = "multi_thread")] +#[tokio::test] async fn main() -> anyhow::Result<()> { let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), - wasip1: WasiPreview1Factor, + wasi_p1: WasiPreview1Factor, variables: VariablesFactor::default(), outbound_networking_factor: OutboundNetworkingFactor, - // outbound_http_factor: OutboundHttpFactor, + outbound_http_factor: OutboundHttpFactor, }; factors.variables.add_provider_type(StaticVariables)?; @@ -79,11 +81,14 @@ async fn main() -> anyhow::Result<()> { struct TestSource; impl RuntimeConfigSource for TestSource { - fn config_keys(&self) -> impl IntoIterator { + fn factor_config_keys(&self) -> impl IntoIterator { [spin_factor_variables::RuntimeConfig::KEY] } - fn get_config(&self, key: &str) -> anyhow::Result> { + fn get_factor_config( + &self, + key: &str, + ) -> anyhow::Result> { let Some(table) = toml::toml! { [[variable_provider]] type = "static" From 647b1f5e9eb020c9a41ef628401528e3b6d05a20 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 12 Jun 2024 11:49:12 -0400 Subject: [PATCH 019/195] factors: Add KeyValueFactor Signed-off-by: Lann Martin --- Cargo.lock | 36 ++++- crates/factor-key-value/Cargo.toml | 17 +++ crates/factor-key-value/src/lib.rs | 163 ++++++++++++++++++++++ crates/factor-key-value/src/store.rs | 24 ++++ crates/factor-outbound-http/src/lib.rs | 3 + crates/factor-outbound-http/src/wasi.rs | 11 ++ crates/factor-variables/src/lib.rs | 24 ++-- crates/factor-variables/src/provider.rs | 4 +- crates/factors-derive/src/lib.rs | 14 +- crates/factors/Cargo.toml | 10 +- crates/factors/src/lib.rs | 1 + crates/factors/src/prepare.rs | 4 +- crates/factors/src/runtime_factors.rs | 44 +++--- crates/factors/tests/smoke-app/.gitignore | 2 + crates/factors/tests/smoke-app/Cargo.toml | 15 ++ crates/factors/tests/smoke-app/spin.toml | 25 ++++ crates/factors/tests/smoke-app/src/lib.rs | 19 +++ crates/factors/tests/smoke.rs | 126 +++++++++++++---- 18 files changed, 468 insertions(+), 74 deletions(-) create mode 100644 crates/factor-key-value/Cargo.toml create mode 100644 crates/factor-key-value/src/lib.rs create mode 100644 crates/factor-key-value/src/store.rs create mode 100644 crates/factors/tests/smoke-app/.gitignore create mode 100644 crates/factors/tests/smoke-app/Cargo.toml create mode 100644 crates/factors/tests/smoke-app/spin.toml create mode 100644 crates/factors/tests/smoke-app/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 2801b1fed..a530a00da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2476,6 +2476,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "field-offset" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" +dependencies = [ + "memoffset 0.9.1", + "rustc_version", +] + [[package]] name = "filetime" version = "0.2.23" @@ -3292,12 +3302,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", "http-body 1.0.0", "pin-project-lite", @@ -7548,6 +7558,18 @@ dependencies = [ "toml 0.5.11", ] +[[package]] +name = "spin-factor-key-value" +version = "2.6.0-pre0" +dependencies = [ + "anyhow", + "serde 1.0.197", + "spin-factors", + "spin-key-value", + "spin-world", + "toml 0.8.12", +] + [[package]] name = "spin-factor-outbound-http" version = "2.6.0-pre0" @@ -7600,19 +7622,27 @@ name = "spin-factors" version = "2.6.0-pre0" dependencies = [ "anyhow", + "field-offset", + "http 1.1.0", + "http-body-util", "serde 1.0.197", "serde_json", "spin-app", + "spin-componentize", + "spin-factor-key-value", "spin-factor-outbound-http", "spin-factor-outbound-networking", "spin-factor-variables", "spin-factor-wasi", "spin-factors-derive", + "spin-key-value-sqlite", + "spin-loader", "thiserror", "tokio", "toml 0.8.12", "tracing", "wasmtime", + "wasmtime-wasi-http", ] [[package]] diff --git a/crates/factor-key-value/Cargo.toml b/crates/factor-key-value/Cargo.toml new file mode 100644 index 000000000..89e060e46 --- /dev/null +++ b/crates/factor-key-value/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "spin-factor-key-value" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +serde = { version = "1.0", features = ["rc"] } +spin-factors = { path = "../factors" } +# TODO: merge with this crate +spin-key-value = { path = "../key-value" } +spin-world = { path = "../world" } +toml = "0.8" + +[lints] +workspace = true diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs new file mode 100644 index 000000000..55932e06f --- /dev/null +++ b/crates/factor-key-value/src/lib.rs @@ -0,0 +1,163 @@ +mod store; + +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use anyhow::{bail, ensure}; +use serde::Deserialize; +use spin_factors::{ + anyhow::{self, Context}, + ConfigureAppContext, Factor, FactorInstanceBuilder, FactorRuntimeConfig, InitContext, + InstanceBuilders, PrepareContext, RuntimeFactors, +}; +use spin_key_value::{ + CachingStoreManager, DelegatingStoreManager, KeyValueDispatch, StoreManager, + KEY_VALUE_STORES_KEY, +}; +use store::{store_from_toml_fn, StoreFromToml}; + +pub use store::MakeKeyValueStore; + +#[derive(Default)] +pub struct KeyValueFactor { + store_types: HashMap<&'static str, StoreFromToml>, +} + +impl KeyValueFactor { + pub fn add_store_type(&mut self, store_type: T) -> anyhow::Result<()> { + if self + .store_types + .insert(T::RUNTIME_CONFIG_TYPE, store_from_toml_fn(store_type)) + .is_some() + { + bail!( + "duplicate key value store type {:?}", + T::RUNTIME_CONFIG_TYPE + ); + } + Ok(()) + } +} + +impl Factor for KeyValueFactor { + type RuntimeConfig = RuntimeConfig; + type AppState = AppState; + type InstanceBuilder = InstanceBuilder; + + fn init( + &mut self, + mut ctx: InitContext, + ) -> anyhow::Result<()> { + ctx.link_bindings(spin_world::v1::key_value::add_to_linker)?; + ctx.link_bindings(spin_world::v2::key_value::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + mut ctx: ConfigureAppContext, + ) -> anyhow::Result { + // Build StoreManager from runtime config + let mut stores = HashMap::new(); + if let Some(runtime_config) = ctx.take_runtime_config() { + for (label, StoreConfig { type_, config }) in runtime_config.store_configs { + let store_maker = self + .store_types + .get(type_.as_str()) + .with_context(|| format!("unknown key value store type {type_:?}"))?; + let store = store_maker(config)?; + stores.insert(label, store); + } + } + let delegating_manager = DelegatingStoreManager::new(stores); + let caching_manager = CachingStoreManager::new(delegating_manager); + let store_manager = Arc::new(caching_manager); + + // Build component -> allowed stores map + let mut component_allowed_stores = HashMap::new(); + for component in ctx.app().components() { + let component_id = component.id().to_string(); + let key_value_stores = component + .get_metadata(KEY_VALUE_STORES_KEY)? + .unwrap_or_default() + .into_iter() + .collect::>(); + for label in &key_value_stores { + // TODO: port nicer errors from KeyValueComponent (via error type?) + ensure!( + store_manager.is_defined(label), + "unknown key_value_stores label {label:?} for component {component_id:?}" + ); + } + component_allowed_stores.insert(component_id, key_value_stores); + // TODO: warn (?) on unused store? + } + + Ok(AppState { + store_manager, + component_allowed_stores, + }) + } + + fn prepare( + ctx: PrepareContext, + _builders: &mut InstanceBuilders, + ) -> anyhow::Result { + let app_state = ctx.app_state(); + let allowed_stores = app_state + .component_allowed_stores + .get(ctx.app_component().id()) + .expect("component should be in component_stores") + .clone(); + Ok(InstanceBuilder { + store_manager: app_state.store_manager.clone(), + allowed_stores, + }) + } +} + +#[derive(Deserialize)] +#[serde(transparent)] +pub struct RuntimeConfig { + store_configs: HashMap, +} + +impl FactorRuntimeConfig for RuntimeConfig { + const KEY: &'static str = "key_value_store"; +} + +#[derive(Deserialize)] +struct StoreConfig { + #[serde(rename = "type")] + type_: String, + #[serde(flatten)] + config: toml::Table, +} + +type AppStoreManager = CachingStoreManager; + +pub struct AppState { + store_manager: Arc, + component_allowed_stores: HashMap>, +} + +pub struct InstanceBuilder { + store_manager: Arc, + allowed_stores: HashSet, +} + +impl FactorInstanceBuilder for InstanceBuilder { + type InstanceState = KeyValueDispatch; + + fn build(self) -> anyhow::Result { + let Self { + store_manager, + allowed_stores, + } = self; + let mut dispatch = KeyValueDispatch::new_with_capacity(u32::MAX); + dispatch.init(allowed_stores, store_manager); + Ok(dispatch) + } +} diff --git a/crates/factor-key-value/src/store.rs b/crates/factor-key-value/src/store.rs new file mode 100644 index 000000000..5b0c95658 --- /dev/null +++ b/crates/factor-key-value/src/store.rs @@ -0,0 +1,24 @@ +use std::sync::Arc; + +use serde::de::DeserializeOwned; +use spin_key_value::StoreManager; + +pub trait MakeKeyValueStore: 'static { + const RUNTIME_CONFIG_TYPE: &'static str; + + type RuntimeConfig: DeserializeOwned; + type StoreManager: StoreManager; + + fn make_store(&self, runtime_config: Self::RuntimeConfig) + -> anyhow::Result; +} + +pub(crate) type StoreFromToml = Box anyhow::Result>>; + +pub(crate) fn store_from_toml_fn(provider_type: T) -> StoreFromToml { + Box::new(move |table| { + let runtime_config: T::RuntimeConfig = table.try_into()?; + let provider = provider_type.make_store(runtime_config)?; + Ok(Arc::new(provider)) + }) +} diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 026e52413..3507b5b95 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -7,6 +7,9 @@ use spin_factors::{ SelfInstanceBuilder, }; use wasmtime_wasi_http::WasiHttpCtx; + +pub use wasi::get_wasi_http_view; + pub struct OutboundHttpFactor; impl Factor for OutboundHttpFactor { diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index 3de0d8437..daec51fd0 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -38,3 +38,14 @@ impl<'a> WasiHttpView for MutStates<'a> { self.wasi.table() } } + +// TODO: This is a little weird, organizationally +pub fn get_wasi_http_view( + instance_state: &mut T::InstanceState, +) -> anyhow::Result { + let wasi_and_http_getter = + T::instance_state_getter2::() + .context("failed to get WasiFactor")?; + let (wasi, http) = wasi_and_http_getter.get_states(instance_state); + Ok(MutStates { http, wasi }) +} diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 29c36a975..730d1d1df 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -2,7 +2,7 @@ mod provider; use std::{collections::HashMap, sync::Arc}; -use provider::{provider_from_toml, ProviderFromToml}; +use provider::{provider_from_toml_fn, ProviderFromToml}; use serde::Deserialize; use spin_expressions::ProviderResolver; use spin_factors::{ @@ -10,7 +10,7 @@ use spin_factors::{ ConfigureAppContext, Factor, FactorRuntimeConfig, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; -use spin_world::{async_trait, v1::config as v1_config, v2::variables}; +use spin_world::{async_trait, v1, v2::variables}; pub use provider::{MakeVariablesProvider, StaticVariables}; @@ -26,7 +26,7 @@ impl VariablesFactor { ) -> anyhow::Result<()> { if self .provider_types - .insert(T::RUNTIME_CONFIG_TYPE, provider_from_toml(provider_type)) + .insert(T::RUNTIME_CONFIG_TYPE, provider_from_toml_fn(provider_type)) .is_some() { bail!("duplicate provider type {:?}", T::RUNTIME_CONFIG_TYPE); @@ -44,8 +44,8 @@ impl Factor for VariablesFactor { &mut self, mut ctx: InitContext, ) -> anyhow::Result<()> { - ctx.link_bindings(v1_config::add_to_linker)?; - ctx.link_bindings(variables::add_to_linker)?; + ctx.link_bindings(spin_world::v1::config::add_to_linker)?; + ctx.link_bindings(spin_world::v2::variables::add_to_linker)?; Ok(()) } @@ -69,7 +69,7 @@ impl Factor for VariablesFactor { let provider_maker = self .provider_types .get(type_.as_str()) - .with_context(|| format!("unknown variables provider type {type_}"))?; + .with_context(|| format!("unknown variables provider type {type_:?}"))?; let provider = provider_maker(config)?; resolver.add_provider(provider); } @@ -144,18 +144,18 @@ impl variables::Host for InstanceState { } #[async_trait] -impl v1_config::Host for InstanceState { - async fn get_config(&mut self, key: String) -> Result { +impl v1::config::Host for InstanceState { + async fn get_config(&mut self, key: String) -> Result { ::get(self, key) .await .map_err(|err| match err { - variables::Error::InvalidName(msg) => v1_config::Error::InvalidKey(msg), - variables::Error::Undefined(msg) => v1_config::Error::Provider(msg), - other => v1_config::Error::Other(format!("{other}")), + variables::Error::InvalidName(msg) => v1::config::Error::InvalidKey(msg), + variables::Error::Undefined(msg) => v1::config::Error::Provider(msg), + other => v1::config::Error::Other(format!("{other}")), }) } - fn convert_error(&mut self, err: v1_config::Error) -> anyhow::Result { + fn convert_error(&mut self, err: v1::config::Error) -> anyhow::Result { Ok(err) } } diff --git a/crates/factor-variables/src/provider.rs b/crates/factor-variables/src/provider.rs index 83d634bf9..f3ee94811 100644 --- a/crates/factor-variables/src/provider.rs +++ b/crates/factor-variables/src/provider.rs @@ -15,7 +15,9 @@ pub trait MakeVariablesProvider: 'static { pub(crate) type ProviderFromToml = Box anyhow::Result>>; -pub(crate) fn provider_from_toml(provider_type: T) -> ProviderFromToml { +pub(crate) fn provider_from_toml_fn( + provider_type: T, +) -> ProviderFromToml { Box::new(move |table| { let runtime_config: T::RuntimeConfig = table.try_into()?; let provider = provider_type.make_provider(runtime_config)?; diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index b7e30734c..d64bc88b0 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -56,6 +56,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let TypeId = quote!(::std::any::TypeId); let factors_crate = format_ident!("spin_factors"); let factors_path = quote!(::#factors_crate); + let field_offset = quote!(#factors_path::__internal::field_offset); let wasmtime = quote!(#factors_path::wasmtime); let Result = quote!(#factors_path::Result); let Factor = quote!(#factors_path::Factor); @@ -172,16 +173,23 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { None } - fn instance_state_offset() -> Option { + fn instance_state_offset() -> Option< + #field_offset::FieldOffset< + Self::InstanceState, + <::InstanceBuilder as #FactorInstanceBuilder>::InstanceState, + > + > { let type_id = #TypeId::of::(); #( if type_id == #TypeId::of::<#factor_types>() { - return Some(std::mem::offset_of!(Self::InstanceState, #factor_names)); + let offset = #field_offset::offset_of!(Self::InstanceState => #factor_names); + return Some( + unsafe { ::std::mem::transmute(offset) } + ); } )* None } - } #vis struct #app_state_name { diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 117e89472..7a2e5f1d7 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -6,6 +6,7 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" +field-offset = "0.3.6" serde = "1.0" spin-app = { path = "../app" } spin-factors-derive = { path = "../factors-derive" } @@ -14,14 +15,21 @@ tracing = { workspace = true } wasmtime = { workspace = true } [dev-dependencies] +http = "1.1.0" +http-body-util = "0.1.2" serde_json = "1.0" +spin-componentize = { path = "../componentize" } spin-factors-derive = { path = "../factors-derive", features = ["expander"] } +spin-factor-key-value = { path = "../factor-key-value" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } -tokio = { version = "1", features = ["macros", "rt"] } +spin-key-value-sqlite = { path = "../key-value-sqlite" } +spin-loader = { path = "../loader" } +tokio = { version = "1", features = ["macros", "rt", "sync"] } toml = "0.8" +wasmtime-wasi-http = { workspace = true } [lints] workspace = true diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 208adb84f..b0203d6d5 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -28,4 +28,5 @@ pub type Result = wasmtime::Result; #[doc(hidden)] pub mod __internal { pub use crate::runtime_config::RuntimeConfigTracker; + pub use field_offset; } diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index 525f77404..851cce1f5 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -5,7 +5,7 @@ use anyhow::Context; use crate::{AppComponent, Factor, RuntimeFactors}; pub trait FactorInstanceBuilder: Any { - type InstanceState; + type InstanceState: Send + 'static; fn build(self) -> anyhow::Result; } @@ -18,7 +18,7 @@ impl FactorInstanceBuilder for () { } } -pub trait SelfInstanceBuilder: 'static {} +pub trait SelfInstanceBuilder: Send + 'static {} impl FactorInstanceBuilder for T { type InstanceState = Self; diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 4df65d475..73f746039 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,4 +1,4 @@ -use std::{any::TypeId, marker::PhantomData}; +use field_offset::FieldOffset; use crate::{factor::FactorInstanceState, Factor}; @@ -15,7 +15,8 @@ pub trait RuntimeFactors: Sized + 'static { ) -> Option>; #[doc(hidden)] - fn instance_state_offset() -> Option; + fn instance_state_offset( + ) -> Option>>; fn instance_state_getter() -> Option> { StateGetter::new() @@ -26,16 +27,14 @@ pub trait RuntimeFactors: Sized + 'static { } } -pub struct StateGetter { - offset: isize, - _phantom: PhantomData F>, +pub struct StateGetter { + offset: FieldOffset>, } impl StateGetter { fn new() -> Option { Some(Self { - offset: T::instance_state_offset::()?.try_into().unwrap(), - _phantom: PhantomData, + offset: T::instance_state_offset::()?, }) } @@ -43,8 +42,7 @@ impl StateGetter { &self, instance_state: &'a mut T::InstanceState, ) -> &'a mut FactorInstanceState { - let ptr = instance_state as *mut T::InstanceState; - unsafe { &mut *(ptr.offset(self.offset) as *mut FactorInstanceState) } + self.offset.apply_mut(instance_state) } } @@ -56,23 +54,21 @@ impl Clone for StateGetter { impl Copy for StateGetter {} -pub struct StateGetter2 { - offset1: isize, - offset2: isize, - _phantom: PhantomData (F1, F2)>, +pub struct StateGetter2 { + // Invariant: offsets must point at non-overlapping objects + offset1: FieldOffset>, + offset2: FieldOffset>, } impl StateGetter2 { fn new() -> Option { - // Only safe if F1 and F2 are different (and so do not alias) - if TypeId::of::() == TypeId::of::() { + let offset1 = T::instance_state_offset::()?; + let offset2 = T::instance_state_offset::()?; + // Make sure the two states don't point to the same field + if offset1.get_byte_offset() == offset2.get_byte_offset() { return None; } - Some(StateGetter2 { - offset1: T::instance_state_offset::()?.try_into().unwrap(), - offset2: T::instance_state_offset::()?.try_into().unwrap(), - _phantom: PhantomData, - }) + Some(StateGetter2 { offset1, offset2 }) } pub fn get_states<'a>( @@ -85,8 +81,8 @@ impl StateGetter2 { let ptr = instance_state as *mut T::InstanceState; unsafe { ( - &mut *(ptr.offset(self.offset1) as *mut FactorInstanceState), - &mut *(ptr.offset(self.offset2) as *mut FactorInstanceState), + &mut *(self.offset1.apply_ptr_mut(ptr) as *mut FactorInstanceState), + &mut *(self.offset2.apply_ptr_mut(ptr) as *mut FactorInstanceState), ) } } @@ -99,3 +95,7 @@ impl Clone for StateGetter2 Copy for StateGetter2 {} + +// TODO: This seems fine, but then again I don't understand why `FieldOffset`'s +// own `Sync`ness depends on `U`... +unsafe impl Sync for StateGetter2 {} diff --git a/crates/factors/tests/smoke-app/.gitignore b/crates/factors/tests/smoke-app/.gitignore new file mode 100644 index 000000000..386474fa5 --- /dev/null +++ b/crates/factors/tests/smoke-app/.gitignore @@ -0,0 +1,2 @@ +target/ +.spin/ diff --git a/crates/factors/tests/smoke-app/Cargo.toml b/crates/factors/tests/smoke-app/Cargo.toml new file mode 100644 index 000000000..d4fc36af1 --- /dev/null +++ b/crates/factors/tests/smoke-app/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "smoke-app" +authors = ["Lann Martin "] +description = "" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +anyhow = "1" +spin-sdk = "3.0.1" + +[workspace] diff --git a/crates/factors/tests/smoke-app/spin.toml b/crates/factors/tests/smoke-app/spin.toml new file mode 100644 index 000000000..a800b49fe --- /dev/null +++ b/crates/factors/tests/smoke-app/spin.toml @@ -0,0 +1,25 @@ +spin_manifest_version = 2 + +[application] +name = "smoke-app" +version = "0.1.0" +authors = ["Lann Martin "] +description = "" + +[variables] +host = { required = true } +other = { default = "other value" } + +[[trigger.http]] +route = "/..." +component = "smoke-app" + +[component.smoke-app] +source = "target/wasm32-wasi/release/smoke_app.wasm" +allowed_outbound_hosts = ["https://{{ host }}"] +key_value_stores = ["default"] +variables = { "other" = "<{{ other }}>"} + +[component.smoke-app.build] +command = "cargo build --target wasm32-wasi --release" +watch = ["src/**/*.rs", "Cargo.toml"] diff --git a/crates/factors/tests/smoke-app/src/lib.rs b/crates/factors/tests/smoke-app/src/lib.rs new file mode 100644 index 000000000..214373d29 --- /dev/null +++ b/crates/factors/tests/smoke-app/src/lib.rs @@ -0,0 +1,19 @@ +use spin_sdk::http::{IntoResponse, Request, Response}; +use spin_sdk::http_component; + +/// A simple Spin HTTP component. +#[http_component] +fn handle_smoke_app(_req: Request) -> anyhow::Result { + let var_val = spin_sdk::variables::get("other")?; + let kv_val = { + let store = spin_sdk::key_value::Store::open_default()?; + store.set("k", b"v")?; + store.get("k")? + }; + let body = format!("Test response\nVariable: {var_val}\nKV: {kv_val:?}"); + Ok(Response::builder() + .status(200) + .header("content-type", "text/plain") + .body(body) + .build()) +} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 6dfdf4899..6a749c806 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,50 +1,49 @@ +use std::path::PathBuf; + +use anyhow::bail; +use http_body_util::BodyExt; +use serde::Deserialize; use spin_app::App; +use spin_factor_key_value::{KeyValueFactor, MakeKeyValueStore}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{preview1::WasiPreview1Factor, DummyFilesMounter, WasiFactor}; use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; +use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; +use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] struct Factors { wasi: WasiFactor, wasi_p1: WasiPreview1Factor, variables: VariablesFactor, - outbound_networking_factor: OutboundNetworkingFactor, - outbound_http_factor: OutboundHttpFactor, + outbound_networking: OutboundNetworkingFactor, + outbound_http: OutboundHttpFactor, + key_value: KeyValueFactor, } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), wasi_p1: WasiPreview1Factor, variables: VariablesFactor::default(), - outbound_networking_factor: OutboundNetworkingFactor, - outbound_http_factor: OutboundHttpFactor, + outbound_networking: OutboundNetworkingFactor, + outbound_http: OutboundHttpFactor, + key_value: KeyValueFactor::default(), }; + factors.variables.add_provider_type(StaticVariables)?; - let locked = serde_json::from_value(serde_json::json!({ - "spin_lock_version": 1, - "variables": { - "foo": {} - }, - "triggers": [], - "components": [{ - "id": "test", - "metadata": { - "allowed_outbound_hosts": ["http://{{ foo }}"] - }, - "source": { - "content_type": "application/wasm", - "content": {"inline": "KGNvbXBvbmVudCk="} - }, - "config": { - "test_var": "{{foo}}" - } - }] - }))?; + factors.key_value.add_store_type(TestSpinKeyValueStore)?; + + let locked = spin_loader::from_file( + "tests/smoke-app/spin.toml", + spin_loader::FilesMountStrategy::Direct, + None, + ) + .await?; let app = App::inert(locked); let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; @@ -56,25 +55,60 @@ async fn main() -> anyhow::Result<()> { .unwrap(); let configured_app = factors.configure_app(app, TestSource)?; - let data = factors.build_store_data(&configured_app, "test")?; + let data = factors.build_store_data(&configured_app, "smoke-app")?; assert_eq!( data.variables .resolver() - .resolve("test", "test_var".try_into().unwrap()) + .resolve("smoke-app", "other".try_into().unwrap()) .await .unwrap(), - "bar" + "" ); let mut store = wasmtime::Store::new(&engine, data); - let component = wasmtime::component::Component::new(&engine, b"(component)")?; - let _instance = linker.instantiate_async(&mut store, &component).await?; + let component = configured_app.app().components().next().unwrap(); + let wasm_path = component + .source() + .content + .source + .as_deref() + .unwrap() + .strip_prefix("file://") + .unwrap(); + let wasm_bytes = std::fs::read(wasm_path)?; + let component_bytes = spin_componentize::componentize_if_necessary(&wasm_bytes)?; + let component = wasmtime::component::Component::new(&engine, component_bytes)?; + let instance = linker.instantiate_async(&mut store, &component).await?; let module = wasmtime::Module::new(&engine, b"(module)")?; let _module_instance = module_linker.instantiate_async(&mut store, &module).await?; + // Invoke handler + let req = http::Request::get("/").body(Default::default()).unwrap(); + let mut wasi_http_view = + spin_factor_outbound_http::get_wasi_http_view::(store.data_mut())?; + let request = wasi_http_view.new_incoming_request(req)?; + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + let response = wasi_http_view.new_response_outparam(response_tx)?; + drop(wasi_http_view); + + let guest = wasmtime_wasi_http::proxy::Proxy::new(&mut store, &instance)?; + let call_task = tokio::spawn(async move { + guest + .wasi_http_incoming_handler() + .call_handle(&mut store, request, response) + .await + }); + let resp_task = tokio::spawn(async { + let resp = response_rx.await.unwrap().unwrap(); + let body = resp.into_body().collect().await.unwrap().to_bytes(); + eprintln!("Response: {body:?}"); + }); + let (call_res, resp_res) = tokio::join!(call_task, resp_task); + let _ = call_res?; + resp_res?; Ok(()) } @@ -94,6 +128,9 @@ impl RuntimeConfigSource for TestSource { type = "static" [variable_provider.values] foo = "bar" + + [key_value_store.default] + type = "spin" } .remove(key) else { return Ok(None); @@ -102,3 +139,32 @@ impl RuntimeConfigSource for TestSource { Ok(Some(config)) } } + +struct TestSpinKeyValueStore; + +impl MakeKeyValueStore for TestSpinKeyValueStore { + const RUNTIME_CONFIG_TYPE: &'static str = "spin"; + + type RuntimeConfig = TestSpinKeyValueRuntimeConfig; + + type StoreManager = KeyValueSqlite; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + let location = match runtime_config.path { + Some(_) => { + // TODO(lann): need state_dir to derive default store path + bail!("spin key value runtime config not implemented") + } + None => DatabaseLocation::InMemory, + }; + Ok(KeyValueSqlite::new(location)) + } +} + +#[derive(Deserialize)] +struct TestSpinKeyValueRuntimeConfig { + path: Option, +} From 6668cb8b70a8fdbd65acca061d176aeeb240e234 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 13 Jun 2024 09:21:37 -0400 Subject: [PATCH 020/195] factors: Replace some transmute with Any::downcast_* Signed-off-by: Lann Martin --- crates/factors-derive/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index d64bc88b0..39b730034 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -53,6 +53,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { factor_types.push(&field.ty); } + let Any = quote!(::std::any::Any); let TypeId = quote!(::std::any::TypeId); let factors_crate = format_ident!("spin_factors"); let factors_path = quote!(::#factors_crate); @@ -146,11 +147,10 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { type InstanceState = #state_name; fn app_state(app_state: &Self::AppState) -> Option<&F::AppState> { - let type_id = #TypeId::of::(); #( - if type_id == #TypeId::of::<#factor_types>() { - unsafe { - return Some(::std::mem::transmute(&app_state.#factor_names)); + if let Some(state) = &app_state.#factor_names { + if let Some(state) = ::downcast_ref(state) { + return Some(state) } } )* @@ -165,7 +165,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { if type_id == #TypeId::of::<#factor_types>() { return Some( builders.#factor_names.as_mut().map(|builder| { - unsafe { ::std::mem::transmute(builder) } + ::downcast_mut(builder).unwrap() }) ); } From d281f4a5d94b7100747c27340fa3feec8750afa8 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 13 Jun 2024 17:20:20 -0400 Subject: [PATCH 021/195] factors: Tweak Factor::prepare signature Signed-off-by: Lann Martin --- crates/factor-key-value/src/lib.rs | 1 + crates/factor-outbound-http/src/lib.rs | 1 + crates/factor-outbound-networking/src/lib.rs | 1 + crates/factor-variables/src/lib.rs | 1 + crates/factor-wasi/src/lib.rs | 4 ++-- crates/factor-wasi/src/preview1.rs | 1 + crates/factors-derive/src/lib.rs | 2 +- crates/factors/src/factor.rs | 1 + crates/factors/src/prepare.rs | 8 +------- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index 55932e06f..efca63a16 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -102,6 +102,7 @@ impl Factor for KeyValueFactor { } fn prepare( + &self, ctx: PrepareContext, _builders: &mut InstanceBuilders, ) -> anyhow::Result { diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 3507b5b95..15ec5f8ed 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -36,6 +36,7 @@ impl Factor for OutboundHttpFactor { } fn prepare( + &self, _ctx: PrepareContext, builders: &mut InstanceBuilders, ) -> anyhow::Result { diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index b4f88c714..66b5e3732 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -49,6 +49,7 @@ impl Factor for OutboundNetworkingFactor { } fn prepare( + &self, ctx: PrepareContext, builders: &mut InstanceBuilders, ) -> anyhow::Result { diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 730d1d1df..498e96c89 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -81,6 +81,7 @@ impl Factor for VariablesFactor { } fn prepare( + &self, ctx: PrepareContext, _builders: &mut InstanceBuilders, ) -> anyhow::Result { diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 226477bb3..2789ac2a8 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -78,6 +78,7 @@ impl Factor for WasiFactor { } fn prepare( + &self, ctx: PrepareContext, _builders: &mut InstanceBuilders, ) -> anyhow::Result { @@ -92,8 +93,7 @@ impl Factor for WasiFactor { let mount_ctx = MountFilesContext { wasi_ctx: &mut wasi_ctx, }; - ctx.factor() - .files_mounter + self.files_mounter .mount_files(ctx.app_component(), mount_ctx)?; Ok(InstanceBuilder { wasi_ctx }) diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs index 743708668..3839e3c68 100644 --- a/crates/factor-wasi/src/preview1.rs +++ b/crates/factor-wasi/src/preview1.rs @@ -23,6 +23,7 @@ impl Factor for WasiPreview1Factor { } fn prepare( + &self, _ctx: spin_factors::PrepareContext, _builders: &mut spin_factors::InstanceBuilders, ) -> anyhow::Result { diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 39b730034..d18415330 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -121,8 +121,8 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #( builders.#factor_names = Some( #Factor::prepare::( + &self.#factor_names, #factors_path::PrepareContext::new( - &self.#factor_names, configured_app.app_state::<#factor_types>().unwrap(), &app_component, ), diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 2f3b25838..a62d6e669 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -38,6 +38,7 @@ pub trait Factor: Any + Sized { /// This method is given access to the app component being instantiated and /// to any other factors' instance builders that have already been prepared. fn prepare( + &self, ctx: PrepareContext, _builders: &mut InstanceBuilders, ) -> anyhow::Result; diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index 851cce1f5..7e3c7f6d6 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -32,25 +32,19 @@ impl FactorInstanceBuilder for T { /// already-initialized [`FactorInstanceBuilder`]s, allowing for /// inter-[`Factor`] dependencies. pub struct PrepareContext<'a, F: Factor> { - pub(crate) factor: &'a F, pub(crate) app_state: &'a F::AppState, pub(crate) app_component: &'a AppComponent<'a>, } impl<'a, F: Factor> PrepareContext<'a, F> { #[doc(hidden)] - pub fn new(factor: &'a F, app_state: &'a F::AppState, app_component: &'a AppComponent) -> Self { + pub fn new(app_state: &'a F::AppState, app_component: &'a AppComponent) -> Self { Self { - factor, app_state, app_component, } } - pub fn factor(&self) -> &F { - self.factor - } - pub fn app_state(&self) -> &F::AppState { self.app_state } From 87d8f4c41d4750390fad278990e4cf7109caa569 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 18 Jun 2024 13:48:41 -0400 Subject: [PATCH 022/195] factors: Remove module code Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/lib.rs | 4 +- crates/factor-wasi/src/lib.rs | 61 ++++++++++++-------------- crates/factor-wasi/src/preview1.rs | 47 -------------------- crates/factors-derive/src/lib.rs | 6 +-- crates/factors/src/factor.rs | 46 +++---------------- crates/factors/src/lib.rs | 1 - crates/factors/tests/smoke.rs | 12 +---- 7 files changed, 41 insertions(+), 136 deletions(-) delete mode 100644 crates/factor-wasi/src/preview1.rs diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 15ec5f8ed..36000239b 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -22,9 +22,7 @@ impl Factor for OutboundHttpFactor { mut ctx: spin_factors::InitContext, ) -> anyhow::Result<()> { ctx.link_bindings(spin_world::v1::http::add_to_linker)?; - if let Some(linker) = ctx.linker() { - wasi::add_to_linker::(linker)?; - } + wasi::add_to_linker::(ctx.linker())?; Ok(()) } diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 2789ac2a8..9d80131b6 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,5 +1,3 @@ -pub mod preview1; - use std::{future::Future, net::SocketAddr, path::Path}; use spin_factors::{ @@ -37,36 +35,35 @@ impl Factor for WasiFactor { } let get_data = ctx.get_data_fn(); let closure = type_annotate(move |data| WasiImpl(get_data(data))); - if let Some(linker) = ctx.linker() { - use wasmtime_wasi::bindings; - bindings::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; - bindings::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; - bindings::filesystem::types::add_to_linker_get_host(linker, closure)?; - bindings::filesystem::preopens::add_to_linker_get_host(linker, closure)?; - bindings::io::error::add_to_linker_get_host(linker, closure)?; - bindings::io::poll::add_to_linker_get_host(linker, closure)?; - bindings::io::streams::add_to_linker_get_host(linker, closure)?; - bindings::random::random::add_to_linker_get_host(linker, closure)?; - bindings::random::insecure::add_to_linker_get_host(linker, closure)?; - bindings::random::insecure_seed::add_to_linker_get_host(linker, closure)?; - bindings::cli::exit::add_to_linker_get_host(linker, closure)?; - bindings::cli::environment::add_to_linker_get_host(linker, closure)?; - bindings::cli::stdin::add_to_linker_get_host(linker, closure)?; - bindings::cli::stdout::add_to_linker_get_host(linker, closure)?; - bindings::cli::stderr::add_to_linker_get_host(linker, closure)?; - bindings::cli::terminal_input::add_to_linker_get_host(linker, closure)?; - bindings::cli::terminal_output::add_to_linker_get_host(linker, closure)?; - bindings::cli::terminal_stdin::add_to_linker_get_host(linker, closure)?; - bindings::cli::terminal_stdout::add_to_linker_get_host(linker, closure)?; - bindings::cli::terminal_stderr::add_to_linker_get_host(linker, closure)?; - bindings::sockets::tcp::add_to_linker_get_host(linker, closure)?; - bindings::sockets::tcp_create_socket::add_to_linker_get_host(linker, closure)?; - bindings::sockets::udp::add_to_linker_get_host(linker, closure)?; - bindings::sockets::udp_create_socket::add_to_linker_get_host(linker, closure)?; - bindings::sockets::instance_network::add_to_linker_get_host(linker, closure)?; - bindings::sockets::network::add_to_linker_get_host(linker, closure)?; - bindings::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; - } + let linker = ctx.linker(); + use wasmtime_wasi::bindings; + bindings::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; + bindings::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; + bindings::filesystem::types::add_to_linker_get_host(linker, closure)?; + bindings::filesystem::preopens::add_to_linker_get_host(linker, closure)?; + bindings::io::error::add_to_linker_get_host(linker, closure)?; + bindings::io::poll::add_to_linker_get_host(linker, closure)?; + bindings::io::streams::add_to_linker_get_host(linker, closure)?; + bindings::random::random::add_to_linker_get_host(linker, closure)?; + bindings::random::insecure::add_to_linker_get_host(linker, closure)?; + bindings::random::insecure_seed::add_to_linker_get_host(linker, closure)?; + bindings::cli::exit::add_to_linker_get_host(linker, closure)?; + bindings::cli::environment::add_to_linker_get_host(linker, closure)?; + bindings::cli::stdin::add_to_linker_get_host(linker, closure)?; + bindings::cli::stdout::add_to_linker_get_host(linker, closure)?; + bindings::cli::stderr::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_input::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_output::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_stdin::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_stdout::add_to_linker_get_host(linker, closure)?; + bindings::cli::terminal_stderr::add_to_linker_get_host(linker, closure)?; + bindings::sockets::tcp::add_to_linker_get_host(linker, closure)?; + bindings::sockets::tcp_create_socket::add_to_linker_get_host(linker, closure)?; + bindings::sockets::udp::add_to_linker_get_host(linker, closure)?; + bindings::sockets::udp_create_socket::add_to_linker_get_host(linker, closure)?; + bindings::sockets::instance_network::add_to_linker_get_host(linker, closure)?; + bindings::sockets::network::add_to_linker_get_host(linker, closure)?; + bindings::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; Ok(()) } diff --git a/crates/factor-wasi/src/preview1.rs b/crates/factor-wasi/src/preview1.rs deleted file mode 100644 index 3839e3c68..000000000 --- a/crates/factor-wasi/src/preview1.rs +++ /dev/null @@ -1,47 +0,0 @@ -use spin_factors::{anyhow, Factor, FactorInstanceBuilder, InitContext, RuntimeFactors}; -use wasmtime_wasi::{preview1::WasiP1Ctx, WasiCtxBuilder}; - -pub struct WasiPreview1Factor; - -impl Factor for WasiPreview1Factor { - type RuntimeConfig = (); - type AppState = (); - type InstanceBuilder = InstanceBuilder; - - fn init( - &mut self, - mut ctx: InitContext, - ) -> anyhow::Result<()> { - ctx.link_module_bindings(wasmtime_wasi::preview1::add_to_linker_async) - } - - fn configure_app( - &self, - _ctx: spin_factors::ConfigureAppContext, - ) -> anyhow::Result { - Ok(()) - } - - fn prepare( - &self, - _ctx: spin_factors::PrepareContext, - _builders: &mut spin_factors::InstanceBuilders, - ) -> anyhow::Result { - Ok(InstanceBuilder { - wasi_ctx: WasiCtxBuilder::new(), - }) - } -} - -pub struct InstanceBuilder { - wasi_ctx: WasiCtxBuilder, -} - -impl FactorInstanceBuilder for InstanceBuilder { - type InstanceState = WasiP1Ctx; - - fn build(self) -> anyhow::Result { - let Self { mut wasi_ctx } = self; - Ok(wasi_ctx.build_p1()) - } -} diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index d18415330..0b980570f 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -70,15 +70,13 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #[allow(clippy::needless_option_as_deref)] pub fn init( &mut self, - mut linker: Option<&mut #wasmtime::component::Linker<#state_name>>, - mut module_linker: Option<&mut #wasmtime::Linker<#state_name>>, + linker: &mut #wasmtime::component::Linker<#state_name>, ) -> #Result<()> { #( #Factor::init::( &mut self.#factor_names, #factors_path::InitContext::::new( - linker.as_deref_mut(), - module_linker.as_deref_mut(), + linker, |state| &mut state.#factor_names, ) )?; diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index a62d6e669..a3d074795 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -4,7 +4,7 @@ use anyhow::Context; use crate::{ prepare::FactorInstanceBuilder, runtime_config::RuntimeConfigTracker, App, FactorRuntimeConfig, - InstanceBuilders, Linker, ModuleLinker, PrepareContext, RuntimeConfigSource, RuntimeFactors, + InstanceBuilders, Linker, PrepareContext, RuntimeConfigSource, RuntimeFactors, }; pub trait Factor: Any + Sized { @@ -65,31 +65,18 @@ pub(crate) type GetDataFn = /// An InitContext is passed to [`Factor::init`], giving access to the global /// common [`wasmtime::component::Linker`]. pub struct InitContext<'a, T: RuntimeFactors, F: Factor> { - pub(crate) linker: Option<&'a mut Linker>, - pub(crate) module_linker: Option<&'a mut ModuleLinker>, + pub(crate) linker: &'a mut Linker, pub(crate) get_data: GetDataFn, } impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { #[doc(hidden)] - pub fn new( - linker: Option<&'a mut Linker>, - module_linker: Option<&'a mut ModuleLinker>, - get_data: GetDataFn, - ) -> Self { - Self { - linker, - module_linker, - get_data, - } + pub fn new(linker: &'a mut Linker, get_data: GetDataFn) -> Self { + Self { linker, get_data } } - pub fn linker(&mut self) -> Option<&mut Linker> { - self.linker.as_deref_mut() - } - - pub fn module_linker(&mut self) -> Option<&mut ModuleLinker> { - self.module_linker.as_deref_mut() + pub fn linker(&mut self) -> &mut Linker { + self.linker } pub fn get_data_fn(&self) -> GetDataFn { @@ -104,26 +91,7 @@ impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { ) -> anyhow::Result<()>, ) -> anyhow::Result<()> where { - if let Some(linker) = self.linker.as_deref_mut() { - add_to_linker(linker, self.get_data) - } else { - Ok(()) - } - } - - pub fn link_module_bindings( - &mut self, - add_to_linker: impl Fn( - &mut ModuleLinker, - fn(&mut T::InstanceState) -> &mut FactorInstanceState, - ) -> anyhow::Result<()>, - ) -> anyhow::Result<()> -where { - if let Some(linker) = self.module_linker.as_deref_mut() { - add_to_linker(linker, self.get_data) - } else { - Ok(()) - } + add_to_linker(self.linker, self.get_data) } } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index b0203d6d5..bfdbf0f54 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -16,7 +16,6 @@ pub use crate::{ }; pub type Linker = wasmtime::component::Linker<::InstanceState>; -pub type ModuleLinker = wasmtime::Linker<::InstanceState>; // Temporary wrappers while refactoring pub type App = spin_app::App<'static, spin_app::InertLoader>; diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 6a749c806..dcbc043f2 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -8,7 +8,7 @@ use spin_factor_key_value::{KeyValueFactor, MakeKeyValueStore}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; -use spin_factor_wasi::{preview1::WasiPreview1Factor, DummyFilesMounter, WasiFactor}; +use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; use wasmtime_wasi_http::WasiHttpView; @@ -16,7 +16,6 @@ use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] struct Factors { wasi: WasiFactor, - wasi_p1: WasiPreview1Factor, variables: VariablesFactor, outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, @@ -27,7 +26,6 @@ struct Factors { async fn main() -> anyhow::Result<()> { let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), - wasi_p1: WasiPreview1Factor, variables: VariablesFactor::default(), outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, @@ -48,11 +46,8 @@ async fn main() -> anyhow::Result<()> { let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; let mut linker = wasmtime::component::Linker::new(&engine); - let mut module_linker = wasmtime::Linker::new(&engine); - factors - .init(Some(&mut linker), Some(&mut module_linker)) - .unwrap(); + factors.init(&mut linker).unwrap(); let configured_app = factors.configure_app(app, TestSource)?; let data = factors.build_store_data(&configured_app, "smoke-app")?; @@ -82,9 +77,6 @@ async fn main() -> anyhow::Result<()> { let component = wasmtime::component::Component::new(&engine, component_bytes)?; let instance = linker.instantiate_async(&mut store, &component).await?; - let module = wasmtime::Module::new(&engine, b"(module)")?; - let _module_instance = module_linker.instantiate_async(&mut store, &module).await?; - // Invoke handler let req = http::Request::get("/").body(Default::default()).unwrap(); let mut wasi_http_view = From 431d66c53773c5600a281d0ab72bfa203d5f79bb Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 1 Jul 2024 11:14:20 +0200 Subject: [PATCH 023/195] Small nits Signed-off-by: Ryan Levick --- Cargo.lock | 116 ++++++++++++++++++++++------------ crates/factors/tests/smoke.rs | 7 +- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a530a00da..ce128fcac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1566,7 +1566,8 @@ dependencies = [ [[package]] name = "cranelift-bforest" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6b33d7e757a887989eb18b35712b2a67d96171ec3149d1bfb657b29b7b367c" dependencies = [ "cranelift-entity", ] @@ -1574,7 +1575,8 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9acf15cb22be42d07c3b57d7856329cb228b7315d385346149df2566ad5e4aa" dependencies = [ "bumpalo", "cranelift-bforest", @@ -1595,7 +1597,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e934d301392b73b3f8b0540391fb82465a0f179a3cee7c726482ac4727efcc97" dependencies = [ "cranelift-codegen-shared", ] @@ -1603,12 +1606,14 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb2a2566b3d54b854dfb288b3b187f6d3d17d6f762c92898207eba302931da" [[package]] name = "cranelift-control" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0100f33b704cdacd01ad66ff41f8c5030d57cbff078e2a4e49ab1822591299fa" dependencies = [ "arbitrary", ] @@ -1616,7 +1621,8 @@ dependencies = [ [[package]] name = "cranelift-entity" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8cfdc315e5d18997093e040a8d234bea1ac1e118a716d3e30f40d449e78207b" dependencies = [ "serde 1.0.197", "serde_derive", @@ -1625,7 +1631,8 @@ dependencies = [ [[package]] name = "cranelift-frontend" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f74b84f16af2e982b0c0c72233503d9d55cbfe3865dbe807ca28dc6642a28b5" dependencies = [ "cranelift-codegen", "log", @@ -1636,12 +1643,14 @@ dependencies = [ [[package]] name = "cranelift-isle" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adf306d3dde705fb94bd48082f01d38c4ededc74293a4c007805f610bf08bc6e" [[package]] name = "cranelift-native" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea0ebdef7aff4a79bcbc8b6495f31315f16b3bf311152f472eaa8d679352581" dependencies = [ "cranelift-codegen", "libc", @@ -1651,7 +1660,8 @@ dependencies = [ [[package]] name = "cranelift-wasm" version = "0.109.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d549108a1942065cdbac3bb96c2952afa0e1b9a3beff4b08c4308ac72257576d" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -7560,19 +7570,19 @@ dependencies = [ [[package]] name = "spin-factor-key-value" -version = "2.6.0-pre0" +version = "2.7.0-pre0" dependencies = [ "anyhow", "serde 1.0.197", "spin-factors", "spin-key-value", "spin-world", - "toml 0.8.12", + "toml 0.8.14", ] [[package]] name = "spin-factor-outbound-http" -version = "2.6.0-pre0" +version = "2.7.0-pre0" dependencies = [ "anyhow", "http 1.1.0", @@ -7586,7 +7596,7 @@ dependencies = [ [[package]] name = "spin-factor-outbound-networking" -version = "2.6.0-pre0" +version = "2.7.0-pre0" dependencies = [ "futures-util", "ipnet", @@ -7599,18 +7609,18 @@ dependencies = [ [[package]] name = "spin-factor-variables" -version = "2.6.0-pre0" +version = "2.7.0-pre0" dependencies = [ "serde 1.0.197", "spin-expressions", "spin-factors", "spin-world", - "toml 0.8.12", + "toml 0.8.14", ] [[package]] name = "spin-factor-wasi" -version = "2.6.0-pre0" +version = "2.7.0-pre0" dependencies = [ "cap-primitives 3.0.0", "spin-factors", @@ -7619,7 +7629,7 @@ dependencies = [ [[package]] name = "spin-factors" -version = "2.6.0-pre0" +version = "2.7.0-pre0" dependencies = [ "anyhow", "field-offset", @@ -7639,7 +7649,7 @@ dependencies = [ "spin-loader", "thiserror", "tokio", - "toml 0.8.12", + "toml 0.8.14", "tracing", "wasmtime", "wasmtime-wasi-http", @@ -7647,7 +7657,7 @@ dependencies = [ [[package]] name = "spin-factors-derive" -version = "2.6.0-pre0" +version = "2.7.0-pre0" dependencies = [ "expander", "proc-macro2", @@ -9523,7 +9533,8 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi-common" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86fd41e1e26ff6af9451c6a332a5ce5f5283ca51e87d875cdd9a05305598ee3" dependencies = [ "anyhow", "bitflags 2.5.0", @@ -9856,7 +9867,8 @@ dependencies = [ [[package]] name = "wasmtime" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "786d8b5e7a4d54917c5ebe555b9667337e5f93383f49bddaaeec2eba68093b45" dependencies = [ "addr2line", "anyhow", @@ -9911,7 +9923,8 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d697d99c341d4a9ffb72f3af7a02124d233eeb59aee010f36d88e97cca553d5e" dependencies = [ "cfg-if", ] @@ -9919,7 +9932,8 @@ dependencies = [ [[package]] name = "wasmtime-cache" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916610f9ae9a6c22deb25bba2e6247ba9f00b093d30620875203b91328a1adfa" dependencies = [ "anyhow", "base64 0.21.7", @@ -9938,7 +9952,8 @@ dependencies = [ [[package]] name = "wasmtime-component-macro" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29b462b068e73b5b27fae092a27f47e5937cabf6b26be2779c978698a52feca" dependencies = [ "anyhow", "proc-macro2", @@ -9952,12 +9967,14 @@ dependencies = [ [[package]] name = "wasmtime-component-util" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d2912c53d9054984b380dfbd7579f9c3681b2a73b903a56bd71a1c4f175f1e" [[package]] name = "wasmtime-cranelift" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3975deafea000457ba84355c7c0fce0372937204f77026510b7b454f28a3a65" dependencies = [ "anyhow", "cfg-if", @@ -9980,7 +9997,8 @@ dependencies = [ [[package]] name = "wasmtime-environ" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f444e900e848b884d8a8a2949b6f5b92af642a3e663ff8fbe78731143a55be61" dependencies = [ "anyhow", "cpp_demangle", @@ -10004,7 +10022,8 @@ dependencies = [ [[package]] name = "wasmtime-fiber" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ded58eb2d1bf0dcd2182d0ccd7055c4b10b50d711514f1d73f61515d0fa829d" dependencies = [ "anyhow", "cc", @@ -10018,7 +10037,8 @@ dependencies = [ [[package]] name = "wasmtime-jit-debug" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bc54198c6720f098210a85efb3ba8c078d1de4d373cdb6778850a66ae088d11" dependencies = [ "object 0.36.0", "once_cell", @@ -10029,7 +10049,8 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5afe2f0499542f9a4bcfa1b55bfdda803b6ade4e7c93c6b99e0f39dba44b0a91" dependencies = [ "anyhow", "cfg-if", @@ -10040,12 +10061,14 @@ dependencies = [ [[package]] name = "wasmtime-slab" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7de1f2bec5bbb35d532e61c85c049dc84ae671df60492f90b954ecf21169e7" [[package]] name = "wasmtime-types" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "412463e9000e14cf6856be48628d2213c20c153e29ffc22b036980c892ea6964" dependencies = [ "cranelift-entity", "serde 1.0.197", @@ -10057,7 +10080,8 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5a9bc4f44ceeb168e9e8e3be4e0b4beb9095b468479663a9e24c667e36826f" dependencies = [ "proc-macro2", "quote", @@ -10067,7 +10091,8 @@ dependencies = [ [[package]] name = "wasmtime-wasi" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8abb1301089ed8e0b4840f539cba316a73ac382090f1b25d22d8c8eed8df49c7" dependencies = [ "anyhow", "async-trait", @@ -10097,7 +10122,8 @@ dependencies = [ [[package]] name = "wasmtime-wasi-http" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "315cadc284b808cfbd6be9295da4009144c106723f09b421ce6c6d89275cfdb7" dependencies = [ "anyhow", "async-trait", @@ -10119,7 +10145,8 @@ dependencies = [ [[package]] name = "wasmtime-winch" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed4db238a0241df2d15f79ad17b3a37a27f2ea6cb885894d81b42ae107544466" dependencies = [ "anyhow", "cranelift-codegen", @@ -10135,7 +10162,8 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc077306b38288262e5ba01d4b21532a6987416cdc0aedf04bb06c22a68fdc" dependencies = [ "anyhow", "heck 0.4.1", @@ -10314,7 +10342,8 @@ dependencies = [ [[package]] name = "wiggle" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29830e5d01c182d24b94092c697aa7ab0ee97d22e78a2bf40ca91eae6ebca5c2" dependencies = [ "anyhow", "async-trait", @@ -10328,7 +10357,8 @@ dependencies = [ [[package]] name = "wiggle-generate" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "557567f2793508760cd855f7659b7a0b9dc4dbc451f53f1415d6943a15311ade" dependencies = [ "anyhow", "heck 0.4.1", @@ -10342,7 +10372,8 @@ dependencies = [ [[package]] name = "wiggle-macro" version = "22.0.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc26129a8aea20b62c961d1b9ab4a3c3b56b10042ed85d004f8678af0f21ba6e" dependencies = [ "proc-macro2", "quote", @@ -10384,7 +10415,8 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" version = "0.20.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=69a78fc1359769f320b5968042cb9d988336dd54#69a78fc1359769f320b5968042cb9d988336dd54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c6915884e731b2db0d8cf08cb64474cb69221a161675fd3c135f91febc3daa" dependencies = [ "anyhow", "cranelift-codegen", diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index dcbc043f2..59738b4cf 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use anyhow::bail; +use anyhow::{bail, Context}; use http_body_util::BodyExt; use serde::Deserialize; use spin_app::App; @@ -23,7 +23,7 @@ struct Factors { } #[tokio::test(flavor = "multi_thread")] -async fn main() -> anyhow::Result<()> { +async fn smoke_test_works() -> anyhow::Result<()> { let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), variables: VariablesFactor::default(), @@ -72,7 +72,8 @@ async fn main() -> anyhow::Result<()> { .unwrap() .strip_prefix("file://") .unwrap(); - let wasm_bytes = std::fs::read(wasm_path)?; + let wasm_bytes = std::fs::read(wasm_path) + .with_context(|| format!("wasm binary not found at '{wasm_path}'. Did you remember to run `spin build` in the `smoke-app` directory?"))?; let component_bytes = spin_componentize::componentize_if_necessary(&wasm_bytes)?; let component = wasmtime::component::Component::new(&engine, component_bytes)?; let instance = linker.instantiate_async(&mut store, &component).await?; From d68f40bf40bbe9a62c893f2912db1dda928ae551 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 1 Jul 2024 14:50:33 +0200 Subject: [PATCH 024/195] Get rid of field offsets in favor of dynamic Any lookup. Signed-off-by: Ryan Levick --- Cargo.lock | 11 --- crates/factor-outbound-http/src/wasi.rs | 46 ++++++------- crates/factors-derive/src/lib.rs | 48 ++++++------- crates/factors/Cargo.toml | 1 - crates/factors/src/factor.rs | 6 +- crates/factors/src/lib.rs | 5 +- crates/factors/src/runtime_factors.rs | 92 ++----------------------- crates/factors/tests/smoke.rs | 9 +-- 8 files changed, 59 insertions(+), 159 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce128fcac..f3f4a66d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2486,16 +2486,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "field-offset" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" -dependencies = [ - "memoffset 0.9.1", - "rustc_version", -] - [[package]] name = "filetime" version = "0.2.23" @@ -7632,7 +7622,6 @@ name = "spin-factors" version = "2.7.0-pre0" dependencies = [ "anyhow", - "field-offset", "http 1.1.0", "http-body-util", "serde 1.0.197", diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index daec51fd0..dd6b5415f 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -1,21 +1,14 @@ -use anyhow::Context; -use spin_factors::{Linker, RuntimeFactors}; +use spin_factors::{GetFactorState, Linker, RuntimeFactors}; use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; pub(crate) fn add_to_linker(linker: &mut Linker) -> anyhow::Result<()> { - fn type_annotate(f: F) -> F + fn type_annotate(f: F) -> F where - F: Fn(&mut T) -> WasiHttpImpl, + F: Fn(&mut T) -> WasiHttpImpl>, { f } - let wasi_and_http_getter = - T::instance_state_getter2::() - .context("failed to get WasiFactor")?; - let host_getter = type_annotate(move |data| { - let (wasi, http) = wasi_and_http_getter.get_states(data); - WasiHttpImpl(MutStates { http, wasi }) - }); + let host_getter = type_annotate(move |data| WasiHttpImpl(MutStates { inner: data })); wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker_get_host( linker, host_getter, @@ -24,28 +17,35 @@ pub(crate) fn add_to_linker(linker: &mut Linker) -> anyhow Ok(()) } -struct MutStates<'a> { - http: &'a mut crate::InstanceState, - wasi: &'a mut spin_factor_wasi::InstanceState, +struct MutStates<'a, T> { + inner: &'a mut T, } -impl<'a> WasiHttpView for MutStates<'a> { +impl<'a, T> WasiHttpView for MutStates<'a, T> +where + T: GetFactorState + Send, +{ fn ctx(&mut self) -> &mut wasmtime_wasi_http::WasiHttpCtx { - &mut self.http.wasi_http_ctx + &mut self + .inner + .get::() + .expect("failed to get `OutboundHttpFactor`") + .wasi_http_ctx } fn table(&mut self) -> &mut spin_factors::wasmtime::component::ResourceTable { - self.wasi.table() + self.inner + .get::() + .expect("failed to get `WasiFactor`") + .table() } } // TODO: This is a little weird, organizationally pub fn get_wasi_http_view( instance_state: &mut T::InstanceState, -) -> anyhow::Result { - let wasi_and_http_getter = - T::instance_state_getter2::() - .context("failed to get WasiFactor")?; - let (wasi, http) = wasi_and_http_getter.get_states(instance_state); - Ok(MutStates { http, wasi }) +) -> impl WasiHttpView + '_ { + MutStates { + inner: instance_state, + } } diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 0b980570f..fba2f8b05 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -54,10 +54,11 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } let Any = quote!(::std::any::Any); + let Box = quote!(::std::boxed::Box); + let Send = quote!(::std::marker::Send); let TypeId = quote!(::std::any::TypeId); let factors_crate = format_ident!("spin_factors"); let factors_path = quote!(::#factors_crate); - let field_offset = quote!(#factors_path::__internal::field_offset); let wasmtime = quote!(#factors_path::wasmtime); let Result = quote!(#factors_path::Result); let Factor = quote!(#factors_path::Factor); @@ -77,7 +78,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &mut self.#factor_names, #factors_path::InitContext::::new( linker, - |state| &mut state.#factor_names, + |state| #factors_path::GetFactorState::get::<#factor_types>(state).unwrap(), ) )?; )* @@ -129,11 +130,16 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { ); )* Ok(#state_name { - #( - #factor_names: #FactorInstanceBuilder::build( - builders.#factor_names.unwrap(), - )?, - )* + factors: vec![ + #( + ( + #TypeId::of::<#factor_types>(), + #Box::new( + #FactorInstanceBuilder::build(builders.#factor_names.unwrap())? + ) as #Box + ), + )* + ].into_iter().collect() }) } @@ -170,24 +176,6 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { )* None } - - fn instance_state_offset() -> Option< - #field_offset::FieldOffset< - Self::InstanceState, - <::InstanceBuilder as #FactorInstanceBuilder>::InstanceState, - > - > { - let type_id = #TypeId::of::(); - #( - if type_id == #TypeId::of::<#factor_types>() { - let offset = #field_offset::offset_of!(Self::InstanceState => #factor_names); - return Some( - unsafe { ::std::mem::transmute(offset) } - ); - } - )* - None - } } #vis struct #app_state_name { @@ -203,9 +191,13 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } #vis struct #state_name { - #( - pub #factor_names: <<#factor_types as #Factor>::InstanceBuilder as #FactorInstanceBuilder>::InstanceState, - )* + factors: ::std::collections::HashMap<#TypeId, #Box>, + } + + impl #factors_path::GetFactorState for #state_name { + fn get(&mut self) -> ::std::option::Option<&mut #factors_path::FactorInstanceState> { + self.factors.get_mut(&#TypeId::of::())?.downcast_mut() + } } }) } diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 7a2e5f1d7..d99884e21 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -6,7 +6,6 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" -field-offset = "0.3.6" serde = "1.0" spin-app = { path = "../app" } spin-factors-derive = { path = "../factors-derive" } diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index a3d074795..509e55c61 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -56,7 +56,8 @@ pub trait Factor: Any + Sized { } } -pub(crate) type FactorInstanceState = +/// The instance state of the given [`Factor`] `F`. +pub type FactorInstanceState = <::InstanceBuilder as FactorInstanceBuilder>::InstanceState; pub(crate) type GetDataFn = @@ -89,8 +90,7 @@ impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { &mut Linker, fn(&mut T::InstanceState) -> &mut FactorInstanceState, ) -> anyhow::Result<()>, - ) -> anyhow::Result<()> -where { + ) -> anyhow::Result<()> { add_to_linker(self.linker, self.get_data) } } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index bfdbf0f54..660f7d82f 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -9,10 +9,10 @@ pub use wasmtime; pub use spin_factors_derive::RuntimeFactors; pub use crate::{ - factor::{ConfigureAppContext, ConfiguredApp, Factor, InitContext}, + factor::{ConfigureAppContext, ConfiguredApp, Factor, FactorInstanceState, InitContext}, prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext, SelfInstanceBuilder}, runtime_config::{FactorRuntimeConfig, RuntimeConfigSource}, - runtime_factors::RuntimeFactors, + runtime_factors::{GetFactorState, RuntimeFactors}, }; pub type Linker = wasmtime::component::Linker<::InstanceState>; @@ -27,5 +27,4 @@ pub type Result = wasmtime::Result; #[doc(hidden)] pub mod __internal { pub use crate::runtime_config::RuntimeConfigTracker; - pub use field_offset; } diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 73f746039..05f46ff14 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,101 +1,21 @@ -use field_offset::FieldOffset; - use crate::{factor::FactorInstanceState, Factor}; /// Implemented by `#[derive(RuntimeFactors)]` pub trait RuntimeFactors: Sized + 'static { type AppState; type InstanceBuilders; - type InstanceState: Send + 'static; + type InstanceState: GetFactorState + Send + 'static; fn app_state(app_state: &Self::AppState) -> Option<&F::AppState>; fn instance_builder_mut( builders: &mut Self::InstanceBuilders, ) -> Option>; - - #[doc(hidden)] - fn instance_state_offset( - ) -> Option>>; - - fn instance_state_getter() -> Option> { - StateGetter::new() - } - - fn instance_state_getter2() -> Option> { - StateGetter2::new() - } -} - -pub struct StateGetter { - offset: FieldOffset>, -} - -impl StateGetter { - fn new() -> Option { - Some(Self { - offset: T::instance_state_offset::()?, - }) - } - - pub fn get_state<'a>( - &self, - instance_state: &'a mut T::InstanceState, - ) -> &'a mut FactorInstanceState { - self.offset.apply_mut(instance_state) - } -} - -impl Clone for StateGetter { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for StateGetter {} - -pub struct StateGetter2 { - // Invariant: offsets must point at non-overlapping objects - offset1: FieldOffset>, - offset2: FieldOffset>, -} - -impl StateGetter2 { - fn new() -> Option { - let offset1 = T::instance_state_offset::()?; - let offset2 = T::instance_state_offset::()?; - // Make sure the two states don't point to the same field - if offset1.get_byte_offset() == offset2.get_byte_offset() { - return None; - } - Some(StateGetter2 { offset1, offset2 }) - } - - pub fn get_states<'a>( - &self, - instance_state: &'a mut T::InstanceState, - ) -> ( - &'a mut FactorInstanceState, - &'a mut FactorInstanceState, - ) { - let ptr = instance_state as *mut T::InstanceState; - unsafe { - ( - &mut *(self.offset1.apply_ptr_mut(ptr) as *mut FactorInstanceState), - &mut *(self.offset2.apply_ptr_mut(ptr) as *mut FactorInstanceState), - ) - } - } } -impl Clone for StateGetter2 { - fn clone(&self) -> Self { - *self - } +/// Get the state of a particular Factor from the overall InstanceState +/// +/// Implemented by `#[derive(RuntimeFactors)]` +pub trait GetFactorState { + fn get(&mut self) -> Option<&mut FactorInstanceState>; } - -impl Copy for StateGetter2 {} - -// TODO: This seems fine, but then again I don't understand why `FieldOffset`'s -// own `Sync`ness depends on `U`... -unsafe impl Sync for StateGetter2 {} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 59738b4cf..1d05546d1 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -9,7 +9,7 @@ use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; -use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; +use spin_factors::{FactorRuntimeConfig, GetFactorState, RuntimeConfigSource, RuntimeFactors}; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; use wasmtime_wasi_http::WasiHttpView; @@ -50,10 +50,11 @@ async fn smoke_test_works() -> anyhow::Result<()> { factors.init(&mut linker).unwrap(); let configured_app = factors.configure_app(app, TestSource)?; - let data = factors.build_store_data(&configured_app, "smoke-app")?; + let mut data = factors.build_store_data(&configured_app, "smoke-app")?; + let variables = data.get::().unwrap(); assert_eq!( - data.variables + variables .resolver() .resolve("smoke-app", "other".try_into().unwrap()) .await @@ -81,7 +82,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { // Invoke handler let req = http::Request::get("/").body(Default::default()).unwrap(); let mut wasi_http_view = - spin_factor_outbound_http::get_wasi_http_view::(store.data_mut())?; + spin_factor_outbound_http::get_wasi_http_view::(store.data_mut()); let request = wasi_http_view.new_incoming_request(req)?; let (response_tx, response_rx) = tokio::sync::oneshot::channel(); let response = wasi_http_view.new_response_outparam(response_tx)?; From 7c0e58da41befb236c699c917d9f3790c857605d Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 8 Jul 2024 12:01:57 -0400 Subject: [PATCH 025/195] factors: Make instance state fields again Signed-off-by: Lann Martin --- crates/factors-derive/src/lib.rs | 28 +++++++++++++--------------- crates/factors/tests/smoke.rs | 7 +++---- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index fba2f8b05..7c4c62e54 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -54,7 +54,6 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } let Any = quote!(::std::any::Any); - let Box = quote!(::std::boxed::Box); let Send = quote!(::std::marker::Send); let TypeId = quote!(::std::any::TypeId); let factors_crate = format_ident!("spin_factors"); @@ -78,7 +77,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &mut self.#factor_names, #factors_path::InitContext::::new( linker, - |state| #factors_path::GetFactorState::get::<#factor_types>(state).unwrap(), + |state| &mut state.#factor_names, ) )?; )* @@ -130,19 +129,11 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { ); )* Ok(#state_name { - factors: vec![ - #( - ( - #TypeId::of::<#factor_types>(), - #Box::new( - #FactorInstanceBuilder::build(builders.#factor_names.unwrap())? - ) as #Box - ), - )* - ].into_iter().collect() + #( + #factor_names: #FactorInstanceBuilder::build(builders.#factor_names.unwrap())?, + )* }) } - } impl #factors_path::RuntimeFactors for #name { @@ -191,12 +182,19 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } #vis struct #state_name { - factors: ::std::collections::HashMap<#TypeId, #Box>, + #( + pub #factor_names: #factors_path::FactorInstanceState<#factor_types>, + )* } impl #factors_path::GetFactorState for #state_name { fn get(&mut self) -> ::std::option::Option<&mut #factors_path::FactorInstanceState> { - self.factors.get_mut(&#TypeId::of::())?.downcast_mut() + #( + if let Some(state) = (&mut self.#factor_names as &mut (dyn #Any + #Send)).downcast_mut() { + return Some(state) + } + )* + None } } }) diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 1d05546d1..b46da230c 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -9,7 +9,7 @@ use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; -use spin_factors::{FactorRuntimeConfig, GetFactorState, RuntimeConfigSource, RuntimeFactors}; +use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; use wasmtime_wasi_http::WasiHttpView; @@ -50,11 +50,10 @@ async fn smoke_test_works() -> anyhow::Result<()> { factors.init(&mut linker).unwrap(); let configured_app = factors.configure_app(app, TestSource)?; - let mut data = factors.build_store_data(&configured_app, "smoke-app")?; + let data = factors.build_store_data(&configured_app, "smoke-app")?; - let variables = data.get::().unwrap(); assert_eq!( - variables + data.variables .resolver() .resolve("smoke-app", "other".try_into().unwrap()) .await From 5d9c9f8c9a9ab519a2ebc17132654845de175042 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 9 Jul 2024 16:24:24 -0400 Subject: [PATCH 026/195] factors: Add spin-factors-test crate This provides a TestEnvironment type to aid factor authors in writing tests. In order to enable TestEnvironment, move all of the generated RuntimeFactors methcods into the trait. Exercise TestEnvironment in a spin-factor-variables test. Signed-off-by: Lann Martin --- Cargo.lock | 13 +++++ crates/factor-variables/Cargo.toml | 4 ++ crates/factor-variables/tests/factor.rs | 44 ++++++++++++++++ crates/factors-derive/src/lib.rs | 21 ++++---- crates/factors-test/Cargo.toml | 15 ++++++ crates/factors-test/src/lib.rs | 69 +++++++++++++++++++++++++ crates/factors/src/factor.rs | 2 +- crates/factors/src/lib.rs | 1 + crates/factors/src/runtime_factors.rs | 16 +++++- 9 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 crates/factor-variables/tests/factor.rs create mode 100644 crates/factors-test/Cargo.toml create mode 100644 crates/factors-test/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6a4e3afef..337b4aab5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7562,7 +7562,9 @@ dependencies = [ "serde 1.0.197", "spin-expressions", "spin-factors", + "spin-factors-test", "spin-world", + "tokio", "toml 0.8.14", ] @@ -7612,6 +7614,17 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "spin-factors-test" +version = "2.7.0-pre0" +dependencies = [ + "spin-app", + "spin-factors", + "spin-loader", + "tempfile", + "toml 0.8.14", +] + [[package]] name = "spin-http" version = "2.7.0-pre0" diff --git a/crates/factor-variables/Cargo.toml b/crates/factor-variables/Cargo.toml index 5e5cf5543..30465c209 100644 --- a/crates/factor-variables/Cargo.toml +++ b/crates/factor-variables/Cargo.toml @@ -11,5 +11,9 @@ spin-factors = { path = "../factors" } spin-world = { path = "../world" } toml = "0.8" +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + [lints] workspace = true diff --git a/crates/factor-variables/tests/factor.rs b/crates/factor-variables/tests/factor.rs new file mode 100644 index 000000000..5f57e76eb --- /dev/null +++ b/crates/factor-variables/tests/factor.rs @@ -0,0 +1,44 @@ +use spin_factor_variables::{StaticVariables, VariablesFactor}; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; + +#[derive(RuntimeFactors)] +struct TestFactors { + variables: VariablesFactor, +} + +#[tokio::test] +async fn static_provider_works() -> anyhow::Result<()> { + let mut factors = TestFactors { + variables: VariablesFactor::default(), + }; + factors.variables.add_provider_type(StaticVariables)?; + + let mut env = TestEnvironment { + manifest: toml! { + spin_manifest_version = 2 + application.name = "test-app" + [[trigger.test]] + + [variables] + foo = { required = true } + + [component.test-component] + source = "does-not-exist.wasm" + variables = { baz = "<{{ foo }}>" } + }, + runtime_config: toml! { + [[variable_provider]] + type = "static" + values = { foo = "bar" } + }, + }; + let state = env.build_instance_state(factors).await?; + let val = state + .variables + .resolver() + .resolve("test-component", "baz".try_into().unwrap()) + .await?; + assert_eq!(val, ""); + Ok(()) +} diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 7c4c62e54..b6c92ca7b 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -66,9 +66,13 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let FactorInstanceBuilder = quote!(#factors_path::FactorInstanceBuilder); Ok(quote! { - impl #name { + impl #factors_path::RuntimeFactors for #name { + type AppState = #app_state_name; + type InstanceBuilders = #builders_name; + type InstanceState = #state_name; + #[allow(clippy::needless_option_as_deref)] - pub fn init( + fn init( &mut self, linker: &mut #wasmtime::component::Linker<#state_name>, ) -> #Result<()> { @@ -84,7 +88,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { Ok(()) } - pub fn configure_app( + fn configure_app( &self, app: #factors_path::App, runtime_config: impl #factors_path::RuntimeConfigSource @@ -109,7 +113,10 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { Ok(#ConfiguredApp::new(app, app_state)) } - pub fn build_store_data(&self, configured_app: &#ConfiguredApp, component_id: &str) -> #Result<#state_name> { + fn build_store_data( + &self, configured_app: &#ConfiguredApp, + component_id: &str, + ) -> #Result { let app_component = configured_app.app().get_component(component_id).ok_or_else(|| { #wasmtime::Error::msg("unknown component") })?; @@ -134,12 +141,6 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { )* }) } - } - - impl #factors_path::RuntimeFactors for #name { - type AppState = #app_state_name; - type InstanceBuilders = #builders_name; - type InstanceState = #state_name; fn app_state(app_state: &Self::AppState) -> Option<&F::AppState> { #( diff --git a/crates/factors-test/Cargo.toml b/crates/factors-test/Cargo.toml new file mode 100644 index 000000000..59b9dd1d8 --- /dev/null +++ b/crates/factors-test/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "spin-factors-test" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +spin-app = { path = "../app" } +spin-factors = { path = "../factors" } +spin-loader = { path = "../loader" } +tempfile = "3.10.1" +toml = "0.8.14" + +[lints] +workspace = true diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs new file mode 100644 index 000000000..7f0334e8f --- /dev/null +++ b/crates/factors-test/src/lib.rs @@ -0,0 +1,69 @@ +use spin_app::locked::LockedApp; +use spin_factors::{ + anyhow::{self, Context}, + serde::de::DeserializeOwned, + wasmtime::{Config, Engine}, + App, Linker, RuntimeConfigSource, RuntimeFactors, +}; +use spin_loader::FilesMountStrategy; + +pub use toml::toml; + +#[derive(Default)] +pub struct TestEnvironment { + pub manifest: toml::Table, + pub runtime_config: toml::Table, +} + +impl TestEnvironment { + /// Starting from a new _uninitialized_ [`RuntimeFactors`], run through the + /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`]. + pub async fn build_instance_state( + &mut self, + mut factors: T, + ) -> anyhow::Result { + let mut linker = Self::new_linker::(); + factors.init(&mut linker)?; + + let locked_app = self.build_locked_app().await?; + let app = App::inert(locked_app); + let runtime_config = TomlRuntimeConfig(&self.runtime_config); + let configured_app = factors.configure_app(app, runtime_config)?; + + let component = configured_app + .app() + .components() + .next() + .context("no components")?; + factors.build_store_data(&configured_app, component.id()) + } + + pub fn new_linker() -> Linker { + let engine = Engine::new(Config::new().async_support(true)).expect("engine"); + Linker::::new(&engine) + } + + pub async fn build_locked_app(&self) -> anyhow::Result { + let toml_str = toml::to_string(&self.manifest).context("failed serializing manifest")?; + let dir = tempfile::tempdir().context("failed creating tempdir")?; + let path = dir.path().join("spin.toml"); + std::fs::write(&path, toml_str).context("failed writing manifest")?; + spin_loader::from_file(&path, FilesMountStrategy::Direct, None).await + } +} + +pub struct TomlRuntimeConfig<'a>(&'a toml::Table); + +impl RuntimeConfigSource for TomlRuntimeConfig<'_> { + fn factor_config_keys(&self) -> impl IntoIterator { + self.0.keys().map(|key| key.as_str()) + } + + fn get_factor_config(&self, key: &str) -> anyhow::Result> { + let Some(val) = self.0.get(key) else { + return Ok(None); + }; + let config = val.clone().try_into()?; + Ok(Some(config)) + } +} diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 509e55c61..0d81f2ebe 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -15,7 +15,7 @@ pub trait Factor: Any + Sized { type InstanceBuilder: FactorInstanceBuilder; /// Initializes this Factor for a runtime. This will be called at most once, - /// before any call to [`FactorInstancePreparer::new`] + /// before any call to [`FactorInstanceBuilder::new`] fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { // TODO: Should `ctx` always be immut? Rename this param/type? _ = &mut ctx; diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 660f7d82f..5d568e6aa 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -4,6 +4,7 @@ mod runtime_config; mod runtime_factors; pub use anyhow; +pub use serde; pub use wasmtime; pub use spin_factors_derive::RuntimeFactors; diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 05f46ff14..9222d183c 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,4 +1,4 @@ -use crate::{factor::FactorInstanceState, Factor}; +use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, Linker, RuntimeConfigSource}; /// Implemented by `#[derive(RuntimeFactors)]` pub trait RuntimeFactors: Sized + 'static { @@ -6,6 +6,20 @@ pub trait RuntimeFactors: Sized + 'static { type InstanceBuilders; type InstanceState: GetFactorState + Send + 'static; + fn init(&mut self, linker: &mut Linker) -> anyhow::Result<()>; + + fn configure_app( + &self, + app: App, + runtime_config: impl RuntimeConfigSource, + ) -> anyhow::Result>; + + fn build_store_data( + &self, + configured_app: &ConfiguredApp, + component_id: &str, + ) -> anyhow::Result; + fn app_state(app_state: &Self::AppState) -> Option<&F::AppState>; fn instance_builder_mut( From f7c8914720a776e3b60451f09909a823d5f8be52 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 10 Jul 2024 17:21:37 -0400 Subject: [PATCH 027/195] factors: Fix derive macro in rust-analyzer Rust analyzer doesn't like `include!`s that reach outside of a crate's own OUT_DIR: https://github.com/rust-lang/rust-analyzer/issues/17040 Fix this by requiring any crate that wants expanded output (in tests) to have a build script that sets a magic env var. Signed-off-by: Lann Martin --- Cargo.lock | 30 +++++++++++-------- crates/factor-variables/build.rs | 6 ++++ .../tests/{factor.rs => factor_test.rs} | 0 crates/factors-derive/Cargo.toml | 2 +- crates/factors-derive/src/lib.rs | 10 +++++-- crates/factors-test/Cargo.toml | 1 + 6 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 crates/factor-variables/build.rs rename crates/factor-variables/tests/{factor.rs => factor_test.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index 337b4aab5..97f108940 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2409,13 +2409,14 @@ dependencies = [ [[package]] name = "expander" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e83c02035136f1592a47964ea60c05a50e4ed8b5892cfac197063850898d4d" +checksum = "e2c470c71d91ecbd179935b24170459e926382eaaa86b590b78814e180d8a8e2" dependencies = [ "blake2", + "file-guard", "fs-err", - "prettier-please", + "prettyplease", "proc-macro2", "quote", "syn 2.0.58", @@ -2486,6 +2487,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "file-guard" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ef72acf95ec3d7dbf61275be556299490a245f017cf084bd23b4f68cf9407c" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "filetime" version = "0.2.23" @@ -5819,16 +5830,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "prettier-please" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22020dfcf177fcc7bf5deaf7440af371400c67c0de14c399938d8ed4fb4645d3" -dependencies = [ - "proc-macro2", - "syn 2.0.58", -] - [[package]] name = "prettyplease" version = "0.2.17" @@ -7574,6 +7575,8 @@ version = "2.7.0-pre0" dependencies = [ "cap-primitives 3.0.0", "spin-factors", + "spin-factors-test", + "tokio", "wasmtime-wasi", ] @@ -7620,6 +7623,7 @@ version = "2.7.0-pre0" dependencies = [ "spin-app", "spin-factors", + "spin-factors-derive", "spin-loader", "tempfile", "toml 0.8.14", diff --git a/crates/factor-variables/build.rs b/crates/factor-variables/build.rs new file mode 100644 index 000000000..bba8ce3c9 --- /dev/null +++ b/crates/factor-variables/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + // Enable spin-factors-derive to emit expanded macro output. + let out_dir = std::env::var("OUT_DIR").unwrap(); + println!("cargo:rustc-env=SPIN_FACTORS_DERIVE_EXPAND_DIR={out_dir}"); +} diff --git a/crates/factor-variables/tests/factor.rs b/crates/factor-variables/tests/factor_test.rs similarity index 100% rename from crates/factor-variables/tests/factor.rs rename to crates/factor-variables/tests/factor_test.rs diff --git a/crates/factors-derive/Cargo.toml b/crates/factors-derive/Cargo.toml index 859a3c329..387276149 100644 --- a/crates/factors-derive/Cargo.toml +++ b/crates/factors-derive/Cargo.toml @@ -11,7 +11,7 @@ proc-macro = true expander = ["dep:expander"] [dependencies] -expander = { version = "2.1.0", optional = true } +expander = { version = "2.2.1", optional = true } proc-macro2 = "1.0.79" quote = "1.0.35" syn = "2.0.52" diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index b6c92ca7b..a9b24a03e 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -8,9 +8,13 @@ pub fn derive_factors(input: proc_macro::TokenStream) -> proc_macro::TokenStream let expanded = expand_factors(&input).unwrap_or_else(|err| err.into_compile_error()); #[cfg(feature = "expander")] - let expanded = expander::Expander::new("factors") - .write_to_out_dir(expanded) - .unwrap(); + let expanded = if let Some(dest_dir) = std::env::var_os("SPIN_FACTORS_DERIVE_EXPAND_DIR") { + expander::Expander::new("factors") + .write_to(expanded, std::path::Path::new(&dest_dir)) + .unwrap() + } else { + expanded + }; expanded.into() } diff --git a/crates/factors-test/Cargo.toml b/crates/factors-test/Cargo.toml index 59b9dd1d8..23e606bde 100644 --- a/crates/factors-test/Cargo.toml +++ b/crates/factors-test/Cargo.toml @@ -7,6 +7,7 @@ edition = { workspace = true } [dependencies] spin-app = { path = "../app" } spin-factors = { path = "../factors" } +spin-factors-derive = { path = "../factors-derive", features = ["expander"] } spin-loader = { path = "../loader" } tempfile = "3.10.1" toml = "0.8.14" From 2ab09d0853d03a7a7c5e9a462009bf73155ae670 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 10 Jul 2024 17:24:52 -0400 Subject: [PATCH 028/195] factors: Extend and add a test to spin-factor-wasi Signed-off-by: Lann Martin --- crates/factor-wasi/Cargo.toml | 5 ++ crates/factor-wasi/build.rs | 6 ++ crates/factor-wasi/src/lib.rs | 103 ++++++++++++++++++++---- crates/factor-wasi/tests/factor_test.rs | 37 +++++++++ 4 files changed, 136 insertions(+), 15 deletions(-) create mode 100644 crates/factor-wasi/build.rs create mode 100644 crates/factor-wasi/tests/factor_test.rs diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index d1572e555..4edc9b1c3 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -7,7 +7,12 @@ edition = { workspace = true } [dependencies] cap-primitives = "3.0.0" spin-factors = { path = "../factors" } +tokio = { version = "1" } wasmtime-wasi = { workspace = true } +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + [lints] workspace = true diff --git a/crates/factor-wasi/build.rs b/crates/factor-wasi/build.rs new file mode 100644 index 000000000..bba8ce3c9 --- /dev/null +++ b/crates/factor-wasi/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + // Enable spin-factors-derive to emit expanded macro output. + let out_dir = std::env::var("OUT_DIR").unwrap(); + println!("cargo:rustc-env=SPIN_FACTORS_DERIVE_EXPAND_DIR={out_dir}"); +} diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 9d80131b6..588c22e7d 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -4,7 +4,12 @@ use spin_factors::{ anyhow, AppComponent, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, }; -use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiImpl, WasiView}; +use tokio::io::{AsyncRead, AsyncWrite}; +use wasmtime_wasi::{ + pipe::{AsyncReadStream, AsyncWriteStream}, + AsyncStdinStream, AsyncStdoutStream, DirPerms, FilePerms, ResourceTable, StdinStream, + StdoutStream, WasiCtx, WasiCtxBuilder, WasiImpl, WasiView, +}; pub struct WasiFactor { files_mounter: Box, @@ -81,19 +86,17 @@ impl Factor for WasiFactor { ) -> anyhow::Result { let mut wasi_ctx = WasiCtxBuilder::new(); - // Apply environment variables - for (key, val) in ctx.app_component().environment() { - wasi_ctx.env(key, val); - } - // Mount files - let mount_ctx = MountFilesContext { - wasi_ctx: &mut wasi_ctx, - }; + let mount_ctx = MountFilesContext { ctx: &mut wasi_ctx }; self.files_mounter .mount_files(ctx.app_component(), mount_ctx)?; - Ok(InstanceBuilder { wasi_ctx }) + let mut builder = InstanceBuilder { ctx: wasi_ctx }; + + // Apply environment variables + builder.env(ctx.app_component().environment()); + + Ok(builder) } } @@ -122,7 +125,7 @@ impl FilesMounter for DummyFilesMounter { } pub struct MountFilesContext<'a> { - wasi_ctx: &'a mut WasiCtxBuilder, + ctx: &'a mut WasiCtxBuilder, } impl<'a> MountFilesContext<'a> { @@ -138,21 +141,91 @@ impl<'a> MountFilesContext<'a> { } else { (DirPerms::READ, FilePerms::READ) }; - self.wasi_ctx + self.ctx .preopened_dir(host_path, guest_path, dir_perms, file_perms)?; Ok(()) } } pub struct InstanceBuilder { - wasi_ctx: WasiCtxBuilder, + ctx: WasiCtxBuilder, +} + +impl InstanceBuilder { + /// Sets the WASI `stdin` descriptor to the given [`StdinStream`]. + pub fn stdin(&mut self, stdin: impl StdinStream + 'static) { + self.ctx.stdin(stdin); + } + + /// Sets the WASI `stdin` descriptor to the given [`AsyncRead`]er. + pub fn stdin_pipe(&mut self, r: impl AsyncRead + Send + Unpin + 'static) { + self.stdin(AsyncStdinStream::new(AsyncReadStream::new(r))); + } + + /// Sets the WASI `stdout` descriptor to the given [`StdoutStream`]. + pub fn stdout(&mut self, stdout: impl StdoutStream + 'static) { + self.ctx.stdout(stdout); + } + + /// Sets the WASI `stdout` descriptor to the given [`AsyncWrite`]r. + pub fn stdout_pipe(&mut self, w: impl AsyncWrite + Send + Unpin + 'static) { + self.stdout(AsyncStdoutStream::new(AsyncWriteStream::new( + 1024 * 1024, + w, + ))); + } + + /// Sets the WASI `stderr` descriptor to the given [`StdoutStream`]. + pub fn stderr(&mut self, stderr: impl StdoutStream + 'static) { + self.ctx.stderr(stderr); + } + + /// Sets the WASI `stderr` descriptor to the given [`AsyncWrite`]r. + pub fn stderr_pipe(&mut self, w: impl AsyncWrite + Send + Unpin + 'static) { + self.stderr(AsyncStdoutStream::new(AsyncWriteStream::new( + 1024 * 1024, + w, + ))); + } + + /// Appends the given strings to the WASI 'args'. + pub fn args(&mut self, args: impl IntoIterator>) { + for arg in args { + self.ctx.arg(arg); + } + } + + /// Sets the given key/value string entries on the WASI 'env'. + pub fn env(&mut self, vars: impl IntoIterator, impl AsRef)>) { + for (k, v) in vars { + self.ctx.env(k, v); + } + } + + /// "Mounts" the given `host_path` into the WASI filesystem at the given + /// `guest_path`. + pub fn preopened_dir( + &mut self, + host_path: impl AsRef, + guest_path: impl AsRef, + writable: bool, + ) -> anyhow::Result<()> { + let (dir_perms, file_perms) = if writable { + (DirPerms::all(), FilePerms::all()) + } else { + (DirPerms::READ, FilePerms::READ) + }; + self.ctx + .preopened_dir(host_path, guest_path, dir_perms, file_perms)?; + Ok(()) + } } impl FactorInstanceBuilder for InstanceBuilder { type InstanceState = InstanceState; fn build(self) -> anyhow::Result { - let InstanceBuilder { mut wasi_ctx } = self; + let InstanceBuilder { ctx: mut wasi_ctx } = self; Ok(InstanceState { ctx: wasi_ctx.build(), table: Default::default(), @@ -166,7 +239,7 @@ impl InstanceBuilder { F: Fn(SocketAddr) -> Fut + Send + Sync + Clone + 'static, Fut: Future + Send + Sync, { - self.wasi_ctx.socket_addr_check(move |addr, addr_use| { + self.ctx.socket_addr_check(move |addr, addr_use| { let check = check.clone(); Box::pin(async move { match addr_use { diff --git a/crates/factor-wasi/tests/factor_test.rs b/crates/factor-wasi/tests/factor_test.rs new file mode 100644 index 000000000..d0e3aa7d8 --- /dev/null +++ b/crates/factor-wasi/tests/factor_test.rs @@ -0,0 +1,37 @@ +use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use wasmtime_wasi::{bindings::cli::environment::Host, WasiImpl}; + +#[derive(RuntimeFactors)] +struct TestFactors { + wasi: WasiFactor, +} + +#[tokio::test] +async fn environment_works() -> anyhow::Result<()> { + let factors = TestFactors { + wasi: WasiFactor::new(DummyFilesMounter), + }; + + let mut env = TestEnvironment { + manifest: toml! { + spin_manifest_version = 2 + application.name = "test-app" + [[trigger.test]] + + [component.test-component] + source = "does-not-exist.wasm" + environment = { FOO = "bar" } + }, + ..Default::default() + }; + let mut state = env.build_instance_state(factors).await?; + let mut wasi = WasiImpl(&mut state.wasi); + let val = wasi + .get_environment()? + .into_iter() + .find_map(|(key, val)| (key == "FOO").then_some(val)); + assert_eq!(val.as_deref(), Some("bar")); + Ok(()) +} From 889f19a0822076cb9178f9596778d794f916a2e3 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 11 Jul 2024 10:11:52 -0400 Subject: [PATCH 029/195] factors: Add TestEnvironment::default_manifest_extend Avoids some boilerplate in factor test setup. Use this to establish a `test_env` pattern for factor tests. Signed-off-by: Lann Martin --- crates/factor-variables/tests/factor_test.rs | 37 ++++++++++---------- crates/factor-wasi/tests/factor_test.rs | 21 +++++------ crates/factors-test/src/lib.rs | 35 ++++++++++++++++-- 3 files changed, 59 insertions(+), 34 deletions(-) diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index 5f57e76eb..e5e11ed16 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -7,6 +7,23 @@ struct TestFactors { variables: VariablesFactor, } +fn test_env() -> TestEnvironment { + let mut env = TestEnvironment::default_manifest_extend(toml! { + [variables] + foo = { required = true } + + [component.test-component] + source = "does-not-exist.wasm" + variables = { baz = "<{{ foo }}>" } + }); + env.runtime_config = toml! { + [[variable_provider]] + type = "static" + values = { foo = "bar" } + }; + env +} + #[tokio::test] async fn static_provider_works() -> anyhow::Result<()> { let mut factors = TestFactors { @@ -14,25 +31,7 @@ async fn static_provider_works() -> anyhow::Result<()> { }; factors.variables.add_provider_type(StaticVariables)?; - let mut env = TestEnvironment { - manifest: toml! { - spin_manifest_version = 2 - application.name = "test-app" - [[trigger.test]] - - [variables] - foo = { required = true } - - [component.test-component] - source = "does-not-exist.wasm" - variables = { baz = "<{{ foo }}>" } - }, - runtime_config: toml! { - [[variable_provider]] - type = "static" - values = { foo = "bar" } - }, - }; + let env = test_env(); let state = env.build_instance_state(factors).await?; let val = state .variables diff --git a/crates/factor-wasi/tests/factor_test.rs b/crates/factor-wasi/tests/factor_test.rs index d0e3aa7d8..3b9632152 100644 --- a/crates/factor-wasi/tests/factor_test.rs +++ b/crates/factor-wasi/tests/factor_test.rs @@ -8,24 +8,21 @@ struct TestFactors { wasi: WasiFactor, } +fn test_env() -> TestEnvironment { + TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + environment = { FOO = "bar" } + }) +} + #[tokio::test] async fn environment_works() -> anyhow::Result<()> { let factors = TestFactors { wasi: WasiFactor::new(DummyFilesMounter), }; - let mut env = TestEnvironment { - manifest: toml! { - spin_manifest_version = 2 - application.name = "test-app" - [[trigger.test]] - - [component.test-component] - source = "does-not-exist.wasm" - environment = { FOO = "bar" } - }, - ..Default::default() - }; + let env = test_env(); let mut state = env.build_instance_state(factors).await?; let mut wasi = WasiImpl(&mut state.wasi); let val = wasi diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 7f0334e8f..a32d76dc9 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -9,17 +9,46 @@ use spin_loader::FilesMountStrategy; pub use toml::toml; -#[derive(Default)] pub struct TestEnvironment { pub manifest: toml::Table, pub runtime_config: toml::Table, } +impl Default for TestEnvironment { + fn default() -> Self { + let manifest = toml! { + spin_manifest_version = 2 + + [application] + name = "test-app" + + [[trigger.test-trigger]] + }; + Self { + manifest, + runtime_config: Default::default(), + } + } +} + impl TestEnvironment { + /// Builds a TestEnvironment by extending a default manifest with the given + /// manifest TOML. + /// + /// The default manifest includes boilerplate like the + /// `spin_manifest_version` and `[application]` section, so you typically + /// need to pass only a `[component.test-component]` section. + pub fn default_manifest_extend(manifest_merge: toml::Table) -> Self { + let mut env = Self::default(); + env.manifest.extend(manifest_merge); + env + } + /// Starting from a new _uninitialized_ [`RuntimeFactors`], run through the - /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`]. + /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`] + /// for the first component defined in the manifest. pub async fn build_instance_state( - &mut self, + &self, mut factors: T, ) -> anyhow::Result { let mut linker = Self::new_linker::(); From c02338c934610412935af1eb2f6861263c86ec89 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 11 Jul 2024 12:35:43 -0400 Subject: [PATCH 030/195] factors: Port WASI RCs to WasiFactor Signed-off-by: Lann Martin --- Cargo.lock | 2 + crates/factor-wasi/Cargo.toml | 2 + crates/factor-wasi/src/lib.rs | 7 + crates/factor-wasi/src/wasi_2023_10_18.rs | 1918 +++++++++++++++++++++ crates/factor-wasi/src/wasi_2023_11_10.rs | 1722 ++++++++++++++++++ crates/factor-wasi/tests/factor_test.rs | 2 +- 6 files changed, 3652 insertions(+), 1 deletion(-) create mode 100644 crates/factor-wasi/src/wasi_2023_10_18.rs create mode 100644 crates/factor-wasi/src/wasi_2023_11_10.rs diff --git a/Cargo.lock b/Cargo.lock index 97f108940..8dda2cc27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7573,10 +7573,12 @@ dependencies = [ name = "spin-factor-wasi" version = "2.7.0-pre0" dependencies = [ + "async-trait", "cap-primitives 3.0.0", "spin-factors", "spin-factors-test", "tokio", + "wasmtime", "wasmtime-wasi", ] diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index 4edc9b1c3..6fb2dfc8e 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -5,9 +5,11 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] +async-trait = "0.1" cap-primitives = "3.0.0" spin-factors = { path = "../factors" } tokio = { version = "1" } +wasmtime = { workspace = true } wasmtime-wasi = { workspace = true } [dev-dependencies] diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 588c22e7d..d35c01f33 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,3 +1,6 @@ +pub mod wasi_2023_10_18; +pub mod wasi_2023_11_10; + use std::{future::Future, net::SocketAddr, path::Path}; use spin_factors::{ @@ -69,6 +72,10 @@ impl Factor for WasiFactor { bindings::sockets::instance_network::add_to_linker_get_host(linker, closure)?; bindings::sockets::network::add_to_linker_get_host(linker, closure)?; bindings::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; + + wasi_2023_10_18::add_to_linker(linker, closure)?; + wasi_2023_11_10::add_to_linker(linker, closure)?; + Ok(()) } diff --git a/crates/factor-wasi/src/wasi_2023_10_18.rs b/crates/factor-wasi/src/wasi_2023_10_18.rs new file mode 100644 index 000000000..5c7d01ad9 --- /dev/null +++ b/crates/factor-wasi/src/wasi_2023_10_18.rs @@ -0,0 +1,1918 @@ +#![doc(hidden)] // internal implementation detail used in tests and spin-trigger + +use async_trait::async_trait; +use spin_factors::anyhow::{self, Result}; +use std::mem; +use wasmtime::component::{Linker, Resource}; +use wasmtime_wasi::{Pollable, TrappableError, WasiImpl, WasiView}; + +use crate::InstanceState; + +mod latest { + pub use wasmtime_wasi::bindings::*; +} + +mod bindings { + use super::latest; + pub use super::UdpSocket; + + wasmtime::component::bindgen!({ + path: "../../wit", + interfaces: r#" + // NB: this is handling the historical behavior where Spin supported + // more than "just" this snaphsot of the proxy world but additionally + // other CLI-related interfaces. + include wasi:cli/reactor@0.2.0-rc-2023-10-18; + "#, + async: { + only_imports: [ + "[method]descriptor.access-at", + "[method]descriptor.advise", + "[method]descriptor.change-directory-permissions-at", + "[method]descriptor.change-file-permissions-at", + "[method]descriptor.create-directory-at", + "[method]descriptor.get-flags", + "[method]descriptor.get-type", + "[method]descriptor.is-same-object", + "[method]descriptor.link-at", + "[method]descriptor.lock-exclusive", + "[method]descriptor.lock-shared", + "[method]descriptor.metadata-hash", + "[method]descriptor.metadata-hash-at", + "[method]descriptor.open-at", + "[method]descriptor.read", + "[method]descriptor.read-directory", + "[method]descriptor.readlink-at", + "[method]descriptor.remove-directory-at", + "[method]descriptor.rename-at", + "[method]descriptor.set-size", + "[method]descriptor.set-times", + "[method]descriptor.set-times-at", + "[method]descriptor.stat", + "[method]descriptor.stat-at", + "[method]descriptor.symlink-at", + "[method]descriptor.sync", + "[method]descriptor.sync-data", + "[method]descriptor.try-lock-exclusive", + "[method]descriptor.try-lock-shared", + "[method]descriptor.unlink-file-at", + "[method]descriptor.unlock", + "[method]descriptor.write", + "[method]input-stream.read", + "[method]input-stream.blocking-read", + "[method]input-stream.blocking-skip", + "[method]input-stream.skip", + "[method]output-stream.forward", + "[method]output-stream.splice", + "[method]output-stream.blocking-splice", + "[method]output-stream.blocking-flush", + "[method]output-stream.blocking-write", + "[method]output-stream.blocking-write-and-flush", + "[method]output-stream.blocking-write-zeroes-and-flush", + "[method]directory-entry-stream.read-directory-entry", + "poll-list", + "poll-one", + + "[method]tcp-socket.start-bind", + "[method]tcp-socket.start-connect", + "[method]udp-socket.finish-connect", + "[method]udp-socket.receive", + "[method]udp-socket.send", + "[method]udp-socket.start-bind", + "[method]udp-socket.stream", + "[method]outgoing-datagram-stream.send", + ], + }, + with: { + "wasi:io/poll/pollable": latest::io::poll::Pollable, + "wasi:io/streams/input-stream": latest::io::streams::InputStream, + "wasi:io/streams/output-stream": latest::io::streams::OutputStream, + "wasi:io/streams/error": latest::io::streams::Error, + "wasi:filesystem/types/directory-entry-stream": latest::filesystem::types::DirectoryEntryStream, + "wasi:filesystem/types/descriptor": latest::filesystem::types::Descriptor, + "wasi:cli/terminal-input/terminal-input": latest::cli::terminal_input::TerminalInput, + "wasi:cli/terminal-output/terminal-output": latest::cli::terminal_output::TerminalOutput, + "wasi:sockets/tcp/tcp-socket": latest::sockets::tcp::TcpSocket, + "wasi:sockets/udp/udp-socket": UdpSocket, + "wasi:sockets/network/network": latest::sockets::network::Network, + "wasi:sockets/ip-name-lookup/resolve-address-stream": latest::sockets::ip_name_lookup::ResolveAddressStream, + }, + trappable_imports: true, + }); +} + +mod wasi { + pub use super::bindings::wasi::{ + cli0_2_0_rc_2023_10_18 as cli, clocks0_2_0_rc_2023_10_18 as clocks, + filesystem0_2_0_rc_2023_10_18 as filesystem, io0_2_0_rc_2023_10_18 as io, + random0_2_0_rc_2023_10_18 as random, sockets0_2_0_rc_2023_10_18 as sockets, + }; +} + +use wasi::cli::terminal_input::TerminalInput; +use wasi::cli::terminal_output::TerminalOutput; +use wasi::clocks::monotonic_clock::Instant; +use wasi::clocks::wall_clock::Datetime; +use wasi::filesystem::types::{ + AccessType, Advice, Descriptor, DescriptorFlags, DescriptorStat, DescriptorType, + DirectoryEntry, DirectoryEntryStream, Error, ErrorCode as FsErrorCode, Filesize, + MetadataHashValue, Modes, NewTimestamp, OpenFlags, PathFlags, +}; +use wasi::io::streams::{InputStream, OutputStream, StreamError}; +use wasi::sockets::ip_name_lookup::{IpAddress, ResolveAddressStream}; +use wasi::sockets::network::{Ipv4SocketAddress, Ipv6SocketAddress}; +use wasi::sockets::tcp::{ + ErrorCode as SocketErrorCode, IpAddressFamily, IpSocketAddress, Network, ShutdownType, + TcpSocket, +}; +use wasi::sockets::udp::Datagram; + +pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> +where + T: Send, + F: Fn(&mut T) -> WasiImpl<&mut InstanceState> + Send + Sync + Copy + 'static, +{ + fn type_annotate(f: F) -> F + where + F: Fn(&mut T) -> WasiImpl<&mut U>, + { + f + } + let closure = type_annotate(closure); + wasi::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; + wasi::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; + wasi::filesystem::types::add_to_linker_get_host(linker, closure)?; + wasi::filesystem::preopens::add_to_linker_get_host(linker, closure)?; + wasi::io::poll::add_to_linker_get_host(linker, closure)?; + wasi::io::streams::add_to_linker_get_host(linker, closure)?; + wasi::random::random::add_to_linker_get_host(linker, closure)?; + wasi::random::insecure::add_to_linker_get_host(linker, closure)?; + wasi::random::insecure_seed::add_to_linker_get_host(linker, closure)?; + wasi::cli::exit::add_to_linker_get_host(linker, closure)?; + wasi::cli::environment::add_to_linker_get_host(linker, closure)?; + wasi::cli::stdin::add_to_linker_get_host(linker, closure)?; + wasi::cli::stdout::add_to_linker_get_host(linker, closure)?; + wasi::cli::stderr::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_input::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_output::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_stdin::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_stdout::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_stderr::add_to_linker_get_host(linker, closure)?; + wasi::sockets::tcp::add_to_linker_get_host(linker, closure)?; + wasi::sockets::tcp_create_socket::add_to_linker_get_host(linker, closure)?; + wasi::sockets::udp::add_to_linker_get_host(linker, closure)?; + wasi::sockets::udp_create_socket::add_to_linker_get_host(linker, closure)?; + wasi::sockets::instance_network::add_to_linker_get_host(linker, closure)?; + wasi::sockets::network::add_to_linker_get_host(linker, closure)?; + wasi::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; + Ok(()) +} + +impl wasi::clocks::monotonic_clock::Host for WasiImpl +where + T: WasiView, +{ + fn now(&mut self) -> wasmtime::Result { + latest::clocks::monotonic_clock::Host::now(self) + } + + fn resolution(&mut self) -> wasmtime::Result { + latest::clocks::monotonic_clock::Host::resolution(self) + } + + fn subscribe(&mut self, when: Instant, absolute: bool) -> wasmtime::Result> { + if absolute { + latest::clocks::monotonic_clock::Host::subscribe_instant(self, when) + } else { + latest::clocks::monotonic_clock::Host::subscribe_duration(self, when) + } + } +} + +impl wasi::clocks::wall_clock::Host for WasiImpl +where + T: WasiView, +{ + fn now(&mut self) -> wasmtime::Result { + Ok(latest::clocks::wall_clock::Host::now(self)?.into()) + } + + fn resolution(&mut self) -> wasmtime::Result { + Ok(latest::clocks::wall_clock::Host::resolution(self)?.into()) + } +} + +impl wasi::filesystem::types::Host for WasiImpl +where + T: WasiView, +{ + fn filesystem_error_code( + &mut self, + err: Resource, + ) -> wasmtime::Result> { + Ok(latest::filesystem::types::Host::filesystem_error_code(self, err)?.map(|e| e.into())) + } +} + +#[async_trait] +impl wasi::filesystem::types::HostDescriptor for WasiImpl +where + T: WasiView, +{ + fn read_via_stream( + &mut self, + self_: Resource, + offset: Filesize, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::read_via_stream( + self, self_, offset, + )) + } + + fn write_via_stream( + &mut self, + self_: Resource, + offset: Filesize, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::write_via_stream( + self, self_, offset, + )) + } + + fn append_via_stream( + &mut self, + self_: Resource, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::append_via_stream(self, self_)) + } + + async fn advise( + &mut self, + self_: Resource, + offset: Filesize, + length: Filesize, + advice: Advice, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::advise( + self, + self_, + offset, + length, + advice.into(), + ) + .await, + ) + } + + async fn sync_data( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::sync_data(self, self_).await) + } + + async fn get_flags( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::get_flags(self, self_).await) + } + + async fn get_type( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::get_type(self, self_).await) + } + + async fn set_size( + &mut self, + self_: Resource, + size: Filesize, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::set_size(self, self_, size).await) + } + + async fn set_times( + &mut self, + self_: Resource, + data_access_timestamp: NewTimestamp, + data_modification_timestamp: NewTimestamp, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::set_times( + self, + self_, + data_access_timestamp.into(), + data_modification_timestamp.into(), + ) + .await, + ) + } + + async fn read( + &mut self, + self_: Resource, + length: Filesize, + offset: Filesize, + ) -> wasmtime::Result, bool), FsErrorCode>> { + convert_result( + latest::filesystem::types::HostDescriptor::read(self, self_, length, offset).await, + ) + } + + async fn write( + &mut self, + self_: Resource, + buffer: Vec, + offset: Filesize, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::write(self, self_, buffer, offset).await, + ) + } + + async fn read_directory( + &mut self, + self_: Resource, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::read_directory(self, self_).await) + } + + async fn sync( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::sync(self, self_).await) + } + + async fn create_directory_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::create_directory_at(self, self_, path).await, + ) + } + + async fn stat( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::stat(self, self_).await) + } + + async fn stat_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::stat_at( + self, + self_, + path_flags.into(), + path, + ) + .await, + ) + } + + async fn set_times_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + data_access_timestamp: NewTimestamp, + data_modification_timestamp: NewTimestamp, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::set_times_at( + self, + self_, + path_flags.into(), + path, + data_access_timestamp.into(), + data_modification_timestamp.into(), + ) + .await, + ) + } + + async fn link_at( + &mut self, + self_: Resource, + old_path_flags: PathFlags, + old_path: String, + new_descriptor: Resource, + new_path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::link_at( + self, + self_, + old_path_flags.into(), + old_path, + new_descriptor, + new_path, + ) + .await, + ) + } + + async fn open_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + open_flags: OpenFlags, + flags: DescriptorFlags, + _modes: Modes, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result( + latest::filesystem::types::HostDescriptor::open_at( + self, + self_, + path_flags.into(), + path, + open_flags.into(), + flags.into(), + ) + .await, + ) + } + + async fn readlink_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::readlink_at(self, self_, path).await, + ) + } + + async fn remove_directory_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::remove_directory_at(self, self_, path).await, + ) + } + + async fn rename_at( + &mut self, + self_: Resource, + old_path: String, + new_descriptor: Resource, + new_path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::rename_at( + self, + self_, + old_path, + new_descriptor, + new_path, + ) + .await, + ) + } + + async fn symlink_at( + &mut self, + self_: Resource, + old_path: String, + new_path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::symlink_at(self, self_, old_path, new_path) + .await, + ) + } + + async fn access_at( + &mut self, + _self_: Resource, + _path_flags: PathFlags, + _path: String, + _type_: AccessType, + ) -> wasmtime::Result> { + anyhow::bail!("access-at API is no longer supported in the latest snapshot") + } + + async fn unlink_file_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::unlink_file_at(self, self_, path).await, + ) + } + + async fn change_file_permissions_at( + &mut self, + _self_: Resource, + _path_flags: PathFlags, + _path: String, + _modes: Modes, + ) -> wasmtime::Result> { + anyhow::bail!( + "change-file-permissions-at API is no longer supported in the latest snapshot" + ) + } + + async fn change_directory_permissions_at( + &mut self, + _self_: Resource, + _path_flags: PathFlags, + _path: String, + _modes: Modes, + ) -> wasmtime::Result> { + anyhow::bail!( + "change-directory-permissions-at API is no longer supported in the latest snapshot" + ) + } + + async fn lock_shared( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("lock-shared API is no longer supported in the latest snapshot") + } + + async fn lock_exclusive( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("lock-exclusive API is no longer supported in the latest snapshot") + } + + async fn try_lock_shared( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("try-lock-shared API is no longer supported in the latest snapshot") + } + + async fn try_lock_exclusive( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("try-lock-exclusive API is no longer supported in the latest snapshot") + } + + async fn unlock( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("unlock API is no longer supported in the latest snapshot") + } + + async fn is_same_object( + &mut self, + self_: Resource, + other: Resource, + ) -> wasmtime::Result { + latest::filesystem::types::HostDescriptor::is_same_object(self, self_, other).await + } + + async fn metadata_hash( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::metadata_hash(self, self_).await) + } + + async fn metadata_hash_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::metadata_hash_at( + self, + self_, + path_flags.into(), + path, + ) + .await, + ) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::filesystem::types::HostDescriptor::drop(self, rep) + } +} + +#[async_trait] +impl wasi::filesystem::types::HostDirectoryEntryStream for WasiImpl +where + T: WasiView, +{ + async fn read_directory_entry( + &mut self, + self_: Resource, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result( + latest::filesystem::types::HostDirectoryEntryStream::read_directory_entry(self, self_) + .await + .map(|e| e.map(DirectoryEntry::from)), + ) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::filesystem::types::HostDirectoryEntryStream::drop(self, rep) + } +} + +impl wasi::filesystem::preopens::Host for WasiImpl +where + T: WasiView, +{ + fn get_directories(&mut self) -> wasmtime::Result, String)>> { + latest::filesystem::preopens::Host::get_directories(self) + } +} + +#[async_trait] +impl wasi::io::poll::Host for WasiImpl +where + T: WasiView, +{ + async fn poll_list(&mut self, list: Vec>) -> wasmtime::Result> { + latest::io::poll::Host::poll(self, list).await + } + + async fn poll_one(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::poll::HostPollable::block(self, rep).await + } +} + +impl wasi::io::poll::HostPollable for WasiImpl +where + T: WasiView, +{ + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::poll::HostPollable::drop(self, rep) + } +} + +impl wasi::io::streams::Host for WasiImpl where T: WasiView {} + +impl wasi::io::streams::HostError for WasiImpl +where + T: WasiView, +{ + fn to_debug_string(&mut self, self_: Resource) -> wasmtime::Result { + latest::io::error::HostError::to_debug_string(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::error::HostError::drop(self, rep) + } +} + +#[async_trait] +impl wasi::io::streams::HostInputStream for WasiImpl +where + T: WasiView, +{ + async fn read( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result, StreamError>> { + let result = latest::io::streams::HostInputStream::read(self, self_, len).await; + convert_stream_result(self, result) + } + + async fn blocking_read( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result, StreamError>> { + let result = latest::io::streams::HostInputStream::blocking_read(self, self_, len).await; + convert_stream_result(self, result) + } + + async fn skip( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostInputStream::skip(self, self_, len).await; + convert_stream_result(self, result) + } + + async fn blocking_skip( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostInputStream::blocking_skip(self, self_, len).await; + convert_stream_result(self, result) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + latest::io::streams::HostInputStream::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::streams::HostInputStream::drop(self, rep) + } +} + +#[async_trait] +impl wasi::io::streams::HostOutputStream for WasiImpl +where + T: WasiView, +{ + fn check_write( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::check_write(self, self_); + convert_stream_result(self, result) + } + + fn write( + &mut self, + self_: Resource, + contents: Vec, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::write(self, self_, contents); + convert_stream_result(self, result) + } + + async fn blocking_write_and_flush( + &mut self, + self_: Resource, + contents: Vec, + ) -> wasmtime::Result> { + let result = + latest::io::streams::HostOutputStream::blocking_write_and_flush(self, self_, contents) + .await; + convert_stream_result(self, result) + } + + fn flush( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::flush(self, self_); + convert_stream_result(self, result) + } + + async fn blocking_flush( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::blocking_flush(self, self_).await; + convert_stream_result(self, result) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + latest::io::streams::HostOutputStream::subscribe(self, self_) + } + + fn write_zeroes( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::write_zeroes(self, self_, len); + convert_stream_result(self, result) + } + + async fn blocking_write_zeroes_and_flush( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::blocking_write_zeroes_and_flush( + self, self_, len, + ) + .await; + convert_stream_result(self, result) + } + + async fn splice( + &mut self, + self_: Resource, + src: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::splice(self, self_, src, len).await; + convert_stream_result(self, result) + } + + async fn blocking_splice( + &mut self, + self_: Resource, + src: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = + latest::io::streams::HostOutputStream::blocking_splice(self, self_, src, len).await; + convert_stream_result(self, result) + } + + async fn forward( + &mut self, + _self_: Resource, + _src: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("forward API no longer supported") + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::streams::HostOutputStream::drop(self, rep) + } +} + +impl wasi::random::random::Host for WasiImpl +where + T: WasiView, +{ + fn get_random_bytes(&mut self, len: u64) -> wasmtime::Result> { + latest::random::random::Host::get_random_bytes(self, len) + } + + fn get_random_u64(&mut self) -> wasmtime::Result { + latest::random::random::Host::get_random_u64(self) + } +} + +impl wasi::random::insecure::Host for WasiImpl +where + T: WasiView, +{ + fn get_insecure_random_bytes(&mut self, len: u64) -> wasmtime::Result> { + latest::random::insecure::Host::get_insecure_random_bytes(self, len) + } + + fn get_insecure_random_u64(&mut self) -> wasmtime::Result { + latest::random::insecure::Host::get_insecure_random_u64(self) + } +} + +impl wasi::random::insecure_seed::Host for WasiImpl +where + T: WasiView, +{ + fn insecure_seed(&mut self) -> wasmtime::Result<(u64, u64)> { + latest::random::insecure_seed::Host::insecure_seed(self) + } +} + +impl wasi::cli::exit::Host for WasiImpl +where + T: WasiView, +{ + fn exit(&mut self, status: Result<(), ()>) -> wasmtime::Result<()> { + latest::cli::exit::Host::exit(self, status) + } +} + +impl wasi::cli::environment::Host for WasiImpl +where + T: WasiView, +{ + fn get_environment(&mut self) -> wasmtime::Result> { + latest::cli::environment::Host::get_environment(self) + } + + fn get_arguments(&mut self) -> wasmtime::Result> { + latest::cli::environment::Host::get_arguments(self) + } + + fn initial_cwd(&mut self) -> wasmtime::Result> { + latest::cli::environment::Host::initial_cwd(self) + } +} + +impl wasi::cli::stdin::Host for WasiImpl +where + T: WasiView, +{ + fn get_stdin(&mut self) -> wasmtime::Result> { + latest::cli::stdin::Host::get_stdin(self) + } +} + +impl wasi::cli::stdout::Host for WasiImpl +where + T: WasiView, +{ + fn get_stdout(&mut self) -> wasmtime::Result> { + latest::cli::stdout::Host::get_stdout(self) + } +} + +impl wasi::cli::stderr::Host for WasiImpl +where + T: WasiView, +{ + fn get_stderr(&mut self) -> wasmtime::Result> { + latest::cli::stderr::Host::get_stderr(self) + } +} + +impl wasi::cli::terminal_stdin::Host for WasiImpl +where + T: WasiView, +{ + fn get_terminal_stdin(&mut self) -> wasmtime::Result>> { + latest::cli::terminal_stdin::Host::get_terminal_stdin(self) + } +} + +impl wasi::cli::terminal_stdout::Host for WasiImpl +where + T: WasiView, +{ + fn get_terminal_stdout(&mut self) -> wasmtime::Result>> { + latest::cli::terminal_stdout::Host::get_terminal_stdout(self) + } +} + +impl wasi::cli::terminal_stderr::Host for WasiImpl +where + T: WasiView, +{ + fn get_terminal_stderr(&mut self) -> wasmtime::Result>> { + latest::cli::terminal_stderr::Host::get_terminal_stderr(self) + } +} + +impl wasi::cli::terminal_input::Host for WasiImpl where T: WasiView {} + +impl wasi::cli::terminal_input::HostTerminalInput for WasiImpl +where + T: WasiView, +{ + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::cli::terminal_input::HostTerminalInput::drop(self, rep) + } +} + +impl wasi::cli::terminal_output::Host for WasiImpl where T: WasiView {} + +impl wasi::cli::terminal_output::HostTerminalOutput for WasiImpl +where + T: WasiView, +{ + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::cli::terminal_output::HostTerminalOutput::drop(self, rep) + } +} + +impl wasi::sockets::tcp::Host for WasiImpl where T: WasiView {} + +#[async_trait] +impl wasi::sockets::tcp::HostTcpSocket for WasiImpl +where + T: WasiView, +{ + async fn start_bind( + &mut self, + self_: Resource, + network: Resource, + local_address: IpSocketAddress, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::start_bind( + self, + self_, + network, + local_address.into(), + ) + .await, + ) + } + + fn finish_bind( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::finish_bind( + self, self_, + )) + } + + async fn start_connect( + &mut self, + self_: Resource, + network: Resource, + remote_address: IpSocketAddress, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::start_connect( + self, + self_, + network, + remote_address.into(), + ) + .await, + ) + } + + fn finish_connect( + &mut self, + self_: Resource, + ) -> wasmtime::Result, Resource), SocketErrorCode>> + { + convert_result(latest::sockets::tcp::HostTcpSocket::finish_connect( + self, self_, + )) + } + + fn start_listen( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::start_listen( + self, self_, + )) + } + + fn finish_listen( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::finish_listen( + self, self_, + )) + } + + fn accept( + &mut self, + self_: Resource, + ) -> wasmtime::Result< + Result< + ( + Resource, + Resource, + Resource, + ), + SocketErrorCode, + >, + > { + convert_result(latest::sockets::tcp::HostTcpSocket::accept(self, self_)) + } + + fn local_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::local_address( + self, self_, + )) + } + + fn remote_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::remote_address( + self, self_, + )) + } + + fn address_family(&mut self, self_: Resource) -> wasmtime::Result { + latest::sockets::tcp::HostTcpSocket::address_family(self, self_).map(|e| e.into()) + } + + fn ipv6_only( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn set_ipv6_only( + &mut self, + _self_: Resource, + _value: bool, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn set_listen_backlog_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::set_listen_backlog_size(self, self_, value), + ) + } + + fn keep_alive( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_enabled( + self, self_, + )) + } + + fn set_keep_alive( + &mut self, + self_: Resource, + value: bool, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::set_keep_alive_enabled( + self, self_, value, + )) + } + + fn no_delay( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("no-delay API no longer supported") + } + + fn set_no_delay( + &mut self, + _self_: Resource, + _value: bool, + ) -> wasmtime::Result> { + anyhow::bail!("set-no-delay API no longer supported") + } + + fn unicast_hop_limit( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::hop_limit(self, self_)) + } + + fn set_unicast_hop_limit( + &mut self, + self_: Resource, + value: u8, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::set_hop_limit( + self, self_, value, + )) + } + + fn receive_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::receive_buffer_size( + self, self_, + )) + } + + fn set_receive_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::set_receive_buffer_size(self, self_, value), + ) + } + + fn send_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::send_buffer_size( + self, self_, + )) + } + + fn set_send_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::set_send_buffer_size( + self, self_, value, + )) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + latest::sockets::tcp::HostTcpSocket::subscribe(self, self_) + } + + fn shutdown( + &mut self, + self_: Resource, + shutdown_type: ShutdownType, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::shutdown( + self, + self_, + shutdown_type.into(), + )) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::tcp::HostTcpSocket::drop(self, rep) + } +} + +impl wasi::sockets::tcp_create_socket::Host for WasiImpl +where + T: WasiView, +{ + fn create_tcp_socket( + &mut self, + address_family: IpAddressFamily, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result(latest::sockets::tcp_create_socket::Host::create_tcp_socket( + self, + address_family.into(), + )) + } +} + +impl wasi::sockets::udp::Host for WasiImpl where T: WasiView {} + +/// Between the snapshot of WASI that this file is implementing and the current +/// implementation of WASI UDP sockets were redesigned slightly to deal with +/// a different way of managing incoming and outgoing datagrams. This means +/// that this snapshot's `{start,finish}_connect`, `send`, and `receive` +/// methods are no longer natively implemented, so they're polyfilled by this +/// implementation. +pub enum UdpSocket { + Initial(Resource), + Connecting(Resource, IpSocketAddress), + Connected { + socket: Resource, + incoming: Resource, + outgoing: Resource, + }, + Dummy, +} + +impl UdpSocket { + async fn finish_connect( + table: &mut WasiImpl, + socket: &Resource, + explicit: bool, + ) -> wasmtime::Result> { + let state = table.table().get_mut(socket)?; + let (new_socket, addr) = match mem::replace(state, UdpSocket::Dummy) { + // Implicit finishes will call `stream` for sockets in the initial + // state. + UdpSocket::Initial(socket) if !explicit => (socket, None), + // Implicit finishes won't try to reconnect a socket. + UdpSocket::Connected { .. } if !explicit => return Ok(Ok(())), + // Only explicit finishes can transition from the `Connecting` state. + UdpSocket::Connecting(socket, addr) if explicit => (socket, Some(addr)), + _ => return Ok(Err(SocketErrorCode::ConcurrencyConflict)), + }; + let borrow = Resource::new_borrow(new_socket.rep()); + let result = convert_result( + latest::sockets::udp::HostUdpSocket::stream(table, borrow, addr.map(|a| a.into())) + .await, + )?; + let (incoming, outgoing) = match result { + Ok(pair) => pair, + Err(e) => return Ok(Err(e)), + }; + *table.table().get_mut(socket)? = UdpSocket::Connected { + socket: new_socket, + incoming, + outgoing, + }; + Ok(Ok(())) + } + + fn inner(&self) -> wasmtime::Result> { + let r = match self { + UdpSocket::Initial(r) => r, + UdpSocket::Connecting(r, _) => r, + UdpSocket::Connected { socket, .. } => socket, + UdpSocket::Dummy => anyhow::bail!("invalid udp socket state"), + }; + Ok(Resource::new_borrow(r.rep())) + } +} + +#[async_trait] +impl wasi::sockets::udp::HostUdpSocket for WasiImpl +where + T: WasiView, +{ + async fn start_bind( + &mut self, + self_: Resource, + network: Resource, + local_address: IpSocketAddress, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result( + latest::sockets::udp::HostUdpSocket::start_bind( + self, + socket, + network, + local_address.into(), + ) + .await, + ) + } + + fn finish_bind( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::finish_bind( + self, socket, + )) + } + + fn start_connect( + &mut self, + self_: Resource, + _network: Resource, + remote_address: IpSocketAddress, + ) -> wasmtime::Result> { + let socket = self.table().get_mut(&self_)?; + let (new_state, result) = match mem::replace(socket, UdpSocket::Dummy) { + UdpSocket::Initial(socket) => (UdpSocket::Connecting(socket, remote_address), Ok(())), + other => (other, Err(SocketErrorCode::ConcurrencyConflict)), + }; + *socket = new_state; + Ok(result) + } + + async fn finish_connect( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + UdpSocket::finish_connect(self, &self_, true).await + } + + async fn receive( + &mut self, + self_: Resource, + max_results: u64, + ) -> wasmtime::Result, SocketErrorCode>> { + // If the socket is in the `initial` state then complete the connect, + // otherwise verify we're connected. + if let Err(e) = UdpSocket::finish_connect(self, &self_, true).await? { + return Ok(Err(e)); + } + + // Use our connected state to acquire the `incoming-datagram-stream` + // resource, then receive some datagrams. + let incoming = match self.table().get(&self_)? { + UdpSocket::Connected { incoming, .. } => Resource::new_borrow(incoming.rep()), + _ => return Ok(Err(SocketErrorCode::ConcurrencyConflict)), + }; + let result: Result, _> = convert_result( + latest::sockets::udp::HostIncomingDatagramStream::receive(self, incoming, max_results), + )?; + match result { + Ok(datagrams) => Ok(Ok(datagrams + .into_iter() + .map(|datagram| datagram.into()) + .collect())), + Err(e) => Ok(Err(e)), + } + } + + async fn send( + &mut self, + self_: Resource, + mut datagrams: Vec, + ) -> wasmtime::Result> { + // If the socket is in the `initial` state then complete the connect, + // otherwise verify we're connected. + if let Err(e) = UdpSocket::finish_connect(self, &self_, true).await? { + return Ok(Err(e)); + } + + // Use our connected state to acquire the `outgoing-datagram-stream` + // resource. + let outgoing = match self.table().get(&self_)? { + UdpSocket::Connected { outgoing, .. } => Resource::new_borrow(outgoing.rep()), + _ => return Ok(Err(SocketErrorCode::ConcurrencyConflict)), + }; + + // Acquire a sending permit for some datagrams, truncating our list to + // that size if we have one. + let outgoing2 = Resource::new_borrow(outgoing.rep()); + match convert_result( + latest::sockets::udp::HostOutgoingDatagramStream::check_send(self, outgoing2), + )? { + Ok(n) => { + if datagrams.len() as u64 > n { + datagrams.truncate(n as usize); + } + } + Err(e) => return Ok(Err(e)), + } + + // Send off the datagrams. + convert_result( + latest::sockets::udp::HostOutgoingDatagramStream::send( + self, + outgoing, + datagrams + .into_iter() + .map(|d| latest::sockets::udp::OutgoingDatagram { + data: d.data, + remote_address: Some(d.remote_address.into()), + }) + .collect(), + ) + .await, + ) + } + + fn local_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::local_address( + self, socket, + )) + } + + fn remote_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::remote_address( + self, socket, + )) + } + + fn address_family(&mut self, self_: Resource) -> wasmtime::Result { + let socket = self.table().get(&self_)?.inner()?; + latest::sockets::udp::HostUdpSocket::address_family(self, socket).map(|e| e.into()) + } + + fn ipv6_only( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn set_ipv6_only( + &mut self, + _self_: Resource, + _value: bool, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn unicast_hop_limit( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::unicast_hop_limit( + self, socket, + )) + } + + fn set_unicast_hop_limit( + &mut self, + self_: Resource, + value: u8, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::set_unicast_hop_limit( + self, socket, value, + )) + } + + fn receive_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::receive_buffer_size( + self, socket, + )) + } + + fn set_receive_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result( + latest::sockets::udp::HostUdpSocket::set_receive_buffer_size(self, socket, value), + ) + } + + fn send_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::send_buffer_size( + self, socket, + )) + } + + fn set_send_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + convert_result(latest::sockets::udp::HostUdpSocket::set_send_buffer_size( + self, socket, value, + )) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + let socket = self.table().get(&self_)?.inner()?; + latest::sockets::udp::HostUdpSocket::subscribe(self, socket) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + let me = self.table().delete(rep)?; + let socket = match me { + UdpSocket::Initial(s) => s, + UdpSocket::Connecting(s, _) => s, + UdpSocket::Connected { + socket, + incoming, + outgoing, + } => { + latest::sockets::udp::HostIncomingDatagramStream::drop(self, incoming)?; + latest::sockets::udp::HostOutgoingDatagramStream::drop(self, outgoing)?; + socket + } + UdpSocket::Dummy => return Ok(()), + }; + latest::sockets::udp::HostUdpSocket::drop(self, socket) + } +} + +impl wasi::sockets::udp_create_socket::Host for WasiImpl +where + T: WasiView, +{ + fn create_udp_socket( + &mut self, + address_family: IpAddressFamily, + ) -> wasmtime::Result, SocketErrorCode>> { + let result = convert_result(latest::sockets::udp_create_socket::Host::create_udp_socket( + self, + address_family.into(), + ))?; + let socket = match result { + Ok(socket) => socket, + Err(e) => return Ok(Err(e)), + }; + let socket = self.table().push(UdpSocket::Initial(socket))?; + Ok(Ok(socket)) + } +} + +impl wasi::sockets::instance_network::Host for WasiImpl +where + T: WasiView, +{ + fn instance_network(&mut self) -> wasmtime::Result> { + latest::sockets::instance_network::Host::instance_network(self) + } +} + +impl wasi::sockets::network::Host for WasiImpl where T: WasiView {} + +impl wasi::sockets::network::HostNetwork for WasiImpl +where + T: WasiView, +{ + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::network::HostNetwork::drop(self, rep) + } +} + +impl wasi::sockets::ip_name_lookup::Host for WasiImpl +where + T: WasiView, +{ + fn resolve_addresses( + &mut self, + network: Resource, + name: String, + _address_family: Option, + _include_unavailable: bool, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result(latest::sockets::ip_name_lookup::Host::resolve_addresses( + self, network, name, + )) + } +} + +impl wasi::sockets::ip_name_lookup::HostResolveAddressStream for WasiImpl +where + T: WasiView, +{ + fn resolve_next_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result( + latest::sockets::ip_name_lookup::HostResolveAddressStream::resolve_next_address( + self, self_, + ) + .map(|e| e.map(|e| e.into())), + ) + } + + fn subscribe( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::sockets::ip_name_lookup::HostResolveAddressStream::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::ip_name_lookup::HostResolveAddressStream::drop(self, rep) + } +} + +pub fn convert_result( + result: Result>, +) -> wasmtime::Result> +where + T2: From, + E: std::error::Error + Send + Sync + 'static, + E2: From, +{ + match result { + Ok(e) => Ok(Ok(e.into())), + Err(e) => Ok(Err(e.downcast()?.into())), + } +} + +fn convert_stream_result( + mut view: impl WasiView, + result: Result, +) -> wasmtime::Result> +where + T2: From, +{ + match result { + Ok(e) => Ok(Ok(e.into())), + Err(wasmtime_wasi::StreamError::Closed) => Ok(Err(StreamError::Closed)), + Err(wasmtime_wasi::StreamError::LastOperationFailed(e)) => { + let e = view.table().push(e)?; + Ok(Err(StreamError::LastOperationFailed(e))) + } + Err(wasmtime_wasi::StreamError::Trap(e)) => Err(e), + } +} + +macro_rules! convert { + () => {}; + ($kind:ident $from:path [<=>] $to:path { $($body:tt)* } $($rest:tt)*) => { + convert!($kind $from => $to { $($body)* }); + convert!($kind $to => $from { $($body)* }); + + convert!($($rest)*); + }; + (struct $from:ty => $to:path { $($field:ident,)* } $($rest:tt)*) => { + impl From<$from> for $to { + fn from(e: $from) -> $to { + $to { + $( $field: e.$field.into(), )* + } + } + } + + convert!($($rest)*); + }; + (enum $from:path => $to:path { $($variant:ident $(($e:ident))?,)* } $($rest:tt)*) => { + impl From<$from> for $to { + fn from(e: $from) -> $to { + use $from as A; + use $to as B; + match e { + $( + A::$variant $(($e))? => B::$variant $(($e.into()))?, + )* + } + } + } + + convert!($($rest)*); + }; + (flags $from:path => $to:path { $($flag:ident,)* } $($rest:tt)*) => { + impl From<$from> for $to { + fn from(e: $from) -> $to { + use $from as A; + use $to as B; + let mut out = B::empty(); + $( + if e.contains(A::$flag) { + out |= B::$flag; + } + )* + out + } + } + + convert!($($rest)*); + }; +} + +pub(crate) use convert; + +convert! { + struct latest::clocks::wall_clock::Datetime [<=>] Datetime { + seconds, + nanoseconds, + } + + enum latest::filesystem::types::ErrorCode => FsErrorCode { + Access, + WouldBlock, + Already, + BadDescriptor, + Busy, + Deadlock, + Quota, + Exist, + FileTooLarge, + IllegalByteSequence, + InProgress, + Interrupted, + Invalid, + Io, + IsDirectory, + Loop, + TooManyLinks, + MessageSize, + NameTooLong, + NoDevice, + NoEntry, + NoLock, + InsufficientMemory, + InsufficientSpace, + NotDirectory, + NotEmpty, + NotRecoverable, + Unsupported, + NoTty, + NoSuchDevice, + Overflow, + NotPermitted, + Pipe, + ReadOnly, + InvalidSeek, + TextFileBusy, + CrossDevice, + } + + enum Advice => latest::filesystem::types::Advice { + Normal, + Sequential, + Random, + WillNeed, + DontNeed, + NoReuse, + } + + flags DescriptorFlags [<=>] latest::filesystem::types::DescriptorFlags { + READ, + WRITE, + FILE_INTEGRITY_SYNC, + DATA_INTEGRITY_SYNC, + REQUESTED_WRITE_SYNC, + MUTATE_DIRECTORY, + } + + enum DescriptorType [<=>] latest::filesystem::types::DescriptorType { + Unknown, + BlockDevice, + CharacterDevice, + Directory, + Fifo, + SymbolicLink, + RegularFile, + Socket, + } + + enum NewTimestamp => latest::filesystem::types::NewTimestamp { + NoChange, + Now, + Timestamp(e), + } + + flags PathFlags => latest::filesystem::types::PathFlags { + SYMLINK_FOLLOW, + } + + flags OpenFlags => latest::filesystem::types::OpenFlags { + CREATE, + DIRECTORY, + EXCLUSIVE, + TRUNCATE, + } + + struct latest::filesystem::types::MetadataHashValue => MetadataHashValue { + lower, + upper, + } + + struct latest::filesystem::types::DirectoryEntry => DirectoryEntry { + type_, + name, + } + + enum latest::sockets::network::ErrorCode => SocketErrorCode { + Unknown, + AccessDenied, + NotSupported, + InvalidArgument, + OutOfMemory, + Timeout, + ConcurrencyConflict, + NotInProgress, + WouldBlock, + InvalidState, + NewSocketLimit, + AddressNotBindable, + AddressInUse, + RemoteUnreachable, + ConnectionRefused, + ConnectionReset, + ConnectionAborted, + DatagramTooLarge, + NameUnresolvable, + TemporaryResolverFailure, + PermanentResolverFailure, + } + + enum latest::sockets::network::IpAddress [<=>] IpAddress { + Ipv4(e), + Ipv6(e), + } + + enum latest::sockets::network::IpSocketAddress [<=>] IpSocketAddress { + Ipv4(e), + Ipv6(e), + } + + struct latest::sockets::network::Ipv4SocketAddress [<=>] Ipv4SocketAddress { + port, + address, + } + + struct latest::sockets::network::Ipv6SocketAddress [<=>] Ipv6SocketAddress { + port, + flow_info, + scope_id, + address, + } + + enum latest::sockets::network::IpAddressFamily [<=>] IpAddressFamily { + Ipv4, + Ipv6, + } + + enum ShutdownType => latest::sockets::tcp::ShutdownType { + Receive, + Send, + Both, + } + + struct latest::sockets::udp::IncomingDatagram => Datagram { + data, + remote_address, + } +} + +impl From for DescriptorStat { + fn from(e: latest::filesystem::types::DescriptorStat) -> DescriptorStat { + DescriptorStat { + type_: e.type_.into(), + link_count: e.link_count, + size: e.size, + data_access_timestamp: e.data_access_timestamp.map(|e| e.into()), + data_modification_timestamp: e.data_modification_timestamp.map(|e| e.into()), + status_change_timestamp: e.status_change_timestamp.map(|e| e.into()), + } + } +} diff --git a/crates/factor-wasi/src/wasi_2023_11_10.rs b/crates/factor-wasi/src/wasi_2023_11_10.rs new file mode 100644 index 000000000..5135f516c --- /dev/null +++ b/crates/factor-wasi/src/wasi_2023_11_10.rs @@ -0,0 +1,1722 @@ +#![doc(hidden)] // internal implementation detail used in tests and spin-trigger + +use super::wasi_2023_10_18::{convert, convert_result}; +use async_trait::async_trait; +use spin_factors::anyhow::{self, Result}; +use wasmtime::component::{Linker, Resource}; +use wasmtime_wasi::{WasiImpl, WasiView}; + +use crate::InstanceState; + +mod latest { + pub use wasmtime_wasi::bindings::*; +} + +mod bindings { + use super::latest; + + wasmtime::component::bindgen!({ + path: "../../wit", + interfaces: r#" + // NB: this is handling the historical behavior where Spin supported + // more than "just" this snapshot of the proxy world but additionally + // other CLI-related interfaces. + include wasi:cli/reactor@0.2.0-rc-2023-11-10; + "#, + async: { + only_imports: [ + "[method]descriptor.advise", + "[method]descriptor.create-directory-at", + "[method]descriptor.get-flags", + "[method]descriptor.get-type", + "[method]descriptor.is-same-object", + "[method]descriptor.link-at", + "[method]descriptor.metadata-hash", + "[method]descriptor.metadata-hash-at", + "[method]descriptor.open-at", + "[method]descriptor.read", + "[method]descriptor.read-directory", + "[method]descriptor.readlink-at", + "[method]descriptor.remove-directory-at", + "[method]descriptor.rename-at", + "[method]descriptor.set-size", + "[method]descriptor.set-times", + "[method]descriptor.set-times-at", + "[method]descriptor.stat", + "[method]descriptor.stat-at", + "[method]descriptor.symlink-at", + "[method]descriptor.sync", + "[method]descriptor.sync-data", + "[method]descriptor.unlink-file-at", + "[method]descriptor.write", + "[method]input-stream.read", + "[method]input-stream.blocking-read", + "[method]input-stream.blocking-skip", + "[method]input-stream.skip", + "[method]output-stream.splice", + "[method]output-stream.blocking-splice", + "[method]output-stream.blocking-flush", + "[method]output-stream.blocking-write", + "[method]output-stream.blocking-write-and-flush", + "[method]output-stream.blocking-write-zeroes-and-flush", + "[method]directory-entry-stream.read-directory-entry", + "[method]pollable.block", + "[method]pollable.ready", + "poll", + + "[method]tcp-socket.start-bind", + "[method]tcp-socket.start-connect", + "[method]udp-socket.start-bind", + "[method]udp-socket.stream", + "[method]outgoing-datagram-stream.send", + ] + }, + with: { + "wasi:io/poll/pollable": latest::io::poll::Pollable, + "wasi:io/streams/input-stream": latest::io::streams::InputStream, + "wasi:io/streams/output-stream": latest::io::streams::OutputStream, + "wasi:io/error/error": latest::io::error::Error, + "wasi:filesystem/types/directory-entry-stream": latest::filesystem::types::DirectoryEntryStream, + "wasi:filesystem/types/descriptor": latest::filesystem::types::Descriptor, + "wasi:cli/terminal-input/terminal-input": latest::cli::terminal_input::TerminalInput, + "wasi:cli/terminal-output/terminal-output": latest::cli::terminal_output::TerminalOutput, + "wasi:sockets/tcp/tcp-socket": latest::sockets::tcp::TcpSocket, + "wasi:sockets/udp/udp-socket": latest::sockets::udp::UdpSocket, + "wasi:sockets/udp/outgoing-datagram-stream": latest::sockets::udp::OutgoingDatagramStream, + "wasi:sockets/udp/incoming-datagram-stream": latest::sockets::udp::IncomingDatagramStream, + "wasi:sockets/network/network": latest::sockets::network::Network, + "wasi:sockets/ip-name-lookup/resolve-address-stream": latest::sockets::ip_name_lookup::ResolveAddressStream, + }, + trappable_imports: true, + }); +} + +mod wasi { + pub use super::bindings::wasi::{ + cli0_2_0_rc_2023_11_10 as cli, clocks0_2_0_rc_2023_11_10 as clocks, + filesystem0_2_0_rc_2023_11_10 as filesystem, io0_2_0_rc_2023_11_10 as io, + random0_2_0_rc_2023_11_10 as random, sockets0_2_0_rc_2023_11_10 as sockets, + }; +} + +use wasi::cli::terminal_input::TerminalInput; +use wasi::cli::terminal_output::TerminalOutput; +use wasi::clocks::monotonic_clock::{Duration, Instant}; +use wasi::clocks::wall_clock::Datetime; +use wasi::filesystem::types::{ + Advice, Descriptor, DescriptorFlags, DescriptorStat, DescriptorType, DirectoryEntry, + DirectoryEntryStream, ErrorCode as FsErrorCode, Filesize, MetadataHashValue, NewTimestamp, + OpenFlags, PathFlags, +}; +use wasi::io::poll::Pollable; +use wasi::io::streams::{Error as IoError, InputStream, OutputStream, StreamError}; +use wasi::sockets::ip_name_lookup::{IpAddress, ResolveAddressStream}; +use wasi::sockets::network::{Ipv4SocketAddress, Ipv6SocketAddress}; +use wasi::sockets::tcp::{ + ErrorCode as SocketErrorCode, IpAddressFamily, IpSocketAddress, Network, ShutdownType, + TcpSocket, +}; +use wasi::sockets::udp::{ + IncomingDatagram, IncomingDatagramStream, OutgoingDatagram, OutgoingDatagramStream, UdpSocket, +}; + +pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> +where + T: Send, + F: Fn(&mut T) -> WasiImpl<&mut InstanceState> + Send + Sync + Copy + 'static, +{ + fn type_annotate(f: F) -> F + where + F: Fn(&mut T) -> WasiImpl<&mut U>, + { + f + } + let closure = type_annotate(closure); + wasi::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; + wasi::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; + wasi::filesystem::types::add_to_linker_get_host(linker, closure)?; + wasi::filesystem::preopens::add_to_linker_get_host(linker, closure)?; + wasi::io::error::add_to_linker_get_host(linker, closure)?; + wasi::io::poll::add_to_linker_get_host(linker, closure)?; + wasi::io::streams::add_to_linker_get_host(linker, closure)?; + wasi::random::random::add_to_linker_get_host(linker, closure)?; + wasi::random::insecure::add_to_linker_get_host(linker, closure)?; + wasi::random::insecure_seed::add_to_linker_get_host(linker, closure)?; + wasi::cli::exit::add_to_linker_get_host(linker, closure)?; + wasi::cli::environment::add_to_linker_get_host(linker, closure)?; + wasi::cli::stdin::add_to_linker_get_host(linker, closure)?; + wasi::cli::stdout::add_to_linker_get_host(linker, closure)?; + wasi::cli::stderr::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_input::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_output::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_stdin::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_stdout::add_to_linker_get_host(linker, closure)?; + wasi::cli::terminal_stderr::add_to_linker_get_host(linker, closure)?; + wasi::sockets::tcp::add_to_linker_get_host(linker, closure)?; + wasi::sockets::tcp_create_socket::add_to_linker_get_host(linker, closure)?; + wasi::sockets::udp::add_to_linker_get_host(linker, closure)?; + wasi::sockets::udp_create_socket::add_to_linker_get_host(linker, closure)?; + wasi::sockets::instance_network::add_to_linker_get_host(linker, closure)?; + wasi::sockets::network::add_to_linker_get_host(linker, closure)?; + wasi::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; + Ok(()) +} + +impl wasi::clocks::monotonic_clock::Host for WasiImpl +where + T: WasiView, +{ + fn now(&mut self) -> wasmtime::Result { + latest::clocks::monotonic_clock::Host::now(self) + } + + fn resolution(&mut self) -> wasmtime::Result { + latest::clocks::monotonic_clock::Host::resolution(self) + } + + fn subscribe_instant(&mut self, when: Instant) -> wasmtime::Result> { + latest::clocks::monotonic_clock::Host::subscribe_instant(self, when) + } + + fn subscribe_duration(&mut self, when: Duration) -> wasmtime::Result> { + latest::clocks::monotonic_clock::Host::subscribe_duration(self, when) + } +} + +impl wasi::clocks::wall_clock::Host for WasiImpl +where + T: WasiView, +{ + fn now(&mut self) -> wasmtime::Result { + Ok(latest::clocks::wall_clock::Host::now(self)?.into()) + } + + fn resolution(&mut self) -> wasmtime::Result { + Ok(latest::clocks::wall_clock::Host::resolution(self)?.into()) + } +} + +impl wasi::filesystem::types::Host for WasiImpl +where + T: WasiView, +{ + fn filesystem_error_code( + &mut self, + err: Resource, + ) -> wasmtime::Result> { + Ok(latest::filesystem::types::Host::filesystem_error_code(self, err)?.map(|e| e.into())) + } +} + +#[async_trait] +impl wasi::filesystem::types::HostDescriptor for WasiImpl +where + T: WasiView, +{ + fn read_via_stream( + &mut self, + self_: Resource, + offset: Filesize, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::read_via_stream( + self, self_, offset, + )) + } + + fn write_via_stream( + &mut self, + self_: Resource, + offset: Filesize, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::write_via_stream( + self, self_, offset, + )) + } + + fn append_via_stream( + &mut self, + self_: Resource, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::append_via_stream(self, self_)) + } + + async fn advise( + &mut self, + self_: Resource, + offset: Filesize, + length: Filesize, + advice: Advice, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::advise( + self, + self_, + offset, + length, + advice.into(), + ) + .await, + ) + } + + async fn sync_data( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::sync_data(self, self_).await) + } + + async fn get_flags( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::get_flags(self, self_).await) + } + + async fn get_type( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::get_type(self, self_).await) + } + + async fn set_size( + &mut self, + self_: Resource, + size: Filesize, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::set_size(self, self_, size).await) + } + + async fn set_times( + &mut self, + self_: Resource, + data_access_timestamp: NewTimestamp, + data_modification_timestamp: NewTimestamp, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::set_times( + self, + self_, + data_access_timestamp.into(), + data_modification_timestamp.into(), + ) + .await, + ) + } + + async fn read( + &mut self, + self_: Resource, + length: Filesize, + offset: Filesize, + ) -> wasmtime::Result, bool), FsErrorCode>> { + convert_result( + latest::filesystem::types::HostDescriptor::read(self, self_, length, offset).await, + ) + } + + async fn write( + &mut self, + self_: Resource, + buffer: Vec, + offset: Filesize, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::write(self, self_, buffer, offset).await, + ) + } + + async fn read_directory( + &mut self, + self_: Resource, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result(latest::filesystem::types::HostDescriptor::read_directory(self, self_).await) + } + + async fn sync( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::sync(self, self_).await) + } + + async fn create_directory_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::create_directory_at(self, self_, path).await, + ) + } + + async fn stat( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::stat(self, self_).await) + } + + async fn stat_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::stat_at( + self, + self_, + path_flags.into(), + path, + ) + .await, + ) + } + + async fn set_times_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + data_access_timestamp: NewTimestamp, + data_modification_timestamp: NewTimestamp, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::set_times_at( + self, + self_, + path_flags.into(), + path, + data_access_timestamp.into(), + data_modification_timestamp.into(), + ) + .await, + ) + } + + async fn link_at( + &mut self, + self_: Resource, + old_path_flags: PathFlags, + old_path: String, + new_descriptor: Resource, + new_path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::link_at( + self, + self_, + old_path_flags.into(), + old_path, + new_descriptor, + new_path, + ) + .await, + ) + } + + async fn open_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + open_flags: OpenFlags, + flags: DescriptorFlags, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result( + latest::filesystem::types::HostDescriptor::open_at( + self, + self_, + path_flags.into(), + path, + open_flags.into(), + flags.into(), + ) + .await, + ) + } + + async fn readlink_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::readlink_at(self, self_, path).await, + ) + } + + async fn remove_directory_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::remove_directory_at(self, self_, path).await, + ) + } + + async fn rename_at( + &mut self, + self_: Resource, + old_path: String, + new_descriptor: Resource, + new_path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::rename_at( + self, + self_, + old_path, + new_descriptor, + new_path, + ) + .await, + ) + } + + async fn symlink_at( + &mut self, + self_: Resource, + old_path: String, + new_path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::symlink_at(self, self_, old_path, new_path) + .await, + ) + } + + async fn unlink_file_at( + &mut self, + self_: Resource, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::unlink_file_at(self, self_, path).await, + ) + } + + async fn is_same_object( + &mut self, + self_: Resource, + other: Resource, + ) -> wasmtime::Result { + latest::filesystem::types::HostDescriptor::is_same_object(self, self_, other).await + } + + async fn metadata_hash( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::filesystem::types::HostDescriptor::metadata_hash(self, self_).await) + } + + async fn metadata_hash_at( + &mut self, + self_: Resource, + path_flags: PathFlags, + path: String, + ) -> wasmtime::Result> { + convert_result( + latest::filesystem::types::HostDescriptor::metadata_hash_at( + self, + self_, + path_flags.into(), + path, + ) + .await, + ) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::filesystem::types::HostDescriptor::drop(self, rep) + } +} + +#[async_trait] +impl wasi::filesystem::types::HostDirectoryEntryStream for WasiImpl +where + T: WasiView, +{ + async fn read_directory_entry( + &mut self, + self_: Resource, + ) -> wasmtime::Result, FsErrorCode>> { + convert_result( + latest::filesystem::types::HostDirectoryEntryStream::read_directory_entry(self, self_) + .await + .map(|e| e.map(DirectoryEntry::from)), + ) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::filesystem::types::HostDirectoryEntryStream::drop(self, rep) + } +} + +impl wasi::filesystem::preopens::Host for WasiImpl +where + T: WasiView, +{ + fn get_directories(&mut self) -> wasmtime::Result, String)>> { + latest::filesystem::preopens::Host::get_directories(self) + } +} + +#[async_trait] +impl wasi::io::poll::Host for WasiImpl +where + T: WasiView, +{ + async fn poll(&mut self, list: Vec>) -> wasmtime::Result> { + latest::io::poll::Host::poll(self, list).await + } +} + +#[async_trait] +impl wasi::io::poll::HostPollable for WasiImpl +where + T: WasiView, +{ + async fn block(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::poll::HostPollable::block(self, rep).await + } + + async fn ready(&mut self, rep: Resource) -> wasmtime::Result { + latest::io::poll::HostPollable::ready(self, rep).await + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::poll::HostPollable::drop(self, rep) + } +} + +impl wasi::io::error::Host for WasiImpl where T: WasiView {} + +impl wasi::io::error::HostError for WasiImpl +where + T: WasiView, +{ + fn to_debug_string(&mut self, self_: Resource) -> wasmtime::Result { + latest::io::error::HostError::to_debug_string(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::error::HostError::drop(self, rep) + } +} + +fn convert_stream_result( + mut view: impl WasiView, + result: Result, +) -> wasmtime::Result> +where + T2: From, +{ + match result { + Ok(e) => Ok(Ok(e.into())), + Err(wasmtime_wasi::StreamError::Closed) => Ok(Err(StreamError::Closed)), + Err(wasmtime_wasi::StreamError::LastOperationFailed(e)) => { + let e = view.table().push(e)?; + Ok(Err(StreamError::LastOperationFailed(e))) + } + Err(wasmtime_wasi::StreamError::Trap(e)) => Err(e), + } +} + +impl wasi::io::streams::Host for WasiImpl where T: WasiView {} + +#[async_trait] +impl wasi::io::streams::HostInputStream for WasiImpl +where + T: WasiView, +{ + async fn read( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result, StreamError>> { + let result = latest::io::streams::HostInputStream::read(self, self_, len).await; + convert_stream_result(self, result) + } + + async fn blocking_read( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result, StreamError>> { + let result = latest::io::streams::HostInputStream::blocking_read(self, self_, len).await; + convert_stream_result(self, result) + } + + async fn skip( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostInputStream::skip(self, self_, len).await; + convert_stream_result(self, result) + } + + async fn blocking_skip( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostInputStream::blocking_skip(self, self_, len).await; + convert_stream_result(self, result) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + latest::io::streams::HostInputStream::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::streams::HostInputStream::drop(self, rep) + } +} + +#[async_trait] +impl wasi::io::streams::HostOutputStream for WasiImpl +where + T: WasiView, +{ + fn check_write( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::check_write(self, self_); + convert_stream_result(self, result) + } + + fn write( + &mut self, + self_: Resource, + contents: Vec, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::write(self, self_, contents); + convert_stream_result(self, result) + } + + async fn blocking_write_and_flush( + &mut self, + self_: Resource, + contents: Vec, + ) -> wasmtime::Result> { + let result = + latest::io::streams::HostOutputStream::blocking_write_and_flush(self, self_, contents) + .await; + convert_stream_result(self, result) + } + + fn flush( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::flush(self, self_); + convert_stream_result(self, result) + } + + async fn blocking_flush( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::blocking_flush(self, self_).await; + convert_stream_result(self, result) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + latest::io::streams::HostOutputStream::subscribe(self, self_) + } + + fn write_zeroes( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::write_zeroes(self, self_, len); + convert_stream_result(self, result) + } + + async fn blocking_write_zeroes_and_flush( + &mut self, + self_: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::blocking_write_zeroes_and_flush( + self, self_, len, + ) + .await; + convert_stream_result(self, result) + } + + async fn splice( + &mut self, + self_: Resource, + src: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = latest::io::streams::HostOutputStream::splice(self, self_, src, len).await; + convert_stream_result(self, result) + } + + async fn blocking_splice( + &mut self, + self_: Resource, + src: Resource, + len: u64, + ) -> wasmtime::Result> { + let result = + latest::io::streams::HostOutputStream::blocking_splice(self, self_, src, len).await; + convert_stream_result(self, result) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::io::streams::HostOutputStream::drop(self, rep) + } +} + +impl wasi::random::random::Host for WasiImpl +where + T: WasiView, +{ + fn get_random_bytes(&mut self, len: u64) -> wasmtime::Result> { + latest::random::random::Host::get_random_bytes(self, len) + } + + fn get_random_u64(&mut self) -> wasmtime::Result { + latest::random::random::Host::get_random_u64(self) + } +} + +impl wasi::random::insecure::Host for WasiImpl +where + T: WasiView, +{ + fn get_insecure_random_bytes(&mut self, len: u64) -> wasmtime::Result> { + latest::random::insecure::Host::get_insecure_random_bytes(self, len) + } + + fn get_insecure_random_u64(&mut self) -> wasmtime::Result { + latest::random::insecure::Host::get_insecure_random_u64(self) + } +} + +impl wasi::random::insecure_seed::Host for WasiImpl +where + T: WasiView, +{ + fn insecure_seed(&mut self) -> wasmtime::Result<(u64, u64)> { + latest::random::insecure_seed::Host::insecure_seed(self) + } +} + +impl wasi::cli::exit::Host for WasiImpl +where + T: WasiView, +{ + fn exit(&mut self, status: Result<(), ()>) -> wasmtime::Result<()> { + latest::cli::exit::Host::exit(self, status) + } +} + +impl wasi::cli::environment::Host for WasiImpl +where + T: WasiView, +{ + fn get_environment(&mut self) -> wasmtime::Result> { + latest::cli::environment::Host::get_environment(self) + } + + fn get_arguments(&mut self) -> wasmtime::Result> { + latest::cli::environment::Host::get_arguments(self) + } + + fn initial_cwd(&mut self) -> wasmtime::Result> { + latest::cli::environment::Host::initial_cwd(self) + } +} + +impl wasi::cli::stdin::Host for WasiImpl +where + T: WasiView, +{ + fn get_stdin(&mut self) -> wasmtime::Result> { + latest::cli::stdin::Host::get_stdin(self) + } +} + +impl wasi::cli::stdout::Host for WasiImpl +where + T: WasiView, +{ + fn get_stdout(&mut self) -> wasmtime::Result> { + latest::cli::stdout::Host::get_stdout(self) + } +} + +impl wasi::cli::stderr::Host for WasiImpl +where + T: WasiView, +{ + fn get_stderr(&mut self) -> wasmtime::Result> { + latest::cli::stderr::Host::get_stderr(self) + } +} + +impl wasi::cli::terminal_stdin::Host for WasiImpl +where + T: WasiView, +{ + fn get_terminal_stdin(&mut self) -> wasmtime::Result>> { + latest::cli::terminal_stdin::Host::get_terminal_stdin(self) + } +} + +impl wasi::cli::terminal_stdout::Host for WasiImpl +where + T: WasiView, +{ + fn get_terminal_stdout(&mut self) -> wasmtime::Result>> { + latest::cli::terminal_stdout::Host::get_terminal_stdout(self) + } +} + +impl wasi::cli::terminal_stderr::Host for WasiImpl +where + T: WasiView, +{ + fn get_terminal_stderr(&mut self) -> wasmtime::Result>> { + latest::cli::terminal_stderr::Host::get_terminal_stderr(self) + } +} + +impl wasi::cli::terminal_input::Host for WasiImpl where T: WasiView {} + +impl wasi::cli::terminal_input::HostTerminalInput for WasiImpl +where + T: WasiView, +{ + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::cli::terminal_input::HostTerminalInput::drop(self, rep) + } +} + +impl wasi::cli::terminal_output::Host for WasiImpl where T: WasiView {} + +impl wasi::cli::terminal_output::HostTerminalOutput for WasiImpl +where + T: WasiView, +{ + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::cli::terminal_output::HostTerminalOutput::drop(self, rep) + } +} + +impl wasi::sockets::tcp::Host for WasiImpl where T: WasiView {} + +#[async_trait] +impl wasi::sockets::tcp::HostTcpSocket for WasiImpl +where + T: WasiView, +{ + async fn start_bind( + &mut self, + self_: Resource, + network: Resource, + local_address: IpSocketAddress, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::start_bind( + self, + self_, + network, + local_address.into(), + ) + .await, + ) + } + + fn finish_bind( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::finish_bind( + self, self_, + )) + } + + async fn start_connect( + &mut self, + self_: Resource, + network: Resource, + remote_address: IpSocketAddress, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::start_connect( + self, + self_, + network, + remote_address.into(), + ) + .await, + ) + } + + fn finish_connect( + &mut self, + self_: Resource, + ) -> wasmtime::Result, Resource), SocketErrorCode>> + { + convert_result(latest::sockets::tcp::HostTcpSocket::finish_connect( + self, self_, + )) + } + + fn start_listen( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::start_listen( + self, self_, + )) + } + + fn finish_listen( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::finish_listen( + self, self_, + )) + } + + fn accept( + &mut self, + self_: Resource, + ) -> wasmtime::Result< + Result< + ( + Resource, + Resource, + Resource, + ), + SocketErrorCode, + >, + > { + convert_result(latest::sockets::tcp::HostTcpSocket::accept(self, self_)) + } + + fn local_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::local_address( + self, self_, + )) + } + + fn remote_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::remote_address( + self, self_, + )) + } + + fn address_family(&mut self, self_: Resource) -> wasmtime::Result { + latest::sockets::tcp::HostTcpSocket::address_family(self, self_).map(|e| e.into()) + } + + fn ipv6_only( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn set_ipv6_only( + &mut self, + _self_: Resource, + _value: bool, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn set_listen_backlog_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::set_listen_backlog_size(self, self_, value), + ) + } + + fn is_listening(&mut self, self_: Resource) -> wasmtime::Result { + latest::sockets::tcp::HostTcpSocket::is_listening(self, self_) + } + + fn keep_alive_enabled( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_enabled( + self, self_, + )) + } + + fn set_keep_alive_enabled( + &mut self, + self_: Resource, + value: bool, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::set_keep_alive_enabled( + self, self_, value, + )) + } + + fn keep_alive_idle_time( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_idle_time( + self, self_, + )) + } + + fn set_keep_alive_idle_time( + &mut self, + self_: Resource, + value: Duration, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::set_keep_alive_idle_time(self, self_, value), + ) + } + + fn keep_alive_interval( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_interval( + self, self_, + )) + } + + fn set_keep_alive_interval( + &mut self, + self_: Resource, + value: Duration, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::set_keep_alive_interval(self, self_, value), + ) + } + + fn keep_alive_count( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_count( + self, self_, + )) + } + + fn set_keep_alive_count( + &mut self, + self_: Resource, + value: u32, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::set_keep_alive_count( + self, self_, value, + )) + } + + fn hop_limit( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::hop_limit(self, self_)) + } + + fn set_hop_limit( + &mut self, + self_: Resource, + value: u8, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::set_hop_limit( + self, self_, value, + )) + } + + fn receive_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::receive_buffer_size( + self, self_, + )) + } + + fn set_receive_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::tcp::HostTcpSocket::set_receive_buffer_size(self, self_, value), + ) + } + + fn send_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::send_buffer_size( + self, self_, + )) + } + + fn set_send_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::set_send_buffer_size( + self, self_, value, + )) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + latest::sockets::tcp::HostTcpSocket::subscribe(self, self_) + } + + fn shutdown( + &mut self, + self_: Resource, + shutdown_type: ShutdownType, + ) -> wasmtime::Result> { + convert_result(latest::sockets::tcp::HostTcpSocket::shutdown( + self, + self_, + shutdown_type.into(), + )) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::tcp::HostTcpSocket::drop(self, rep) + } +} + +impl wasi::sockets::tcp_create_socket::Host for WasiImpl +where + T: WasiView, +{ + fn create_tcp_socket( + &mut self, + address_family: IpAddressFamily, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result(latest::sockets::tcp_create_socket::Host::create_tcp_socket( + self, + address_family.into(), + )) + } +} + +impl wasi::sockets::udp::Host for WasiImpl where T: WasiView {} + +#[async_trait] +impl wasi::sockets::udp::HostUdpSocket for WasiImpl +where + T: WasiView, +{ + async fn start_bind( + &mut self, + self_: Resource, + network: Resource, + local_address: IpSocketAddress, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::udp::HostUdpSocket::start_bind( + self, + self_, + network, + local_address.into(), + ) + .await, + ) + } + + fn finish_bind( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::finish_bind( + self, self_, + )) + } + + fn local_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::local_address( + self, self_, + )) + } + + fn remote_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::remote_address( + self, self_, + )) + } + + fn address_family(&mut self, self_: Resource) -> wasmtime::Result { + latest::sockets::udp::HostUdpSocket::address_family(self, self_).map(|e| e.into()) + } + + fn ipv6_only( + &mut self, + _self_: Resource, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn set_ipv6_only( + &mut self, + _self_: Resource, + _value: bool, + ) -> wasmtime::Result> { + anyhow::bail!("ipv6-only API no longer supported") + } + + fn unicast_hop_limit( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::unicast_hop_limit( + self, self_, + )) + } + + fn set_unicast_hop_limit( + &mut self, + self_: Resource, + value: u8, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::set_unicast_hop_limit( + self, self_, value, + )) + } + + fn receive_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::receive_buffer_size( + self, self_, + )) + } + + fn set_receive_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::udp::HostUdpSocket::set_receive_buffer_size(self, self_, value), + ) + } + + fn send_buffer_size( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::send_buffer_size( + self, self_, + )) + } + + fn set_send_buffer_size( + &mut self, + self_: Resource, + value: u64, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostUdpSocket::set_send_buffer_size( + self, self_, value, + )) + } + + async fn stream( + &mut self, + self_: Resource, + remote_address: Option, + ) -> wasmtime::Result< + Result< + ( + Resource, + Resource, + ), + SocketErrorCode, + >, + > { + convert_result( + latest::sockets::udp::HostUdpSocket::stream( + self, + self_, + remote_address.map(|a| a.into()), + ) + .await, + ) + } + + fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { + latest::sockets::udp::HostUdpSocket::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::udp::HostUdpSocket::drop(self, rep) + } +} + +#[async_trait] +impl wasi::sockets::udp::HostOutgoingDatagramStream for WasiImpl +where + T: WasiView, +{ + fn check_send( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + convert_result(latest::sockets::udp::HostOutgoingDatagramStream::check_send(self, self_)) + } + + async fn send( + &mut self, + self_: Resource, + datagrams: Vec, + ) -> wasmtime::Result> { + convert_result( + latest::sockets::udp::HostOutgoingDatagramStream::send( + self, + self_, + datagrams.into_iter().map(|d| d.into()).collect(), + ) + .await, + ) + } + + fn subscribe( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::sockets::udp::HostOutgoingDatagramStream::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::udp::HostOutgoingDatagramStream::drop(self, rep) + } +} + +impl wasi::sockets::udp::HostIncomingDatagramStream for WasiImpl +where + T: WasiView, +{ + fn receive( + &mut self, + self_: Resource, + max_results: u64, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result(latest::sockets::udp::HostIncomingDatagramStream::receive( + self, + self_, + max_results, + )) + .map(|r| r.map(|r: Vec<_>| r.into_iter().map(|d| d.into()).collect())) + } + + fn subscribe( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::sockets::udp::HostIncomingDatagramStream::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::udp::HostIncomingDatagramStream::drop(self, rep) + } +} + +impl wasi::sockets::udp_create_socket::Host for WasiImpl +where + T: WasiView, +{ + fn create_udp_socket( + &mut self, + address_family: IpAddressFamily, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result(latest::sockets::udp_create_socket::Host::create_udp_socket( + self, + address_family.into(), + )) + } +} + +impl wasi::sockets::instance_network::Host for WasiImpl +where + T: WasiView, +{ + fn instance_network(&mut self) -> wasmtime::Result> { + latest::sockets::instance_network::Host::instance_network(self) + } +} + +impl wasi::sockets::network::Host for WasiImpl where T: WasiView {} + +impl wasi::sockets::network::HostNetwork for WasiImpl +where + T: WasiView, +{ + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::network::HostNetwork::drop(self, rep) + } +} + +impl wasi::sockets::ip_name_lookup::Host for WasiImpl +where + T: WasiView, +{ + fn resolve_addresses( + &mut self, + network: Resource, + name: String, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result(latest::sockets::ip_name_lookup::Host::resolve_addresses( + self, network, name, + )) + } +} + +impl wasi::sockets::ip_name_lookup::HostResolveAddressStream for WasiImpl +where + T: WasiView, +{ + fn resolve_next_address( + &mut self, + self_: Resource, + ) -> wasmtime::Result, SocketErrorCode>> { + convert_result( + latest::sockets::ip_name_lookup::HostResolveAddressStream::resolve_next_address( + self, self_, + ) + .map(|e| e.map(|e| e.into())), + ) + } + + fn subscribe( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::sockets::ip_name_lookup::HostResolveAddressStream::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::sockets::ip_name_lookup::HostResolveAddressStream::drop(self, rep) + } +} + +convert! { + struct latest::clocks::wall_clock::Datetime [<=>] Datetime { + seconds, + nanoseconds, + } + + enum latest::filesystem::types::ErrorCode => FsErrorCode { + Access, + WouldBlock, + Already, + BadDescriptor, + Busy, + Deadlock, + Quota, + Exist, + FileTooLarge, + IllegalByteSequence, + InProgress, + Interrupted, + Invalid, + Io, + IsDirectory, + Loop, + TooManyLinks, + MessageSize, + NameTooLong, + NoDevice, + NoEntry, + NoLock, + InsufficientMemory, + InsufficientSpace, + NotDirectory, + NotEmpty, + NotRecoverable, + Unsupported, + NoTty, + NoSuchDevice, + Overflow, + NotPermitted, + Pipe, + ReadOnly, + InvalidSeek, + TextFileBusy, + CrossDevice, + } + + enum Advice => latest::filesystem::types::Advice { + Normal, + Sequential, + Random, + WillNeed, + DontNeed, + NoReuse, + } + + flags DescriptorFlags [<=>] latest::filesystem::types::DescriptorFlags { + READ, + WRITE, + FILE_INTEGRITY_SYNC, + DATA_INTEGRITY_SYNC, + REQUESTED_WRITE_SYNC, + MUTATE_DIRECTORY, + } + + enum DescriptorType [<=>] latest::filesystem::types::DescriptorType { + Unknown, + BlockDevice, + CharacterDevice, + Directory, + Fifo, + SymbolicLink, + RegularFile, + Socket, + } + + enum NewTimestamp => latest::filesystem::types::NewTimestamp { + NoChange, + Now, + Timestamp(e), + } + + flags PathFlags => latest::filesystem::types::PathFlags { + SYMLINK_FOLLOW, + } + + flags OpenFlags => latest::filesystem::types::OpenFlags { + CREATE, + DIRECTORY, + EXCLUSIVE, + TRUNCATE, + } + + struct latest::filesystem::types::MetadataHashValue => MetadataHashValue { + lower, + upper, + } + + struct latest::filesystem::types::DirectoryEntry => DirectoryEntry { + type_, + name, + } + + + enum latest::sockets::network::ErrorCode => SocketErrorCode { + Unknown, + AccessDenied, + NotSupported, + InvalidArgument, + OutOfMemory, + Timeout, + ConcurrencyConflict, + NotInProgress, + WouldBlock, + InvalidState, + NewSocketLimit, + AddressNotBindable, + AddressInUse, + RemoteUnreachable, + ConnectionRefused, + ConnectionReset, + ConnectionAborted, + DatagramTooLarge, + NameUnresolvable, + TemporaryResolverFailure, + PermanentResolverFailure, + } + + enum latest::sockets::network::IpAddress [<=>] IpAddress { + Ipv4(e), + Ipv6(e), + } + + enum latest::sockets::network::IpSocketAddress [<=>] IpSocketAddress { + Ipv4(e), + Ipv6(e), + } + + struct latest::sockets::network::Ipv4SocketAddress [<=>] Ipv4SocketAddress { + port, + address, + } + + struct latest::sockets::network::Ipv6SocketAddress [<=>] Ipv6SocketAddress { + port, + flow_info, + scope_id, + address, + } + + enum latest::sockets::network::IpAddressFamily [<=>] IpAddressFamily { + Ipv4, + Ipv6, + } + + enum ShutdownType => latest::sockets::tcp::ShutdownType { + Receive, + Send, + Both, + } + + struct latest::sockets::udp::IncomingDatagram => IncomingDatagram { + data, + remote_address, + } +} + +impl From for DescriptorStat { + fn from(e: latest::filesystem::types::DescriptorStat) -> DescriptorStat { + DescriptorStat { + type_: e.type_.into(), + link_count: e.link_count, + size: e.size, + data_access_timestamp: e.data_access_timestamp.map(|e| e.into()), + data_modification_timestamp: e.data_modification_timestamp.map(|e| e.into()), + status_change_timestamp: e.status_change_timestamp.map(|e| e.into()), + } + } +} + +impl From for latest::sockets::udp::OutgoingDatagram { + fn from(d: OutgoingDatagram) -> Self { + Self { + data: d.data, + remote_address: d.remote_address.map(|a| a.into()), + } + } +} diff --git a/crates/factor-wasi/tests/factor_test.rs b/crates/factor-wasi/tests/factor_test.rs index 3b9632152..e502f4ae7 100644 --- a/crates/factor-wasi/tests/factor_test.rs +++ b/crates/factor-wasi/tests/factor_test.rs @@ -21,10 +21,10 @@ async fn environment_works() -> anyhow::Result<()> { let factors = TestFactors { wasi: WasiFactor::new(DummyFilesMounter), }; - let env = test_env(); let mut state = env.build_instance_state(factors).await?; let mut wasi = WasiImpl(&mut state.wasi); + let val = wasi .get_environment()? .into_iter() From 46c0e9ead956c0e174a998d180ebbe6e653090a2 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 11 Jul 2024 19:08:11 +0200 Subject: [PATCH 031/195] Add a llm-factors Signed-off-by: Ryan Levick --- Cargo.lock | 14 ++++ crates/factor-llm/Cargo.toml | 24 +++++++ crates/factor-llm/src/host.rs | 84 ++++++++++++++++++++++ crates/factor-llm/src/lib.rs | 114 ++++++++++++++++++++++++++++++ crates/factor-llm/tests/factor.rs | 65 +++++++++++++++++ 5 files changed, 301 insertions(+) create mode 100644 crates/factor-llm/Cargo.toml create mode 100644 crates/factor-llm/src/host.rs create mode 100644 crates/factor-llm/src/lib.rs create mode 100644 crates/factor-llm/tests/factor.rs diff --git a/Cargo.lock b/Cargo.lock index 97f108940..238d90ef7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2422,6 +2422,20 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "factor-llm" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "spin-factors", + "spin-factors-test", + "spin-locked-app", + "spin-world", + "tokio", + "tracing", +] + [[package]] name = "fallible-iterator" version = "0.2.0" diff --git a/crates/factor-llm/Cargo.toml b/crates/factor-llm/Cargo.toml new file mode 100644 index 000000000..bcdc1e81f --- /dev/null +++ b/crates/factor-llm/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "factor-llm" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +spin-factors = { path = "../factors" } +spin-locked-app = { path = "../locked-app" } +spin-world = { path = "../world" } +tracing = { workspace = true } + +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/factor-llm/src/host.rs b/crates/factor-llm/src/host.rs new file mode 100644 index 000000000..2eded3b7e --- /dev/null +++ b/crates/factor-llm/src/host.rs @@ -0,0 +1,84 @@ +use async_trait::async_trait; +use spin_world::v1::llm::{self as v1}; +use spin_world::v2::llm::{self as v2}; + +use crate::LlmDispatch; + +#[async_trait] +impl v2::Host for LlmDispatch { + async fn infer( + &mut self, + model: v2::InferencingModel, + prompt: String, + params: Option, + ) -> Result { + if !self.allowed_models.contains(&model) { + return Err(access_denied_error(&model)); + } + self.engine + .infer( + model, + prompt, + params.unwrap_or(v2::InferencingParams { + max_tokens: 100, + repeat_penalty: 1.1, + repeat_penalty_last_n_token_count: 64, + temperature: 0.8, + top_k: 40, + top_p: 0.9, + }), + ) + .await + } + + async fn generate_embeddings( + &mut self, + m: v1::EmbeddingModel, + data: Vec, + ) -> Result { + if !self.allowed_models.contains(&m) { + return Err(access_denied_error(&m)); + } + self.engine.generate_embeddings(m, data).await + } + + fn convert_error(&mut self, error: v2::Error) -> anyhow::Result { + Ok(error) + } +} + +#[async_trait] +impl v1::Host for LlmDispatch { + async fn infer( + &mut self, + model: v1::InferencingModel, + prompt: String, + params: Option, + ) -> Result { + ::infer(self, model, prompt, params.map(Into::into)) + .await + .map(Into::into) + .map_err(Into::into) + } + + async fn generate_embeddings( + &mut self, + model: v1::EmbeddingModel, + data: Vec, + ) -> Result { + ::generate_embeddings(self, model, data) + .await + .map(Into::into) + .map_err(Into::into) + } + + fn convert_error(&mut self, error: v1::Error) -> anyhow::Result { + Ok(error) + } +} + +fn access_denied_error(model: &str) -> v2::Error { + v2::Error::InvalidInput(format!( + "The component does not have access to use '{model}'. To give the component access, add '{model}' to the 'ai_models' key for the component in your spin.toml manifest" + )) +} diff --git a/crates/factor-llm/src/lib.rs b/crates/factor-llm/src/lib.rs new file mode 100644 index 000000000..9d922929d --- /dev/null +++ b/crates/factor-llm/src/lib.rs @@ -0,0 +1,114 @@ +mod host; + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use spin_factors::{ + ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, + SelfInstanceBuilder, +}; +use spin_locked_app::MetadataKey; +use spin_world::v1::llm::{self as v1}; +use spin_world::v2::llm::{self as v2}; + +pub const ALLOWED_MODELS_KEY: MetadataKey> = MetadataKey::new("ai_models"); + +pub struct LlmFactor { + create_engine: Box Box + Send + Sync>, +} + +impl LlmFactor { + pub fn new(create_engine: F) -> Self + where + F: Fn() -> Box + Send + Sync + 'static, + { + Self { + create_engine: Box::new(create_engine), + } + } +} + +impl Factor for LlmFactor { + type RuntimeConfig = (); + type AppState = AppState; + type InstanceBuilder = LlmDispatch; + + fn init( + &mut self, + mut ctx: spin_factors::InitContext, + ) -> anyhow::Result<()> { + ctx.link_bindings(spin_world::v1::llm::add_to_linker)?; + ctx.link_bindings(spin_world::v2::llm::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + ctx: ConfigureAppContext, + ) -> anyhow::Result { + let component_allowed_models = ctx + .app() + .components() + .map(|component| { + Ok(( + component.id().to_string(), + component + .get_metadata(ALLOWED_MODELS_KEY)? + .unwrap_or_default() + .into_iter() + .collect::>() + .into(), + )) + }) + .collect::>()?; + Ok(AppState { + component_allowed_models, + }) + } + + fn prepare( + &self, + ctx: PrepareContext, + _builders: &mut InstanceBuilders, + ) -> anyhow::Result { + let allowed_models = ctx + .app_state() + .component_allowed_models + .get(ctx.app_component().id()) + .cloned() + .unwrap_or_default(); + + Ok(LlmDispatch { + engine: (self.create_engine)(), + allowed_models, + }) + } +} + +pub struct AppState { + component_allowed_models: HashMap>>, +} + +pub struct LlmDispatch { + engine: Box, + pub allowed_models: Arc>, +} + +impl SelfInstanceBuilder for LlmDispatch {} + +#[async_trait] +pub trait LlmEngine: Send + Sync { + async fn infer( + &mut self, + model: v1::InferencingModel, + prompt: String, + params: v2::InferencingParams, + ) -> Result; + + async fn generate_embeddings( + &mut self, + model: v2::EmbeddingModel, + data: Vec, + ) -> Result; +} diff --git a/crates/factor-llm/tests/factor.rs b/crates/factor-llm/tests/factor.rs new file mode 100644 index 000000000..4fd15ff4e --- /dev/null +++ b/crates/factor-llm/tests/factor.rs @@ -0,0 +1,65 @@ +use std::collections::HashSet; + +use factor_llm::{LlmEngine, LlmFactor}; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use spin_world::v1::llm::{self as v1}; +use spin_world::v2::llm::{self as v2, Host}; + +#[derive(RuntimeFactors)] +struct TestFactors { + llm: LlmFactor, +} + +#[tokio::test] +async fn static_provider_works() -> anyhow::Result<()> { + let factors = TestFactors { + llm: LlmFactor::new(|| Box::new(FakeLLm) as _), + }; + + let env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + ai_models = ["llama2-chat"] + }); + let mut state = env.build_instance_state(factors).await?; + assert_eq!( + &*state.llm.allowed_models, + &["llama2-chat".to_owned()] + .into_iter() + .collect::>() + ); + + assert!(matches!( + state + .llm + .infer("no-model".into(), "some prompt".into(), Default::default()) + .await, + Err(v2::Error::InvalidInput(msg)) if msg.contains("The component does not have access to use") + )); + Ok(()) +} + +struct FakeLLm; + +#[async_trait::async_trait] +impl LlmEngine for FakeLLm { + async fn infer( + &mut self, + model: v1::InferencingModel, + prompt: String, + params: v2::InferencingParams, + ) -> Result { + let _ = (model, prompt, params); + todo!() + } + + async fn generate_embeddings( + &mut self, + model: v2::EmbeddingModel, + data: Vec, + ) -> Result { + let _ = (model, data); + todo!() + } +} From 030e0ff33420594ae41d20336437a8f95bf260bd Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 11 Jul 2024 21:15:42 +0200 Subject: [PATCH 032/195] PR feedback Signed-off-by: Ryan Levick --- crates/factor-llm/src/host.rs | 6 +++--- crates/factor-llm/src/lib.rs | 8 ++++---- crates/factor-llm/tests/factor.rs | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/factor-llm/src/host.rs b/crates/factor-llm/src/host.rs index 2eded3b7e..748f97b1a 100644 --- a/crates/factor-llm/src/host.rs +++ b/crates/factor-llm/src/host.rs @@ -2,10 +2,10 @@ use async_trait::async_trait; use spin_world::v1::llm::{self as v1}; use spin_world::v2::llm::{self as v2}; -use crate::LlmDispatch; +use crate::InstanceState; #[async_trait] -impl v2::Host for LlmDispatch { +impl v2::Host for InstanceState { async fn infer( &mut self, model: v2::InferencingModel, @@ -48,7 +48,7 @@ impl v2::Host for LlmDispatch { } #[async_trait] -impl v1::Host for LlmDispatch { +impl v1::Host for InstanceState { async fn infer( &mut self, model: v1::InferencingModel, diff --git a/crates/factor-llm/src/lib.rs b/crates/factor-llm/src/lib.rs index 9d922929d..13899ec3b 100644 --- a/crates/factor-llm/src/lib.rs +++ b/crates/factor-llm/src/lib.rs @@ -32,7 +32,7 @@ impl LlmFactor { impl Factor for LlmFactor { type RuntimeConfig = (); type AppState = AppState; - type InstanceBuilder = LlmDispatch; + type InstanceBuilder = InstanceState; fn init( &mut self, @@ -79,7 +79,7 @@ impl Factor for LlmFactor { .cloned() .unwrap_or_default(); - Ok(LlmDispatch { + Ok(InstanceState { engine: (self.create_engine)(), allowed_models, }) @@ -90,12 +90,12 @@ pub struct AppState { component_allowed_models: HashMap>>, } -pub struct LlmDispatch { +pub struct InstanceState { engine: Box, pub allowed_models: Arc>, } -impl SelfInstanceBuilder for LlmDispatch {} +impl SelfInstanceBuilder for InstanceState {} #[async_trait] pub trait LlmEngine: Send + Sync { diff --git a/crates/factor-llm/tests/factor.rs b/crates/factor-llm/tests/factor.rs index 4fd15ff4e..da074d33f 100644 --- a/crates/factor-llm/tests/factor.rs +++ b/crates/factor-llm/tests/factor.rs @@ -12,7 +12,7 @@ struct TestFactors { } #[tokio::test] -async fn static_provider_works() -> anyhow::Result<()> { +async fn llm_works() -> anyhow::Result<()> { let factors = TestFactors { llm: LlmFactor::new(|| Box::new(FakeLLm) as _), }; From 5fd07dacfd3630ab1869d44c2de38dfc743aefcf Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 11 Jul 2024 16:41:29 -0400 Subject: [PATCH 033/195] factors: Port wasi-http RCs to OutboundHttpFactor And add a simple test. Signed-off-by: Lann Martin --- Cargo.lock | 4 + crates/factor-outbound-http/Cargo.toml | 6 + crates/factor-outbound-http/src/lib.rs | 2 + crates/factor-outbound-http/src/wasi.rs | 35 +- .../src/wasi_2023_10_18.rs | 630 +++++++++++++++ .../src/wasi_2023_11_10.rs | 765 ++++++++++++++++++ .../factor-outbound-http/tests/factor_test.rs | 59 ++ crates/factor-wasi/src/lib.rs | 4 +- crates/factor-wasi/src/wasi_2023_10_18.rs | 13 +- crates/factor-wasi/src/wasi_2023_11_10.rs | 6 +- crates/factors/tests/smoke.rs | 3 +- 11 files changed, 1497 insertions(+), 30 deletions(-) create mode 100644 crates/factor-outbound-http/src/wasi_2023_10_18.rs create mode 100644 crates/factor-outbound-http/src/wasi_2023_11_10.rs create mode 100644 crates/factor-outbound-http/tests/factor_test.rs diff --git a/Cargo.lock b/Cargo.lock index 8dda2cc27..823b49257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7536,10 +7536,14 @@ dependencies = [ "anyhow", "http 1.1.0", "spin-factor-outbound-networking", + "spin-factor-variables", "spin-factor-wasi", "spin-factors", + "spin-factors-test", "spin-world", + "tokio", "tracing", + "wasmtime", "wasmtime-wasi-http", ] diff --git a/crates/factor-outbound-http/Cargo.toml b/crates/factor-outbound-http/Cargo.toml index e922255e3..7cf2044d8 100644 --- a/crates/factor-outbound-http/Cargo.toml +++ b/crates/factor-outbound-http/Cargo.toml @@ -12,7 +12,13 @@ spin-factor-wasi = { path = "../factor-wasi" } spin-factors = { path = "../factors" } spin-world = { path = "../world" } tracing = { workspace = true } +wasmtime = { workspace = true } wasmtime-wasi-http = { workspace = true } +[dev-dependencies] +spin-factor-variables = { path = "../factor-variables" } +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + [lints] workspace = true diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 36000239b..fd9344dcc 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -1,5 +1,7 @@ mod spin; mod wasi; +mod wasi_2023_10_18; +mod wasi_2023_11_10; use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; use spin_factors::{ diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index dd6b5415f..834a619dd 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -1,23 +1,27 @@ +use http::Request; use spin_factors::{GetFactorState, Linker, RuntimeFactors}; -use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; +use wasmtime_wasi_http::{bindings::http::types::ErrorCode, WasiHttpImpl, WasiHttpView}; + +use crate::{wasi_2023_10_18, wasi_2023_11_10}; pub(crate) fn add_to_linker(linker: &mut Linker) -> anyhow::Result<()> { fn type_annotate(f: F) -> F where - F: Fn(&mut T) -> WasiHttpImpl>, + F: Fn(&mut T) -> WasiHttpImpl>, { f } - let host_getter = type_annotate(move |data| WasiHttpImpl(MutStates { inner: data })); - wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker_get_host( - linker, - host_getter, - )?; - wasmtime_wasi_http::bindings::http::types::add_to_linker_get_host(linker, host_getter)?; + let closure = type_annotate(move |data| WasiHttpImpl(MutStates { inner: data })); + wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; + wasmtime_wasi_http::bindings::http::types::add_to_linker_get_host(linker, closure)?; + + wasi_2023_10_18::add_to_linker(linker, closure)?; + wasi_2023_11_10::add_to_linker(linker, closure)?; + Ok(()) } -struct MutStates<'a, T> { +pub(crate) struct MutStates<'a, T> { inner: &'a mut T, } @@ -39,11 +43,20 @@ where .expect("failed to get `WasiFactor`") .table() } + + fn send_request( + &mut self, + _request: Request, + _config: wasmtime_wasi_http::types::OutgoingRequestConfig, + ) -> wasmtime_wasi_http::HttpResult { + // TODO: port implementation from spin-trigger-http + Err(ErrorCode::HttpRequestDenied.into()) + } } // TODO: This is a little weird, organizationally -pub fn get_wasi_http_view( - instance_state: &mut T::InstanceState, +pub fn get_wasi_http_view( + instance_state: &mut T, ) -> impl WasiHttpView + '_ { MutStates { inner: instance_state, diff --git a/crates/factor-outbound-http/src/wasi_2023_10_18.rs b/crates/factor-outbound-http/src/wasi_2023_10_18.rs new file mode 100644 index 000000000..f99025d1b --- /dev/null +++ b/crates/factor-outbound-http/src/wasi_2023_10_18.rs @@ -0,0 +1,630 @@ +use anyhow::Result; +use spin_factors::GetFactorState; +use wasmtime::component::{Linker, Resource}; +use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; + +mod latest { + pub use wasmtime_wasi_http::bindings::wasi::*; + pub mod http { + pub use wasmtime_wasi_http::bindings::http::*; + } +} + +mod bindings { + use super::latest; + + wasmtime::component::bindgen!({ + path: "../../wit", + interfaces: r#" + include wasi:http/proxy@0.2.0-rc-2023-10-18; + "#, + with: { + "wasi:io/poll/pollable": latest::io::poll::Pollable, + "wasi:io/streams/input-stream": latest::io::streams::InputStream, + "wasi:io/streams/output-stream": latest::io::streams::OutputStream, + "wasi:io/streams/error": latest::io::streams::Error, + "wasi:http/types/incoming-response": latest::http::types::IncomingResponse, + "wasi:http/types/incoming-request": latest::http::types::IncomingRequest, + "wasi:http/types/incoming-body": latest::http::types::IncomingBody, + "wasi:http/types/outgoing-response": latest::http::types::OutgoingResponse, + "wasi:http/types/outgoing-request": latest::http::types::OutgoingRequest, + "wasi:http/types/outgoing-body": latest::http::types::OutgoingBody, + "wasi:http/types/fields": latest::http::types::Fields, + "wasi:http/types/response-outparam": latest::http::types::ResponseOutparam, + "wasi:http/types/future-incoming-response": latest::http::types::FutureIncomingResponse, + "wasi:http/types/future-trailers": latest::http::types::FutureTrailers, + }, + trappable_imports: true, + }); +} + +mod wasi { + pub use super::bindings::wasi::{http0_2_0_rc_2023_10_18 as http, io0_2_0_rc_2023_10_18 as io}; +} + +use wasi::http::types::{ + Error as HttpError, Fields, FutureIncomingResponse, FutureTrailers, Headers, IncomingBody, + IncomingRequest, IncomingResponse, Method, OutgoingBody, OutgoingRequest, OutgoingResponse, + RequestOptions, ResponseOutparam, Scheme, StatusCode, Trailers, +}; +use wasi::io::poll::Pollable; +use wasi::io::streams::{InputStream, OutputStream}; + +use crate::wasi::MutStates; + +pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> +where + T: GetFactorState + Send, + F: Fn(&mut T) -> WasiHttpImpl> + Send + Sync + Copy + 'static, +{ + wasi::http::types::add_to_linker_get_host(linker, closure)?; + wasi::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; + Ok(()) +} + +impl wasi::http::types::Host for WasiHttpImpl where T: WasiHttpView + Send {} + +impl wasi::http::types::HostFields for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn new( + &mut self, + entries: Vec<(String, Vec)>, + ) -> wasmtime::Result> { + match latest::http::types::HostFields::from_list(self, entries)? { + Ok(fields) => Ok(fields), + Err(e) => Err(e.into()), + } + } + + fn get( + &mut self, + self_: wasmtime::component::Resource, + name: String, + ) -> wasmtime::Result>> { + latest::http::types::HostFields::get(self, self_, name) + } + + fn set( + &mut self, + self_: wasmtime::component::Resource, + name: String, + value: Vec>, + ) -> wasmtime::Result<()> { + latest::http::types::HostFields::set(self, self_, name, value)??; + Ok(()) + } + + fn delete( + &mut self, + self_: wasmtime::component::Resource, + name: String, + ) -> wasmtime::Result<()> { + latest::http::types::HostFields::delete(self, self_, name)??; + Ok(()) + } + + fn append( + &mut self, + self_: wasmtime::component::Resource, + name: String, + value: Vec, + ) -> wasmtime::Result<()> { + latest::http::types::HostFields::append(self, self_, name, value)??; + Ok(()) + } + + fn entries( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result)>> { + latest::http::types::HostFields::entries(self, self_) + } + + fn clone( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostFields::clone(self, self_) + } + + fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { + latest::http::types::HostFields::drop(self, rep) + } +} + +impl wasi::http::types::HostIncomingRequest for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn method( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result { + latest::http::types::HostIncomingRequest::method(self, self_).map(|e| e.into()) + } + + fn path_with_query( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::path_with_query(self, self_) + } + + fn scheme( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::scheme(self, self_).map(|e| e.map(|e| e.into())) + } + + fn authority( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::authority(self, self_) + } + + fn headers( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::headers(self, self_) + } + + fn consume( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostIncomingRequest::consume(self, self_) + } + + fn drop( + &mut self, + rep: wasmtime::component::Resource, + ) -> wasmtime::Result<()> { + latest::http::types::HostIncomingRequest::drop(self, rep) + } +} + +impl wasi::http::types::HostIncomingResponse for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn status( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result { + latest::http::types::HostIncomingResponse::status(self, self_) + } + + fn headers( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingResponse::headers(self, self_) + } + + fn consume( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostIncomingResponse::consume(self, self_) + } + + fn drop( + &mut self, + rep: wasmtime::component::Resource, + ) -> wasmtime::Result<()> { + latest::http::types::HostIncomingResponse::drop(self, rep) + } +} + +impl wasi::http::types::HostIncomingBody for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn stream( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostIncomingBody::stream(self, self_) + } + + fn finish( + &mut self, + this: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingBody::finish(self, this) + } + + fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { + latest::http::types::HostIncomingBody::drop(self, rep) + } +} + +impl wasi::http::types::HostOutgoingRequest for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn new( + &mut self, + method: Method, + path_with_query: Option, + scheme: Option, + authority: Option, + headers: wasmtime::component::Resource, + ) -> wasmtime::Result> { + let headers = latest::http::types::HostFields::clone(self, headers)?; + let request = latest::http::types::HostOutgoingRequest::new(self, headers)?; + let borrow = || Resource::new_borrow(request.rep()); + + if let Err(()) = + latest::http::types::HostOutgoingRequest::set_method(self, borrow(), method.into())? + { + latest::http::types::HostOutgoingRequest::drop(self, request)?; + anyhow::bail!("invalid method supplied"); + } + + if let Err(()) = latest::http::types::HostOutgoingRequest::set_path_with_query( + self, + borrow(), + path_with_query, + )? { + latest::http::types::HostOutgoingRequest::drop(self, request)?; + anyhow::bail!("invalid path-with-query supplied"); + } + + // Historical WASI would fill in an empty authority with a port which + // got just enough working to get things through. Current WASI requires + // the authority, though, so perform the translation manually here. + let authority = authority.unwrap_or_else(|| match &scheme { + Some(Scheme::Http) | Some(Scheme::Other(_)) => ":80".to_string(), + Some(Scheme::Https) | None => ":443".to_string(), + }); + if let Err(()) = latest::http::types::HostOutgoingRequest::set_scheme( + self, + borrow(), + scheme.map(|s| s.into()), + )? { + latest::http::types::HostOutgoingRequest::drop(self, request)?; + anyhow::bail!("invalid scheme supplied"); + } + + if let Err(()) = latest::http::types::HostOutgoingRequest::set_authority( + self, + borrow(), + Some(authority), + )? { + latest::http::types::HostOutgoingRequest::drop(self, request)?; + anyhow::bail!("invalid authority supplied"); + } + + Ok(request) + } + + fn write( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostOutgoingRequest::body(self, self_) + } + + fn drop( + &mut self, + rep: wasmtime::component::Resource, + ) -> wasmtime::Result<()> { + latest::http::types::HostOutgoingRequest::drop(self, rep) + } +} + +impl wasi::http::types::HostOutgoingResponse for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn new( + &mut self, + status_code: StatusCode, + headers: wasmtime::component::Resource, + ) -> wasmtime::Result> { + let headers = latest::http::types::HostFields::clone(self, headers)?; + let response = latest::http::types::HostOutgoingResponse::new(self, headers)?; + let borrow = || Resource::new_borrow(response.rep()); + + if let Err(()) = + latest::http::types::HostOutgoingResponse::set_status_code(self, borrow(), status_code)? + { + latest::http::types::HostOutgoingResponse::drop(self, response)?; + anyhow::bail!("invalid status code supplied"); + } + + Ok(response) + } + + fn write( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostOutgoingResponse::body(self, self_) + } + + fn drop( + &mut self, + rep: wasmtime::component::Resource, + ) -> wasmtime::Result<()> { + latest::http::types::HostOutgoingResponse::drop(self, rep) + } +} + +impl wasi::http::types::HostOutgoingBody for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn write( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostOutgoingBody::write(self, self_) + } + + fn finish( + &mut self, + this: wasmtime::component::Resource, + trailers: Option>, + ) -> wasmtime::Result<()> { + latest::http::types::HostOutgoingBody::finish(self, this, trailers)?; + Ok(()) + } + + fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { + latest::http::types::HostOutgoingBody::drop(self, rep) + } +} + +impl wasi::http::types::HostResponseOutparam for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn set( + &mut self, + param: wasmtime::component::Resource, + response: Result, HttpError>, + ) -> wasmtime::Result<()> { + let response = response.map_err(|err| { + // TODO: probably need to figure out a better mapping between + // errors, but that seems like it would require string matching, + // which also seems not great. + let msg = match err { + HttpError::InvalidUrl(s) => format!("invalid url: {s}"), + HttpError::TimeoutError(s) => format!("timeout: {s}"), + HttpError::ProtocolError(s) => format!("protocol error: {s}"), + HttpError::UnexpectedError(s) => format!("unexpected error: {s}"), + }; + latest::http::types::ErrorCode::InternalError(Some(msg)) + }); + latest::http::types::HostResponseOutparam::set(self, param, response) + } + + fn drop( + &mut self, + rep: wasmtime::component::Resource, + ) -> wasmtime::Result<()> { + latest::http::types::HostResponseOutparam::drop(self, rep) + } +} + +impl wasi::http::types::HostFutureTrailers for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn subscribe( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostFutureTrailers::subscribe(self, self_) + } + + fn get( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result, HttpError>>> { + match latest::http::types::HostFutureTrailers::get(self, self_)? { + Some(Ok(Ok(Some(trailers)))) => Ok(Some(Ok(trailers))), + // Return an empty trailers if no trailers popped out since this + // version of WASI couldn't represent the lack of trailers. + Some(Ok(Ok(None))) => Ok(Some(Ok(latest::http::types::HostFields::new(self)?))), + Some(Ok(Err(e))) => Ok(Some(Err(e.into()))), + Some(Err(())) => Err(anyhow::anyhow!("trailers have already been retrieved")), + None => Ok(None), + } + } + + fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { + latest::http::types::HostFutureTrailers::drop(self, rep) + } +} + +impl wasi::http::types::HostFutureIncomingResponse for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn get( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result< + Option, HttpError>, ()>>, + > { + match latest::http::types::HostFutureIncomingResponse::get(self, self_)? { + None => Ok(None), + Some(Ok(Ok(response))) => Ok(Some(Ok(Ok(response)))), + Some(Ok(Err(e))) => Ok(Some(Ok(Err(e.into())))), + Some(Err(())) => Ok(Some(Err(()))), + } + } + + fn subscribe( + &mut self, + self_: wasmtime::component::Resource, + ) -> wasmtime::Result> { + latest::http::types::HostFutureIncomingResponse::subscribe(self, self_) + } + + fn drop( + &mut self, + rep: wasmtime::component::Resource, + ) -> wasmtime::Result<()> { + latest::http::types::HostFutureIncomingResponse::drop(self, rep) + } +} + +impl wasi::http::outgoing_handler::Host for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn handle( + &mut self, + request: wasmtime::component::Resource, + options: Option, + ) -> wasmtime::Result, HttpError>> + { + let options = match options { + Some(RequestOptions { + connect_timeout_ms, + first_byte_timeout_ms, + between_bytes_timeout_ms, + }) => { + let options = latest::http::types::HostRequestOptions::new(self)?; + let borrow = || Resource::new_borrow(request.rep()); + + if let Some(ms) = connect_timeout_ms { + if let Err(()) = latest::http::types::HostRequestOptions::set_connect_timeout( + self, + borrow(), + Some(ms.into()), + )? { + latest::http::types::HostRequestOptions::drop(self, options)?; + anyhow::bail!("invalid connect timeout supplied"); + } + } + + if let Some(ms) = first_byte_timeout_ms { + if let Err(()) = + latest::http::types::HostRequestOptions::set_first_byte_timeout( + self, + borrow(), + Some(ms.into()), + )? + { + latest::http::types::HostRequestOptions::drop(self, options)?; + anyhow::bail!("invalid first byte timeout supplied"); + } + } + + if let Some(ms) = between_bytes_timeout_ms { + if let Err(()) = + latest::http::types::HostRequestOptions::set_between_bytes_timeout( + self, + borrow(), + Some(ms.into()), + )? + { + latest::http::types::HostRequestOptions::drop(self, options)?; + anyhow::bail!("invalid between bytes timeout supplied"); + } + } + + Some(options) + } + None => None, + }; + match latest::http::outgoing_handler::Host::handle(self, request, options) { + Ok(resp) => Ok(Ok(resp)), + Err(e) => Ok(Err(e.downcast()?.into())), + } + } +} + +macro_rules! convert { + () => {}; + ($kind:ident $from:path [<=>] $to:path { $($body:tt)* } $($rest:tt)*) => { + convert!($kind $from => $to { $($body)* }); + convert!($kind $to => $from { $($body)* }); + + convert!($($rest)*); + }; + (struct $from:ty => $to:path { $($field:ident,)* } $($rest:tt)*) => { + impl From<$from> for $to { + fn from(e: $from) -> $to { + $to { + $( $field: e.$field.into(), )* + } + } + } + + convert!($($rest)*); + }; + (enum $from:path => $to:path { $($variant:ident $(($e:ident))?,)* } $($rest:tt)*) => { + impl From<$from> for $to { + fn from(e: $from) -> $to { + use $from as A; + use $to as B; + match e { + $( + A::$variant $(($e))? => B::$variant $(($e.into()))?, + )* + } + } + } + + convert!($($rest)*); + }; + (flags $from:path => $to:path { $($flag:ident,)* } $($rest:tt)*) => { + impl From<$from> for $to { + fn from(e: $from) -> $to { + use $from as A; + use $to as B; + let mut out = B::empty(); + $( + if e.contains(A::$flag) { + out |= B::$flag; + } + )* + out + } + } + + convert!($($rest)*); + }; +} + +pub(crate) use convert; + +convert! { + enum latest::http::types::Method [<=>] Method { + Get, + Head, + Post, + Put, + Delete, + Connect, + Options, + Trace, + Patch, + Other(e), + } + + enum latest::http::types::Scheme [<=>] Scheme { + Http, + Https, + Other(e), + } +} + +impl From for HttpError { + fn from(e: latest::http::types::ErrorCode) -> HttpError { + // TODO: should probably categorize this better given the typed info + // we have in `e`. + HttpError::UnexpectedError(e.to_string()) + } +} diff --git a/crates/factor-outbound-http/src/wasi_2023_11_10.rs b/crates/factor-outbound-http/src/wasi_2023_11_10.rs new file mode 100644 index 000000000..5295ed86a --- /dev/null +++ b/crates/factor-outbound-http/src/wasi_2023_11_10.rs @@ -0,0 +1,765 @@ +#![doc(hidden)] // internal implementation detail used in tests and spin-trigger + +use super::wasi_2023_10_18::convert; +use anyhow::Result; +use spin_factors::GetFactorState; +use wasmtime::component::{Linker, Resource}; +use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; + +mod latest { + pub use wasmtime_wasi_http::bindings::wasi::*; + pub mod http { + pub use wasmtime_wasi_http::bindings::http::*; + } +} + +mod bindings { + use super::latest; + + wasmtime::component::bindgen!({ + path: "../../wit", + interfaces: r#" + include wasi:http/proxy@0.2.0-rc-2023-11-10; + "#, + with: { + "wasi:io/poll/pollable": latest::io::poll::Pollable, + "wasi:io/streams/input-stream": latest::io::streams::InputStream, + "wasi:io/streams/output-stream": latest::io::streams::OutputStream, + "wasi:io/error/error": latest::io::error::Error, + "wasi:http/types/incoming-response": latest::http::types::IncomingResponse, + "wasi:http/types/incoming-request": latest::http::types::IncomingRequest, + "wasi:http/types/incoming-body": latest::http::types::IncomingBody, + "wasi:http/types/outgoing-response": latest::http::types::OutgoingResponse, + "wasi:http/types/outgoing-request": latest::http::types::OutgoingRequest, + "wasi:http/types/outgoing-body": latest::http::types::OutgoingBody, + "wasi:http/types/fields": latest::http::types::Fields, + "wasi:http/types/response-outparam": latest::http::types::ResponseOutparam, + "wasi:http/types/future-incoming-response": latest::http::types::FutureIncomingResponse, + "wasi:http/types/future-trailers": latest::http::types::FutureTrailers, + "wasi:http/types/request-options": latest::http::types::RequestOptions, + }, + trappable_imports: true, + }); +} + +mod wasi { + pub use super::bindings::wasi::{http0_2_0_rc_2023_11_10 as http, io0_2_0_rc_2023_11_10 as io}; +} + +use wasi::http::types::{ + DnsErrorPayload, ErrorCode as HttpErrorCode, FieldSizePayload, Fields, FutureIncomingResponse, + FutureTrailers, HeaderError, Headers, IncomingBody, IncomingRequest, IncomingResponse, Method, + OutgoingBody, OutgoingRequest, OutgoingResponse, RequestOptions, ResponseOutparam, Scheme, + StatusCode, TlsAlertReceivedPayload, Trailers, +}; +use wasi::io::poll::Pollable; +use wasi::io::streams::{Error as IoError, InputStream, OutputStream}; + +use crate::wasi::MutStates; + +pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> +where + T: GetFactorState + Send, + F: Fn(&mut T) -> WasiHttpImpl> + Send + Sync + Copy + 'static, +{ + wasi::http::types::add_to_linker_get_host(linker, closure)?; + wasi::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; + Ok(()) +} + +impl wasi::http::types::Host for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn http_error_code( + &mut self, + error: Resource, + ) -> wasmtime::Result> { + latest::http::types::Host::http_error_code(self, error).map(|e| e.map(|e| e.into())) + } +} + +impl wasi::http::types::HostRequestOptions for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn new(&mut self) -> wasmtime::Result> { + latest::http::types::HostRequestOptions::new(self) + } + + fn connect_timeout_ms( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostRequestOptions::connect_timeout(self, self_) + } + + fn set_connect_timeout_ms( + &mut self, + self_: Resource, + duration: Option, + ) -> wasmtime::Result> { + latest::http::types::HostRequestOptions::set_connect_timeout(self, self_, duration) + } + + fn first_byte_timeout_ms( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostRequestOptions::first_byte_timeout(self, self_) + } + + fn set_first_byte_timeout_ms( + &mut self, + self_: Resource, + duration: Option, + ) -> wasmtime::Result> { + latest::http::types::HostRequestOptions::set_first_byte_timeout(self, self_, duration) + } + + fn between_bytes_timeout_ms( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostRequestOptions::between_bytes_timeout(self, self_) + } + + fn set_between_bytes_timeout_ms( + &mut self, + self_: Resource, + duration: Option, + ) -> wasmtime::Result> { + latest::http::types::HostRequestOptions::set_between_bytes_timeout(self, self_, duration) + } + + fn drop(&mut self, self_: Resource) -> wasmtime::Result<()> { + latest::http::types::HostRequestOptions::drop(self, self_) + } +} + +impl wasi::http::types::HostFields for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn new(&mut self) -> wasmtime::Result> { + latest::http::types::HostFields::new(self) + } + + fn from_list( + &mut self, + entries: Vec<(String, Vec)>, + ) -> wasmtime::Result, HeaderError>> { + latest::http::types::HostFields::from_list(self, entries).map(|r| r.map_err(|e| e.into())) + } + + fn get(&mut self, self_: Resource, name: String) -> wasmtime::Result>> { + latest::http::types::HostFields::get(self, self_, name) + } + + fn set( + &mut self, + self_: Resource, + name: String, + value: Vec>, + ) -> wasmtime::Result> { + latest::http::types::HostFields::set(self, self_, name, value) + .map(|r| r.map_err(|e| e.into())) + } + + fn delete( + &mut self, + self_: Resource, + name: String, + ) -> wasmtime::Result> { + latest::http::types::HostFields::delete(self, self_, name).map(|r| r.map_err(|e| e.into())) + } + + fn append( + &mut self, + self_: Resource, + name: String, + value: Vec, + ) -> wasmtime::Result> { + latest::http::types::HostFields::append(self, self_, name, value) + .map(|r| r.map_err(|e| e.into())) + } + + fn entries(&mut self, self_: Resource) -> wasmtime::Result)>> { + latest::http::types::HostFields::entries(self, self_) + } + + fn clone(&mut self, self_: Resource) -> wasmtime::Result> { + latest::http::types::HostFields::clone(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostFields::drop(self, rep) + } +} + +impl wasi::http::types::HostIncomingRequest for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn method(&mut self, self_: Resource) -> wasmtime::Result { + latest::http::types::HostIncomingRequest::method(self, self_).map(|e| e.into()) + } + + fn path_with_query( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::path_with_query(self, self_) + } + + fn scheme(&mut self, self_: Resource) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::scheme(self, self_).map(|e| e.map(|e| e.into())) + } + + fn authority(&mut self, self_: Resource) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::authority(self, self_) + } + + fn headers(&mut self, self_: Resource) -> wasmtime::Result> { + latest::http::types::HostIncomingRequest::headers(self, self_) + } + + fn consume( + &mut self, + self_: Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostIncomingRequest::consume(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostIncomingRequest::drop(self, rep) + } +} + +impl wasi::http::types::HostIncomingResponse for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn status(&mut self, self_: Resource) -> wasmtime::Result { + latest::http::types::HostIncomingResponse::status(self, self_) + } + + fn headers( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingResponse::headers(self, self_) + } + + fn consume( + &mut self, + self_: Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostIncomingResponse::consume(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostIncomingResponse::drop(self, rep) + } +} + +impl wasi::http::types::HostIncomingBody for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn stream( + &mut self, + self_: Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostIncomingBody::stream(self, self_) + } + + fn finish( + &mut self, + this: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostIncomingBody::finish(self, this) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostIncomingBody::drop(self, rep) + } +} + +impl wasi::http::types::HostOutgoingRequest for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn new(&mut self, headers: Resource) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::new(self, headers) + } + + fn method(&mut self, self_: Resource) -> wasmtime::Result { + latest::http::types::HostOutgoingRequest::method(self, self_).map(|m| m.into()) + } + + fn set_method( + &mut self, + self_: Resource, + method: Method, + ) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::set_method(self, self_, method.into()) + } + + fn path_with_query( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::path_with_query(self, self_) + } + + fn set_path_with_query( + &mut self, + self_: Resource, + path_with_query: Option, + ) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::set_path_with_query(self, self_, path_with_query) + } + + fn scheme(&mut self, self_: Resource) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::scheme(self, self_).map(|s| s.map(|s| s.into())) + } + + fn set_scheme( + &mut self, + self_: Resource, + scheme: Option, + ) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::set_scheme(self, self_, scheme.map(|s| s.into())) + } + + fn authority(&mut self, self_: Resource) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::authority(self, self_) + } + + fn set_authority( + &mut self, + self_: Resource, + authority: Option, + ) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::set_authority(self, self_, authority) + } + + fn headers(&mut self, self_: Resource) -> wasmtime::Result> { + latest::http::types::HostOutgoingRequest::headers(self, self_) + } + + fn body( + &mut self, + self_: Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostOutgoingRequest::body(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostOutgoingRequest::drop(self, rep) + } +} + +impl wasi::http::types::HostOutgoingResponse for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn new(&mut self, headers: Resource) -> wasmtime::Result> { + let headers = latest::http::types::HostFields::clone(self, headers)?; + latest::http::types::HostOutgoingResponse::new(self, headers) + } + + fn status_code(&mut self, self_: Resource) -> wasmtime::Result { + latest::http::types::HostOutgoingResponse::status_code(self, self_) + } + + fn set_status_code( + &mut self, + self_: Resource, + status_code: StatusCode, + ) -> wasmtime::Result> { + latest::http::types::HostOutgoingResponse::set_status_code(self, self_, status_code) + } + + fn headers( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostOutgoingResponse::headers(self, self_) + } + + fn body( + &mut self, + self_: Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostOutgoingResponse::body(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostOutgoingResponse::drop(self, rep) + } +} + +impl wasi::http::types::HostOutgoingBody for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn write( + &mut self, + self_: Resource, + ) -> wasmtime::Result, ()>> { + latest::http::types::HostOutgoingBody::write(self, self_) + } + + fn finish( + &mut self, + this: Resource, + trailers: Option>, + ) -> wasmtime::Result> { + match latest::http::types::HostOutgoingBody::finish(self, this, trailers) { + Ok(()) => Ok(Ok(())), + Err(e) => Ok(Err(e.downcast()?.into())), + } + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostOutgoingBody::drop(self, rep) + } +} + +impl wasi::http::types::HostResponseOutparam for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn set( + &mut self, + param: Resource, + response: Result, HttpErrorCode>, + ) -> wasmtime::Result<()> { + latest::http::types::HostResponseOutparam::set(self, param, response.map_err(|e| e.into())) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostResponseOutparam::drop(self, rep) + } +} + +impl wasi::http::types::HostFutureTrailers for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn subscribe( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostFutureTrailers::subscribe(self, self_) + } + + fn get( + &mut self, + self_: Resource, + ) -> wasmtime::Result>, HttpErrorCode>>> { + match latest::http::types::HostFutureTrailers::get(self, self_)? { + Some(Ok(Ok(trailers))) => Ok(Some(Ok(trailers))), + Some(Ok(Err(e))) => Ok(Some(Err(e.into()))), + Some(Err(())) => Err(anyhow::anyhow!("trailers have already been retrieved")), + None => Ok(None), + } + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostFutureTrailers::drop(self, rep) + } +} + +impl wasi::http::types::HostFutureIncomingResponse for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn get( + &mut self, + self_: Resource, + ) -> wasmtime::Result, HttpErrorCode>, ()>>> + { + match latest::http::types::HostFutureIncomingResponse::get(self, self_)? { + None => Ok(None), + Some(Ok(Ok(response))) => Ok(Some(Ok(Ok(response)))), + Some(Ok(Err(e))) => Ok(Some(Ok(Err(e.into())))), + Some(Err(())) => Ok(Some(Err(()))), + } + } + + fn subscribe( + &mut self, + self_: Resource, + ) -> wasmtime::Result> { + latest::http::types::HostFutureIncomingResponse::subscribe(self, self_) + } + + fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { + latest::http::types::HostFutureIncomingResponse::drop(self, rep) + } +} + +impl wasi::http::outgoing_handler::Host for WasiHttpImpl +where + T: WasiHttpView + Send, +{ + fn handle( + &mut self, + request: Resource, + options: Option>, + ) -> wasmtime::Result, HttpErrorCode>> { + match latest::http::outgoing_handler::Host::handle(self, request, options) { + Ok(resp) => Ok(Ok(resp)), + Err(e) => Ok(Err(e.downcast()?.into())), + } + } +} + +convert! { + enum latest::http::types::Method [<=>] Method { + Get, + Head, + Post, + Put, + Delete, + Connect, + Options, + Trace, + Patch, + Other(e), + } + + enum latest::http::types::Scheme [<=>] Scheme { + Http, + Https, + Other(e), + } + + enum latest::http::types::HeaderError => HeaderError { + InvalidSyntax, + Forbidden, + Immutable, + } + + struct latest::http::types::DnsErrorPayload [<=>] DnsErrorPayload { + rcode, + info_code, + } + + struct latest::http::types::TlsAlertReceivedPayload [<=>] TlsAlertReceivedPayload { + alert_id, + alert_message, + } + + struct latest::http::types::FieldSizePayload [<=>] FieldSizePayload { + field_name, + field_size, + } +} + +impl From for HttpErrorCode { + fn from(e: latest::http::types::ErrorCode) -> Self { + match e { + latest::http::types::ErrorCode::DnsTimeout => HttpErrorCode::DnsTimeout, + latest::http::types::ErrorCode::DnsError(e) => HttpErrorCode::DnsError(e.into()), + latest::http::types::ErrorCode::DestinationNotFound => { + HttpErrorCode::DestinationNotFound + } + latest::http::types::ErrorCode::DestinationUnavailable => { + HttpErrorCode::DestinationUnavailable + } + latest::http::types::ErrorCode::DestinationIpProhibited => { + HttpErrorCode::DestinationIpProhibited + } + latest::http::types::ErrorCode::DestinationIpUnroutable => { + HttpErrorCode::DestinationIpUnroutable + } + latest::http::types::ErrorCode::ConnectionRefused => HttpErrorCode::ConnectionRefused, + latest::http::types::ErrorCode::ConnectionTerminated => { + HttpErrorCode::ConnectionTerminated + } + latest::http::types::ErrorCode::ConnectionTimeout => HttpErrorCode::ConnectionTimeout, + latest::http::types::ErrorCode::ConnectionReadTimeout => { + HttpErrorCode::ConnectionReadTimeout + } + latest::http::types::ErrorCode::ConnectionWriteTimeout => { + HttpErrorCode::ConnectionWriteTimeout + } + latest::http::types::ErrorCode::ConnectionLimitReached => { + HttpErrorCode::ConnectionLimitReached + } + latest::http::types::ErrorCode::TlsProtocolError => HttpErrorCode::TlsProtocolError, + latest::http::types::ErrorCode::TlsCertificateError => { + HttpErrorCode::TlsCertificateError + } + latest::http::types::ErrorCode::TlsAlertReceived(e) => { + HttpErrorCode::TlsAlertReceived(e.into()) + } + latest::http::types::ErrorCode::HttpRequestDenied => HttpErrorCode::HttpRequestDenied, + latest::http::types::ErrorCode::HttpRequestLengthRequired => { + HttpErrorCode::HttpRequestLengthRequired + } + latest::http::types::ErrorCode::HttpRequestBodySize(e) => { + HttpErrorCode::HttpRequestBodySize(e) + } + latest::http::types::ErrorCode::HttpRequestMethodInvalid => { + HttpErrorCode::HttpRequestMethodInvalid + } + latest::http::types::ErrorCode::HttpRequestUriInvalid => { + HttpErrorCode::HttpRequestUriInvalid + } + latest::http::types::ErrorCode::HttpRequestUriTooLong => { + HttpErrorCode::HttpRequestUriTooLong + } + latest::http::types::ErrorCode::HttpRequestHeaderSectionSize(e) => { + HttpErrorCode::HttpRequestHeaderSectionSize(e) + } + latest::http::types::ErrorCode::HttpRequestHeaderSize(e) => { + HttpErrorCode::HttpRequestHeaderSize(e.map(|e| e.into())) + } + latest::http::types::ErrorCode::HttpRequestTrailerSectionSize(e) => { + HttpErrorCode::HttpRequestTrailerSectionSize(e) + } + latest::http::types::ErrorCode::HttpRequestTrailerSize(e) => { + HttpErrorCode::HttpRequestTrailerSize(e.into()) + } + latest::http::types::ErrorCode::HttpResponseIncomplete => { + HttpErrorCode::HttpResponseIncomplete + } + latest::http::types::ErrorCode::HttpResponseHeaderSectionSize(e) => { + HttpErrorCode::HttpResponseHeaderSectionSize(e) + } + latest::http::types::ErrorCode::HttpResponseHeaderSize(e) => { + HttpErrorCode::HttpResponseHeaderSize(e.into()) + } + latest::http::types::ErrorCode::HttpResponseBodySize(e) => { + HttpErrorCode::HttpResponseBodySize(e) + } + latest::http::types::ErrorCode::HttpResponseTrailerSectionSize(e) => { + HttpErrorCode::HttpResponseTrailerSectionSize(e) + } + latest::http::types::ErrorCode::HttpResponseTrailerSize(e) => { + HttpErrorCode::HttpResponseTrailerSize(e.into()) + } + latest::http::types::ErrorCode::HttpResponseTransferCoding(e) => { + HttpErrorCode::HttpResponseTransferCoding(e) + } + latest::http::types::ErrorCode::HttpResponseContentCoding(e) => { + HttpErrorCode::HttpResponseContentCoding(e) + } + latest::http::types::ErrorCode::HttpResponseTimeout => { + HttpErrorCode::HttpResponseTimeout + } + latest::http::types::ErrorCode::HttpUpgradeFailed => HttpErrorCode::HttpUpgradeFailed, + latest::http::types::ErrorCode::HttpProtocolError => HttpErrorCode::HttpProtocolError, + latest::http::types::ErrorCode::LoopDetected => HttpErrorCode::LoopDetected, + latest::http::types::ErrorCode::ConfigurationError => HttpErrorCode::ConfigurationError, + latest::http::types::ErrorCode::InternalError(e) => HttpErrorCode::InternalError(e), + } + } +} + +impl From for latest::http::types::ErrorCode { + fn from(e: HttpErrorCode) -> Self { + match e { + HttpErrorCode::DnsTimeout => latest::http::types::ErrorCode::DnsTimeout, + HttpErrorCode::DnsError(e) => latest::http::types::ErrorCode::DnsError(e.into()), + HttpErrorCode::DestinationNotFound => { + latest::http::types::ErrorCode::DestinationNotFound + } + HttpErrorCode::DestinationUnavailable => { + latest::http::types::ErrorCode::DestinationUnavailable + } + HttpErrorCode::DestinationIpProhibited => { + latest::http::types::ErrorCode::DestinationIpProhibited + } + HttpErrorCode::DestinationIpUnroutable => { + latest::http::types::ErrorCode::DestinationIpUnroutable + } + HttpErrorCode::ConnectionRefused => latest::http::types::ErrorCode::ConnectionRefused, + HttpErrorCode::ConnectionTerminated => { + latest::http::types::ErrorCode::ConnectionTerminated + } + HttpErrorCode::ConnectionTimeout => latest::http::types::ErrorCode::ConnectionTimeout, + HttpErrorCode::ConnectionReadTimeout => { + latest::http::types::ErrorCode::ConnectionReadTimeout + } + HttpErrorCode::ConnectionWriteTimeout => { + latest::http::types::ErrorCode::ConnectionWriteTimeout + } + HttpErrorCode::ConnectionLimitReached => { + latest::http::types::ErrorCode::ConnectionLimitReached + } + HttpErrorCode::TlsProtocolError => latest::http::types::ErrorCode::TlsProtocolError, + HttpErrorCode::TlsCertificateError => { + latest::http::types::ErrorCode::TlsCertificateError + } + HttpErrorCode::TlsAlertReceived(e) => { + latest::http::types::ErrorCode::TlsAlertReceived(e.into()) + } + HttpErrorCode::HttpRequestDenied => latest::http::types::ErrorCode::HttpRequestDenied, + HttpErrorCode::HttpRequestLengthRequired => { + latest::http::types::ErrorCode::HttpRequestLengthRequired + } + HttpErrorCode::HttpRequestBodySize(e) => { + latest::http::types::ErrorCode::HttpRequestBodySize(e) + } + HttpErrorCode::HttpRequestMethodInvalid => { + latest::http::types::ErrorCode::HttpRequestMethodInvalid + } + HttpErrorCode::HttpRequestUriInvalid => { + latest::http::types::ErrorCode::HttpRequestUriInvalid + } + HttpErrorCode::HttpRequestUriTooLong => { + latest::http::types::ErrorCode::HttpRequestUriTooLong + } + HttpErrorCode::HttpRequestHeaderSectionSize(e) => { + latest::http::types::ErrorCode::HttpRequestHeaderSectionSize(e) + } + HttpErrorCode::HttpRequestHeaderSize(e) => { + latest::http::types::ErrorCode::HttpRequestHeaderSize(e.map(|e| e.into())) + } + HttpErrorCode::HttpRequestTrailerSectionSize(e) => { + latest::http::types::ErrorCode::HttpRequestTrailerSectionSize(e) + } + HttpErrorCode::HttpRequestTrailerSize(e) => { + latest::http::types::ErrorCode::HttpRequestTrailerSize(e.into()) + } + HttpErrorCode::HttpResponseIncomplete => { + latest::http::types::ErrorCode::HttpResponseIncomplete + } + HttpErrorCode::HttpResponseHeaderSectionSize(e) => { + latest::http::types::ErrorCode::HttpResponseHeaderSectionSize(e) + } + HttpErrorCode::HttpResponseHeaderSize(e) => { + latest::http::types::ErrorCode::HttpResponseHeaderSize(e.into()) + } + HttpErrorCode::HttpResponseBodySize(e) => { + latest::http::types::ErrorCode::HttpResponseBodySize(e) + } + HttpErrorCode::HttpResponseTrailerSectionSize(e) => { + latest::http::types::ErrorCode::HttpResponseTrailerSectionSize(e) + } + HttpErrorCode::HttpResponseTrailerSize(e) => { + latest::http::types::ErrorCode::HttpResponseTrailerSize(e.into()) + } + HttpErrorCode::HttpResponseTransferCoding(e) => { + latest::http::types::ErrorCode::HttpResponseTransferCoding(e) + } + HttpErrorCode::HttpResponseContentCoding(e) => { + latest::http::types::ErrorCode::HttpResponseContentCoding(e) + } + HttpErrorCode::HttpResponseTimeout => { + latest::http::types::ErrorCode::HttpResponseTimeout + } + HttpErrorCode::HttpUpgradeFailed => latest::http::types::ErrorCode::HttpUpgradeFailed, + HttpErrorCode::HttpProtocolError => latest::http::types::ErrorCode::HttpProtocolError, + HttpErrorCode::LoopDetected => latest::http::types::ErrorCode::LoopDetected, + HttpErrorCode::ConfigurationError => latest::http::types::ErrorCode::ConfigurationError, + HttpErrorCode::InternalError(e) => latest::http::types::ErrorCode::InternalError(e), + } + } +} diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs new file mode 100644 index 000000000..0a9655270 --- /dev/null +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -0,0 +1,59 @@ +use std::time::Duration; + +use anyhow::bail; +use http::Request; +use spin_factor_outbound_http::{get_wasi_http_view, OutboundHttpFactor}; +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_variables::VariablesFactor; +use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use wasmtime_wasi_http::{ + bindings::http::types::ErrorCode, types::OutgoingRequestConfig, WasiHttpView, +}; + +#[derive(RuntimeFactors)] +struct TestFactors { + wasi: WasiFactor, + variables: VariablesFactor, + networking: OutboundNetworkingFactor, + http: OutboundHttpFactor, +} + +fn test_env() -> TestEnvironment { + TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + allowed_outbound_hosts = ["http://allowed.test"] + }) +} + +#[tokio::test] +async fn disallowed_host_fails() -> anyhow::Result<()> { + let factors = TestFactors { + wasi: WasiFactor::new(DummyFilesMounter), + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + http: OutboundHttpFactor, + }; + let env = test_env(); + let mut state = env.build_instance_state(factors).await?; + let mut wasi_http = get_wasi_http_view(&mut state); + + let req = Request::get("https://denied.test").body(Default::default())?; + let res = wasi_http.send_request(req, test_request_config()); + let Err(err) = res else { + bail!("expected Err, got Ok"); + }; + assert!(matches!(err.downcast()?, ErrorCode::HttpRequestDenied)); + Ok(()) +} + +fn test_request_config() -> OutgoingRequestConfig { + OutgoingRequestConfig { + use_tls: false, + connect_timeout: Duration::from_secs(60), + first_byte_timeout: Duration::from_secs(60), + between_bytes_timeout: Duration::from_secs(60), + } +} diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index d35c01f33..0868c7df5 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,5 +1,5 @@ -pub mod wasi_2023_10_18; -pub mod wasi_2023_11_10; +mod wasi_2023_10_18; +mod wasi_2023_11_10; use std::{future::Future, net::SocketAddr, path::Path}; diff --git a/crates/factor-wasi/src/wasi_2023_10_18.rs b/crates/factor-wasi/src/wasi_2023_10_18.rs index 5c7d01ad9..43f3cbbaa 100644 --- a/crates/factor-wasi/src/wasi_2023_10_18.rs +++ b/crates/factor-wasi/src/wasi_2023_10_18.rs @@ -1,13 +1,9 @@ -#![doc(hidden)] // internal implementation detail used in tests and spin-trigger - use async_trait::async_trait; use spin_factors::anyhow::{self, Result}; use std::mem; use wasmtime::component::{Linker, Resource}; use wasmtime_wasi::{Pollable, TrappableError, WasiImpl, WasiView}; -use crate::InstanceState; - mod latest { pub use wasmtime_wasi::bindings::*; } @@ -127,18 +123,13 @@ use wasi::sockets::tcp::{ }; use wasi::sockets::udp::Datagram; +use crate::InstanceState; + pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where T: Send, F: Fn(&mut T) -> WasiImpl<&mut InstanceState> + Send + Sync + Copy + 'static, { - fn type_annotate(f: F) -> F - where - F: Fn(&mut T) -> WasiImpl<&mut U>, - { - f - } - let closure = type_annotate(closure); wasi::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; wasi::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; wasi::filesystem::types::add_to_linker_get_host(linker, closure)?; diff --git a/crates/factor-wasi/src/wasi_2023_11_10.rs b/crates/factor-wasi/src/wasi_2023_11_10.rs index 5135f516c..a7131c423 100644 --- a/crates/factor-wasi/src/wasi_2023_11_10.rs +++ b/crates/factor-wasi/src/wasi_2023_11_10.rs @@ -1,13 +1,9 @@ -#![doc(hidden)] // internal implementation detail used in tests and spin-trigger - use super::wasi_2023_10_18::{convert, convert_result}; use async_trait::async_trait; use spin_factors::anyhow::{self, Result}; use wasmtime::component::{Linker, Resource}; use wasmtime_wasi::{WasiImpl, WasiView}; -use crate::InstanceState; - mod latest { pub use wasmtime_wasi::bindings::*; } @@ -120,6 +116,8 @@ use wasi::sockets::udp::{ IncomingDatagram, IncomingDatagramStream, OutgoingDatagram, OutgoingDatagramStream, UdpSocket, }; +use crate::InstanceState; + pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where T: Send, diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index b46da230c..99719a47c 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -80,8 +80,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { // Invoke handler let req = http::Request::get("/").body(Default::default()).unwrap(); - let mut wasi_http_view = - spin_factor_outbound_http::get_wasi_http_view::(store.data_mut()); + let mut wasi_http_view = spin_factor_outbound_http::get_wasi_http_view(store.data_mut()); let request = wasi_http_view.new_incoming_request(req)?; let (response_tx, response_rx) = tokio::sync::oneshot::channel(); let response = wasi_http_view.new_response_outparam(response_tx)?; From 3c4a87cca7c0afbc9620ca02d61202443cb51ea1 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 12 Jul 2024 10:05:07 +0200 Subject: [PATCH 034/195] Add test for llm-factor host impl Signed-off-by: Ryan Levick --- crates/factor-llm/tests/factor.rs | 87 ++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 7 deletions(-) diff --git a/crates/factor-llm/tests/factor.rs b/crates/factor-llm/tests/factor.rs index da074d33f..5337ff25f 100644 --- a/crates/factor-llm/tests/factor.rs +++ b/crates/factor-llm/tests/factor.rs @@ -13,8 +13,34 @@ struct TestFactors { #[tokio::test] async fn llm_works() -> anyhow::Result<()> { + let handle = Box::new(|op| match op { + Operation::Inference { + model, + prompt, + params, + } => { + assert_eq!(model, "llama2-chat"); + assert_eq!(prompt, "some prompt"); + assert_eq!(params.max_tokens, 100); + Ok(v2::InferencingResult { + text: "response".to_owned(), + usage: v2::InferencingUsage { + prompt_token_count: 1, + generated_token_count: 1, + }, + } + .into()) + } + Operation::Embedding { .. } => { + todo!("add test for embeddings") + } + }); let factors = TestFactors { - llm: LlmFactor::new(|| Box::new(FakeLLm) as _), + llm: LlmFactor::new(move || { + Box::new(FakeLLm { + handle: handle.clone(), + }) as _ + }), }; let env = TestEnvironment::default_manifest_extend(toml! { @@ -33,14 +59,51 @@ async fn llm_works() -> anyhow::Result<()> { assert!(matches!( state .llm - .infer("no-model".into(), "some prompt".into(), Default::default()) + .infer("unknown-model".into(), "some prompt".into(), Default::default()) .await, Err(v2::Error::InvalidInput(msg)) if msg.contains("The component does not have access to use") )); + + state + .llm + .infer("llama2-chat".into(), "some prompt".into(), None) + .await?; Ok(()) } -struct FakeLLm; +struct FakeLLm { + handle: Box Result + Sync + Send>, +} + +#[allow(dead_code)] +enum Operation { + Inference { + model: v1::InferencingModel, + prompt: String, + params: v2::InferencingParams, + }, + Embedding { + model: v2::EmbeddingModel, + data: Vec, + }, +} + +enum OperationResult { + Inferencing(v2::InferencingResult), + Embeddings(v2::EmbeddingsResult), +} + +impl From for OperationResult { + fn from(e: v2::EmbeddingsResult) -> Self { + OperationResult::Embeddings(e) + } +} + +impl From for OperationResult { + fn from(i: v2::InferencingResult) -> Self { + OperationResult::Inferencing(i) + } +} #[async_trait::async_trait] impl LlmEngine for FakeLLm { @@ -50,8 +113,15 @@ impl LlmEngine for FakeLLm { prompt: String, params: v2::InferencingParams, ) -> Result { - let _ = (model, prompt, params); - todo!() + let OperationResult::Inferencing(i) = (self.handle)(Operation::Inference { + model, + prompt, + params, + })? + else { + panic!("test incorrectly configured. inferencing operation returned embeddings result") + }; + Ok(i) } async fn generate_embeddings( @@ -59,7 +129,10 @@ impl LlmEngine for FakeLLm { model: v2::EmbeddingModel, data: Vec, ) -> Result { - let _ = (model, data); - todo!() + let OperationResult::Embeddings(e) = (self.handle)(Operation::Embedding { model, data })? + else { + panic!("test incorrectly configured. embeddings operation returned inferencing result") + }; + Ok(e) } } From 59ec44bb64fdfaab517643a772f737c0abb57197 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 12 Jul 2024 13:54:33 -0400 Subject: [PATCH 035/195] factors: Add Error type Signed-off-by: Lann Martin --- crates/factors-derive/src/lib.rs | 13 ++++--- crates/factors-test/src/lib.rs | 2 +- crates/factors/src/factor.rs | 15 ++++---- crates/factors/src/lib.rs | 49 +++++++++++++++++++++++++-- crates/factors/src/prepare.rs | 17 ++++++---- crates/factors/src/runtime_config.rs | 27 ++++++--------- crates/factors/src/runtime_factors.rs | 6 ++-- 7 files changed, 87 insertions(+), 42 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index a9b24a03e..305795887 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -68,6 +68,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let ConfiguredApp = quote!(#factors_path::ConfiguredApp); let RuntimeConfigTracker = quote!(#factors_path::__internal::RuntimeConfigTracker); let FactorInstanceBuilder = quote!(#factors_path::FactorInstanceBuilder); + let runtime_factor_error = quote!(#factors_path::__internal::runtime_factor_error); Ok(quote! { impl #factors_path::RuntimeFactors for #name { @@ -87,7 +88,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { linker, |state| &mut state.#factor_names, ) - )?; + ).map_err(|err| #runtime_factor_error::<#factor_types>("init", err))?; )* Ok(()) } @@ -110,7 +111,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &app_state, &mut runtime_config_tracker, )?, - )? + ).map_err(|err| #runtime_factor_error::<#factor_types>("configure_app", err))? ); )* runtime_config_tracker.validate_all_keys_used()?; @@ -122,7 +123,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { component_id: &str, ) -> #Result { let app_component = configured_app.app().get_component(component_id).ok_or_else(|| { - #wasmtime::Error::msg("unknown component") + #factors_path::Error::UnknownComponent(component_id.to_string()) })?; let mut builders = #builders_name { #( #factor_names: None, )* @@ -136,12 +137,14 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &app_component, ), &mut #factors_path::InstanceBuilders::new(&mut builders), - )? + ).map_err(|err| #runtime_factor_error::<#factor_types>("prepare", err))? ); )* Ok(#state_name { #( - #factor_names: #FactorInstanceBuilder::build(builders.#factor_names.unwrap())?, + #factor_names: #FactorInstanceBuilder::build( + builders.#factor_names.unwrap() + ).map_err(|err| #runtime_factor_error::<#factor_types>("build", err))?, )* }) } diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index a32d76dc9..15e40f0e0 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -64,7 +64,7 @@ impl TestEnvironment { .components() .next() .context("no components")?; - factors.build_store_data(&configured_app, component.id()) + Ok(factors.build_store_data(&configured_app, component.id())?) } pub fn new_linker() -> Linker { diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 0d81f2ebe..746589af8 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -1,10 +1,9 @@ use std::{any::Any, collections::HashMap}; -use anyhow::Context; - use crate::{ - prepare::FactorInstanceBuilder, runtime_config::RuntimeConfigTracker, App, FactorRuntimeConfig, - InstanceBuilders, Linker, PrepareContext, RuntimeConfigSource, RuntimeFactors, + prepare::FactorInstanceBuilder, runtime_config::RuntimeConfigTracker, App, Error, + FactorRuntimeConfig, InstanceBuilders, Linker, PrepareContext, RuntimeConfigSource, + RuntimeFactors, }; pub trait Factor: Any + Sized { @@ -107,7 +106,7 @@ impl<'a, T: RuntimeFactors, F: Factor> ConfigureAppContext<'a, T, F> { app: &'a App, app_state: &'a T::AppState, runtime_config_tracker: &mut RuntimeConfigTracker, - ) -> anyhow::Result { + ) -> crate::Result { let runtime_config = runtime_config_tracker.get_config::()?; Ok(Self { app, @@ -121,7 +120,7 @@ impl<'a, T: RuntimeFactors, F: Factor> ConfigureAppContext<'a, T, F> { } pub fn app_state(&self) -> crate::Result<&U::AppState> { - T::app_state::(self.app_state).context("no such factor") + T::app_state::(self.app_state).ok_or(Error::no_such_factor::()) } pub fn runtime_config(&self) -> Option<&F::RuntimeConfig> { @@ -148,7 +147,7 @@ impl ConfiguredApp { &self.app } - pub fn app_state(&self) -> crate::Result<&F::AppState> { - T::app_state::(&self.app_state).context("no such factor") + pub fn app_state(&self) -> crate::Result<&U::AppState> { + T::app_state::(&self.app_state).ok_or(Error::no_such_factor::()) } } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 5d568e6aa..6ba85293c 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -22,10 +22,55 @@ pub type Linker = wasmtime::component::Linker<::Instance pub type App = spin_app::App<'static, spin_app::InertLoader>; pub type AppComponent<'a> = spin_app::AppComponent<'a, spin_app::InertLoader>; -// TODO: Add a real Error type -pub type Result = wasmtime::Result; +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("factor dependency ordering error: {0}")] + DependencyOrderingError(String), + #[error("no such factor: {0}")] + NoSuchFactor(&'static str), + #[error("{factor} requested already-consumed key {key:?}")] + RuntimeConfigReusedKey { factor: &'static str, key: String }, + #[error("runtime config error: {0}")] + RuntimeConfigSource(#[source] anyhow::Error), + #[error("unused runtime config key(s): {}", keys.join(", "))] + RuntimeConfigUnusedKeys { keys: Vec }, + #[error("{factor} {method} failed: {source}")] + RuntimeFactorError { + factor: &'static str, + method: &'static str, + source: anyhow::Error, + }, + #[error("unknown component: {0}")] + UnknownComponent(String), +} + +impl Error { + fn no_such_factor() -> Self { + Self::NoSuchFactor(std::any::type_name::()) + } + + fn runtime_config_reused_key(key: impl Into) -> Self { + Self::RuntimeConfigReusedKey { + factor: std::any::type_name::(), + key: key.into(), + } + } +} #[doc(hidden)] pub mod __internal { pub use crate::runtime_config::RuntimeConfigTracker; + + pub fn runtime_factor_error( + method: &'static str, + source: anyhow::Error, + ) -> crate::Error { + crate::Error::RuntimeFactorError { + factor: std::any::type_name::(), + method, + source, + } + } } diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index 7e3c7f6d6..9619e943e 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -1,8 +1,6 @@ use std::any::Any; -use anyhow::Context; - -use crate::{AppComponent, Factor, RuntimeFactors}; +use crate::{AppComponent, Error, Factor, RuntimeFactors}; pub trait FactorInstanceBuilder: Any { type InstanceState: Send + 'static; @@ -69,9 +67,14 @@ impl<'a, T: RuntimeFactors> InstanceBuilders<'a, T> { /// Fails if the current [`RuntimeFactors`] does not include the given /// [`Factor`] or if the given [`Factor`]'s builder has not been prepared /// yet (because it is sequenced after this factor). - pub fn get_mut(&mut self) -> crate::Result<&mut F::InstanceBuilder> { - T::instance_builder_mut::(self.inner) - .context("no such factor")? - .context("builder not prepared") + pub fn get_mut(&mut self) -> crate::Result<&mut U::InstanceBuilder> { + T::instance_builder_mut::(self.inner) + .ok_or(Error::no_such_factor::())? + .ok_or_else(|| { + Error::DependencyOrderingError(format!( + "{factor} builder requested before it was prepared", + factor = std::any::type_name::() + )) + }) } } diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index 372c9ffe6..e9736b6ce 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -1,9 +1,8 @@ use std::collections::HashSet; -use anyhow::bail; use serde::de::DeserializeOwned; -use crate::Factor; +use crate::{Error, Factor}; pub const NO_RUNTIME_CONFIG: &str = ""; @@ -72,30 +71,26 @@ impl RuntimeConfigTracker { } #[doc(hidden)] - pub fn validate_all_keys_used(self) -> anyhow::Result<()> { + pub fn validate_all_keys_used(self) -> crate::Result<()> { if !self.unused_keys.is_empty() { - bail!( - "unused runtime config key(s): {keys}", - keys = self - .unused_keys - .iter() - .map(|key| format!("{key:?}")) - .collect::>() - .join(", ") - ); + return Err(Error::RuntimeConfigUnusedKeys { + keys: self.unused_keys.into_iter().collect(), + }); } Ok(()) } - pub fn get_config(&mut self) -> anyhow::Result> { - let key = F::RuntimeConfig::KEY; + pub fn get_config(&mut self) -> crate::Result> { + let key = T::RuntimeConfig::KEY; if key == NO_RUNTIME_CONFIG { return Ok(None); } if !self.used_keys.insert(key) { - bail!("already used runtime config key {key:?}"); + return Err(Error::runtime_config_reused_key::(key)); } self.unused_keys.remove(key); - self.source.get_factor_config::(key) + self.source + .get_factor_config::(key) + .map_err(Error::RuntimeConfigSource) } } diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 9222d183c..43bf4e20e 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -6,19 +6,19 @@ pub trait RuntimeFactors: Sized + 'static { type InstanceBuilders; type InstanceState: GetFactorState + Send + 'static; - fn init(&mut self, linker: &mut Linker) -> anyhow::Result<()>; + fn init(&mut self, linker: &mut Linker) -> crate::Result<()>; fn configure_app( &self, app: App, runtime_config: impl RuntimeConfigSource, - ) -> anyhow::Result>; + ) -> crate::Result>; fn build_store_data( &self, configured_app: &ConfiguredApp, component_id: &str, - ) -> anyhow::Result; + ) -> crate::Result; fn app_state(app_state: &Self::AppState) -> Option<&F::AppState>; From c6f2bfa9e063793230d9e24f304c01dac35f7883 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 12 Jul 2024 14:47:39 -0400 Subject: [PATCH 036/195] Remove Factor::runtime_config_json_schema Signed-off-by: Lann Martin --- crates/factors/src/factor.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 0d81f2ebe..60b63d1d1 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -1,4 +1,4 @@ -use std::{any::Any, collections::HashMap}; +use std::any::Any; use anyhow::Context; @@ -17,7 +17,6 @@ pub trait Factor: Any + Sized { /// Initializes this Factor for a runtime. This will be called at most once, /// before any call to [`FactorInstanceBuilder::new`] fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { - // TODO: Should `ctx` always be immut? Rename this param/type? _ = &mut ctx; Ok(()) } @@ -42,18 +41,6 @@ pub trait Factor: Any + Sized { ctx: PrepareContext, _builders: &mut InstanceBuilders, ) -> anyhow::Result; - - /// Returns [JSON Schema](https://json-schema.org/) for this factor's - /// runtime config. - /// - /// Note that this represents only a fragment of an entire JSON document (a - /// "child instance" in JSON Schema terms), so `$schema` isn't needed. - /// - /// The default implementation returns an empty schema, which accepts any - /// configuration. - fn runtime_config_json_schema(&self) -> impl serde::Serialize { - HashMap::<(), ()>::new() - } } /// The instance state of the given [`Factor`] `F`. From a33d4ce52cf21a0e2607fa784bbe42792d8cc09b Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Fri, 12 Jul 2024 15:09:01 -0500 Subject: [PATCH 037/195] Add redis key value factor Signed-off-by: Kate Goldenring --- Cargo.lock | 11 +++++++++++ crates/factor-key-value-redis/Cargo.toml | 15 +++++++++++++++ crates/factor-key-value-redis/src/lib.rs | 24 ++++++++++++++++++++++++ crates/factors/Cargo.toml | 1 + crates/factors/tests/smoke.rs | 6 ++++++ 5 files changed, 57 insertions(+) create mode 100644 crates/factor-key-value-redis/Cargo.toml create mode 100644 crates/factor-key-value-redis/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 97f108940..5dea4f6ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7529,6 +7529,16 @@ dependencies = [ "toml 0.8.14", ] +[[package]] +name = "spin-factor-key-value-redis" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "serde 1.0.197", + "spin-factor-key-value", + "spin-key-value-redis", +] + [[package]] name = "spin-factor-outbound-http" version = "2.7.0-pre0" @@ -7592,6 +7602,7 @@ dependencies = [ "spin-app", "spin-componentize", "spin-factor-key-value", + "spin-factor-key-value-redis", "spin-factor-outbound-http", "spin-factor-outbound-networking", "spin-factor-variables", diff --git a/crates/factor-key-value-redis/Cargo.toml b/crates/factor-key-value-redis/Cargo.toml new file mode 100644 index 000000000..1c19c58ff --- /dev/null +++ b/crates/factor-key-value-redis/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "spin-factor-key-value-redis" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +serde = { version = "1.0", features = ["rc"] } +spin-factor-key-value = { path = "../factor-key-value" } +# TODO: merge with this crate +spin-key-value-redis = { path = "../key-value-redis" } + +[lints] +workspace = true diff --git a/crates/factor-key-value-redis/src/lib.rs b/crates/factor-key-value-redis/src/lib.rs new file mode 100644 index 000000000..4fd45620e --- /dev/null +++ b/crates/factor-key-value-redis/src/lib.rs @@ -0,0 +1,24 @@ +use serde::Deserialize; +use spin_factor_key_value::MakeKeyValueStore; +use spin_key_value_redis::KeyValueRedis; +pub struct RedisKeyValueStore; + +#[derive(Deserialize)] +pub struct RedisKeyValueRuntimeConfig { + url: String, +} + +impl MakeKeyValueStore for RedisKeyValueStore { + const RUNTIME_CONFIG_TYPE: &'static str = "redis"; + + type RuntimeConfig = RedisKeyValueRuntimeConfig; + + type StoreManager = KeyValueRedis; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + KeyValueRedis::new(runtime_config.url) + } +} diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index d99884e21..ae287f977 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -20,6 +20,7 @@ serde_json = "1.0" spin-componentize = { path = "../componentize" } spin-factors-derive = { path = "../factors-derive", features = ["expander"] } spin-factor-key-value = { path = "../factor-key-value" } +spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-variables = { path = "../factor-variables" } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index b46da230c..a55b71008 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -11,6 +11,7 @@ use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; +use spin_factor_key_value_redis::RedisKeyValueStore; use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] @@ -36,6 +37,8 @@ async fn smoke_test_works() -> anyhow::Result<()> { factors.key_value.add_store_type(TestSpinKeyValueStore)?; + factors.key_value.add_store_type(RedisKeyValueStore)?; + let locked = spin_loader::from_file( "tests/smoke-app/spin.toml", spin_loader::FilesMountStrategy::Direct, @@ -124,6 +127,9 @@ impl RuntimeConfigSource for TestSource { [key_value_store.default] type = "spin" + [key_value_store.other] + type = "redis" + url = "redis://localhost:6379" } .remove(key) else { return Ok(None); From cd652bae373af804a539ea61e4ab20a0f4d1bc60 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Fri, 12 Jul 2024 15:58:03 -0500 Subject: [PATCH 038/195] Add support for configuring default KV store Signed-off-by: Kate Goldenring --- crates/factor-key-value/src/lib.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index efca63a16..f066429d8 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -20,11 +20,19 @@ use store::{store_from_toml_fn, StoreFromToml}; pub use store::MakeKeyValueStore; -#[derive(Default)] pub struct KeyValueFactor { store_types: HashMap<&'static str, StoreFromToml>, + default_store_type: &'static str, } +impl Default for KeyValueFactor { + fn default() -> KeyValueFactor { + KeyValueFactor { + store_types: HashMap::default(), + default_store_type: "spin", + } + } +} impl KeyValueFactor { pub fn add_store_type(&mut self, store_type: T) -> anyhow::Result<()> { if self @@ -61,8 +69,12 @@ impl Factor for KeyValueFactor { ) -> anyhow::Result { // Build StoreManager from runtime config let mut stores = HashMap::new(); + let mut add_default_store = true; if let Some(runtime_config) = ctx.take_runtime_config() { for (label, StoreConfig { type_, config }) in runtime_config.store_configs { + if label == "default" { + add_default_store = false; + } let store_maker = self .store_types .get(type_.as_str()) @@ -71,6 +83,19 @@ impl Factor for KeyValueFactor { stores.insert(label, store); } } + if add_default_store { + let store_maker = self + .store_types + .get(self.default_store_type) + .with_context(|| { + format!( + "default key value store {} does not exist", + self.default_store_type + ) + })?; + let store = store_maker(toml::value::Table::new())?; + stores.insert("default".to_string(), store); + } let delegating_manager = DelegatingStoreManager::new(stores); let caching_manager = CachingStoreManager::new(delegating_manager); let store_manager = Arc::new(caching_manager); From f3afe0f4dd8d94d5bc471ac5045301772826cb8b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 12 Jul 2024 10:40:42 +0200 Subject: [PATCH 039/195] Add some more documentation to factors Signed-off-by: Ryan Levick --- crates/factors-derive/src/lib.rs | 2 +- crates/factors-test/src/lib.rs | 23 +++++++++----- crates/factors/src/factor.rs | 41 ++++++++++++++++++++++--- crates/factors/src/lib.rs | 1 + crates/factors/src/prepare.rs | 19 ++++++++++-- crates/factors/src/runtime_config.rs | 15 ++++++--- crates/factors/src/runtime_factors.rs | 44 +++++++++++++++++++++++++-- crates/factors/tests/smoke.rs | 2 +- 8 files changed, 122 insertions(+), 25 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 305795887..2f7bf3c1b 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -118,7 +118,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { Ok(#ConfiguredApp::new(app, app_state)) } - fn build_store_data( + fn build_instance_state( &self, configured_app: &#ConfiguredApp, component_id: &str, ) -> #Result { diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 15e40f0e0..4f18019b9 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -9,8 +9,11 @@ use spin_loader::FilesMountStrategy; pub use toml::toml; +/// A test environment for building [`RuntimeFactors`] instances. pub struct TestEnvironment { + /// The `spin.toml` manifest. pub manifest: toml::Table, + /// The runtime config. pub runtime_config: toml::Table, } @@ -54,21 +57,24 @@ impl TestEnvironment { let mut linker = Self::new_linker::(); factors.init(&mut linker)?; - let locked_app = self.build_locked_app().await?; + let locked_app = self + .build_locked_app() + .await + .context("failed to build locked app")?; let app = App::inert(locked_app); let runtime_config = TomlRuntimeConfig(&self.runtime_config); let configured_app = factors.configure_app(app, runtime_config)?; - let component = configured_app - .app() - .components() - .next() - .context("no components")?; - Ok(factors.build_store_data(&configured_app, component.id())?) + let component = + configured_app.app().components().next().context( + "expected configured app to have at least one component, but it did not", + )?; + Ok(factors.build_instance_state(&configured_app, component.id())?) } pub fn new_linker() -> Linker { - let engine = Engine::new(Config::new().async_support(true)).expect("engine"); + let engine = Engine::new(Config::new().async_support(true)) + .expect("wasmtime engine failed to initialize"); Linker::::new(&engine) } @@ -81,6 +87,7 @@ impl TestEnvironment { } } +/// A [`RuntimeConfigSource`] that reads from a TOML table. pub struct TomlRuntimeConfig<'a>(&'a toml::Table); impl RuntimeConfigSource for TomlRuntimeConfig<'_> { diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 746589af8..2b3bc79fa 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -6,15 +6,27 @@ use crate::{ RuntimeFactors, }; +/// A contained (i.e., "factored") piece of runtime functionality. pub trait Factor: Any + Sized { + /// The particular runtime configuration relevant to this factor. + /// + /// Runtime configuration allows for user provided customization of the + /// factor's behavior on a per app basis. type RuntimeConfig: FactorRuntimeConfig; + /// The application state of this factor. + /// + /// This state *may* be cached by the runtime across multiple requests. type AppState; + /// The builder of instance state for this factor. type InstanceBuilder: FactorInstanceBuilder; - /// Initializes this Factor for a runtime. This will be called at most once, - /// before any call to [`FactorInstanceBuilder::new`] + /// Initializes this `Factor` for a runtime once at runtime startup. + /// + /// This will be called at most once, before any call to [`FactorInstanceBuilder::new`]. + /// `InitContext` provides access to a wasmtime `Linker`, so this is where any bindgen + /// `add_to_linker` calls go. fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { // TODO: Should `ctx` always be immut? Rename this param/type? _ = &mut ctx; @@ -24,18 +36,27 @@ pub trait Factor: Any + Sized { /// Performs factor-specific validation and configuration for the given /// [`App`]. /// + /// `ConfigureAppContext` gives access to: + /// - The `spin_app::App` + /// - This factors's `RuntimeConfig` + /// - The `AppState` for any factors configured before this one + /// /// A runtime may - but is not required to - reuse the returned config - /// across multiple instances. Note that this may be called without any call - /// to `init` in cases where only validation is needed. + /// across multiple instances. + /// + /// This method may be called without any call to `init` or prepare in + /// cases where only validation is needed (e.g., `spin doctor`). fn configure_app( &self, ctx: ConfigureAppContext, ) -> anyhow::Result; - /// Prepares an instance builder for this factor. + /// Creates a new `FactorInstanceBuilder`, which will later build per-instance + /// state for this factor. /// /// This method is given access to the app component being instantiated and /// to any other factors' instance builders that have already been prepared. + /// As such this is primary place for inter-factor dependencies. fn prepare( &self, ctx: PrepareContext, @@ -75,14 +96,17 @@ impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { Self { linker, get_data } } + /// Returns a mutable reference to the [`wasmtime::component::Linker`]. pub fn linker(&mut self) -> &mut Linker { self.linker } + /// Returns a function that can be used to get the instance state for this factor. pub fn get_data_fn(&self) -> GetDataFn { self.get_data } + /// Convenience method to link a binding to the linker. pub fn link_bindings( &mut self, add_to_linker: impl Fn( @@ -115,23 +139,28 @@ impl<'a, T: RuntimeFactors, F: Factor> ConfigureAppContext<'a, T, F> { }) } + /// Get the [`App`] being configured. pub fn app(&self) -> &App { self.app } + /// Get the app state related to the given factor. pub fn app_state(&self) -> crate::Result<&U::AppState> { T::app_state::(self.app_state).ok_or(Error::no_such_factor::()) } + /// Get a reference to the runtime configuration for the given factor. pub fn runtime_config(&self) -> Option<&F::RuntimeConfig> { self.runtime_config.as_ref() } + /// Take ownership of the runtime configuration for the given factor. pub fn take_runtime_config(&mut self) -> Option { self.runtime_config.take() } } +#[doc(hidden)] pub struct ConfiguredApp { app: App, app_state: T::AppState, @@ -143,10 +172,12 @@ impl ConfiguredApp { Self { app, app_state } } + /// Get the configured [`App`]. pub fn app(&self) -> &App { &self.app } + /// Get the configured app's state related to the given factor. pub fn app_state(&self) -> crate::Result<&U::AppState> { T::app_state::(&self.app_state).ok_or(Error::no_such_factor::()) } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 6ba85293c..80dc94af7 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -16,6 +16,7 @@ pub use crate::{ runtime_factors::{GetFactorState, RuntimeFactors}, }; +/// A [`wasmtime::component::Linker`] used for a [`RuntimeFactors`] collection. pub type Linker = wasmtime::component::Linker<::InstanceState>; // Temporary wrappers while refactoring diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index 9619e943e..1c66d7568 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -2,9 +2,16 @@ use std::any::Any; use crate::{AppComponent, Error, Factor, RuntimeFactors}; +/// A builder for a [`Factor`]'s per instance state. pub trait FactorInstanceBuilder: Any { + /// The per instance state of the factor. + /// + /// This is equivalent to the existing `HostComponent::Data` and ends up + /// being stored in the `wasmtime::Store`. Any `bindgen` traits for this + /// factor will be implemented on this type. type InstanceState: Send + 'static; + /// Build the per instance state of the factor. fn build(self) -> anyhow::Result; } @@ -16,6 +23,7 @@ impl FactorInstanceBuilder for () { } } +/// A helper trait for when the type implementing [`FactorInstanceBuilder`] is also the instance state. pub trait SelfInstanceBuilder: Send + 'static {} impl FactorInstanceBuilder for T { @@ -26,9 +34,9 @@ impl FactorInstanceBuilder for T { } } -/// A PrepareContext is passed to [`Factor::prepare`], giving access to any -/// already-initialized [`FactorInstanceBuilder`]s, allowing for -/// inter-[`Factor`] dependencies. +/// A PrepareContext is passed to [`Factor::prepare`]. +/// +/// This gives the factor access to app state and the app component. pub struct PrepareContext<'a, F: Factor> { pub(crate) app_state: &'a F::AppState, pub(crate) app_component: &'a AppComponent<'a>, @@ -43,15 +51,20 @@ impl<'a, F: Factor> PrepareContext<'a, F> { } } + /// Get the app state related to the factor. pub fn app_state(&self) -> &F::AppState { self.app_state } + /// Get the app component. pub fn app_component(&self) -> &AppComponent { self.app_component } } +/// The collection of all the already prepared `InstanceBuilder`s. +/// +/// Use `InstanceBuilders::get_mut` to get a mutable reference to a specific factor's instance builder. pub struct InstanceBuilders<'a, T: RuntimeFactors> { pub(crate) inner: &'a mut T::InstanceBuilders, } diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index e9736b6ce..67ed29c50 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -13,6 +13,7 @@ pub const NO_RUNTIME_CONFIG: &str = ""; /// to be shared between Factors, one Factor can be selected as the owner /// and the others will have a dependency relationship with that owner. pub trait FactorRuntimeConfig: DeserializeOwned { + /// The key used to identify this runtime configuration in a [`RuntimeConfigSource`]. const KEY: &'static str; } @@ -20,6 +21,7 @@ impl FactorRuntimeConfig for () { const KEY: &'static str = NO_RUNTIME_CONFIG; } +/// The source of runtime configuration for a Factor. pub trait RuntimeConfigSource { /// Returns an iterator of factor config keys available in this source. /// @@ -49,6 +51,10 @@ impl RuntimeConfigSource for () { } } +/// Tracks runtime configuration keys used by the runtime. +/// +/// This ensures that the runtime config source does not have any unused keys. +#[doc(hidden)] pub struct RuntimeConfigTracker { source: S, used_keys: HashSet<&'static str>, @@ -80,17 +86,18 @@ impl RuntimeConfigTracker { Ok(()) } - pub fn get_config(&mut self) -> crate::Result> { - let key = T::RuntimeConfig::KEY; + /// Get the runtime configuration for a factor. + pub(crate) fn get_config(&mut self) -> crate::Result> { + let key = F::RuntimeConfig::KEY; if key == NO_RUNTIME_CONFIG { return Ok(None); } if !self.used_keys.insert(key) { - return Err(Error::runtime_config_reused_key::(key)); + return Err(Error::runtime_config_reused_key::(key)); } self.unused_keys.remove(key); self.source - .get_factor_config::(key) + .get_factor_config::(key) .map_err(Error::RuntimeConfigSource) } } diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 43bf4e20e..34f435add 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,27 +1,65 @@ use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, Linker, RuntimeConfigSource}; -/// Implemented by `#[derive(RuntimeFactors)]` +/// A collection of `Factor`s that are initialized and configured together. +/// +/// Implemented by `#[derive(RuntimeFactors)]` and should not be implemented manually. +/// +/// # Example +/// +/// A typical usage of `RuntimeFactors` would look something like the following pseudo-code: +/// +/// ```no_run +/// #[derive(RuntimeFactors)] +/// struct MyFactors { +/// // ... +/// } +/// // Initialize the factors collection +/// let factors = MyFactors { /* .. */ }; +/// // Initialize each factor with a linker +/// factors.init(&mut linker)?; +/// // Configure the factors with an app and runtime config +/// let configured_app = factors.configure_app(app, runtime_config)?; +/// // Build the instance state for the factors +/// let data factors.build_instance_state(&configured_app, component.id()) +/// // Initialize a `wasmtime` store with the instance state +/// let mut store = wasmtime::Store::new(&engine, data); +/// // Instantiate the component +/// let instance = linker.instantiate_async(&mut store, &component).await?; +/// ``` pub trait RuntimeFactors: Sized + 'static { + /// The per application state of all the factors. type AppState; - type InstanceBuilders; + /// The per instance state of the factors. type InstanceState: GetFactorState + Send + 'static; + /// The collection of all the `InstanceBuilder`s of the factors. + type InstanceBuilders; + /// Initialize the factors with a linker. + /// + /// Each factor's `init` is called in turn. fn init(&mut self, linker: &mut Linker) -> crate::Result<()>; + /// Configure the factors with an app and runtime config. fn configure_app( &self, app: App, runtime_config: impl RuntimeConfigSource, ) -> crate::Result>; - fn build_store_data( + /// Build the instance state for the factors. + fn build_instance_state( &self, configured_app: &ConfiguredApp, component_id: &str, ) -> crate::Result; + /// Get the app state related to a particular factor. fn app_state(app_state: &Self::AppState) -> Option<&F::AppState>; + /// Get the instance builder of a particular factor. + /// + /// The outer `Option` is `None` if the factor has not been registered with this `Factors` collection, + /// and the inner `Option` is `None` if the factor has not been prepared yet. fn instance_builder_mut( builders: &mut Self::InstanceBuilders, ) -> Option>; diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index b46da230c..919f30637 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -50,7 +50,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { factors.init(&mut linker).unwrap(); let configured_app = factors.configure_app(app, TestSource)?; - let data = factors.build_store_data(&configured_app, "smoke-app")?; + let data = factors.build_instance_state(&configured_app, "smoke-app")?; assert_eq!( data.variables From e3ee311b80ee283c8928ec5d4aa4da6d7fa6e94b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 15 Jul 2024 16:05:38 +0200 Subject: [PATCH 040/195] Add SQLite Factor Signed-off-by: Ryan Levick --- Cargo.lock | 11 ++ crates/factor-sqlite/Cargo.toml | 19 +++ crates/factor-sqlite/src/lib.rs | 242 ++++++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 crates/factor-sqlite/Cargo.toml create mode 100644 crates/factor-sqlite/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 238d90ef7..ef2051adc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2436,6 +2436,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "factor-sqlite" +version = "2.7.0-pre0" +dependencies = [ + "async-trait", + "spin-factors", + "spin-locked-app", + "spin-world", + "table", +] + [[package]] name = "fallible-iterator" version = "0.2.0" diff --git a/crates/factor-sqlite/Cargo.toml b/crates/factor-sqlite/Cargo.toml new file mode 100644 index 000000000..e858f5fd9 --- /dev/null +++ b/crates/factor-sqlite/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "factor-sqlite" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +async-trait = "0.1" +spin-factors = { path = "../factors" } +spin-locked-app = { path = "../locked-app" } +spin-world = { path = "../world" } +table = { path = "../table" } + +[lints] +workspace = true diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs new file mode 100644 index 000000000..6326bcb08 --- /dev/null +++ b/crates/factor-sqlite/src/lib.rs @@ -0,0 +1,242 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use spin_factors::wasmtime::component::Resource; +use spin_factors::{anyhow, Factor, RuntimeFactors, SelfInstanceBuilder}; +use spin_locked_app::MetadataKey; +use spin_world::v1::sqlite as v1; +use spin_world::v2::sqlite as v2; + +pub struct SqliteFactor { + connections_store: Arc, +} + +impl SqliteFactor { + pub fn new(connections_store: Arc) -> Self { + Self { connections_store } + } +} + +pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); + +impl Factor for SqliteFactor { + type RuntimeConfig = (); + type AppState = AppState; + type InstanceBuilder = InstanceState; + + fn init( + &mut self, + mut ctx: spin_factors::InitContext, + ) -> anyhow::Result<()> { + ctx.link_bindings(v1::add_to_linker)?; + ctx.link_bindings(v2::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + ctx: spin_factors::ConfigureAppContext, + ) -> anyhow::Result { + let allowed_databases = ctx + .app() + .components() + .map(|component| { + Ok(( + component.id().to_string(), + component + .get_metadata(ALLOWED_DATABASES_KEY)? + .unwrap_or_default() + .into_iter() + .collect::>() + .into(), + )) + }) + .collect::>()?; + Ok(AppState { allowed_databases }) + } + + fn prepare( + &self, + ctx: spin_factors::PrepareContext, + _builders: &mut spin_factors::InstanceBuilders, + ) -> spin_factors::anyhow::Result { + let allowed_databases = ctx + .app_state() + .allowed_databases + .get(ctx.app_component().id()) + .cloned() + .unwrap_or_default(); + Ok(InstanceState { + connections: table::Table::new(256), + allowed_databases, + connections_store: self.connections_store.clone(), + }) + } +} + +pub struct AppState { + allowed_databases: HashMap>>, +} + +pub struct InstanceState { + allowed_databases: Arc>, + connections: table::Table>, + connections_store: Arc, +} + +impl InstanceState { + fn get_connection( + &self, + connection: Resource, + ) -> Result<&Arc, v2::Error> { + self.connections + .get(connection.rep()) + .ok_or(v2::Error::InvalidConnection) + } +} + +impl SelfInstanceBuilder for InstanceState {} + +impl v2::Host for InstanceState { + fn convert_error(&mut self, error: v2::Error) -> anyhow::Result { + Ok(error) + } +} + +#[async_trait] +impl v2::HostConnection for InstanceState { + async fn open(&mut self, database: String) -> Result, v2::Error> { + if !self.allowed_databases.contains(&database) { + return Err(v2::Error::AccessDenied); + } + self.connections_store + .get_connection(&database) + .await + .and_then(|conn| conn.ok_or(v2::Error::NoSuchDatabase)) + .and_then(|conn| { + self.connections + .push(conn) + .map_err(|()| v2::Error::Io("too many connections opened".to_string())) + }) + .map(Resource::new_own) + } + + async fn execute( + &mut self, + connection: Resource, + query: String, + parameters: Vec, + ) -> Result { + let conn = match self.get_connection(connection) { + Ok(c) => c, + Err(err) => return Err(err), + }; + conn.query(&query, parameters).await + } + + fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { + let _ = self.connections.remove(connection.rep()); + Ok(()) + } +} + +#[async_trait] +impl v1::Host for InstanceState { + async fn open(&mut self, database: String) -> Result { + let result = ::open(self, database).await; + result.map_err(to_legacy_error).map(|s| s.rep()) + } + + async fn execute( + &mut self, + connection: u32, + query: String, + parameters: Vec, + ) -> Result { + let this = Resource::new_borrow(connection); + let result = ::execute( + self, + this, + query, + parameters.into_iter().map(from_legacy_value).collect(), + ) + .await; + result.map_err(to_legacy_error).map(to_legacy_query_result) + } + + async fn close(&mut self, connection: u32) -> anyhow::Result<()> { + ::drop(self, Resource::new_own(connection)) + } + + fn convert_error(&mut self, error: v1::Error) -> anyhow::Result { + Ok(error) + } +} + +fn to_legacy_error(error: v2::Error) -> v1::Error { + match error { + v2::Error::NoSuchDatabase => v1::Error::NoSuchDatabase, + v2::Error::AccessDenied => v1::Error::AccessDenied, + v2::Error::InvalidConnection => v1::Error::InvalidConnection, + v2::Error::DatabaseFull => v1::Error::DatabaseFull, + v2::Error::Io(s) => v1::Error::Io(s), + } +} + +fn to_legacy_query_result(result: v2::QueryResult) -> v1::QueryResult { + v1::QueryResult { + columns: result.columns, + rows: result.rows.into_iter().map(to_legacy_row_result).collect(), + } +} + +fn to_legacy_row_result(result: v2::RowResult) -> v1::RowResult { + v1::RowResult { + values: result.values.into_iter().map(to_legacy_value).collect(), + } +} + +fn to_legacy_value(value: v2::Value) -> v1::Value { + match value { + v2::Value::Integer(i) => v1::Value::Integer(i), + v2::Value::Real(r) => v1::Value::Real(r), + v2::Value::Text(t) => v1::Value::Text(t), + v2::Value::Blob(b) => v1::Value::Blob(b), + v2::Value::Null => v1::Value::Null, + } +} + +fn from_legacy_value(value: v1::Value) -> v2::Value { + match value { + v1::Value::Integer(i) => v2::Value::Integer(i), + v1::Value::Real(r) => v2::Value::Real(r), + v1::Value::Text(t) => v2::Value::Text(t), + v1::Value::Blob(b) => v2::Value::Blob(b), + v1::Value::Null => v2::Value::Null, + } +} + +/// A store of connections for all accessible databases for an application +#[async_trait] +pub trait ConnectionsStore: Send + Sync { + /// Get a `Connection` for a specific database + async fn get_connection( + &self, + database: &str, + ) -> Result>, v2::Error>; + + fn has_connection_for(&self, database: &str) -> bool; +} + +/// A trait abstracting over operations to a SQLite database +#[async_trait] +pub trait Connection: Send + Sync { + async fn query( + &self, + query: &str, + parameters: Vec, + ) -> Result; + + async fn execute_batch(&self, statements: &str) -> anyhow::Result<()>; +} From 2bcb053561557c2defe942829299289c268b6544 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 15 Jul 2024 16:10:00 +0200 Subject: [PATCH 041/195] Move SQLite factor host impl to own module. Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/host.rs | 162 +++++++++++++++++++++++++++++++ crates/factor-sqlite/src/lib.rs | 152 ++--------------------------- 2 files changed, 170 insertions(+), 144 deletions(-) create mode 100644 crates/factor-sqlite/src/host.rs diff --git a/crates/factor-sqlite/src/host.rs b/crates/factor-sqlite/src/host.rs new file mode 100644 index 000000000..4eea5eac5 --- /dev/null +++ b/crates/factor-sqlite/src/host.rs @@ -0,0 +1,162 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use async_trait::async_trait; + +use spin_factors::wasmtime::component::Resource; +use spin_factors::{anyhow, SelfInstanceBuilder}; +use spin_world::v1::sqlite as v1; +use spin_world::v2::sqlite as v2; + +use crate::{Connection, ConnectionsStore}; + +pub struct InstanceState { + allowed_databases: Arc>, + connections: table::Table>, + connections_store: Arc, +} + +impl InstanceState { + pub fn new( + allowed_databases: Arc>, + connections_store: Arc, + ) -> Self { + Self { + allowed_databases, + connections: table::Table::new(256), + connections_store, + } + } +} + +impl InstanceState { + fn get_connection( + &self, + connection: Resource, + ) -> Result<&Arc, v2::Error> { + self.connections + .get(connection.rep()) + .ok_or(v2::Error::InvalidConnection) + } +} + +impl SelfInstanceBuilder for InstanceState {} + +impl v2::Host for InstanceState { + fn convert_error(&mut self, error: v2::Error) -> anyhow::Result { + Ok(error) + } +} + +#[async_trait] +impl v2::HostConnection for InstanceState { + async fn open(&mut self, database: String) -> Result, v2::Error> { + if !self.allowed_databases.contains(&database) { + return Err(v2::Error::AccessDenied); + } + self.connections_store + .get_connection(&database) + .await + .and_then(|conn| conn.ok_or(v2::Error::NoSuchDatabase)) + .and_then(|conn| { + self.connections + .push(conn) + .map_err(|()| v2::Error::Io("too many connections opened".to_string())) + }) + .map(Resource::new_own) + } + + async fn execute( + &mut self, + connection: Resource, + query: String, + parameters: Vec, + ) -> Result { + let conn = match self.get_connection(connection) { + Ok(c) => c, + Err(err) => return Err(err), + }; + conn.query(&query, parameters).await + } + + fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { + let _ = self.connections.remove(connection.rep()); + Ok(()) + } +} + +#[async_trait] +impl v1::Host for InstanceState { + async fn open(&mut self, database: String) -> Result { + let result = ::open(self, database).await; + result.map_err(to_legacy_error).map(|s| s.rep()) + } + + async fn execute( + &mut self, + connection: u32, + query: String, + parameters: Vec, + ) -> Result { + let this = Resource::new_borrow(connection); + let result = ::execute( + self, + this, + query, + parameters.into_iter().map(from_legacy_value).collect(), + ) + .await; + result.map_err(to_legacy_error).map(to_legacy_query_result) + } + + async fn close(&mut self, connection: u32) -> anyhow::Result<()> { + ::drop(self, Resource::new_own(connection)) + } + + fn convert_error(&mut self, error: v1::Error) -> anyhow::Result { + Ok(error) + } +} + +fn to_legacy_error(error: v2::Error) -> v1::Error { + match error { + v2::Error::NoSuchDatabase => v1::Error::NoSuchDatabase, + v2::Error::AccessDenied => v1::Error::AccessDenied, + v2::Error::InvalidConnection => v1::Error::InvalidConnection, + v2::Error::DatabaseFull => v1::Error::DatabaseFull, + v2::Error::Io(s) => v1::Error::Io(s), + } +} + +fn to_legacy_query_result(result: v2::QueryResult) -> v1::QueryResult { + v1::QueryResult { + columns: result.columns, + rows: result.rows.into_iter().map(to_legacy_row_result).collect(), + } +} + +fn to_legacy_row_result(result: v2::RowResult) -> v1::RowResult { + v1::RowResult { + values: result.values.into_iter().map(to_legacy_value).collect(), + } +} + +fn to_legacy_value(value: v2::Value) -> v1::Value { + match value { + v2::Value::Integer(i) => v1::Value::Integer(i), + v2::Value::Real(r) => v1::Value::Real(r), + v2::Value::Text(t) => v1::Value::Text(t), + v2::Value::Blob(b) => v1::Value::Blob(b), + v2::Value::Null => v1::Value::Null, + } +} + +fn from_legacy_value(value: v1::Value) -> v2::Value { + match value { + v1::Value::Integer(i) => v2::Value::Integer(i), + v1::Value::Real(r) => v2::Value::Real(r), + v1::Value::Text(t) => v2::Value::Text(t), + v1::Value::Blob(b) => v2::Value::Blob(b), + v1::Value::Null => v2::Value::Null, + } +} diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 6326bcb08..5d56e5543 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -1,9 +1,12 @@ +mod host; + use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use host::InstanceState; + use async_trait::async_trait; -use spin_factors::wasmtime::component::Resource; -use spin_factors::{anyhow, Factor, RuntimeFactors, SelfInstanceBuilder}; +use spin_factors::{anyhow, Factor, RuntimeFactors}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; @@ -67,11 +70,10 @@ impl Factor for SqliteFactor { .get(ctx.app_component().id()) .cloned() .unwrap_or_default(); - Ok(InstanceState { - connections: table::Table::new(256), + Ok(InstanceState::new( allowed_databases, - connections_store: self.connections_store.clone(), - }) + self.connections_store.clone(), + )) } } @@ -79,144 +81,6 @@ pub struct AppState { allowed_databases: HashMap>>, } -pub struct InstanceState { - allowed_databases: Arc>, - connections: table::Table>, - connections_store: Arc, -} - -impl InstanceState { - fn get_connection( - &self, - connection: Resource, - ) -> Result<&Arc, v2::Error> { - self.connections - .get(connection.rep()) - .ok_or(v2::Error::InvalidConnection) - } -} - -impl SelfInstanceBuilder for InstanceState {} - -impl v2::Host for InstanceState { - fn convert_error(&mut self, error: v2::Error) -> anyhow::Result { - Ok(error) - } -} - -#[async_trait] -impl v2::HostConnection for InstanceState { - async fn open(&mut self, database: String) -> Result, v2::Error> { - if !self.allowed_databases.contains(&database) { - return Err(v2::Error::AccessDenied); - } - self.connections_store - .get_connection(&database) - .await - .and_then(|conn| conn.ok_or(v2::Error::NoSuchDatabase)) - .and_then(|conn| { - self.connections - .push(conn) - .map_err(|()| v2::Error::Io("too many connections opened".to_string())) - }) - .map(Resource::new_own) - } - - async fn execute( - &mut self, - connection: Resource, - query: String, - parameters: Vec, - ) -> Result { - let conn = match self.get_connection(connection) { - Ok(c) => c, - Err(err) => return Err(err), - }; - conn.query(&query, parameters).await - } - - fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { - let _ = self.connections.remove(connection.rep()); - Ok(()) - } -} - -#[async_trait] -impl v1::Host for InstanceState { - async fn open(&mut self, database: String) -> Result { - let result = ::open(self, database).await; - result.map_err(to_legacy_error).map(|s| s.rep()) - } - - async fn execute( - &mut self, - connection: u32, - query: String, - parameters: Vec, - ) -> Result { - let this = Resource::new_borrow(connection); - let result = ::execute( - self, - this, - query, - parameters.into_iter().map(from_legacy_value).collect(), - ) - .await; - result.map_err(to_legacy_error).map(to_legacy_query_result) - } - - async fn close(&mut self, connection: u32) -> anyhow::Result<()> { - ::drop(self, Resource::new_own(connection)) - } - - fn convert_error(&mut self, error: v1::Error) -> anyhow::Result { - Ok(error) - } -} - -fn to_legacy_error(error: v2::Error) -> v1::Error { - match error { - v2::Error::NoSuchDatabase => v1::Error::NoSuchDatabase, - v2::Error::AccessDenied => v1::Error::AccessDenied, - v2::Error::InvalidConnection => v1::Error::InvalidConnection, - v2::Error::DatabaseFull => v1::Error::DatabaseFull, - v2::Error::Io(s) => v1::Error::Io(s), - } -} - -fn to_legacy_query_result(result: v2::QueryResult) -> v1::QueryResult { - v1::QueryResult { - columns: result.columns, - rows: result.rows.into_iter().map(to_legacy_row_result).collect(), - } -} - -fn to_legacy_row_result(result: v2::RowResult) -> v1::RowResult { - v1::RowResult { - values: result.values.into_iter().map(to_legacy_value).collect(), - } -} - -fn to_legacy_value(value: v2::Value) -> v1::Value { - match value { - v2::Value::Integer(i) => v1::Value::Integer(i), - v2::Value::Real(r) => v1::Value::Real(r), - v2::Value::Text(t) => v1::Value::Text(t), - v2::Value::Blob(b) => v1::Value::Blob(b), - v2::Value::Null => v1::Value::Null, - } -} - -fn from_legacy_value(value: v1::Value) -> v2::Value { - match value { - v1::Value::Integer(i) => v2::Value::Integer(i), - v1::Value::Real(r) => v2::Value::Real(r), - v1::Value::Text(t) => v2::Value::Text(t), - v1::Value::Blob(b) => v2::Value::Blob(b), - v1::Value::Null => v2::Value::Null, - } -} - /// A store of connections for all accessible databases for an application #[async_trait] pub trait ConnectionsStore: Send + Sync { From ef3f87d9b09c13b843a630a77ec86cee184d0e5a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 15 Jul 2024 10:12:08 -0400 Subject: [PATCH 042/195] factors: Break up the Error::RuntimeFactorError monopoly Signed-off-by: Lann Martin --- crates/factors-derive/src/lib.rs | 10 ++--- crates/factors/src/lib.rs | 63 +++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 2f7bf3c1b..f5ca04d98 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -64,11 +64,11 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let factors_path = quote!(::#factors_crate); let wasmtime = quote!(#factors_path::wasmtime); let Result = quote!(#factors_path::Result); + let Error = quote!(#factors_path::Error); let Factor = quote!(#factors_path::Factor); let ConfiguredApp = quote!(#factors_path::ConfiguredApp); let RuntimeConfigTracker = quote!(#factors_path::__internal::RuntimeConfigTracker); let FactorInstanceBuilder = quote!(#factors_path::FactorInstanceBuilder); - let runtime_factor_error = quote!(#factors_path::__internal::runtime_factor_error); Ok(quote! { impl #factors_path::RuntimeFactors for #name { @@ -88,7 +88,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { linker, |state| &mut state.#factor_names, ) - ).map_err(|err| #runtime_factor_error::<#factor_types>("init", err))?; + ).map_err(#Error::factor_init_error::<#factor_types>)?; )* Ok(()) } @@ -111,7 +111,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &app_state, &mut runtime_config_tracker, )?, - ).map_err(|err| #runtime_factor_error::<#factor_types>("configure_app", err))? + ).map_err(#Error::factor_configure_app_error::<#factor_types>)? ); )* runtime_config_tracker.validate_all_keys_used()?; @@ -137,14 +137,14 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &app_component, ), &mut #factors_path::InstanceBuilders::new(&mut builders), - ).map_err(|err| #runtime_factor_error::<#factor_types>("prepare", err))? + ).map_err(#Error::factor_prepare_error::<#factor_types>)? ); )* Ok(#state_name { #( #factor_names: #FactorInstanceBuilder::build( builders.#factor_names.unwrap() - ).map_err(|err| #runtime_factor_error::<#factor_types>("build", err))?, + ).map_err(#Error::factor_build_error::<#factor_types>)?, )* }) } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 80dc94af7..381e64e84 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -29,6 +29,26 @@ pub type Result = std::result::Result; pub enum Error { #[error("factor dependency ordering error: {0}")] DependencyOrderingError(String), + #[error("{factor}::InstanceBuilder::build failed: {source}")] + FactorBuildError { + factor: &'static str, + source: anyhow::Error, + }, + #[error("{factor}::configure_app failed: {source}")] + FactorConfigureAppError { + factor: &'static str, + source: anyhow::Error, + }, + #[error("{factor}::init failed: {source}")] + FactorInitError { + factor: &'static str, + source: anyhow::Error, + }, + #[error("{factor}::prepare failed: {source}")] + FactorPrepareError { + factor: &'static str, + source: anyhow::Error, + }, #[error("no such factor: {0}")] NoSuchFactor(&'static str), #[error("{factor} requested already-consumed key {key:?}")] @@ -37,12 +57,6 @@ pub enum Error { RuntimeConfigSource(#[source] anyhow::Error), #[error("unused runtime config key(s): {}", keys.join(", "))] RuntimeConfigUnusedKeys { keys: Vec }, - #[error("{factor} {method} failed: {source}")] - RuntimeFactorError { - factor: &'static str, - method: &'static str, - source: anyhow::Error, - }, #[error("unknown component: {0}")] UnknownComponent(String), } @@ -58,20 +72,35 @@ impl Error { key: key.into(), } } + + // These helpers are used by factors-derive + + #[doc(hidden)] + pub fn factor_init_error(source: anyhow::Error) -> Self { + let factor = std::any::type_name::(); + Self::FactorInitError { factor, source } + } + + #[doc(hidden)] + pub fn factor_configure_app_error(source: anyhow::Error) -> Self { + let factor = std::any::type_name::(); + Self::FactorConfigureAppError { factor, source } + } + + #[doc(hidden)] + pub fn factor_prepare_error(source: anyhow::Error) -> Self { + let factor = std::any::type_name::(); + Self::FactorPrepareError { factor, source } + } + + #[doc(hidden)] + pub fn factor_build_error(source: anyhow::Error) -> Self { + let factor = std::any::type_name::(); + Self::FactorBuildError { factor, source } + } } #[doc(hidden)] pub mod __internal { pub use crate::runtime_config::RuntimeConfigTracker; - - pub fn runtime_factor_error( - method: &'static str, - source: anyhow::Error, - ) -> crate::Error { - crate::Error::RuntimeFactorError { - factor: std::any::type_name::(), - method, - source, - } - } } From 84fdf3b431a60e126fb2c1a7466c8f34d232f6e0 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 15 Jul 2024 18:45:36 +0200 Subject: [PATCH 043/195] Runtime config for sqlite Signed-off-by: Ryan Levick --- Cargo.lock | 2 + crates/factor-sqlite/Cargo.toml | 2 + crates/factor-sqlite/src/host.rs | 25 ++++----- crates/factor-sqlite/src/lib.rs | 90 +++++++++++++++++++++++++------- 4 files changed, 88 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef2051adc..f9335ef61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2441,10 +2441,12 @@ name = "factor-sqlite" version = "2.7.0-pre0" dependencies = [ "async-trait", + "serde 1.0.197", "spin-factors", "spin-locked-app", "spin-world", "table", + "toml 0.8.14", ] [[package]] diff --git a/crates/factor-sqlite/Cargo.toml b/crates/factor-sqlite/Cargo.toml index e858f5fd9..5b10db1cf 100644 --- a/crates/factor-sqlite/Cargo.toml +++ b/crates/factor-sqlite/Cargo.toml @@ -10,10 +10,12 @@ rust-version.workspace = true [dependencies] async-trait = "0.1" +serde = { version = "1.0", features = ["rc"] } spin-factors = { path = "../factors" } spin-locked-app = { path = "../locked-app" } spin-world = { path = "../world" } table = { path = "../table" } +toml = "0.8" [lints] workspace = true diff --git a/crates/factor-sqlite/src/host.rs b/crates/factor-sqlite/src/host.rs index 4eea5eac5..fff7e5a1f 100644 --- a/crates/factor-sqlite/src/host.rs +++ b/crates/factor-sqlite/src/host.rs @@ -8,28 +8,29 @@ use spin_factors::{anyhow, SelfInstanceBuilder}; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; -use crate::{Connection, ConnectionsStore}; +use crate::{Connection, ConnectionPool}; pub struct InstanceState { allowed_databases: Arc>, connections: table::Table>, - connections_store: Arc, + get_pool: ConnectionPoolGetter, } +/// A function that takes a database label and returns a connection pool, if one exists. +pub type ConnectionPoolGetter = Arc Option> + Send + Sync>; + impl InstanceState { - pub fn new( - allowed_databases: Arc>, - connections_store: Arc, - ) -> Self { + /// Create a new `InstanceState` + /// + /// Takes the list of allowed databases, and a function for getting a connection pool given a database label. + pub fn new(allowed_databases: Arc>, get_pool: ConnectionPoolGetter) -> Self { Self { allowed_databases, connections: table::Table::new(256), - connections_store, + get_pool, } } -} -impl InstanceState { fn get_connection( &self, connection: Resource, @@ -54,10 +55,10 @@ impl v2::HostConnection for InstanceState { if !self.allowed_databases.contains(&database) { return Err(v2::Error::AccessDenied); } - self.connections_store - .get_connection(&database) + (self.get_pool)(&database) + .ok_or(v2::Error::NoSuchDatabase)? + .get_connection() .await - .and_then(|conn| conn.ok_or(v2::Error::NoSuchDatabase)) .and_then(|conn| { self.connections .push(conn) diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 5d56e5543..74cb4411d 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -6,25 +6,31 @@ use std::sync::Arc; use host::InstanceState; use async_trait::async_trait; -use spin_factors::{anyhow, Factor, RuntimeFactors}; +use serde::Deserialize; +use spin_factors::{anyhow, Factor, FactorRuntimeConfig, RuntimeFactors}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; pub struct SqliteFactor { - connections_store: Arc, + runtime_config_resolver: Arc, } impl SqliteFactor { - pub fn new(connections_store: Arc) -> Self { - Self { connections_store } + /// Create a new `SqliteFactor` + pub fn new( + runtime_config_resolver: impl RuntimeConfigResolver + Send + Sync + 'static, + ) -> Self { + Self { + runtime_config_resolver: Arc::new(runtime_config_resolver), + } } } pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); impl Factor for SqliteFactor { - type RuntimeConfig = (); + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; @@ -39,8 +45,16 @@ impl Factor for SqliteFactor { fn configure_app( &self, - ctx: spin_factors::ConfigureAppContext, + mut ctx: spin_factors::ConfigureAppContext, ) -> anyhow::Result { + let mut connection_pools = HashMap::new(); + if let Some(runtime_config) = ctx.take_runtime_config() { + for (database_label, StoreConfig { type_, config }) in runtime_config.store_configs { + let pool = self.runtime_config_resolver.get_pool(&type_, config)?; + connection_pools.insert(database_label, pool); + } + } + let allowed_databases = ctx .app() .components() @@ -56,7 +70,16 @@ impl Factor for SqliteFactor { )) }) .collect::>()?; - Ok(AppState { allowed_databases }) + let resolver = self.runtime_config_resolver.clone(); + Ok(AppState { + allowed_databases, + connection_pools: Arc::new(move |label| { + connection_pools + .get(label) + .cloned() + .or_else(|| resolver.default(label)) + }), + }) } fn prepare( @@ -70,27 +93,56 @@ impl Factor for SqliteFactor { .get(ctx.app_component().id()) .cloned() .unwrap_or_default(); - Ok(InstanceState::new( - allowed_databases, - self.connections_store.clone(), - )) + let connection_pools = ctx.app_state().connection_pools.clone(); + Ok(InstanceState::new(allowed_databases, connection_pools)) } } pub struct AppState { + /// A map from component id to a set of allowed databases allowed_databases: HashMap>>, + /// A map from database name to a connection pool + connection_pools: host::ConnectionPoolGetter, } -/// A store of connections for all accessible databases for an application -#[async_trait] -pub trait ConnectionsStore: Send + Sync { - /// Get a `Connection` for a specific database - async fn get_connection( +#[derive(Deserialize)] +#[serde(transparent)] +pub struct RuntimeConfig { + store_configs: HashMap, +} + +impl FactorRuntimeConfig for RuntimeConfig { + const KEY: &'static str = "sqlite_database"; +} + +#[derive(Deserialize)] +struct StoreConfig { + #[serde(rename = "type")] + type_: String, + #[serde(flatten)] + config: toml::Table, +} + +/// Resolves some piece of runtime configuration to a connection pool +pub trait RuntimeConfigResolver { + /// Get a connection pool for a given runtime configuration type and the raw configuration. + fn get_pool( &self, - database: &str, - ) -> Result>, v2::Error>; + r#type: &str, + config: toml::Table, + ) -> anyhow::Result>; - fn has_connection_for(&self, database: &str) -> bool; + /// If there is no runtime configuration for a given database label, return a default connection pool. + /// + /// If `Option::None` is returned, the database is not allowed. + fn default(&self, label: &str) -> Option>; +} + +/// A store of connections for all accessible databases for an application +#[async_trait] +pub trait ConnectionPool: Send + Sync { + /// Get a `Connection` from the pool + async fn get_connection(&self) -> Result, v2::Error>; } /// A trait abstracting over operations to a SQLite database From fc346aa3414fd96d9987aa5ec8fbcfee191b1e6e Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 15 Jul 2024 19:03:00 +0200 Subject: [PATCH 044/195] Move runtime config to its own module Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/lib.rs | 47 ++++------------------ crates/factor-sqlite/src/runtime_config.rs | 39 ++++++++++++++++++ 2 files changed, 47 insertions(+), 39 deletions(-) create mode 100644 crates/factor-sqlite/src/runtime_config.rs diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 74cb4411d..6b8cd9dd3 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -1,4 +1,5 @@ mod host; +mod runtime_config; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -6,20 +7,19 @@ use std::sync::Arc; use host::InstanceState; use async_trait::async_trait; -use serde::Deserialize; -use spin_factors::{anyhow, Factor, FactorRuntimeConfig, RuntimeFactors}; +use spin_factors::{anyhow, Factor, RuntimeFactors}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; pub struct SqliteFactor { - runtime_config_resolver: Arc, + runtime_config_resolver: Arc, } impl SqliteFactor { /// Create a new `SqliteFactor` pub fn new( - runtime_config_resolver: impl RuntimeConfigResolver + Send + Sync + 'static, + runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + Send + Sync + 'static, ) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), @@ -30,7 +30,7 @@ impl SqliteFactor { pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); impl Factor for SqliteFactor { - type RuntimeConfig = RuntimeConfig; + type RuntimeConfig = runtime_config::RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; @@ -49,7 +49,9 @@ impl Factor for SqliteFactor { ) -> anyhow::Result { let mut connection_pools = HashMap::new(); if let Some(runtime_config) = ctx.take_runtime_config() { - for (database_label, StoreConfig { type_, config }) in runtime_config.store_configs { + for (database_label, runtime_config::StoreConfig { type_, config }) in + runtime_config.store_configs + { let pool = self.runtime_config_resolver.get_pool(&type_, config)?; connection_pools.insert(database_label, pool); } @@ -105,39 +107,6 @@ pub struct AppState { connection_pools: host::ConnectionPoolGetter, } -#[derive(Deserialize)] -#[serde(transparent)] -pub struct RuntimeConfig { - store_configs: HashMap, -} - -impl FactorRuntimeConfig for RuntimeConfig { - const KEY: &'static str = "sqlite_database"; -} - -#[derive(Deserialize)] -struct StoreConfig { - #[serde(rename = "type")] - type_: String, - #[serde(flatten)] - config: toml::Table, -} - -/// Resolves some piece of runtime configuration to a connection pool -pub trait RuntimeConfigResolver { - /// Get a connection pool for a given runtime configuration type and the raw configuration. - fn get_pool( - &self, - r#type: &str, - config: toml::Table, - ) -> anyhow::Result>; - - /// If there is no runtime configuration for a given database label, return a default connection pool. - /// - /// If `Option::None` is returned, the database is not allowed. - fn default(&self, label: &str) -> Option>; -} - /// A store of connections for all accessible databases for an application #[async_trait] pub trait ConnectionPool: Send + Sync { diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs new file mode 100644 index 000000000..2540484a5 --- /dev/null +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -0,0 +1,39 @@ +use std::{collections::HashMap, sync::Arc}; + +use serde::Deserialize; +use spin_factors::{anyhow, FactorRuntimeConfig}; + +use crate::ConnectionPool; + +#[derive(Deserialize)] +#[serde(transparent)] +pub struct RuntimeConfig { + pub store_configs: HashMap, +} + +impl FactorRuntimeConfig for RuntimeConfig { + const KEY: &'static str = "sqlite_database"; +} + +#[derive(Deserialize)] +pub struct StoreConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} + +/// Resolves some piece of runtime configuration to a connection pool +pub trait RuntimeConfigResolver { + /// Get a connection pool for a given runtime configuration type and the raw configuration. + fn get_pool( + &self, + r#type: &str, + config: toml::Table, + ) -> anyhow::Result>; + + /// If there is no runtime configuration for a given database label, return a default connection pool. + /// + /// If `Option::None` is returned, the database is not allowed. + fn default(&self, label: &str) -> Option>; +} From e8cacd588fc5b901a3e0cc6bb39f7691afe03e07 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 15 Jul 2024 19:10:44 +0200 Subject: [PATCH 045/195] Add initial tests Signed-off-by: Ryan Levick --- Cargo.lock | 2 ++ crates/factor-sqlite/Cargo.toml | 4 +++ crates/factor-sqlite/src/host.rs | 6 ++++ crates/factor-sqlite/src/lib.rs | 2 +- crates/factor-sqlite/tests/factor.rs | 48 ++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 crates/factor-sqlite/tests/factor.rs diff --git a/Cargo.lock b/Cargo.lock index f9335ef61..b2c87a190 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2443,9 +2443,11 @@ dependencies = [ "async-trait", "serde 1.0.197", "spin-factors", + "spin-factors-test", "spin-locked-app", "spin-world", "table", + "tokio", "toml 0.8.14", ] diff --git a/crates/factor-sqlite/Cargo.toml b/crates/factor-sqlite/Cargo.toml index 5b10db1cf..f7ad00a55 100644 --- a/crates/factor-sqlite/Cargo.toml +++ b/crates/factor-sqlite/Cargo.toml @@ -17,5 +17,9 @@ spin-world = { path = "../world" } table = { path = "../table" } toml = "0.8" +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + [lints] workspace = true diff --git a/crates/factor-sqlite/src/host.rs b/crates/factor-sqlite/src/host.rs index fff7e5a1f..e12817625 100644 --- a/crates/factor-sqlite/src/host.rs +++ b/crates/factor-sqlite/src/host.rs @@ -16,6 +16,12 @@ pub struct InstanceState { get_pool: ConnectionPoolGetter, } +impl InstanceState { + pub fn allowed_databases(&self) -> &HashSet { + &self.allowed_databases + } +} + /// A function that takes a database label and returns a connection pool, if one exists. pub type ConnectionPoolGetter = Arc Option> + Send + Sync>; diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 6b8cd9dd3..9c26609e5 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -1,5 +1,5 @@ mod host; -mod runtime_config; +pub mod runtime_config; use std::collections::{HashMap, HashSet}; use std::sync::Arc; diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs new file mode 100644 index 000000000..ec83c27b2 --- /dev/null +++ b/crates/factor-sqlite/tests/factor.rs @@ -0,0 +1,48 @@ +use std::{collections::HashSet, sync::Arc}; + +use factor_sqlite::SqliteFactor; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; + +#[derive(RuntimeFactors)] +struct TestFactors { + sqlite: SqliteFactor, +} + +#[tokio::test] +async fn sqlite_works() -> anyhow::Result<()> { + let test_resolver = RuntimeConfigResolver; + let factors = TestFactors { + sqlite: SqliteFactor::new(test_resolver), + }; + let env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + sqlite_databases = ["default"] + }); + let state = env.build_instance_state(factors).await?; + + assert_eq!( + state.sqlite.allowed_databases(), + &["default".into()].into_iter().collect::>() + ); + + Ok(()) +} + +struct RuntimeConfigResolver; + +#[allow(unused_variables)] +impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { + fn get_pool( + &self, + r#type: &str, + config: toml::Table, + ) -> anyhow::Result> { + todo!() + } + + fn default(&self, label: &str) -> Option> { + todo!() + } +} From 99628ee19f76539960097a720a0cd2fe0b0079c7 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 15 Jul 2024 12:56:38 -0400 Subject: [PATCH 046/195] factors: Enhance Factor doc comments Signed-off-by: Lann Martin --- crates/factors/src/factor.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 9778286ed..2a212574a 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -10,8 +10,8 @@ use crate::{ pub trait Factor: Any + Sized { /// The particular runtime configuration relevant to this factor. /// - /// Runtime configuration allows for user provided customization of the - /// factor's behavior on a per app basis. + /// Runtime configuration allows for user-provided customization of the + /// factor's behavior on a per-app basis. type RuntimeConfig: FactorRuntimeConfig; /// The application state of this factor. @@ -41,21 +41,24 @@ pub trait Factor: Any + Sized { /// - The `AppState` for any factors configured before this one /// /// A runtime may - but is not required to - reuse the returned config - /// across multiple instances. + /// across multiple instances. Because this method may be called + /// per-instantiation, it should avoid any blocking operations that could + /// unnecessarily delay execution. /// - /// This method may be called without any call to `init` or prepare in + /// This method may be called without any call to `init` or `prepare` in /// cases where only validation is needed (e.g., `spin doctor`). fn configure_app( &self, ctx: ConfigureAppContext, ) -> anyhow::Result; - /// Creates a new `FactorInstanceBuilder`, which will later build per-instance - /// state for this factor. + /// Creates a new `FactorInstanceBuilder`, which will later build + /// per-instance state for this factor. /// /// This method is given access to the app component being instantiated and /// to any other factors' instance builders that have already been prepared. - /// As such this is primary place for inter-factor dependencies. + /// As such, this is the primary place for inter-factor dependencies to be + /// used. fn prepare( &self, ctx: PrepareContext, From 112c4e4033cfdf888d46c589d5e17524c4563e23 Mon Sep 17 00:00:00 2001 From: Caleb Schoepp Date: Thu, 11 Jul 2024 16:31:23 -0600 Subject: [PATCH 047/195] factors: Add factor-outbound-pg Signed-off-by: Caleb Schoepp --- Cargo.lock | 20 + crates/factor-outbound-pg/Cargo.toml | 27 ++ crates/factor-outbound-pg/src/host.rs | 422 ++++++++++++++++++ crates/factor-outbound-pg/src/lib.rs | 53 +++ .../factor-outbound-pg/tests/factor_test.rs | 50 +++ 5 files changed, 572 insertions(+) create mode 100644 crates/factor-outbound-pg/Cargo.toml create mode 100644 crates/factor-outbound-pg/src/host.rs create mode 100644 crates/factor-outbound-pg/src/lib.rs create mode 100644 crates/factor-outbound-pg/tests/factor_test.rs diff --git a/Cargo.lock b/Cargo.lock index 97f108940..3109e91f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7556,6 +7556,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "spin-factor-outbound-pg" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "native-tls", + "postgres-native-tls", + "spin-core", + "spin-factor-outbound-networking", + "spin-factor-variables", + "spin-factor-wasi", + "spin-factors", + "spin-factors-test", + "spin-world", + "table", + "tokio", + "tokio-postgres", + "tracing", +] + [[package]] name = "spin-factor-variables" version = "2.7.0-pre0" diff --git a/crates/factor-outbound-pg/Cargo.toml b/crates/factor-outbound-pg/Cargo.toml new file mode 100644 index 000000000..ca18e93a1 --- /dev/null +++ b/crates/factor-outbound-pg/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "spin-factor-outbound-pg" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +native-tls = "0.2.11" +postgres-native-tls = "0.5.0" +spin-core = { path = "../core" } +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factors = { path = "../factors" } +spin-world = { path = "../world" } +table = { path = "../table" } +tokio = { version = "1", features = ["rt-multi-thread"] } +tokio-postgres = "0.7.7" +tracing = { workspace = true } + +[dev-dependencies] +spin-factor-variables = { path = "../factor-variables" } +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/factor-outbound-pg/src/host.rs b/crates/factor-outbound-pg/src/host.rs new file mode 100644 index 000000000..63bc9ac91 --- /dev/null +++ b/crates/factor-outbound-pg/src/host.rs @@ -0,0 +1,422 @@ +use anyhow::{anyhow, Result}; +use native_tls::TlsConnector; +use postgres_native_tls::MakeTlsConnector; +use spin_core::{async_trait, wasmtime::component::Resource}; +use spin_world::v1::postgres as v1; +use spin_world::v1::rdbms_types as v1_types; +use spin_world::v2::postgres::{self as v2, Connection}; +use spin_world::v2::rdbms_types; +use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue, RowSet}; +use tokio_postgres::{ + config::SslMode, + types::{ToSql, Type}, + Client, NoTls, Row, Socket, +}; +use tracing::instrument; +use tracing::Level; + +use crate::InstanceState; + +impl InstanceState { + async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { + self.connections + .push( + build_client(address) + .await + .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, + ) + .map_err(|_| v2::Error::ConnectionFailed("too many connections".into())) + .map(Resource::new_own) + } + + async fn get_client(&mut self, connection: Resource) -> Result<&Client, v2::Error> { + self.connections + .get(connection.rep()) + .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) + } + + async fn is_address_allowed(&self, address: &str) -> Result { + let Ok(config) = address.parse::() else { + return Ok(false); + }; + for (i, host) in config.get_hosts().iter().enumerate() { + match host { + tokio_postgres::config::Host::Tcp(address) => { + let ports = config.get_ports(); + // The port we use is either: + // * The port at the same index as the host + // * The first port if there is only one port + let port = + ports + .get(i) + .or_else(|| if ports.len() == 1 { ports.get(1) } else { None }); + let port_str = port.map(|p| format!(":{}", p)).unwrap_or_default(); + let url = format!("{address}{port_str}"); + // TODO: Should I be unwrapping this? + if !self.allowed_hosts.check_url(&url, "postgres").await? { + return Ok(false); + } + } + #[cfg(unix)] + tokio_postgres::config::Host::Unix(_) => return Ok(false), + } + } + Ok(true) + } +} + +#[async_trait] +impl v2::Host for InstanceState {} + +#[async_trait] +impl v2::HostConnection for InstanceState { + #[instrument(name = "spin_outbound_pg.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql"))] + async fn open(&mut self, address: String) -> Result, v2::Error> { + if !self + .is_address_allowed(&address) + .await + .map_err(|e| v2::Error::Other(e.to_string()))? + { + return Err(v2::Error::ConnectionFailed(format!( + "address {address} is not permitted" + ))); + } + self.open_connection(&address).await + } + + #[instrument(name = "spin_outbound_pg.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql", otel.name = statement))] + async fn execute( + &mut self, + connection: Resource, + statement: String, + params: Vec, + ) -> Result { + let params: Vec<&(dyn ToSql + Sync)> = params + .iter() + .map(to_sql_parameter) + .collect::>>() + .map_err(|e| v2::Error::ValueConversionFailed(format!("{:?}", e)))?; + + let nrow = self + .get_client(connection) + .await? + .execute(&statement, params.as_slice()) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + Ok(nrow) + } + + #[instrument(name = "spin_outbound_pg.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql", otel.name = statement))] + async fn query( + &mut self, + connection: Resource, + statement: String, + params: Vec, + ) -> Result { + let params: Vec<&(dyn ToSql + Sync)> = params + .iter() + .map(to_sql_parameter) + .collect::>>() + .map_err(|e| v2::Error::BadParameter(format!("{:?}", e)))?; + + let results = self + .get_client(connection) + .await? + .query(&statement, params.as_slice()) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + if results.is_empty() { + return Ok(RowSet { + columns: vec![], + rows: vec![], + }); + } + + let columns = infer_columns(&results[0]); + let rows = results + .iter() + .map(convert_row) + .collect::, _>>() + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + Ok(RowSet { columns, rows }) + } + + fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { + self.connections.remove(connection.rep()); + Ok(()) + } +} + +impl rdbms_types::Host for InstanceState { + fn convert_error(&mut self, error: v2::Error) -> Result { + Ok(error) + } +} + +fn to_sql_parameter(value: &ParameterValue) -> anyhow::Result<&(dyn ToSql + Sync)> { + match value { + ParameterValue::Boolean(v) => Ok(v), + ParameterValue::Int32(v) => Ok(v), + ParameterValue::Int64(v) => Ok(v), + ParameterValue::Int8(v) => Ok(v), + ParameterValue::Int16(v) => Ok(v), + ParameterValue::Floating32(v) => Ok(v), + ParameterValue::Floating64(v) => Ok(v), + ParameterValue::Uint8(_) + | ParameterValue::Uint16(_) + | ParameterValue::Uint32(_) + | ParameterValue::Uint64(_) => Err(anyhow!("Postgres does not support unsigned integers")), + ParameterValue::Str(v) => Ok(v), + ParameterValue::Binary(v) => Ok(v), + ParameterValue::DbNull => Ok(&PgNull), + } +} + +fn infer_columns(row: &Row) -> Vec { + let mut result = Vec::with_capacity(row.len()); + for index in 0..row.len() { + result.push(infer_column(row, index)); + } + result +} + +fn infer_column(row: &Row, index: usize) -> Column { + let column = &row.columns()[index]; + let name = column.name().to_owned(); + let data_type = convert_data_type(column.type_()); + Column { name, data_type } +} + +fn convert_data_type(pg_type: &Type) -> DbDataType { + match *pg_type { + Type::BOOL => DbDataType::Boolean, + Type::BYTEA => DbDataType::Binary, + Type::FLOAT4 => DbDataType::Floating32, + Type::FLOAT8 => DbDataType::Floating64, + Type::INT2 => DbDataType::Int16, + Type::INT4 => DbDataType::Int32, + Type::INT8 => DbDataType::Int64, + Type::TEXT | Type::VARCHAR | Type::BPCHAR => DbDataType::Str, + _ => { + tracing::debug!("Couldn't convert Postgres type {} to WIT", pg_type.name(),); + DbDataType::Other + } + } +} + +fn convert_row(row: &Row) -> Result, tokio_postgres::Error> { + let mut result = Vec::with_capacity(row.len()); + for index in 0..row.len() { + result.push(convert_entry(row, index)?); + } + Ok(result) +} + +fn convert_entry(row: &Row, index: usize) -> Result { + let column = &row.columns()[index]; + let value = match column.type_() { + &Type::BOOL => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Boolean(v), + None => DbValue::DbNull, + } + } + &Type::BYTEA => { + let value: Option> = row.try_get(index)?; + match value { + Some(v) => DbValue::Binary(v), + None => DbValue::DbNull, + } + } + &Type::FLOAT4 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Floating32(v), + None => DbValue::DbNull, + } + } + &Type::FLOAT8 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Floating64(v), + None => DbValue::DbNull, + } + } + &Type::INT2 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Int16(v), + None => DbValue::DbNull, + } + } + &Type::INT4 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Int32(v), + None => DbValue::DbNull, + } + } + &Type::INT8 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Int64(v), + None => DbValue::DbNull, + } + } + &Type::TEXT | &Type::VARCHAR | &Type::BPCHAR => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Str(v), + None => DbValue::DbNull, + } + } + t => { + tracing::debug!( + "Couldn't convert Postgres type {} in column {}", + t.name(), + column.name() + ); + DbValue::Unsupported + } + }; + Ok(value) +} + +async fn build_client(address: &str) -> anyhow::Result { + let config = address.parse::()?; + + tracing::debug!("Build new connection: {}", address); + + if config.get_ssl_mode() == SslMode::Disable { + connect(config).await + } else { + connect_tls(config).await + } +} + +async fn connect(config: tokio_postgres::Config) -> anyhow::Result { + let (client, connection) = config.connect(NoTls).await?; + + spawn(connection); + + Ok(client) +} + +async fn connect_tls(config: tokio_postgres::Config) -> anyhow::Result { + let builder = TlsConnector::builder(); + let connector = MakeTlsConnector::new(builder.build()?); + let (client, connection) = config.connect(connector).await?; + + spawn(connection); + + Ok(client) +} + +fn spawn(connection: tokio_postgres::Connection) +where + T: tokio_postgres::tls::TlsStream + std::marker::Unpin + std::marker::Send + 'static, +{ + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Postgres connection error: {}", e); + } + }); +} + +/// Although the Postgres crate converts Rust Option::None to Postgres NULL, +/// it enforces the type of the Option as it does so. (For example, trying to +/// pass an Option::::None to a VARCHAR column fails conversion.) As we +/// do not know expected column types, we instead use a "neutral" custom type +/// which allows conversion to any type but always tells the Postgres crate to +/// treat it as a SQL NULL. +struct PgNull; + +impl ToSql for PgNull { + fn to_sql( + &self, + _ty: &Type, + _out: &mut tokio_postgres::types::private::BytesMut, + ) -> Result> + where + Self: Sized, + { + Ok(tokio_postgres::types::IsNull::Yes) + } + + fn accepts(_ty: &Type) -> bool + where + Self: Sized, + { + true + } + + fn to_sql_checked( + &self, + _ty: &Type, + _out: &mut tokio_postgres::types::private::BytesMut, + ) -> Result> { + Ok(tokio_postgres::types::IsNull::Yes) + } +} + +impl std::fmt::Debug for PgNull { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NULL").finish() + } +} + +/// Delegate a function call to the v2::HostConnection implementation +macro_rules! delegate { + ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ + if !$self.is_address_allowed(&$address).await.map_err(|e| v2::Error::Other(e.to_string()))? { + return Err(v1::PgError::ConnectionFailed(format!( + "address {} is not permitted", $address + ))); + } + let connection = match $self.open_connection(&$address).await { + Ok(c) => c, + Err(e) => return Err(e.into()), + }; + ::$name($self, connection, $($arg),*) + .await + .map_err(|e| e.into()) + }}; +} + +#[async_trait] +impl v1::Host for InstanceState { + async fn execute( + &mut self, + address: String, + statement: String, + params: Vec, + ) -> Result { + delegate!(self.execute( + address, + statement, + params.into_iter().map(Into::into).collect() + )) + } + + async fn query( + &mut self, + address: String, + statement: String, + params: Vec, + ) -> Result { + delegate!(self.query( + address, + statement, + params.into_iter().map(Into::into).collect() + )) + .map(Into::into) + } + + fn convert_pg_error(&mut self, error: v1::PgError) -> Result { + Ok(error) + } +} diff --git a/crates/factor-outbound-pg/src/lib.rs b/crates/factor-outbound-pg/src/lib.rs new file mode 100644 index 000000000..143666932 --- /dev/null +++ b/crates/factor-outbound-pg/src/lib.rs @@ -0,0 +1,53 @@ +mod host; + +use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; +use spin_factors::{ + anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, + SelfInstanceBuilder, +}; +use tokio_postgres::Client; + +pub struct OutboundPgFactor; + +impl Factor for OutboundPgFactor { + type RuntimeConfig = (); + type AppState = (); + type InstanceBuilder = InstanceState; + + fn init( + &mut self, + mut ctx: spin_factors::InitContext, + ) -> anyhow::Result<()> { + ctx.link_bindings(spin_world::v1::postgres::add_to_linker)?; + ctx.link_bindings(spin_world::v2::postgres::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + _ctx: ConfigureAppContext, + ) -> anyhow::Result { + Ok(()) + } + + fn prepare( + &self, + _ctx: PrepareContext, + builders: &mut InstanceBuilders, + ) -> anyhow::Result { + let allowed_hosts = builders + .get_mut::()? + .allowed_hosts(); + Ok(InstanceState { + allowed_hosts, + connections: Default::default(), + }) + } +} + +pub struct InstanceState { + allowed_hosts: OutboundAllowedHosts, + connections: table::Table, +} + +impl SelfInstanceBuilder for InstanceState {} diff --git a/crates/factor-outbound-pg/tests/factor_test.rs b/crates/factor-outbound-pg/tests/factor_test.rs new file mode 100644 index 000000000..4f2f78852 --- /dev/null +++ b/crates/factor-outbound-pg/tests/factor_test.rs @@ -0,0 +1,50 @@ +use anyhow::bail; +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_outbound_pg::OutboundPgFactor; +use spin_factor_variables::{StaticVariables, VariablesFactor}; +use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use spin_world::v2::postgres::HostConnection; +use spin_world::v2::rdbms_types::Error as PgError; + +#[derive(RuntimeFactors)] +struct TestFactors { + wasi: WasiFactor, + variables: VariablesFactor, + networking: OutboundNetworkingFactor, + pg: OutboundPgFactor, +} + +fn test_env() -> TestEnvironment { + TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + }) +} + +#[tokio::test] +async fn disallowed_host_fails() -> anyhow::Result<()> { + let mut factors = TestFactors { + wasi: WasiFactor::new(DummyFilesMounter), + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + pg: OutboundPgFactor, + }; + factors.variables.add_provider_type(StaticVariables)?; + + let env = test_env(); + let mut state = env.build_instance_state(factors).await?; + + let res = state + .pg + .open("postgres://postgres.test:5432/test".to_string()) + .await; + let Err(err) = res else { + bail!("expected Err, got Ok"); + }; + println!("err: {:?}", err); + assert!(matches!(err, PgError::ConnectionFailed(_))); + + Ok(()) +} From f5910a75e92d53857cfd1521a2d17fcf66b9c426 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 15 Jul 2024 16:43:27 -0400 Subject: [PATCH 048/195] factors: Add RuntimeFactors::prepare This breaks RuntimeFactors::build_instance_state into two steps, mirroring Factor's methods. Some triggers will need access to these builders, e.g. WAGI needs access to WasiFactor's stdin configuration. Signed-off-by: Lann Martin --- crates/factors-derive/src/lib.rs | 11 +++++++++-- crates/factors-test/src/lib.rs | 4 +++- crates/factors/src/runtime_factors.rs | 10 ++++++++-- crates/factors/tests/smoke.rs | 5 +++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index f5ca04d98..98e427cec 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -118,10 +118,10 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { Ok(#ConfiguredApp::new(app, app_state)) } - fn build_instance_state( + fn prepare( &self, configured_app: &#ConfiguredApp, component_id: &str, - ) -> #Result { + ) -> #Result { let app_component = configured_app.app().get_component(component_id).ok_or_else(|| { #factors_path::Error::UnknownComponent(component_id.to_string()) })?; @@ -140,6 +140,13 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { ).map_err(#Error::factor_prepare_error::<#factor_types>)? ); )* + Ok(builders) + } + + fn build_instance_state( + &self, + builders: Self::InstanceBuilders, + ) -> #Result { Ok(#state_name { #( #factor_names: #FactorInstanceBuilder::build( diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 4f18019b9..856f36671 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -69,7 +69,9 @@ impl TestEnvironment { configured_app.app().components().next().context( "expected configured app to have at least one component, but it did not", )?; - Ok(factors.build_instance_state(&configured_app, component.id())?) + let builders = factors.prepare(&configured_app, component.id())?; + + Ok(factors.build_instance_state(builders)?) } pub fn new_linker() -> Linker { diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 34f435add..adf9c6c2d 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -46,11 +46,17 @@ pub trait RuntimeFactors: Sized + 'static { runtime_config: impl RuntimeConfigSource, ) -> crate::Result>; - /// Build the instance state for the factors. - fn build_instance_state( + /// Prepare the factors' instance state builders. + fn prepare( &self, configured_app: &ConfiguredApp, component_id: &str, + ) -> crate::Result; + + /// Build the instance state for the factors. + fn build_instance_state( + &self, + builders: Self::InstanceBuilders, ) -> crate::Result; /// Get the app state related to a particular factor. diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 070645b0c..b54ae3879 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -5,13 +5,13 @@ use http_body_util::BodyExt; use serde::Deserialize; use spin_app::App; use spin_factor_key_value::{KeyValueFactor, MakeKeyValueStore}; +use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; -use spin_factor_key_value_redis::RedisKeyValueStore; use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] @@ -53,7 +53,8 @@ async fn smoke_test_works() -> anyhow::Result<()> { factors.init(&mut linker).unwrap(); let configured_app = factors.configure_app(app, TestSource)?; - let data = factors.build_instance_state(&configured_app, "smoke-app")?; + let builders = factors.prepare(&configured_app, "smoke-app")?; + let data = factors.build_instance_state(builders)?; assert_eq!( data.variables From 3b3d42d152475373b1eaa381d18d02913206ad24 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 16 Jul 2024 12:09:07 +0200 Subject: [PATCH 049/195] Small cleanup Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/lib.rs | 33 +++++++++++++--------- crates/factor-sqlite/src/runtime_config.rs | 9 ++++-- crates/factor-sqlite/tests/factor.rs | 5 ++-- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 9c26609e5..491c8e1d0 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -13,13 +13,13 @@ use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; pub struct SqliteFactor { - runtime_config_resolver: Arc, + runtime_config_resolver: Arc, } impl SqliteFactor { /// Create a new `SqliteFactor` pub fn new( - runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + Send + Sync + 'static, + runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, ) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), @@ -27,8 +27,6 @@ impl SqliteFactor { } } -pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); - impl Factor for SqliteFactor { type RuntimeConfig = runtime_config::RuntimeConfig; type AppState = AppState; @@ -49,10 +47,17 @@ impl Factor for SqliteFactor { ) -> anyhow::Result { let mut connection_pools = HashMap::new(); if let Some(runtime_config) = ctx.take_runtime_config() { - for (database_label, runtime_config::StoreConfig { type_, config }) in - runtime_config.store_configs + for ( + database_label, + runtime_config::StoreConfig { + type_: database_kind, + config, + }, + ) in runtime_config.store_configs { - let pool = self.runtime_config_resolver.get_pool(&type_, config)?; + let pool = self + .runtime_config_resolver + .get_pool(&database_kind, config)?; connection_pools.insert(database_label, pool); } } @@ -75,7 +80,7 @@ impl Factor for SqliteFactor { let resolver = self.runtime_config_resolver.clone(); Ok(AppState { allowed_databases, - connection_pools: Arc::new(move |label| { + get_connection_pool: Arc::new(move |label| { connection_pools .get(label) .cloned() @@ -95,16 +100,18 @@ impl Factor for SqliteFactor { .get(ctx.app_component().id()) .cloned() .unwrap_or_default(); - let connection_pools = ctx.app_state().connection_pools.clone(); - Ok(InstanceState::new(allowed_databases, connection_pools)) + let get_connection_pool = ctx.app_state().get_connection_pool.clone(); + Ok(InstanceState::new(allowed_databases, get_connection_pool)) } } +pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); + pub struct AppState { - /// A map from component id to a set of allowed databases + /// A map from component id to a set of allowed database labels. allowed_databases: HashMap>>, - /// A map from database name to a connection pool - connection_pools: host::ConnectionPoolGetter, + /// A function for mapping from database name to a connection pool + get_connection_pool: host::ConnectionPoolGetter, } /// A store of connections for all accessible databases for an application diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index 2540484a5..0803d9937 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -24,11 +24,14 @@ pub struct StoreConfig { } /// Resolves some piece of runtime configuration to a connection pool -pub trait RuntimeConfigResolver { - /// Get a connection pool for a given runtime configuration type and the raw configuration. +pub trait RuntimeConfigResolver: Send + Sync { + /// Get a connection pool for a given database kind and the raw configuration. + /// + /// `database_kind` is equivalent to the `type` field in the + /// `[sqlite_database.$databasename]` runtime configuration table. fn get_pool( &self, - r#type: &str, + database_kind: &str, config: toml::Table, ) -> anyhow::Result>; diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index ec83c27b2..d1a876950 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -32,17 +32,18 @@ async fn sqlite_works() -> anyhow::Result<()> { struct RuntimeConfigResolver; -#[allow(unused_variables)] impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { fn get_pool( &self, - r#type: &str, + database_kind: &str, config: toml::Table, ) -> anyhow::Result> { + let _ = (database_kind, config); todo!() } fn default(&self, label: &str) -> Option> { + let _ = label; todo!() } } From ef443e6f57ea85bfb4662e12d40ae62bf69180dc Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 16 Jul 2024 14:31:31 +0200 Subject: [PATCH 050/195] Add runtime config impl that Spin uses to factor-sqlite Signed-off-by: Ryan Levick --- Cargo.lock | 3 + crates/factor-sqlite/Cargo.toml | 13 + crates/factor-sqlite/src/lib.rs | 23 +- crates/factor-sqlite/src/runtime_config.rs | 3 + .../factor-sqlite/src/runtime_config/spin.rs | 237 ++++++++++++++++++ 5 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 crates/factor-sqlite/src/runtime_config/spin.rs diff --git a/Cargo.lock b/Cargo.lock index b2c87a190..b2afa04e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2445,6 +2445,9 @@ dependencies = [ "spin-factors", "spin-factors-test", "spin-locked-app", + "spin-sqlite", + "spin-sqlite-inproc", + "spin-sqlite-libsql", "spin-world", "table", "tokio", diff --git a/crates/factor-sqlite/Cargo.toml b/crates/factor-sqlite/Cargo.toml index f7ad00a55..24442bd33 100644 --- a/crates/factor-sqlite/Cargo.toml +++ b/crates/factor-sqlite/Cargo.toml @@ -15,11 +15,24 @@ spin-factors = { path = "../factors" } spin-locked-app = { path = "../locked-app" } spin-world = { path = "../world" } table = { path = "../table" } +tokio = "1" toml = "0.8" +spin-sqlite = { path = "../sqlite", optional = true } +spin-sqlite-inproc = { path = "../sqlite-inproc", optional = true } +spin-sqlite-libsql = { path = "../sqlite-libsql", optional = true } [dev-dependencies] spin-factors-test = { path = "../factors-test" } tokio = { version = "1", features = ["macros", "rt"] } +[features] +default = ["spin-cli"] +# Includes the runtime configuration handling used by the Spin CLI +spin-cli = [ + "dep:spin-sqlite", + "dep:spin-sqlite-inproc", + "dep:spin-sqlite-libsql", +] + [lints] workspace = true diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 491c8e1d0..68e95e992 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -114,13 +114,34 @@ pub struct AppState { get_connection_pool: host::ConnectionPoolGetter, } -/// A store of connections for all accessible databases for an application +/// A pool of connections for a particular SQLite database #[async_trait] pub trait ConnectionPool: Send + Sync { /// Get a `Connection` from the pool async fn get_connection(&self) -> Result, v2::Error>; } +/// A simple [`ConnectionPool`] that always creates a new connection. +pub struct SimpleConnectionPool( + Box anyhow::Result> + Send + Sync>, +); + +impl SimpleConnectionPool { + /// Create a new `SimpleConnectionPool` with the given connection factory. + pub fn new( + factory: impl Fn() -> anyhow::Result> + Send + Sync + 'static, + ) -> Self { + Self(Box::new(factory)) + } +} + +#[async_trait::async_trait] +impl ConnectionPool for SimpleConnectionPool { + async fn get_connection(&self) -> Result, v2::Error> { + (self.0)().map_err(|_| v2::Error::InvalidConnection) + } +} + /// A trait abstracting over operations to a SQLite database #[async_trait] pub trait Connection: Send + Sync { diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index 0803d9937..2089d079a 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "spin-cli")] +pub mod spin; + use std::{collections::HashMap, sync::Arc}; use serde::Deserialize; diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs new file mode 100644 index 000000000..674e3e2b7 --- /dev/null +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -0,0 +1,237 @@ +//! Spin's default handling of the runtime configuration for SQLite databases. + +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use serde::Deserialize; +use spin_factors::anyhow::{self, Context as _}; +use spin_world::v2::sqlite as v2; +use tokio::sync::OnceCell; + +use crate::{Connection, ConnectionPool, SimpleConnectionPool}; + +use super::RuntimeConfigResolver; + +/// Spin's default handling of the runtime configuration for SQLite databases. +/// +/// This type implements the [`RuntimeConfigResolver`] trait and provides a way to +/// opt into the default behavior of Spin's SQLite database handling. +pub struct SpinSqliteRuntimeConfig { + state_dir: PathBuf, + base_path: Option, +} + +impl SpinSqliteRuntimeConfig { + /// Create a new `SpinSqliteRuntimeConfig` + /// + /// This takes as arguments: + /// * the state directory path (i.e., the path to the `.spin` file). This + /// is used to derive the default path a local SQLite database file. + /// * the base path from which relative paths referenced in configuration are resolved + /// (this should most likely be the path to the runtime-config file). If + /// `None`, the current working directory is used. + pub fn new(state_dir: PathBuf, base_path: Option) -> Self { + Self { + state_dir, + base_path, + } + } +} + +impl RuntimeConfigResolver for SpinSqliteRuntimeConfig { + fn get_pool( + &self, + database_kind: &str, + config: toml::Table, + ) -> anyhow::Result> { + let pool = match database_kind { + "spin" => { + let config: LocalDatabase = config.try_into()?; + config.pool(self.base_path.as_deref())? + } + "libsql" => { + let config: LibSqlDatabase = config.try_into()?; + config.pool()? + } + _ => anyhow::bail!("Unknown database kind: {}", database_kind), + }; + Ok(Arc::new(pool)) + } + + fn default(&self, label: &str) -> Option> { + // Only default the database labeled "default". + if label != "default" { + return None; + } + + let path = self.state_dir.join(DEFAULT_SQLITE_DB_FILENAME); + let factory = move || { + let location = spin_sqlite_inproc::InProcDatabaseLocation::Path(path.clone()); + let connection = spin_sqlite_inproc::InProcConnection::new(location)?; + Ok(Arc::new(connection) as _) + }; + let pool = SimpleConnectionPool::new(factory); + Some(Arc::new(pool)) + } +} + +const DEFAULT_SQLITE_DB_FILENAME: &str = "sqlite_db.db"; + +#[async_trait::async_trait] +impl Connection for spin_sqlite_inproc::InProcConnection { + async fn query( + &self, + query: &str, + parameters: Vec, + ) -> Result { + ::query(self, query, parameters).await + } + + async fn execute_batch(&self, statements: &str) -> anyhow::Result<()> { + ::execute_batch(self, statements).await + } +} + +/// A wrapper around a libSQL connection that implements the [`Connection`] trait. +struct LibSqlConnection { + url: String, + token: String, + // Since the libSQL client can only be created asynchronously, we wait until + // we're in the `Connection` implementation to create. Since we only want to do + // this once, we use a `OnceCell` to store it. + inner: OnceCell, +} + +impl LibSqlConnection { + fn new(url: String, token: String) -> Self { + Self { + url, + token, + inner: OnceCell::new(), + } + } + + async fn get_client(&self) -> Result<&spin_sqlite_libsql::LibsqlClient, v2::Error> { + self.inner + .get_or_try_init(|| async { + spin_sqlite_libsql::LibsqlClient::create(self.url.clone(), self.token.clone()) + .await + .context("failed to create SQLite client") + }) + .await + .map_err(|_| v2::Error::InvalidConnection) + } +} + +#[async_trait::async_trait] +impl Connection for LibSqlConnection { + async fn query( + &self, + query: &str, + parameters: Vec, + ) -> Result { + let client = self.get_client().await?; + ::query( + client, query, parameters, + ) + .await + } + + async fn execute_batch(&self, statements: &str) -> anyhow::Result<()> { + let client = self.get_client().await?; + ::execute_batch( + client, statements, + ) + .await + } +} + +/// Configuration for a local SQLite database. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalDatabase { + pub path: Option, +} + +impl LocalDatabase { + /// Create a new connection pool for a local database. + fn pool(self, base_path: Option<&Path>) -> anyhow::Result { + let location = match self.path { + Some(path) => { + // TODO: `base_path` should be passed in from the runtime config + let path = resolve_relative_path(&path, base_path)?; + // Create the store's parent directory if necessary + // unwrapping the parent is fine, because `resolve_relative_path`` will always return a path with a parent + std::fs::create_dir_all(path.parent().unwrap()) + .context("Failed to create sqlite database directory")?; + spin_sqlite_inproc::InProcDatabaseLocation::Path(path) + } + None => spin_sqlite_inproc::InProcDatabaseLocation::InMemory, + }; + let factory = move || { + let connection = spin_sqlite_inproc::InProcConnection::new(location.clone())?; + Ok(Arc::new(connection) as _) + }; + Ok(SimpleConnectionPool::new(factory)) + } +} + +/// Resolve a relative path against an optional base path. +fn resolve_relative_path(path: &Path, base_path: Option<&Path>) -> anyhow::Result { + if path.is_absolute() { + return Ok(path.to_owned()); + } + let base_path = match base_path { + Some(base_path) => base_path + .parent() + .with_context(|| { + format!( + "failed to get parent of runtime config file path \"{}\"", + base_path.display() + ) + })? + .to_owned(), + None => std::env::current_dir().context("failed to get current directory")?, + }; + Ok(base_path.join(path)) +} + +/// Configuration for a libSQL database. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LibSqlDatabase { + url: String, + token: String, +} + +impl LibSqlDatabase { + /// Create a new connection pool for a libSQL database. + fn pool(self) -> anyhow::Result { + let url = check_url(&self.url) + .with_context(|| { + format!( + "unexpected libSQL URL '{}' in runtime config file ", + self.url + ) + })? + .to_owned(); + let factory = move || { + let connection = LibSqlConnection::new(url.clone(), self.token.clone()); + Ok(Arc::new(connection) as _) + }; + Ok(SimpleConnectionPool::new(factory)) + } +} + +// Checks an incoming url is in the shape we expect +fn check_url(url: &str) -> anyhow::Result<&str> { + if url.starts_with("https://") || url.starts_with("http://") { + Ok(url) + } else { + Err(anyhow::anyhow!( + "URL does not start with 'https://' or 'http://'. Spin currently only supports talking to libSQL databases over HTTP(S)" + )) + } +} From 36be557b5d684ee0a709b48eeb8a9b8cb3b62c89 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 16 Jul 2024 11:47:50 -0400 Subject: [PATCH 051/195] factors: Enhance OutboundNetworkingFactor - Make dependency on WasiFactor optional - Fix wasi socket addr config and add test - Add support for explicit 'tcp' and 'udp' schemes in allowed_outbound_hosts for wasi sockets Signed-off-by: Lann Martin --- Cargo.lock | 3 + crates/factor-outbound-networking/Cargo.toml | 5 ++ crates/factor-outbound-networking/src/lib.rs | 55 ++++++++++--------- .../tests/factor_test.rs | 55 +++++++++++++++++++ crates/factor-wasi/src/lib.rs | 8 ++- 5 files changed, 99 insertions(+), 27 deletions(-) create mode 100644 crates/factor-outbound-networking/tests/factor_test.rs diff --git a/Cargo.lock b/Cargo.lock index eb604ee8f..6d8cb0881 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7576,8 +7576,11 @@ dependencies = [ "spin-factor-variables", "spin-factor-wasi", "spin-factors", + "spin-factors-test", "spin-outbound-networking", + "tokio", "tracing", + "wasmtime-wasi", ] [[package]] diff --git a/crates/factor-outbound-networking/Cargo.toml b/crates/factor-outbound-networking/Cargo.toml index 13dd49c9e..c24284d13 100644 --- a/crates/factor-outbound-networking/Cargo.toml +++ b/crates/factor-outbound-networking/Cargo.toml @@ -14,5 +14,10 @@ spin-factors = { path = "../factors" } spin-outbound-networking = { path = "../outbound-networking" } tracing = { workspace = true } +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } +wasmtime-wasi = { workspace = true } + [lints] workspace = true diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index 66b5e3732..b04262e3f 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -5,10 +5,10 @@ use futures_util::{ FutureExt, }; use spin_factor_variables::VariablesFactor; -use spin_factor_wasi::WasiFactor; +use spin_factor_wasi::{SocketAddrUse, WasiFactor}; use spin_factors::{ anyhow::{self, Context}, - ConfigureAppContext, Factor, FactorInstanceBuilder, InstanceBuilders, PrepareContext, + ConfigureAppContext, Error, Factor, FactorInstanceBuilder, InstanceBuilders, PrepareContext, RuntimeFactors, }; use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; @@ -67,31 +67,36 @@ impl Factor for OutboundNetworkingFactor { .map(|res| res.map(Arc::new).map_err(Arc::new)) .boxed() .shared(); - // let prepared_resolver = resolver.prepare().await?; - // let allowed_hosts = AllowedHostsConfig::parse( - // .context("missing component allowed hosts")?, - // &prepared_resolver, - // )?; - - // Update Wasi socket allowed ports - let wasi_preparer = builders.get_mut::()?; - let hosts_future = allowed_hosts_future.clone(); - wasi_preparer.outbound_socket_addr_check(move |addr| { - let hosts_future = hosts_future.clone(); - async move { - match hosts_future.await { - Ok(allowed_hosts) => { - // TODO: verify this actually works... - spin_outbound_networking::check_url(&addr.to_string(), "*", &allowed_hosts) - } - Err(err) => { - // TODO: should this trap (somehow)? - tracing::error!(%err, "allowed_outbound_hosts variable resolution failed"); - false + + match builders.get_mut::() { + Ok(wasi_builder) => { + // Update Wasi socket allowed ports + let hosts_future = allowed_hosts_future.clone(); + wasi_builder.outbound_socket_addr_check(move |addr, addr_use| { + let hosts_future = hosts_future.clone(); + async move { + match hosts_future.await { + Ok(allowed_hosts) => { + // TODO: validate against existing spin-core behavior + let scheme = match addr_use { + SocketAddrUse::TcpBind => return false, + SocketAddrUse::TcpConnect => "tcp", + SocketAddrUse::UdpBind | SocketAddrUse::UdpConnect | SocketAddrUse::UdpOutgoingDatagram => "udp", + }; + spin_outbound_networking::check_url(&addr.to_string(),scheme, &allowed_hosts) + } + Err(err) => { + // TODO: should this trap (somehow)? + tracing::error!(%err, "allowed_outbound_hosts variable resolution failed"); + false + } + } } - } + }); } - }); + Err(Error::NoSuchFactor(_)) => (), // no WasiFactor to configure; that's OK + Err(err) => return Err(err.into()), + } Ok(InstanceBuilder { allowed_hosts_future, }) diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs new file mode 100644 index 000000000..1e00d05a5 --- /dev/null +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -0,0 +1,55 @@ +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_variables::VariablesFactor; +use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use wasmtime_wasi::{bindings::sockets::instance_network::Host, SocketAddrUse, WasiImpl, WasiView}; + +#[derive(RuntimeFactors)] +struct TestFactors { + wasi: WasiFactor, + variables: VariablesFactor, + networking: OutboundNetworkingFactor, +} + +fn test_env() -> TestEnvironment { + TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + allowed_outbound_hosts = ["*://192.0.2.1:12345"] + }) +} + +#[tokio::test] +async fn configures_wasi_socket_addr_check() -> anyhow::Result<()> { + let factors = TestFactors { + wasi: WasiFactor::new(DummyFilesMounter), + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + }; + + let env = test_env(); + let mut state = env.build_instance_state(factors).await?; + let mut wasi = WasiImpl(&mut state.wasi); + + let network_resource = wasi.instance_network()?; + let network = wasi.table().get(&network_resource)?; + + network + .check_socket_addr( + "192.0.2.1:12345".parse().unwrap(), + SocketAddrUse::TcpConnect, + ) + .await?; + for not_allowed in ["192.0.2.1:25", "192.0.2.2:12345"] { + assert_eq!( + network + .check_socket_addr(not_allowed.parse().unwrap(), SocketAddrUse::TcpConnect) + .await + .unwrap_err() + .kind(), + std::io::ErrorKind::PermissionDenied + ); + } + Ok(()) +} diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 588c22e7d..3dd769b9a 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -11,6 +11,8 @@ use wasmtime_wasi::{ StdoutStream, WasiCtx, WasiCtxBuilder, WasiImpl, WasiView, }; +pub use wasmtime_wasi::SocketAddrUse; + pub struct WasiFactor { files_mounter: Box, } @@ -236,7 +238,7 @@ impl FactorInstanceBuilder for InstanceBuilder { impl InstanceBuilder { pub fn outbound_socket_addr_check(&mut self, check: F) where - F: Fn(SocketAddr) -> Fut + Send + Sync + Clone + 'static, + F: Fn(SocketAddr, SocketAddrUse) -> Fut + Send + Sync + Clone + 'static, Fut: Future + Send + Sync, { self.ctx.socket_addr_check(move |addr, addr_use| { @@ -247,7 +249,9 @@ impl InstanceBuilder { wasmtime_wasi::SocketAddrUse::TcpConnect | wasmtime_wasi::SocketAddrUse::UdpBind | wasmtime_wasi::SocketAddrUse::UdpConnect - | wasmtime_wasi::SocketAddrUse::UdpOutgoingDatagram => check(addr).await, + | wasmtime_wasi::SocketAddrUse::UdpOutgoingDatagram => { + check(addr, addr_use).await + } } }) }); From 8a8b243e5f378e7c112edb3e5c68173ecf16ac79 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 16 Jul 2024 12:34:47 -0400 Subject: [PATCH 052/195] factors: Test OutboundNetworkingFactor wasi dependency optionality Signed-off-by: Lann Martin --- .../tests/factor_test.rs | 16 ++++++++++++++++ crates/factors-test/src/lib.rs | 7 +++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs index 1e00d05a5..75d155d36 100644 --- a/crates/factor-outbound-networking/tests/factor_test.rs +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -53,3 +53,19 @@ async fn configures_wasi_socket_addr_check() -> anyhow::Result<()> { } Ok(()) } + +#[tokio::test] +async fn wasi_factor_is_optional() -> anyhow::Result<()> { + #[derive(RuntimeFactors)] + struct WithoutWasi { + variables: VariablesFactor, + networking: OutboundNetworkingFactor, + } + TestEnvironment::default() + .build_instance_state(WithoutWasi { + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + }) + .await?; + Ok(()) +} diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 856f36671..0308bd746 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -26,6 +26,9 @@ impl Default for TestEnvironment { name = "test-app" [[trigger.test-trigger]] + + [component.empty] + source = "does-not-exist.wasm" }; Self { manifest, @@ -49,7 +52,7 @@ impl TestEnvironment { /// Starting from a new _uninitialized_ [`RuntimeFactors`], run through the /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`] - /// for the first component defined in the manifest. + /// for the last component defined in the manifest. pub async fn build_instance_state( &self, mut factors: T, @@ -66,7 +69,7 @@ impl TestEnvironment { let configured_app = factors.configure_app(app, runtime_config)?; let component = - configured_app.app().components().next().context( + configured_app.app().components().last().context( "expected configured app to have at least one component, but it did not", )?; let builders = factors.prepare(&configured_app, component.id())?; From b2b91e06cbba3e987fe8d8c8284244a5a206b220 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Tue, 16 Jul 2024 18:34:46 -0500 Subject: [PATCH 053/195] factors: handle default kv store resolution Signed-off-by: Kate Goldenring --- crates/factor-key-value/src/lib.rs | 111 ++++++------------ crates/factor-key-value/src/runtime_config.rs | 42 +++++++ .../factor-key-value/src/spin_cli_resolver.rs | 47 ++++++++ crates/factor-key-value/src/store.rs | 5 +- crates/factors/tests/smoke.rs | 14 +-- crates/key-value-sqlite/src/lib.rs | 11 +- crates/key-value/src/lib.rs | 2 +- crates/key-value/src/util.rs | 26 ++-- .../trigger/src/runtime_config/key_value.rs | 7 +- 9 files changed, 165 insertions(+), 100 deletions(-) create mode 100644 crates/factor-key-value/src/runtime_config.rs create mode 100644 crates/factor-key-value/src/spin_cli_resolver.rs diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index f066429d8..73f09342f 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -1,3 +1,5 @@ +mod runtime_config; +pub mod spin_cli_resolver; mod store; use std::{ @@ -5,47 +7,29 @@ use std::{ sync::Arc, }; -use anyhow::{bail, ensure}; -use serde::Deserialize; +use anyhow::ensure; +use runtime_config::RuntimeConfig; use spin_factors::{ - anyhow::{self, Context}, - ConfigureAppContext, Factor, FactorInstanceBuilder, FactorRuntimeConfig, InitContext, - InstanceBuilders, PrepareContext, RuntimeFactors, + ConfigureAppContext, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, + PrepareContext, RuntimeFactors, }; use spin_key_value::{ - CachingStoreManager, DelegatingStoreManager, KeyValueDispatch, StoreManager, + CachingStoreManager, DelegatingStoreManager, DefaultManagerGetter, KeyValueDispatch, StoreManager, KEY_VALUE_STORES_KEY, }; -use store::{store_from_toml_fn, StoreFromToml}; - pub use store::MakeKeyValueStore; pub struct KeyValueFactor { - store_types: HashMap<&'static str, StoreFromToml>, - default_store_type: &'static str, + runtime_config_resolver: Arc, } -impl Default for KeyValueFactor { - fn default() -> KeyValueFactor { - KeyValueFactor { - store_types: HashMap::default(), - default_store_type: "spin", - } - } -} impl KeyValueFactor { - pub fn add_store_type(&mut self, store_type: T) -> anyhow::Result<()> { - if self - .store_types - .insert(T::RUNTIME_CONFIG_TYPE, store_from_toml_fn(store_type)) - .is_some() - { - bail!( - "duplicate key value store type {:?}", - T::RUNTIME_CONFIG_TYPE - ); + pub fn new( + runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, + ) -> Self { + Self { + runtime_config_resolver: Arc::new(runtime_config_resolver), } - Ok(()) } } @@ -68,37 +52,29 @@ impl Factor for KeyValueFactor { mut ctx: ConfigureAppContext, ) -> anyhow::Result { // Build StoreManager from runtime config - let mut stores = HashMap::new(); - let mut add_default_store = true; + let mut store_managers: HashMap> = HashMap::new(); if let Some(runtime_config) = ctx.take_runtime_config() { - for (label, StoreConfig { type_, config }) in runtime_config.store_configs { - if label == "default" { - add_default_store = false; - } - let store_maker = self - .store_types - .get(type_.as_str()) - .with_context(|| format!("unknown key value store type {type_:?}"))?; - let store = store_maker(config)?; - stores.insert(label, store); + for ( + store_label, + runtime_config::StoreConfig { + type_: store_kind, + config, + }, + ) in runtime_config.store_configs + { + let store = self + .runtime_config_resolver + .get_store(&store_kind, config)?; + store_managers.insert(store_label, store); } } - if add_default_store { - let store_maker = self - .store_types - .get(self.default_store_type) - .with_context(|| { - format!( - "default key value store {} does not exist", - self.default_store_type - ) - })?; - let store = store_maker(toml::value::Table::new())?; - stores.insert("default".to_string(), store); - } - let delegating_manager = DelegatingStoreManager::new(stores); + let resolver_clone = self.runtime_config_resolver.clone(); + let default_fn: DefaultManagerGetter = + Arc::new(move |label| resolver_clone.default(label)); + + let delegating_manager = DelegatingStoreManager::new(store_managers, default_fn); let caching_manager = CachingStoreManager::new(delegating_manager); - let store_manager = Arc::new(caching_manager); + let store_manager_manager = Arc::new(caching_manager); // Build component -> allowed stores map let mut component_allowed_stores = HashMap::new(); @@ -112,7 +88,8 @@ impl Factor for KeyValueFactor { for label in &key_value_stores { // TODO: port nicer errors from KeyValueComponent (via error type?) ensure!( - store_manager.is_defined(label), + store_manager_manager.is_defined(label) + || self.runtime_config_resolver.default(label).is_some(), "unknown key_value_stores label {label:?} for component {component_id:?}" ); } @@ -121,7 +98,7 @@ impl Factor for KeyValueFactor { } Ok(AppState { - store_manager, + store_manager: store_manager_manager, component_allowed_stores, }) } @@ -144,24 +121,6 @@ impl Factor for KeyValueFactor { } } -#[derive(Deserialize)] -#[serde(transparent)] -pub struct RuntimeConfig { - store_configs: HashMap, -} - -impl FactorRuntimeConfig for RuntimeConfig { - const KEY: &'static str = "key_value_store"; -} - -#[derive(Deserialize)] -struct StoreConfig { - #[serde(rename = "type")] - type_: String, - #[serde(flatten)] - config: toml::Table, -} - type AppStoreManager = CachingStoreManager; pub struct AppState { diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs new file mode 100644 index 000000000..d0c0f9f63 --- /dev/null +++ b/crates/factor-key-value/src/runtime_config.rs @@ -0,0 +1,42 @@ +use std::{collections::HashMap, sync::Arc}; + +use serde::Deserialize; +use spin_factors::{anyhow, FactorRuntimeConfig}; +use spin_key_value::StoreManager; + +#[derive(Deserialize)] +#[serde(transparent)] +pub struct RuntimeConfig { + pub store_configs: HashMap, +} + +impl FactorRuntimeConfig for RuntimeConfig { + const KEY: &'static str = "key_value_store"; +} + +#[derive(Deserialize)] +pub struct StoreConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} + +/// Resolves some piece of runtime configuration to a connection pool +pub trait RuntimeConfigResolver: Send + Sync { + /// Get a store manager for a given store kind and the raw configuration. + /// + /// `store_kind` is equivalent to the `type` field in the + /// `[key_value_store.$storename]` runtime configuration table. + fn get_store( + &self, + store_kind: &str, + config: toml::Table, + ) -> anyhow::Result>; + + /// Returns a default store manager for a given label. Should only be called + /// if there is no runtime configuration for the label. + /// + /// If `Option::None` is returned, the database is not allowed. + fn default(&self, label: &str) -> Option>; +} diff --git a/crates/factor-key-value/src/spin_cli_resolver.rs b/crates/factor-key-value/src/spin_cli_resolver.rs new file mode 100644 index 000000000..b7e334776 --- /dev/null +++ b/crates/factor-key-value/src/spin_cli_resolver.rs @@ -0,0 +1,47 @@ +use crate::runtime_config::RuntimeConfigResolver; +use crate::store::{store_from_toml_fn, MakeKeyValueStore, StoreFromToml}; +use spin_key_value::StoreManager; +use std::{collections::HashMap, sync::Arc}; + +#[derive(Default)] +pub struct SpinCliRuntimeConfigResolver { + store_types: HashMap<&'static str, StoreFromToml>, +} + +impl SpinCliRuntimeConfigResolver { + pub fn add_store_type(&mut self, store_type: T) -> anyhow::Result<()> { + if self + .store_types + .insert(T::RUNTIME_CONFIG_TYPE, store_from_toml_fn(store_type)) + .is_some() + { + anyhow::bail!( + "duplicate key value store type {:?}", + T::RUNTIME_CONFIG_TYPE + ); + } + Ok(()) + } +} + +impl RuntimeConfigResolver for SpinCliRuntimeConfigResolver { + fn get_store( + &self, + store_kind: &str, + config: toml::Table, + ) -> anyhow::Result> { + let store_from_toml = self + .store_types + .get(store_kind) + .ok_or_else(|| anyhow::anyhow!("unknown store kind: {}", store_kind))?; + store_from_toml(config) + } + + fn default(&self, label: &str) -> Option> { + if label == "default" { + self.get_store("spin", toml::value::Table::new()).ok() + } else { + None + } + } +} diff --git a/crates/factor-key-value/src/store.rs b/crates/factor-key-value/src/store.rs index 5b0c95658..96310401c 100644 --- a/crates/factor-key-value/src/store.rs +++ b/crates/factor-key-value/src/store.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use serde::de::DeserializeOwned; use spin_key_value::StoreManager; -pub trait MakeKeyValueStore: 'static { +pub trait MakeKeyValueStore: 'static + Send + Sync { const RUNTIME_CONFIG_TYPE: &'static str; type RuntimeConfig: DeserializeOwned; @@ -13,7 +13,8 @@ pub trait MakeKeyValueStore: 'static { -> anyhow::Result; } -pub(crate) type StoreFromToml = Box anyhow::Result>>; +pub(crate) type StoreFromToml = + Box anyhow::Result> + Send + Sync>; pub(crate) fn store_from_toml_fn(provider_type: T) -> StoreFromToml { Box::new(move |table| { diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index b54ae3879..60d1a9476 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -4,7 +4,7 @@ use anyhow::{bail, Context}; use http_body_util::BodyExt; use serde::Deserialize; use spin_app::App; -use spin_factor_key_value::{KeyValueFactor, MakeKeyValueStore}; +use spin_factor_key_value::{KeyValueFactor, MakeKeyValueStore, spin_cli_resolver::SpinCliRuntimeConfigResolver}; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; @@ -25,20 +25,20 @@ struct Factors { #[tokio::test(flavor = "multi_thread")] async fn smoke_test_works() -> anyhow::Result<()> { + let mut key_value_resolver = SpinCliRuntimeConfigResolver::default(); + key_value_resolver.add_store_type(TestSpinKeyValueStore)?; + key_value_resolver.add_store_type(RedisKeyValueStore)?; + let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), variables: VariablesFactor::default(), outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, - key_value: KeyValueFactor::default(), + key_value: KeyValueFactor::new(key_value_resolver), }; factors.variables.add_provider_type(StaticVariables)?; - factors.key_value.add_store_type(TestSpinKeyValueStore)?; - - factors.key_value.add_store_type(RedisKeyValueStore)?; - let locked = spin_loader::from_file( "tests/smoke-app/spin.toml", spin_loader::FilesMountStrategy::Direct, @@ -126,8 +126,6 @@ impl RuntimeConfigSource for TestSource { [variable_provider.values] foo = "bar" - [key_value_store.default] - type = "spin" [key_value_store.other] type = "redis" url = "redis://localhost:6379" diff --git a/crates/key-value-sqlite/src/lib.rs b/crates/key-value-sqlite/src/lib.rs index d84887ab7..337667b69 100644 --- a/crates/key-value-sqlite/src/lib.rs +++ b/crates/key-value-sqlite/src/lib.rs @@ -159,10 +159,13 @@ mod test { .into_iter() .map(ToOwned::to_owned) .collect(), - Arc::new(DelegatingStoreManager::new([( - "default".to_owned(), - Arc::new(KeyValueSqlite::new(DatabaseLocation::InMemory)) as _, - )])), + Arc::new(DelegatingStoreManager::new( + [( + "default".to_owned(), + Arc::new(KeyValueSqlite::new(DatabaseLocation::InMemory)) as _, + )], + Arc::new(|_: &str| -> Option> { None }), + )), ); assert!(matches!( diff --git a/crates/key-value/src/lib.rs b/crates/key-value/src/lib.rs index dc84fe797..31e8b5a55 100644 --- a/crates/key-value/src/lib.rs +++ b/crates/key-value/src/lib.rs @@ -9,7 +9,7 @@ mod host_component; mod util; pub use host_component::{manager, KeyValueComponent}; -pub use util::{CachingStoreManager, DelegatingStoreManager, EmptyStoreManager}; +pub use util::{CachingStoreManager, DelegatingStoreManager, DefaultManagerGetter, EmptyStoreManager}; pub const KEY_VALUE_STORES_KEY: MetadataKey> = MetadataKey::new("key_value_stores"); diff --git a/crates/key-value/src/util.rs b/crates/key-value/src/util.rs index f689e0f21..f9b0a1520 100644 --- a/crates/key-value/src/util.rs +++ b/crates/key-value/src/util.rs @@ -28,25 +28,37 @@ impl StoreManager for EmptyStoreManager { } } +/// A function that takes a store label and returns the default store manager, if one exists. +pub type DefaultManagerGetter = Arc Option> + Send + Sync>; + pub struct DelegatingStoreManager { delegates: HashMap>, + default_manager: DefaultManagerGetter, } impl DelegatingStoreManager { - pub fn new(delegates: impl IntoIterator)>) -> Self { + pub fn new( + delegates: impl IntoIterator)>, + default_manager: DefaultManagerGetter, + ) -> Self { let delegates = delegates.into_iter().collect(); - Self { delegates } + Self { + delegates, + default_manager, + } } } #[async_trait] impl StoreManager for DelegatingStoreManager { async fn get(&self, name: &str) -> Result, Error> { - self.delegates - .get(name) - .ok_or(Error::NoSuchStore)? - .get(name) - .await + match self.delegates.get(name) { + Some(store) => store.get(name).await, + None => { + let store = (self.default_manager)(name).ok_or(Error::NoSuchStore)?; + store.get(name).await + } + } } fn is_defined(&self, store_name: &str) -> bool { diff --git a/crates/trigger/src/runtime_config/key_value.rs b/crates/trigger/src/runtime_config/key_value.rs index b186c4819..a7b45f947 100644 --- a/crates/trigger/src/runtime_config/key_value.rs +++ b/crates/trigger/src/runtime_config/key_value.rs @@ -47,8 +47,11 @@ pub async fn build_key_value_component( bail!("Failed to access key-value store to set requested entries"); } } - - let delegating_manager = DelegatingStoreManager::new(stores); + // This is a temporary addition while factors work is in progress + // The default manager should already be added for all default labels + // and this should never be called. + let default_manager_fn = |_: &str| -> Option> { None }; + let delegating_manager = DelegatingStoreManager::new(stores, Arc::new(default_manager_fn)); let caching_manager = Arc::new(CachingStoreManager::new(delegating_manager)); Ok(KeyValueComponent::new(spin_key_value::manager(move |_| { caching_manager.clone() From 717c743ef55ac4abeb708f1226b93cf8919884b4 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 17 Jul 2024 16:14:17 +0200 Subject: [PATCH 054/195] Simplify path handling Signed-off-by: Ryan Levick --- .../factor-sqlite/src/runtime_config/spin.rs | 63 ++++++++++--------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 674e3e2b7..fa8b618a5 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -19,23 +19,33 @@ use super::RuntimeConfigResolver; /// This type implements the [`RuntimeConfigResolver`] trait and provides a way to /// opt into the default behavior of Spin's SQLite database handling. pub struct SpinSqliteRuntimeConfig { - state_dir: PathBuf, - base_path: Option, + default_database_dir: PathBuf, + local_database_dir: PathBuf, } impl SpinSqliteRuntimeConfig { /// Create a new `SpinSqliteRuntimeConfig` /// /// This takes as arguments: - /// * the state directory path (i.e., the path to the `.spin` file). This - /// is used to derive the default path a local SQLite database file. - /// * the base path from which relative paths referenced in configuration are resolved - /// (this should most likely be the path to the runtime-config file). If - /// `None`, the current working directory is used. - pub fn new(state_dir: PathBuf, base_path: Option) -> Self { + /// * the directory to use as the default location for SQLite databases. Usually this + /// will be the path to the `.spin` state directory. + /// * the *absolute* path to the directory from which relative paths to local SQLite + /// databases are resolved. (this should most likely be the path to the runtime-config + /// file or the current working dir). + /// + /// Panics if either `default_database_dir` or `local_database_dir` are not absolute paths. + pub fn new(default_database_dir: PathBuf, local_database_dir: PathBuf) -> Self { + assert!( + default_database_dir.is_absolute(), + "default_database_dir must be an absolute path" + ); + assert!( + local_database_dir.is_absolute(), + "local_database_dir must be an absolute path" + ); Self { - state_dir, - base_path, + default_database_dir, + local_database_dir, } } } @@ -49,7 +59,7 @@ impl RuntimeConfigResolver for SpinSqliteRuntimeConfig { let pool = match database_kind { "spin" => { let config: LocalDatabase = config.try_into()?; - config.pool(self.base_path.as_deref())? + config.pool(&self.local_database_dir)? } "libsql" => { let config: LibSqlDatabase = config.try_into()?; @@ -66,7 +76,7 @@ impl RuntimeConfigResolver for SpinSqliteRuntimeConfig { return None; } - let path = self.state_dir.join(DEFAULT_SQLITE_DB_FILENAME); + let path = self.default_database_dir.join(DEFAULT_SQLITE_DB_FILENAME); let factory = move || { let location = spin_sqlite_inproc::InProcDatabaseLocation::Path(path.clone()); let connection = spin_sqlite_inproc::InProcConnection::new(location)?; @@ -157,11 +167,12 @@ pub struct LocalDatabase { impl LocalDatabase { /// Create a new connection pool for a local database. - fn pool(self, base_path: Option<&Path>) -> anyhow::Result { + /// + /// `base_dir` is the base directory path from which `path` is resolved if it is a relative path. + fn pool(self, base_dir: &Path) -> anyhow::Result { let location = match self.path { Some(path) => { - // TODO: `base_path` should be passed in from the runtime config - let path = resolve_relative_path(&path, base_path)?; + let path = resolve_relative_path(&path, base_dir); // Create the store's parent directory if necessary // unwrapping the parent is fine, because `resolve_relative_path`` will always return a path with a parent std::fs::create_dir_all(path.parent().unwrap()) @@ -178,24 +189,14 @@ impl LocalDatabase { } } -/// Resolve a relative path against an optional base path. -fn resolve_relative_path(path: &Path, base_path: Option<&Path>) -> anyhow::Result { +/// Resolve a relative path against a base dir. +/// +/// If the path is absolute, it is returned as is. Otherwise, it is resolved against the base dir. +fn resolve_relative_path(path: &Path, base_dir: &Path) -> PathBuf { if path.is_absolute() { - return Ok(path.to_owned()); + return path.to_owned(); } - let base_path = match base_path { - Some(base_path) => base_path - .parent() - .with_context(|| { - format!( - "failed to get parent of runtime config file path \"{}\"", - base_path.display() - ) - })? - .to_owned(), - None => std::env::current_dir().context("failed to get current directory")?, - }; - Ok(base_path.join(path)) + base_dir.join(path) } /// Configuration for a libSQL database. From f683a642a9e95b4a58774a7b5bdd76851c801bed Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 16 Jul 2024 15:44:23 -0400 Subject: [PATCH 055/195] factors: Manage single ResourceTable in RuntimeFactors InstanceState Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/lib.rs | 4 +- crates/factor-outbound-http/src/wasi.rs | 74 ++++++++++--------- .../src/wasi_2023_10_18.rs | 6 +- .../src/wasi_2023_11_10.rs | 6 +- .../factor-outbound-http/tests/factor_test.rs | 4 +- .../tests/factor_test.rs | 4 +- crates/factor-wasi/src/lib.rs | 41 ++++++---- crates/factor-wasi/src/wasi_2023_10_18.rs | 4 +- crates/factor-wasi/src/wasi_2023_11_10.rs | 11 +-- crates/factor-wasi/tests/factor_test.rs | 5 +- crates/factors-derive/src/lib.rs | 20 ++++- crates/factors/src/factor.rs | 39 +++++++--- crates/factors/src/lib.rs | 2 +- crates/factors/src/runtime_factors.rs | 18 ++++- crates/factors/tests/smoke.rs | 8 +- 15 files changed, 150 insertions(+), 96 deletions(-) diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index fd9344dcc..5cc845c6f 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -10,8 +10,6 @@ use spin_factors::{ }; use wasmtime_wasi_http::WasiHttpCtx; -pub use wasi::get_wasi_http_view; - pub struct OutboundHttpFactor; impl Factor for OutboundHttpFactor { @@ -24,7 +22,7 @@ impl Factor for OutboundHttpFactor { mut ctx: spin_factors::InitContext, ) -> anyhow::Result<()> { ctx.link_bindings(spin_world::v1::http::add_to_linker)?; - wasi::add_to_linker::(ctx.linker())?; + wasi::add_to_linker::(&mut ctx)?; Ok(()) } diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index 834a619dd..42cf59953 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -1,17 +1,31 @@ use http::Request; -use spin_factors::{GetFactorState, Linker, RuntimeFactors}; -use wasmtime_wasi_http::{bindings::http::types::ErrorCode, WasiHttpImpl, WasiHttpView}; +use spin_factors::{ + wasmtime::component::ResourceTable, RuntimeFactors, RuntimeFactorsInstanceState, +}; +use wasmtime_wasi_http::{ + bindings::http::types::ErrorCode, WasiHttpCtx, WasiHttpImpl, WasiHttpView, +}; -use crate::{wasi_2023_10_18, wasi_2023_11_10}; +use crate::{wasi_2023_10_18, wasi_2023_11_10, OutboundHttpFactor}; -pub(crate) fn add_to_linker(linker: &mut Linker) -> anyhow::Result<()> { - fn type_annotate(f: F) -> F +pub(crate) fn add_to_linker( + ctx: &mut spin_factors::InitContext, +) -> anyhow::Result<()> { + fn type_annotate(f: F) -> F where - F: Fn(&mut T) -> WasiHttpImpl>, + F: Fn(&mut T) -> WasiHttpImpl, { f } - let closure = type_annotate(move |data| WasiHttpImpl(MutStates { inner: data })); + let get_data_with_table = ctx.get_data_with_table_fn(); + let closure = type_annotate(move |data| { + let (state, table) = get_data_with_table(data); + WasiHttpImpl(WasiHttpImplInner { + ctx: &mut state.wasi_http_ctx, + table, + }) + }); + let linker = ctx.linker(); wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; wasmtime_wasi_http::bindings::http::types::add_to_linker_get_host(linker, closure)?; @@ -21,27 +35,30 @@ pub(crate) fn add_to_linker(linker: &mut Linker) -> anyhow Ok(()) } -pub(crate) struct MutStates<'a, T> { - inner: &'a mut T, +impl OutboundHttpFactor { + pub fn get_wasi_http_impl( + runtime_instance_state: &mut impl RuntimeFactorsInstanceState, + ) -> Option> { + let (state, table) = runtime_instance_state.get_with_table::()?; + Some(WasiHttpImpl(WasiHttpImplInner { + ctx: &mut state.wasi_http_ctx, + table, + })) + } +} + +pub(crate) struct WasiHttpImplInner<'a> { + ctx: &'a mut WasiHttpCtx, + table: &'a mut ResourceTable, } -impl<'a, T> WasiHttpView for MutStates<'a, T> -where - T: GetFactorState + Send, -{ - fn ctx(&mut self) -> &mut wasmtime_wasi_http::WasiHttpCtx { - &mut self - .inner - .get::() - .expect("failed to get `OutboundHttpFactor`") - .wasi_http_ctx +impl<'a> WasiHttpView for WasiHttpImplInner<'a> { + fn ctx(&mut self) -> &mut WasiHttpCtx { + self.ctx } - fn table(&mut self) -> &mut spin_factors::wasmtime::component::ResourceTable { - self.inner - .get::() - .expect("failed to get `WasiFactor`") - .table() + fn table(&mut self) -> &mut ResourceTable { + self.table } fn send_request( @@ -53,12 +70,3 @@ where Err(ErrorCode::HttpRequestDenied.into()) } } - -// TODO: This is a little weird, organizationally -pub fn get_wasi_http_view( - instance_state: &mut T, -) -> impl WasiHttpView + '_ { - MutStates { - inner: instance_state, - } -} diff --git a/crates/factor-outbound-http/src/wasi_2023_10_18.rs b/crates/factor-outbound-http/src/wasi_2023_10_18.rs index f99025d1b..92a9cd4cc 100644 --- a/crates/factor-outbound-http/src/wasi_2023_10_18.rs +++ b/crates/factor-outbound-http/src/wasi_2023_10_18.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use spin_factors::GetFactorState; use wasmtime::component::{Linker, Resource}; use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; @@ -50,12 +49,11 @@ use wasi::http::types::{ use wasi::io::poll::Pollable; use wasi::io::streams::{InputStream, OutputStream}; -use crate::wasi::MutStates; +use crate::wasi::WasiHttpImplInner; pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where - T: GetFactorState + Send, - F: Fn(&mut T) -> WasiHttpImpl> + Send + Sync + Copy + 'static, + F: Fn(&mut T) -> WasiHttpImpl + Send + Sync + Copy + 'static, { wasi::http::types::add_to_linker_get_host(linker, closure)?; wasi::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; diff --git a/crates/factor-outbound-http/src/wasi_2023_11_10.rs b/crates/factor-outbound-http/src/wasi_2023_11_10.rs index 5295ed86a..439003d15 100644 --- a/crates/factor-outbound-http/src/wasi_2023_11_10.rs +++ b/crates/factor-outbound-http/src/wasi_2023_11_10.rs @@ -2,7 +2,6 @@ use super::wasi_2023_10_18::convert; use anyhow::Result; -use spin_factors::GetFactorState; use wasmtime::component::{Linker, Resource}; use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; @@ -55,12 +54,11 @@ use wasi::http::types::{ use wasi::io::poll::Pollable; use wasi::io::streams::{Error as IoError, InputStream, OutputStream}; -use crate::wasi::MutStates; +use crate::wasi::WasiHttpImplInner; pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where - T: GetFactorState + Send, - F: Fn(&mut T) -> WasiHttpImpl> + Send + Sync + Copy + 'static, + F: Fn(&mut T) -> WasiHttpImpl + Send + Sync + Copy + 'static, { wasi::http::types::add_to_linker_get_host(linker, closure)?; wasi::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index 0a9655270..e1dc70bbc 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -2,7 +2,7 @@ use std::time::Duration; use anyhow::bail; use http::Request; -use spin_factor_outbound_http::{get_wasi_http_view, OutboundHttpFactor}; +use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; @@ -38,7 +38,7 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { }; let env = test_env(); let mut state = env.build_instance_state(factors).await?; - let mut wasi_http = get_wasi_http_view(&mut state); + let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); let req = Request::get("https://denied.test").body(Default::default())?; let res = wasi_http.send_request(req, test_request_config()); diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs index 75d155d36..0bd409b75 100644 --- a/crates/factor-outbound-networking/tests/factor_test.rs +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -3,7 +3,7 @@ use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; -use wasmtime_wasi::{bindings::sockets::instance_network::Host, SocketAddrUse, WasiImpl, WasiView}; +use wasmtime_wasi::{bindings::sockets::instance_network::Host, SocketAddrUse, WasiView}; #[derive(RuntimeFactors)] struct TestFactors { @@ -30,7 +30,7 @@ async fn configures_wasi_socket_addr_check() -> anyhow::Result<()> { let env = test_env(); let mut state = env.build_instance_state(factors).await?; - let mut wasi = WasiImpl(&mut state.wasi); + let mut wasi = WasiFactor::get_wasi_impl(&mut state).unwrap(); let network_resource = wasi.instance_network()?; let network = wasi.table().get(&network_resource)?; diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index d1fa1e344..47479236c 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -5,7 +5,7 @@ use std::{future::Future, net::SocketAddr, path::Path}; use spin_factors::{ anyhow, AppComponent, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, - PrepareContext, RuntimeFactors, + PrepareContext, RuntimeFactors, RuntimeFactorsInstanceState, }; use tokio::io::{AsyncRead, AsyncWrite}; use wasmtime_wasi::{ @@ -26,6 +26,16 @@ impl WasiFactor { files_mounter: Box::new(files_mounter), } } + + pub fn get_wasi_impl( + runtime_instance_state: &mut impl RuntimeFactorsInstanceState, + ) -> Option> { + let (state, table) = runtime_instance_state.get_with_table::()?; + Some(WasiImpl(WasiImplInner { + ctx: &mut state.ctx, + table, + })) + } } impl Factor for WasiFactor { @@ -37,14 +47,20 @@ impl Factor for WasiFactor { &mut self, mut ctx: InitContext, ) -> anyhow::Result<()> { - fn type_annotate(f: F) -> F + fn type_annotate(f: F) -> F where - F: Fn(&mut T) -> WasiImpl<&mut U>, + F: Fn(&mut T) -> WasiImpl, { f } - let get_data = ctx.get_data_fn(); - let closure = type_annotate(move |data| WasiImpl(get_data(data))); + let get_data_with_table = ctx.get_data_with_table_fn(); + let closure = type_annotate(move |data| { + let (state, table) = get_data_with_table(data); + WasiImpl(WasiImplInner { + ctx: &mut state.ctx, + table, + }) + }); let linker = ctx.linker(); use wasmtime_wasi::bindings; bindings::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; @@ -237,7 +253,6 @@ impl FactorInstanceBuilder for InstanceBuilder { let InstanceBuilder { ctx: mut wasi_ctx } = self; Ok(InstanceState { ctx: wasi_ctx.build(), - table: Default::default(), }) } } @@ -267,21 +282,19 @@ impl InstanceBuilder { pub struct InstanceState { ctx: WasiCtx, - table: ResourceTable, } -impl InstanceState { - pub fn table(&mut self) -> &mut ResourceTable { - &mut self.table - } +struct WasiImplInner<'a> { + ctx: &'a mut WasiCtx, + table: &'a mut ResourceTable, } -impl WasiView for InstanceState { +impl<'a> WasiView for WasiImplInner<'a> { fn ctx(&mut self) -> &mut WasiCtx { - &mut self.ctx + self.ctx } fn table(&mut self) -> &mut ResourceTable { - &mut self.table + self.table } } diff --git a/crates/factor-wasi/src/wasi_2023_10_18.rs b/crates/factor-wasi/src/wasi_2023_10_18.rs index 43f3cbbaa..386da6e72 100644 --- a/crates/factor-wasi/src/wasi_2023_10_18.rs +++ b/crates/factor-wasi/src/wasi_2023_10_18.rs @@ -123,12 +123,12 @@ use wasi::sockets::tcp::{ }; use wasi::sockets::udp::Datagram; -use crate::InstanceState; +use crate::WasiImplInner; pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where T: Send, - F: Fn(&mut T) -> WasiImpl<&mut InstanceState> + Send + Sync + Copy + 'static, + F: Fn(&mut T) -> WasiImpl + Send + Sync + Copy + 'static, { wasi::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; wasi::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; diff --git a/crates/factor-wasi/src/wasi_2023_11_10.rs b/crates/factor-wasi/src/wasi_2023_11_10.rs index a7131c423..a98687885 100644 --- a/crates/factor-wasi/src/wasi_2023_11_10.rs +++ b/crates/factor-wasi/src/wasi_2023_11_10.rs @@ -116,20 +116,13 @@ use wasi::sockets::udp::{ IncomingDatagram, IncomingDatagramStream, OutgoingDatagram, OutgoingDatagramStream, UdpSocket, }; -use crate::InstanceState; +use crate::WasiImplInner; pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where T: Send, - F: Fn(&mut T) -> WasiImpl<&mut InstanceState> + Send + Sync + Copy + 'static, + F: Fn(&mut T) -> WasiImpl + Send + Sync + Copy + 'static, { - fn type_annotate(f: F) -> F - where - F: Fn(&mut T) -> WasiImpl<&mut U>, - { - f - } - let closure = type_annotate(closure); wasi::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; wasi::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; wasi::filesystem::types::add_to_linker_get_host(linker, closure)?; diff --git a/crates/factor-wasi/tests/factor_test.rs b/crates/factor-wasi/tests/factor_test.rs index e502f4ae7..3f7084336 100644 --- a/crates/factor-wasi/tests/factor_test.rs +++ b/crates/factor-wasi/tests/factor_test.rs @@ -1,7 +1,7 @@ use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; -use wasmtime_wasi::{bindings::cli::environment::Host, WasiImpl}; +use wasmtime_wasi::bindings::cli::environment::Host; #[derive(RuntimeFactors)] struct TestFactors { @@ -23,8 +23,7 @@ async fn environment_works() -> anyhow::Result<()> { }; let env = test_env(); let mut state = env.build_instance_state(factors).await?; - let mut wasi = WasiImpl(&mut state.wasi); - + let mut wasi = WasiFactor::get_wasi_impl(&mut state).unwrap(); let val = wasi .get_environment()? .into_iter() diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 98e427cec..0731c7a07 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -63,6 +63,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let factors_crate = format_ident!("spin_factors"); let factors_path = quote!(::#factors_crate); let wasmtime = quote!(#factors_path::wasmtime); + let ResourceTable = quote!(#wasmtime::component::ResourceTable); let Result = quote!(#factors_path::Result); let Error = quote!(#factors_path::Error); let Factor = quote!(#factors_path::Factor); @@ -87,6 +88,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #factors_path::InitContext::::new( linker, |state| &mut state.#factor_names, + |state| (&mut state.#factor_names, &mut state.__table), ) ).map_err(#Error::factor_init_error::<#factor_types>)?; )* @@ -148,6 +150,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { builders: Self::InstanceBuilders, ) -> #Result { Ok(#state_name { + __table: #ResourceTable::new(), #( #factor_names: #FactorInstanceBuilder::build( builders.#factor_names.unwrap() @@ -197,20 +200,31 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } #vis struct #state_name { + __table: #ResourceTable, #( pub #factor_names: #factors_path::FactorInstanceState<#factor_types>, )* } - impl #factors_path::GetFactorState for #state_name { - fn get(&mut self) -> ::std::option::Option<&mut #factors_path::FactorInstanceState> { + impl #factors_path::RuntimeFactorsInstanceState for #state_name { + fn get_with_table( + &mut self + ) -> ::std::option::Option<(&mut #factors_path::FactorInstanceState, &mut #ResourceTable)> { #( if let Some(state) = (&mut self.#factor_names as &mut (dyn #Any + #Send)).downcast_mut() { - return Some(state) + return Some((state, &mut self.__table)) } )* None } + + fn table(&self) -> &#ResourceTable { + &self.__table + } + + fn table_mut(&mut self) -> &mut #ResourceTable { + &mut self.__table + } } }) } diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 2a212574a..8ec6ee8a2 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -1,5 +1,7 @@ use std::any::Any; +use wasmtime::component::ResourceTable; + use crate::{ prepare::FactorInstanceBuilder, runtime_config::RuntimeConfigTracker, App, Error, FactorRuntimeConfig, InstanceBuilders, Linker, PrepareContext, RuntimeConfigSource, @@ -70,20 +72,33 @@ pub trait Factor: Any + Sized { pub type FactorInstanceState = <::InstanceBuilder as FactorInstanceBuilder>::InstanceState; -pub(crate) type GetDataFn = - fn(&mut ::InstanceState) -> &mut FactorInstanceState; +pub(crate) type GetDataFn = + fn(&mut ::InstanceState) -> &mut FactorInstanceState; + +pub(crate) type GetDataWithTableFn = fn( + &mut ::InstanceState, +) -> (&mut FactorInstanceState, &mut ResourceTable); /// An InitContext is passed to [`Factor::init`], giving access to the global /// common [`wasmtime::component::Linker`]. -pub struct InitContext<'a, T: RuntimeFactors, F: Factor> { +pub struct InitContext<'a, T: RuntimeFactors, U: Factor> { pub(crate) linker: &'a mut Linker, - pub(crate) get_data: GetDataFn, + pub(crate) get_data: GetDataFn, + pub(crate) get_data_with_table: GetDataWithTableFn, } -impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { +impl<'a, T: RuntimeFactors, U: Factor> InitContext<'a, T, U> { #[doc(hidden)] - pub fn new(linker: &'a mut Linker, get_data: GetDataFn) -> Self { - Self { linker, get_data } + pub fn new( + linker: &'a mut Linker, + get_data: GetDataFn, + get_data_with_table: GetDataWithTableFn, + ) -> Self { + Self { + linker, + get_data, + get_data_with_table, + } } /// Returns a mutable reference to the [`wasmtime::component::Linker`]. @@ -92,16 +107,22 @@ impl<'a, T: RuntimeFactors, F: Factor> InitContext<'a, T, F> { } /// Returns a function that can be used to get the instance state for this factor. - pub fn get_data_fn(&self) -> GetDataFn { + pub fn get_data_fn(&self) -> GetDataFn { self.get_data } + /// Returns a function that can be used to get the instance state for this + /// factor along with the instance's [`ResourceTable`]. + pub fn get_data_with_table_fn(&self) -> GetDataWithTableFn { + self.get_data_with_table + } + /// Convenience method to link a binding to the linker. pub fn link_bindings( &mut self, add_to_linker: impl Fn( &mut Linker, - fn(&mut T::InstanceState) -> &mut FactorInstanceState, + fn(&mut T::InstanceState) -> &mut FactorInstanceState, ) -> anyhow::Result<()>, ) -> anyhow::Result<()> { add_to_linker(self.linker, self.get_data) diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 381e64e84..6a9185a20 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -13,7 +13,7 @@ pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, FactorInstanceState, InitContext}, prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext, SelfInstanceBuilder}, runtime_config::{FactorRuntimeConfig, RuntimeConfigSource}, - runtime_factors::{GetFactorState, RuntimeFactors}, + runtime_factors::{RuntimeFactors, RuntimeFactorsInstanceState}, }; /// A [`wasmtime::component::Linker`] used for a [`RuntimeFactors`] collection. diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index adf9c6c2d..1293dc13f 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,3 +1,5 @@ +use wasmtime::component::ResourceTable; + use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, Linker, RuntimeConfigSource}; /// A collection of `Factor`s that are initialized and configured together. @@ -30,7 +32,7 @@ pub trait RuntimeFactors: Sized + 'static { /// The per application state of all the factors. type AppState; /// The per instance state of the factors. - type InstanceState: GetFactorState + Send + 'static; + type InstanceState: RuntimeFactorsInstanceState; /// The collection of all the `InstanceBuilder`s of the factors. type InstanceBuilders; @@ -74,6 +76,16 @@ pub trait RuntimeFactors: Sized + 'static { /// Get the state of a particular Factor from the overall InstanceState /// /// Implemented by `#[derive(RuntimeFactors)]` -pub trait GetFactorState { - fn get(&mut self) -> Option<&mut FactorInstanceState>; +pub trait RuntimeFactorsInstanceState: Send + 'static { + fn get_with_table( + &mut self, + ) -> Option<(&mut FactorInstanceState, &mut ResourceTable)>; + + fn get(&mut self) -> Option<&mut FactorInstanceState> { + self.get_with_table::().map(|(state, _)| state) + } + + fn table(&self) -> &ResourceTable; + + fn table_mut(&mut self) -> &mut ResourceTable; } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 1c53853bb..342a36034 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -84,11 +84,11 @@ async fn smoke_test_works() -> anyhow::Result<()> { // Invoke handler let req = http::Request::get("/").body(Default::default()).unwrap(); - let mut wasi_http_view = spin_factor_outbound_http::get_wasi_http_view(store.data_mut()); - let request = wasi_http_view.new_incoming_request(req)?; + let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(store.data_mut()).unwrap(); + let request = wasi_http.new_incoming_request(req)?; let (response_tx, response_rx) = tokio::sync::oneshot::channel(); - let response = wasi_http_view.new_response_outparam(response_tx)?; - drop(wasi_http_view); + let response = wasi_http.new_response_outparam(response_tx)?; + drop(wasi_http); let guest = wasmtime_wasi_http::proxy::Proxy::new(&mut store, &instance)?; let call_task = tokio::spawn(async move { From 4cb982db9c44672f2083d1168ec12f5bd16e1b1a Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 17 Jul 2024 18:19:36 +0200 Subject: [PATCH 056/195] ensure allowed sqlite databases are configured Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/lib.rs | 66 +++++++++++++++++----- crates/factor-sqlite/tests/factor.rs | 82 ++++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 19 deletions(-) diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 68e95e992..5c4c1b64b 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -68,24 +68,31 @@ impl Factor for SqliteFactor { .map(|component| { Ok(( component.id().to_string(), - component - .get_metadata(ALLOWED_DATABASES_KEY)? - .unwrap_or_default() - .into_iter() - .collect::>() - .into(), + Arc::new( + component + .get_metadata(ALLOWED_DATABASES_KEY)? + .unwrap_or_default() + .into_iter() + .collect::>(), + ), )) }) - .collect::>()?; + .collect::>>()?; let resolver = self.runtime_config_resolver.clone(); + let get_connection_pool: host::ConnectionPoolGetter = Arc::new(move |label| { + connection_pools + .get(label) + .cloned() + .or_else(|| resolver.default(label)) + }); + + ensure_allowed_databases_are_configured(&allowed_databases, |label| { + get_connection_pool(label).is_some() + })?; + Ok(AppState { allowed_databases, - get_connection_pool: Arc::new(move |label| { - connection_pools - .get(label) - .cloned() - .or_else(|| resolver.default(label)) - }), + get_connection_pool, }) } @@ -105,6 +112,39 @@ impl Factor for SqliteFactor { } } +/// Ensure that all the databases in the allowed databases list for each component are configured +fn ensure_allowed_databases_are_configured( + allowed_databases: &HashMap>>, + is_configured: impl Fn(&str) -> bool, +) -> anyhow::Result<()> { + let mut errors = Vec::new(); + for (component_id, allowed_dbs) in allowed_databases { + for allowed in allowed_dbs.iter() { + if !is_configured(allowed) { + errors.push(format!( + "- Component {component_id} uses database '{allowed}'" + )); + } + } + } + + if !errors.is_empty() { + let prologue = vec![ + "One or more components use SQLite databases which are not defined.", + "Check the spelling, or pass a runtime configuration file that defines these stores.", + "See https://developer.fermyon.com/spin/dynamic-configuration#sqlite-storage-runtime-configuration", + "Details:", + ]; + let lines: Vec<_> = prologue + .into_iter() + .map(|s| s.to_owned()) + .chain(errors) + .collect(); + return Err(anyhow::anyhow!(lines.join("\n"))); + } + Ok(()) +} + pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); pub struct AppState { diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index d1a876950..eb40212d9 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -1,7 +1,10 @@ use std::{collections::HashSet, sync::Arc}; use factor_sqlite::SqliteFactor; -use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors::{ + anyhow::{self, bail}, + RuntimeFactors, +}; use spin_factors_test::{toml, TestEnvironment}; #[derive(RuntimeFactors)] @@ -11,7 +14,7 @@ struct TestFactors { #[tokio::test] async fn sqlite_works() -> anyhow::Result<()> { - let test_resolver = RuntimeConfigResolver; + let test_resolver = RuntimeConfigResolver::new(Some("default")); let factors = TestFactors { sqlite: SqliteFactor::new(test_resolver), }; @@ -30,7 +33,60 @@ async fn sqlite_works() -> anyhow::Result<()> { Ok(()) } -struct RuntimeConfigResolver; +#[tokio::test] +async fn errors_when_non_configured_database_used() -> anyhow::Result<()> { + let test_resolver = RuntimeConfigResolver::new(None); + let factors = TestFactors { + sqlite: SqliteFactor::new(test_resolver), + }; + let env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + sqlite_databases = ["foo"] + }); + let Err(err) = env.build_instance_state(factors).await else { + bail!("Expected build_instance_state to error but it did not"); + }; + + assert!(err + .to_string() + .contains("One or more components use SQLite databases which are not defined.")); + + Ok(()) +} + +#[tokio::test] +async fn no_error_when_database_is_configured() -> anyhow::Result<()> { + let test_resolver = RuntimeConfigResolver::new(None); + let factors = TestFactors { + sqlite: SqliteFactor::new(test_resolver), + }; + let mut env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + sqlite_databases = ["foo"] + }); + env.runtime_config = toml! { + [sqlite_database.foo] + type = "sqlite" + }; + assert!(env.build_instance_state(factors).await.is_ok()); + + Ok(()) +} + +/// Will return an `InvalidConnectionPool` for all runtime configured databases and the supplied default database. +struct RuntimeConfigResolver { + default: Option, +} + +impl RuntimeConfigResolver { + fn new(default: Option<&str>) -> Self { + Self { + default: default.map(Into::into), + } + } +} impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { fn get_pool( @@ -39,11 +95,25 @@ impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResol config: toml::Table, ) -> anyhow::Result> { let _ = (database_kind, config); - todo!() + Ok(Arc::new(InvalidConnectionPool)) } fn default(&self, label: &str) -> Option> { - let _ = label; - todo!() + let Some(default) = &self.default else { + return None; + }; + (default == label).then_some(Arc::new(InvalidConnectionPool)) + } +} + +/// A connection pool that always returns an error. +struct InvalidConnectionPool; + +#[async_trait::async_trait] +impl factor_sqlite::ConnectionPool for InvalidConnectionPool { + async fn get_connection( + &self, + ) -> Result, spin_world::v2::sqlite::Error> { + Err(spin_world::v2::sqlite::Error::InvalidConnection) } } From 41a5796603c80617eebed2eac7a12d8f475bfaae Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Wed, 17 Jul 2024 11:17:11 -0500 Subject: [PATCH 057/195] Enable configuring default label and store mappings on KV provider Signed-off-by: Kate Goldenring --- ...cli_resolver.rs => delegating_resolver.rs} | 33 ++++++++++++++----- crates/factor-key-value/src/lib.rs | 10 +++--- crates/factor-key-value/src/runtime_config.rs | 2 +- crates/factors/tests/smoke.rs | 7 ++-- crates/key-value/src/lib.rs | 4 ++- crates/key-value/src/util.rs | 14 ++++---- 6 files changed, 45 insertions(+), 25 deletions(-) rename crates/factor-key-value/src/{spin_cli_resolver.rs => delegating_resolver.rs} (57%) diff --git a/crates/factor-key-value/src/spin_cli_resolver.rs b/crates/factor-key-value/src/delegating_resolver.rs similarity index 57% rename from crates/factor-key-value/src/spin_cli_resolver.rs rename to crates/factor-key-value/src/delegating_resolver.rs index b7e334776..235a2afcb 100644 --- a/crates/factor-key-value/src/spin_cli_resolver.rs +++ b/crates/factor-key-value/src/delegating_resolver.rs @@ -4,11 +4,29 @@ use spin_key_value::StoreManager; use std::{collections::HashMap, sync::Arc}; #[derive(Default)] -pub struct SpinCliRuntimeConfigResolver { +pub struct DelegatingRuntimeConfigResolver { store_types: HashMap<&'static str, StoreFromToml>, + defaults: HashMap<&'static str, StoreConfig>, } -impl SpinCliRuntimeConfigResolver { +type StoreConfig = (&'static str, toml::value::Table); + +impl DelegatingRuntimeConfigResolver { + pub fn new() -> Self { + Self::default() + } + + pub fn add_default_store( + &mut self, + label: &'static str, + store_kind: &'static str, + config: toml::value::Table, + ) { + self.defaults.insert(label, (store_kind, config)); + } +} + +impl DelegatingRuntimeConfigResolver { pub fn add_store_type(&mut self, store_type: T) -> anyhow::Result<()> { if self .store_types @@ -24,7 +42,7 @@ impl SpinCliRuntimeConfigResolver { } } -impl RuntimeConfigResolver for SpinCliRuntimeConfigResolver { +impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { fn get_store( &self, store_kind: &str, @@ -37,11 +55,8 @@ impl RuntimeConfigResolver for SpinCliRuntimeConfigResolver { store_from_toml(config) } - fn default(&self, label: &str) -> Option> { - if label == "default" { - self.get_store("spin", toml::value::Table::new()).ok() - } else { - None - } + fn default_store(&self, label: &str) -> Option> { + let (store_kind, config) = self.defaults.get(label)?; + self.get_store(store_kind, config.to_owned()).ok() } } diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index 73f09342f..7cf274922 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -1,5 +1,5 @@ mod runtime_config; -pub mod spin_cli_resolver; +pub mod delegating_resolver; mod store; use std::{ @@ -14,8 +14,8 @@ use spin_factors::{ PrepareContext, RuntimeFactors, }; use spin_key_value::{ - CachingStoreManager, DelegatingStoreManager, DefaultManagerGetter, KeyValueDispatch, StoreManager, - KEY_VALUE_STORES_KEY, + CachingStoreManager, DefaultManagerGetter, DelegatingStoreManager, KeyValueDispatch, + StoreManager, KEY_VALUE_STORES_KEY, }; pub use store::MakeKeyValueStore; @@ -70,7 +70,7 @@ impl Factor for KeyValueFactor { } let resolver_clone = self.runtime_config_resolver.clone(); let default_fn: DefaultManagerGetter = - Arc::new(move |label| resolver_clone.default(label)); + Arc::new(move |label| resolver_clone.default_store(label)); let delegating_manager = DelegatingStoreManager::new(store_managers, default_fn); let caching_manager = CachingStoreManager::new(delegating_manager); @@ -89,7 +89,7 @@ impl Factor for KeyValueFactor { // TODO: port nicer errors from KeyValueComponent (via error type?) ensure!( store_manager_manager.is_defined(label) - || self.runtime_config_resolver.default(label).is_some(), + || self.runtime_config_resolver.default_store(label).is_some(), "unknown key_value_stores label {label:?} for component {component_id:?}" ); } diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index d0c0f9f63..721cf5be4 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -38,5 +38,5 @@ pub trait RuntimeConfigResolver: Send + Sync { /// if there is no runtime configuration for the label. /// /// If `Option::None` is returned, the database is not allowed. - fn default(&self, label: &str) -> Option>; + fn default_store(&self, label: &str) -> Option>; } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 60d1a9476..e4cac5d1a 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -4,7 +4,9 @@ use anyhow::{bail, Context}; use http_body_util::BodyExt; use serde::Deserialize; use spin_app::App; -use spin_factor_key_value::{KeyValueFactor, MakeKeyValueStore, spin_cli_resolver::SpinCliRuntimeConfigResolver}; +use spin_factor_key_value::{ + delegating_resolver::DelegatingRuntimeConfigResolver, KeyValueFactor, MakeKeyValueStore, +}; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; @@ -25,7 +27,8 @@ struct Factors { #[tokio::test(flavor = "multi_thread")] async fn smoke_test_works() -> anyhow::Result<()> { - let mut key_value_resolver = SpinCliRuntimeConfigResolver::default(); + let mut key_value_resolver = DelegatingRuntimeConfigResolver::default(); + key_value_resolver.add_default_store("default", "spin", toml::value::Table::new()); key_value_resolver.add_store_type(TestSpinKeyValueStore)?; key_value_resolver.add_store_type(RedisKeyValueStore)?; diff --git a/crates/key-value/src/lib.rs b/crates/key-value/src/lib.rs index 31e8b5a55..7caddf40a 100644 --- a/crates/key-value/src/lib.rs +++ b/crates/key-value/src/lib.rs @@ -9,7 +9,9 @@ mod host_component; mod util; pub use host_component::{manager, KeyValueComponent}; -pub use util::{CachingStoreManager, DelegatingStoreManager, DefaultManagerGetter, EmptyStoreManager}; +pub use util::{ + CachingStoreManager, DefaultManagerGetter, DelegatingStoreManager, EmptyStoreManager, +}; pub const KEY_VALUE_STORES_KEY: MetadataKey> = MetadataKey::new("key_value_stores"); diff --git a/crates/key-value/src/util.rs b/crates/key-value/src/util.rs index f9b0a1520..78498e3a0 100644 --- a/crates/key-value/src/util.rs +++ b/crates/key-value/src/util.rs @@ -52,13 +52,13 @@ impl DelegatingStoreManager { #[async_trait] impl StoreManager for DelegatingStoreManager { async fn get(&self, name: &str) -> Result, Error> { - match self.delegates.get(name) { - Some(store) => store.get(name).await, - None => { - let store = (self.default_manager)(name).ok_or(Error::NoSuchStore)?; - store.get(name).await - } - } + let store = match self.delegates.get(name) { + Some(store) => store, + None => { + &(self.default_manager)(name).ok_or(Error::NoSuchStore)? + } + }; + store.get(name).await } fn is_defined(&self, store_name: &str) -> bool { From 55c2ffce1ccfea7dce385a6996c997d444e4e58b Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 17 Jul 2024 16:49:50 -0400 Subject: [PATCH 058/195] factors: Allow instance state to be nested in store data Signed-off-by: Lann Martin --- crates/factor-key-value/src/lib.rs | 5 +---- crates/factor-llm/src/lib.rs | 2 +- crates/factor-outbound-http/src/lib.rs | 2 +- crates/factor-outbound-http/src/wasi.rs | 6 ++---- crates/factor-sqlite/src/lib.rs | 4 ++-- crates/factor-variables/src/lib.rs | 5 +---- crates/factor-wasi/src/lib.rs | 4 ++-- crates/factors-derive/src/lib.rs | 22 +++++++++++++++------- crates/factors-test/src/lib.rs | 10 +++++----- crates/factors/src/factor.rs | 21 +++++++++------------ crates/factors/src/lib.rs | 3 --- crates/factors/src/runtime_factors.rs | 17 +++++++++++------ crates/factors/tests/smoke.rs | 25 +++++++++++++++++++++---- 13 files changed, 71 insertions(+), 55 deletions(-) diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index f066429d8..dc77e713b 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -54,10 +54,7 @@ impl Factor for KeyValueFactor { type AppState = AppState; type InstanceBuilder = InstanceBuilder; - fn init( - &mut self, - mut ctx: InitContext, - ) -> anyhow::Result<()> { + fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { ctx.link_bindings(spin_world::v1::key_value::add_to_linker)?; ctx.link_bindings(spin_world::v2::key_value::add_to_linker)?; Ok(()) diff --git a/crates/factor-llm/src/lib.rs b/crates/factor-llm/src/lib.rs index 13899ec3b..3e40f36a2 100644 --- a/crates/factor-llm/src/lib.rs +++ b/crates/factor-llm/src/lib.rs @@ -34,7 +34,7 @@ impl Factor for LlmFactor { type AppState = AppState; type InstanceBuilder = InstanceState; - fn init( + fn init( &mut self, mut ctx: spin_factors::InitContext, ) -> anyhow::Result<()> { diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 5cc845c6f..e17babf2e 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -17,7 +17,7 @@ impl Factor for OutboundHttpFactor { type AppState = (); type InstanceBuilder = InstanceState; - fn init( + fn init( &mut self, mut ctx: spin_factors::InitContext, ) -> anyhow::Result<()> { diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index 42cf59953..f5b99497e 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -1,14 +1,12 @@ use http::Request; -use spin_factors::{ - wasmtime::component::ResourceTable, RuntimeFactors, RuntimeFactorsInstanceState, -}; +use spin_factors::{wasmtime::component::ResourceTable, RuntimeFactorsInstanceState}; use wasmtime_wasi_http::{ bindings::http::types::ErrorCode, WasiHttpCtx, WasiHttpImpl, WasiHttpView, }; use crate::{wasi_2023_10_18, wasi_2023_11_10, OutboundHttpFactor}; -pub(crate) fn add_to_linker( +pub(crate) fn add_to_linker( ctx: &mut spin_factors::InitContext, ) -> anyhow::Result<()> { fn type_annotate(f: F) -> F diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 68e95e992..a2673a44b 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use host::InstanceState; use async_trait::async_trait; -use spin_factors::{anyhow, Factor, RuntimeFactors}; +use spin_factors::{anyhow, Factor}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; @@ -32,7 +32,7 @@ impl Factor for SqliteFactor { type AppState = AppState; type InstanceBuilder = InstanceState; - fn init( + fn init( &mut self, mut ctx: spin_factors::InitContext, ) -> anyhow::Result<()> { diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 498e96c89..596dc6e74 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -40,10 +40,7 @@ impl Factor for VariablesFactor { type AppState = AppState; type InstanceBuilder = InstanceState; - fn init( - &mut self, - mut ctx: InitContext, - ) -> anyhow::Result<()> { + fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { ctx.link_bindings(spin_world::v1::config::add_to_linker)?; ctx.link_bindings(spin_world::v2::variables::add_to_linker)?; Ok(()) diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 47479236c..62f98c146 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -43,9 +43,9 @@ impl Factor for WasiFactor { type AppState = (); type InstanceBuilder = InstanceBuilder; - fn init( + fn init( &mut self, - mut ctx: InitContext, + mut ctx: InitContext, ) -> anyhow::Result<()> { fn type_annotate(f: F) -> F where diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 0731c7a07..c65f47485 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -77,18 +77,20 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { type InstanceBuilders = #builders_name; type InstanceState = #state_name; - #[allow(clippy::needless_option_as_deref)] - fn init( + fn init + Send + 'static>( &mut self, - linker: &mut #wasmtime::component::Linker<#state_name>, + linker: &mut #wasmtime::component::Linker, ) -> #Result<()> { #( - #Factor::init::( + #Factor::init::( &mut self.#factor_names, - #factors_path::InitContext::::new( + #factors_path::InitContext::::new( linker, - |state| &mut state.#factor_names, - |state| (&mut state.#factor_names, &mut state.__table), + |data| &mut data.as_mut().#factor_names, + |data| { + let state = data.as_mut(); + (&mut state.#factor_names, &mut state.__table) + }, ) ).map_err(#Error::factor_init_error::<#factor_types>)?; )* @@ -226,5 +228,11 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &mut self.__table } } + + impl AsMut<#state_name> for #state_name { + fn as_mut(&mut self) -> &mut Self { + self + } + } }) } diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 0308bd746..103671dfe 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -2,8 +2,8 @@ use spin_app::locked::LockedApp; use spin_factors::{ anyhow::{self, Context}, serde::de::DeserializeOwned, - wasmtime::{Config, Engine}, - App, Linker, RuntimeConfigSource, RuntimeFactors, + wasmtime::{component::Linker, Config, Engine}, + App, RuntimeConfigSource, RuntimeFactors, }; use spin_loader::FilesMountStrategy; @@ -57,8 +57,8 @@ impl TestEnvironment { &self, mut factors: T, ) -> anyhow::Result { - let mut linker = Self::new_linker::(); - factors.init(&mut linker)?; + let mut linker = Self::new_linker::(); + factors.init::(&mut linker)?; let locked_app = self .build_locked_app() @@ -77,7 +77,7 @@ impl TestEnvironment { Ok(factors.build_instance_state(builders)?) } - pub fn new_linker() -> Linker { + pub fn new_linker() -> Linker { let engine = Engine::new(Config::new().async_support(true)) .expect("wasmtime engine failed to initialize"); Linker::::new(&engine) diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 8ec6ee8a2..14da60951 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -1,11 +1,10 @@ use std::any::Any; -use wasmtime::component::ResourceTable; +use wasmtime::component::{Linker, ResourceTable}; use crate::{ prepare::FactorInstanceBuilder, runtime_config::RuntimeConfigTracker, App, Error, - FactorRuntimeConfig, InstanceBuilders, Linker, PrepareContext, RuntimeConfigSource, - RuntimeFactors, + FactorRuntimeConfig, InstanceBuilders, PrepareContext, RuntimeConfigSource, RuntimeFactors, }; /// A contained (i.e., "factored") piece of runtime functionality. @@ -29,7 +28,7 @@ pub trait Factor: Any + Sized { /// This will be called at most once, before any call to [`FactorInstanceBuilder::new`]. /// `InitContext` provides access to a wasmtime `Linker`, so this is where any bindgen /// `add_to_linker` calls go. - fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { + fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { _ = &mut ctx; Ok(()) } @@ -72,22 +71,20 @@ pub trait Factor: Any + Sized { pub type FactorInstanceState = <::InstanceBuilder as FactorInstanceBuilder>::InstanceState; -pub(crate) type GetDataFn = - fn(&mut ::InstanceState) -> &mut FactorInstanceState; +pub(crate) type GetDataFn = fn(&mut T) -> &mut FactorInstanceState; -pub(crate) type GetDataWithTableFn = fn( - &mut ::InstanceState, -) -> (&mut FactorInstanceState, &mut ResourceTable); +pub(crate) type GetDataWithTableFn = + fn(&mut T) -> (&mut FactorInstanceState, &mut ResourceTable); /// An InitContext is passed to [`Factor::init`], giving access to the global /// common [`wasmtime::component::Linker`]. -pub struct InitContext<'a, T: RuntimeFactors, U: Factor> { +pub struct InitContext<'a, T, U: Factor> { pub(crate) linker: &'a mut Linker, pub(crate) get_data: GetDataFn, pub(crate) get_data_with_table: GetDataWithTableFn, } -impl<'a, T: RuntimeFactors, U: Factor> InitContext<'a, T, U> { +impl<'a, T, U: Factor> InitContext<'a, T, U> { #[doc(hidden)] pub fn new( linker: &'a mut Linker, @@ -122,7 +119,7 @@ impl<'a, T: RuntimeFactors, U: Factor> InitContext<'a, T, U> { &mut self, add_to_linker: impl Fn( &mut Linker, - fn(&mut T::InstanceState) -> &mut FactorInstanceState, + fn(&mut T) -> &mut FactorInstanceState, ) -> anyhow::Result<()>, ) -> anyhow::Result<()> { add_to_linker(self.linker, self.get_data) diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 6a9185a20..569b76e9b 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -16,9 +16,6 @@ pub use crate::{ runtime_factors::{RuntimeFactors, RuntimeFactorsInstanceState}, }; -/// A [`wasmtime::component::Linker`] used for a [`RuntimeFactors`] collection. -pub type Linker = wasmtime::component::Linker<::InstanceState>; - // Temporary wrappers while refactoring pub type App = spin_app::App<'static, spin_app::InertLoader>; pub type AppComponent<'a> = spin_app::AppComponent<'a, spin_app::InertLoader>; diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 1293dc13f..1c7e8e1e8 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,6 +1,6 @@ -use wasmtime::component::ResourceTable; +use wasmtime::component::{Linker, ResourceTable}; -use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, Linker, RuntimeConfigSource}; +use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, RuntimeConfigSource}; /// A collection of `Factor`s that are initialized and configured together. /// @@ -10,7 +10,7 @@ use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, Linker, Run /// /// A typical usage of `RuntimeFactors` would look something like the following pseudo-code: /// -/// ```no_run +/// ```ignore /// #[derive(RuntimeFactors)] /// struct MyFactors { /// // ... @@ -21,8 +21,10 @@ use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, Linker, Run /// factors.init(&mut linker)?; /// // Configure the factors with an app and runtime config /// let configured_app = factors.configure_app(app, runtime_config)?; +/// // Prepare instance state builders +/// let builders = factors.prepare(&configured_app, "component-id")?; /// // Build the instance state for the factors -/// let data factors.build_instance_state(&configured_app, component.id()) +/// let data = factors.build_instance_state(builders)?; /// // Initialize a `wasmtime` store with the instance state /// let mut store = wasmtime::Store::new(&engine, data); /// // Instantiate the component @@ -39,7 +41,10 @@ pub trait RuntimeFactors: Sized + 'static { /// Initialize the factors with a linker. /// /// Each factor's `init` is called in turn. - fn init(&mut self, linker: &mut Linker) -> crate::Result<()>; + fn init + Send + 'static>( + &mut self, + linker: &mut Linker, + ) -> crate::Result<()>; /// Configure the factors with an app and runtime config. fn configure_app( @@ -76,7 +81,7 @@ pub trait RuntimeFactors: Sized + 'static { /// Get the state of a particular Factor from the overall InstanceState /// /// Implemented by `#[derive(RuntimeFactors)]` -pub trait RuntimeFactorsInstanceState: Send + 'static { +pub trait RuntimeFactorsInstanceState: AsMut + Send + 'static { fn get_with_table( &mut self, ) -> Option<(&mut FactorInstanceState, &mut ResourceTable)>; diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 342a36034..ed6cdd72f 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -23,6 +23,17 @@ struct Factors { key_value: KeyValueFactor, } +struct Data { + factors_instance_state: FactorsInstanceState, + _other_data: usize, +} + +impl AsMut for Data { + fn as_mut(&mut self) -> &mut FactorsInstanceState { + &mut self.factors_instance_state + } +} + #[tokio::test(flavor = "multi_thread")] async fn smoke_test_works() -> anyhow::Result<()> { let mut factors = Factors { @@ -50,14 +61,15 @@ async fn smoke_test_works() -> anyhow::Result<()> { let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; let mut linker = wasmtime::component::Linker::new(&engine); - factors.init(&mut linker).unwrap(); + factors.init::(&mut linker).unwrap(); let configured_app = factors.configure_app(app, TestSource)?; let builders = factors.prepare(&configured_app, "smoke-app")?; - let data = factors.build_instance_state(builders)?; + let state = factors.build_instance_state(builders)?; assert_eq!( - data.variables + state + .variables .resolver() .resolve("smoke-app", "other".try_into().unwrap()) .await @@ -65,6 +77,11 @@ async fn smoke_test_works() -> anyhow::Result<()> { "" ); + let data = Data { + factors_instance_state: state, + _other_data: 1, + }; + let mut store = wasmtime::Store::new(&engine, data); let component = configured_app.app().components().next().unwrap(); @@ -84,7 +101,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { // Invoke handler let req = http::Request::get("/").body(Default::default()).unwrap(); - let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(store.data_mut()).unwrap(); + let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(store.data_mut().as_mut()).unwrap(); let request = wasi_http.new_incoming_request(req)?; let (response_tx, response_rx) = tokio::sync::oneshot::channel(); let response = wasi_http.new_response_outparam(response_tx)?; From c7f7ba415e89a0f6e6122c0b85c1b81514cddbd7 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 18 Jul 2024 10:09:35 -0400 Subject: [PATCH 059/195] factors: Enhance Factor doc comments Signed-off-by: Lann Martin --- crates/factors/src/factor.rs | 9 ++++++--- crates/factors/src/runtime_factors.rs | 7 ++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 14da60951..72c57371f 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -25,9 +25,12 @@ pub trait Factor: Any + Sized { /// Initializes this `Factor` for a runtime once at runtime startup. /// - /// This will be called at most once, before any call to [`FactorInstanceBuilder::new`]. - /// `InitContext` provides access to a wasmtime `Linker`, so this is where any bindgen - /// `add_to_linker` calls go. + /// This will be called at most once, before any call to + /// [`Factor::prepare`]. `InitContext` provides access to a wasmtime + /// `Linker`, so this is where any bindgen `add_to_linker` calls go. + /// + /// The type parameter `T` here is the same as the [`wasmtime::Store`] type + /// parameter `T`, which will contain the [`RuntimeFactors::InstanceState`]. fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { _ = &mut ctx; Ok(()) diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 1c7e8e1e8..78154b7eb 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -38,15 +38,16 @@ pub trait RuntimeFactors: Sized + 'static { /// The collection of all the `InstanceBuilder`s of the factors. type InstanceBuilders; - /// Initialize the factors with a linker. + /// Initialize the factors with the given linker. /// - /// Each factor's `init` is called in turn. + /// Each factor's `init` is called in turn. Must be called once before + /// [`RuntimeFactors::prepare`]. fn init + Send + 'static>( &mut self, linker: &mut Linker, ) -> crate::Result<()>; - /// Configure the factors with an app and runtime config. + /// Configure the factors with the given app and runtime config. fn configure_app( &self, app: App, From d13415e52373c4bea0c074a646b8f7ce0c557e56 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Wed, 17 Jul 2024 13:37:15 -0500 Subject: [PATCH 060/195] factor: add KV SQLite factor with default path resolution Signed-off-by: Kate Goldenring --- Cargo.lock | 12 ++++- crates/factor-key-value-spin/Cargo.toml | 15 ++++++ crates/factor-key-value-spin/src/lib.rs | 72 +++++++++++++++++++++++++ crates/factor-key-value/src/lib.rs | 2 +- crates/factor-key-value/src/store.rs | 8 ++- crates/factors/Cargo.toml | 2 +- crates/factors/tests/smoke.rs | 44 ++++----------- crates/key-value/src/util.rs | 12 ++--- 8 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 crates/factor-key-value-spin/Cargo.toml create mode 100644 crates/factor-key-value-spin/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 721340087..24138a407 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7568,6 +7568,16 @@ dependencies = [ "spin-key-value-redis", ] +[[package]] +name = "spin-factor-key-value-spin" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "serde 1.0.197", + "spin-factor-key-value", + "spin-key-value-sqlite", +] + [[package]] name = "spin-factor-outbound-http" version = "2.7.0-pre0" @@ -7635,12 +7645,12 @@ dependencies = [ "spin-componentize", "spin-factor-key-value", "spin-factor-key-value-redis", + "spin-factor-key-value-spin", "spin-factor-outbound-http", "spin-factor-outbound-networking", "spin-factor-variables", "spin-factor-wasi", "spin-factors-derive", - "spin-key-value-sqlite", "spin-loader", "thiserror", "tokio", diff --git a/crates/factor-key-value-spin/Cargo.toml b/crates/factor-key-value-spin/Cargo.toml new file mode 100644 index 000000000..29ca47c3f --- /dev/null +++ b/crates/factor-key-value-spin/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "spin-factor-key-value-spin" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +serde = { version = "1.0", features = ["rc"] } +spin-factor-key-value = { path = "../factor-key-value" } +# TODO: merge with this crate +spin-key-value-sqlite = { path = "../key-value-sqlite" } + +[lints] +workspace = true diff --git a/crates/factor-key-value-spin/src/lib.rs b/crates/factor-key-value-spin/src/lib.rs new file mode 100644 index 000000000..db19b54d3 --- /dev/null +++ b/crates/factor-key-value-spin/src/lib.rs @@ -0,0 +1,72 @@ +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use spin_factor_key_value::MakeKeyValueStore; +use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; + +pub struct SpinKeyValueStore { + base_path: PathBuf, +} + +impl SpinKeyValueStore { + pub fn new(base_path: Option) -> anyhow::Result { + let base_path = match base_path { + Some(path) => path, + None => std::env::current_dir().context("failed to get current directory")?, + }; + Ok(Self { base_path }) + } +} + +#[derive(Deserialize, Serialize)] +pub struct SpinKeyValueRuntimeConfig { + path: Option, +} + +impl SpinKeyValueRuntimeConfig { + const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; + + pub fn default(state_dir: Option) -> Self { + let path = state_dir.map(|dir| dir.join(Self::DEFAULT_SPIN_STORE_FILENAME)); + Self { path } + } +} + +/// Resolve a relative path against a base dir. +/// +/// If the path is absolute, it is returned as is. Otherwise, it is resolved against the base dir. +fn resolve_relative_path(path: &Path, base_dir: &Path) -> PathBuf { + if path.is_absolute() { + return path.to_owned(); + } + base_dir.join(path) +} + +impl MakeKeyValueStore for SpinKeyValueStore { + const RUNTIME_CONFIG_TYPE: &'static str = "spin"; + + type RuntimeConfig = SpinKeyValueRuntimeConfig; + + type StoreManager = KeyValueSqlite; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + let location = match runtime_config.path { + Some(path) => { + let path = resolve_relative_path(&path, &self.base_path); + // Create the store's parent directory if necessary + fs::create_dir_all(path.parent().unwrap()) + .context("Failed to create key value store")?; + DatabaseLocation::Path(path) + } + None => DatabaseLocation::InMemory, + }; + Ok(KeyValueSqlite::new(location)) + } +} diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index 7cf274922..c2a1e76ab 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -1,5 +1,5 @@ -mod runtime_config; pub mod delegating_resolver; +mod runtime_config; mod store; use std::{ diff --git a/crates/factor-key-value/src/store.rs b/crates/factor-key-value/src/store.rs index 96310401c..647f90172 100644 --- a/crates/factor-key-value/src/store.rs +++ b/crates/factor-key-value/src/store.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use anyhow::Context; use serde::de::DeserializeOwned; use spin_key_value::StoreManager; @@ -18,8 +19,11 @@ pub(crate) type StoreFromToml = pub(crate) fn store_from_toml_fn(provider_type: T) -> StoreFromToml { Box::new(move |table| { - let runtime_config: T::RuntimeConfig = table.try_into()?; - let provider = provider_type.make_store(runtime_config)?; + let runtime_config: T::RuntimeConfig = + table.try_into().context("could not parse runtime config")?; + let provider = provider_type + .make_store(runtime_config) + .context("could not make store")?; Ok(Arc::new(provider)) }) } diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index ae287f977..7306e5294 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -21,11 +21,11 @@ spin-componentize = { path = "../componentize" } spin-factors-derive = { path = "../factors-derive", features = ["expander"] } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-key-value-redis = { path = "../factor-key-value-redis" } +spin-factor-key-value-spin = { path = "../factor-key-value-spin" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } -spin-key-value-sqlite = { path = "../key-value-sqlite" } spin-loader = { path = "../loader" } tokio = { version = "1", features = ["macros", "rt", "sync"] } toml = "0.8" diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index e4cac5d1a..1de2ff653 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,19 +1,18 @@ use std::path::PathBuf; -use anyhow::{bail, Context}; +use anyhow::Context; use http_body_util::BodyExt; -use serde::Deserialize; use spin_app::App; use spin_factor_key_value::{ delegating_resolver::DelegatingRuntimeConfigResolver, KeyValueFactor, MakeKeyValueStore, }; use spin_factor_key_value_redis::RedisKeyValueStore; +use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; -use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] @@ -28,8 +27,14 @@ struct Factors { #[tokio::test(flavor = "multi_thread")] async fn smoke_test_works() -> anyhow::Result<()> { let mut key_value_resolver = DelegatingRuntimeConfigResolver::default(); - key_value_resolver.add_default_store("default", "spin", toml::value::Table::new()); - key_value_resolver.add_store_type(TestSpinKeyValueStore)?; + let default_config = + SpinKeyValueRuntimeConfig::default(Some(PathBuf::from("tests/smoke-app/.spin"))); + key_value_resolver.add_default_store( + "default", + SpinKeyValueStore::RUNTIME_CONFIG_TYPE, + toml::value::Table::try_from(default_config)?, + ); + key_value_resolver.add_store_type(SpinKeyValueStore::new(None)?)?; key_value_resolver.add_store_type(RedisKeyValueStore)?; let mut factors = Factors { @@ -140,32 +145,3 @@ impl RuntimeConfigSource for TestSource { Ok(Some(config)) } } - -struct TestSpinKeyValueStore; - -impl MakeKeyValueStore for TestSpinKeyValueStore { - const RUNTIME_CONFIG_TYPE: &'static str = "spin"; - - type RuntimeConfig = TestSpinKeyValueRuntimeConfig; - - type StoreManager = KeyValueSqlite; - - fn make_store( - &self, - runtime_config: Self::RuntimeConfig, - ) -> anyhow::Result { - let location = match runtime_config.path { - Some(_) => { - // TODO(lann): need state_dir to derive default store path - bail!("spin key value runtime config not implemented") - } - None => DatabaseLocation::InMemory, - }; - Ok(KeyValueSqlite::new(location)) - } -} - -#[derive(Deserialize)] -struct TestSpinKeyValueRuntimeConfig { - path: Option, -} diff --git a/crates/key-value/src/util.rs b/crates/key-value/src/util.rs index 78498e3a0..0af704e62 100644 --- a/crates/key-value/src/util.rs +++ b/crates/key-value/src/util.rs @@ -52,13 +52,11 @@ impl DelegatingStoreManager { #[async_trait] impl StoreManager for DelegatingStoreManager { async fn get(&self, name: &str) -> Result, Error> { - let store = match self.delegates.get(name) { - Some(store) => store, - None => { - &(self.default_manager)(name).ok_or(Error::NoSuchStore)? - } - }; - store.get(name).await + let store = match self.delegates.get(name) { + Some(store) => store, + None => &(self.default_manager)(name).ok_or(Error::NoSuchStore)?, + }; + store.get(name).await } fn is_defined(&self, store_name: &str) -> bool { From ae292ecf6809906af5efadc2e943ff1ad0728081 Mon Sep 17 00:00:00 2001 From: Rohit Dandamudi Date: Tue, 9 Jul 2024 11:52:52 -0700 Subject: [PATCH 061/195] Add inital work on factors outbound redis Signed-off-by: Rohit Dandamudi --- Cargo.lock | 16 + crates/factor-outbound-redis/Cargo.toml | 20 ++ .../factor-outbound-redis/src/factor_redis.rs | 316 ++++++++++++++++++ crates/factor-outbound-redis/src/lib.rs | 66 ++++ crates/factors/Cargo.toml | 1 + crates/factors/tests/smoke.rs | 1 + 6 files changed, 420 insertions(+) create mode 100644 crates/factor-outbound-redis/Cargo.toml create mode 100644 crates/factor-outbound-redis/src/factor_redis.rs create mode 100644 crates/factor-outbound-redis/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index f947c7c49..745b37a2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7635,6 +7635,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "spin-factor-outbound-redis" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "redis 0.21.7", + "spin-core", + "spin-factor-outbound-networking", + "spin-factor-wasi", + "spin-factors", + "spin-world", + "table", + "tracing", +] + [[package]] name = "spin-factor-variables" version = "2.7.0-pre0" @@ -7677,6 +7692,7 @@ dependencies = [ "spin-factor-key-value-spin", "spin-factor-outbound-http", "spin-factor-outbound-networking", + "spin-factor-outbound-redis", "spin-factor-variables", "spin-factor-wasi", "spin-factors-derive", diff --git a/crates/factor-outbound-redis/Cargo.toml b/crates/factor-outbound-redis/Cargo.toml new file mode 100644 index 000000000..4e3b25496 --- /dev/null +++ b/crates/factor-outbound-redis/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "spin-factor-outbound-redis" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors = { path = "../factors" } +spin-core = { path = "../core" } +spin-world = { path = "../world" } +tracing = { workspace = true } +table = { path = "../table" } +redis = { version = "0.21", features = ["tokio-comp", "tokio-native-tls-comp", "aio"] } + +# wasmtime-wasi-http = { workspace = true } +[lints] +workspace = true diff --git a/crates/factor-outbound-redis/src/factor_redis.rs b/crates/factor-outbound-redis/src/factor_redis.rs new file mode 100644 index 000000000..af4f737ec --- /dev/null +++ b/crates/factor-outbound-redis/src/factor_redis.rs @@ -0,0 +1,316 @@ +use anyhow::Result; +use redis::{aio::Connection, AsyncCommands, FromRedisValue, Value}; +use spin_core::{async_trait, wasmtime::component::Resource}; +use spin_world::v1::{redis as v1, redis_types}; +use spin_world::v2::redis::{ + self as v2, Connection as RedisConnection, Error, RedisParameter, RedisResult, +}; + +// pub use host_component::OutboundRedisComponent; +use tracing::{instrument, Level}; + +struct RedisResults(Vec); + +impl FromRedisValue for RedisResults { + fn from_redis_value(value: &Value) -> redis::RedisResult { + fn append(values: &mut Vec, value: &Value) { + match value { + Value::Nil | Value::Okay => (), + Value::Int(v) => values.push(RedisResult::Int64(*v)), + Value::Data(bytes) => values.push(RedisResult::Binary(bytes.to_owned())), + Value::Bulk(bulk) => bulk.iter().for_each(|value| append(values, value)), + Value::Status(message) => values.push(RedisResult::Status(message.to_owned())), + } + } + + let mut values = Vec::new(); + append(&mut values, value); + Ok(RedisResults(values)) + } +} + +// pub struct crate::InstanceState{ +// allowed_hosts: spin_outbound_networking::AllowedHostsConfig, +// connections: table::Table, +// } + +// impl Default for crate::InstanceState{ +// fn default() -> Self { +// Self { +// allowed_hosts: Default::default(), +// connections: table::Table::new(1024), +// } +// } +// } + +impl crate::InstanceState { + async fn is_address_allowed(&self, address: &str) -> bool { + // TODO: handle better + self.allowed_hosts.check_url(address, "redis").await.unwrap() + } + + async fn establish_connection( + &mut self, + address: String, + ) -> Result, Error> { + let conn = redis::Client::open(address.as_str()) + .map_err(|_| Error::InvalidAddress)? + .get_async_connection() + .await + .map_err(other_error)?; + self.connections + .push(conn) + .map(Resource::new_own) + .map_err(|_| Error::TooManyConnections) + } +} + +impl v2::Host for crate::InstanceState { + fn convert_error(&mut self, error: Error) -> Result { + Ok(error) + } +} + +#[async_trait] +impl v2::HostConnection for crate::InstanceState{ + #[instrument(name = "spin_outbound_redis.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis"))] + async fn open(&mut self, address: String) -> Result, Error> { + if !self.is_address_allowed(&address) { + return Err(Error::InvalidAddress); + } + + self.establish_connection(address).await + } + + #[instrument(name = "spin_outbound_redis.publish", skip(self, connection, payload), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("PUBLISH {}", channel)))] + async fn publish( + &mut self, + connection: Resource, + channel: String, + payload: Vec, + ) -> Result<(), Error> { + let conn = self.get_conn(connection).await.map_err(other_error)?; + conn.publish(&channel, &payload) + .await + .map_err(other_error)?; + Ok(()) + } + + #[instrument(name = "spin_outbound_redis.get", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("GET {}", key)))] + async fn get( + &mut self, + connection: Resource, + key: String, + ) -> Result>, Error> { + let conn = self.get_conn(connection).await.map_err(other_error)?; + let value = conn.get(&key).await.map_err(other_error)?; + Ok(value) + } + + #[instrument(name = "spin_outbound_redis.set", skip(self, connection, value), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SET {}", key)))] + async fn set( + &mut self, + connection: Resource, + key: String, + value: Vec, + ) -> Result<(), Error> { + let conn = self.get_conn(connection).await.map_err(other_error)?; + conn.set(&key, &value).await.map_err(other_error)?; + Ok(()) + } + + #[instrument(name = "spin_outbound_redis.incr", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("INCRBY {} 1", key)))] + async fn incr( + &mut self, + connection: Resource, + key: String, + ) -> Result { + let conn = self.get_conn(connection).await.map_err(other_error)?; + let value = conn.incr(&key, 1).await.map_err(other_error)?; + Ok(value) + } + + #[instrument(name = "spin_outbound_redis.del", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("DEL {}", keys.join(" "))))] + async fn del( + &mut self, + connection: Resource, + keys: Vec, + ) -> Result { + let conn = self.get_conn(connection).await.map_err(other_error)?; + let value = conn.del(&keys).await.map_err(other_error)?; + Ok(value) + } + + #[instrument(name = "spin_outbound_redis.sadd", skip(self, connection, values), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SADD {} {}", key, values.join(" "))))] + async fn sadd( + &mut self, + connection: Resource, + key: String, + values: Vec, + ) -> Result { + let conn = self.get_conn(connection).await.map_err(other_error)?; + let value = conn.sadd(&key, &values).await.map_err(|e| { + if e.kind() == redis::ErrorKind::TypeError { + Error::TypeError + } else { + Error::Other(e.to_string()) + } + })?; + Ok(value) + } + + #[instrument(name = "spin_outbound_redis.smembers", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SMEMBERS {}", key)))] + async fn smembers( + &mut self, + connection: Resource, + key: String, + ) -> Result, Error> { + let conn = self.get_conn(connection).await.map_err(other_error)?; + let value = conn.smembers(&key).await.map_err(other_error)?; + Ok(value) + } + + #[instrument(name = "spin_outbound_redis.srem", skip(self, connection, values), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SREM {} {}", key, values.join(" "))))] + async fn srem( + &mut self, + connection: Resource, + key: String, + values: Vec, + ) -> Result { + let conn = self.get_conn(connection).await.map_err(other_error)?; + let value = conn.srem(&key, &values).await.map_err(other_error)?; + Ok(value) + } + + #[instrument(name = "spin_outbound_redis.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("{}", command)))] + async fn execute( + &mut self, + connection: Resource, + command: String, + arguments: Vec, + ) -> Result, Error> { + let conn = self.get_conn(connection).await?; + let mut cmd = redis::cmd(&command); + arguments.iter().for_each(|value| match value { + RedisParameter::Int64(v) => { + cmd.arg(v); + } + RedisParameter::Binary(v) => { + cmd.arg(v); + } + }); + + cmd.query_async::<_, RedisResults>(conn) + .await + .map(|values| values.0) + .map_err(other_error) + } + + fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { + self.connections.remove(connection.rep()); + Ok(()) + } +} + +fn other_error(e: impl std::fmt::Display) -> Error { + Error::Other(e.to_string()) +} + +/// Delegate a function call to the v2::HostConnection implementation +macro_rules! delegate { + ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ + if !$self.is_address_allowed(&$address) { + return Err(v1::Error::Error); + } + let connection = match $self.establish_connection($address).await { + Ok(c) => c, + Err(_) => return Err(v1::Error::Error), + }; + ::$name($self, connection, $($arg),*) + .await + .map_err(|_| v1::Error::Error) + }}; +} + +#[async_trait] +impl v1::Host for crate::InstanceState{ + async fn publish( + &mut self, + address: String, + channel: String, + payload: Vec, + ) -> Result<(), v1::Error> { + delegate!(self.publish(address, channel, payload)) + } + + async fn get(&mut self, address: String, key: String) -> Result, v1::Error> { + delegate!(self.get(address, key)).map(|v| v.unwrap_or_default()) + } + + async fn set(&mut self, address: String, key: String, value: Vec) -> Result<(), v1::Error> { + delegate!(self.set(address, key, value)) + } + + async fn incr(&mut self, address: String, key: String) -> Result { + delegate!(self.incr(address, key)) + } + + async fn del(&mut self, address: String, keys: Vec) -> Result { + delegate!(self.del(address, keys)).map(|v| v as i64) + } + + async fn sadd( + &mut self, + address: String, + key: String, + values: Vec, + ) -> Result { + delegate!(self.sadd(address, key, values)).map(|v| v as i64) + } + + async fn smembers(&mut self, address: String, key: String) -> Result, v1::Error> { + delegate!(self.smembers(address, key)) + } + + async fn srem( + &mut self, + address: String, + key: String, + values: Vec, + ) -> Result { + delegate!(self.srem(address, key, values)).map(|v| v as i64) + } + + async fn execute( + &mut self, + address: String, + command: String, + arguments: Vec, + ) -> Result, v1::Error> { + delegate!(self.execute( + address, + command, + arguments.into_iter().map(Into::into).collect() + )) + .map(|v| v.into_iter().map(Into::into).collect()) + } +} + +impl redis_types::Host for crate::InstanceState{ + fn convert_error(&mut self, error: redis_types::Error) -> Result { + Ok(error) + } +} + +impl crate::InstanceState{ + async fn get_conn( + &mut self, + connection: Resource, + ) -> Result<&mut Connection, Error> { + self.connections + .get_mut(connection.rep()) + .ok_or(Error::Other( + "could not find connection for resource".into(), + )) + } +} diff --git a/crates/factor-outbound-redis/src/lib.rs b/crates/factor-outbound-redis/src/lib.rs new file mode 100644 index 000000000..1a55694ef --- /dev/null +++ b/crates/factor-outbound-redis/src/lib.rs @@ -0,0 +1,66 @@ +mod factor_redis; + +use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; +use spin_factors::{ + anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, + SelfInstanceBuilder, +}; + +use redis::aio::Connection; + +// use wasmtime_wasi_http::WasiHttpCtx; +pub struct OutboundRedisFactor; + +impl Factor for OutboundRedisFactor { + type RuntimeConfig = (); + type AppState = (); + type InstanceBuilder = InstanceState; + + fn init( + &mut self, + mut ctx: spin_factors::InitContext, + ) -> anyhow::Result<()> { + ctx.link_bindings(spin_world::v1::redis::add_to_linker)?; + ctx.link_bindings(spin_world::v2::redis::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + _ctx: ConfigureAppContext, + ) -> anyhow::Result { + Ok(()) + } + + fn prepare( + &self, + _ctx: PrepareContext, + builders: &mut InstanceBuilders, + ) -> anyhow::Result { + let allowed_hosts = builders + .get_mut::()? + .allowed_hosts(); + Ok(InstanceState { + allowed_hosts, + connections: table::Table::new(1024), + }) + } +} + +pub struct InstanceState { + allowed_hosts: OutboundAllowedHosts, + connections: table::Table, + +} + + +// impl Default for InstanceState { +// fn default() -> Self { +// Self { +// allowed_hosts: Default::default(), +// connections: table::Table::new(1024), +// } +// } +// } + +impl SelfInstanceBuilder for InstanceState {} diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 7306e5294..5b24e0486 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -24,6 +24,7 @@ spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-key-value-spin = { path = "../factor-key-value-spin" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } spin-loader = { path = "../loader" } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 7ec8cc059..5c26c1edf 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -22,6 +22,7 @@ struct Factors { outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, key_value: KeyValueFactor, + redis: OutboundRedisFactor, } struct Data { From ac98109ed468a27a599b59ea7b79ad490451fe73 Mon Sep 17 00:00:00 2001 From: Rohit Dandamudi Date: Wed, 10 Jul 2024 17:27:00 -0700 Subject: [PATCH 062/195] Add outbound redis factors unit test Clean some factor_outbound_redis files Signed-off-by: Rohit Dandamudi --- Cargo.lock | 4 +- crates/factor-outbound-redis/Cargo.toml | 8 +- .../src/{factor_redis.rs => host.rs} | 97 +++++++++---------- crates/factor-outbound-redis/src/lib.rs | 28 +----- .../tests/factor_test.rs | 56 +++++++++++ crates/factors/Cargo.toml | 1 - crates/factors/tests/smoke.rs | 1 - 7 files changed, 115 insertions(+), 80 deletions(-) rename crates/factor-outbound-redis/src/{factor_redis.rs => host.rs} (92%) create mode 100644 crates/factor-outbound-redis/tests/factor_test.rs diff --git a/Cargo.lock b/Cargo.lock index 745b37a2c..cf9ea14a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7643,10 +7643,13 @@ dependencies = [ "redis 0.21.7", "spin-core", "spin-factor-outbound-networking", + "spin-factor-variables", "spin-factor-wasi", "spin-factors", + "spin-factors-test", "spin-world", "table", + "tokio", "tracing", ] @@ -7692,7 +7695,6 @@ dependencies = [ "spin-factor-key-value-spin", "spin-factor-outbound-http", "spin-factor-outbound-networking", - "spin-factor-outbound-redis", "spin-factor-variables", "spin-factor-wasi", "spin-factors-derive", diff --git a/crates/factor-outbound-redis/Cargo.toml b/crates/factor-outbound-redis/Cargo.toml index 4e3b25496..330e070a4 100644 --- a/crates/factor-outbound-redis/Cargo.toml +++ b/crates/factor-outbound-redis/Cargo.toml @@ -7,7 +7,6 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" spin-factor-outbound-networking = { path = "../factor-outbound-networking" } -spin-factor-wasi = { path = "../factor-wasi" } spin-factors = { path = "../factors" } spin-core = { path = "../core" } spin-world = { path = "../world" } @@ -15,6 +14,13 @@ tracing = { workspace = true } table = { path = "../table" } redis = { version = "0.21", features = ["tokio-comp", "tokio-native-tls-comp", "aio"] } + +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } +spin-factor-variables = { path = "../factor-variables" } +spin-factor-wasi = { path = "../factor-wasi" } + # wasmtime-wasi-http = { workspace = true } [lints] workspace = true diff --git a/crates/factor-outbound-redis/src/factor_redis.rs b/crates/factor-outbound-redis/src/host.rs similarity index 92% rename from crates/factor-outbound-redis/src/factor_redis.rs rename to crates/factor-outbound-redis/src/host.rs index af4f737ec..baf3bb3c9 100644 --- a/crates/factor-outbound-redis/src/factor_redis.rs +++ b/crates/factor-outbound-redis/src/host.rs @@ -1,52 +1,21 @@ use anyhow::Result; use redis::{aio::Connection, AsyncCommands, FromRedisValue, Value}; use spin_core::{async_trait, wasmtime::component::Resource}; +use spin_factor_outbound_networking::OutboundAllowedHosts; use spin_world::v1::{redis as v1, redis_types}; use spin_world::v2::redis::{ self as v2, Connection as RedisConnection, Error, RedisParameter, RedisResult, }; - -// pub use host_component::OutboundRedisComponent; use tracing::{instrument, Level}; -struct RedisResults(Vec); - -impl FromRedisValue for RedisResults { - fn from_redis_value(value: &Value) -> redis::RedisResult { - fn append(values: &mut Vec, value: &Value) { - match value { - Value::Nil | Value::Okay => (), - Value::Int(v) => values.push(RedisResult::Int64(*v)), - Value::Data(bytes) => values.push(RedisResult::Binary(bytes.to_owned())), - Value::Bulk(bulk) => bulk.iter().for_each(|value| append(values, value)), - Value::Status(message) => values.push(RedisResult::Status(message.to_owned())), - } - } - - let mut values = Vec::new(); - append(&mut values, value); - Ok(RedisResults(values)) - } +pub struct InstanceState { + pub allowed_hosts: OutboundAllowedHosts, + pub connections: table::Table, } -// pub struct crate::InstanceState{ -// allowed_hosts: spin_outbound_networking::AllowedHostsConfig, -// connections: table::Table, -// } - -// impl Default for crate::InstanceState{ -// fn default() -> Self { -// Self { -// allowed_hosts: Default::default(), -// connections: table::Table::new(1024), -// } -// } -// } - -impl crate::InstanceState { - async fn is_address_allowed(&self, address: &str) -> bool { - // TODO: handle better - self.allowed_hosts.check_url(address, "redis").await.unwrap() +impl InstanceState { + async fn is_address_allowed(&self, address: &str) -> Result { + self.allowed_hosts.check_url(address, "redis").await } async fn establish_connection( @@ -63,6 +32,17 @@ impl crate::InstanceState { .map(Resource::new_own) .map_err(|_| Error::TooManyConnections) } + + async fn get_conn( + &mut self, + connection: Resource, + ) -> Result<&mut Connection, Error> { + self.connections + .get_mut(connection.rep()) + .ok_or(Error::Other( + "could not find connection for resource".into(), + )) + } } impl v2::Host for crate::InstanceState { @@ -72,10 +52,14 @@ impl v2::Host for crate::InstanceState { } #[async_trait] -impl v2::HostConnection for crate::InstanceState{ +impl v2::HostConnection for crate::InstanceState { #[instrument(name = "spin_outbound_redis.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis"))] async fn open(&mut self, address: String) -> Result, Error> { - if !self.is_address_allowed(&address) { + if !self + .is_address_allowed(&address) + .await + .map_err(|e| v2::Error::Other(e.to_string()))? + { return Err(Error::InvalidAddress); } @@ -219,7 +203,7 @@ fn other_error(e: impl std::fmt::Display) -> Error { /// Delegate a function call to the v2::HostConnection implementation macro_rules! delegate { ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ - if !$self.is_address_allowed(&$address) { + if !$self.is_address_allowed(&$address).await.map_err(|_| v1::Error::Error)? { return Err(v1::Error::Error); } let connection = match $self.establish_connection($address).await { @@ -233,7 +217,7 @@ macro_rules! delegate { } #[async_trait] -impl v1::Host for crate::InstanceState{ +impl v1::Host for crate::InstanceState { async fn publish( &mut self, address: String, @@ -296,21 +280,28 @@ impl v1::Host for crate::InstanceState{ } } -impl redis_types::Host for crate::InstanceState{ +impl redis_types::Host for crate::InstanceState { fn convert_error(&mut self, error: redis_types::Error) -> Result { Ok(error) } } -impl crate::InstanceState{ - async fn get_conn( - &mut self, - connection: Resource, - ) -> Result<&mut Connection, Error> { - self.connections - .get_mut(connection.rep()) - .ok_or(Error::Other( - "could not find connection for resource".into(), - )) +struct RedisResults(Vec); + +impl FromRedisValue for RedisResults { + fn from_redis_value(value: &Value) -> redis::RedisResult { + fn append(values: &mut Vec, value: &Value) { + match value { + Value::Nil | Value::Okay => (), + Value::Int(v) => values.push(RedisResult::Int64(*v)), + Value::Data(bytes) => values.push(RedisResult::Binary(bytes.to_owned())), + Value::Bulk(bulk) => bulk.iter().for_each(|value| append(values, value)), + Value::Status(message) => values.push(RedisResult::Status(message.to_owned())), + } + } + + let mut values = Vec::new(); + append(&mut values, value); + Ok(RedisResults(values)) } } diff --git a/crates/factor-outbound-redis/src/lib.rs b/crates/factor-outbound-redis/src/lib.rs index 1a55694ef..cb84cc29c 100644 --- a/crates/factor-outbound-redis/src/lib.rs +++ b/crates/factor-outbound-redis/src/lib.rs @@ -1,14 +1,12 @@ -mod factor_redis; +mod host; -use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; +use host::InstanceState; +use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factors::{ anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; - -use redis::aio::Connection; - -// use wasmtime_wasi_http::WasiHttpCtx; +use std::marker::Send; pub struct OutboundRedisFactor; impl Factor for OutboundRedisFactor { @@ -16,7 +14,7 @@ impl Factor for OutboundRedisFactor { type AppState = (); type InstanceBuilder = InstanceState; - fn init( + fn init( &mut self, mut ctx: spin_factors::InitContext, ) -> anyhow::Result<()> { @@ -47,20 +45,4 @@ impl Factor for OutboundRedisFactor { } } -pub struct InstanceState { - allowed_hosts: OutboundAllowedHosts, - connections: table::Table, - -} - - -// impl Default for InstanceState { -// fn default() -> Self { -// Self { -// allowed_hosts: Default::default(), -// connections: table::Table::new(1024), -// } -// } -// } - impl SelfInstanceBuilder for InstanceState {} diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs new file mode 100644 index 000000000..552a871cc --- /dev/null +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -0,0 +1,56 @@ +use anyhow::bail; +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_outbound_redis::OutboundRedisFactor; +use spin_factor_variables::{StaticVariables, VariablesFactor}; +use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use spin_world::v2::redis::{Error, HostConnection}; + +#[derive(RuntimeFactors)] +struct TestFactors { + wasi: WasiFactor, + variables: VariablesFactor, + networking: OutboundNetworkingFactor, + redis: OutboundRedisFactor, +} + +fn get_test_factors() -> TestFactors { + TestFactors { + wasi: WasiFactor::new(DummyFilesMounter), + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + redis: OutboundRedisFactor, + } +} + +#[tokio::test] +async fn no_outbound_hosts_fails() -> anyhow::Result<()> { + let mut factors = get_test_factors(); + + factors.variables.add_provider_type(StaticVariables)?; + + let env = TestEnvironment { + manifest: toml! { + spin_manifest_version = 2 + application.name = "test-app" + [[trigger.test]] + + [component.test-component] + source = "does-not-exist.wasm" + }, + ..Default::default() + }; + let mut state = env.build_instance_state(factors).await?; + let connection = state + .redis + .open("redis://redis.test:8080".to_string()) + .await; + + let Err(err) = connection else { + bail!("expected Error, got Ok"); + }; + + assert!(matches!(err, Error::InvalidAddress)); + Ok(()) +} diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 5b24e0486..7306e5294 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -24,7 +24,6 @@ spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-key-value-spin = { path = "../factor-key-value-spin" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } -spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } spin-loader = { path = "../loader" } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 5c26c1edf..7ec8cc059 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -22,7 +22,6 @@ struct Factors { outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, key_value: KeyValueFactor, - redis: OutboundRedisFactor, } struct Data { From 0907fec07a1ae77c2cfa031a324cc2b243a7f022 Mon Sep 17 00:00:00 2001 From: Rohit Dandamudi Date: Thu, 18 Jul 2024 13:12:15 -0700 Subject: [PATCH 063/195] fix: change the func impl to reflect trait Also remove explicit prelude usage Signed-off-by: Rohit Dandamudi --- crates/factor-outbound-pg/src/lib.rs | 2 +- crates/factor-outbound-redis/src/lib.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/factor-outbound-pg/src/lib.rs b/crates/factor-outbound-pg/src/lib.rs index 143666932..db2283cbf 100644 --- a/crates/factor-outbound-pg/src/lib.rs +++ b/crates/factor-outbound-pg/src/lib.rs @@ -14,7 +14,7 @@ impl Factor for OutboundPgFactor { type AppState = (); type InstanceBuilder = InstanceState; - fn init( + fn init( &mut self, mut ctx: spin_factors::InitContext, ) -> anyhow::Result<()> { diff --git a/crates/factor-outbound-redis/src/lib.rs b/crates/factor-outbound-redis/src/lib.rs index cb84cc29c..bcd983091 100644 --- a/crates/factor-outbound-redis/src/lib.rs +++ b/crates/factor-outbound-redis/src/lib.rs @@ -6,7 +6,6 @@ use spin_factors::{ anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; -use std::marker::Send; pub struct OutboundRedisFactor; impl Factor for OutboundRedisFactor { From 7cda1ea16179c2f26210c8ff616dffbc813c8ba9 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 18 Jul 2024 17:27:02 -0400 Subject: [PATCH 064/195] factors: Pare down spin-core and spin-app These features are moving as part of the migration to factors. Signed-off-by: Lann Martin --- Cargo.lock | 20 +- crates/app/Cargo.toml | 2 - crates/app/src/host_component.rs | 123 - crates/app/src/lib.rs | 245 +- crates/core/Cargo.toml | 26 +- crates/core/src/host_component.rs | 305 --- crates/core/src/io.rs | 34 - crates/core/src/lib.rs | 278 +- crates/core/src/preview1.rs | 212 -- crates/core/src/store.rs | 546 +--- crates/core/src/wasi_2023_10_18.rs | 2465 ------------------ crates/core/src/wasi_2023_11_10.rs | 2455 ----------------- crates/core/tests/core-wasi-test/src/main.rs | 11 - crates/core/tests/integration_test.rs | 261 +- crates/factor-wasi/Cargo.toml | 1 + crates/factor-wasi/src/lib.rs | 10 +- crates/factors-test/src/lib.rs | 4 +- crates/factors/src/lib.rs | 4 +- crates/factors/src/prepare.rs | 4 +- crates/factors/tests/smoke.rs | 2 +- 20 files changed, 175 insertions(+), 6833 deletions(-) delete mode 100644 crates/app/src/host_component.rs delete mode 100644 crates/core/src/host_component.rs delete mode 100644 crates/core/src/io.rs delete mode 100644 crates/core/src/preview1.rs delete mode 100644 crates/core/src/wasi_2023_10_18.rs delete mode 100644 crates/core/src/wasi_2023_11_10.rs diff --git a/Cargo.lock b/Cargo.lock index cf9ea14a0..4fbcc3c94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7341,10 +7341,8 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "async-trait", - "ouroboros", "serde 1.0.197", "serde_json", - "spin-core", "spin-locked-app", "spin-serde", "thiserror", @@ -7493,24 +7491,17 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "async-trait", - "bytes", - "cap-primitives 3.0.0", - "cap-std 3.0.0", "crossbeam-channel", - "futures", - "http 1.1.0", - "io-extras", - "rustix 0.37.27", + "serde_json", "spin-componentize", - "spin-telemetry", - "system-interface", - "tempfile", + "spin-factor-wasi", + "spin-factors", + "spin-factors-test", + "spin-locked-app", "tokio", "tracing", - "wasi-common", "wasmtime", "wasmtime-wasi", - "wasmtime-wasi-http", ] [[package]] @@ -7672,6 +7663,7 @@ version = "2.7.0-pre0" dependencies = [ "async-trait", "cap-primitives 3.0.0", + "spin-app", "spin-factors", "spin-factors-test", "tokio", diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index fadb5e350..2f0d0cf18 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -7,10 +7,8 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" async-trait = "0.1" -ouroboros = "0.18.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -spin-core = { path = "../core" } spin-locked-app = { path = "../locked-app" } spin-serde = { path = "../serde" } thiserror = "1.0" diff --git a/crates/app/src/host_component.rs b/crates/app/src/host_component.rs deleted file mode 100644 index c5ab96c4e..000000000 --- a/crates/app/src/host_component.rs +++ /dev/null @@ -1,123 +0,0 @@ -use std::{any::Any, sync::Arc}; - -use anyhow::Context; -use spin_core::{ - AnyHostComponentDataHandle, EngineBuilder, HostComponent, HostComponentDataHandle, - HostComponentsData, -}; - -use crate::{App, AppComponent}; - -/// A trait for "dynamic" Spin host components. -/// -/// This extends [`HostComponent`] to support per-[`AppComponent`] dynamic -/// runtime configuration. -/// -/// Dynamic host components differ from regular host components in that they can be -/// configured on a per-component basis. -pub trait DynamicHostComponent: HostComponent { - /// Called on [`AppComponent`] instance initialization. - /// - /// The `data` returned by [`HostComponent::build_data`] is passed, along - /// with a reference to the `component` being instantiated. - fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()>; - - /// Called on [`App`] load to validate any configuration needed by this - /// host component. - /// - /// Note that the _absence_ of configuration should not be treated as an - /// error here, as the app may not use this host component at all. - #[allow(unused_variables)] - fn validate_app(&self, app: &App) -> anyhow::Result<()> { - Ok(()) - } -} - -impl DynamicHostComponent for Arc { - fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()> { - (**self).update_data(data, component) - } -} - -/// A version of `DynamicHostComponent` which can be made into a trait object. -/// -/// This is only implemented for `T: DynamicHostComponent`. We want to make `DynamicHostComponent` -/// into a trait object so that we can store them into a heterogeneous collection in `DynamicHostComponents`. -/// -/// `DynamicHostComponent` can't be made into a trait object itself since `HostComponent::add_to_linker` -/// does not have a `self` parameter (and thus cannot be add to the object's vtable). -trait DynSafeDynamicHostComponent { - /// The moral equivalent to `DynamicHostComponent::update_data` - fn update_data_any(&self, data: &mut dyn Any, component: &AppComponent) -> anyhow::Result<()>; - /// The moral equivalent to `DynamicHostComponent::validate_app` - fn validate_app(&self, app: &App) -> anyhow::Result<()>; -} - -impl DynSafeDynamicHostComponent for T -where - T::Data: Any, -{ - fn update_data_any(&self, data: &mut dyn Any, component: &AppComponent) -> anyhow::Result<()> { - let data = data.downcast_mut().context("wrong data type")?; - self.update_data(data, component) - } - - fn validate_app(&self, app: &App) -> anyhow::Result<()> { - T::validate_app(self, app) - } -} - -struct DynamicHostComponentWithHandle { - host_component: Arc, - handle: AnyHostComponentDataHandle, -} - -/// A heterogeneous collection of dynamic host components. -/// -/// This is stored in an `AppLoader` so that the host components -/// can be referenced and updated at a later point. This is effectively -/// what makes a `DynamicHostComponent` "dynamic" and differentiates it from -/// a regular `HostComponent`. -#[derive(Default)] -pub(crate) struct DynamicHostComponents { - host_components: Vec, -} - -impl DynamicHostComponents { - pub fn add_dynamic_host_component( - &mut self, - engine_builder: &mut EngineBuilder, - host_component: DHC, - ) -> anyhow::Result> { - let host_component = Arc::new(host_component); - let handle = engine_builder.add_host_component(host_component.clone())?; - self.host_components.push(DynamicHostComponentWithHandle { - host_component, - handle: handle.into(), - }); - Ok(handle.into()) - } - - pub fn update_data( - &self, - host_components_data: &mut HostComponentsData, - component: &AppComponent, - ) -> anyhow::Result<()> { - for DynamicHostComponentWithHandle { - host_component, - handle, - } in &self.host_components - { - let data = host_components_data.get_or_insert_any(*handle); - host_component.update_data_any(data.as_mut(), component)?; - } - Ok(()) - } - - pub fn validate_app(&self, app: &App) -> anyhow::Result<()> { - for DynamicHostComponentWithHandle { host_component, .. } in &self.host_components { - host_component.validate_app(app)?; - } - Ok(()) - } -} diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 97077032d..22d97ceba 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -6,22 +6,16 @@ #![deny(missing_docs)] -mod host_component; +use serde::Deserialize; use serde_json::Value; +use spin_locked_app::MetadataExt; + +use locked::{ContentPath, LockedApp, LockedComponent, LockedComponentSource, LockedTrigger}; + pub use spin_locked_app::locked; pub use spin_locked_app::values; pub use spin_locked_app::{Error, MetadataKey, Result}; -use ouroboros::self_referencing; -use serde::Deserialize; -use spin_core::{wasmtime, Engine, EngineBuilder, HostComponentDataHandle, StoreBuilder}; - -use host_component::DynamicHostComponents; -use locked::{ContentPath, LockedApp, LockedComponent, LockedComponentSource, LockedTrigger}; -use spin_locked_app::MetadataExt; - -pub use async_trait::async_trait; -pub use host_component::DynamicHostComponent; pub use locked::Variable; /// MetadataKey for extracting the application name. @@ -33,130 +27,28 @@ pub const APP_DESCRIPTION_KEY: MetadataKey = MetadataKey::new("description"); /// MetadataKey for extracting the OCI image digest. pub const OCI_IMAGE_DIGEST_KEY: MetadataKey = MetadataKey::new("oci_image_digest"); -/// A trait for implementing the low-level operations needed to load an [`App`]. -// TODO(lann): Should this migrate to spin-loader? -#[async_trait] -pub trait Loader { - /// Called with an implementation-defined `uri` pointing to some - /// representation of a [`LockedApp`], which will be loaded. - async fn load_app(&self, uri: &str) -> anyhow::Result; - - /// Called with a [`LockedComponentSource`] pointing to a Wasm component - /// binary, which will be loaded. - async fn load_component( - &self, - engine: &wasmtime::Engine, - source: &LockedComponentSource, - ) -> anyhow::Result; - - /// Called with a [`LockedComponentSource`] pointing to a Wasm module - /// binary, which will be loaded. - async fn load_module( - &self, - engine: &wasmtime::Engine, - source: &LockedComponentSource, - ) -> anyhow::Result; - - /// Called with an [`AppComponent`]; any `files` configured with the - /// component should be "mounted" into the `store_builder`, via e.g. - /// [`StoreBuilder::read_only_preopened_dir`]. - async fn mount_files( - &self, - store_builder: &mut StoreBuilder, - component: &AppComponent, - ) -> anyhow::Result<()>; -} - -/// An `AppLoader` holds an implementation of [`Loader`] along with -/// [`DynamicHostComponent`]s configuration. -pub struct AppLoader { - inner: Box, - dynamic_host_components: DynamicHostComponents, +/// An `App` holds loaded configuration for a Spin application. +#[derive(Debug)] +pub struct App { + id: String, + locked: LockedApp, } -impl AppLoader { - /// Creates a new [`AppLoader`]. - pub fn new(loader: impl Loader + Send + Sync + 'static) -> Self { +impl App { + /// Returns a new app for the given runtime-specific identifier and locked + /// app. + pub fn new(id: impl Into, locked: LockedApp) -> Self { Self { - inner: Box::new(loader), - dynamic_host_components: Default::default(), - } - } - - /// Adds a [`DynamicHostComponent`] to the given [`EngineBuilder`] and - /// configures this [`AppLoader`] to update it on component instantiation. - /// - /// This calls [`EngineBuilder::add_host_component`] for you; it should not - /// be called separately. - pub fn add_dynamic_host_component( - &mut self, - engine_builder: &mut EngineBuilder, - host_component: DHC, - ) -> anyhow::Result> { - self.dynamic_host_components - .add_dynamic_host_component(engine_builder, host_component) - } - - /// Loads an [`App`] from the given `Loader`-implementation-specific `uri`. - pub async fn load_app(&self, uri: String) -> Result { - let locked = self - .inner - .load_app(&uri) - .await - .map_err(Error::LoaderError)?; - let app = App { - loader: self, - uri, + id: id.into(), locked, - }; - self.dynamic_host_components - .validate_app(&app) - .map_err(Error::ValidationError)?; - Ok(app) - } - - /// Loads an [`OwnedApp`] from the given `Loader`-implementation-specific - /// `uri`; the [`OwnedApp`] takes ownership of this [`AppLoader`]. - pub async fn load_owned_app(self, uri: String) -> Result { - OwnedApp::try_new_async(self, |loader| Box::pin(loader.load_app(uri))).await - } -} - -impl std::fmt::Debug for AppLoader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AppLoader").finish() + } } -} - -#[self_referencing] -#[derive(Debug)] -pub struct OwnedApp { - loader: AppLoader, - - #[borrows(loader)] - #[covariant] - app: App<'this>, -} -impl OwnedApp { - /// Returns a reference to the owned [`App`]. - pub fn borrowed(&self) -> &App { - self.borrow_app() + /// Returns a runtime-specific identifier for this app. + pub fn id(&self) -> &str { + &self.id } -} -/// An `App` holds loaded configuration for a Spin application. -/// -/// Note: The `L` param is an implementation detail to support the -/// [`App::inert`] constructor. -#[derive(Debug)] -pub struct App<'a, L = AppLoader> { - loader: &'a L, - uri: String, - locked: LockedApp, -} - -impl<'a, L> App<'a, L> { /// Deserializes typed metadata for this app. /// /// Returns `Ok(None)` if there is no metadata for the given `key` and an @@ -186,7 +78,7 @@ impl<'a, L> App<'a, L> { } /// Returns an iterator of [`AppComponent`]s defined for this app. - pub fn components(&self) -> impl Iterator> { + pub fn components(&self) -> impl Iterator> { self.locked .components .iter() @@ -195,13 +87,13 @@ impl<'a, L> App<'a, L> { /// Returns the [`AppComponent`] with the given `component_id`, or `None` /// if it doesn't exist. - pub fn get_component(&self, component_id: &str) -> Option> { + pub fn get_component(&self, component_id: &str) -> Option> { self.components() .find(|component| component.locked.id == component_id) } /// Returns an iterator of [`AppTrigger`]s defined for this app. - pub fn triggers(&self) -> impl Iterator> { + pub fn triggers(&self) -> impl Iterator> + '_ { self.locked .triggers .iter() @@ -211,7 +103,7 @@ impl<'a, L> App<'a, L> { /// Returns the trigger metadata for a specific trigger type. pub fn get_trigger_metadata<'this, T: Deserialize<'this> + Default>( &'this self, - trigger_type: &'a str, + trigger_type: &str, ) -> Result> { let Some(value) = self.get_trigger_metadata_value(trigger_type) else { return Ok(None); @@ -240,10 +132,10 @@ impl<'a, L> App<'a, L> { /// Returns an iterator of [`AppTrigger`]s defined for this app with /// the given `trigger_type`. - pub fn triggers_with_type( + pub fn triggers_with_type<'a>( &'a self, trigger_type: &'a str, - ) -> impl Iterator> { + ) -> impl Iterator { self.triggers() .filter(move |trigger| trigger.locked.trigger_type == trigger_type) } @@ -256,36 +148,14 @@ impl<'a, L> App<'a, L> { } } -impl<'a> App<'a> { - /// Returns a [`Loader`]-implementation-specific URI for this app. - pub fn uri(&self) -> &str { - &self.uri - } -} - -#[doc(hidden)] -pub struct InertLoader; - -impl App<'static, InertLoader> { - /// Return an "inert" App which does not have an associated [`AppLoader`] - /// and cannot be used to instantiate components. - pub fn inert(locked: LockedApp) -> Self { - App { - loader: &InertLoader, - uri: "".into(), - locked, - } - } -} - /// An `AppComponent` holds configuration for a Spin application component. -pub struct AppComponent<'a, L = AppLoader> { +pub struct AppComponent<'a> { /// The app this component belongs to. - pub app: &'a App<'a, L>, + pub app: &'a App, locked: &'a LockedComponent, } -impl<'a, L> AppComponent<'a, L> { +impl<'a> AppComponent<'a> { /// Returns this component's app-unique ID. pub fn id(&self) -> &str { &self.locked.id @@ -336,65 +206,14 @@ impl<'a, L> AppComponent<'a, L> { } } -impl<'a> AppComponent<'a> { - /// Loads and returns the [`spin_core::Component`] for this component. - pub async fn load_component( - &self, - engine: &Engine, - ) -> Result { - self.app - .loader - .inner - .load_component(engine.as_ref(), &self.locked.source) - .await - .map_err(Error::LoaderError) - } - - /// Loads and returns the [`spin_core::Module`] for this component. - pub async fn load_module( - &self, - engine: &Engine, - ) -> Result { - self.app - .loader - .inner - .load_module(engine.as_ref(), &self.locked.source) - .await - .map_err(Error::LoaderError) - } - - /// Updates the given [`StoreBuilder`] with configuration for this component. - /// - /// In particular, the WASI 'env' and "preloaded dirs" are set up, and any - /// [`DynamicHostComponent`]s associated with the source [`AppLoader`] are - /// configured. - pub async fn apply_store_config(&self, builder: &mut StoreBuilder) -> Result<()> { - builder.env(&self.locked.env).map_err(Error::CoreError)?; - - let loader = self.app.loader; - loader - .inner - .mount_files(builder, self) - .await - .map_err(Error::LoaderError)?; - - loader - .dynamic_host_components - .update_data(builder.host_components_data(), self) - .map_err(Error::HostComponentError)?; - - Ok(()) - } -} - /// An `AppTrigger` holds configuration for a Spin application trigger. -pub struct AppTrigger<'a, L = AppLoader> { +pub struct AppTrigger<'a> { /// The app this trigger belongs to. - pub app: &'a App<'a, L>, + pub app: &'a App, locked: &'a LockedTrigger, } -impl<'a, L> AppTrigger<'a, L> { +impl<'a> AppTrigger<'a> { /// Returns this trigger's app-unique ID. pub fn id(&self) -> &str { &self.locked.id @@ -414,7 +233,7 @@ impl<'a, L> AppTrigger<'a, L> { /// /// This is a convenience wrapper that looks up the component based on the /// 'component' metadata value which is conventionally a component ID. - pub fn component(&self) -> Result> { + pub fn component(&self) -> Result> { let id = &self.locked.id; let common_config: CommonTriggerConfig = self.typed_config()?; let component_id = common_config.component.ok_or_else(|| { diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index cba62cd50..175888c76 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -10,25 +10,13 @@ async-trait = "0.1" crossbeam-channel = "0.5" tracing = { workspace = true } wasmtime = { workspace = true } -wasmtime-wasi = { workspace = true } -wasmtime-wasi-http = { workspace = true } -wasi-common-preview1 = { workspace = true } -system-interface = { version = "0.27.0", features = ["cap_std_impls"] } -cap-std = "3.0.0" -cap-primitives = "3.0.0" -tokio = "1.0" -bytes = "1.0" -spin-telemetry = { path = "../telemetry" } -http = "1.0" - -[target.'cfg(unix)'.dependencies] -rustix = "0.37.19" - -[target.'cfg(windows)'.dependencies] -io-extras = "0.18.0" [dev-dependencies] -tempfile = "3" -tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } +serde_json = "1" spin-componentize = { workspace = true } -futures = "0.3" +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors = { path = "../factors" } +spin-factors-test = { path = "../factors-test" } +spin-locked-app = { path = "../locked-app" } +wasmtime-wasi = { workspace = true } \ No newline at end of file diff --git a/crates/core/src/host_component.rs b/crates/core/src/host_component.rs deleted file mode 100644 index a2b428a5f..000000000 --- a/crates/core/src/host_component.rs +++ /dev/null @@ -1,305 +0,0 @@ -use std::{ - any::{type_name, Any, TypeId}, - collections::HashMap, - marker::PhantomData, - sync::Arc, -}; - -use anyhow::{bail, Result}; - -use super::{Data, Linker}; - -/// A trait for Spin "host components". -/// -/// A Spin host component is an interface provided to Spin components that is -/// implemented by the host. This trait is designed to be compatible with -/// [`wasmtime::component::bindgen`]'s generated bindings. -/// -/// # Example -/// -/// ```ignore -/// use spin_core::my_interface; -/// -/// #[derive(Default)] -/// struct MyHostComponent { -/// // ... -/// } -/// -/// #[async_trait] -/// impl my_interface::Host for MyHostComponent { -/// // ... -/// } -/// -/// impl HostComponent for MyHostComponent { -/// type Data = Self; -/// -/// fn add_to_linker( -/// linker: &mut Linker, -/// get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, -/// ) -> anyhow::Result<()> { -/// my_interface::add_to_linker(linker, get) -/// } -/// -/// fn build_data(&self) -> Self::Data { -/// Default::default() -/// } -/// } -/// ``` -pub trait HostComponent: Send + Sync + 'static { - /// Host component runtime data. - type Data: Send + Sized + 'static; - - /// Add this component to the given Linker, using the given runtime state-getting handle. - /// - /// This function signature mirrors those generated by [`wasmtime::component::bindgen`]. - fn add_to_linker( - linker: &mut Linker, - get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> Result<()>; - - /// Builds new host component runtime data for [`HostComponentsData`]. - fn build_data(&self) -> Self::Data; -} - -impl HostComponent for Arc { - type Data = HC::Data; - - fn add_to_linker( - linker: &mut Linker, - get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> Result<()> { - HC::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - (**self).build_data() - } -} - -/// An opaque handle which can be passed to [`HostComponentsData`] to access -/// host component data. -#[derive(Clone, Copy)] -pub struct AnyHostComponentDataHandle(usize); - -impl From> for AnyHostComponentDataHandle { - fn from(value: HostComponentDataHandle) -> Self { - value.inner - } -} - -/// An opaque handle returned by [`crate::EngineBuilder::add_host_component`] -/// which can be passed to [`HostComponentsData`] to access or set associated -/// [`HostComponent::Data`]. -pub struct HostComponentDataHandle { - inner: AnyHostComponentDataHandle, - _phantom: PhantomData HC::Data>, -} - -impl HostComponentDataHandle { - fn from_any(handle: AnyHostComponentDataHandle) -> Self { - Self { - inner: handle, - _phantom: PhantomData, - } - } -} - -impl Clone for HostComponentDataHandle { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for HostComponentDataHandle {} - -impl From>> for HostComponentDataHandle { - fn from(value: HostComponentDataHandle>) -> Self { - Self::from_any(value.inner) - } -} - -#[doc(hidden)] -pub trait DynSafeHostComponent { - fn build_data_box(&self) -> AnyData; -} - -impl DynSafeHostComponent for T -where - T::Data: Any + Send, -{ - fn build_data_box(&self) -> AnyData { - Box::new(self.build_data()) - } -} - -type BoxHostComponent = Box; - -#[derive(Default)] -pub struct HostComponentsBuilder { - handles: HashMap, - host_components: Vec, -} - -impl HostComponentsBuilder { - pub fn add_host_component( - &mut self, - linker: &mut Linker, - host_component: HC, - ) -> Result> { - let type_id = TypeId::of::(); - if self.handles.contains_key(&type_id) { - bail!( - "already have a host component of type {}", - type_name::() - ) - } - - let handle = AnyHostComponentDataHandle(self.host_components.len()); - self.handles.insert(type_id, handle); - - self.host_components.push(Box::new(host_component)); - HC::add_to_linker(linker, move |data| { - data.host_components_data - .get_or_insert_any(handle) - .downcast_mut() - .unwrap() - })?; - Ok(HostComponentDataHandle:: { - inner: handle, - _phantom: PhantomData, - }) - } - - pub fn build(self) -> HostComponents { - HostComponents { - handles: self.handles, - host_components: Arc::new(self.host_components), - } - } -} - -pub struct HostComponents { - handles: HashMap, - host_components: Arc>, -} - -impl HostComponents { - pub fn builder() -> HostComponentsBuilder { - Default::default() - } - - pub fn new_data(&self) -> HostComponentsData { - // Fill with `None` - let data = std::iter::repeat_with(Default::default) - .take(self.host_components.len()) - .collect(); - HostComponentsData { - data, - host_components: self.host_components.clone(), - } - } - - pub fn find_handle(&self) -> Option> { - self.handles - .get(&TypeId::of::()) - .map(|handle| HostComponentDataHandle::from_any(*handle)) - } -} - -type AnyData = Box; - -/// Holds a heterogenous set of [`HostComponent::Data`]s. -pub struct HostComponentsData { - data: Vec>, - host_components: Arc>, -} - -impl HostComponentsData { - /// Sets the [`HostComponent::Data`] for the given `handle`. - pub fn set(&mut self, handle: HostComponentDataHandle, data: HC::Data) { - self.data[handle.inner.0] = Some(Box::new(data)); - } - - /// Retrieves a mutable reference to [`HostComponent::Data`] for the given `handle`. - /// - /// If unset, the data will be initialized with [`HostComponent::build_data`]. - /// - /// # Panics - /// - /// If the given handle was not obtained from the same [`HostComponentsBuilder`] that - /// was the source of this [`HostComponentsData`], this function may panic. - pub fn get_or_insert( - &mut self, - handle: HostComponentDataHandle, - ) -> &mut HC::Data { - let data = self.get_or_insert_any(handle.inner); - data.downcast_mut().unwrap() - } - - /// Retrieves a mutable reference to [`HostComponent::Data`] for the given `handle`. - /// - /// If unset, the data will be initialized with [`HostComponent::build_data`]. - /// - /// # Panics - /// - /// If the given handle was not obtained from the same [`HostComponentsBuilder`] that - /// was the source of this [`HostComponentsData`], this function may panic. - pub fn get_or_insert_any(&mut self, handle: AnyHostComponentDataHandle) -> &mut AnyData { - let idx = handle.0; - self.data[idx].get_or_insert_with(|| self.host_components[idx].build_data_box()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - struct TestHC; - - impl HostComponent for TestHC { - type Data = u8; - - fn add_to_linker( - _linker: &mut Linker, - _get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> Result<()> { - Ok(()) - } - - fn build_data(&self) -> Self::Data { - 0 - } - } - - #[test] - fn host_components_data() { - let engine = wasmtime::Engine::default(); - let mut linker: crate::Linker<()> = crate::Linker::new(&engine); - - let mut builder = HostComponents::builder(); - let handle1 = builder - .add_host_component(&mut linker, Arc::new(TestHC)) - .unwrap(); - let handle2 = builder.add_host_component(&mut linker, TestHC).unwrap(); - let host_components = builder.build(); - let mut hc_data = host_components.new_data(); - - assert_eq!(hc_data.get_or_insert(handle1), &0); - - hc_data.set(handle2, 1); - assert_eq!(hc_data.get_or_insert(handle2), &1); - } - - #[test] - fn find_handle() { - let engine = wasmtime::Engine::default(); - let mut linker: crate::Linker<()> = crate::Linker::new(&engine); - - let mut builder = HostComponents::builder(); - builder.add_host_component(&mut linker, TestHC).unwrap(); - let host_components = builder.build(); - let handle = host_components.find_handle::().unwrap(); - let mut hc_data = host_components.new_data(); - assert_eq!(hc_data.get_or_insert(handle), &0); - } -} diff --git a/crates/core/src/io.rs b/crates/core/src/io.rs deleted file mode 100644 index a8bb7bd3d..000000000 --- a/crates/core/src/io.rs +++ /dev/null @@ -1,34 +0,0 @@ -use wasmtime_wasi::{pipe::MemoryOutputPipe, HostOutputStream}; - -/// An in-memory stdio output buffer. -#[derive(Clone)] -pub struct OutputBuffer(MemoryOutputPipe); - -impl OutputBuffer { - /// Clones the buffered output from this buffer. - pub fn contents(&self) -> bytes::Bytes { - self.0.contents() - } - - pub(crate) fn writer(&self) -> impl HostOutputStream { - self.0.clone() - } -} - -impl Default for OutputBuffer { - fn default() -> Self { - Self(MemoryOutputPipe::new(usize::MAX)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn take_what_you_write() { - let buf = OutputBuffer::default(); - buf.writer().write(b"foo".to_vec().into()).unwrap(); - assert_eq!(buf.contents().as_ref(), b"foo"); - } -} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index b9dc8fbb9..cd02404ba 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,33 +1,23 @@ //! Spin core execution engine //! -//! This crate provides low-level Wasm and WASI functionality required by Spin. -//! Most of this functionality consists of wrappers around [`wasmtime`] and -//! [`wasi_common`] that narrows the flexibility of `wasmtime` to the set of -//! features used by Spin (such as only supporting `wasmtime`'s async calling style). +//! This crate provides low-level Wasm functionality required by Spin. Most of +//! this functionality consists of wrappers around [`wasmtime`] that narrow the +//! flexibility of `wasmtime` to the set of features used by Spin (such as only +//! supporting `wasmtime`'s async calling style). #![deny(missing_docs)] -mod host_component; -mod io; mod limits; -mod preview1; mod store; -pub mod wasi_2023_10_18; -pub mod wasi_2023_11_10; use std::sync::OnceLock; use std::{path::PathBuf, time::Duration}; use anyhow::Result; use crossbeam_channel::Sender; -use http::Request; -use tracing::{field::Empty, instrument}; +use tracing::instrument; +use wasmtime::component::{InstancePre, Linker}; use wasmtime::{InstanceAllocationStrategy, PoolingAllocationConfig}; -use wasmtime_wasi::ResourceTable; -use wasmtime_wasi_http::body::HyperOutgoingBody; -use wasmtime_wasi_http::types::{default_send_request, WasiHttpCtx, WasiHttpView}; - -use self::host_component::{HostComponents, HostComponentsBuilder}; pub use async_trait::async_trait; pub use wasmtime::{ @@ -35,13 +25,8 @@ pub use wasmtime::{ component::{Component, Instance}, Instance as ModuleInstance, Module, Trap, }; -pub use wasmtime_wasi::I32Exit; -pub use host_component::{ - AnyHostComponentDataHandle, HostComponent, HostComponentDataHandle, HostComponentsData, -}; -pub use io::OutputBuffer; -pub use store::{Store, StoreBuilder, Wasi, WasiVersion}; +pub use store::{Store, StoreBuilder}; /// The default [`EngineBuilder::epoch_tick_interval`]. pub const DEFAULT_EPOCH_TICK_INTERVAL: Duration = Duration::from_millis(10); @@ -205,183 +190,43 @@ fn use_pooling_allocator_by_default() -> bool { } /// Host state data associated with individual [Store]s and [Instance]s. -pub struct Data { - inner: T, - wasi: Wasi, - host_components_data: HostComponentsData, +#[derive(Default)] +pub struct State { store_limits: limits::StoreLimitsAsync, - table: ResourceTable, } -impl Data { +impl State { /// Get the amount of memory in bytes consumed by instances in the store pub fn memory_consumed(&self) -> u64 { self.store_limits.memory_consumed() } } -impl AsRef for Data { - fn as_ref(&self) -> &T { - &self.inner - } -} - -impl AsMut for Data { - fn as_mut(&mut self) -> &mut T { - &mut self.inner - } -} - -impl wasmtime_wasi::WasiView for Data { - fn table(&mut self) -> &mut ResourceTable { - &mut self.table - } - - fn ctx(&mut self) -> &mut wasmtime_wasi::WasiCtx { - match &mut self.wasi { - Wasi::Preview1(_) => panic!("using WASI Preview 1 functions with Preview 2 store"), - Wasi::Preview2 { wasi_ctx, .. } => wasi_ctx, - } - } -} - -impl WasiHttpView for Data { - fn ctx(&mut self) -> &mut WasiHttpCtx { - match &mut self.wasi { - Wasi::Preview1(_) => panic!("using WASI Preview 1 functions with Preview 2 store"), - Wasi::Preview2 { wasi_http_ctx, .. } => wasi_http_ctx, - } - } - - fn table(&mut self) -> &mut ResourceTable { - &mut self.table - } - - #[instrument( - name = "spin_core.send_request", - skip_all, - fields( - otel.kind = "client", - url.full = %request.uri(), - http.request.method = %request.method(), - otel.name = %request.method(), - http.response.status_code = Empty, - server.address = Empty, - server.port = Empty, - ), - )] - fn send_request( - &mut self, - mut request: Request, - config: wasmtime_wasi_http::types::OutgoingRequestConfig, - ) -> wasmtime_wasi_http::HttpResult - where - Self: Sized, - { - spin_telemetry::inject_trace_context(&mut request); - T::send_request(self, request, config) - } -} - -/// Handler for wasi-http based requests -pub trait OutboundWasiHttpHandler { - /// Send the request - fn send_request( - data: &mut Data, - request: Request, - config: wasmtime_wasi_http::types::OutgoingRequestConfig, - ) -> wasmtime_wasi_http::HttpResult - where - Self: Sized; -} - -impl OutboundWasiHttpHandler for () { - fn send_request( - _data: &mut Data, - request: Request, - config: wasmtime_wasi_http::types::OutgoingRequestConfig, - ) -> wasmtime_wasi_http::HttpResult - where - Self: Sized, - { - Ok(default_send_request(request, config)) - } -} - -/// An alias for [`wasmtime::Linker`] specialized to [`Data`]. -pub type ModuleLinker = wasmtime::Linker>; - -/// An alias for [`wasmtime::component::Linker`] specialized to [`Data`]. -pub type Linker = wasmtime::component::Linker>; - /// A builder interface for configuring a new [`Engine`]. /// /// A new [`EngineBuilder`] can be obtained with [`Engine::builder`]. pub struct EngineBuilder { engine: wasmtime::Engine, linker: Linker, - module_linker: ModuleLinker, - host_components_builder: HostComponentsBuilder, epoch_tick_interval: Duration, epoch_ticker_thread: bool, } -impl EngineBuilder { +impl EngineBuilder { fn new(config: &Config) -> Result { let engine = wasmtime::Engine::new(&config.inner)?; let linker: Linker = Linker::new(&engine); - let mut module_linker = ModuleLinker::new(&engine); - - wasi_common_preview1::tokio::add_to_linker(&mut module_linker, |data| { - match &mut data.wasi { - Wasi::Preview1(ctx) => ctx, - Wasi::Preview2 { .. } => { - panic!("using WASI Preview 2 functions with Preview 1 store") - } - } - })?; - Ok(Self { engine, linker, - module_linker, - host_components_builder: HostComponents::builder(), epoch_tick_interval: DEFAULT_EPOCH_TICK_INTERVAL, epoch_ticker_thread: true, }) } -} - -impl EngineBuilder { - /// Adds definition(s) to the built [`Engine`]. - /// - /// This method's signature is meant to be used with - /// [`wasmtime::component::bindgen`]'s generated `add_to_linker` functions, e.g.: - /// - /// ```ignore - /// use spin_core::my_interface; - /// // ... - /// let mut builder: EngineBuilder = Engine::builder(); - /// builder.link_import(my_interface::add_to_linker)?; - /// ``` - pub fn link_import( - &mut self, - f: impl FnOnce(&mut Linker, fn(&mut Data) -> &mut T) -> Result<()>, - ) -> Result<()> { - f(&mut self.linker, Data::as_mut) - } - /// Adds a [`HostComponent`] to the built [`Engine`]. - /// - /// Returns a [`HostComponentDataHandle`] which can be passed to - /// [`HostComponentsData`] to access or set associated - /// [`HostComponent::Data`] for an instance. - pub fn add_host_component( - &mut self, - host_component: HC, - ) -> Result> { - self.host_components_builder - .add_host_component(&mut self.linker, host_component) + /// Returns a reference to the [`Linker`] for this [`Engine`]. + pub fn linker(&mut self) -> &mut Linker { + &mut self.linker } /// Sets the epoch tick internal for the built [`Engine`]. @@ -427,14 +272,9 @@ impl EngineBuilder { /// Builds an [`Engine`] from this builder. pub fn build(self) -> Engine { let epoch_ticker_signal = self.maybe_spawn_epoch_ticker(); - - let host_components = self.host_components_builder.build(); - Engine { inner: self.engine, linker: self.linker, - module_linker: self.module_linker, - host_components, epoch_tick_interval: self.epoch_tick_interval, _epoch_ticker_signal: epoch_ticker_signal, } @@ -446,50 +286,26 @@ impl EngineBuilder { pub struct Engine { inner: wasmtime::Engine, linker: Linker, - module_linker: ModuleLinker, - host_components: HostComponents, epoch_tick_interval: Duration, // Matching receiver closes on drop _epoch_ticker_signal: Option>, } -impl Engine { +impl Engine { /// Creates a new [`EngineBuilder`] with the given [`Config`]. pub fn builder(config: &Config) -> Result> { EngineBuilder::new(config) } /// Creates a new [`StoreBuilder`]. - pub fn store_builder(&self, wasi_version: WasiVersion) -> StoreBuilder { - StoreBuilder::new( - self.inner.clone(), - self.epoch_tick_interval, - &self.host_components, - wasi_version, - ) + pub fn store_builder(&self) -> StoreBuilder { + StoreBuilder::new(self.inner.clone(), self.epoch_tick_interval) } /// Creates a new [`InstancePre`] for the given [`Component`]. #[instrument(skip_all, level = "debug")] pub fn instantiate_pre(&self, component: &Component) -> Result> { - let inner = self.linker.instantiate_pre(component)?; - Ok(InstancePre { inner }) - } - - /// Creates a new [`ModuleInstancePre`] for the given [`Module`]. - #[instrument(skip_all, level = "debug")] - pub fn module_instantiate_pre(&self, module: &Module) -> Result> { - let inner = self.module_linker.instantiate_pre(module)?; - Ok(ModuleInstancePre { inner }) - } - - /// Find the [`HostComponentDataHandle`] for a [`HostComponent`] if configured for this engine. - /// Note: [`DynamicHostComponent`]s are implicitly wrapped in `Arc`s and need to be explicitly - /// typed as such here, e.g. `find_host_component_handle::>()`. - pub fn find_host_component_handle( - &self, - ) -> Option> { - self.host_components.find_handle() + self.linker.instantiate_pre(component) } } @@ -498,61 +314,3 @@ impl AsRef for Engine { &self.inner } } - -/// A pre-initialized instance that is ready to be instantiated. -/// -/// See [`wasmtime::component::InstancePre`] for more information. -pub struct InstancePre { - inner: wasmtime::component::InstancePre>, -} - -impl InstancePre { - /// Instantiates this instance with the given [`Store`]. - #[instrument(skip_all, level = "debug")] - pub async fn instantiate_async(&self, store: &mut Store) -> Result { - self.inner.instantiate_async(store).await - } -} - -impl Clone for InstancePre { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } -} - -impl AsRef>> for InstancePre { - fn as_ref(&self) -> &wasmtime::component::InstancePre> { - &self.inner - } -} - -/// A pre-initialized module instance that is ready to be instantiated. -/// -/// See [`wasmtime::InstancePre`] for more information. -pub struct ModuleInstancePre { - inner: wasmtime::InstancePre>, -} - -impl ModuleInstancePre { - /// Instantiates this instance with the given [`Store`]. - #[instrument(skip_all, level = "debug")] - pub async fn instantiate_async(&self, store: &mut Store) -> Result { - self.inner.instantiate_async(store).await - } -} - -impl Clone for ModuleInstancePre { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } -} - -impl AsRef>> for ModuleInstancePre { - fn as_ref(&self) -> &wasmtime::InstancePre> { - &self.inner - } -} diff --git a/crates/core/src/preview1.rs b/crates/core/src/preview1.rs deleted file mode 100644 index 3b1cd967c..000000000 --- a/crates/core/src/preview1.rs +++ /dev/null @@ -1,212 +0,0 @@ -//! Ports of `ReadOnlyDir` and `ReadOnlyFile` to Preview 1 API. -//! Adapted from https://github.com/bytecodealliance/preview2-prototyping/pull/121 - -use std::{any::Any, path::PathBuf}; - -use wasi_common_preview1::{ - dir::{OpenResult, ReaddirCursor, ReaddirEntity}, - file::{Advice, FdFlags, FileType, Filestat, OFlags}, - Error, ErrorExt, SystemTimeSpec, WasiDir, WasiFile, -}; - -pub struct ReadOnlyDir(pub Box); - -#[async_trait::async_trait] -impl WasiDir for ReadOnlyDir { - fn as_any(&self) -> &dyn Any { - self - } - - async fn open_file( - &self, - symlink_follow: bool, - path: &str, - oflags: OFlags, - read: bool, - write: bool, - fdflags: FdFlags, - ) -> Result { - if write { - Err(Error::perm()) - } else { - let open_result = self - .0 - .open_file(symlink_follow, path, oflags, read, write, fdflags) - .await?; - Ok(match open_result { - OpenResult::File(f) => OpenResult::File(Box::new(ReadOnlyFile(f))), - OpenResult::Dir(d) => OpenResult::Dir(Box::new(ReadOnlyDir(d))), - }) - } - } - - async fn create_dir(&self, _path: &str) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn readdir( - &self, - cursor: ReaddirCursor, - ) -> Result> + Send>, Error> { - self.0.readdir(cursor).await - } - - async fn symlink(&self, _old_path: &str, _new_path: &str) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn remove_dir(&self, _path: &str) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn unlink_file(&self, _path: &str) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn read_link(&self, path: &str) -> Result { - self.0.read_link(path).await - } - - async fn get_filestat(&self) -> Result { - self.0.get_filestat().await - } - - async fn get_path_filestat( - &self, - path: &str, - follow_symlinks: bool, - ) -> Result { - self.0.get_path_filestat(path, follow_symlinks).await - } - - async fn rename( - &self, - _path: &str, - _dest_dir: &dyn WasiDir, - _dest_path: &str, - ) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn hard_link( - &self, - _path: &str, - _target_dir: &dyn WasiDir, - _target_path: &str, - ) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn set_times( - &self, - _path: &str, - _atime: Option, - _mtime: Option, - _follow_symlinks: bool, - ) -> Result<(), Error> { - Err(Error::perm()) - } -} - -pub struct ReadOnlyFile(pub Box); - -#[async_trait::async_trait] -impl WasiFile for ReadOnlyFile { - fn as_any(&self) -> &dyn Any { - self - } - - async fn get_filetype(&self) -> Result { - self.0.get_filetype().await - } - - #[cfg(unix)] - fn pollable(&self) -> Option { - self.0.pollable() - } - - #[cfg(windows)] - fn pollable(&self) -> Option { - self.0.pollable() - } - - fn isatty(&self) -> bool { - self.0.isatty() - } - - async fn datasync(&self) -> Result<(), Error> { - self.0.datasync().await - } - - async fn sync(&self) -> Result<(), Error> { - self.0.sync().await - } - - async fn get_fdflags(&self) -> Result { - self.0.get_fdflags().await - } - - async fn set_fdflags(&mut self, _flags: FdFlags) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn get_filestat(&self) -> Result { - self.0.get_filestat().await - } - - async fn set_filestat_size(&self, _size: u64) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn advise(&self, offset: u64, len: u64, advice: Advice) -> Result<(), Error> { - self.0.advise(offset, len, advice).await - } - - async fn set_times( - &self, - _atime: Option, - _mtime: Option, - ) -> Result<(), Error> { - Err(Error::perm()) - } - - async fn read_vectored<'a>(&self, bufs: &mut [std::io::IoSliceMut<'a>]) -> Result { - self.0.read_vectored(bufs).await - } - - async fn read_vectored_at<'a>( - &self, - bufs: &mut [std::io::IoSliceMut<'a>], - offset: u64, - ) -> Result { - self.0.read_vectored_at(bufs, offset).await - } - - async fn write_vectored_at<'a>( - &self, - _bufs: &[std::io::IoSlice<'a>], - _offset: u64, - ) -> Result { - Err(Error::perm()) - } - - async fn seek(&self, pos: std::io::SeekFrom) -> Result { - self.0.seek(pos).await - } - - async fn peek(&self, buf: &mut [u8]) -> Result { - self.0.peek(buf).await - } - - fn num_ready_bytes(&self) -> Result { - self.0.num_ready_bytes() - } - - async fn readable(&self) -> Result<(), Error> { - self.0.readable().await - } - - async fn writable(&self) -> Result<(), Error> { - Err(Error::perm()) - } -} diff --git a/crates/core/src/store.rs b/crates/core/src/store.rs index eeb36006b..b2b950b5f 100644 --- a/crates/core/src/store.rs +++ b/crates/core/src/store.rs @@ -1,70 +1,11 @@ -use anyhow::{anyhow, Result}; -use bytes::Bytes; -use cap_primitives::net::Pool; -use cap_std::ipnet::{IpNet, Ipv4Net, Ipv6Net}; -use std::{ - io::{Read, Write}, - mem, - net::{Ipv4Addr, Ipv6Addr}, - path::{Path, PathBuf}, - sync::{Arc, Mutex}, - time::{Duration, Instant}, -}; -use system_interface::io::ReadReady; -use tokio::io::{AsyncRead, AsyncWrite}; -use wasi_common_preview1 as wasi_preview1; -use wasmtime_wasi::{ - self as wasi_preview2, HostInputStream, HostOutputStream, StdinStream, StdoutStream, - StreamError, StreamResult, Subscribe, -}; -use wasmtime_wasi_http::types::WasiHttpCtx; +use anyhow::Result; +use std::time::{Duration, Instant}; -use crate::{ - async_trait, - host_component::{HostComponents, HostComponentsData}, - io::OutputBuffer, - limits::StoreLimitsAsync, - preview1, Data, -}; +use crate::{limits::StoreLimitsAsync, State}; #[cfg(doc)] use crate::EngineBuilder; -/// Wrapper for the Preview 1 and Preview 2 versions of `WasiCtx`. -/// -/// Currently, only WAGI uses Preview 1, while everything else uses Preview 2 (possibly via an adapter). WAGI is -/// stuck on Preview 1 and modules because there's no reliable way to wrap an arbitrary Preview 1 command in a -/// component -- the Preview 1 -> 2 adapter only works with modules that either export `canonical_abi_realloc` -/// (e.g. native Spin apps) or use a recent version of `wasi-sdk`, which contains patches to allow the adapter to -/// safely allocate memory via `memory.grow`. -/// -/// In theory, someone could build a WAGI app using a new-enough version of `wasi-sdk` and wrap it in a component -/// using the adapter, but that wouldn't add any value beyond leaving it as a module, and any toolchain capable of -/// natively producing components will be capable enough to produce native Spin apps, so we probably won't ever -/// support WAGI components. -/// -// TODO: As of this writing, the plan is to merge the WASI Preview 1 and Preview 2 implementations together, at -// which point we'll be able to avoid all the duplication here and below. -pub enum Wasi { - /// Preview 1 `WasiCtx` - Preview1(wasi_preview1::WasiCtx), - /// Preview 2 `WasiCtx` - Preview2 { - /// `wasi-cli` context - wasi_ctx: wasi_preview2::WasiCtx, - - /// `wasi-http` context - wasi_http_ctx: WasiHttpCtx, - }, -} - -/// The version of Wasi being used -#[allow(missing_docs)] -pub enum WasiVersion { - Preview1, - Preview2, -} - /// A `Store` holds the runtime state of a Spin instance. /// /// In general, a `Store` is expected to live only for the lifetime of a single @@ -72,16 +13,11 @@ pub enum WasiVersion { /// /// A `Store` can be built with a [`StoreBuilder`]. pub struct Store { - inner: wasmtime::Store>, + inner: wasmtime::Store, epoch_tick_interval: Duration, } impl Store { - /// Returns a mutable reference to the [`HostComponentsData`] of this [`Store`]. - pub fn host_components_data(&mut self) -> &mut HostComponentsData { - &mut self.inner.data_mut().host_components_data - } - /// Sets the execution deadline. /// /// This is a rough deadline; an instance will trap some time after this @@ -104,20 +40,20 @@ impl Store { } } -impl AsRef>> for Store { - fn as_ref(&self) -> &wasmtime::Store> { +impl AsRef> for Store { + fn as_ref(&self) -> &wasmtime::Store { &self.inner } } -impl AsMut>> for Store { - fn as_mut(&mut self) -> &mut wasmtime::Store> { +impl AsMut> for Store { + fn as_mut(&mut self) -> &mut wasmtime::Store { &mut self.inner } } impl wasmtime::AsContext for Store { - type Data = Data; + type Data = T; fn as_context(&self) -> wasmtime::StoreContext<'_, Self::Data> { self.inner.as_context() @@ -136,27 +72,16 @@ impl wasmtime::AsContextMut for Store { pub struct StoreBuilder { engine: wasmtime::Engine, epoch_tick_interval: Duration, - wasi: std::result::Result, - host_components_data: HostComponentsData, store_limits: StoreLimitsAsync, - net_pool: Pool, } impl StoreBuilder { // Called by Engine::store_builder. - pub(crate) fn new( - engine: wasmtime::Engine, - epoch_tick_interval: Duration, - host_components: &HostComponents, - wasi: WasiVersion, - ) -> Self { + pub(crate) fn new(engine: wasmtime::Engine, epoch_tick_interval: Duration) -> Self { Self { engine, epoch_tick_interval, - wasi: Ok(wasi.into()), - host_components_data: host_components.new_data(), store_limits: StoreLimitsAsync::default(), - net_pool: Pool::default(), } } @@ -168,281 +93,15 @@ impl StoreBuilder { self.store_limits = StoreLimitsAsync::new(Some(max_memory_size), None); } - /// Inherit stdin from the host process. - pub fn inherit_stdin(&mut self) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(ctx) => { - ctx.set_stdin(Box::new(wasi_common_preview1::tokio::stdio::stdin())) - } - WasiCtxBuilder::Preview2(ctx) => { - ctx.inherit_stdin(); - } - }); - } - - /// Insert IP network with a given port range - pub fn insert_ip_net_port_range( - &mut self, - ip_net: IpNet, - ports_start: u16, - ports_end: Option, - ) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(_) => { - panic!("Enabling network only allowed in preview2") - } - WasiCtxBuilder::Preview2(_) => {} - }); - - self.net_pool.insert_ip_net_port_range( - ip_net, - ports_start, - ports_end, - cap_primitives::ambient_authority(), - ); - } - - /// Allow unrestricted outbound access to the host network. - pub fn inherit_network(&mut self) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(_) => { - panic!("Enabling network only allowed in preview2") - } - WasiCtxBuilder::Preview2(_) => { - // TODO: ctx.allow_udp(false); - } - }); - - // Allow access to 0.0.0.0/0, i.e. all IPv4 addresses - self.net_pool.insert_ip_net_port_any( - IpNet::V4(Ipv4Net::new(Ipv4Addr::new(0, 0, 0, 0), 0).unwrap()), - cap_primitives::ambient_authority(), - ); - // Allow access to 0:/0, i.e. all IPv6 addresses - self.net_pool.insert_ip_net_port_any( - IpNet::V6(Ipv6Net::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0), 0).unwrap()), - cap_primitives::ambient_authority(), - ); - } - - /// Sets the WASI `stdin` descriptor to the given [`Read`]er. - pub fn stdin_pipe( - &mut self, - r: impl AsyncRead + Read + ReadReady + Send + Sync + Unpin + 'static, - ) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(ctx) => { - ctx.set_stdin(Box::new(wasi_preview1::pipe::ReadPipe::new(r))) - } - WasiCtxBuilder::Preview2(ctx) => { - ctx.stdin(PipeStdinStream::new(r)); - } - }) - } - - /// Inherit stdin from the host process. - pub fn inherit_stdout(&mut self) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(ctx) => { - ctx.set_stdout(Box::new(wasi_common_preview1::tokio::stdio::stdout())) - } - WasiCtxBuilder::Preview2(ctx) => { - ctx.inherit_stdout(); - } - }); - } - - /// Sets the WASI `stdout` descriptor to the given [`Write`]er. - pub fn stdout(&mut self, w: Box) -> Result<()> { - self.try_with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(ctx) => { - ctx.set_stdout(w); - Ok(()) - } - WasiCtxBuilder::Preview2(_) => Err(anyhow!( - "`Store::stdout` only supported with WASI Preview 1" - )), - }) - } - - /// Sets the WASI `stdout` descriptor to the given [`Write`]er. - pub fn stdout_pipe(&mut self, w: impl AsyncWrite + Write + Send + Sync + Unpin + 'static) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(ctx) => { - ctx.set_stdout(Box::new(wasi_preview1::pipe::WritePipe::new(w))) - } - WasiCtxBuilder::Preview2(ctx) => { - ctx.stdout(PipeStdoutStream::new(w)); - } - }) - } - - /// Sets the WASI `stdout` descriptor to an in-memory buffer which can be - /// retrieved after execution from the returned [`OutputBuffer`]. - pub fn stdout_buffered(&mut self) -> Result { - let buffer = OutputBuffer::default(); - // This only needs to work with Preview 2 since WAGI does its own thing with Preview 1: - self.try_with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(_) => Err(anyhow!( - "`Store::stdout_buffered` only supported with WASI Preview 2" - )), - WasiCtxBuilder::Preview2(ctx) => { - ctx.stdout(BufferStdoutStream(buffer.clone())); - Ok(()) - } - })?; - Ok(buffer) - } - - /// Inherit stdin from the host process. - pub fn inherit_stderr(&mut self) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(ctx) => { - ctx.set_stderr(Box::new(wasi_common_preview1::tokio::stdio::stderr())) - } - WasiCtxBuilder::Preview2(ctx) => { - ctx.inherit_stderr(); - } - }); - } - - /// Sets the WASI `stderr` descriptor to the given [`Write`]er. - pub fn stderr_pipe(&mut self, w: impl AsyncWrite + Write + Send + Sync + Unpin + 'static) { - self.with_wasi(|wasi| match wasi { - WasiCtxBuilder::Preview1(ctx) => { - ctx.set_stderr(Box::new(wasi_preview1::pipe::WritePipe::new(w))) - } - WasiCtxBuilder::Preview2(ctx) => { - ctx.stderr(PipeStdoutStream::new(w)); - } - }) - } - - /// Appends the given strings to the the WASI 'args'. - pub fn args<'b>(&mut self, args: impl IntoIterator) -> Result<()> { - self.try_with_wasi(|wasi| { - for arg in args { - match wasi { - WasiCtxBuilder::Preview1(ctx) => ctx.push_arg(arg)?, - WasiCtxBuilder::Preview2(ctx) => { - ctx.arg(arg); - } - } - } - Ok(()) - }) - } - - /// Sets the given key/value string entries on the the WASI 'env'. - pub fn env( - &mut self, - vars: impl IntoIterator, impl AsRef)>, - ) -> Result<()> { - self.try_with_wasi(|wasi| { - for (k, v) in vars { - match wasi { - WasiCtxBuilder::Preview1(ctx) => ctx.push_env(k.as_ref(), v.as_ref())?, - WasiCtxBuilder::Preview2(ctx) => { - ctx.env(k, v); - } - } - } - - Ok(()) - }) - } - - /// "Mounts" the given `host_path` into the WASI filesystem at the given - /// `guest_path` with read-only capabilities. - pub fn read_only_preopened_dir( - &mut self, - host_path: impl AsRef, - guest_path: PathBuf, - ) -> Result<()> { - self.preopened_dir_impl(host_path, guest_path, false) - } - - /// "Mounts" the given `host_path` into the WASI filesystem at the given - /// `guest_path` with read and write capabilities. - pub fn read_write_preopened_dir( - &mut self, - host_path: impl AsRef, - guest_path: PathBuf, - ) -> Result<()> { - self.preopened_dir_impl(host_path, guest_path, true) - } - - fn preopened_dir_impl( - &mut self, - host_path: impl AsRef, - guest_path: PathBuf, - writable: bool, - ) -> Result<()> { - let cap_std_dir = - cap_std::fs::Dir::open_ambient_dir(host_path.as_ref(), cap_std::ambient_authority())?; - let path = guest_path - .to_str() - .ok_or_else(|| anyhow!("non-utf8 path: {}", guest_path.display()))?; - - self.try_with_wasi(|wasi| { - match wasi { - WasiCtxBuilder::Preview1(ctx) => { - let mut dir = - Box::new(wasi_common_preview1::tokio::Dir::from_cap_std(cap_std_dir)) as _; - if !writable { - dir = Box::new(preview1::ReadOnlyDir(dir)); - } - ctx.push_preopened_dir(dir, path)?; - } - WasiCtxBuilder::Preview2(ctx) => { - let dir_perms = if writable { - wasi_preview2::DirPerms::all() - } else { - wasi_preview2::DirPerms::READ - }; - let file_perms = wasi_preview2::FilePerms::all(); - - ctx.preopened_dir(host_path.as_ref(), path, dir_perms, file_perms)?; - } - } - Ok(()) - }) - } - - /// Returns a mutable reference to the built - pub fn host_components_data(&mut self) -> &mut HostComponentsData { - &mut self.host_components_data - } - /// Builds a [`Store`] from this builder with given host state data. /// - /// If `T: Default`, it may be preferable to use [`Store::build`]. - pub fn build_with_data(mut self, inner_data: T) -> Result> { - let net_pool = mem::take(&mut self.net_pool); - self.with_wasi(move |wasi| match wasi { - WasiCtxBuilder::Preview1(_) => {} - WasiCtxBuilder::Preview2(ctx) => { - ctx.socket_addr_check(move |addr, _| { - let net_pool = net_pool.clone(); - Box::pin(async move { net_pool.check_addr(&addr).is_ok() }) - }); - } - }); - - let wasi = self.wasi.map_err(anyhow::Error::msg)?.build(); - - let mut inner = wasmtime::Store::new( - &self.engine, - Data { - inner: inner_data, - wasi, - host_components_data: self.host_components_data, - store_limits: self.store_limits, - table: wasi_preview2::ResourceTable::new(), - }, - ); + /// The `T` parameter must provide access to a [`State`] via `impl + /// AsMut`. + pub fn build>(self, mut data: T) -> Result> { + data.as_mut().store_limits = self.store_limits; - inner.limiter_async(move |data| &mut data.store_limits); + let mut inner = wasmtime::Store::new(&self.engine, data); + inner.limiter_async(|data| &mut data.as_mut().store_limits); // With epoch interruption enabled, there must be _some_ deadline set // or execution will trap immediately. Since this is a delta, we need @@ -455,177 +114,4 @@ impl StoreBuilder { epoch_tick_interval: self.epoch_tick_interval, }) } - - /// Builds a [`Store`] from this builder with `Default` host state data. - pub fn build(self) -> Result> { - self.build_with_data(T::default()) - } - - fn with_wasi(&mut self, f: impl FnOnce(&mut WasiCtxBuilder)) { - let _ = self.try_with_wasi(|wasi| { - f(wasi); - Ok(()) - }); - } - - fn try_with_wasi(&mut self, f: impl FnOnce(&mut WasiCtxBuilder) -> Result<()>) -> Result<()> { - let wasi = self - .wasi - .as_mut() - .map_err(|err| anyhow!("StoreBuilder already failed: {}", err))?; - - match f(wasi) { - Ok(()) => Ok(()), - Err(err) => { - self.wasi = Err(err.to_string()); - Err(err) - } - } - } -} - -struct PipeStdinStream { - buffer: Vec, - inner: Arc>, -} - -impl PipeStdinStream { - fn new(inner: T) -> Self { - Self { - buffer: vec![0_u8; 64 * 1024], - inner: Arc::new(Mutex::new(inner)), - } - } -} - -impl Clone for PipeStdinStream { - fn clone(&self) -> Self { - Self { - buffer: vec![0_u8; 64 * 1024], - inner: self.inner.clone(), - } - } -} - -impl HostInputStream for PipeStdinStream { - fn read(&mut self, size: usize) -> StreamResult { - let size = size.min(self.buffer.len()); - - let count = self - .inner - .lock() - .unwrap() - .read(&mut self.buffer[..size]) - .map_err(|e| StreamError::LastOperationFailed(anyhow::anyhow!(e)))?; - - Ok(Bytes::copy_from_slice(&self.buffer[..count])) - } -} - -#[async_trait] -impl Subscribe for PipeStdinStream { - async fn ready(&mut self) {} -} - -impl StdinStream for PipeStdinStream { - fn stream(&self) -> Box { - Box::new(self.clone()) - } - - fn isatty(&self) -> bool { - false - } -} - -struct PipeStdoutStream(Arc>); - -impl Clone for PipeStdoutStream { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -impl PipeStdoutStream { - fn new(inner: T) -> Self { - Self(Arc::new(Mutex::new(inner))) - } -} - -impl HostOutputStream for PipeStdoutStream { - fn write(&mut self, bytes: Bytes) -> Result<(), StreamError> { - self.0 - .lock() - .unwrap() - .write_all(&bytes) - .map_err(|e| StreamError::LastOperationFailed(anyhow::anyhow!(e))) - } - - fn flush(&mut self) -> Result<(), StreamError> { - self.0 - .lock() - .unwrap() - .flush() - .map_err(|e| StreamError::LastOperationFailed(anyhow::anyhow!(e))) - } - - fn check_write(&mut self) -> Result { - Ok(1024 * 1024) - } -} - -impl StdoutStream for PipeStdoutStream { - fn stream(&self) -> Box { - Box::new(self.clone()) - } - - fn isatty(&self) -> bool { - false - } -} - -#[async_trait] -impl Subscribe for PipeStdoutStream { - async fn ready(&mut self) {} -} - -struct BufferStdoutStream(OutputBuffer); - -impl StdoutStream for BufferStdoutStream { - fn stream(&self) -> Box { - Box::new(self.0.writer()) - } - - fn isatty(&self) -> bool { - false - } -} - -/// A builder of a `WasiCtx` for all versions of Wasi -#[allow(clippy::large_enum_variant)] -enum WasiCtxBuilder { - Preview1(wasi_preview1::WasiCtx), - Preview2(wasi_preview2::WasiCtxBuilder), -} - -impl From for WasiCtxBuilder { - fn from(value: WasiVersion) -> Self { - match value { - WasiVersion::Preview1 => { - Self::Preview1(wasi_common_preview1::tokio::WasiCtxBuilder::new().build()) - } - WasiVersion::Preview2 => Self::Preview2(wasi_preview2::WasiCtxBuilder::new()), - } - } -} - -impl WasiCtxBuilder { - fn build(self) -> Wasi { - match self { - WasiCtxBuilder::Preview1(ctx) => Wasi::Preview1(ctx), - WasiCtxBuilder::Preview2(mut b) => Wasi::Preview2 { - wasi_ctx: b.build(), - wasi_http_ctx: WasiHttpCtx::new(), - }, - } - } } diff --git a/crates/core/src/wasi_2023_10_18.rs b/crates/core/src/wasi_2023_10_18.rs deleted file mode 100644 index 9918f5a5e..000000000 --- a/crates/core/src/wasi_2023_10_18.rs +++ /dev/null @@ -1,2465 +0,0 @@ -#![doc(hidden)] // internal implementation detail used in tests and spin-trigger - -use anyhow::Result; -use async_trait::async_trait; -use std::mem; -use wasmtime::component::{Linker, Resource}; -use wasmtime_wasi::{TrappableError, WasiImpl, WasiView}; -use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; - -mod latest { - pub use wasmtime_wasi::bindings::*; - pub mod http { - pub use wasmtime_wasi_http::bindings::http::*; - } -} - -mod bindings { - use super::latest; - pub use super::UdpSocket; - - wasmtime::component::bindgen!({ - path: "../../wit", - interfaces: r#" - include wasi:http/proxy@0.2.0-rc-2023-10-18; - - // NB: this is handling the historical behavior where Spin supported - // more than "just" this snaphsot of the proxy world but additionally - // other CLI-related interfaces. - include wasi:cli/reactor@0.2.0-rc-2023-10-18; - "#, - async: { - only_imports: [ - "[method]descriptor.access-at", - "[method]descriptor.advise", - "[method]descriptor.change-directory-permissions-at", - "[method]descriptor.change-file-permissions-at", - "[method]descriptor.create-directory-at", - "[method]descriptor.get-flags", - "[method]descriptor.get-type", - "[method]descriptor.is-same-object", - "[method]descriptor.link-at", - "[method]descriptor.lock-exclusive", - "[method]descriptor.lock-shared", - "[method]descriptor.metadata-hash", - "[method]descriptor.metadata-hash-at", - "[method]descriptor.open-at", - "[method]descriptor.read", - "[method]descriptor.read-directory", - "[method]descriptor.readlink-at", - "[method]descriptor.remove-directory-at", - "[method]descriptor.rename-at", - "[method]descriptor.set-size", - "[method]descriptor.set-times", - "[method]descriptor.set-times-at", - "[method]descriptor.stat", - "[method]descriptor.stat-at", - "[method]descriptor.symlink-at", - "[method]descriptor.sync", - "[method]descriptor.sync-data", - "[method]descriptor.try-lock-exclusive", - "[method]descriptor.try-lock-shared", - "[method]descriptor.unlink-file-at", - "[method]descriptor.unlock", - "[method]descriptor.write", - "[method]input-stream.read", - "[method]input-stream.blocking-read", - "[method]input-stream.blocking-skip", - "[method]input-stream.skip", - "[method]output-stream.forward", - "[method]output-stream.splice", - "[method]output-stream.blocking-splice", - "[method]output-stream.blocking-flush", - "[method]output-stream.blocking-write", - "[method]output-stream.blocking-write-and-flush", - "[method]output-stream.blocking-write-zeroes-and-flush", - "[method]directory-entry-stream.read-directory-entry", - "poll-list", - "poll-one", - - "[method]tcp-socket.start-bind", - "[method]tcp-socket.start-connect", - "[method]udp-socket.finish-connect", - "[method]udp-socket.receive", - "[method]udp-socket.send", - "[method]udp-socket.start-bind", - "[method]udp-socket.stream", - "[method]outgoing-datagram-stream.send", - ], - }, - with: { - "wasi:io/poll/pollable": latest::io::poll::Pollable, - "wasi:io/streams/input-stream": latest::io::streams::InputStream, - "wasi:io/streams/output-stream": latest::io::streams::OutputStream, - "wasi:io/streams/error": latest::io::streams::Error, - "wasi:filesystem/types/directory-entry-stream": latest::filesystem::types::DirectoryEntryStream, - "wasi:filesystem/types/descriptor": latest::filesystem::types::Descriptor, - "wasi:cli/terminal-input/terminal-input": latest::cli::terminal_input::TerminalInput, - "wasi:cli/terminal-output/terminal-output": latest::cli::terminal_output::TerminalOutput, - "wasi:sockets/tcp/tcp-socket": latest::sockets::tcp::TcpSocket, - "wasi:sockets/udp/udp-socket": UdpSocket, - "wasi:sockets/network/network": latest::sockets::network::Network, - "wasi:sockets/ip-name-lookup/resolve-address-stream": latest::sockets::ip_name_lookup::ResolveAddressStream, - "wasi:http/types/incoming-response": latest::http::types::IncomingResponse, - "wasi:http/types/incoming-request": latest::http::types::IncomingRequest, - "wasi:http/types/incoming-body": latest::http::types::IncomingBody, - "wasi:http/types/outgoing-response": latest::http::types::OutgoingResponse, - "wasi:http/types/outgoing-request": latest::http::types::OutgoingRequest, - "wasi:http/types/outgoing-body": latest::http::types::OutgoingBody, - "wasi:http/types/fields": latest::http::types::Fields, - "wasi:http/types/response-outparam": latest::http::types::ResponseOutparam, - "wasi:http/types/future-incoming-response": latest::http::types::FutureIncomingResponse, - "wasi:http/types/future-trailers": latest::http::types::FutureTrailers, - }, - trappable_imports: true, - }); -} - -mod wasi { - pub use super::bindings::wasi::{ - cli0_2_0_rc_2023_10_18 as cli, clocks0_2_0_rc_2023_10_18 as clocks, - filesystem0_2_0_rc_2023_10_18 as filesystem, http0_2_0_rc_2023_10_18 as http, - io0_2_0_rc_2023_10_18 as io, random0_2_0_rc_2023_10_18 as random, - sockets0_2_0_rc_2023_10_18 as sockets, - }; -} - -pub mod exports { - pub mod wasi { - pub use super::super::bindings::exports::wasi::http0_2_0_rc_2023_10_18 as http; - } -} - -use wasi::cli::terminal_input::TerminalInput; -use wasi::cli::terminal_output::TerminalOutput; -use wasi::clocks::monotonic_clock::Instant; -use wasi::clocks::wall_clock::Datetime; -use wasi::filesystem::types::{ - AccessType, Advice, Descriptor, DescriptorFlags, DescriptorStat, DescriptorType, - DirectoryEntry, DirectoryEntryStream, Error, ErrorCode as FsErrorCode, Filesize, - MetadataHashValue, Modes, NewTimestamp, OpenFlags, PathFlags, -}; -use wasi::http::types::{ - Error as HttpError, Fields, FutureIncomingResponse, FutureTrailers, Headers, IncomingBody, - IncomingRequest, IncomingResponse, Method, OutgoingBody, OutgoingRequest, OutgoingResponse, - RequestOptions, ResponseOutparam, Scheme, StatusCode, Trailers, -}; -use wasi::io::poll::Pollable; -use wasi::io::streams::{InputStream, OutputStream, StreamError}; -use wasi::sockets::ip_name_lookup::{IpAddress, ResolveAddressStream}; -use wasi::sockets::network::{Ipv4SocketAddress, Ipv6SocketAddress}; -use wasi::sockets::tcp::{ - ErrorCode as SocketErrorCode, IpAddressFamily, IpSocketAddress, Network, ShutdownType, - TcpSocket, -}; -use wasi::sockets::udp::Datagram; - -pub fn add_to_linker(linker: &mut Linker) -> Result<()> -where - T: WasiView + WasiHttpView, -{ - // interfaces from the "command" world - fn type_annotate_wasi(f: F) -> F - where - F: Fn(&mut T) -> WasiImpl<&mut T>, - { - f - } - let closure = type_annotate_wasi::(|t| WasiImpl(t)); - wasi::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; - wasi::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; - wasi::filesystem::types::add_to_linker_get_host(linker, closure)?; - wasi::filesystem::preopens::add_to_linker_get_host(linker, closure)?; - wasi::io::poll::add_to_linker_get_host(linker, closure)?; - wasi::io::streams::add_to_linker_get_host(linker, closure)?; - wasi::random::random::add_to_linker_get_host(linker, closure)?; - wasi::random::insecure::add_to_linker_get_host(linker, closure)?; - wasi::random::insecure_seed::add_to_linker_get_host(linker, closure)?; - wasi::cli::exit::add_to_linker_get_host(linker, closure)?; - wasi::cli::environment::add_to_linker_get_host(linker, closure)?; - wasi::cli::stdin::add_to_linker_get_host(linker, closure)?; - wasi::cli::stdout::add_to_linker_get_host(linker, closure)?; - wasi::cli::stderr::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_input::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_output::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_stdin::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_stdout::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_stderr::add_to_linker_get_host(linker, closure)?; - wasi::sockets::tcp::add_to_linker_get_host(linker, closure)?; - wasi::sockets::tcp_create_socket::add_to_linker_get_host(linker, closure)?; - wasi::sockets::udp::add_to_linker_get_host(linker, closure)?; - wasi::sockets::udp_create_socket::add_to_linker_get_host(linker, closure)?; - wasi::sockets::instance_network::add_to_linker_get_host(linker, closure)?; - wasi::sockets::network::add_to_linker_get_host(linker, closure)?; - wasi::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; - - fn type_annotate_http(f: F) -> F - where - F: Fn(&mut T) -> WasiHttpImpl<&mut T>, - { - f - } - let closure = type_annotate_http::(|t| WasiHttpImpl(t)); - wasi::http::types::add_to_linker_get_host(linker, closure)?; - wasi::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; - Ok(()) -} - -impl wasi::clocks::monotonic_clock::Host for WasiImpl -where - T: WasiView, -{ - fn now(&mut self) -> wasmtime::Result { - latest::clocks::monotonic_clock::Host::now(self) - } - - fn resolution(&mut self) -> wasmtime::Result { - latest::clocks::monotonic_clock::Host::resolution(self) - } - - fn subscribe(&mut self, when: Instant, absolute: bool) -> wasmtime::Result> { - if absolute { - latest::clocks::monotonic_clock::Host::subscribe_instant(self, when) - } else { - latest::clocks::monotonic_clock::Host::subscribe_duration(self, when) - } - } -} - -impl wasi::clocks::wall_clock::Host for WasiImpl -where - T: WasiView, -{ - fn now(&mut self) -> wasmtime::Result { - Ok(latest::clocks::wall_clock::Host::now(self)?.into()) - } - - fn resolution(&mut self) -> wasmtime::Result { - Ok(latest::clocks::wall_clock::Host::resolution(self)?.into()) - } -} - -impl wasi::filesystem::types::Host for WasiImpl -where - T: WasiView, -{ - fn filesystem_error_code( - &mut self, - err: Resource, - ) -> wasmtime::Result> { - Ok(latest::filesystem::types::Host::filesystem_error_code(self, err)?.map(|e| e.into())) - } -} - -#[async_trait] -impl wasi::filesystem::types::HostDescriptor for WasiImpl -where - T: WasiView, -{ - fn read_via_stream( - &mut self, - self_: Resource, - offset: Filesize, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::read_via_stream( - self, self_, offset, - )) - } - - fn write_via_stream( - &mut self, - self_: Resource, - offset: Filesize, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::write_via_stream( - self, self_, offset, - )) - } - - fn append_via_stream( - &mut self, - self_: Resource, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::append_via_stream(self, self_)) - } - - async fn advise( - &mut self, - self_: Resource, - offset: Filesize, - length: Filesize, - advice: Advice, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::advise( - self, - self_, - offset, - length, - advice.into(), - ) - .await, - ) - } - - async fn sync_data( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::sync_data(self, self_).await) - } - - async fn get_flags( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::get_flags(self, self_).await) - } - - async fn get_type( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::get_type(self, self_).await) - } - - async fn set_size( - &mut self, - self_: Resource, - size: Filesize, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::set_size(self, self_, size).await) - } - - async fn set_times( - &mut self, - self_: Resource, - data_access_timestamp: NewTimestamp, - data_modification_timestamp: NewTimestamp, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::set_times( - self, - self_, - data_access_timestamp.into(), - data_modification_timestamp.into(), - ) - .await, - ) - } - - async fn read( - &mut self, - self_: Resource, - length: Filesize, - offset: Filesize, - ) -> wasmtime::Result, bool), FsErrorCode>> { - convert_result( - latest::filesystem::types::HostDescriptor::read(self, self_, length, offset).await, - ) - } - - async fn write( - &mut self, - self_: Resource, - buffer: Vec, - offset: Filesize, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::write(self, self_, buffer, offset).await, - ) - } - - async fn read_directory( - &mut self, - self_: Resource, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::read_directory(self, self_).await) - } - - async fn sync( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::sync(self, self_).await) - } - - async fn create_directory_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::create_directory_at(self, self_, path).await, - ) - } - - async fn stat( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::stat(self, self_).await) - } - - async fn stat_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::stat_at( - self, - self_, - path_flags.into(), - path, - ) - .await, - ) - } - - async fn set_times_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - data_access_timestamp: NewTimestamp, - data_modification_timestamp: NewTimestamp, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::set_times_at( - self, - self_, - path_flags.into(), - path, - data_access_timestamp.into(), - data_modification_timestamp.into(), - ) - .await, - ) - } - - async fn link_at( - &mut self, - self_: Resource, - old_path_flags: PathFlags, - old_path: String, - new_descriptor: Resource, - new_path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::link_at( - self, - self_, - old_path_flags.into(), - old_path, - new_descriptor, - new_path, - ) - .await, - ) - } - - async fn open_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - open_flags: OpenFlags, - flags: DescriptorFlags, - _modes: Modes, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result( - latest::filesystem::types::HostDescriptor::open_at( - self, - self_, - path_flags.into(), - path, - open_flags.into(), - flags.into(), - ) - .await, - ) - } - - async fn readlink_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::readlink_at(self, self_, path).await, - ) - } - - async fn remove_directory_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::remove_directory_at(self, self_, path).await, - ) - } - - async fn rename_at( - &mut self, - self_: Resource, - old_path: String, - new_descriptor: Resource, - new_path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::rename_at( - self, - self_, - old_path, - new_descriptor, - new_path, - ) - .await, - ) - } - - async fn symlink_at( - &mut self, - self_: Resource, - old_path: String, - new_path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::symlink_at(self, self_, old_path, new_path) - .await, - ) - } - - async fn access_at( - &mut self, - _self_: Resource, - _path_flags: PathFlags, - _path: String, - _type_: AccessType, - ) -> wasmtime::Result> { - anyhow::bail!("access-at API is no longer supported in the latest snapshot") - } - - async fn unlink_file_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::unlink_file_at(self, self_, path).await, - ) - } - - async fn change_file_permissions_at( - &mut self, - _self_: Resource, - _path_flags: PathFlags, - _path: String, - _modes: Modes, - ) -> wasmtime::Result> { - anyhow::bail!( - "change-file-permissions-at API is no longer supported in the latest snapshot" - ) - } - - async fn change_directory_permissions_at( - &mut self, - _self_: Resource, - _path_flags: PathFlags, - _path: String, - _modes: Modes, - ) -> wasmtime::Result> { - anyhow::bail!( - "change-directory-permissions-at API is no longer supported in the latest snapshot" - ) - } - - async fn lock_shared( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("lock-shared API is no longer supported in the latest snapshot") - } - - async fn lock_exclusive( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("lock-exclusive API is no longer supported in the latest snapshot") - } - - async fn try_lock_shared( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("try-lock-shared API is no longer supported in the latest snapshot") - } - - async fn try_lock_exclusive( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("try-lock-exclusive API is no longer supported in the latest snapshot") - } - - async fn unlock( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("unlock API is no longer supported in the latest snapshot") - } - - async fn is_same_object( - &mut self, - self_: Resource, - other: Resource, - ) -> wasmtime::Result { - latest::filesystem::types::HostDescriptor::is_same_object(self, self_, other).await - } - - async fn metadata_hash( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::metadata_hash(self, self_).await) - } - - async fn metadata_hash_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::metadata_hash_at( - self, - self_, - path_flags.into(), - path, - ) - .await, - ) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::filesystem::types::HostDescriptor::drop(self, rep) - } -} - -#[async_trait] -impl wasi::filesystem::types::HostDirectoryEntryStream for WasiImpl -where - T: WasiView, -{ - async fn read_directory_entry( - &mut self, - self_: Resource, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result( - latest::filesystem::types::HostDirectoryEntryStream::read_directory_entry(self, self_) - .await - .map(|e| e.map(DirectoryEntry::from)), - ) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::filesystem::types::HostDirectoryEntryStream::drop(self, rep) - } -} - -impl wasi::filesystem::preopens::Host for WasiImpl -where - T: WasiView, -{ - fn get_directories(&mut self) -> wasmtime::Result, String)>> { - latest::filesystem::preopens::Host::get_directories(self) - } -} - -#[async_trait] -impl wasi::io::poll::Host for WasiImpl -where - T: WasiView, -{ - async fn poll_list(&mut self, list: Vec>) -> wasmtime::Result> { - latest::io::poll::Host::poll(self, list).await - } - - async fn poll_one(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::poll::HostPollable::block(self, rep).await - } -} - -impl wasi::io::poll::HostPollable for WasiImpl -where - T: WasiView, -{ - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::poll::HostPollable::drop(self, rep) - } -} - -impl wasi::io::streams::Host for WasiImpl where T: WasiView {} - -impl wasi::io::streams::HostError for WasiImpl -where - T: WasiView, -{ - fn to_debug_string(&mut self, self_: Resource) -> wasmtime::Result { - latest::io::error::HostError::to_debug_string(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::error::HostError::drop(self, rep) - } -} - -#[async_trait] -impl wasi::io::streams::HostInputStream for WasiImpl -where - T: WasiView, -{ - async fn read( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result, StreamError>> { - let result = latest::io::streams::HostInputStream::read(self, self_, len).await; - convert_stream_result(self, result) - } - - async fn blocking_read( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result, StreamError>> { - let result = latest::io::streams::HostInputStream::blocking_read(self, self_, len).await; - convert_stream_result(self, result) - } - - async fn skip( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostInputStream::skip(self, self_, len).await; - convert_stream_result(self, result) - } - - async fn blocking_skip( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostInputStream::blocking_skip(self, self_, len).await; - convert_stream_result(self, result) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - latest::io::streams::HostInputStream::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::streams::HostInputStream::drop(self, rep) - } -} - -#[async_trait] -impl wasi::io::streams::HostOutputStream for WasiImpl -where - T: WasiView, -{ - fn check_write( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::check_write(self, self_); - convert_stream_result(self, result) - } - - fn write( - &mut self, - self_: Resource, - contents: Vec, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::write(self, self_, contents); - convert_stream_result(self, result) - } - - async fn blocking_write_and_flush( - &mut self, - self_: Resource, - contents: Vec, - ) -> wasmtime::Result> { - let result = - latest::io::streams::HostOutputStream::blocking_write_and_flush(self, self_, contents) - .await; - convert_stream_result(self, result) - } - - fn flush( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::flush(self, self_); - convert_stream_result(self, result) - } - - async fn blocking_flush( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::blocking_flush(self, self_).await; - convert_stream_result(self, result) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - latest::io::streams::HostOutputStream::subscribe(self, self_) - } - - fn write_zeroes( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::write_zeroes(self, self_, len); - convert_stream_result(self, result) - } - - async fn blocking_write_zeroes_and_flush( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::blocking_write_zeroes_and_flush( - self, self_, len, - ) - .await; - convert_stream_result(self, result) - } - - async fn splice( - &mut self, - self_: Resource, - src: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::splice(self, self_, src, len).await; - convert_stream_result(self, result) - } - - async fn blocking_splice( - &mut self, - self_: Resource, - src: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = - latest::io::streams::HostOutputStream::blocking_splice(self, self_, src, len).await; - convert_stream_result(self, result) - } - - async fn forward( - &mut self, - _self_: Resource, - _src: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("forward API no longer supported") - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::streams::HostOutputStream::drop(self, rep) - } -} - -impl wasi::random::random::Host for WasiImpl -where - T: WasiView, -{ - fn get_random_bytes(&mut self, len: u64) -> wasmtime::Result> { - latest::random::random::Host::get_random_bytes(self, len) - } - - fn get_random_u64(&mut self) -> wasmtime::Result { - latest::random::random::Host::get_random_u64(self) - } -} - -impl wasi::random::insecure::Host for WasiImpl -where - T: WasiView, -{ - fn get_insecure_random_bytes(&mut self, len: u64) -> wasmtime::Result> { - latest::random::insecure::Host::get_insecure_random_bytes(self, len) - } - - fn get_insecure_random_u64(&mut self) -> wasmtime::Result { - latest::random::insecure::Host::get_insecure_random_u64(self) - } -} - -impl wasi::random::insecure_seed::Host for WasiImpl -where - T: WasiView, -{ - fn insecure_seed(&mut self) -> wasmtime::Result<(u64, u64)> { - latest::random::insecure_seed::Host::insecure_seed(self) - } -} - -impl wasi::cli::exit::Host for WasiImpl -where - T: WasiView, -{ - fn exit(&mut self, status: Result<(), ()>) -> wasmtime::Result<()> { - latest::cli::exit::Host::exit(self, status) - } -} - -impl wasi::cli::environment::Host for WasiImpl -where - T: WasiView, -{ - fn get_environment(&mut self) -> wasmtime::Result> { - latest::cli::environment::Host::get_environment(self) - } - - fn get_arguments(&mut self) -> wasmtime::Result> { - latest::cli::environment::Host::get_arguments(self) - } - - fn initial_cwd(&mut self) -> wasmtime::Result> { - latest::cli::environment::Host::initial_cwd(self) - } -} - -impl wasi::cli::stdin::Host for WasiImpl -where - T: WasiView, -{ - fn get_stdin(&mut self) -> wasmtime::Result> { - latest::cli::stdin::Host::get_stdin(self) - } -} - -impl wasi::cli::stdout::Host for WasiImpl -where - T: WasiView, -{ - fn get_stdout(&mut self) -> wasmtime::Result> { - latest::cli::stdout::Host::get_stdout(self) - } -} - -impl wasi::cli::stderr::Host for WasiImpl -where - T: WasiView, -{ - fn get_stderr(&mut self) -> wasmtime::Result> { - latest::cli::stderr::Host::get_stderr(self) - } -} - -impl wasi::cli::terminal_stdin::Host for WasiImpl -where - T: WasiView, -{ - fn get_terminal_stdin(&mut self) -> wasmtime::Result>> { - latest::cli::terminal_stdin::Host::get_terminal_stdin(self) - } -} - -impl wasi::cli::terminal_stdout::Host for WasiImpl -where - T: WasiView, -{ - fn get_terminal_stdout(&mut self) -> wasmtime::Result>> { - latest::cli::terminal_stdout::Host::get_terminal_stdout(self) - } -} - -impl wasi::cli::terminal_stderr::Host for WasiImpl -where - T: WasiView, -{ - fn get_terminal_stderr(&mut self) -> wasmtime::Result>> { - latest::cli::terminal_stderr::Host::get_terminal_stderr(self) - } -} - -impl wasi::cli::terminal_input::Host for WasiImpl where T: WasiView {} - -impl wasi::cli::terminal_input::HostTerminalInput for WasiImpl -where - T: WasiView, -{ - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::cli::terminal_input::HostTerminalInput::drop(self, rep) - } -} - -impl wasi::cli::terminal_output::Host for WasiImpl where T: WasiView {} - -impl wasi::cli::terminal_output::HostTerminalOutput for WasiImpl -where - T: WasiView, -{ - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::cli::terminal_output::HostTerminalOutput::drop(self, rep) - } -} - -impl wasi::sockets::tcp::Host for WasiImpl where T: WasiView {} - -#[async_trait] -impl wasi::sockets::tcp::HostTcpSocket for WasiImpl -where - T: WasiView, -{ - async fn start_bind( - &mut self, - self_: Resource, - network: Resource, - local_address: IpSocketAddress, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::start_bind( - self, - self_, - network, - local_address.into(), - ) - .await, - ) - } - - fn finish_bind( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::finish_bind( - self, self_, - )) - } - - async fn start_connect( - &mut self, - self_: Resource, - network: Resource, - remote_address: IpSocketAddress, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::start_connect( - self, - self_, - network, - remote_address.into(), - ) - .await, - ) - } - - fn finish_connect( - &mut self, - self_: Resource, - ) -> wasmtime::Result, Resource), SocketErrorCode>> - { - convert_result(latest::sockets::tcp::HostTcpSocket::finish_connect( - self, self_, - )) - } - - fn start_listen( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::start_listen( - self, self_, - )) - } - - fn finish_listen( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::finish_listen( - self, self_, - )) - } - - fn accept( - &mut self, - self_: Resource, - ) -> wasmtime::Result< - Result< - ( - Resource, - Resource, - Resource, - ), - SocketErrorCode, - >, - > { - convert_result(latest::sockets::tcp::HostTcpSocket::accept(self, self_)) - } - - fn local_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::local_address( - self, self_, - )) - } - - fn remote_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::remote_address( - self, self_, - )) - } - - fn address_family(&mut self, self_: Resource) -> wasmtime::Result { - latest::sockets::tcp::HostTcpSocket::address_family(self, self_).map(|e| e.into()) - } - - fn ipv6_only( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn set_ipv6_only( - &mut self, - _self_: Resource, - _value: bool, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn set_listen_backlog_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::set_listen_backlog_size(self, self_, value), - ) - } - - fn keep_alive( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_enabled( - self, self_, - )) - } - - fn set_keep_alive( - &mut self, - self_: Resource, - value: bool, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::set_keep_alive_enabled( - self, self_, value, - )) - } - - fn no_delay( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("no-delay API no longer supported") - } - - fn set_no_delay( - &mut self, - _self_: Resource, - _value: bool, - ) -> wasmtime::Result> { - anyhow::bail!("set-no-delay API no longer supported") - } - - fn unicast_hop_limit( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::hop_limit(self, self_)) - } - - fn set_unicast_hop_limit( - &mut self, - self_: Resource, - value: u8, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::set_hop_limit( - self, self_, value, - )) - } - - fn receive_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::receive_buffer_size( - self, self_, - )) - } - - fn set_receive_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::set_receive_buffer_size(self, self_, value), - ) - } - - fn send_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::send_buffer_size( - self, self_, - )) - } - - fn set_send_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::set_send_buffer_size( - self, self_, value, - )) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - latest::sockets::tcp::HostTcpSocket::subscribe(self, self_) - } - - fn shutdown( - &mut self, - self_: Resource, - shutdown_type: ShutdownType, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::shutdown( - self, - self_, - shutdown_type.into(), - )) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::tcp::HostTcpSocket::drop(self, rep) - } -} - -impl wasi::sockets::tcp_create_socket::Host for WasiImpl -where - T: WasiView, -{ - fn create_tcp_socket( - &mut self, - address_family: IpAddressFamily, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result(latest::sockets::tcp_create_socket::Host::create_tcp_socket( - self, - address_family.into(), - )) - } -} - -impl wasi::sockets::udp::Host for WasiImpl where T: WasiView {} - -/// Between the snapshot of WASI that this file is implementing and the current -/// implementation of WASI UDP sockets were redesigned slightly to deal with -/// a different way of managing incoming and outgoing datagrams. This means -/// that this snapshot's `{start,finish}_connect`, `send`, and `receive` -/// methods are no longer natively implemented, so they're polyfilled by this -/// implementation. -pub enum UdpSocket { - Initial(Resource), - Connecting(Resource, IpSocketAddress), - Connected { - socket: Resource, - incoming: Resource, - outgoing: Resource, - }, - Dummy, -} - -impl UdpSocket { - async fn finish_connect( - table: &mut WasiImpl, - socket: &Resource, - explicit: bool, - ) -> wasmtime::Result> { - let state = table.table().get_mut(socket)?; - let (new_socket, addr) = match mem::replace(state, UdpSocket::Dummy) { - // Implicit finishes will call `stream` for sockets in the initial - // state. - UdpSocket::Initial(socket) if !explicit => (socket, None), - // Implicit finishes won't try to reconnect a socket. - UdpSocket::Connected { .. } if !explicit => return Ok(Ok(())), - // Only explicit finishes can transition from the `Connecting` state. - UdpSocket::Connecting(socket, addr) if explicit => (socket, Some(addr)), - _ => return Ok(Err(SocketErrorCode::ConcurrencyConflict)), - }; - let borrow = Resource::new_borrow(new_socket.rep()); - let result = convert_result( - latest::sockets::udp::HostUdpSocket::stream(table, borrow, addr.map(|a| a.into())) - .await, - )?; - let (incoming, outgoing) = match result { - Ok(pair) => pair, - Err(e) => return Ok(Err(e)), - }; - *table.table().get_mut(socket)? = UdpSocket::Connected { - socket: new_socket, - incoming, - outgoing, - }; - Ok(Ok(())) - } - - fn inner(&self) -> wasmtime::Result> { - let r = match self { - UdpSocket::Initial(r) => r, - UdpSocket::Connecting(r, _) => r, - UdpSocket::Connected { socket, .. } => socket, - UdpSocket::Dummy => anyhow::bail!("invalid udp socket state"), - }; - Ok(Resource::new_borrow(r.rep())) - } -} - -#[async_trait] -impl wasi::sockets::udp::HostUdpSocket for WasiImpl -where - T: WasiView, -{ - async fn start_bind( - &mut self, - self_: Resource, - network: Resource, - local_address: IpSocketAddress, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result( - latest::sockets::udp::HostUdpSocket::start_bind( - self, - socket, - network, - local_address.into(), - ) - .await, - ) - } - - fn finish_bind( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::finish_bind( - self, socket, - )) - } - - fn start_connect( - &mut self, - self_: Resource, - _network: Resource, - remote_address: IpSocketAddress, - ) -> wasmtime::Result> { - let socket = self.table().get_mut(&self_)?; - let (new_state, result) = match mem::replace(socket, UdpSocket::Dummy) { - UdpSocket::Initial(socket) => (UdpSocket::Connecting(socket, remote_address), Ok(())), - other => (other, Err(SocketErrorCode::ConcurrencyConflict)), - }; - *socket = new_state; - Ok(result) - } - - async fn finish_connect( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - UdpSocket::finish_connect(self, &self_, true).await - } - - async fn receive( - &mut self, - self_: Resource, - max_results: u64, - ) -> wasmtime::Result, SocketErrorCode>> { - // If the socket is in the `initial` state then complete the connect, - // otherwise verify we're connected. - if let Err(e) = UdpSocket::finish_connect(self, &self_, true).await? { - return Ok(Err(e)); - } - - // Use our connected state to acquire the `incoming-datagram-stream` - // resource, then receive some datagrams. - let incoming = match self.table().get(&self_)? { - UdpSocket::Connected { incoming, .. } => Resource::new_borrow(incoming.rep()), - _ => return Ok(Err(SocketErrorCode::ConcurrencyConflict)), - }; - let result: Result, _> = convert_result( - latest::sockets::udp::HostIncomingDatagramStream::receive(self, incoming, max_results), - )?; - match result { - Ok(datagrams) => Ok(Ok(datagrams - .into_iter() - .map(|datagram| datagram.into()) - .collect())), - Err(e) => Ok(Err(e)), - } - } - - async fn send( - &mut self, - self_: Resource, - mut datagrams: Vec, - ) -> wasmtime::Result> { - // If the socket is in the `initial` state then complete the connect, - // otherwise verify we're connected. - if let Err(e) = UdpSocket::finish_connect(self, &self_, true).await? { - return Ok(Err(e)); - } - - // Use our connected state to acquire the `outgoing-datagram-stream` - // resource. - let outgoing = match self.table().get(&self_)? { - UdpSocket::Connected { outgoing, .. } => Resource::new_borrow(outgoing.rep()), - _ => return Ok(Err(SocketErrorCode::ConcurrencyConflict)), - }; - - // Acquire a sending permit for some datagrams, truncating our list to - // that size if we have one. - let outgoing2 = Resource::new_borrow(outgoing.rep()); - match convert_result( - latest::sockets::udp::HostOutgoingDatagramStream::check_send(self, outgoing2), - )? { - Ok(n) => { - if datagrams.len() as u64 > n { - datagrams.truncate(n as usize); - } - } - Err(e) => return Ok(Err(e)), - } - - // Send off the datagrams. - convert_result( - latest::sockets::udp::HostOutgoingDatagramStream::send( - self, - outgoing, - datagrams - .into_iter() - .map(|d| latest::sockets::udp::OutgoingDatagram { - data: d.data, - remote_address: Some(d.remote_address.into()), - }) - .collect(), - ) - .await, - ) - } - - fn local_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::local_address( - self, socket, - )) - } - - fn remote_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::remote_address( - self, socket, - )) - } - - fn address_family(&mut self, self_: Resource) -> wasmtime::Result { - let socket = self.table().get(&self_)?.inner()?; - latest::sockets::udp::HostUdpSocket::address_family(self, socket).map(|e| e.into()) - } - - fn ipv6_only( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn set_ipv6_only( - &mut self, - _self_: Resource, - _value: bool, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn unicast_hop_limit( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::unicast_hop_limit( - self, socket, - )) - } - - fn set_unicast_hop_limit( - &mut self, - self_: Resource, - value: u8, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::set_unicast_hop_limit( - self, socket, value, - )) - } - - fn receive_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::receive_buffer_size( - self, socket, - )) - } - - fn set_receive_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result( - latest::sockets::udp::HostUdpSocket::set_receive_buffer_size(self, socket, value), - ) - } - - fn send_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::send_buffer_size( - self, socket, - )) - } - - fn set_send_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - convert_result(latest::sockets::udp::HostUdpSocket::set_send_buffer_size( - self, socket, value, - )) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - let socket = self.table().get(&self_)?.inner()?; - latest::sockets::udp::HostUdpSocket::subscribe(self, socket) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - let me = self.table().delete(rep)?; - let socket = match me { - UdpSocket::Initial(s) => s, - UdpSocket::Connecting(s, _) => s, - UdpSocket::Connected { - socket, - incoming, - outgoing, - } => { - latest::sockets::udp::HostIncomingDatagramStream::drop(self, incoming)?; - latest::sockets::udp::HostOutgoingDatagramStream::drop(self, outgoing)?; - socket - } - UdpSocket::Dummy => return Ok(()), - }; - latest::sockets::udp::HostUdpSocket::drop(self, socket) - } -} - -impl wasi::sockets::udp_create_socket::Host for WasiImpl -where - T: WasiView, -{ - fn create_udp_socket( - &mut self, - address_family: IpAddressFamily, - ) -> wasmtime::Result, SocketErrorCode>> { - let result = convert_result(latest::sockets::udp_create_socket::Host::create_udp_socket( - self, - address_family.into(), - ))?; - let socket = match result { - Ok(socket) => socket, - Err(e) => return Ok(Err(e)), - }; - let socket = self.table().push(UdpSocket::Initial(socket))?; - Ok(Ok(socket)) - } -} - -impl wasi::sockets::instance_network::Host for WasiImpl -where - T: WasiView, -{ - fn instance_network(&mut self) -> wasmtime::Result> { - latest::sockets::instance_network::Host::instance_network(self) - } -} - -impl wasi::sockets::network::Host for WasiImpl where T: WasiView {} - -impl wasi::sockets::network::HostNetwork for WasiImpl -where - T: WasiView, -{ - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::network::HostNetwork::drop(self, rep) - } -} - -impl wasi::sockets::ip_name_lookup::Host for WasiImpl -where - T: WasiView, -{ - fn resolve_addresses( - &mut self, - network: Resource, - name: String, - _address_family: Option, - _include_unavailable: bool, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result(latest::sockets::ip_name_lookup::Host::resolve_addresses( - self, network, name, - )) - } -} - -impl wasi::sockets::ip_name_lookup::HostResolveAddressStream for WasiImpl -where - T: WasiView, -{ - fn resolve_next_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result( - latest::sockets::ip_name_lookup::HostResolveAddressStream::resolve_next_address( - self, self_, - ) - .map(|e| e.map(|e| e.into())), - ) - } - - fn subscribe( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::sockets::ip_name_lookup::HostResolveAddressStream::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::ip_name_lookup::HostResolveAddressStream::drop(self, rep) - } -} - -impl wasi::http::types::Host for WasiHttpImpl where T: WasiHttpView + Send {} - -impl wasi::http::types::HostFields for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn new( - &mut self, - entries: Vec<(String, Vec)>, - ) -> wasmtime::Result> { - match latest::http::types::HostFields::from_list(self, entries)? { - Ok(fields) => Ok(fields), - Err(e) => Err(e.into()), - } - } - - fn get( - &mut self, - self_: wasmtime::component::Resource, - name: String, - ) -> wasmtime::Result>> { - latest::http::types::HostFields::get(self, self_, name) - } - - fn set( - &mut self, - self_: wasmtime::component::Resource, - name: String, - value: Vec>, - ) -> wasmtime::Result<()> { - latest::http::types::HostFields::set(self, self_, name, value)??; - Ok(()) - } - - fn delete( - &mut self, - self_: wasmtime::component::Resource, - name: String, - ) -> wasmtime::Result<()> { - latest::http::types::HostFields::delete(self, self_, name)??; - Ok(()) - } - - fn append( - &mut self, - self_: wasmtime::component::Resource, - name: String, - value: Vec, - ) -> wasmtime::Result<()> { - latest::http::types::HostFields::append(self, self_, name, value)??; - Ok(()) - } - - fn entries( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result)>> { - latest::http::types::HostFields::entries(self, self_) - } - - fn clone( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostFields::clone(self, self_) - } - - fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { - latest::http::types::HostFields::drop(self, rep) - } -} - -impl wasi::http::types::HostIncomingRequest for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn method( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result { - latest::http::types::HostIncomingRequest::method(self, self_).map(|e| e.into()) - } - - fn path_with_query( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::path_with_query(self, self_) - } - - fn scheme( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::scheme(self, self_).map(|e| e.map(|e| e.into())) - } - - fn authority( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::authority(self, self_) - } - - fn headers( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::headers(self, self_) - } - - fn consume( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostIncomingRequest::consume(self, self_) - } - - fn drop( - &mut self, - rep: wasmtime::component::Resource, - ) -> wasmtime::Result<()> { - latest::http::types::HostIncomingRequest::drop(self, rep) - } -} - -impl wasi::http::types::HostIncomingResponse for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn status( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result { - latest::http::types::HostIncomingResponse::status(self, self_) - } - - fn headers( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingResponse::headers(self, self_) - } - - fn consume( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostIncomingResponse::consume(self, self_) - } - - fn drop( - &mut self, - rep: wasmtime::component::Resource, - ) -> wasmtime::Result<()> { - latest::http::types::HostIncomingResponse::drop(self, rep) - } -} - -impl wasi::http::types::HostIncomingBody for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn stream( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostIncomingBody::stream(self, self_) - } - - fn finish( - &mut self, - this: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingBody::finish(self, this) - } - - fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { - latest::http::types::HostIncomingBody::drop(self, rep) - } -} - -impl wasi::http::types::HostOutgoingRequest for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn new( - &mut self, - method: Method, - path_with_query: Option, - scheme: Option, - authority: Option, - headers: wasmtime::component::Resource, - ) -> wasmtime::Result> { - let headers = latest::http::types::HostFields::clone(self, headers)?; - let request = latest::http::types::HostOutgoingRequest::new(self, headers)?; - let borrow = || Resource::new_borrow(request.rep()); - - if let Err(()) = - latest::http::types::HostOutgoingRequest::set_method(self, borrow(), method.into())? - { - latest::http::types::HostOutgoingRequest::drop(self, request)?; - anyhow::bail!("invalid method supplied"); - } - - if let Err(()) = latest::http::types::HostOutgoingRequest::set_path_with_query( - self, - borrow(), - path_with_query, - )? { - latest::http::types::HostOutgoingRequest::drop(self, request)?; - anyhow::bail!("invalid path-with-query supplied"); - } - - // Historical WASI would fill in an empty authority with a port which - // got just enough working to get things through. Current WASI requires - // the authority, though, so perform the translation manually here. - let authority = authority.unwrap_or_else(|| match &scheme { - Some(Scheme::Http) | Some(Scheme::Other(_)) => ":80".to_string(), - Some(Scheme::Https) | None => ":443".to_string(), - }); - if let Err(()) = latest::http::types::HostOutgoingRequest::set_scheme( - self, - borrow(), - scheme.map(|s| s.into()), - )? { - latest::http::types::HostOutgoingRequest::drop(self, request)?; - anyhow::bail!("invalid scheme supplied"); - } - - if let Err(()) = latest::http::types::HostOutgoingRequest::set_authority( - self, - borrow(), - Some(authority), - )? { - latest::http::types::HostOutgoingRequest::drop(self, request)?; - anyhow::bail!("invalid authority supplied"); - } - - Ok(request) - } - - fn write( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostOutgoingRequest::body(self, self_) - } - - fn drop( - &mut self, - rep: wasmtime::component::Resource, - ) -> wasmtime::Result<()> { - latest::http::types::HostOutgoingRequest::drop(self, rep) - } -} - -impl wasi::http::types::HostOutgoingResponse for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn new( - &mut self, - status_code: StatusCode, - headers: wasmtime::component::Resource, - ) -> wasmtime::Result> { - let headers = latest::http::types::HostFields::clone(self, headers)?; - let response = latest::http::types::HostOutgoingResponse::new(self, headers)?; - let borrow = || Resource::new_borrow(response.rep()); - - if let Err(()) = - latest::http::types::HostOutgoingResponse::set_status_code(self, borrow(), status_code)? - { - latest::http::types::HostOutgoingResponse::drop(self, response)?; - anyhow::bail!("invalid status code supplied"); - } - - Ok(response) - } - - fn write( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostOutgoingResponse::body(self, self_) - } - - fn drop( - &mut self, - rep: wasmtime::component::Resource, - ) -> wasmtime::Result<()> { - latest::http::types::HostOutgoingResponse::drop(self, rep) - } -} - -impl wasi::http::types::HostOutgoingBody for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn write( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostOutgoingBody::write(self, self_) - } - - fn finish( - &mut self, - this: wasmtime::component::Resource, - trailers: Option>, - ) -> wasmtime::Result<()> { - latest::http::types::HostOutgoingBody::finish(self, this, trailers)?; - Ok(()) - } - - fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { - latest::http::types::HostOutgoingBody::drop(self, rep) - } -} - -impl wasi::http::types::HostResponseOutparam for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn set( - &mut self, - param: wasmtime::component::Resource, - response: Result, HttpError>, - ) -> wasmtime::Result<()> { - let response = response.map_err(|err| { - // TODO: probably need to figure out a better mapping between - // errors, but that seems like it would require string matching, - // which also seems not great. - let msg = match err { - HttpError::InvalidUrl(s) => format!("invalid url: {s}"), - HttpError::TimeoutError(s) => format!("timeout: {s}"), - HttpError::ProtocolError(s) => format!("protocol error: {s}"), - HttpError::UnexpectedError(s) => format!("unexpected error: {s}"), - }; - latest::http::types::ErrorCode::InternalError(Some(msg)) - }); - latest::http::types::HostResponseOutparam::set(self, param, response) - } - - fn drop( - &mut self, - rep: wasmtime::component::Resource, - ) -> wasmtime::Result<()> { - latest::http::types::HostResponseOutparam::drop(self, rep) - } -} - -impl wasi::http::types::HostFutureTrailers for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn subscribe( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostFutureTrailers::subscribe(self, self_) - } - - fn get( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result, HttpError>>> { - match latest::http::types::HostFutureTrailers::get(self, self_)? { - Some(Ok(Ok(Some(trailers)))) => Ok(Some(Ok(trailers))), - // Return an empty trailers if no trailers popped out since this - // version of WASI couldn't represent the lack of trailers. - Some(Ok(Ok(None))) => Ok(Some(Ok(latest::http::types::HostFields::new(self)?))), - Some(Ok(Err(e))) => Ok(Some(Err(e.into()))), - Some(Err(())) => Err(anyhow::anyhow!("trailers have already been retrieved")), - None => Ok(None), - } - } - - fn drop(&mut self, rep: wasmtime::component::Resource) -> wasmtime::Result<()> { - latest::http::types::HostFutureTrailers::drop(self, rep) - } -} - -impl wasi::http::types::HostFutureIncomingResponse for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn get( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result< - Option, HttpError>, ()>>, - > { - match latest::http::types::HostFutureIncomingResponse::get(self, self_)? { - None => Ok(None), - Some(Ok(Ok(response))) => Ok(Some(Ok(Ok(response)))), - Some(Ok(Err(e))) => Ok(Some(Ok(Err(e.into())))), - Some(Err(())) => Ok(Some(Err(()))), - } - } - - fn subscribe( - &mut self, - self_: wasmtime::component::Resource, - ) -> wasmtime::Result> { - latest::http::types::HostFutureIncomingResponse::subscribe(self, self_) - } - - fn drop( - &mut self, - rep: wasmtime::component::Resource, - ) -> wasmtime::Result<()> { - latest::http::types::HostFutureIncomingResponse::drop(self, rep) - } -} - -impl wasi::http::outgoing_handler::Host for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn handle( - &mut self, - request: wasmtime::component::Resource, - options: Option, - ) -> wasmtime::Result, HttpError>> - { - let options = match options { - Some(RequestOptions { - connect_timeout_ms, - first_byte_timeout_ms, - between_bytes_timeout_ms, - }) => { - let options = latest::http::types::HostRequestOptions::new(self)?; - let borrow = || Resource::new_borrow(request.rep()); - - if let Some(ms) = connect_timeout_ms { - if let Err(()) = latest::http::types::HostRequestOptions::set_connect_timeout( - self, - borrow(), - Some(ms.into()), - )? { - latest::http::types::HostRequestOptions::drop(self, options)?; - anyhow::bail!("invalid connect timeout supplied"); - } - } - - if let Some(ms) = first_byte_timeout_ms { - if let Err(()) = - latest::http::types::HostRequestOptions::set_first_byte_timeout( - self, - borrow(), - Some(ms.into()), - )? - { - latest::http::types::HostRequestOptions::drop(self, options)?; - anyhow::bail!("invalid first byte timeout supplied"); - } - } - - if let Some(ms) = between_bytes_timeout_ms { - if let Err(()) = - latest::http::types::HostRequestOptions::set_between_bytes_timeout( - self, - borrow(), - Some(ms.into()), - )? - { - latest::http::types::HostRequestOptions::drop(self, options)?; - anyhow::bail!("invalid between bytes timeout supplied"); - } - } - - Some(options) - } - None => None, - }; - match latest::http::outgoing_handler::Host::handle(self, request, options) { - Ok(resp) => Ok(Ok(resp)), - Err(e) => Ok(Err(e.downcast()?.into())), - } - } -} - -pub fn convert_result( - result: Result>, -) -> wasmtime::Result> -where - T2: From, - E: std::error::Error + Send + Sync + 'static, - E2: From, -{ - match result { - Ok(e) => Ok(Ok(e.into())), - Err(e) => Ok(Err(e.downcast()?.into())), - } -} - -fn convert_stream_result( - mut view: impl WasiView, - result: Result, -) -> wasmtime::Result> -where - T2: From, -{ - match result { - Ok(e) => Ok(Ok(e.into())), - Err(wasmtime_wasi::StreamError::Closed) => Ok(Err(StreamError::Closed)), - Err(wasmtime_wasi::StreamError::LastOperationFailed(e)) => { - let e = view.table().push(e)?; - Ok(Err(StreamError::LastOperationFailed(e))) - } - Err(wasmtime_wasi::StreamError::Trap(e)) => Err(e), - } -} - -macro_rules! convert { - () => {}; - ($kind:ident $from:path [<=>] $to:path { $($body:tt)* } $($rest:tt)*) => { - convert!($kind $from => $to { $($body)* }); - convert!($kind $to => $from { $($body)* }); - - convert!($($rest)*); - }; - (struct $from:ty => $to:path { $($field:ident,)* } $($rest:tt)*) => { - impl From<$from> for $to { - fn from(e: $from) -> $to { - $to { - $( $field: e.$field.into(), )* - } - } - } - - convert!($($rest)*); - }; - (enum $from:path => $to:path { $($variant:ident $(($e:ident))?,)* } $($rest:tt)*) => { - impl From<$from> for $to { - fn from(e: $from) -> $to { - use $from as A; - use $to as B; - match e { - $( - A::$variant $(($e))? => B::$variant $(($e.into()))?, - )* - } - } - } - - convert!($($rest)*); - }; - (flags $from:path => $to:path { $($flag:ident,)* } $($rest:tt)*) => { - impl From<$from> for $to { - fn from(e: $from) -> $to { - use $from as A; - use $to as B; - let mut out = B::empty(); - $( - if e.contains(A::$flag) { - out |= B::$flag; - } - )* - out - } - } - - convert!($($rest)*); - }; -} - -pub(crate) use convert; - -convert! { - struct latest::clocks::wall_clock::Datetime [<=>] Datetime { - seconds, - nanoseconds, - } - - enum latest::filesystem::types::ErrorCode => FsErrorCode { - Access, - WouldBlock, - Already, - BadDescriptor, - Busy, - Deadlock, - Quota, - Exist, - FileTooLarge, - IllegalByteSequence, - InProgress, - Interrupted, - Invalid, - Io, - IsDirectory, - Loop, - TooManyLinks, - MessageSize, - NameTooLong, - NoDevice, - NoEntry, - NoLock, - InsufficientMemory, - InsufficientSpace, - NotDirectory, - NotEmpty, - NotRecoverable, - Unsupported, - NoTty, - NoSuchDevice, - Overflow, - NotPermitted, - Pipe, - ReadOnly, - InvalidSeek, - TextFileBusy, - CrossDevice, - } - - enum Advice => latest::filesystem::types::Advice { - Normal, - Sequential, - Random, - WillNeed, - DontNeed, - NoReuse, - } - - flags DescriptorFlags [<=>] latest::filesystem::types::DescriptorFlags { - READ, - WRITE, - FILE_INTEGRITY_SYNC, - DATA_INTEGRITY_SYNC, - REQUESTED_WRITE_SYNC, - MUTATE_DIRECTORY, - } - - enum DescriptorType [<=>] latest::filesystem::types::DescriptorType { - Unknown, - BlockDevice, - CharacterDevice, - Directory, - Fifo, - SymbolicLink, - RegularFile, - Socket, - } - - enum NewTimestamp => latest::filesystem::types::NewTimestamp { - NoChange, - Now, - Timestamp(e), - } - - flags PathFlags => latest::filesystem::types::PathFlags { - SYMLINK_FOLLOW, - } - - flags OpenFlags => latest::filesystem::types::OpenFlags { - CREATE, - DIRECTORY, - EXCLUSIVE, - TRUNCATE, - } - - struct latest::filesystem::types::MetadataHashValue => MetadataHashValue { - lower, - upper, - } - - struct latest::filesystem::types::DirectoryEntry => DirectoryEntry { - type_, - name, - } - - enum latest::sockets::network::ErrorCode => SocketErrorCode { - Unknown, - AccessDenied, - NotSupported, - InvalidArgument, - OutOfMemory, - Timeout, - ConcurrencyConflict, - NotInProgress, - WouldBlock, - InvalidState, - NewSocketLimit, - AddressNotBindable, - AddressInUse, - RemoteUnreachable, - ConnectionRefused, - ConnectionReset, - ConnectionAborted, - DatagramTooLarge, - NameUnresolvable, - TemporaryResolverFailure, - PermanentResolverFailure, - } - - enum latest::sockets::network::IpAddress [<=>] IpAddress { - Ipv4(e), - Ipv6(e), - } - - enum latest::sockets::network::IpSocketAddress [<=>] IpSocketAddress { - Ipv4(e), - Ipv6(e), - } - - struct latest::sockets::network::Ipv4SocketAddress [<=>] Ipv4SocketAddress { - port, - address, - } - - struct latest::sockets::network::Ipv6SocketAddress [<=>] Ipv6SocketAddress { - port, - flow_info, - scope_id, - address, - } - - enum latest::sockets::network::IpAddressFamily [<=>] IpAddressFamily { - Ipv4, - Ipv6, - } - - enum ShutdownType => latest::sockets::tcp::ShutdownType { - Receive, - Send, - Both, - } - - struct latest::sockets::udp::IncomingDatagram => Datagram { - data, - remote_address, - } - - enum latest::http::types::Method [<=>] Method { - Get, - Head, - Post, - Put, - Delete, - Connect, - Options, - Trace, - Patch, - Other(e), - } - - enum latest::http::types::Scheme [<=>] Scheme { - Http, - Https, - Other(e), - } -} - -impl From for DescriptorStat { - fn from(e: latest::filesystem::types::DescriptorStat) -> DescriptorStat { - DescriptorStat { - type_: e.type_.into(), - link_count: e.link_count, - size: e.size, - data_access_timestamp: e.data_access_timestamp.map(|e| e.into()), - data_modification_timestamp: e.data_modification_timestamp.map(|e| e.into()), - status_change_timestamp: e.status_change_timestamp.map(|e| e.into()), - } - } -} - -impl From for HttpError { - fn from(e: latest::http::types::ErrorCode) -> HttpError { - // TODO: should probably categorize this better given the typed info - // we have in `e`. - HttpError::UnexpectedError(e.to_string()) - } -} diff --git a/crates/core/src/wasi_2023_11_10.rs b/crates/core/src/wasi_2023_11_10.rs deleted file mode 100644 index a7d554deb..000000000 --- a/crates/core/src/wasi_2023_11_10.rs +++ /dev/null @@ -1,2455 +0,0 @@ -#![doc(hidden)] // internal implementation detail used in tests and spin-trigger - -use super::wasi_2023_10_18::{convert, convert_result}; -use anyhow::Result; -use async_trait::async_trait; -use wasmtime::component::{Linker, Resource}; -use wasmtime_wasi::{WasiImpl, WasiView}; -use wasmtime_wasi_http::{WasiHttpImpl, WasiHttpView}; - -mod latest { - pub use wasmtime_wasi::bindings::*; - pub mod http { - pub use wasmtime_wasi_http::bindings::http::*; - } -} - -mod bindings { - use super::latest; - - wasmtime::component::bindgen!({ - path: "../../wit", - interfaces: r#" - include wasi:http/proxy@0.2.0-rc-2023-11-10; - - // NB: this is handling the historical behavior where Spin supported - // more than "just" this snapshot of the proxy world but additionally - // other CLI-related interfaces. - include wasi:cli/reactor@0.2.0-rc-2023-11-10; - "#, - async: { - only_imports: [ - "[method]descriptor.advise", - "[method]descriptor.create-directory-at", - "[method]descriptor.get-flags", - "[method]descriptor.get-type", - "[method]descriptor.is-same-object", - "[method]descriptor.link-at", - "[method]descriptor.metadata-hash", - "[method]descriptor.metadata-hash-at", - "[method]descriptor.open-at", - "[method]descriptor.read", - "[method]descriptor.read-directory", - "[method]descriptor.readlink-at", - "[method]descriptor.remove-directory-at", - "[method]descriptor.rename-at", - "[method]descriptor.set-size", - "[method]descriptor.set-times", - "[method]descriptor.set-times-at", - "[method]descriptor.stat", - "[method]descriptor.stat-at", - "[method]descriptor.symlink-at", - "[method]descriptor.sync", - "[method]descriptor.sync-data", - "[method]descriptor.unlink-file-at", - "[method]descriptor.write", - "[method]input-stream.read", - "[method]input-stream.blocking-read", - "[method]input-stream.blocking-skip", - "[method]input-stream.skip", - "[method]output-stream.splice", - "[method]output-stream.blocking-splice", - "[method]output-stream.blocking-flush", - "[method]output-stream.blocking-write", - "[method]output-stream.blocking-write-and-flush", - "[method]output-stream.blocking-write-zeroes-and-flush", - "[method]directory-entry-stream.read-directory-entry", - "[method]pollable.block", - "[method]pollable.ready", - "poll", - - "[method]tcp-socket.start-bind", - "[method]tcp-socket.start-connect", - "[method]udp-socket.start-bind", - "[method]udp-socket.stream", - "[method]outgoing-datagram-stream.send", - ] - }, - with: { - "wasi:io/poll/pollable": latest::io::poll::Pollable, - "wasi:io/streams/input-stream": latest::io::streams::InputStream, - "wasi:io/streams/output-stream": latest::io::streams::OutputStream, - "wasi:io/error/error": latest::io::error::Error, - "wasi:filesystem/types/directory-entry-stream": latest::filesystem::types::DirectoryEntryStream, - "wasi:filesystem/types/descriptor": latest::filesystem::types::Descriptor, - "wasi:cli/terminal-input/terminal-input": latest::cli::terminal_input::TerminalInput, - "wasi:cli/terminal-output/terminal-output": latest::cli::terminal_output::TerminalOutput, - "wasi:sockets/tcp/tcp-socket": latest::sockets::tcp::TcpSocket, - "wasi:sockets/udp/udp-socket": latest::sockets::udp::UdpSocket, - "wasi:sockets/udp/outgoing-datagram-stream": latest::sockets::udp::OutgoingDatagramStream, - "wasi:sockets/udp/incoming-datagram-stream": latest::sockets::udp::IncomingDatagramStream, - "wasi:sockets/network/network": latest::sockets::network::Network, - "wasi:sockets/ip-name-lookup/resolve-address-stream": latest::sockets::ip_name_lookup::ResolveAddressStream, - "wasi:http/types/incoming-response": latest::http::types::IncomingResponse, - "wasi:http/types/incoming-request": latest::http::types::IncomingRequest, - "wasi:http/types/incoming-body": latest::http::types::IncomingBody, - "wasi:http/types/outgoing-response": latest::http::types::OutgoingResponse, - "wasi:http/types/outgoing-request": latest::http::types::OutgoingRequest, - "wasi:http/types/outgoing-body": latest::http::types::OutgoingBody, - "wasi:http/types/fields": latest::http::types::Fields, - "wasi:http/types/response-outparam": latest::http::types::ResponseOutparam, - "wasi:http/types/future-incoming-response": latest::http::types::FutureIncomingResponse, - "wasi:http/types/future-trailers": latest::http::types::FutureTrailers, - "wasi:http/types/request-options": latest::http::types::RequestOptions, - }, - trappable_imports: true, - }); -} - -mod wasi { - pub use super::bindings::wasi::{ - cli0_2_0_rc_2023_11_10 as cli, clocks0_2_0_rc_2023_11_10 as clocks, - filesystem0_2_0_rc_2023_11_10 as filesystem, http0_2_0_rc_2023_11_10 as http, - io0_2_0_rc_2023_11_10 as io, random0_2_0_rc_2023_11_10 as random, - sockets0_2_0_rc_2023_11_10 as sockets, - }; -} - -pub mod exports { - pub mod wasi { - pub use super::super::bindings::exports::wasi::http0_2_0_rc_2023_11_10 as http; - } -} - -use wasi::cli::terminal_input::TerminalInput; -use wasi::cli::terminal_output::TerminalOutput; -use wasi::clocks::monotonic_clock::{Duration, Instant}; -use wasi::clocks::wall_clock::Datetime; -use wasi::filesystem::types::{ - Advice, Descriptor, DescriptorFlags, DescriptorStat, DescriptorType, DirectoryEntry, - DirectoryEntryStream, ErrorCode as FsErrorCode, Filesize, MetadataHashValue, NewTimestamp, - OpenFlags, PathFlags, -}; -use wasi::http::types::{ - DnsErrorPayload, ErrorCode as HttpErrorCode, FieldSizePayload, Fields, FutureIncomingResponse, - FutureTrailers, HeaderError, Headers, IncomingBody, IncomingRequest, IncomingResponse, Method, - OutgoingBody, OutgoingRequest, OutgoingResponse, RequestOptions, ResponseOutparam, Scheme, - StatusCode, TlsAlertReceivedPayload, Trailers, -}; -use wasi::io::poll::Pollable; -use wasi::io::streams::{Error as IoError, InputStream, OutputStream, StreamError}; -use wasi::sockets::ip_name_lookup::{IpAddress, ResolveAddressStream}; -use wasi::sockets::network::{Ipv4SocketAddress, Ipv6SocketAddress}; -use wasi::sockets::tcp::{ - ErrorCode as SocketErrorCode, IpAddressFamily, IpSocketAddress, Network, ShutdownType, - TcpSocket, -}; -use wasi::sockets::udp::{ - IncomingDatagram, IncomingDatagramStream, OutgoingDatagram, OutgoingDatagramStream, UdpSocket, -}; - -pub fn add_to_linker(linker: &mut Linker) -> Result<()> -where - T: WasiView + WasiHttpView, -{ - // interfaces from the "command" world - fn type_annotate_wasi(f: F) -> F - where - F: Fn(&mut T) -> WasiImpl<&mut T>, - { - f - } - let closure = type_annotate_wasi::(|t| WasiImpl(t)); - wasi::clocks::monotonic_clock::add_to_linker_get_host(linker, closure)?; - wasi::clocks::wall_clock::add_to_linker_get_host(linker, closure)?; - wasi::filesystem::types::add_to_linker_get_host(linker, closure)?; - wasi::filesystem::preopens::add_to_linker_get_host(linker, closure)?; - wasi::io::error::add_to_linker_get_host(linker, closure)?; - wasi::io::poll::add_to_linker_get_host(linker, closure)?; - wasi::io::streams::add_to_linker_get_host(linker, closure)?; - wasi::random::random::add_to_linker_get_host(linker, closure)?; - wasi::random::insecure::add_to_linker_get_host(linker, closure)?; - wasi::random::insecure_seed::add_to_linker_get_host(linker, closure)?; - wasi::cli::exit::add_to_linker_get_host(linker, closure)?; - wasi::cli::environment::add_to_linker_get_host(linker, closure)?; - wasi::cli::stdin::add_to_linker_get_host(linker, closure)?; - wasi::cli::stdout::add_to_linker_get_host(linker, closure)?; - wasi::cli::stderr::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_input::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_output::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_stdin::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_stdout::add_to_linker_get_host(linker, closure)?; - wasi::cli::terminal_stderr::add_to_linker_get_host(linker, closure)?; - wasi::sockets::tcp::add_to_linker_get_host(linker, closure)?; - wasi::sockets::tcp_create_socket::add_to_linker_get_host(linker, closure)?; - wasi::sockets::udp::add_to_linker_get_host(linker, closure)?; - wasi::sockets::udp_create_socket::add_to_linker_get_host(linker, closure)?; - wasi::sockets::instance_network::add_to_linker_get_host(linker, closure)?; - wasi::sockets::network::add_to_linker_get_host(linker, closure)?; - wasi::sockets::ip_name_lookup::add_to_linker_get_host(linker, closure)?; - - fn type_annotate_http(f: F) -> F - where - F: Fn(&mut T) -> WasiHttpImpl<&mut T>, - { - f - } - let closure = type_annotate_http::(|t| WasiHttpImpl(t)); - wasi::http::types::add_to_linker_get_host(linker, closure)?; - wasi::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; - Ok(()) -} - -impl wasi::clocks::monotonic_clock::Host for WasiImpl -where - T: WasiView, -{ - fn now(&mut self) -> wasmtime::Result { - latest::clocks::monotonic_clock::Host::now(self) - } - - fn resolution(&mut self) -> wasmtime::Result { - latest::clocks::monotonic_clock::Host::resolution(self) - } - - fn subscribe_instant(&mut self, when: Instant) -> wasmtime::Result> { - latest::clocks::monotonic_clock::Host::subscribe_instant(self, when) - } - - fn subscribe_duration(&mut self, when: Duration) -> wasmtime::Result> { - latest::clocks::monotonic_clock::Host::subscribe_duration(self, when) - } -} - -impl wasi::clocks::wall_clock::Host for WasiImpl -where - T: WasiView, -{ - fn now(&mut self) -> wasmtime::Result { - Ok(latest::clocks::wall_clock::Host::now(self)?.into()) - } - - fn resolution(&mut self) -> wasmtime::Result { - Ok(latest::clocks::wall_clock::Host::resolution(self)?.into()) - } -} - -impl wasi::filesystem::types::Host for WasiImpl -where - T: WasiView, -{ - fn filesystem_error_code( - &mut self, - err: Resource, - ) -> wasmtime::Result> { - Ok(latest::filesystem::types::Host::filesystem_error_code(self, err)?.map(|e| e.into())) - } -} - -#[async_trait] -impl wasi::filesystem::types::HostDescriptor for WasiImpl -where - T: WasiView, -{ - fn read_via_stream( - &mut self, - self_: Resource, - offset: Filesize, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::read_via_stream( - self, self_, offset, - )) - } - - fn write_via_stream( - &mut self, - self_: Resource, - offset: Filesize, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::write_via_stream( - self, self_, offset, - )) - } - - fn append_via_stream( - &mut self, - self_: Resource, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::append_via_stream(self, self_)) - } - - async fn advise( - &mut self, - self_: Resource, - offset: Filesize, - length: Filesize, - advice: Advice, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::advise( - self, - self_, - offset, - length, - advice.into(), - ) - .await, - ) - } - - async fn sync_data( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::sync_data(self, self_).await) - } - - async fn get_flags( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::get_flags(self, self_).await) - } - - async fn get_type( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::get_type(self, self_).await) - } - - async fn set_size( - &mut self, - self_: Resource, - size: Filesize, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::set_size(self, self_, size).await) - } - - async fn set_times( - &mut self, - self_: Resource, - data_access_timestamp: NewTimestamp, - data_modification_timestamp: NewTimestamp, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::set_times( - self, - self_, - data_access_timestamp.into(), - data_modification_timestamp.into(), - ) - .await, - ) - } - - async fn read( - &mut self, - self_: Resource, - length: Filesize, - offset: Filesize, - ) -> wasmtime::Result, bool), FsErrorCode>> { - convert_result( - latest::filesystem::types::HostDescriptor::read(self, self_, length, offset).await, - ) - } - - async fn write( - &mut self, - self_: Resource, - buffer: Vec, - offset: Filesize, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::write(self, self_, buffer, offset).await, - ) - } - - async fn read_directory( - &mut self, - self_: Resource, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result(latest::filesystem::types::HostDescriptor::read_directory(self, self_).await) - } - - async fn sync( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::sync(self, self_).await) - } - - async fn create_directory_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::create_directory_at(self, self_, path).await, - ) - } - - async fn stat( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::stat(self, self_).await) - } - - async fn stat_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::stat_at( - self, - self_, - path_flags.into(), - path, - ) - .await, - ) - } - - async fn set_times_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - data_access_timestamp: NewTimestamp, - data_modification_timestamp: NewTimestamp, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::set_times_at( - self, - self_, - path_flags.into(), - path, - data_access_timestamp.into(), - data_modification_timestamp.into(), - ) - .await, - ) - } - - async fn link_at( - &mut self, - self_: Resource, - old_path_flags: PathFlags, - old_path: String, - new_descriptor: Resource, - new_path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::link_at( - self, - self_, - old_path_flags.into(), - old_path, - new_descriptor, - new_path, - ) - .await, - ) - } - - async fn open_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - open_flags: OpenFlags, - flags: DescriptorFlags, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result( - latest::filesystem::types::HostDescriptor::open_at( - self, - self_, - path_flags.into(), - path, - open_flags.into(), - flags.into(), - ) - .await, - ) - } - - async fn readlink_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::readlink_at(self, self_, path).await, - ) - } - - async fn remove_directory_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::remove_directory_at(self, self_, path).await, - ) - } - - async fn rename_at( - &mut self, - self_: Resource, - old_path: String, - new_descriptor: Resource, - new_path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::rename_at( - self, - self_, - old_path, - new_descriptor, - new_path, - ) - .await, - ) - } - - async fn symlink_at( - &mut self, - self_: Resource, - old_path: String, - new_path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::symlink_at(self, self_, old_path, new_path) - .await, - ) - } - - async fn unlink_file_at( - &mut self, - self_: Resource, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::unlink_file_at(self, self_, path).await, - ) - } - - async fn is_same_object( - &mut self, - self_: Resource, - other: Resource, - ) -> wasmtime::Result { - latest::filesystem::types::HostDescriptor::is_same_object(self, self_, other).await - } - - async fn metadata_hash( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::filesystem::types::HostDescriptor::metadata_hash(self, self_).await) - } - - async fn metadata_hash_at( - &mut self, - self_: Resource, - path_flags: PathFlags, - path: String, - ) -> wasmtime::Result> { - convert_result( - latest::filesystem::types::HostDescriptor::metadata_hash_at( - self, - self_, - path_flags.into(), - path, - ) - .await, - ) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::filesystem::types::HostDescriptor::drop(self, rep) - } -} - -#[async_trait] -impl wasi::filesystem::types::HostDirectoryEntryStream for WasiImpl -where - T: WasiView, -{ - async fn read_directory_entry( - &mut self, - self_: Resource, - ) -> wasmtime::Result, FsErrorCode>> { - convert_result( - latest::filesystem::types::HostDirectoryEntryStream::read_directory_entry(self, self_) - .await - .map(|e| e.map(DirectoryEntry::from)), - ) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::filesystem::types::HostDirectoryEntryStream::drop(self, rep) - } -} - -impl wasi::filesystem::preopens::Host for WasiImpl -where - T: WasiView, -{ - fn get_directories(&mut self) -> wasmtime::Result, String)>> { - latest::filesystem::preopens::Host::get_directories(self) - } -} - -#[async_trait] -impl wasi::io::poll::Host for WasiImpl -where - T: WasiView, -{ - async fn poll(&mut self, list: Vec>) -> wasmtime::Result> { - latest::io::poll::Host::poll(self, list).await - } -} - -#[async_trait] -impl wasi::io::poll::HostPollable for WasiImpl -where - T: WasiView, -{ - async fn block(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::poll::HostPollable::block(self, rep).await - } - - async fn ready(&mut self, rep: Resource) -> wasmtime::Result { - latest::io::poll::HostPollable::ready(self, rep).await - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::poll::HostPollable::drop(self, rep) - } -} - -impl wasi::io::error::Host for WasiImpl where T: WasiView {} - -impl wasi::io::error::HostError for WasiImpl -where - T: WasiView, -{ - fn to_debug_string(&mut self, self_: Resource) -> wasmtime::Result { - latest::io::error::HostError::to_debug_string(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::error::HostError::drop(self, rep) - } -} - -fn convert_stream_result( - mut view: impl WasiView, - result: Result, -) -> wasmtime::Result> -where - T2: From, -{ - match result { - Ok(e) => Ok(Ok(e.into())), - Err(wasmtime_wasi::StreamError::Closed) => Ok(Err(StreamError::Closed)), - Err(wasmtime_wasi::StreamError::LastOperationFailed(e)) => { - let e = view.table().push(e)?; - Ok(Err(StreamError::LastOperationFailed(e))) - } - Err(wasmtime_wasi::StreamError::Trap(e)) => Err(e), - } -} - -impl wasi::io::streams::Host for WasiImpl where T: WasiView {} - -#[async_trait] -impl wasi::io::streams::HostInputStream for WasiImpl -where - T: WasiView, -{ - async fn read( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result, StreamError>> { - let result = latest::io::streams::HostInputStream::read(self, self_, len).await; - convert_stream_result(self, result) - } - - async fn blocking_read( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result, StreamError>> { - let result = latest::io::streams::HostInputStream::blocking_read(self, self_, len).await; - convert_stream_result(self, result) - } - - async fn skip( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostInputStream::skip(self, self_, len).await; - convert_stream_result(self, result) - } - - async fn blocking_skip( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostInputStream::blocking_skip(self, self_, len).await; - convert_stream_result(self, result) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - latest::io::streams::HostInputStream::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::streams::HostInputStream::drop(self, rep) - } -} - -#[async_trait] -impl wasi::io::streams::HostOutputStream for WasiImpl -where - T: WasiView, -{ - fn check_write( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::check_write(self, self_); - convert_stream_result(self, result) - } - - fn write( - &mut self, - self_: Resource, - contents: Vec, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::write(self, self_, contents); - convert_stream_result(self, result) - } - - async fn blocking_write_and_flush( - &mut self, - self_: Resource, - contents: Vec, - ) -> wasmtime::Result> { - let result = - latest::io::streams::HostOutputStream::blocking_write_and_flush(self, self_, contents) - .await; - convert_stream_result(self, result) - } - - fn flush( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::flush(self, self_); - convert_stream_result(self, result) - } - - async fn blocking_flush( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::blocking_flush(self, self_).await; - convert_stream_result(self, result) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - latest::io::streams::HostOutputStream::subscribe(self, self_) - } - - fn write_zeroes( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::write_zeroes(self, self_, len); - convert_stream_result(self, result) - } - - async fn blocking_write_zeroes_and_flush( - &mut self, - self_: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::blocking_write_zeroes_and_flush( - self, self_, len, - ) - .await; - convert_stream_result(self, result) - } - - async fn splice( - &mut self, - self_: Resource, - src: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = latest::io::streams::HostOutputStream::splice(self, self_, src, len).await; - convert_stream_result(self, result) - } - - async fn blocking_splice( - &mut self, - self_: Resource, - src: Resource, - len: u64, - ) -> wasmtime::Result> { - let result = - latest::io::streams::HostOutputStream::blocking_splice(self, self_, src, len).await; - convert_stream_result(self, result) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::io::streams::HostOutputStream::drop(self, rep) - } -} - -impl wasi::random::random::Host for WasiImpl -where - T: WasiView, -{ - fn get_random_bytes(&mut self, len: u64) -> wasmtime::Result> { - latest::random::random::Host::get_random_bytes(self, len) - } - - fn get_random_u64(&mut self) -> wasmtime::Result { - latest::random::random::Host::get_random_u64(self) - } -} - -impl wasi::random::insecure::Host for WasiImpl -where - T: WasiView, -{ - fn get_insecure_random_bytes(&mut self, len: u64) -> wasmtime::Result> { - latest::random::insecure::Host::get_insecure_random_bytes(self, len) - } - - fn get_insecure_random_u64(&mut self) -> wasmtime::Result { - latest::random::insecure::Host::get_insecure_random_u64(self) - } -} - -impl wasi::random::insecure_seed::Host for WasiImpl -where - T: WasiView, -{ - fn insecure_seed(&mut self) -> wasmtime::Result<(u64, u64)> { - latest::random::insecure_seed::Host::insecure_seed(self) - } -} - -impl wasi::cli::exit::Host for WasiImpl -where - T: WasiView, -{ - fn exit(&mut self, status: Result<(), ()>) -> wasmtime::Result<()> { - latest::cli::exit::Host::exit(self, status) - } -} - -impl wasi::cli::environment::Host for WasiImpl -where - T: WasiView, -{ - fn get_environment(&mut self) -> wasmtime::Result> { - latest::cli::environment::Host::get_environment(self) - } - - fn get_arguments(&mut self) -> wasmtime::Result> { - latest::cli::environment::Host::get_arguments(self) - } - - fn initial_cwd(&mut self) -> wasmtime::Result> { - latest::cli::environment::Host::initial_cwd(self) - } -} - -impl wasi::cli::stdin::Host for WasiImpl -where - T: WasiView, -{ - fn get_stdin(&mut self) -> wasmtime::Result> { - latest::cli::stdin::Host::get_stdin(self) - } -} - -impl wasi::cli::stdout::Host for WasiImpl -where - T: WasiView, -{ - fn get_stdout(&mut self) -> wasmtime::Result> { - latest::cli::stdout::Host::get_stdout(self) - } -} - -impl wasi::cli::stderr::Host for WasiImpl -where - T: WasiView, -{ - fn get_stderr(&mut self) -> wasmtime::Result> { - latest::cli::stderr::Host::get_stderr(self) - } -} - -impl wasi::cli::terminal_stdin::Host for WasiImpl -where - T: WasiView, -{ - fn get_terminal_stdin(&mut self) -> wasmtime::Result>> { - latest::cli::terminal_stdin::Host::get_terminal_stdin(self) - } -} - -impl wasi::cli::terminal_stdout::Host for WasiImpl -where - T: WasiView, -{ - fn get_terminal_stdout(&mut self) -> wasmtime::Result>> { - latest::cli::terminal_stdout::Host::get_terminal_stdout(self) - } -} - -impl wasi::cli::terminal_stderr::Host for WasiImpl -where - T: WasiView, -{ - fn get_terminal_stderr(&mut self) -> wasmtime::Result>> { - latest::cli::terminal_stderr::Host::get_terminal_stderr(self) - } -} - -impl wasi::cli::terminal_input::Host for WasiImpl where T: WasiView {} - -impl wasi::cli::terminal_input::HostTerminalInput for WasiImpl -where - T: WasiView, -{ - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::cli::terminal_input::HostTerminalInput::drop(self, rep) - } -} - -impl wasi::cli::terminal_output::Host for WasiImpl where T: WasiView {} - -impl wasi::cli::terminal_output::HostTerminalOutput for WasiImpl -where - T: WasiView, -{ - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::cli::terminal_output::HostTerminalOutput::drop(self, rep) - } -} - -impl wasi::sockets::tcp::Host for WasiImpl where T: WasiView {} - -#[async_trait] -impl wasi::sockets::tcp::HostTcpSocket for WasiImpl -where - T: WasiView, -{ - async fn start_bind( - &mut self, - self_: Resource, - network: Resource, - local_address: IpSocketAddress, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::start_bind( - self, - self_, - network, - local_address.into(), - ) - .await, - ) - } - - fn finish_bind( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::finish_bind( - self, self_, - )) - } - - async fn start_connect( - &mut self, - self_: Resource, - network: Resource, - remote_address: IpSocketAddress, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::start_connect( - self, - self_, - network, - remote_address.into(), - ) - .await, - ) - } - - fn finish_connect( - &mut self, - self_: Resource, - ) -> wasmtime::Result, Resource), SocketErrorCode>> - { - convert_result(latest::sockets::tcp::HostTcpSocket::finish_connect( - self, self_, - )) - } - - fn start_listen( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::start_listen( - self, self_, - )) - } - - fn finish_listen( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::finish_listen( - self, self_, - )) - } - - fn accept( - &mut self, - self_: Resource, - ) -> wasmtime::Result< - Result< - ( - Resource, - Resource, - Resource, - ), - SocketErrorCode, - >, - > { - convert_result(latest::sockets::tcp::HostTcpSocket::accept(self, self_)) - } - - fn local_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::local_address( - self, self_, - )) - } - - fn remote_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::remote_address( - self, self_, - )) - } - - fn address_family(&mut self, self_: Resource) -> wasmtime::Result { - latest::sockets::tcp::HostTcpSocket::address_family(self, self_).map(|e| e.into()) - } - - fn ipv6_only( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn set_ipv6_only( - &mut self, - _self_: Resource, - _value: bool, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn set_listen_backlog_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::set_listen_backlog_size(self, self_, value), - ) - } - - fn is_listening(&mut self, self_: Resource) -> wasmtime::Result { - latest::sockets::tcp::HostTcpSocket::is_listening(self, self_) - } - - fn keep_alive_enabled( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_enabled( - self, self_, - )) - } - - fn set_keep_alive_enabled( - &mut self, - self_: Resource, - value: bool, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::set_keep_alive_enabled( - self, self_, value, - )) - } - - fn keep_alive_idle_time( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_idle_time( - self, self_, - )) - } - - fn set_keep_alive_idle_time( - &mut self, - self_: Resource, - value: Duration, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::set_keep_alive_idle_time(self, self_, value), - ) - } - - fn keep_alive_interval( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_interval( - self, self_, - )) - } - - fn set_keep_alive_interval( - &mut self, - self_: Resource, - value: Duration, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::set_keep_alive_interval(self, self_, value), - ) - } - - fn keep_alive_count( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::keep_alive_count( - self, self_, - )) - } - - fn set_keep_alive_count( - &mut self, - self_: Resource, - value: u32, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::set_keep_alive_count( - self, self_, value, - )) - } - - fn hop_limit( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::hop_limit(self, self_)) - } - - fn set_hop_limit( - &mut self, - self_: Resource, - value: u8, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::set_hop_limit( - self, self_, value, - )) - } - - fn receive_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::receive_buffer_size( - self, self_, - )) - } - - fn set_receive_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::tcp::HostTcpSocket::set_receive_buffer_size(self, self_, value), - ) - } - - fn send_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::send_buffer_size( - self, self_, - )) - } - - fn set_send_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::set_send_buffer_size( - self, self_, value, - )) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - latest::sockets::tcp::HostTcpSocket::subscribe(self, self_) - } - - fn shutdown( - &mut self, - self_: Resource, - shutdown_type: ShutdownType, - ) -> wasmtime::Result> { - convert_result(latest::sockets::tcp::HostTcpSocket::shutdown( - self, - self_, - shutdown_type.into(), - )) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::tcp::HostTcpSocket::drop(self, rep) - } -} - -impl wasi::sockets::tcp_create_socket::Host for WasiImpl -where - T: WasiView, -{ - fn create_tcp_socket( - &mut self, - address_family: IpAddressFamily, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result(latest::sockets::tcp_create_socket::Host::create_tcp_socket( - self, - address_family.into(), - )) - } -} - -impl wasi::sockets::udp::Host for WasiImpl where T: WasiView {} - -#[async_trait] -impl wasi::sockets::udp::HostUdpSocket for WasiImpl -where - T: WasiView, -{ - async fn start_bind( - &mut self, - self_: Resource, - network: Resource, - local_address: IpSocketAddress, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::udp::HostUdpSocket::start_bind( - self, - self_, - network, - local_address.into(), - ) - .await, - ) - } - - fn finish_bind( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::finish_bind( - self, self_, - )) - } - - fn local_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::local_address( - self, self_, - )) - } - - fn remote_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::remote_address( - self, self_, - )) - } - - fn address_family(&mut self, self_: Resource) -> wasmtime::Result { - latest::sockets::udp::HostUdpSocket::address_family(self, self_).map(|e| e.into()) - } - - fn ipv6_only( - &mut self, - _self_: Resource, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn set_ipv6_only( - &mut self, - _self_: Resource, - _value: bool, - ) -> wasmtime::Result> { - anyhow::bail!("ipv6-only API no longer supported") - } - - fn unicast_hop_limit( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::unicast_hop_limit( - self, self_, - )) - } - - fn set_unicast_hop_limit( - &mut self, - self_: Resource, - value: u8, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::set_unicast_hop_limit( - self, self_, value, - )) - } - - fn receive_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::receive_buffer_size( - self, self_, - )) - } - - fn set_receive_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::udp::HostUdpSocket::set_receive_buffer_size(self, self_, value), - ) - } - - fn send_buffer_size( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::send_buffer_size( - self, self_, - )) - } - - fn set_send_buffer_size( - &mut self, - self_: Resource, - value: u64, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostUdpSocket::set_send_buffer_size( - self, self_, value, - )) - } - - async fn stream( - &mut self, - self_: Resource, - remote_address: Option, - ) -> wasmtime::Result< - Result< - ( - Resource, - Resource, - ), - SocketErrorCode, - >, - > { - convert_result( - latest::sockets::udp::HostUdpSocket::stream( - self, - self_, - remote_address.map(|a| a.into()), - ) - .await, - ) - } - - fn subscribe(&mut self, self_: Resource) -> wasmtime::Result> { - latest::sockets::udp::HostUdpSocket::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::udp::HostUdpSocket::drop(self, rep) - } -} - -#[async_trait] -impl wasi::sockets::udp::HostOutgoingDatagramStream for WasiImpl -where - T: WasiView, -{ - fn check_send( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - convert_result(latest::sockets::udp::HostOutgoingDatagramStream::check_send(self, self_)) - } - - async fn send( - &mut self, - self_: Resource, - datagrams: Vec, - ) -> wasmtime::Result> { - convert_result( - latest::sockets::udp::HostOutgoingDatagramStream::send( - self, - self_, - datagrams.into_iter().map(|d| d.into()).collect(), - ) - .await, - ) - } - - fn subscribe( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::sockets::udp::HostOutgoingDatagramStream::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::udp::HostOutgoingDatagramStream::drop(self, rep) - } -} - -impl wasi::sockets::udp::HostIncomingDatagramStream for WasiImpl -where - T: WasiView, -{ - fn receive( - &mut self, - self_: Resource, - max_results: u64, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result(latest::sockets::udp::HostIncomingDatagramStream::receive( - self, - self_, - max_results, - )) - .map(|r| r.map(|r: Vec<_>| r.into_iter().map(|d| d.into()).collect())) - } - - fn subscribe( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::sockets::udp::HostIncomingDatagramStream::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::udp::HostIncomingDatagramStream::drop(self, rep) - } -} - -impl wasi::sockets::udp_create_socket::Host for WasiImpl -where - T: WasiView, -{ - fn create_udp_socket( - &mut self, - address_family: IpAddressFamily, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result(latest::sockets::udp_create_socket::Host::create_udp_socket( - self, - address_family.into(), - )) - } -} - -impl wasi::sockets::instance_network::Host for WasiImpl -where - T: WasiView, -{ - fn instance_network(&mut self) -> wasmtime::Result> { - latest::sockets::instance_network::Host::instance_network(self) - } -} - -impl wasi::sockets::network::Host for WasiImpl where T: WasiView {} - -impl wasi::sockets::network::HostNetwork for WasiImpl -where - T: WasiView, -{ - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::network::HostNetwork::drop(self, rep) - } -} - -impl wasi::sockets::ip_name_lookup::Host for WasiImpl -where - T: WasiView, -{ - fn resolve_addresses( - &mut self, - network: Resource, - name: String, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result(latest::sockets::ip_name_lookup::Host::resolve_addresses( - self, network, name, - )) - } -} - -impl wasi::sockets::ip_name_lookup::HostResolveAddressStream for WasiImpl -where - T: WasiView, -{ - fn resolve_next_address( - &mut self, - self_: Resource, - ) -> wasmtime::Result, SocketErrorCode>> { - convert_result( - latest::sockets::ip_name_lookup::HostResolveAddressStream::resolve_next_address( - self, self_, - ) - .map(|e| e.map(|e| e.into())), - ) - } - - fn subscribe( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::sockets::ip_name_lookup::HostResolveAddressStream::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::sockets::ip_name_lookup::HostResolveAddressStream::drop(self, rep) - } -} - -impl wasi::http::types::Host for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn http_error_code( - &mut self, - error: Resource, - ) -> wasmtime::Result> { - latest::http::types::Host::http_error_code(self, error).map(|e| e.map(|e| e.into())) - } -} - -impl wasi::http::types::HostRequestOptions for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn new(&mut self) -> wasmtime::Result> { - latest::http::types::HostRequestOptions::new(self) - } - - fn connect_timeout_ms( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostRequestOptions::connect_timeout(self, self_) - } - - fn set_connect_timeout_ms( - &mut self, - self_: Resource, - duration: Option, - ) -> wasmtime::Result> { - latest::http::types::HostRequestOptions::set_connect_timeout(self, self_, duration) - } - - fn first_byte_timeout_ms( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostRequestOptions::first_byte_timeout(self, self_) - } - - fn set_first_byte_timeout_ms( - &mut self, - self_: Resource, - duration: Option, - ) -> wasmtime::Result> { - latest::http::types::HostRequestOptions::set_first_byte_timeout(self, self_, duration) - } - - fn between_bytes_timeout_ms( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostRequestOptions::between_bytes_timeout(self, self_) - } - - fn set_between_bytes_timeout_ms( - &mut self, - self_: Resource, - duration: Option, - ) -> wasmtime::Result> { - latest::http::types::HostRequestOptions::set_between_bytes_timeout(self, self_, duration) - } - - fn drop(&mut self, self_: Resource) -> wasmtime::Result<()> { - latest::http::types::HostRequestOptions::drop(self, self_) - } -} - -impl wasi::http::types::HostFields for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn new(&mut self) -> wasmtime::Result> { - latest::http::types::HostFields::new(self) - } - - fn from_list( - &mut self, - entries: Vec<(String, Vec)>, - ) -> wasmtime::Result, HeaderError>> { - latest::http::types::HostFields::from_list(self, entries).map(|r| r.map_err(|e| e.into())) - } - - fn get(&mut self, self_: Resource, name: String) -> wasmtime::Result>> { - latest::http::types::HostFields::get(self, self_, name) - } - - fn set( - &mut self, - self_: Resource, - name: String, - value: Vec>, - ) -> wasmtime::Result> { - latest::http::types::HostFields::set(self, self_, name, value) - .map(|r| r.map_err(|e| e.into())) - } - - fn delete( - &mut self, - self_: Resource, - name: String, - ) -> wasmtime::Result> { - latest::http::types::HostFields::delete(self, self_, name).map(|r| r.map_err(|e| e.into())) - } - - fn append( - &mut self, - self_: Resource, - name: String, - value: Vec, - ) -> wasmtime::Result> { - latest::http::types::HostFields::append(self, self_, name, value) - .map(|r| r.map_err(|e| e.into())) - } - - fn entries(&mut self, self_: Resource) -> wasmtime::Result)>> { - latest::http::types::HostFields::entries(self, self_) - } - - fn clone(&mut self, self_: Resource) -> wasmtime::Result> { - latest::http::types::HostFields::clone(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostFields::drop(self, rep) - } -} - -impl wasi::http::types::HostIncomingRequest for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn method(&mut self, self_: Resource) -> wasmtime::Result { - latest::http::types::HostIncomingRequest::method(self, self_).map(|e| e.into()) - } - - fn path_with_query( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::path_with_query(self, self_) - } - - fn scheme(&mut self, self_: Resource) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::scheme(self, self_).map(|e| e.map(|e| e.into())) - } - - fn authority(&mut self, self_: Resource) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::authority(self, self_) - } - - fn headers(&mut self, self_: Resource) -> wasmtime::Result> { - latest::http::types::HostIncomingRequest::headers(self, self_) - } - - fn consume( - &mut self, - self_: Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostIncomingRequest::consume(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostIncomingRequest::drop(self, rep) - } -} - -impl wasi::http::types::HostIncomingResponse for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn status(&mut self, self_: Resource) -> wasmtime::Result { - latest::http::types::HostIncomingResponse::status(self, self_) - } - - fn headers( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingResponse::headers(self, self_) - } - - fn consume( - &mut self, - self_: Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostIncomingResponse::consume(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostIncomingResponse::drop(self, rep) - } -} - -impl wasi::http::types::HostIncomingBody for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn stream( - &mut self, - self_: Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostIncomingBody::stream(self, self_) - } - - fn finish( - &mut self, - this: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostIncomingBody::finish(self, this) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostIncomingBody::drop(self, rep) - } -} - -impl wasi::http::types::HostOutgoingRequest for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn new(&mut self, headers: Resource) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::new(self, headers) - } - - fn method(&mut self, self_: Resource) -> wasmtime::Result { - latest::http::types::HostOutgoingRequest::method(self, self_).map(|m| m.into()) - } - - fn set_method( - &mut self, - self_: Resource, - method: Method, - ) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::set_method(self, self_, method.into()) - } - - fn path_with_query( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::path_with_query(self, self_) - } - - fn set_path_with_query( - &mut self, - self_: Resource, - path_with_query: Option, - ) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::set_path_with_query(self, self_, path_with_query) - } - - fn scheme(&mut self, self_: Resource) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::scheme(self, self_).map(|s| s.map(|s| s.into())) - } - - fn set_scheme( - &mut self, - self_: Resource, - scheme: Option, - ) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::set_scheme(self, self_, scheme.map(|s| s.into())) - } - - fn authority(&mut self, self_: Resource) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::authority(self, self_) - } - - fn set_authority( - &mut self, - self_: Resource, - authority: Option, - ) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::set_authority(self, self_, authority) - } - - fn headers(&mut self, self_: Resource) -> wasmtime::Result> { - latest::http::types::HostOutgoingRequest::headers(self, self_) - } - - fn body( - &mut self, - self_: Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostOutgoingRequest::body(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostOutgoingRequest::drop(self, rep) - } -} - -impl wasi::http::types::HostOutgoingResponse for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn new(&mut self, headers: Resource) -> wasmtime::Result> { - let headers = latest::http::types::HostFields::clone(self, headers)?; - latest::http::types::HostOutgoingResponse::new(self, headers) - } - - fn status_code(&mut self, self_: Resource) -> wasmtime::Result { - latest::http::types::HostOutgoingResponse::status_code(self, self_) - } - - fn set_status_code( - &mut self, - self_: Resource, - status_code: StatusCode, - ) -> wasmtime::Result> { - latest::http::types::HostOutgoingResponse::set_status_code(self, self_, status_code) - } - - fn headers( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostOutgoingResponse::headers(self, self_) - } - - fn body( - &mut self, - self_: Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostOutgoingResponse::body(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostOutgoingResponse::drop(self, rep) - } -} - -impl wasi::http::types::HostOutgoingBody for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn write( - &mut self, - self_: Resource, - ) -> wasmtime::Result, ()>> { - latest::http::types::HostOutgoingBody::write(self, self_) - } - - fn finish( - &mut self, - this: Resource, - trailers: Option>, - ) -> wasmtime::Result> { - match latest::http::types::HostOutgoingBody::finish(self, this, trailers) { - Ok(()) => Ok(Ok(())), - Err(e) => Ok(Err(e.downcast()?.into())), - } - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostOutgoingBody::drop(self, rep) - } -} - -impl wasi::http::types::HostResponseOutparam for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn set( - &mut self, - param: Resource, - response: Result, HttpErrorCode>, - ) -> wasmtime::Result<()> { - latest::http::types::HostResponseOutparam::set(self, param, response.map_err(|e| e.into())) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostResponseOutparam::drop(self, rep) - } -} - -impl wasi::http::types::HostFutureTrailers for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn subscribe( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostFutureTrailers::subscribe(self, self_) - } - - fn get( - &mut self, - self_: Resource, - ) -> wasmtime::Result>, HttpErrorCode>>> { - match latest::http::types::HostFutureTrailers::get(self, self_)? { - Some(Ok(Ok(trailers))) => Ok(Some(Ok(trailers))), - Some(Ok(Err(e))) => Ok(Some(Err(e.into()))), - Some(Err(())) => Err(anyhow::anyhow!("trailers have already been retrieved")), - None => Ok(None), - } - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostFutureTrailers::drop(self, rep) - } -} - -impl wasi::http::types::HostFutureIncomingResponse for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn get( - &mut self, - self_: Resource, - ) -> wasmtime::Result, HttpErrorCode>, ()>>> - { - match latest::http::types::HostFutureIncomingResponse::get(self, self_)? { - None => Ok(None), - Some(Ok(Ok(response))) => Ok(Some(Ok(Ok(response)))), - Some(Ok(Err(e))) => Ok(Some(Ok(Err(e.into())))), - Some(Err(())) => Ok(Some(Err(()))), - } - } - - fn subscribe( - &mut self, - self_: Resource, - ) -> wasmtime::Result> { - latest::http::types::HostFutureIncomingResponse::subscribe(self, self_) - } - - fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { - latest::http::types::HostFutureIncomingResponse::drop(self, rep) - } -} - -impl wasi::http::outgoing_handler::Host for WasiHttpImpl -where - T: WasiHttpView + Send, -{ - fn handle( - &mut self, - request: Resource, - options: Option>, - ) -> wasmtime::Result, HttpErrorCode>> { - match latest::http::outgoing_handler::Host::handle(self, request, options) { - Ok(resp) => Ok(Ok(resp)), - Err(e) => Ok(Err(e.downcast()?.into())), - } - } -} - -convert! { - struct latest::clocks::wall_clock::Datetime [<=>] Datetime { - seconds, - nanoseconds, - } - - enum latest::filesystem::types::ErrorCode => FsErrorCode { - Access, - WouldBlock, - Already, - BadDescriptor, - Busy, - Deadlock, - Quota, - Exist, - FileTooLarge, - IllegalByteSequence, - InProgress, - Interrupted, - Invalid, - Io, - IsDirectory, - Loop, - TooManyLinks, - MessageSize, - NameTooLong, - NoDevice, - NoEntry, - NoLock, - InsufficientMemory, - InsufficientSpace, - NotDirectory, - NotEmpty, - NotRecoverable, - Unsupported, - NoTty, - NoSuchDevice, - Overflow, - NotPermitted, - Pipe, - ReadOnly, - InvalidSeek, - TextFileBusy, - CrossDevice, - } - - enum Advice => latest::filesystem::types::Advice { - Normal, - Sequential, - Random, - WillNeed, - DontNeed, - NoReuse, - } - - flags DescriptorFlags [<=>] latest::filesystem::types::DescriptorFlags { - READ, - WRITE, - FILE_INTEGRITY_SYNC, - DATA_INTEGRITY_SYNC, - REQUESTED_WRITE_SYNC, - MUTATE_DIRECTORY, - } - - enum DescriptorType [<=>] latest::filesystem::types::DescriptorType { - Unknown, - BlockDevice, - CharacterDevice, - Directory, - Fifo, - SymbolicLink, - RegularFile, - Socket, - } - - enum NewTimestamp => latest::filesystem::types::NewTimestamp { - NoChange, - Now, - Timestamp(e), - } - - flags PathFlags => latest::filesystem::types::PathFlags { - SYMLINK_FOLLOW, - } - - flags OpenFlags => latest::filesystem::types::OpenFlags { - CREATE, - DIRECTORY, - EXCLUSIVE, - TRUNCATE, - } - - struct latest::filesystem::types::MetadataHashValue => MetadataHashValue { - lower, - upper, - } - - struct latest::filesystem::types::DirectoryEntry => DirectoryEntry { - type_, - name, - } - - - enum latest::sockets::network::ErrorCode => SocketErrorCode { - Unknown, - AccessDenied, - NotSupported, - InvalidArgument, - OutOfMemory, - Timeout, - ConcurrencyConflict, - NotInProgress, - WouldBlock, - InvalidState, - NewSocketLimit, - AddressNotBindable, - AddressInUse, - RemoteUnreachable, - ConnectionRefused, - ConnectionReset, - ConnectionAborted, - DatagramTooLarge, - NameUnresolvable, - TemporaryResolverFailure, - PermanentResolverFailure, - } - - enum latest::sockets::network::IpAddress [<=>] IpAddress { - Ipv4(e), - Ipv6(e), - } - - enum latest::sockets::network::IpSocketAddress [<=>] IpSocketAddress { - Ipv4(e), - Ipv6(e), - } - - struct latest::sockets::network::Ipv4SocketAddress [<=>] Ipv4SocketAddress { - port, - address, - } - - struct latest::sockets::network::Ipv6SocketAddress [<=>] Ipv6SocketAddress { - port, - flow_info, - scope_id, - address, - } - - enum latest::sockets::network::IpAddressFamily [<=>] IpAddressFamily { - Ipv4, - Ipv6, - } - - enum ShutdownType => latest::sockets::tcp::ShutdownType { - Receive, - Send, - Both, - } - - struct latest::sockets::udp::IncomingDatagram => IncomingDatagram { - data, - remote_address, - } - - enum latest::http::types::Method [<=>] Method { - Get, - Head, - Post, - Put, - Delete, - Connect, - Options, - Trace, - Patch, - Other(e), - } - - enum latest::http::types::Scheme [<=>] Scheme { - Http, - Https, - Other(e), - } - - enum latest::http::types::HeaderError => HeaderError { - InvalidSyntax, - Forbidden, - Immutable, - } - - struct latest::http::types::DnsErrorPayload [<=>] DnsErrorPayload { - rcode, - info_code, - } - - struct latest::http::types::TlsAlertReceivedPayload [<=>] TlsAlertReceivedPayload { - alert_id, - alert_message, - } - - struct latest::http::types::FieldSizePayload [<=>] FieldSizePayload { - field_name, - field_size, - } -} - -impl From for DescriptorStat { - fn from(e: latest::filesystem::types::DescriptorStat) -> DescriptorStat { - DescriptorStat { - type_: e.type_.into(), - link_count: e.link_count, - size: e.size, - data_access_timestamp: e.data_access_timestamp.map(|e| e.into()), - data_modification_timestamp: e.data_modification_timestamp.map(|e| e.into()), - status_change_timestamp: e.status_change_timestamp.map(|e| e.into()), - } - } -} - -impl From for latest::sockets::udp::OutgoingDatagram { - fn from(d: OutgoingDatagram) -> Self { - Self { - data: d.data, - remote_address: d.remote_address.map(|a| a.into()), - } - } -} - -impl From for HttpErrorCode { - fn from(e: latest::http::types::ErrorCode) -> Self { - match e { - latest::http::types::ErrorCode::DnsTimeout => HttpErrorCode::DnsTimeout, - latest::http::types::ErrorCode::DnsError(e) => HttpErrorCode::DnsError(e.into()), - latest::http::types::ErrorCode::DestinationNotFound => { - HttpErrorCode::DestinationNotFound - } - latest::http::types::ErrorCode::DestinationUnavailable => { - HttpErrorCode::DestinationUnavailable - } - latest::http::types::ErrorCode::DestinationIpProhibited => { - HttpErrorCode::DestinationIpProhibited - } - latest::http::types::ErrorCode::DestinationIpUnroutable => { - HttpErrorCode::DestinationIpUnroutable - } - latest::http::types::ErrorCode::ConnectionRefused => HttpErrorCode::ConnectionRefused, - latest::http::types::ErrorCode::ConnectionTerminated => { - HttpErrorCode::ConnectionTerminated - } - latest::http::types::ErrorCode::ConnectionTimeout => HttpErrorCode::ConnectionTimeout, - latest::http::types::ErrorCode::ConnectionReadTimeout => { - HttpErrorCode::ConnectionReadTimeout - } - latest::http::types::ErrorCode::ConnectionWriteTimeout => { - HttpErrorCode::ConnectionWriteTimeout - } - latest::http::types::ErrorCode::ConnectionLimitReached => { - HttpErrorCode::ConnectionLimitReached - } - latest::http::types::ErrorCode::TlsProtocolError => HttpErrorCode::TlsProtocolError, - latest::http::types::ErrorCode::TlsCertificateError => { - HttpErrorCode::TlsCertificateError - } - latest::http::types::ErrorCode::TlsAlertReceived(e) => { - HttpErrorCode::TlsAlertReceived(e.into()) - } - latest::http::types::ErrorCode::HttpRequestDenied => HttpErrorCode::HttpRequestDenied, - latest::http::types::ErrorCode::HttpRequestLengthRequired => { - HttpErrorCode::HttpRequestLengthRequired - } - latest::http::types::ErrorCode::HttpRequestBodySize(e) => { - HttpErrorCode::HttpRequestBodySize(e) - } - latest::http::types::ErrorCode::HttpRequestMethodInvalid => { - HttpErrorCode::HttpRequestMethodInvalid - } - latest::http::types::ErrorCode::HttpRequestUriInvalid => { - HttpErrorCode::HttpRequestUriInvalid - } - latest::http::types::ErrorCode::HttpRequestUriTooLong => { - HttpErrorCode::HttpRequestUriTooLong - } - latest::http::types::ErrorCode::HttpRequestHeaderSectionSize(e) => { - HttpErrorCode::HttpRequestHeaderSectionSize(e) - } - latest::http::types::ErrorCode::HttpRequestHeaderSize(e) => { - HttpErrorCode::HttpRequestHeaderSize(e.map(|e| e.into())) - } - latest::http::types::ErrorCode::HttpRequestTrailerSectionSize(e) => { - HttpErrorCode::HttpRequestTrailerSectionSize(e) - } - latest::http::types::ErrorCode::HttpRequestTrailerSize(e) => { - HttpErrorCode::HttpRequestTrailerSize(e.into()) - } - latest::http::types::ErrorCode::HttpResponseIncomplete => { - HttpErrorCode::HttpResponseIncomplete - } - latest::http::types::ErrorCode::HttpResponseHeaderSectionSize(e) => { - HttpErrorCode::HttpResponseHeaderSectionSize(e) - } - latest::http::types::ErrorCode::HttpResponseHeaderSize(e) => { - HttpErrorCode::HttpResponseHeaderSize(e.into()) - } - latest::http::types::ErrorCode::HttpResponseBodySize(e) => { - HttpErrorCode::HttpResponseBodySize(e) - } - latest::http::types::ErrorCode::HttpResponseTrailerSectionSize(e) => { - HttpErrorCode::HttpResponseTrailerSectionSize(e) - } - latest::http::types::ErrorCode::HttpResponseTrailerSize(e) => { - HttpErrorCode::HttpResponseTrailerSize(e.into()) - } - latest::http::types::ErrorCode::HttpResponseTransferCoding(e) => { - HttpErrorCode::HttpResponseTransferCoding(e) - } - latest::http::types::ErrorCode::HttpResponseContentCoding(e) => { - HttpErrorCode::HttpResponseContentCoding(e) - } - latest::http::types::ErrorCode::HttpResponseTimeout => { - HttpErrorCode::HttpResponseTimeout - } - latest::http::types::ErrorCode::HttpUpgradeFailed => HttpErrorCode::HttpUpgradeFailed, - latest::http::types::ErrorCode::HttpProtocolError => HttpErrorCode::HttpProtocolError, - latest::http::types::ErrorCode::LoopDetected => HttpErrorCode::LoopDetected, - latest::http::types::ErrorCode::ConfigurationError => HttpErrorCode::ConfigurationError, - latest::http::types::ErrorCode::InternalError(e) => HttpErrorCode::InternalError(e), - } - } -} - -impl From for latest::http::types::ErrorCode { - fn from(e: HttpErrorCode) -> Self { - match e { - HttpErrorCode::DnsTimeout => latest::http::types::ErrorCode::DnsTimeout, - HttpErrorCode::DnsError(e) => latest::http::types::ErrorCode::DnsError(e.into()), - HttpErrorCode::DestinationNotFound => { - latest::http::types::ErrorCode::DestinationNotFound - } - HttpErrorCode::DestinationUnavailable => { - latest::http::types::ErrorCode::DestinationUnavailable - } - HttpErrorCode::DestinationIpProhibited => { - latest::http::types::ErrorCode::DestinationIpProhibited - } - HttpErrorCode::DestinationIpUnroutable => { - latest::http::types::ErrorCode::DestinationIpUnroutable - } - HttpErrorCode::ConnectionRefused => latest::http::types::ErrorCode::ConnectionRefused, - HttpErrorCode::ConnectionTerminated => { - latest::http::types::ErrorCode::ConnectionTerminated - } - HttpErrorCode::ConnectionTimeout => latest::http::types::ErrorCode::ConnectionTimeout, - HttpErrorCode::ConnectionReadTimeout => { - latest::http::types::ErrorCode::ConnectionReadTimeout - } - HttpErrorCode::ConnectionWriteTimeout => { - latest::http::types::ErrorCode::ConnectionWriteTimeout - } - HttpErrorCode::ConnectionLimitReached => { - latest::http::types::ErrorCode::ConnectionLimitReached - } - HttpErrorCode::TlsProtocolError => latest::http::types::ErrorCode::TlsProtocolError, - HttpErrorCode::TlsCertificateError => { - latest::http::types::ErrorCode::TlsCertificateError - } - HttpErrorCode::TlsAlertReceived(e) => { - latest::http::types::ErrorCode::TlsAlertReceived(e.into()) - } - HttpErrorCode::HttpRequestDenied => latest::http::types::ErrorCode::HttpRequestDenied, - HttpErrorCode::HttpRequestLengthRequired => { - latest::http::types::ErrorCode::HttpRequestLengthRequired - } - HttpErrorCode::HttpRequestBodySize(e) => { - latest::http::types::ErrorCode::HttpRequestBodySize(e) - } - HttpErrorCode::HttpRequestMethodInvalid => { - latest::http::types::ErrorCode::HttpRequestMethodInvalid - } - HttpErrorCode::HttpRequestUriInvalid => { - latest::http::types::ErrorCode::HttpRequestUriInvalid - } - HttpErrorCode::HttpRequestUriTooLong => { - latest::http::types::ErrorCode::HttpRequestUriTooLong - } - HttpErrorCode::HttpRequestHeaderSectionSize(e) => { - latest::http::types::ErrorCode::HttpRequestHeaderSectionSize(e) - } - HttpErrorCode::HttpRequestHeaderSize(e) => { - latest::http::types::ErrorCode::HttpRequestHeaderSize(e.map(|e| e.into())) - } - HttpErrorCode::HttpRequestTrailerSectionSize(e) => { - latest::http::types::ErrorCode::HttpRequestTrailerSectionSize(e) - } - HttpErrorCode::HttpRequestTrailerSize(e) => { - latest::http::types::ErrorCode::HttpRequestTrailerSize(e.into()) - } - HttpErrorCode::HttpResponseIncomplete => { - latest::http::types::ErrorCode::HttpResponseIncomplete - } - HttpErrorCode::HttpResponseHeaderSectionSize(e) => { - latest::http::types::ErrorCode::HttpResponseHeaderSectionSize(e) - } - HttpErrorCode::HttpResponseHeaderSize(e) => { - latest::http::types::ErrorCode::HttpResponseHeaderSize(e.into()) - } - HttpErrorCode::HttpResponseBodySize(e) => { - latest::http::types::ErrorCode::HttpResponseBodySize(e) - } - HttpErrorCode::HttpResponseTrailerSectionSize(e) => { - latest::http::types::ErrorCode::HttpResponseTrailerSectionSize(e) - } - HttpErrorCode::HttpResponseTrailerSize(e) => { - latest::http::types::ErrorCode::HttpResponseTrailerSize(e.into()) - } - HttpErrorCode::HttpResponseTransferCoding(e) => { - latest::http::types::ErrorCode::HttpResponseTransferCoding(e) - } - HttpErrorCode::HttpResponseContentCoding(e) => { - latest::http::types::ErrorCode::HttpResponseContentCoding(e) - } - HttpErrorCode::HttpResponseTimeout => { - latest::http::types::ErrorCode::HttpResponseTimeout - } - HttpErrorCode::HttpUpgradeFailed => latest::http::types::ErrorCode::HttpUpgradeFailed, - HttpErrorCode::HttpProtocolError => latest::http::types::ErrorCode::HttpProtocolError, - HttpErrorCode::LoopDetected => latest::http::types::ErrorCode::LoopDetected, - HttpErrorCode::ConfigurationError => latest::http::types::ErrorCode::ConfigurationError, - HttpErrorCode::InternalError(e) => latest::http::types::ErrorCode::InternalError(e), - } - } -} diff --git a/crates/core/tests/core-wasi-test/src/main.rs b/crates/core/tests/core-wasi-test/src/main.rs index a1bc0ed28..77a76c2a1 100644 --- a/crates/core/tests/core-wasi-test/src/main.rs +++ b/crates/core/tests/core-wasi-test/src/main.rs @@ -5,11 +5,6 @@ use std::time::Duration; -wit_bindgen::generate!({ - world: "multiplier", - path: "wit/multiplier.wit" -}); - type Result = std::result::Result<(), Box>; fn main() -> Result { @@ -44,12 +39,6 @@ fn main() -> Result { eprintln!("write {path}"); std::fs::write(path, "content")?; } - "multiply" => { - let input: i32 = args.next().expect("input").parse().expect("i32"); - eprintln!("multiply {input}"); - let output = imports::multiply(input); - println!("{output}"); - } "sleep" => { let duration = Duration::from_millis(args.next().expect("duration_ms").parse().expect("u64")); diff --git a/crates/core/tests/integration_test.rs b/crates/core/tests/integration_test.rs index 726f9aa59..a442bc2dd 100644 --- a/crates/core/tests/integration_test.rs +++ b/crates/core/tests/integration_test.rs @@ -1,86 +1,28 @@ use std::{ - io::Cursor, path::PathBuf, time::{Duration, Instant}, }; use anyhow::Context; -use spin_core::{ - Component, Config, Engine, HostComponent, I32Exit, Store, StoreBuilder, Trap, WasiVersion, -}; -use tempfile::TempDir; +use serde_json::json; +use spin_core::{Component, Config, Engine, State, Store, StoreBuilder, Trap}; +use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; +use spin_factors::{App, RuntimeFactors}; +use spin_locked_app::locked::LockedApp; use tokio::{fs, io::AsyncWrite}; - -#[tokio::test(flavor = "multi_thread")] -async fn test_stdio() { - let stdout = run_core_wasi_test(["echo"], |store_builder| { - store_builder.stdin_pipe(Cursor::new(b"DATA")); - }) - .await - .unwrap(); - - assert_eq!(stdout, "DATA"); -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_read_only_preopened_dir() { - let filename = "test_file"; - let tempdir = TempDir::new().unwrap(); - std::fs::write(tempdir.path().join(filename), "x").unwrap(); - - run_core_wasi_test(["read", filename], |store_builder| { - store_builder - .read_only_preopened_dir(&tempdir, "/".into()) - .unwrap(); - }) - .await - .unwrap(); -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_read_only_preopened_dir_write_fails() { - let filename = "test_file"; - let tempdir = TempDir::new().unwrap(); - std::fs::write(tempdir.path().join(filename), "x").unwrap(); - - let err = run_core_wasi_test(["write", filename], |store_builder| { - store_builder - .read_only_preopened_dir(&tempdir, "/".into()) - .unwrap(); - }) - .await - .unwrap_err(); - let trap = err - .root_cause() // The error returned is a backtrace. We need the root cause. - .downcast_ref::() - .expect("trap error was not an I32Exit"); - assert_eq!(trap.0, 1); -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_read_write_preopened_dir() { - let filename = "test_file"; - let tempdir = TempDir::new().unwrap(); - - run_core_wasi_test(["write", filename], |store_builder| { - store_builder - .read_write_preopened_dir(&tempdir, "/".into()) - .unwrap(); - }) - .await - .unwrap(); - - let content = std::fs::read(tempdir.path().join(filename)).unwrap(); - assert_eq!(content, b"content"); -} +use wasmtime_wasi::I32Exit; #[tokio::test(flavor = "multi_thread")] async fn test_max_memory_size_obeyed() { let max = 10_000_000; let alloc = max / 10; - run_core_wasi_test(["alloc", &format!("{alloc}")], |store_builder| { - store_builder.max_memory_size(max); - }) + run_test( + ["alloc", &format!("{alloc}")], + |store_builder| { + store_builder.max_memory_size(max); + }, + |_| {}, + ) .await .unwrap(); } @@ -89,9 +31,13 @@ async fn test_max_memory_size_obeyed() { async fn test_max_memory_size_violated() { let max = 10_000_000; let alloc = max * 2; - let err = run_core_wasi_test(["alloc", &format!("{alloc}")], |store_builder| { - store_builder.max_memory_size(max); - }) + let err = run_test( + ["alloc", &format!("{alloc}")], + |store_builder| { + store_builder.max_memory_size(max); + }, + |_| {}, + ) .await .unwrap_err(); let trap = err @@ -101,14 +47,14 @@ async fn test_max_memory_size_violated() { assert_eq!(trap.0, 1); } +// FIXME: racy timing test #[tokio::test(flavor = "multi_thread")] async fn test_set_deadline_obeyed() { - run_core_wasi_test_engine( - &test_engine(), + run_test( ["sleep", "20"], |_| {}, |store| { - store.set_deadline(Instant::now() + Duration::from_millis(1000)); + store.set_deadline(Instant::now() + Duration::from_millis(10000)); }, ) .await @@ -117,8 +63,7 @@ async fn test_set_deadline_obeyed() { #[tokio::test(flavor = "multi_thread")] async fn test_set_deadline_violated() { - let err = run_core_wasi_test_engine( - &test_engine(), + let err = run_test( ["sleep", "100"], |_| {}, |store| { @@ -131,83 +76,82 @@ async fn test_set_deadline_violated() { assert_eq!(trap, Trap::Interrupt); } -#[tokio::test(flavor = "multi_thread")] -async fn test_host_component() { - let stdout = run_core_wasi_test(["multiply", "5"], |_| {}).await.unwrap(); - assert_eq!(stdout, "10"); -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_host_component_data_update() { - let engine = test_engine(); - let multiplier_handle = engine - .find_host_component_handle::() - .unwrap(); - - let stdout = run_core_wasi_test_engine( - &engine, - ["multiply", "5"], - |store_builder| { - store_builder - .host_components_data() - .set(multiplier_handle, Multiplier(100)); - }, - |_| {}, - ) - .await - .unwrap(); - assert_eq!(stdout, "500"); -} - #[tokio::test(flavor = "multi_thread")] #[cfg(not(tarpaulin))] async fn test_panic() { - let err = run_core_wasi_test(["panic"], |_| {}).await.unwrap_err(); + let err = run_test(["panic"], |_| {}, |_| {}).await.unwrap_err(); let trap = err.downcast::().expect("trap"); assert_eq!(trap, Trap::UnreachableCodeReached); } -fn test_config() -> Config { - let mut config = Config::default(); - config - .wasmtime_config() - .wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); - config +#[derive(RuntimeFactors)] +struct TestFactors { + wasi: WasiFactor, +} + +struct TestState { + core: State, + factors: TestFactorsInstanceState, } -fn test_engine() -> Engine<()> { - let mut builder = Engine::builder(&test_config()).unwrap(); - builder.add_host_component(MultiplierHostComponent).unwrap(); - builder - .link_import(|l, _| wasmtime_wasi::add_to_linker_async(l)) - .unwrap(); - builder - .link_import(|l, _| spin_core::wasi_2023_10_18::add_to_linker(l)) - .unwrap(); - builder.build() +impl AsMut for TestState { + fn as_mut(&mut self) -> &mut State { + &mut self.core + } } -async fn run_core_wasi_test<'a>( - args: impl IntoIterator, - f: impl FnOnce(&mut StoreBuilder), -) -> anyhow::Result { - run_core_wasi_test_engine(&test_engine(), args, f, |_| {}).await +impl AsMut for TestState { + fn as_mut(&mut self) -> &mut TestFactorsInstanceState { + &mut self.factors + } } -async fn run_core_wasi_test_engine<'a>( - engine: &Engine<()>, - args: impl IntoIterator, +async fn run_test( + args: impl IntoIterator, update_store_builder: impl FnOnce(&mut StoreBuilder), - update_store: impl FnOnce(&mut Store<()>), -) -> anyhow::Result { - let mut store_builder: StoreBuilder = engine.store_builder(WasiVersion::Preview2); - let stdout_buf = store_builder.stdout_buffered()?; - store_builder.stderr_pipe(TestWriter(tokio::io::stdout())); - store_builder.args(args)?; + update_store: impl FnOnce(&mut Store), +) -> anyhow::Result<()> { + let mut factors = TestFactors { + wasi: WasiFactor::new(DummyFilesMounter), + }; + let mut config = Config::default(); + config + .wasmtime_config() + .wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); + + let mut builder = Engine::builder(&config).unwrap(); + factors.init(builder.linker())?; + let engine = builder.build(); + + let mut store_builder = engine.store_builder(); update_store_builder(&mut store_builder); - let mut store = store_builder.build()?; + let locked: LockedApp = serde_json::from_value(json!({ + "spin_lock_version": 1, + "triggers": [], + "components": [{ + "id": "test-component", + "source": { + "content_type": "application/wasm", + "content": {}, + }, + }] + }))?; + let app = App::new("test-app", locked); + let configured_app = factors.configure_app(app, ())?; + let mut builders = factors.prepare(&configured_app, "test-component")?; + // FIXME: it is unfortunate that we have to unwrap here... + builders.wasi.as_mut().unwrap().args(args); + let instance_state = factors.build_instance_state(builders)?; + let state = TestState { + core: State::default(), + factors: instance_state, + }; + + let mut store = store_builder.build(state)?; + update_store(&mut store); + let module_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../target/test-programs/core-wasi-test.wasm"); let component = spin_componentize::componentize_command(&fs::read(module_path).await?)?; @@ -222,48 +166,11 @@ async fn run_core_wasi_test_engine<'a>( .context("missing the expected 'wasi:cli/run@0.2.0' instance")?; instance.typed_func::<(), (Result<(), ()>,)>("run")? }; - update_store(&mut store); func.call_async(&mut store, ()) .await? .0 - .map_err(|()| anyhow::anyhow!("command failed"))?; - - let stdout = String::from_utf8(stdout_buf.contents().to_vec())? - .trim_end() - .into(); - Ok(stdout) -} - -// Simple test HostComponent; multiplies the input by the configured factor -#[derive(Clone)] -struct MultiplierHostComponent; - -mod multiplier { - wasmtime::component::bindgen!("multiplier" in "tests/core-wasi-test/wit"); -} - -impl HostComponent for MultiplierHostComponent { - type Data = Multiplier; - - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - multiplier::imports::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - Multiplier(2) - } -} - -struct Multiplier(i32); - -impl multiplier::imports::Host for Multiplier { - fn multiply(&mut self, a: i32) -> i32 { - self.0 * a - } + .map_err(|()| anyhow::anyhow!("command failed")) } // Write with `print!`, required for test output capture diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index 6fb2dfc8e..3995da46f 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -7,6 +7,7 @@ edition = { workspace = true } [dependencies] async-trait = "0.1" cap-primitives = "3.0.0" +spin-app = { path = "../app" } spin-factors = { path = "../factors" } tokio = { version = "1" } wasmtime = { workspace = true } diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 62f98c146..983b18dea 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -3,9 +3,10 @@ mod wasi_2023_11_10; use std::{future::Future, net::SocketAddr, path::Path}; +use spin_app::AppComponent; use spin_factors::{ - anyhow, AppComponent, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, - PrepareContext, RuntimeFactors, RuntimeFactorsInstanceState, + anyhow, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, + RuntimeFactors, RuntimeFactorsInstanceState, }; use tokio::io::{AsyncRead, AsyncWrite}; use wasmtime_wasi::{ @@ -43,10 +44,7 @@ impl Factor for WasiFactor { type AppState = (); type InstanceBuilder = InstanceBuilder; - fn init( - &mut self, - mut ctx: InitContext, - ) -> anyhow::Result<()> { + fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { fn type_annotate(f: F) -> F where F: Fn(&mut T) -> WasiImpl, diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 103671dfe..7e9c78232 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -58,13 +58,13 @@ impl TestEnvironment { mut factors: T, ) -> anyhow::Result { let mut linker = Self::new_linker::(); - factors.init::(&mut linker)?; + factors.init(&mut linker)?; let locked_app = self .build_locked_app() .await .context("failed to build locked app")?; - let app = App::inert(locked_app); + let app = App::new("test-app", locked_app); let runtime_config = TomlRuntimeConfig(&self.runtime_config); let configured_app = factors.configure_app(app, runtime_config)?; diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 569b76e9b..074468809 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -7,6 +7,7 @@ pub use anyhow; pub use serde; pub use wasmtime; +pub use spin_app::App; pub use spin_factors_derive::RuntimeFactors; pub use crate::{ @@ -17,9 +18,6 @@ pub use crate::{ }; // Temporary wrappers while refactoring -pub type App = spin_app::App<'static, spin_app::InertLoader>; -pub type AppComponent<'a> = spin_app::AppComponent<'a, spin_app::InertLoader>; - pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] diff --git a/crates/factors/src/prepare.rs b/crates/factors/src/prepare.rs index 1c66d7568..318bb15b0 100644 --- a/crates/factors/src/prepare.rs +++ b/crates/factors/src/prepare.rs @@ -1,6 +1,8 @@ use std::any::Any; -use crate::{AppComponent, Error, Factor, RuntimeFactors}; +use spin_app::AppComponent; + +use crate::{Error, Factor, RuntimeFactors}; /// A builder for a [`Factor`]'s per instance state. pub trait FactorInstanceBuilder: Any { diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 7ec8cc059..0a4cbde32 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -69,7 +69,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; let mut linker = wasmtime::component::Linker::new(&engine); - factors.init::(&mut linker).unwrap(); + factors.init(&mut linker).unwrap(); let configured_app = factors.configure_app(app, TestSource)?; let builders = factors.prepare(&configured_app, "smoke-app")?; From 3b66505be3c1e21eb57e0a4333b90ef84d41e8ce Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 19 Jul 2024 09:35:40 -0400 Subject: [PATCH 065/195] Resolve PR feedback Signed-off-by: Lann Martin --- Cargo.lock | 1 - crates/factor-wasi/Cargo.toml | 1 - crates/factor-wasi/src/lib.rs | 5 ++--- crates/factors/src/lib.rs | 4 ++-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4fbcc3c94..dd8ec4bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7663,7 +7663,6 @@ version = "2.7.0-pre0" dependencies = [ "async-trait", "cap-primitives 3.0.0", - "spin-app", "spin-factors", "spin-factors-test", "tokio", diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index 3995da46f..6fb2dfc8e 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -7,7 +7,6 @@ edition = { workspace = true } [dependencies] async-trait = "0.1" cap-primitives = "3.0.0" -spin-app = { path = "../app" } spin-factors = { path = "../factors" } tokio = { version = "1" } wasmtime = { workspace = true } diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 983b18dea..82fdd802b 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -3,10 +3,9 @@ mod wasi_2023_11_10; use std::{future::Future, net::SocketAddr, path::Path}; -use spin_app::AppComponent; use spin_factors::{ - anyhow, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, - RuntimeFactors, RuntimeFactorsInstanceState, + anyhow, AppComponent, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, + PrepareContext, RuntimeFactors, RuntimeFactorsInstanceState, }; use tokio::io::{AsyncRead, AsyncWrite}; use wasmtime_wasi::{ diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 074468809..69f5c20b8 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -7,7 +7,7 @@ pub use anyhow; pub use serde; pub use wasmtime; -pub use spin_app::App; +pub use spin_app::{App, AppComponent}; pub use spin_factors_derive::RuntimeFactors; pub use crate::{ @@ -17,7 +17,7 @@ pub use crate::{ runtime_factors::{RuntimeFactors, RuntimeFactorsInstanceState}, }; -// Temporary wrappers while refactoring +/// Result wrapper type defaulting to use [`Error`]. pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] From e2334e7dc5513a90a996a023938b9bc0fc26e676 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 19 Jul 2024 09:45:16 -0400 Subject: [PATCH 066/195] Remove ouroboros Signed-off-by: Lann Martin --- Cargo.lock | 51 ------------------- crates/locked-app/Cargo.toml | 1 - examples/spin-timer/Cargo.lock | 93 +--------------------------------- supply-chain/config.toml | 8 --- 4 files changed, 1 insertion(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd8ec4bd0..aa48f93f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,12 +64,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "allocator-api2" version = "0.2.16" @@ -5298,31 +5292,6 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -[[package]] -name = "ouroboros" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b7be5a8a3462b752f4be3ff2b2bf2f7f1d00834902e46be2a4d68b87b0573c" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645dcde5f119c2c454a92d0dfa271a2a3b205da92e4292a68ead4bdbfde1f33" -dependencies = [ - "heck 0.4.1", - "itertools 0.12.1", - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.58", -] - [[package]] name = "outbound-http" version = "2.7.0-pre0" @@ -5930,19 +5899,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "proc-macro2-diagnostics" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.58", - "version_check", - "yansi", -] - [[package]] name = "proc-quote" version = "0.4.0" @@ -7896,7 +7852,6 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "async-trait", - "ouroboros", "serde 1.0.197", "serde_json", "spin-serde", @@ -10896,12 +10851,6 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "zbus" version = "3.15.2" diff --git a/crates/locked-app/Cargo.toml b/crates/locked-app/Cargo.toml index 44b25fee5..d0531c52a 100644 --- a/crates/locked-app/Cargo.toml +++ b/crates/locked-app/Cargo.toml @@ -7,7 +7,6 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" async-trait = "0.1" -ouroboros = "0.18.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" spin-serde = { path = "../serde" } diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index e78d13851..e26adced5 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -58,12 +58,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "allocator-api2" version = "0.2.16" @@ -3953,31 +3947,6 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -[[package]] -name = "ouroboros" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b7be5a8a3462b752f4be3ff2b2bf2f7f1d00834902e46be2a4d68b87b0573c" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645dcde5f119c2c454a92d0dfa271a2a3b205da92e4292a68ead4bdbfde1f33" -dependencies = [ - "heck 0.4.1", - "itertools 0.12.1", - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.48", -] - [[package]] name = "outbound-http" version = "2.7.0-pre0" @@ -4500,19 +4469,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "proc-macro2-diagnostics" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", - "version_check", - "yansi", -] - [[package]] name = "prost" version = "0.12.6" @@ -5674,10 +5630,8 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "async-trait", - "ouroboros", "serde 1.0.203", "serde_json", - "spin-core", "spin-locked-app", "spin-serde", "thiserror", @@ -5714,21 +5668,9 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "async-trait", - "bytes", - "cap-primitives", - "cap-std", "crossbeam-channel", - "http 1.1.0", - "io-extras", - "rustix 0.37.27", - "spin-telemetry", - "system-interface", - "tokio", "tracing", - "wasi-common", "wasmtime", - "wasmtime-wasi", - "wasmtime-wasi-http", ] [[package]] @@ -5875,7 +5817,6 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "async-trait", - "ouroboros", "serde 1.0.203", "serde_json", "spin-serde", @@ -6063,6 +6004,7 @@ dependencies = [ name = "spin-world" version = "2.7.0-pre0" dependencies = [ + "async-trait", "wasmtime", ] @@ -7163,33 +7105,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasi-common" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86fd41e1e26ff6af9451c6a332a5ce5f5283ca51e87d875cdd9a05305598ee3" -dependencies = [ - "anyhow", - "bitflags 2.4.2", - "cap-fs-ext", - "cap-rand", - "cap-std", - "cap-time-ext", - "fs-set-times", - "io-extras", - "io-lifetimes 2.0.3", - "log", - "once_cell", - "rustix 0.38.31", - "system-interface", - "thiserror", - "tokio", - "tracing", - "wasmtime", - "wiggle", - "windows-sys 0.52.0", -] - [[package]] name = "wasite" version = "0.1.0" @@ -8249,12 +8164,6 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "zbus" version = "3.15.2" diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 142f4441c..27eaaba2c 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -1104,14 +1104,6 @@ criteria = "safe-to-deploy" version = "6.3.0" criteria = "safe-to-deploy" -[[exemptions.ouroboros]] -version = "0.15.6" -criteria = "safe-to-deploy" - -[[exemptions.ouroboros_macro]] -version = "0.15.6" -criteria = "safe-to-deploy" - [[exemptions.overload]] version = "0.1.1" criteria = "safe-to-deploy" From d6a4c3724a38b54892e24f5065d15d47d8de6fb9 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 19 Jul 2024 09:28:20 -0400 Subject: [PATCH 067/195] Add run-factors-tests.sh This is a temporary hack while the factors branch CI is broken during refactoring. Signed-off-by: Lann Martin --- crates/key-value/src/util.rs | 14 +++++++++----- run-factors-tests.sh | 5 +++++ 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100755 run-factors-tests.sh diff --git a/crates/key-value/src/util.rs b/crates/key-value/src/util.rs index 0af704e62..c2aef7e4d 100644 --- a/crates/key-value/src/util.rs +++ b/crates/key-value/src/util.rs @@ -52,11 +52,15 @@ impl DelegatingStoreManager { #[async_trait] impl StoreManager for DelegatingStoreManager { async fn get(&self, name: &str) -> Result, Error> { - let store = match self.delegates.get(name) { - Some(store) => store, - None => &(self.default_manager)(name).ok_or(Error::NoSuchStore)?, - }; - store.get(name).await + match self.delegates.get(name) { + Some(store) => store.get(name).await, + None => { + (self.default_manager)(name) + .ok_or(Error::NoSuchStore)? + .get(name) + .await + } + } } fn is_defined(&self, store_name: &str) -> bool { diff --git a/run-factors-tests.sh b/run-factors-tests.sh new file mode 100755 index 000000000..a32175177 --- /dev/null +++ b/run-factors-tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# TODO: Remove after enabling CI for factors branch + +cargo test -p '*factor*' \ No newline at end of file From 0a6f9acc76b86938c4d74ffc0fcd8c41327c86b9 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 19 Jul 2024 15:50:25 +0200 Subject: [PATCH 068/195] Remove toml assumption from sqlite Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/lib.rs | 26 ++++++----------- crates/factor-sqlite/src/runtime_config.rs | 28 +++++-------------- .../factor-sqlite/src/runtime_config/spin.rs | 5 ++-- crates/factor-sqlite/tests/factor.rs | 22 +++++++++++---- 4 files changed, 34 insertions(+), 47 deletions(-) diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index bdc5d3abb..3139a86a3 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -7,19 +7,20 @@ use std::sync::Arc; use host::InstanceState; use async_trait::async_trait; +use serde::de::DeserializeOwned; use spin_factors::{anyhow, Factor}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; -pub struct SqliteFactor { - runtime_config_resolver: Arc, +pub struct SqliteFactor { + runtime_config_resolver: Arc>, } -impl SqliteFactor { +impl SqliteFactor { /// Create a new `SqliteFactor` pub fn new( - runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, + runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, ) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), @@ -27,8 +28,8 @@ impl SqliteFactor { } } -impl Factor for SqliteFactor { - type RuntimeConfig = runtime_config::RuntimeConfig; +impl Factor for SqliteFactor { + type RuntimeConfig = runtime_config::RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; @@ -47,17 +48,8 @@ impl Factor for SqliteFactor { ) -> anyhow::Result { let mut connection_pools = HashMap::new(); if let Some(runtime_config) = ctx.take_runtime_config() { - for ( - database_label, - runtime_config::StoreConfig { - type_: database_kind, - config, - }, - ) in runtime_config.store_configs - { - let pool = self - .runtime_config_resolver - .get_pool(&database_kind, config)?; + for (database_label, config) in runtime_config.store_configs { + let pool = self.runtime_config_resolver.get_pool(config)?; connection_pools.insert(database_label, pool); } } diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index 2089d079a..841a5e2c2 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -3,40 +3,26 @@ pub mod spin; use std::{collections::HashMap, sync::Arc}; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize}; use spin_factors::{anyhow, FactorRuntimeConfig}; use crate::ConnectionPool; #[derive(Deserialize)] #[serde(transparent)] -pub struct RuntimeConfig { - pub store_configs: HashMap, +pub struct RuntimeConfig { + pub store_configs: HashMap, } -impl FactorRuntimeConfig for RuntimeConfig { +impl FactorRuntimeConfig for RuntimeConfig { const KEY: &'static str = "sqlite_database"; } -#[derive(Deserialize)] -pub struct StoreConfig { - #[serde(rename = "type")] - pub type_: String, - #[serde(flatten)] - pub config: toml::Table, -} - /// Resolves some piece of runtime configuration to a connection pool -pub trait RuntimeConfigResolver: Send + Sync { - /// Get a connection pool for a given database kind and the raw configuration. +pub trait RuntimeConfigResolver: Send + Sync { + /// Get a connection pool for a given config. /// - /// `database_kind` is equivalent to the `type` field in the - /// `[sqlite_database.$databasename]` runtime configuration table. - fn get_pool( - &self, - database_kind: &str, - config: toml::Table, - ) -> anyhow::Result>; + fn get_pool(&self, config: C) -> anyhow::Result>; /// If there is no runtime configuration for a given database label, return a default connection pool. /// diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index fa8b618a5..f8d8c33d4 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -50,11 +50,10 @@ impl SpinSqliteRuntimeConfig { } } -impl RuntimeConfigResolver for SpinSqliteRuntimeConfig { +impl RuntimeConfigResolver<(&str, toml::Table)> for SpinSqliteRuntimeConfig { fn get_pool( &self, - database_kind: &str, - config: toml::Table, + (database_kind, config): (&str, toml::Table), ) -> anyhow::Result> { let pool = match database_kind { "spin" => { diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index eb40212d9..6ce8b06d2 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -1,6 +1,7 @@ use std::{collections::HashSet, sync::Arc}; use factor_sqlite::SqliteFactor; +use serde::Deserialize; use spin_factors::{ anyhow::{self, bail}, RuntimeFactors, @@ -9,7 +10,7 @@ use spin_factors_test::{toml, TestEnvironment}; #[derive(RuntimeFactors)] struct TestFactors { - sqlite: SqliteFactor, + sqlite: SqliteFactor, } #[tokio::test] @@ -70,7 +71,9 @@ async fn no_error_when_database_is_configured() -> anyhow::Result<()> { [sqlite_database.foo] type = "sqlite" }; - assert!(env.build_instance_state(factors).await.is_ok()); + if let Err(e) = env.build_instance_state(factors).await { + bail!("Expected build_instance_state to succeed but it errored: {e}"); + } Ok(()) } @@ -88,13 +91,12 @@ impl RuntimeConfigResolver { } } -impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { +impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { fn get_pool( &self, - database_kind: &str, - config: toml::Table, + config: RuntimeConfig, ) -> anyhow::Result> { - let _ = (database_kind, config); + let _ = config; Ok(Arc::new(InvalidConnectionPool)) } @@ -117,3 +119,11 @@ impl factor_sqlite::ConnectionPool for InvalidConnectionPool { Err(spin_world::v2::sqlite::Error::InvalidConnection) } } + +#[derive(Deserialize)] +pub struct RuntimeConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} From bbdf0aa5cc8dc22159aa6ba3b0c250d1552184c8 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 19 Jul 2024 16:02:55 +0200 Subject: [PATCH 069/195] Remove toml assumption from key-value Signed-off-by: Ryan Levick --- .../src/delegating_resolver.rs | 35 +++++++++---------- crates/factor-key-value/src/lib.rs | 26 +++++--------- crates/factor-key-value/src/runtime_config.rs | 29 ++++----------- crates/factors/tests/smoke.rs | 11 +++--- 4 files changed, 40 insertions(+), 61 deletions(-) diff --git a/crates/factor-key-value/src/delegating_resolver.rs b/crates/factor-key-value/src/delegating_resolver.rs index 235a2afcb..3644c8e28 100644 --- a/crates/factor-key-value/src/delegating_resolver.rs +++ b/crates/factor-key-value/src/delegating_resolver.rs @@ -1,5 +1,6 @@ use crate::runtime_config::RuntimeConfigResolver; use crate::store::{store_from_toml_fn, MakeKeyValueStore, StoreFromToml}; +use serde::Deserialize; use spin_key_value::StoreManager; use std::{collections::HashMap, sync::Arc}; @@ -9,20 +10,13 @@ pub struct DelegatingRuntimeConfigResolver { defaults: HashMap<&'static str, StoreConfig>, } -type StoreConfig = (&'static str, toml::value::Table); - impl DelegatingRuntimeConfigResolver { pub fn new() -> Self { Self::default() } - pub fn add_default_store( - &mut self, - label: &'static str, - store_kind: &'static str, - config: toml::value::Table, - ) { - self.defaults.insert(label, (store_kind, config)); + pub fn add_default_store(&mut self, label: &'static str, config: StoreConfig) { + self.defaults.insert(label, config); } } @@ -42,21 +36,26 @@ impl DelegatingRuntimeConfigResolver { } } -impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { - fn get_store( - &self, - store_kind: &str, - config: toml::Table, - ) -> anyhow::Result> { +impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { + fn get_store(&self, config: StoreConfig) -> anyhow::Result> { + let store_kind = config.type_.as_str(); let store_from_toml = self .store_types .get(store_kind) .ok_or_else(|| anyhow::anyhow!("unknown store kind: {}", store_kind))?; - store_from_toml(config) + store_from_toml(config.config) } fn default_store(&self, label: &str) -> Option> { - let (store_kind, config) = self.defaults.get(label)?; - self.get_store(store_kind, config.to_owned()).ok() + let config = self.defaults.get(label)?; + self.get_store(config.clone()).ok() } } + +#[derive(Deserialize, Clone)] +pub struct StoreConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index bc2016467..269eed5e1 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::ensure; use runtime_config::RuntimeConfig; +use serde::de::DeserializeOwned; use spin_factors::{ ConfigureAppContext, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, @@ -19,13 +20,13 @@ use spin_key_value::{ }; pub use store::MakeKeyValueStore; -pub struct KeyValueFactor { - runtime_config_resolver: Arc, +pub struct KeyValueFactor { + runtime_config_resolver: Arc>, } -impl KeyValueFactor { +impl KeyValueFactor { pub fn new( - runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, + runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, ) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), @@ -33,8 +34,8 @@ impl KeyValueFactor { } } -impl Factor for KeyValueFactor { - type RuntimeConfig = RuntimeConfig; +impl Factor for KeyValueFactor { + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceBuilder; @@ -51,17 +52,8 @@ impl Factor for KeyValueFactor { // Build StoreManager from runtime config let mut store_managers: HashMap> = HashMap::new(); if let Some(runtime_config) = ctx.take_runtime_config() { - for ( - store_label, - runtime_config::StoreConfig { - type_: store_kind, - config, - }, - ) in runtime_config.store_configs - { - let store = self - .runtime_config_resolver - .get_store(&store_kind, config)?; + for (store_label, config) in runtime_config.store_configs { + let store = self.runtime_config_resolver.get_store(config)?; store_managers.insert(store_label, store); } } diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index 721cf5be4..dbafa7431 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -1,38 +1,23 @@ use std::{collections::HashMap, sync::Arc}; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize}; use spin_factors::{anyhow, FactorRuntimeConfig}; use spin_key_value::StoreManager; #[derive(Deserialize)] #[serde(transparent)] -pub struct RuntimeConfig { - pub store_configs: HashMap, +pub struct RuntimeConfig { + pub store_configs: HashMap, } -impl FactorRuntimeConfig for RuntimeConfig { +impl FactorRuntimeConfig for RuntimeConfig { const KEY: &'static str = "key_value_store"; } -#[derive(Deserialize)] -pub struct StoreConfig { - #[serde(rename = "type")] - pub type_: String, - #[serde(flatten)] - pub config: toml::Table, -} - /// Resolves some piece of runtime configuration to a connection pool -pub trait RuntimeConfigResolver: Send + Sync { - /// Get a store manager for a given store kind and the raw configuration. - /// - /// `store_kind` is equivalent to the `type` field in the - /// `[key_value_store.$storename]` runtime configuration table. - fn get_store( - &self, - store_kind: &str, - config: toml::Table, - ) -> anyhow::Result>; +pub trait RuntimeConfigResolver: Send + Sync { + /// Get a store manager for a given config. + fn get_store(&self, config: C) -> anyhow::Result>; /// Returns a default store manager for a given label. Should only be called /// if there is no runtime configuration for the label. diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 7ec8cc059..5244e3bd2 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -4,7 +4,8 @@ use anyhow::Context; use http_body_util::BodyExt; use spin_app::App; use spin_factor_key_value::{ - delegating_resolver::DelegatingRuntimeConfigResolver, KeyValueFactor, MakeKeyValueStore, + delegating_resolver::{DelegatingRuntimeConfigResolver, StoreConfig}, + KeyValueFactor, MakeKeyValueStore, }; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; @@ -21,7 +22,7 @@ struct Factors { variables: VariablesFactor, outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, - key_value: KeyValueFactor, + key_value: KeyValueFactor, } struct Data { @@ -42,8 +43,10 @@ async fn smoke_test_works() -> anyhow::Result<()> { SpinKeyValueRuntimeConfig::default(Some(PathBuf::from("tests/smoke-app/.spin"))); key_value_resolver.add_default_store( "default", - SpinKeyValueStore::RUNTIME_CONFIG_TYPE, - toml::value::Table::try_from(default_config)?, + StoreConfig { + type_: SpinKeyValueStore::RUNTIME_CONFIG_TYPE.to_string(), + config: toml::value::Table::try_from(default_config)?, + }, ); key_value_resolver.add_store_type(SpinKeyValueStore::new(None)?)?; key_value_resolver.add_store_type(RedisKeyValueStore)?; From eb3ebb224dad475396824439e75dccc3d36fefb0 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 19 Jul 2024 09:32:51 -0400 Subject: [PATCH 070/195] factors: Get factors tests passing with new spin-core Signed-off-by: Lann Martin --- crates/factor-variables/build.rs | 2 +- crates/factor-wasi/build.rs | 2 +- crates/factor-wasi/src/lib.rs | 1 - crates/factors/tests/smoke.rs | 2 +- crates/key-value/src/lib.rs | 4 ++-- crates/sqlite/src/lib.rs | 8 ++++---- run-factors-tests.sh | 5 ++--- 7 files changed, 11 insertions(+), 13 deletions(-) diff --git a/crates/factor-variables/build.rs b/crates/factor-variables/build.rs index bba8ce3c9..c96556b06 100644 --- a/crates/factor-variables/build.rs +++ b/crates/factor-variables/build.rs @@ -1,5 +1,5 @@ fn main() { - println!("cargo::rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=build.rs"); // Enable spin-factors-derive to emit expanded macro output. let out_dir = std::env::var("OUT_DIR").unwrap(); println!("cargo:rustc-env=SPIN_FACTORS_DERIVE_EXPAND_DIR={out_dir}"); diff --git a/crates/factor-wasi/build.rs b/crates/factor-wasi/build.rs index bba8ce3c9..c96556b06 100644 --- a/crates/factor-wasi/build.rs +++ b/crates/factor-wasi/build.rs @@ -1,5 +1,5 @@ fn main() { - println!("cargo::rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=build.rs"); // Enable spin-factors-derive to emit expanded macro output. let out_dir = std::env::var("OUT_DIR").unwrap(); println!("cargo:rustc-env=SPIN_FACTORS_DERIVE_EXPAND_DIR={out_dir}"); diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 82fdd802b..2579b15e6 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -157,7 +157,6 @@ impl<'a> MountFilesContext<'a> { guest_path: impl AsRef, writable: bool, ) -> anyhow::Result<()> { - use wasmtime_wasi::{DirPerms, FilePerms}; let (dir_perms, file_perms) = if writable { (DirPerms::all(), FilePerms::all()) } else { diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 0a4cbde32..ee1bcb6b4 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -64,7 +64,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { None, ) .await?; - let app = App::inert(locked); + let app = App::new("test-app", locked); let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; let mut linker = wasmtime::component::Linker::new(&engine); diff --git a/crates/key-value/src/lib.rs b/crates/key-value/src/lib.rs index 7caddf40a..08a656c5d 100644 --- a/crates/key-value/src/lib.rs +++ b/crates/key-value/src/lib.rs @@ -5,10 +5,10 @@ use spin_world::v2::key_value; use std::{collections::HashSet, sync::Arc}; use table::Table; -mod host_component; +// TODO: Code left for reference; remove after migration to factors +// mod host_component; mod util; -pub use host_component::{manager, KeyValueComponent}; pub use util::{ CachingStoreManager, DefaultManagerGetter, DelegatingStoreManager, EmptyStoreManager, }; diff --git a/crates/sqlite/src/lib.rs b/crates/sqlite/src/lib.rs index a9f589bf6..a56f40474 100644 --- a/crates/sqlite/src/lib.rs +++ b/crates/sqlite/src/lib.rs @@ -1,13 +1,13 @@ -mod host_component; +// TODO: Code left for reference; remove after migration to factors +// mod host_component; -use spin_app::{async_trait, MetadataKey}; +use spin_app::MetadataKey; use spin_core::wasmtime::component::Resource; +use spin_world::async_trait; use spin_world::v1::sqlite::Error as V1SqliteError; use spin_world::v2::sqlite; use std::{collections::HashSet, sync::Arc}; -pub use host_component::SqliteComponent; - pub const DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); /// A store of connections for all accessible databases for an application diff --git a/run-factors-tests.sh b/run-factors-tests.sh index a32175177..497a0f319 100755 --- a/run-factors-tests.sh +++ b/run-factors-tests.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash +# TODO(factors): Remove after enabling CI for factors branch -# TODO: Remove after enabling CI for factors branch - -cargo test -p '*factor*' \ No newline at end of file +cargo test -p '*factor*' From e863c8b5034d540e12b4e20ef9b9e08170899ed5 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 19 Jul 2024 09:40:33 -0400 Subject: [PATCH 071/195] factors: Add simple CI Signed-off-by: Lann Martin --- .github/workflows/factors.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/factors.yml diff --git a/.github/workflows/factors.yml b/.github/workflows/factors.yml new file mode 100644 index 000000000..e7b2c080c --- /dev/null +++ b/.github/workflows/factors.yml @@ -0,0 +1,20 @@ +# TODO: remove after factors branch passes regular CI +name: Factors +on: + push: + branches: ["factors"] + pull_request: + branches: ["factors"] +jobs: + factors-tests: + runs-on: ubuntu-22.04-4core-spin + steps: + - uses: actions/checkout@v3 + - name: setup dependencies + uses: ./.github/actions/spin-ci-dependencies + with: + rust: true + rust-wasm: true + rust-cache: true + - name: Run factors tests + run: ./run-factors-tests.sh From ab4d65c0103e993cd40a08676941411337aa2920 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 19 Jul 2024 09:56:05 -0400 Subject: [PATCH 072/195] Use TODO(factors) Signed-off-by: Lann Martin --- .github/workflows/factors.yml | 2 +- crates/key-value/src/lib.rs | 2 +- crates/sqlite/src/lib.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/factors.yml b/.github/workflows/factors.yml index e7b2c080c..68ecf8fb6 100644 --- a/.github/workflows/factors.yml +++ b/.github/workflows/factors.yml @@ -1,4 +1,4 @@ -# TODO: remove after factors branch passes regular CI +# TODO(factors): remove after factors branch passes regular CI name: Factors on: push: diff --git a/crates/key-value/src/lib.rs b/crates/key-value/src/lib.rs index 08a656c5d..78e494000 100644 --- a/crates/key-value/src/lib.rs +++ b/crates/key-value/src/lib.rs @@ -5,7 +5,7 @@ use spin_world::v2::key_value; use std::{collections::HashSet, sync::Arc}; use table::Table; -// TODO: Code left for reference; remove after migration to factors +// TODO(factors): Code left for reference; remove after migration to factors // mod host_component; mod util; diff --git a/crates/sqlite/src/lib.rs b/crates/sqlite/src/lib.rs index a56f40474..110dfaca0 100644 --- a/crates/sqlite/src/lib.rs +++ b/crates/sqlite/src/lib.rs @@ -1,4 +1,4 @@ -// TODO: Code left for reference; remove after migration to factors +// TODO(factors): Code left for reference; remove after migration to factors // mod host_component; use spin_app::MetadataKey; From b48c0475298009568ae4415df146b6bba9e5af07 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 19 Jul 2024 17:33:18 +0200 Subject: [PATCH 073/195] Prefer associate type Signed-off-by: Ryan Levick --- .../src/delegating_resolver.rs | 4 +++- crates/factor-key-value/src/lib.rs | 4 ++-- crates/factor-key-value/src/runtime_config.rs | 7 ++++-- crates/factor-sqlite/src/lib.rs | 4 ++-- crates/factor-sqlite/src/runtime_config.rs | 6 +++-- .../factor-sqlite/src/runtime_config/spin.rs | 24 ++++++++++++------- crates/factor-sqlite/tests/factor.rs | 15 ++++-------- 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/crates/factor-key-value/src/delegating_resolver.rs b/crates/factor-key-value/src/delegating_resolver.rs index 3644c8e28..73ce2605e 100644 --- a/crates/factor-key-value/src/delegating_resolver.rs +++ b/crates/factor-key-value/src/delegating_resolver.rs @@ -36,7 +36,9 @@ impl DelegatingRuntimeConfigResolver { } } -impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { +impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { + type Config = StoreConfig; + fn get_store(&self, config: StoreConfig) -> anyhow::Result> { let store_kind = config.type_.as_str(); let store_from_toml = self diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index 269eed5e1..e7320445c 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -21,12 +21,12 @@ use spin_key_value::{ pub use store::MakeKeyValueStore; pub struct KeyValueFactor { - runtime_config_resolver: Arc>, + runtime_config_resolver: Arc>, } impl KeyValueFactor { pub fn new( - runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, + runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, ) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index dbafa7431..b7d719570 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -15,9 +15,12 @@ impl FactorRuntimeConfig for RuntimeConfig { } /// Resolves some piece of runtime configuration to a connection pool -pub trait RuntimeConfigResolver: Send + Sync { +pub trait RuntimeConfigResolver: Send + Sync { + /// The type of configuration that this resolver can handle. + type Config: DeserializeOwned; + /// Get a store manager for a given config. - fn get_store(&self, config: C) -> anyhow::Result>; + fn get_store(&self, config: Self::Config) -> anyhow::Result>; /// Returns a default store manager for a given label. Should only be called /// if there is no runtime configuration for the label. diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 3139a86a3..a6eab56bf 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -14,13 +14,13 @@ use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; pub struct SqliteFactor { - runtime_config_resolver: Arc>, + runtime_config_resolver: Arc>, } impl SqliteFactor { /// Create a new `SqliteFactor` pub fn new( - runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, + runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, ) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index 841a5e2c2..df29a1bf1 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -19,10 +19,12 @@ impl FactorRuntimeConfig for RuntimeConfig { } /// Resolves some piece of runtime configuration to a connection pool -pub trait RuntimeConfigResolver: Send + Sync { +pub trait RuntimeConfigResolver: Send + Sync { + type Config; + /// Get a connection pool for a given config. /// - fn get_pool(&self, config: C) -> anyhow::Result>; + fn get_pool(&self, config: Self::Config) -> anyhow::Result>; /// If there is no runtime configuration for a given database label, return a default connection pool. /// diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index f8d8c33d4..003120661 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -50,21 +50,29 @@ impl SpinSqliteRuntimeConfig { } } -impl RuntimeConfigResolver<(&str, toml::Table)> for SpinSqliteRuntimeConfig { - fn get_pool( - &self, - (database_kind, config): (&str, toml::Table), - ) -> anyhow::Result> { +#[derive(Deserialize)] +pub struct RuntimeConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} + +impl RuntimeConfigResolver for SpinSqliteRuntimeConfig { + type Config = RuntimeConfig; + + fn get_pool(&self, config: RuntimeConfig) -> anyhow::Result> { + let database_kind = config.type_.as_str(); let pool = match database_kind { "spin" => { - let config: LocalDatabase = config.try_into()?; + let config: LocalDatabase = config.config.try_into()?; config.pool(&self.local_database_dir)? } "libsql" => { - let config: LibSqlDatabase = config.try_into()?; + let config: LibSqlDatabase = config.config.try_into()?; config.pool()? } - _ => anyhow::bail!("Unknown database kind: {}", database_kind), + _ => anyhow::bail!("Unknown database kind: {database_kind}"), }; Ok(Arc::new(pool)) } diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index 6ce8b06d2..bff9218e3 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -1,7 +1,6 @@ use std::{collections::HashSet, sync::Arc}; -use factor_sqlite::SqliteFactor; -use serde::Deserialize; +use factor_sqlite::{runtime_config::spin::RuntimeConfig, SqliteFactor}; use spin_factors::{ anyhow::{self, bail}, RuntimeFactors, @@ -91,7 +90,9 @@ impl RuntimeConfigResolver { } } -impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { +impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { + type Config = RuntimeConfig; + fn get_pool( &self, config: RuntimeConfig, @@ -119,11 +120,3 @@ impl factor_sqlite::ConnectionPool for InvalidConnectionPool { Err(spin_world::v2::sqlite::Error::InvalidConnection) } } - -#[derive(Deserialize)] -pub struct RuntimeConfig { - #[serde(rename = "type")] - pub type_: String, - #[serde(flatten)] - pub config: toml::Table, -} From 4ec282b1f6fbf9c3b6b281ace76b4df44194129b Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 19 Jul 2024 11:12:20 -0400 Subject: [PATCH 074/195] factors: Add build script for factors smoke test app Signed-off-by: Lann Martin --- Cargo.lock | 1 + crates/factors/Cargo.toml | 3 +++ crates/factors/build.rs | 19 +++++++++++++++++++ crates/factors/tests/smoke-app/.gitignore | 1 + crates/factors/tests/smoke-app/spin.toml | 8 ++------ 5 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 crates/factors/build.rs diff --git a/Cargo.lock b/Cargo.lock index aa48f93f9..e4bd2fdc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7631,6 +7631,7 @@ name = "spin-factors" version = "2.7.0-pre0" dependencies = [ "anyhow", + "cargo-target-dep", "http 1.1.0", "http-body-util", "serde 1.0.197", diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 7306e5294..3efbb2e02 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -31,5 +31,8 @@ tokio = { version = "1", features = ["macros", "rt", "sync"] } toml = "0.8" wasmtime-wasi-http = { workspace = true } +[build-dependencies] +cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "482f269eceb7b1a7e8fc618bf8c082dd24979cf1" } + [lints] workspace = true diff --git a/crates/factors/build.rs b/crates/factors/build.rs new file mode 100644 index 000000000..923b145fe --- /dev/null +++ b/crates/factors/build.rs @@ -0,0 +1,19 @@ +use std::path::Path; + +use cargo_target_dep::build_target_dep; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + // Enable spin-factors-derive to emit expanded macro output. + let out_dir = std::env::var("OUT_DIR").unwrap(); + println!("cargo:rustc-env=SPIN_FACTORS_DERIVE_EXPAND_DIR={out_dir}"); + + let root = "tests/smoke-app"; + build_target_dep(root, Path::new("tests/smoke-app/smoke_app.wasm")) + .release() + .target("wasm32-wasi") + .build(); + println!("cargo:rerun-if-changed={root}/Cargo.toml"); + println!("cargo:rerun-if-changed={root}/Cargo.lock"); + println!("cargo:rerun-if-changed={root}/src"); +} diff --git a/crates/factors/tests/smoke-app/.gitignore b/crates/factors/tests/smoke-app/.gitignore index 386474fa5..671b13b2a 100644 --- a/crates/factors/tests/smoke-app/.gitignore +++ b/crates/factors/tests/smoke-app/.gitignore @@ -1,2 +1,3 @@ target/ .spin/ +*.wasm diff --git a/crates/factors/tests/smoke-app/spin.toml b/crates/factors/tests/smoke-app/spin.toml index a800b49fe..a49ca70f8 100644 --- a/crates/factors/tests/smoke-app/spin.toml +++ b/crates/factors/tests/smoke-app/spin.toml @@ -15,11 +15,7 @@ route = "/..." component = "smoke-app" [component.smoke-app] -source = "target/wasm32-wasi/release/smoke_app.wasm" +source = "smoke_app.wasm" allowed_outbound_hosts = ["https://{{ host }}"] key_value_stores = ["default"] -variables = { "other" = "<{{ other }}>"} - -[component.smoke-app.build] -command = "cargo build --target wasm32-wasi --release" -watch = ["src/**/*.rs", "Cargo.toml"] +variables = { "other" = "<{{ other }}>"} \ No newline at end of file From 10f7b42910feb79ffd9e57844a80bedfc11a9cd5 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 22 Jul 2024 13:44:58 +0200 Subject: [PATCH 075/195] Be generic over config resolver Signed-off-by: Ryan Levick --- crates/factor-key-value/src/lib.rs | 17 +++++++---------- crates/factor-sqlite/src/lib.rs | 18 +++++++++--------- crates/factor-sqlite/src/runtime_config.rs | 2 +- crates/factor-sqlite/tests/factor.rs | 2 +- crates/factors/tests/smoke.rs | 2 +- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index e7320445c..2a7cbeea3 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -8,8 +8,7 @@ use std::{ }; use anyhow::ensure; -use runtime_config::RuntimeConfig; -use serde::de::DeserializeOwned; +use runtime_config::{RuntimeConfig, RuntimeConfigResolver}; use spin_factors::{ ConfigureAppContext, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, @@ -20,22 +19,20 @@ use spin_key_value::{ }; pub use store::MakeKeyValueStore; -pub struct KeyValueFactor { - runtime_config_resolver: Arc>, +pub struct KeyValueFactor { + runtime_config_resolver: Arc, } -impl KeyValueFactor { - pub fn new( - runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, - ) -> Self { +impl KeyValueFactor { + pub fn new(runtime_config_resolver: R) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), } } } -impl Factor for KeyValueFactor { - type RuntimeConfig = RuntimeConfig; +impl Factor for KeyValueFactor { + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceBuilder; diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index a6eab56bf..1c270b9a4 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -7,29 +7,29 @@ use std::sync::Arc; use host::InstanceState; use async_trait::async_trait; -use serde::de::DeserializeOwned; +use runtime_config::RuntimeConfigResolver; use spin_factors::{anyhow, Factor}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; -pub struct SqliteFactor { - runtime_config_resolver: Arc>, +pub struct SqliteFactor { + runtime_config_resolver: Arc, } -impl SqliteFactor { +impl SqliteFactor { /// Create a new `SqliteFactor` - pub fn new( - runtime_config_resolver: impl runtime_config::RuntimeConfigResolver + 'static, - ) -> Self { + /// + /// Takes a `runtime_config_resolver` that can resolve a runtime configuration into a connection pool. + pub fn new(runtime_config_resolver: R) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), } } } -impl Factor for SqliteFactor { - type RuntimeConfig = runtime_config::RuntimeConfig; +impl Factor for SqliteFactor { + type RuntimeConfig = runtime_config::RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index df29a1bf1..10e6f8e72 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -20,7 +20,7 @@ impl FactorRuntimeConfig for RuntimeConfig { /// Resolves some piece of runtime configuration to a connection pool pub trait RuntimeConfigResolver: Send + Sync { - type Config; + type Config: DeserializeOwned; /// Get a connection pool for a given config. /// diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index bff9218e3..6897e015b 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -9,7 +9,7 @@ use spin_factors_test::{toml, TestEnvironment}; #[derive(RuntimeFactors)] struct TestFactors { - sqlite: SqliteFactor, + sqlite: SqliteFactor, } #[tokio::test] diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 5244e3bd2..6ad949c13 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -22,7 +22,7 @@ struct Factors { variables: VariablesFactor, outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, - key_value: KeyValueFactor, + key_value: KeyValueFactor, } struct Data { From fcbf144dcc16aafe87f0c1bf4ad7e8b5b571cb09 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 22 Jul 2024 15:20:24 +0200 Subject: [PATCH 076/195] Add environment variables provider Signed-off-by: Ryan Levick --- Cargo.lock | 2 + crates/factor-variables/Cargo.toml | 3 + crates/factor-variables/src/lib.rs | 10 +- crates/factor-variables/src/provider.rs | 35 +-- crates/factor-variables/src/provider/env.rs | 215 ++++++++++++++++++ .../factor-variables/src/provider/statik.rs | 34 +++ 6 files changed, 267 insertions(+), 32 deletions(-) create mode 100644 crates/factor-variables/src/provider/env.rs create mode 100644 crates/factor-variables/src/provider/statik.rs diff --git a/Cargo.lock b/Cargo.lock index a2c7f42a0..395f90357 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7630,6 +7630,7 @@ dependencies = [ name = "spin-factor-variables" version = "2.7.0-pre0" dependencies = [ + "dotenvy", "serde 1.0.197", "spin-expressions", "spin-factors", @@ -7637,6 +7638,7 @@ dependencies = [ "spin-world", "tokio", "toml 0.8.14", + "tracing", ] [[package]] diff --git a/crates/factor-variables/Cargo.toml b/crates/factor-variables/Cargo.toml index 30465c209..ac7370d70 100644 --- a/crates/factor-variables/Cargo.toml +++ b/crates/factor-variables/Cargo.toml @@ -5,11 +5,14 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] +dotenvy = "0.15" serde = { version = "1.0", features = ["rc"] } spin-expressions = { path = "../expressions" } spin-factors = { path = "../factors" } spin-world = { path = "../world" } +tokio = { version = "1", features = ["rt-multi-thread"] } toml = "0.8" +tracing = { workspace = true } [dev-dependencies] spin-factors-test = { path = "../factors-test" } diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 596dc6e74..22c26022b 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -1,8 +1,7 @@ -mod provider; +pub mod provider; use std::{collections::HashMap, sync::Arc}; -use provider::{provider_from_toml_fn, ProviderFromToml}; use serde::Deserialize; use spin_expressions::ProviderResolver; use spin_factors::{ @@ -16,7 +15,7 @@ pub use provider::{MakeVariablesProvider, StaticVariables}; #[derive(Default)] pub struct VariablesFactor { - provider_types: HashMap<&'static str, ProviderFromToml>, + provider_types: HashMap<&'static str, provider::ProviderFromToml>, } impl VariablesFactor { @@ -26,7 +25,10 @@ impl VariablesFactor { ) -> anyhow::Result<()> { if self .provider_types - .insert(T::RUNTIME_CONFIG_TYPE, provider_from_toml_fn(provider_type)) + .insert( + T::RUNTIME_CONFIG_TYPE, + provider::provider_from_toml_fn(provider_type), + ) .is_some() { bail!("duplicate provider type {:?}", T::RUNTIME_CONFIG_TYPE); diff --git a/crates/factor-variables/src/provider.rs b/crates/factor-variables/src/provider.rs index f3ee94811..f34945f1c 100644 --- a/crates/factor-variables/src/provider.rs +++ b/crates/factor-variables/src/provider.rs @@ -1,7 +1,11 @@ -use std::{collections::HashMap, sync::Arc}; +mod env; +mod statik; -use serde::{de::DeserializeOwned, Deserialize}; -use spin_expressions::{async_trait::async_trait, Key, Provider}; +pub use env::EnvVariables; +pub use statik::StaticVariables; + +use serde::de::DeserializeOwned; +use spin_expressions::Provider; use spin_factors::anyhow; pub trait MakeVariablesProvider: 'static { @@ -24,28 +28,3 @@ pub(crate) fn provider_from_toml_fn( Ok(Box::new(provider)) }) } - -pub struct StaticVariables; - -impl MakeVariablesProvider for StaticVariables { - const RUNTIME_CONFIG_TYPE: &'static str = "static"; - - type RuntimeConfig = StaticVariablesProvider; - type Provider = StaticVariablesProvider; - - fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result { - Ok(runtime_config) - } -} - -#[derive(Debug, Deserialize)] -pub struct StaticVariablesProvider { - values: Arc>, -} - -#[async_trait] -impl Provider for StaticVariablesProvider { - async fn get(&self, key: &Key) -> anyhow::Result> { - Ok(self.values.get(key.as_str()).cloned()) - } -} diff --git a/crates/factor-variables/src/provider/env.rs b/crates/factor-variables/src/provider/env.rs new file mode 100644 index 000000000..485c061eb --- /dev/null +++ b/crates/factor-variables/src/provider/env.rs @@ -0,0 +1,215 @@ +use std::{ + collections::HashMap, + env::VarError, + path::{Path, PathBuf}, + sync::OnceLock, +}; + +use serde::Deserialize; +use spin_expressions::{Key, Provider}; +use spin_factors::anyhow::{self, Context as _}; +use spin_world::async_trait; +use tracing::{instrument, Level}; + +use crate::MakeVariablesProvider; + +/// Creator of a environment variables provider. +pub struct EnvVariables; + +impl MakeVariablesProvider for EnvVariables { + const RUNTIME_CONFIG_TYPE: &'static str = "env"; + + type RuntimeConfig = EnvVariablesConfig; + type Provider = EnvVariablesProvider; + + fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result { + Ok(EnvVariablesProvider::new( + runtime_config.prefix, + |key| std::env::var(key), + runtime_config.dotenv_path, + )) + } +} + +/// Configuration for the environment variables provider. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvVariablesConfig { + /// A prefix to add to variable names when resolving from the environment. + /// + /// Unless empty, joined to the variable name with an underscore. + #[serde(default)] + pub prefix: Option, + /// Optional path to a 'dotenv' file which will be merged into the environment. + #[serde(default)] + pub dotenv_path: Option, +} + +const DEFAULT_ENV_PREFIX: &str = "SPIN_VARIABLE"; + +/// A config Provider that uses environment variables. +pub struct EnvVariablesProvider { + prefix: Option, + env_fetcher: Box Result + Send + Sync>, + dotenv_path: Option, + dotenv_cache: OnceLock>, +} + +impl EnvVariablesProvider { + /// Creates a new EnvProvider. + /// + /// * `prefix` - The string prefix to use to distinguish an environment variable that should be used. + /// If not set, the default prefix is used. + /// * `env_fetcher` - The function to use to fetch an environment variable. + /// * `dotenv_path` - The path to the .env file to load environment variables from. If not set, + /// no .env file is loaded. + pub fn new( + prefix: Option>, + env_fetcher: impl Fn(&str) -> Result + Send + Sync + 'static, + dotenv_path: Option, + ) -> Self { + Self { + prefix: prefix.map(Into::into), + dotenv_path, + env_fetcher: Box::new(env_fetcher), + dotenv_cache: Default::default(), + } + } + + /// Gets the value of a variable from the environment. + fn get_sync(&self, key: &Key) -> anyhow::Result> { + let prefix = self + .prefix + .clone() + .unwrap_or(DEFAULT_ENV_PREFIX.to_string()); + + let upper_key = key.as_ref().to_ascii_uppercase(); + let env_key = format!("{prefix}_{upper_key}"); + + self.query_env(&env_key) + } + + /// Queries the environment for a variable defaulting to dotenv. + fn query_env(&self, env_key: &str) -> anyhow::Result> { + match (self.env_fetcher)(env_key) { + Err(std::env::VarError::NotPresent) => self.get_dotenv(env_key), + other => other + .map(Some) + .with_context(|| format!("failed to resolve env var {env_key}")), + } + } + + fn get_dotenv(&self, key: &str) -> anyhow::Result> { + let Some(dotenv_path) = self.dotenv_path.as_deref() else { + return Ok(None); + }; + let cache = match self.dotenv_cache.get() { + Some(cache) => cache, + None => { + let cache = load_dotenv(dotenv_path)?; + let _ = self.dotenv_cache.set(cache); + // Safe to unwrap because we just set the cache. + // Ensures we always get the first value set. + self.dotenv_cache.get().unwrap() + } + }; + Ok(cache.get(key).cloned()) + } +} + +impl std::fmt::Debug for EnvVariablesProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EnvProvider") + .field("prefix", &self.prefix) + .field("dotenv_path", &self.dotenv_path) + .finish() + } +} + +fn load_dotenv(dotenv_path: &Path) -> anyhow::Result> { + Ok(dotenvy::from_path_iter(dotenv_path) + .into_iter() + .flatten() + .collect::, _>>()?) +} + +#[async_trait] +impl Provider for EnvVariablesProvider { + #[instrument(name = "spin_variables.get_from_env", skip(self), err(level = Level::INFO))] + async fn get(&self, key: &Key) -> anyhow::Result> { + tokio::task::block_in_place(|| self.get_sync(key)) + } +} + +#[cfg(test)] +mod test { + use std::env::temp_dir; + + use super::*; + + struct TestEnv { + map: HashMap, + } + + impl TestEnv { + fn new() -> Self { + Self { + map: Default::default(), + } + } + + fn insert(&mut self, key: &str, value: &str) { + self.map.insert(key.to_string(), value.to_string()); + } + + fn get(&self, key: &str) -> Result { + self.map.get(key).cloned().ok_or(VarError::NotPresent) + } + } + + #[test] + fn provider_get() { + let mut env = TestEnv::new(); + env.insert("TESTING_SPIN_ENV_KEY1", "val"); + let key1 = Key::new("env_key1").unwrap(); + assert_eq!( + EnvVariablesProvider::new(Some("TESTING_SPIN"), move |key| env.get(key), None) + .get_sync(&key1) + .unwrap(), + Some("val".to_string()) + ); + } + + #[test] + fn provider_get_dotenv() { + let dotenv_path = temp_dir().join("spin-env-provider-test"); + std::fs::write(&dotenv_path, b"TESTING_SPIN_ENV_KEY2=dotenv_val").unwrap(); + + let key = Key::new("env_key2").unwrap(); + assert_eq!( + EnvVariablesProvider::new( + Some("TESTING_SPIN"), + |_| Err(VarError::NotPresent), + Some(dotenv_path) + ) + .get_sync(&key) + .unwrap(), + Some("dotenv_val".to_string()) + ); + } + + #[test] + fn provider_get_missing() { + let key = Key::new("definitely_not_set").unwrap(); + assert_eq!( + EnvVariablesProvider::new( + Some("TESTING_SPIN"), + |_| Err(VarError::NotPresent), + Default::default() + ) + .get_sync(&key) + .unwrap(), + None + ); + } +} diff --git a/crates/factor-variables/src/provider/statik.rs b/crates/factor-variables/src/provider/statik.rs new file mode 100644 index 000000000..222c7168e --- /dev/null +++ b/crates/factor-variables/src/provider/statik.rs @@ -0,0 +1,34 @@ +use std::{collections::HashMap, sync::Arc}; + +use serde::Deserialize; +use spin_expressions::{async_trait::async_trait, Key, Provider}; +use spin_factors::anyhow; + +use crate::MakeVariablesProvider; + +/// Creator of a static variables provider. +pub struct StaticVariables; + +impl MakeVariablesProvider for StaticVariables { + const RUNTIME_CONFIG_TYPE: &'static str = "static"; + + type RuntimeConfig = StaticVariablesProvider; + type Provider = StaticVariablesProvider; + + fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result { + Ok(runtime_config) + } +} + +/// A variables provider that reads variables from an static map. +#[derive(Debug, Deserialize)] +pub struct StaticVariablesProvider { + values: Arc>, +} + +#[async_trait] +impl Provider for StaticVariablesProvider { + async fn get(&self, key: &Key) -> anyhow::Result> { + Ok(self.values.get(key.as_str()).cloned()) + } +} From d58314d76e31c81ddd3173fbd06f0e0b53c14676 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Fri, 19 Jul 2024 13:58:54 -0700 Subject: [PATCH 077/195] factors: key value tests, doc comments, and Azure factor Signed-off-by: Kate Goldenring --- Cargo.lock | 77 +++++++ crates/factor-key-value-azure/Cargo.toml | 19 ++ crates/factor-key-value-azure/src/lib.rs | 35 +++ crates/factor-key-value-redis/src/lib.rs | 4 + crates/factor-key-value-spin/src/lib.rs | 10 + crates/factor-key-value/Cargo.toml | 8 + .../src/delegating_resolver.rs | 11 +- crates/factor-key-value/src/lib.rs | 22 +- crates/factor-key-value/src/runtime_config.rs | 4 +- crates/factor-key-value/src/store.rs | 8 +- crates/factor-key-value/tests/test.rs | 214 ++++++++++++++++++ crates/key-value/src/lib.rs | 4 + 12 files changed, 410 insertions(+), 6 deletions(-) create mode 100644 crates/factor-key-value-azure/Cargo.toml create mode 100644 crates/factor-key-value-azure/src/lib.rs create mode 100644 crates/factor-key-value/tests/test.rs diff --git a/Cargo.lock b/Cargo.lock index 395f90357..849b999f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2672,6 +2672,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.30" @@ -6084,6 +6090,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.7.3" @@ -6128,6 +6147,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -6214,6 +6248,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "reborrow" version = "0.5.5" @@ -6349,6 +6392,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "reqwest" version = "0.11.27" @@ -7528,12 +7580,27 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "serde 1.0.197", + "spin-factor-key-value-redis", + "spin-factor-key-value-spin", "spin-factors", + "spin-factors-test", "spin-key-value", "spin-world", + "tempdir", + "tokio", "toml 0.8.14", ] +[[package]] +name = "spin-factor-key-value-azure" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "serde 1.0.197", + "spin-factor-key-value", + "spin-key-value-azure", +] + [[package]] name = "spin-factor-key-value-redis" version = "2.7.0-pre0" @@ -8461,6 +8528,16 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f227968ec00f0e5322f9b8173c7a0cbcff6181a0a5b28e9892491c286277231" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.10.1" diff --git a/crates/factor-key-value-azure/Cargo.toml b/crates/factor-key-value-azure/Cargo.toml new file mode 100644 index 000000000..318855ff9 --- /dev/null +++ b/crates/factor-key-value-azure/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "spin-factor-key-value-azure" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = "1.0" +serde = { version = "1.0", features = ["rc"] } +spin-factor-key-value = { path = "../factor-key-value" } +# TODO: merge with this crate +spin-key-value-azure = { path = "../key-value-azure" } + +[lints] +workspace = true diff --git a/crates/factor-key-value-azure/src/lib.rs b/crates/factor-key-value-azure/src/lib.rs new file mode 100644 index 000000000..cc5f67026 --- /dev/null +++ b/crates/factor-key-value-azure/src/lib.rs @@ -0,0 +1,35 @@ +use serde::Deserialize; +use spin_factor_key_value::MakeKeyValueStore; +use spin_key_value_azure::KeyValueAzureCosmos; + +/// A key-value store that uses Azure Cosmos as the backend. +pub struct AzureKeyValueStore; + +/// Runtime configuration for the Azure Cosmos key-value store. +#[derive(Deserialize)] +pub struct AzureCosmosKeyValueRuntimeConfig { + /// The authorization token for the Azure Cosmos DB account. + key: String, + /// The Azure Cosmos DB account name. + account: String, + /// The Azure Cosmos DB database. + database: String, + /// The Azure Cosmos DB container where data is stored. + /// The CosmosDB container must be created with the default partition key, /id + container: String, +} + +impl MakeKeyValueStore for AzureKeyValueStore { + const RUNTIME_CONFIG_TYPE: &'static str = "azure_cosmos"; + + type RuntimeConfig = AzureCosmosKeyValueRuntimeConfig; + + type StoreManager = KeyValueAzureCosmos; + + fn make_store( + &self, + runtime_config: Self::RuntimeConfig, + ) -> anyhow::Result { + KeyValueAzureCosmos::new(runtime_config.key, runtime_config.account, runtime_config.database, runtime_config.container) + } +} diff --git a/crates/factor-key-value-redis/src/lib.rs b/crates/factor-key-value-redis/src/lib.rs index 4fd45620e..39fadd26f 100644 --- a/crates/factor-key-value-redis/src/lib.rs +++ b/crates/factor-key-value-redis/src/lib.rs @@ -1,10 +1,14 @@ use serde::Deserialize; use spin_factor_key_value::MakeKeyValueStore; use spin_key_value_redis::KeyValueRedis; + +/// A key-value store that uses Redis as the backend. pub struct RedisKeyValueStore; +/// Runtime configuration for the Redis key-value store. #[derive(Deserialize)] pub struct RedisKeyValueRuntimeConfig { + /// The URL of the Redis server. url: String, } diff --git a/crates/factor-key-value-spin/src/lib.rs b/crates/factor-key-value-spin/src/lib.rs index db19b54d3..40d8e7e03 100644 --- a/crates/factor-key-value-spin/src/lib.rs +++ b/crates/factor-key-value-spin/src/lib.rs @@ -8,11 +8,15 @@ use serde::{Deserialize, Serialize}; use spin_factor_key_value::MakeKeyValueStore; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; +/// A key-value store that uses SQLite as the backend. pub struct SpinKeyValueStore { + /// The base path or directory for the SQLite database file. base_path: PathBuf, } impl SpinKeyValueStore { + /// Create a new SpinKeyValueStore with the given base path. + /// If the base path is None, the current directory is used. pub fn new(base_path: Option) -> anyhow::Result { let base_path = match base_path { Some(path) => path, @@ -22,14 +26,20 @@ impl SpinKeyValueStore { } } +/// Runtime configuration for the SQLite key-value store. #[derive(Deserialize, Serialize)] pub struct SpinKeyValueRuntimeConfig { + /// The path to the SQLite database file. path: Option, } impl SpinKeyValueRuntimeConfig { + /// The default filename for the SQLite database. const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; + /// Create a new runtime configuration with the given state directory. + /// If the state directory is None, the database is in-memory. + /// If the state directory is Some, the database is stored in a file in the state directory. pub fn default(state_dir: Option) -> Self { let path = state_dir.map(|dir| dir.join(Self::DEFAULT_SPIN_STORE_FILENAME)); Self { path } diff --git a/crates/factor-key-value/Cargo.toml b/crates/factor-key-value/Cargo.toml index 89e060e46..cab19b014 100644 --- a/crates/factor-key-value/Cargo.toml +++ b/crates/factor-key-value/Cargo.toml @@ -13,5 +13,13 @@ spin-key-value = { path = "../key-value" } spin-world = { path = "../world" } toml = "0.8" +[dev-dependencies] +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } +spin-factor-key-value-spin = { path = "../factor-key-value-spin" } +spin-factor-key-value-redis = { path = "../factor-key-value-redis" } +tempdir = "0.3" + + [lints] workspace = true diff --git a/crates/factor-key-value/src/delegating_resolver.rs b/crates/factor-key-value/src/delegating_resolver.rs index 73ce2605e..f655fd202 100644 --- a/crates/factor-key-value/src/delegating_resolver.rs +++ b/crates/factor-key-value/src/delegating_resolver.rs @@ -4,13 +4,19 @@ use serde::Deserialize; use spin_key_value::StoreManager; use std::{collections::HashMap, sync::Arc}; +/// A RuntimeConfigResolver that delegates to the appropriate key-value store for a given label. +/// The store types are registered with the resolver using `add_store_type`. +/// The default store for a label is registered using `add_default_store`. #[derive(Default)] pub struct DelegatingRuntimeConfigResolver { + /// A map of store types to a function that returns the appropriate store manager from runtime config TOML. store_types: HashMap<&'static str, StoreFromToml>, + /// A map of default store configurations for a label. defaults: HashMap<&'static str, StoreConfig>, } impl DelegatingRuntimeConfigResolver { + /// Create a new DelegatingRuntimeConfigResolver. pub fn new() -> Self { Self::default() } @@ -18,9 +24,8 @@ impl DelegatingRuntimeConfigResolver { pub fn add_default_store(&mut self, label: &'static str, config: StoreConfig) { self.defaults.insert(label, config); } -} -impl DelegatingRuntimeConfigResolver { + /// Adds a store type to the resolver. pub fn add_store_type(&mut self, store_type: T) -> anyhow::Result<()> { if self .store_types @@ -48,6 +53,8 @@ impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { store_from_toml(config.config) } + /// Get the default store manager for the given label. + /// Returns None if no default store is registered for the label. fn default_store(&self, label: &str) -> Option> { let config = self.defaults.get(label)?; self.get_store(config.clone()).ok() diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index 2a7cbeea3..109ced6e0 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -1,6 +1,7 @@ pub mod delegating_resolver; mod runtime_config; mod store; +pub use delegating_resolver::DelegatingRuntimeConfigResolver; use std::{ collections::{HashMap, HashSet}, @@ -19,11 +20,14 @@ use spin_key_value::{ }; pub use store::MakeKeyValueStore; +/// A factor that provides key-value storage. pub struct KeyValueFactor { + /// Resolves runtime configuration into store managers. runtime_config_resolver: Arc, } impl KeyValueFactor { + /// Create a new KeyValueFactor. pub fn new(runtime_config_resolver: R) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), @@ -46,12 +50,17 @@ impl Factor for KeyValueFactor { &self, mut ctx: ConfigureAppContext, ) -> anyhow::Result { - // Build StoreManager from runtime config + // Build store manager from runtime config let mut store_managers: HashMap> = HashMap::new(); if let Some(runtime_config) = ctx.take_runtime_config() { for (store_label, config) in runtime_config.store_configs { let store = self.runtime_config_resolver.get_store(config)?; - store_managers.insert(store_label, store); + if let std::collections::hash_map::Entry::Vacant(e) = + store_managers.entry(store_label) + { + let store = self.runtime_config_resolver.get_store(config)?; + e.insert(store); + } } } let resolver_clone = self.runtime_config_resolver.clone(); @@ -110,12 +119,21 @@ impl Factor for KeyValueFactor { type AppStoreManager = CachingStoreManager; pub struct AppState { + /// The store manager for the app. This is a cache around a delegating store + /// manager. For `get` requests, first checks the cache before delegating to + /// the underlying store manager. store_manager: Arc, + /// The allowed stores for each component. + /// This is a map from component ID to the set of store labels that the component is allowed to use. component_allowed_stores: HashMap>, } pub struct InstanceBuilder { + /// The store manager for the app. This is a cache around a delegating store + /// manager. For `get` requests, first checks the cache before delegating to + /// the underlying store manager. store_manager: Arc, + /// The allowed stores for this component instance. allowed_stores: HashSet, } diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index b7d719570..74fddeb54 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -4,9 +4,11 @@ use serde::{de::DeserializeOwned, Deserialize}; use spin_factors::{anyhow, FactorRuntimeConfig}; use spin_key_value::StoreManager; +/// Runtime configuration for all key value stores. #[derive(Deserialize)] #[serde(transparent)] pub struct RuntimeConfig { + /// Map of store names to store configurations. pub store_configs: HashMap, } @@ -14,7 +16,7 @@ impl FactorRuntimeConfig for RuntimeConfig { const KEY: &'static str = "key_value_store"; } -/// Resolves some piece of runtime configuration to a connection pool +/// Resolves some piece of runtime configuration to a key value store manager. pub trait RuntimeConfigResolver: Send + Sync { /// The type of configuration that this resolver can handle. type Config: DeserializeOwned; diff --git a/crates/factor-key-value/src/store.rs b/crates/factor-key-value/src/store.rs index 647f90172..24c5e95bb 100644 --- a/crates/factor-key-value/src/store.rs +++ b/crates/factor-key-value/src/store.rs @@ -4,19 +4,25 @@ use anyhow::Context; use serde::de::DeserializeOwned; use spin_key_value::StoreManager; +/// Defines the construction of a key value store. pub trait MakeKeyValueStore: 'static + Send + Sync { + /// Unique type identifier for the store. const RUNTIME_CONFIG_TYPE: &'static str; - + /// Runtime configuration for the store. type RuntimeConfig: DeserializeOwned; + /// The store manager for the store. type StoreManager: StoreManager; + /// Creates a new store manager from the runtime configuration. fn make_store(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result; } +/// A function that creates a store manager from a TOML table. pub(crate) type StoreFromToml = Box anyhow::Result> + Send + Sync>; +/// Creates a `StoreFromToml` function from a `MakeKeyValueStore` implementation. pub(crate) fn store_from_toml_fn(provider_type: T) -> StoreFromToml { Box::new(move |table| { let runtime_config: T::RuntimeConfig = diff --git a/crates/factor-key-value/tests/test.rs b/crates/factor-key-value/tests/test.rs new file mode 100644 index 000000000..77f6e1812 --- /dev/null +++ b/crates/factor-key-value/tests/test.rs @@ -0,0 +1,214 @@ +use std::collections::HashSet; +use spin_factor_key_value::{DelegatingRuntimeConfigResolver, KeyValueFactor, MakeKeyValueStore}; +use spin_factor_key_value_redis::RedisKeyValueStore; +use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; +use spin_factors::RuntimeFactors; +use spin_factors_test::{toml, TestEnvironment}; + +#[derive(RuntimeFactors)] +struct TestFactors { + key_value: KeyValueFactor, +} + +fn default_key_value_resolver( +) -> anyhow::Result<(DelegatingRuntimeConfigResolver, tempdir::TempDir)> { + let mut test_resolver = DelegatingRuntimeConfigResolver::new(); + test_resolver.add_store_type(SpinKeyValueStore::new(None)?)?; + let tmp_dir = tempdir::TempDir::new("example")?; + let path = tmp_dir.path().to_path_buf(); + let default_config = SpinKeyValueRuntimeConfig::default(Some(path)); + test_resolver.add_default_store( + "default", + SpinKeyValueStore::RUNTIME_CONFIG_TYPE, + toml::value::Table::try_from(default_config)?, + ); + Ok((test_resolver, tmp_dir)) +} + +#[tokio::test] +async fn default_key_value_works() -> anyhow::Result<()> { + let (test_resolver, dir) = default_key_value_resolver()?; + let factors = TestFactors { + key_value: KeyValueFactor::new(test_resolver), + }; + let env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + key_value_stores = ["default"] + }); + let state = env.build_instance_state(factors).await?; + + assert_eq!( + state.key_value.allowed_stores(), + &["default".into()].into_iter().collect::>() + ); + // Ensure the database directory is created + assert!(dir.path().exists()); + Ok(()) +} + +async fn run_test_with_config_and_stores_for_label( + runtime_config: Option, + store_types: Vec, + labels: Vec<&str>, +) -> anyhow::Result<()> { + let mut test_resolver = DelegatingRuntimeConfigResolver::new(); + for store_type in store_types { + test_resolver.add_store_type(store_type)?; + } + let factors = TestFactors { + key_value: KeyValueFactor::new(test_resolver), + }; + let labels_clone = labels.clone(); + let mut env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + key_value_stores = labels_clone + }); + if let Some(runtime_config) = runtime_config { + env.runtime_config.extend(runtime_config); + } + let state = env.build_instance_state(factors).await?; + // String::new("foo").as + assert_eq!( + labels, + state.key_value.allowed_stores().iter().collect::>() + ); + + Ok(()) +} + +#[tokio::test] +async fn overridden_default_key_value_works() -> anyhow::Result<()> { + let runtime_config = toml::toml! { + [key_value_store.default] + type = "redis" + url = "redis://localhost:6379" + }; + run_test_with_config_and_stores_for_label( + Some(runtime_config), + vec![RedisKeyValueStore], + vec!["default"], + ) + .await +} + +#[tokio::test] +async fn custom_spin_key_value_works() -> anyhow::Result<()> { + let runtime_config = toml::toml! { + [key_value_store.custom] + type = "spin" + }; + run_test_with_config_and_stores_for_label( + Some(runtime_config), + vec![SpinKeyValueStore::new(None)?], + vec!["custom"], + ) + .await +} + +#[tokio::test] +async fn custom_spin_key_value_works_with_absolute_path() -> anyhow::Result<()> { + let tmp_dir = tempdir::TempDir::new("example")?; + let path = tmp_dir.path().join("custom.db"); + let path_str = path.to_str().unwrap(); + let runtime_config = toml::toml! { + [key_value_store.custom] + type = "spin" + path = path_str + }; + run_test_with_config_and_stores_for_label( + Some(runtime_config), + vec![SpinKeyValueStore::new(None)?], + vec!["custom"], + ) + .await?; + assert!(tmp_dir.path().exists()); + Ok(()) +} + +#[tokio::test] +async fn custom_spin_key_value_works_with_relative_path() -> anyhow::Result<()> { + let tmp_dir = tempdir::TempDir::new("example")?; + let path = tmp_dir.path().to_owned(); + let runtime_config = toml::toml! { + [key_value_store.custom] + type = "spin" + path = "custom.db" + }; + run_test_with_config_and_stores_for_label( + Some(runtime_config), + vec![SpinKeyValueStore::new(Some(path))?], + vec!["custom"], + ) + .await?; + assert!(tmp_dir.path().exists()); + Ok(()) +} + +#[tokio::test] +async fn custom_redis_key_value_works() -> anyhow::Result<()> { + let runtime_config = toml::toml! { + [key_value_store.custom] + type = "redis" + url = "redis://localhost:6379" + }; + run_test_with_config_and_stores_for_label( + Some(runtime_config), + vec![RedisKeyValueStore], + vec!["custom"], + ) + .await +} + +#[tokio::test] +async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { + let runtime_config = toml::toml! { + [key_value_store.custom] + type = "spin" + path = "/$$&/bad/path/foo.db" + }; + assert!(run_test_with_config_and_stores_for_label( + Some(runtime_config), + vec![SpinKeyValueStore::new(None)?], + vec!["custom"] + ) + .await + .is_err()); + Ok(()) +} + +#[tokio::test] +async fn multiple_custom_key_value_fails() -> anyhow::Result<()> { + let tmp_dir = tempdir::TempDir::new("example")?; + let runtime_config = toml::toml! { + [key_value_store.custom] + type = "spin" + path = "custom.db" + + [key_value_store.custom] + type = "redis" + url = "redis://localhost:6379" + }; + let mut test_resolver = DelegatingRuntimeConfigResolver::new(); + test_resolver.add_store_type(RedisKeyValueStore)?; + test_resolver.add_store_type(SpinKeyValueStore::new(Some(tmp_dir.path().to_owned()))?)?; + let factors = TestFactors { + key_value: KeyValueFactor::new(test_resolver), + }; + let mut env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + key_value_stores = ["custom"] + }); + env.runtime_config.extend(runtime_config); + let state = env.build_instance_state(factors).await?; + + assert_eq!( + state.key_value.allowed_stores(), + &["custom".into()].into_iter().collect::>() + ); + // Check that the first store in the config was chosen by verifying the existence of the DB directory + assert!(tmp_dir.path().exists()); + Ok(()) +} diff --git a/crates/key-value/src/lib.rs b/crates/key-value/src/lib.rs index 78e494000..c9d2b6354 100644 --- a/crates/key-value/src/lib.rs +++ b/crates/key-value/src/lib.rs @@ -61,6 +61,10 @@ impl KeyValueDispatch { pub fn get_store(&self, store: Resource) -> anyhow::Result<&Arc> { self.stores.get(store.rep()).context("invalid store") } + + pub fn allowed_stores(&self) -> &HashSet { + &self.allowed_stores + } } impl Default for KeyValueDispatch { From 31d7e5ad66c144163e65168ba9660747ada8f837 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Tue, 23 Jul 2024 13:56:22 -0700 Subject: [PATCH 078/195] Remove Spin KV store base path default to current working dir and syntactical nits Signed-off-by: Kate Goldenring --- crates/factor-key-value-azure/src/lib.rs | 7 +++- crates/factor-key-value-spin/src/lib.rs | 18 ++++---- .../src/delegating_resolver.rs | 26 +++++++++--- crates/factor-key-value/src/lib.rs | 29 ++++++++----- crates/factor-key-value/tests/test.rs | 42 ++++++++++++------- crates/factors/tests/smoke.rs | 4 +- 6 files changed, 82 insertions(+), 44 deletions(-) diff --git a/crates/factor-key-value-azure/src/lib.rs b/crates/factor-key-value-azure/src/lib.rs index cc5f67026..1e13cac35 100644 --- a/crates/factor-key-value-azure/src/lib.rs +++ b/crates/factor-key-value-azure/src/lib.rs @@ -30,6 +30,11 @@ impl MakeKeyValueStore for AzureKeyValueStore { &self, runtime_config: Self::RuntimeConfig, ) -> anyhow::Result { - KeyValueAzureCosmos::new(runtime_config.key, runtime_config.account, runtime_config.database, runtime_config.container) + KeyValueAzureCosmos::new( + runtime_config.key, + runtime_config.account, + runtime_config.database, + runtime_config.container, + ) } } diff --git a/crates/factor-key-value-spin/src/lib.rs b/crates/factor-key-value-spin/src/lib.rs index 40d8e7e03..7c107b4e7 100644 --- a/crates/factor-key-value-spin/src/lib.rs +++ b/crates/factor-key-value-spin/src/lib.rs @@ -16,13 +16,8 @@ pub struct SpinKeyValueStore { impl SpinKeyValueStore { /// Create a new SpinKeyValueStore with the given base path. - /// If the base path is None, the current directory is used. - pub fn new(base_path: Option) -> anyhow::Result { - let base_path = match base_path { - Some(path) => path, - None => std::env::current_dir().context("failed to get current directory")?, - }; - Ok(Self { base_path }) + pub fn new(base_path: PathBuf) -> Self { + Self { base_path } } } @@ -38,10 +33,11 @@ impl SpinKeyValueRuntimeConfig { const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; /// Create a new runtime configuration with the given state directory. - /// If the state directory is None, the database is in-memory. - /// If the state directory is Some, the database is stored in a file in the state directory. - pub fn default(state_dir: Option) -> Self { - let path = state_dir.map(|dir| dir.join(Self::DEFAULT_SPIN_STORE_FILENAME)); + /// + /// If the database directory is None, the database is in-memory. + /// If the database directory is Some, the database is stored in a file in the state directory. + pub fn default(default_database_dir: Option) -> Self { + let path = default_database_dir.map(|dir| dir.join(Self::DEFAULT_SPIN_STORE_FILENAME)); Self { path } } } diff --git a/crates/factor-key-value/src/delegating_resolver.rs b/crates/factor-key-value/src/delegating_resolver.rs index f655fd202..89ecc2e1d 100644 --- a/crates/factor-key-value/src/delegating_resolver.rs +++ b/crates/factor-key-value/src/delegating_resolver.rs @@ -1,15 +1,18 @@ use crate::runtime_config::RuntimeConfigResolver; use crate::store::{store_from_toml_fn, MakeKeyValueStore, StoreFromToml}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use spin_key_value::StoreManager; use std::{collections::HashMap, sync::Arc}; -/// A RuntimeConfigResolver that delegates to the appropriate key-value store for a given label. -/// The store types are registered with the resolver using `add_store_type`. -/// The default store for a label is registered using `add_default_store`. +/// A RuntimeConfigResolver that delegates to the appropriate key-value store +/// for a given label. +/// +/// The store types are registered with the resolver using `add_store_type`. The +/// default store for a label is registered using `add_default_store`. #[derive(Default)] pub struct DelegatingRuntimeConfigResolver { - /// A map of store types to a function that returns the appropriate store manager from runtime config TOML. + /// A map of store types to a function that returns the appropriate store + /// manager from runtime config TOML. store_types: HashMap<&'static str, StoreFromToml>, /// A map of default store configurations for a label. defaults: HashMap<&'static str, StoreConfig>, @@ -54,6 +57,7 @@ impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { } /// Get the default store manager for the given label. + /// /// Returns None if no default store is registered for the label. fn default_store(&self, label: &str) -> Option> { let config = self.defaults.get(label)?; @@ -68,3 +72,15 @@ pub struct StoreConfig { #[serde(flatten)] pub config: toml::Table, } + +impl StoreConfig { + pub fn new(type_: String, config: T) -> anyhow::Result + where + T: Serialize, + { + Ok(Self { + type_, + config: toml::value::Table::try_from(config)?, + }) + } +} diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index 109ced6e0..f83b417ab 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -1,7 +1,7 @@ pub mod delegating_resolver; mod runtime_config; mod store; -pub use delegating_resolver::DelegatingRuntimeConfigResolver; +pub use delegating_resolver::{DelegatingRuntimeConfigResolver, StoreConfig}; use std::{ collections::{HashMap, HashSet}, @@ -27,7 +27,9 @@ pub struct KeyValueFactor { } impl KeyValueFactor { - /// Create a new KeyValueFactor. + /// Create a new KeyValueFactor. + /// + /// The `runtime_config_resolver` is used to resolve runtime configuration into store managers. pub fn new(runtime_config_resolver: R) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), @@ -54,10 +56,11 @@ impl Factor for KeyValueFactor { let mut store_managers: HashMap> = HashMap::new(); if let Some(runtime_config) = ctx.take_runtime_config() { for (store_label, config) in runtime_config.store_configs { - let store = self.runtime_config_resolver.get_store(config)?; if let std::collections::hash_map::Entry::Vacant(e) = store_managers.entry(store_label) { + // Only add manager for labels that are not already configured. Runtime config + // takes top-down precedence. let store = self.runtime_config_resolver.get_store(config)?; e.insert(store); } @@ -119,19 +122,25 @@ impl Factor for KeyValueFactor { type AppStoreManager = CachingStoreManager; pub struct AppState { - /// The store manager for the app. This is a cache around a delegating store - /// manager. For `get` requests, first checks the cache before delegating to - /// the underlying store manager. + /// The store manager for the app. + /// + /// This is a cache around a delegating store manager. For `get` requests, + /// first checks the cache before delegating to the underlying store + /// manager. store_manager: Arc, /// The allowed stores for each component. - /// This is a map from component ID to the set of store labels that the component is allowed to use. + /// + /// This is a map from component ID to the set of store labels that the + /// component is allowed to use. component_allowed_stores: HashMap>, } pub struct InstanceBuilder { - /// The store manager for the app. This is a cache around a delegating store - /// manager. For `get` requests, first checks the cache before delegating to - /// the underlying store manager. + /// The store manager for the app. + /// + /// This is a cache around a delegating store manager. For `get` requests, + /// first checks the cache before delegating to the underlying store + /// manager. store_manager: Arc, /// The allowed stores for this component instance. allowed_stores: HashSet, diff --git a/crates/factor-key-value/tests/test.rs b/crates/factor-key-value/tests/test.rs index 77f6e1812..fd2e74b48 100644 --- a/crates/factor-key-value/tests/test.rs +++ b/crates/factor-key-value/tests/test.rs @@ -1,27 +1,32 @@ -use std::collections::HashSet; -use spin_factor_key_value::{DelegatingRuntimeConfigResolver, KeyValueFactor, MakeKeyValueStore}; +use anyhow::Context; +use spin_factor_key_value::{ + DelegatingRuntimeConfigResolver, KeyValueFactor, MakeKeyValueStore, StoreConfig, +}; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factors::RuntimeFactors; use spin_factors_test::{toml, TestEnvironment}; +use std::collections::HashSet; #[derive(RuntimeFactors)] struct TestFactors { - key_value: KeyValueFactor, + key_value: KeyValueFactor, } fn default_key_value_resolver( ) -> anyhow::Result<(DelegatingRuntimeConfigResolver, tempdir::TempDir)> { let mut test_resolver = DelegatingRuntimeConfigResolver::new(); - test_resolver.add_store_type(SpinKeyValueStore::new(None)?)?; + test_resolver.add_store_type(SpinKeyValueStore::new( + std::env::current_dir().context("failed to get current directory")?, + ))?; let tmp_dir = tempdir::TempDir::new("example")?; let path = tmp_dir.path().to_path_buf(); let default_config = SpinKeyValueRuntimeConfig::default(Some(path)); - test_resolver.add_default_store( - "default", - SpinKeyValueStore::RUNTIME_CONFIG_TYPE, - toml::value::Table::try_from(default_config)?, - ); + let store_config = StoreConfig::new( + SpinKeyValueStore::RUNTIME_CONFIG_TYPE.to_string(), + default_config, + )?; + test_resolver.add_default_store("default", store_config); Ok((test_resolver, tmp_dir)) } @@ -69,7 +74,6 @@ async fn run_test_with_config_and_stores_for_label( env.runtime_config.extend(runtime_config); } let state = env.build_instance_state(factors).await?; - // String::new("foo").as assert_eq!( labels, state.key_value.allowed_stores().iter().collect::>() @@ -101,7 +105,9 @@ async fn custom_spin_key_value_works() -> anyhow::Result<()> { }; run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new(None)?], + vec![SpinKeyValueStore::new( + std::env::current_dir().context("failed to get current directory")?, + )], vec!["custom"], ) .await @@ -119,7 +125,9 @@ async fn custom_spin_key_value_works_with_absolute_path() -> anyhow::Result<()> }; run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new(None)?], + vec![SpinKeyValueStore::new( + std::env::current_dir().context("failed to get current directory")?, + )], vec!["custom"], ) .await?; @@ -138,7 +146,7 @@ async fn custom_spin_key_value_works_with_relative_path() -> anyhow::Result<()> }; run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new(Some(path))?], + vec![SpinKeyValueStore::new(path)], vec!["custom"], ) .await?; @@ -170,7 +178,9 @@ async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { }; assert!(run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new(None)?], + vec![SpinKeyValueStore::new( + std::env::current_dir().context("failed to get current directory")? + )], vec!["custom"] ) .await @@ -179,7 +189,7 @@ async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { } #[tokio::test] -async fn multiple_custom_key_value_fails() -> anyhow::Result<()> { +async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { let tmp_dir = tempdir::TempDir::new("example")?; let runtime_config = toml::toml! { [key_value_store.custom] @@ -192,7 +202,7 @@ async fn multiple_custom_key_value_fails() -> anyhow::Result<()> { }; let mut test_resolver = DelegatingRuntimeConfigResolver::new(); test_resolver.add_store_type(RedisKeyValueStore)?; - test_resolver.add_store_type(SpinKeyValueStore::new(Some(tmp_dir.path().to_owned()))?)?; + test_resolver.add_store_type(SpinKeyValueStore::new(tmp_dir.path().to_owned()))?; let factors = TestFactors { key_value: KeyValueFactor::new(test_resolver), }; diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 45a727c85..fb92b509a 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -48,7 +48,9 @@ async fn smoke_test_works() -> anyhow::Result<()> { config: toml::value::Table::try_from(default_config)?, }, ); - key_value_resolver.add_store_type(SpinKeyValueStore::new(None)?)?; + key_value_resolver.add_store_type(SpinKeyValueStore::new( + std::env::current_dir().context("failed to get current directory")?, + ))?; key_value_resolver.add_store_type(RedisKeyValueStore)?; let mut factors = Factors { From 5849e9fd3b1e2efbccda61c63faab7843bc62ede Mon Sep 17 00:00:00 2001 From: Caleb Schoepp Date: Mon, 15 Jul 2024 15:04:48 -0600 Subject: [PATCH 079/195] factors: Add more tests to factor-outbound-pg and refactor it to be generic across pg impl Signed-off-by: Caleb Schoepp --- Cargo.lock | 1 - crates/factor-outbound-pg/Cargo.toml | 1 - crates/factor-outbound-pg/src/client.rs | 284 ++++++++++++++++++ crates/factor-outbound-pg/src/host.rs | 284 +----------------- crates/factor-outbound-pg/src/lib.rs | 32 +- .../factor-outbound-pg/tests/factor_test.rs | 123 +++++++- 6 files changed, 434 insertions(+), 291 deletions(-) create mode 100644 crates/factor-outbound-pg/src/client.rs diff --git a/Cargo.lock b/Cargo.lock index 891e4802c..f2cae7b54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7615,7 +7615,6 @@ dependencies = [ "spin-core", "spin-factor-outbound-networking", "spin-factor-variables", - "spin-factor-wasi", "spin-factors", "spin-factors-test", "spin-world", diff --git a/crates/factor-outbound-pg/Cargo.toml b/crates/factor-outbound-pg/Cargo.toml index ca18e93a1..cd8681a4a 100644 --- a/crates/factor-outbound-pg/Cargo.toml +++ b/crates/factor-outbound-pg/Cargo.toml @@ -19,7 +19,6 @@ tracing = { workspace = true } [dev-dependencies] spin-factor-variables = { path = "../factor-variables" } -spin-factor-wasi = { path = "../factor-wasi" } spin-factors-test = { path = "../factors-test" } tokio = { version = "1", features = ["macros", "rt"] } diff --git a/crates/factor-outbound-pg/src/client.rs b/crates/factor-outbound-pg/src/client.rs new file mode 100644 index 000000000..06a93a631 --- /dev/null +++ b/crates/factor-outbound-pg/src/client.rs @@ -0,0 +1,284 @@ +use anyhow::{anyhow, Result}; +use native_tls::TlsConnector; +use postgres_native_tls::MakeTlsConnector; +use spin_world::async_trait; +use spin_world::v2::postgres::{self as v2}; +use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue, RowSet}; +use tokio_postgres::types::Type; +use tokio_postgres::{config::SslMode, types::ToSql, Row}; +use tokio_postgres::{Client as TokioClient, NoTls, Socket}; + +#[async_trait] +pub trait Client { + async fn build_client(address: &str) -> Result + where + Self: Sized; + + async fn execute( + &self, + statement: String, + params: Vec, + ) -> Result; + + async fn query( + &self, + statement: String, + params: Vec, + ) -> Result; +} + +#[async_trait] +impl Client for TokioClient { + async fn build_client(address: &str) -> Result + where + Self: Sized, + { + let config = address.parse::()?; + + tracing::debug!("Build new connection: {}", address); + + if config.get_ssl_mode() == SslMode::Disable { + let (client, connection) = config.connect(NoTls).await?; + spawn_connection(connection); + Ok(client) + } else { + let builder = TlsConnector::builder(); + let connector = MakeTlsConnector::new(builder.build()?); + let (client, connection) = config.connect(connector).await?; + spawn_connection(connection); + Ok(client) + } + } + + async fn execute( + &self, + statement: String, + params: Vec, + ) -> Result { + let params: Vec<&(dyn ToSql + Sync)> = params + .iter() + .map(to_sql_parameter) + .collect::>>() + .map_err(|e| v2::Error::ValueConversionFailed(format!("{:?}", e)))?; + + self.execute(&statement, params.as_slice()) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e))) + } + + async fn query( + &self, + statement: String, + params: Vec, + ) -> Result { + let params: Vec<&(dyn ToSql + Sync)> = params + .iter() + .map(to_sql_parameter) + .collect::>>() + .map_err(|e| v2::Error::BadParameter(format!("{:?}", e)))?; + + let results = self + .query(&statement, params.as_slice()) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + if results.is_empty() { + return Ok(RowSet { + columns: vec![], + rows: vec![], + }); + } + + let columns = infer_columns(&results[0]); + let rows = results + .iter() + .map(convert_row) + .collect::, _>>() + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + Ok(RowSet { columns, rows }) + } +} + +fn spawn_connection(connection: tokio_postgres::Connection) +where + T: tokio_postgres::tls::TlsStream + std::marker::Unpin + std::marker::Send + 'static, +{ + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Postgres connection error: {}", e); + } + }); +} + +fn to_sql_parameter(value: &ParameterValue) -> Result<&(dyn ToSql + Sync)> { + match value { + ParameterValue::Boolean(v) => Ok(v), + ParameterValue::Int32(v) => Ok(v), + ParameterValue::Int64(v) => Ok(v), + ParameterValue::Int8(v) => Ok(v), + ParameterValue::Int16(v) => Ok(v), + ParameterValue::Floating32(v) => Ok(v), + ParameterValue::Floating64(v) => Ok(v), + ParameterValue::Uint8(_) + | ParameterValue::Uint16(_) + | ParameterValue::Uint32(_) + | ParameterValue::Uint64(_) => Err(anyhow!("Postgres does not support unsigned integers")), + ParameterValue::Str(v) => Ok(v), + ParameterValue::Binary(v) => Ok(v), + ParameterValue::DbNull => Ok(&PgNull), + } +} + +fn infer_columns(row: &Row) -> Vec { + let mut result = Vec::with_capacity(row.len()); + for index in 0..row.len() { + result.push(infer_column(row, index)); + } + result +} + +fn infer_column(row: &Row, index: usize) -> Column { + let column = &row.columns()[index]; + let name = column.name().to_owned(); + let data_type = convert_data_type(column.type_()); + Column { name, data_type } +} + +fn convert_data_type(pg_type: &Type) -> DbDataType { + match *pg_type { + Type::BOOL => DbDataType::Boolean, + Type::BYTEA => DbDataType::Binary, + Type::FLOAT4 => DbDataType::Floating32, + Type::FLOAT8 => DbDataType::Floating64, + Type::INT2 => DbDataType::Int16, + Type::INT4 => DbDataType::Int32, + Type::INT8 => DbDataType::Int64, + Type::TEXT | Type::VARCHAR | Type::BPCHAR => DbDataType::Str, + _ => { + tracing::debug!("Couldn't convert Postgres type {} to WIT", pg_type.name(),); + DbDataType::Other + } + } +} + +fn convert_row(row: &Row) -> Result, tokio_postgres::Error> { + let mut result = Vec::with_capacity(row.len()); + for index in 0..row.len() { + result.push(convert_entry(row, index)?); + } + Ok(result) +} + +fn convert_entry(row: &Row, index: usize) -> Result { + let column = &row.columns()[index]; + let value = match column.type_() { + &Type::BOOL => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Boolean(v), + None => DbValue::DbNull, + } + } + &Type::BYTEA => { + let value: Option> = row.try_get(index)?; + match value { + Some(v) => DbValue::Binary(v), + None => DbValue::DbNull, + } + } + &Type::FLOAT4 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Floating32(v), + None => DbValue::DbNull, + } + } + &Type::FLOAT8 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Floating64(v), + None => DbValue::DbNull, + } + } + &Type::INT2 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Int16(v), + None => DbValue::DbNull, + } + } + &Type::INT4 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Int32(v), + None => DbValue::DbNull, + } + } + &Type::INT8 => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Int64(v), + None => DbValue::DbNull, + } + } + &Type::TEXT | &Type::VARCHAR | &Type::BPCHAR => { + let value: Option = row.try_get(index)?; + match value { + Some(v) => DbValue::Str(v), + None => DbValue::DbNull, + } + } + t => { + tracing::debug!( + "Couldn't convert Postgres type {} in column {}", + t.name(), + column.name() + ); + DbValue::Unsupported + } + }; + Ok(value) +} + +/// Although the Postgres crate converts Rust Option::None to Postgres NULL, +/// it enforces the type of the Option as it does so. (For example, trying to +/// pass an Option::::None to a VARCHAR column fails conversion.) As we +/// do not know expected column types, we instead use a "neutral" custom type +/// which allows conversion to any type but always tells the Postgres crate to +/// treat it as a SQL NULL. +struct PgNull; + +impl ToSql for PgNull { + fn to_sql( + &self, + _ty: &Type, + _out: &mut tokio_postgres::types::private::BytesMut, + ) -> Result> + where + Self: Sized, + { + Ok(tokio_postgres::types::IsNull::Yes) + } + + fn accepts(_ty: &Type) -> bool + where + Self: Sized, + { + true + } + + fn to_sql_checked( + &self, + _ty: &Type, + _out: &mut tokio_postgres::types::private::BytesMut, + ) -> Result> { + Ok(tokio_postgres::types::IsNull::Yes) + } +} + +impl std::fmt::Debug for PgNull { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NULL").finish() + } +} diff --git a/crates/factor-outbound-pg/src/host.rs b/crates/factor-outbound-pg/src/host.rs index 63bc9ac91..1f7be3570 100644 --- a/crates/factor-outbound-pg/src/host.rs +++ b/crates/factor-outbound-pg/src/host.rs @@ -1,27 +1,21 @@ -use anyhow::{anyhow, Result}; -use native_tls::TlsConnector; -use postgres_native_tls::MakeTlsConnector; +use anyhow::Result; use spin_core::{async_trait, wasmtime::component::Resource}; use spin_world::v1::postgres as v1; use spin_world::v1::rdbms_types as v1_types; use spin_world::v2::postgres::{self as v2, Connection}; use spin_world::v2::rdbms_types; -use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue, RowSet}; -use tokio_postgres::{ - config::SslMode, - types::{ToSql, Type}, - Client, NoTls, Row, Socket, -}; +use spin_world::v2::rdbms_types::{ParameterValue, RowSet}; use tracing::instrument; use tracing::Level; +use crate::client::Client; use crate::InstanceState; -impl InstanceState { +impl InstanceState { async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { self.connections .push( - build_client(address) + C::build_client(address) .await .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, ) @@ -29,7 +23,7 @@ impl InstanceState { .map(Resource::new_own) } - async fn get_client(&mut self, connection: Resource) -> Result<&Client, v2::Error> { + async fn get_client(&mut self, connection: Resource) -> Result<&C, v2::Error> { self.connections .get(connection.rep()) .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) @@ -52,7 +46,6 @@ impl InstanceState { .or_else(|| if ports.len() == 1 { ports.get(1) } else { None }); let port_str = port.map(|p| format!(":{}", p)).unwrap_or_default(); let url = format!("{address}{port_str}"); - // TODO: Should I be unwrapping this? if !self.allowed_hosts.check_url(&url, "postgres").await? { return Ok(false); } @@ -66,10 +59,10 @@ impl InstanceState { } #[async_trait] -impl v2::Host for InstanceState {} +impl v2::Host for InstanceState {} #[async_trait] -impl v2::HostConnection for InstanceState { +impl v2::HostConnection for InstanceState { #[instrument(name = "spin_outbound_pg.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql"))] async fn open(&mut self, address: String) -> Result, v2::Error> { if !self @@ -91,20 +84,11 @@ impl v2::HostConnection for InstanceState { statement: String, params: Vec, ) -> Result { - let params: Vec<&(dyn ToSql + Sync)> = params - .iter() - .map(to_sql_parameter) - .collect::>>() - .map_err(|e| v2::Error::ValueConversionFailed(format!("{:?}", e)))?; - - let nrow = self + Ok(self .get_client(connection) .await? - .execute(&statement, params.as_slice()) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - Ok(nrow) + .execute(statement, params) + .await?) } #[instrument(name = "spin_outbound_pg.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql", otel.name = statement))] @@ -114,34 +98,11 @@ impl v2::HostConnection for InstanceState { statement: String, params: Vec, ) -> Result { - let params: Vec<&(dyn ToSql + Sync)> = params - .iter() - .map(to_sql_parameter) - .collect::>>() - .map_err(|e| v2::Error::BadParameter(format!("{:?}", e)))?; - - let results = self + Ok(self .get_client(connection) .await? - .query(&statement, params.as_slice()) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - if results.is_empty() { - return Ok(RowSet { - columns: vec![], - rows: vec![], - }); - } - - let columns = infer_columns(&results[0]); - let rows = results - .iter() - .map(convert_row) - .collect::, _>>() - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - Ok(RowSet { columns, rows }) + .query(statement, params) + .await?) } fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { @@ -150,225 +111,12 @@ impl v2::HostConnection for InstanceState { } } -impl rdbms_types::Host for InstanceState { +impl rdbms_types::Host for InstanceState { fn convert_error(&mut self, error: v2::Error) -> Result { Ok(error) } } -fn to_sql_parameter(value: &ParameterValue) -> anyhow::Result<&(dyn ToSql + Sync)> { - match value { - ParameterValue::Boolean(v) => Ok(v), - ParameterValue::Int32(v) => Ok(v), - ParameterValue::Int64(v) => Ok(v), - ParameterValue::Int8(v) => Ok(v), - ParameterValue::Int16(v) => Ok(v), - ParameterValue::Floating32(v) => Ok(v), - ParameterValue::Floating64(v) => Ok(v), - ParameterValue::Uint8(_) - | ParameterValue::Uint16(_) - | ParameterValue::Uint32(_) - | ParameterValue::Uint64(_) => Err(anyhow!("Postgres does not support unsigned integers")), - ParameterValue::Str(v) => Ok(v), - ParameterValue::Binary(v) => Ok(v), - ParameterValue::DbNull => Ok(&PgNull), - } -} - -fn infer_columns(row: &Row) -> Vec { - let mut result = Vec::with_capacity(row.len()); - for index in 0..row.len() { - result.push(infer_column(row, index)); - } - result -} - -fn infer_column(row: &Row, index: usize) -> Column { - let column = &row.columns()[index]; - let name = column.name().to_owned(); - let data_type = convert_data_type(column.type_()); - Column { name, data_type } -} - -fn convert_data_type(pg_type: &Type) -> DbDataType { - match *pg_type { - Type::BOOL => DbDataType::Boolean, - Type::BYTEA => DbDataType::Binary, - Type::FLOAT4 => DbDataType::Floating32, - Type::FLOAT8 => DbDataType::Floating64, - Type::INT2 => DbDataType::Int16, - Type::INT4 => DbDataType::Int32, - Type::INT8 => DbDataType::Int64, - Type::TEXT | Type::VARCHAR | Type::BPCHAR => DbDataType::Str, - _ => { - tracing::debug!("Couldn't convert Postgres type {} to WIT", pg_type.name(),); - DbDataType::Other - } - } -} - -fn convert_row(row: &Row) -> Result, tokio_postgres::Error> { - let mut result = Vec::with_capacity(row.len()); - for index in 0..row.len() { - result.push(convert_entry(row, index)?); - } - Ok(result) -} - -fn convert_entry(row: &Row, index: usize) -> Result { - let column = &row.columns()[index]; - let value = match column.type_() { - &Type::BOOL => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Boolean(v), - None => DbValue::DbNull, - } - } - &Type::BYTEA => { - let value: Option> = row.try_get(index)?; - match value { - Some(v) => DbValue::Binary(v), - None => DbValue::DbNull, - } - } - &Type::FLOAT4 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Floating32(v), - None => DbValue::DbNull, - } - } - &Type::FLOAT8 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Floating64(v), - None => DbValue::DbNull, - } - } - &Type::INT2 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Int16(v), - None => DbValue::DbNull, - } - } - &Type::INT4 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Int32(v), - None => DbValue::DbNull, - } - } - &Type::INT8 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Int64(v), - None => DbValue::DbNull, - } - } - &Type::TEXT | &Type::VARCHAR | &Type::BPCHAR => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Str(v), - None => DbValue::DbNull, - } - } - t => { - tracing::debug!( - "Couldn't convert Postgres type {} in column {}", - t.name(), - column.name() - ); - DbValue::Unsupported - } - }; - Ok(value) -} - -async fn build_client(address: &str) -> anyhow::Result { - let config = address.parse::()?; - - tracing::debug!("Build new connection: {}", address); - - if config.get_ssl_mode() == SslMode::Disable { - connect(config).await - } else { - connect_tls(config).await - } -} - -async fn connect(config: tokio_postgres::Config) -> anyhow::Result { - let (client, connection) = config.connect(NoTls).await?; - - spawn(connection); - - Ok(client) -} - -async fn connect_tls(config: tokio_postgres::Config) -> anyhow::Result { - let builder = TlsConnector::builder(); - let connector = MakeTlsConnector::new(builder.build()?); - let (client, connection) = config.connect(connector).await?; - - spawn(connection); - - Ok(client) -} - -fn spawn(connection: tokio_postgres::Connection) -where - T: tokio_postgres::tls::TlsStream + std::marker::Unpin + std::marker::Send + 'static, -{ - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::error!("Postgres connection error: {}", e); - } - }); -} - -/// Although the Postgres crate converts Rust Option::None to Postgres NULL, -/// it enforces the type of the Option as it does so. (For example, trying to -/// pass an Option::::None to a VARCHAR column fails conversion.) As we -/// do not know expected column types, we instead use a "neutral" custom type -/// which allows conversion to any type but always tells the Postgres crate to -/// treat it as a SQL NULL. -struct PgNull; - -impl ToSql for PgNull { - fn to_sql( - &self, - _ty: &Type, - _out: &mut tokio_postgres::types::private::BytesMut, - ) -> Result> - where - Self: Sized, - { - Ok(tokio_postgres::types::IsNull::Yes) - } - - fn accepts(_ty: &Type) -> bool - where - Self: Sized, - { - true - } - - fn to_sql_checked( - &self, - _ty: &Type, - _out: &mut tokio_postgres::types::private::BytesMut, - ) -> Result> { - Ok(tokio_postgres::types::IsNull::Yes) - } -} - -impl std::fmt::Debug for PgNull { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("NULL").finish() - } -} - /// Delegate a function call to the v2::HostConnection implementation macro_rules! delegate { ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ @@ -388,7 +136,7 @@ macro_rules! delegate { } #[async_trait] -impl v1::Host for InstanceState { +impl v1::Host for InstanceState { async fn execute( &mut self, address: String, diff --git a/crates/factor-outbound-pg/src/lib.rs b/crates/factor-outbound-pg/src/lib.rs index 143666932..484cc68c3 100644 --- a/crates/factor-outbound-pg/src/lib.rs +++ b/crates/factor-outbound-pg/src/lib.rs @@ -1,18 +1,22 @@ +pub mod client; mod host; +use client::Client; use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; use spin_factors::{ anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; -use tokio_postgres::Client; +use tokio_postgres::Client as PgClient; -pub struct OutboundPgFactor; +pub struct OutboundPgFactor { + _phantom: std::marker::PhantomData, +} -impl Factor for OutboundPgFactor { +impl Factor for OutboundPgFactor { type RuntimeConfig = (); type AppState = (); - type InstanceBuilder = InstanceState; + type InstanceBuilder = InstanceState; fn init( &mut self, @@ -45,9 +49,23 @@ impl Factor for OutboundPgFactor { } } -pub struct InstanceState { +impl Default for OutboundPgFactor { + fn default() -> Self { + Self { + _phantom: Default::default(), + } + } +} + +impl OutboundPgFactor { + pub fn new() -> Self { + Self::default() + } +} + +pub struct InstanceState { allowed_hosts: OutboundAllowedHosts, - connections: table::Table, + connections: table::Table, } -impl SelfInstanceBuilder for InstanceState {} +impl SelfInstanceBuilder for InstanceState {} diff --git a/crates/factor-outbound-pg/tests/factor_test.rs b/crates/factor-outbound-pg/tests/factor_test.rs index 4f2f78852..07f47cc0c 100644 --- a/crates/factor-outbound-pg/tests/factor_test.rs +++ b/crates/factor-outbound-pg/tests/factor_test.rs @@ -1,39 +1,48 @@ -use anyhow::bail; +use anyhow::{bail, Result}; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_outbound_pg::client::Client; use spin_factor_outbound_pg::OutboundPgFactor; use spin_factor_variables::{StaticVariables, VariablesFactor}; -use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; +use spin_world::async_trait; use spin_world::v2::postgres::HostConnection; +use spin_world::v2::postgres::{self as v2}; use spin_world::v2::rdbms_types::Error as PgError; +use spin_world::v2::rdbms_types::{ParameterValue, RowSet}; #[derive(RuntimeFactors)] struct TestFactors { - wasi: WasiFactor, variables: VariablesFactor, networking: OutboundNetworkingFactor, - pg: OutboundPgFactor, + pg: OutboundPgFactor, +} + +fn factors() -> Result { + let mut f = TestFactors { + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + pg: OutboundPgFactor::::new(), + }; + f.variables.add_provider_type(StaticVariables)?; + Ok(f) } fn test_env() -> TestEnvironment { TestEnvironment::default_manifest_extend(toml! { [component.test-component] source = "does-not-exist.wasm" + allowed_outbound_hosts = ["postgres://*:*"] }) } #[tokio::test] async fn disallowed_host_fails() -> anyhow::Result<()> { - let mut factors = TestFactors { - wasi: WasiFactor::new(DummyFilesMounter), - variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, - pg: OutboundPgFactor, - }; - factors.variables.add_provider_type(StaticVariables)?; - - let env = test_env(); + let factors = factors()?; + let env = TestEnvironment::default_manifest_extend(toml! { + [component.test-component] + source = "does-not-exist.wasm" + }); let mut state = env.build_instance_state(factors).await?; let res = state @@ -43,8 +52,94 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { let Err(err) = res else { bail!("expected Err, got Ok"); }; - println!("err: {:?}", err); assert!(matches!(err, PgError::ConnectionFailed(_))); Ok(()) } + +#[tokio::test] +async fn allowed_host_succeeds() -> anyhow::Result<()> { + let factors = factors()?; + let env = test_env(); + let mut state = env.build_instance_state(factors).await?; + + let res = state + .pg + .open("postgres://localhost:5432/test".to_string()) + .await; + let Ok(_) = res else { + bail!("expected Ok, got Err"); + }; + + Ok(()) +} + +#[tokio::test] +async fn exercise_execute() -> anyhow::Result<()> { + let factors = factors()?; + let env = test_env(); + let mut state = env.build_instance_state(factors).await?; + + let connection = state + .pg + .open("postgres://localhost:5432/test".to_string()) + .await?; + + state + .pg + .execute(connection, "SELECT * FROM test".to_string(), vec![]) + .await?; + + Ok(()) +} + +#[tokio::test] +async fn exercise_query() -> anyhow::Result<()> { + let factors = factors()?; + let env = test_env(); + let mut state = env.build_instance_state(factors).await?; + + let connection = state + .pg + .open("postgres://localhost:5432/test".to_string()) + .await?; + + state + .pg + .query(connection, "SELECT * FROM test".to_string(), vec![]) + .await?; + + Ok(()) +} + +// TODO: We can expand this mock to track calls and simulate return values +pub struct MockClient {} + +#[async_trait] +impl Client for MockClient { + async fn build_client(_address: &str) -> anyhow::Result + where + Self: Sized, + { + Ok(MockClient {}) + } + + async fn execute( + &self, + _statement: String, + _params: Vec, + ) -> Result { + Ok(0) + } + + async fn query( + &self, + _statement: String, + _params: Vec, + ) -> Result { + Ok(RowSet { + columns: vec![], + rows: vec![], + }) + } +} From 2a338125d2629ec4ee7954054296880d86fa2237 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 22 Jul 2024 18:16:17 +0200 Subject: [PATCH 080/195] Remove variables factor assumption of toml Signed-off-by: Ryan Levick --- crates/factor-variables/src/lib.rs | 69 ++++++++----------- crates/factor-variables/src/provider.rs | 31 +++------ .../src/{provider => spin_cli}/env.rs | 23 ++++--- crates/factor-variables/src/spin_cli/mod.rs | 17 +++++ .../src/{provider => spin_cli}/statik.rs | 21 +++--- crates/factor-variables/tests/factor_test.rs | 5 +- 6 files changed, 85 insertions(+), 81 deletions(-) rename crates/factor-variables/src/{provider => spin_cli}/env.rs (93%) create mode 100644 crates/factor-variables/src/spin_cli/mod.rs rename crates/factor-variables/src/{provider => spin_cli}/statik.rs (62%) diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 22c26022b..94ad9441c 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -1,44 +1,42 @@ pub mod provider; +pub mod spin_cli; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize}; use spin_expressions::ProviderResolver; use spin_factors::{ - anyhow::{self, bail, Context}, - ConfigureAppContext, Factor, FactorRuntimeConfig, InitContext, InstanceBuilders, + anyhow, ConfigureAppContext, Factor, FactorRuntimeConfig, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; use spin_world::{async_trait, v1, v2::variables}; -pub use provider::{MakeVariablesProvider, StaticVariables}; +pub use provider::MakeVariablesProvider; -#[derive(Default)] -pub struct VariablesFactor { - provider_types: HashMap<&'static str, provider::ProviderFromToml>, +pub struct VariablesFactor { + provider_types: Vec>>, } -impl VariablesFactor { - pub fn add_provider_type( +impl Default for VariablesFactor { + fn default() -> Self { + Self { + provider_types: Default::default(), + } + } +} + +impl VariablesFactor { + pub fn add_provider_type>( &mut self, provider_type: T, ) -> anyhow::Result<()> { - if self - .provider_types - .insert( - T::RUNTIME_CONFIG_TYPE, - provider::provider_from_toml_fn(provider_type), - ) - .is_some() - { - bail!("duplicate provider type {:?}", T::RUNTIME_CONFIG_TYPE); - } + self.provider_types.push(Box::new(provider_type) as _); Ok(()) } } -impl Factor for VariablesFactor { - type RuntimeConfig = RuntimeConfig; +impl Factor for VariablesFactor { + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; @@ -64,13 +62,12 @@ impl Factor for VariablesFactor { } if let Some(runtime_config) = ctx.take_runtime_config() { - for ProviderConfig { type_, config } in runtime_config.provider_configs { - let provider_maker = self - .provider_types - .get(type_.as_str()) - .with_context(|| format!("unknown variables provider type {type_:?}"))?; - let provider = provider_maker(config)?; - resolver.add_provider(provider); + for config in runtime_config.provider_configs { + for make_provider in self.provider_types.iter() { + if let Some(provider) = make_provider.make_provider(&config)? { + resolver.add_provider(provider); + } + } } } @@ -95,22 +92,14 @@ impl Factor for VariablesFactor { #[derive(Deserialize)] #[serde(transparent)] -pub struct RuntimeConfig { - provider_configs: Vec, +pub struct RuntimeConfig { + provider_configs: Vec, } -impl FactorRuntimeConfig for RuntimeConfig { +impl FactorRuntimeConfig for RuntimeConfig { const KEY: &'static str = "variable_provider"; } -#[derive(Deserialize)] -struct ProviderConfig { - #[serde(rename = "type")] - type_: String, - #[serde(flatten)] - config: toml::Table, -} - pub struct AppState { resolver: Arc, } diff --git a/crates/factor-variables/src/provider.rs b/crates/factor-variables/src/provider.rs index f34945f1c..6a8795fb9 100644 --- a/crates/factor-variables/src/provider.rs +++ b/crates/factor-variables/src/provider.rs @@ -1,30 +1,17 @@ -mod env; -mod statik; - -pub use env::EnvVariables; -pub use statik::StaticVariables; - use serde::de::DeserializeOwned; use spin_expressions::Provider; use spin_factors::anyhow; +/// A trait for converting a runtime configuration into a variables provider. pub trait MakeVariablesProvider: 'static { - const RUNTIME_CONFIG_TYPE: &'static str; - + /// Serialized configuration for the provider. type RuntimeConfig: DeserializeOwned; - type Provider: Provider; - - fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result; -} - -pub(crate) type ProviderFromToml = Box anyhow::Result>>; -pub(crate) fn provider_from_toml_fn( - provider_type: T, -) -> ProviderFromToml { - Box::new(move |table| { - let runtime_config: T::RuntimeConfig = table.try_into()?; - let provider = provider_type.make_provider(runtime_config)?; - Ok(Box::new(provider)) - }) + /// Create a variables provider from the given runtime configuration. + /// + /// Returns `Ok(None)` if the provider is not applicable to the given configuration. + fn make_provider( + &self, + runtime_config: &Self::RuntimeConfig, + ) -> anyhow::Result>>; } diff --git a/crates/factor-variables/src/provider/env.rs b/crates/factor-variables/src/spin_cli/env.rs similarity index 93% rename from crates/factor-variables/src/provider/env.rs rename to crates/factor-variables/src/spin_cli/env.rs index 485c061eb..ca4ad6fa8 100644 --- a/crates/factor-variables/src/provider/env.rs +++ b/crates/factor-variables/src/spin_cli/env.rs @@ -13,21 +13,26 @@ use tracing::{instrument, Level}; use crate::MakeVariablesProvider; +use super::RuntimeConfig; + /// Creator of a environment variables provider. pub struct EnvVariables; impl MakeVariablesProvider for EnvVariables { - const RUNTIME_CONFIG_TYPE: &'static str = "env"; - - type RuntimeConfig = EnvVariablesConfig; - type Provider = EnvVariablesProvider; + type RuntimeConfig = RuntimeConfig; - fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result { - Ok(EnvVariablesProvider::new( - runtime_config.prefix, + fn make_provider( + &self, + runtime_config: &Self::RuntimeConfig, + ) -> anyhow::Result>> { + let RuntimeConfig::Env(runtime_config) = runtime_config else { + return Ok(None); + }; + Ok(Some(Box::new(EnvVariablesProvider::new( + runtime_config.prefix.clone(), |key| std::env::var(key), - runtime_config.dotenv_path, - )) + runtime_config.dotenv_path.clone(), + )))) } } diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs new file mode 100644 index 000000000..448625d8d --- /dev/null +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -0,0 +1,17 @@ +//! The runtime configuration for the variables factor used in the Spin CLI. + +mod env; +mod statik; + +pub use env::EnvVariables; +pub use statik::StaticVariables; + +use serde::Deserialize; +use statik::StaticVariablesProvider; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case", tag = "type")] +pub enum RuntimeConfig { + Static(StaticVariablesProvider), + Env(env::EnvVariablesConfig), +} diff --git a/crates/factor-variables/src/provider/statik.rs b/crates/factor-variables/src/spin_cli/statik.rs similarity index 62% rename from crates/factor-variables/src/provider/statik.rs rename to crates/factor-variables/src/spin_cli/statik.rs index 222c7168e..8e31cff12 100644 --- a/crates/factor-variables/src/provider/statik.rs +++ b/crates/factor-variables/src/spin_cli/statik.rs @@ -6,22 +6,27 @@ use spin_factors::anyhow; use crate::MakeVariablesProvider; +use super::RuntimeConfig; + /// Creator of a static variables provider. pub struct StaticVariables; impl MakeVariablesProvider for StaticVariables { - const RUNTIME_CONFIG_TYPE: &'static str = "static"; - - type RuntimeConfig = StaticVariablesProvider; - type Provider = StaticVariablesProvider; - - fn make_provider(&self, runtime_config: Self::RuntimeConfig) -> anyhow::Result { - Ok(runtime_config) + type RuntimeConfig = RuntimeConfig; + + fn make_provider( + &self, + runtime_config: &Self::RuntimeConfig, + ) -> anyhow::Result>> { + let RuntimeConfig::Static(config) = runtime_config else { + return Ok(None); + }; + Ok(Some(Box::new(config.clone()) as _)) } } /// A variables provider that reads variables from an static map. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct StaticVariablesProvider { values: Arc>, } diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index e5e11ed16..734c9fa21 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -1,10 +1,11 @@ -use spin_factor_variables::{StaticVariables, VariablesFactor}; +use spin_factor_variables::spin_cli::{RuntimeConfig, StaticVariables}; +use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; #[derive(RuntimeFactors)] struct TestFactors { - variables: VariablesFactor, + variables: VariablesFactor, } fn test_env() -> TestEnvironment { From b203d75628d712d760d20a057eed8420c5dd6f3c Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 23 Jul 2024 12:11:47 +0200 Subject: [PATCH 081/195] Rename and document for clarity Signed-off-by: Ryan Levick --- crates/factor-outbound-networking/src/lib.rs | 5 +- .../tests/factor_test.rs | 2 +- crates/factor-variables/src/lib.rs | 48 +++++++++++-------- crates/factor-variables/src/provider.rs | 4 +- crates/factor-variables/src/spin_cli/env.rs | 6 +-- crates/factor-variables/src/spin_cli/mod.rs | 3 ++ .../factor-variables/src/spin_cli/statik.rs | 6 +-- crates/factor-variables/tests/factor_test.rs | 8 ++-- crates/factors/tests/smoke.rs | 4 +- 9 files changed, 51 insertions(+), 35 deletions(-) diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index b04262e3f..9d7f04b2d 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -59,7 +59,10 @@ impl Factor for OutboundNetworkingFactor { .get(ctx.app_component().id()) .cloned() .context("missing component allowed hosts")?; - let resolver = builders.get_mut::()?.resolver().clone(); + let resolver = builders + .get_mut::()? + .expression_resolver() + .clone(); let allowed_hosts_future = async move { let prepared = resolver.prepare().await?; AllowedHostsConfig::parse(&hosts, &prepared) diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs index 552a871cc..e0d42dd54 100644 --- a/crates/factor-outbound-redis/tests/factor_test.rs +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -28,7 +28,7 @@ fn get_test_factors() -> TestFactors { async fn no_outbound_hosts_fails() -> anyhow::Result<()> { let mut factors = get_test_factors(); - factors.variables.add_provider_type(StaticVariables)?; + factors.variables.add_provider_resolver(StaticVariables)?; let env = TestEnvironment { manifest: toml! { diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 94ad9441c..ffa7b0787 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -4,33 +4,40 @@ pub mod spin_cli; use std::sync::Arc; use serde::{de::DeserializeOwned, Deserialize}; -use spin_expressions::ProviderResolver; +use spin_expressions::ProviderResolver as ExpressionResolver; use spin_factors::{ anyhow, ConfigureAppContext, Factor, FactorRuntimeConfig, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; use spin_world::{async_trait, v1, v2::variables}; -pub use provider::MakeVariablesProvider; +pub use provider::ProviderResolver; +/// A factor for providing variables to components. +/// +/// The factor is generic over the type of runtime configuration used to configure the providers. pub struct VariablesFactor { - provider_types: Vec>>, + provider_resolvers: Vec>>, } impl Default for VariablesFactor { fn default() -> Self { Self { - provider_types: Default::default(), + provider_resolvers: Default::default(), } } } impl VariablesFactor { - pub fn add_provider_type>( + /// Adds a provider resolver to the factor. + /// + /// Each added provider will be called in order with the runtime configuration. This order + /// will be the order in which the providers are called to resolve variables. + pub fn add_provider_resolver>( &mut self, provider_type: T, ) -> anyhow::Result<()> { - self.provider_types.push(Box::new(provider_type) as _); + self.provider_resolvers.push(Box::new(provider_type)); Ok(()) } } @@ -51,11 +58,11 @@ impl Factor for VariablesFactor { mut ctx: ConfigureAppContext, ) -> anyhow::Result { let app = ctx.app(); - let mut resolver = - ProviderResolver::new(app.variables().map(|(key, val)| (key.clone(), val.clone())))?; + let mut expression_resolver = + ExpressionResolver::new(app.variables().map(|(key, val)| (key.clone(), val.clone())))?; for component in app.components() { - resolver.add_component_variables( + expression_resolver.add_component_variables( component.id(), component.config().map(|(k, v)| (k.into(), v.into())), )?; @@ -63,16 +70,16 @@ impl Factor for VariablesFactor { if let Some(runtime_config) = ctx.take_runtime_config() { for config in runtime_config.provider_configs { - for make_provider in self.provider_types.iter() { - if let Some(provider) = make_provider.make_provider(&config)? { - resolver.add_provider(provider); + for provider_resolver in self.provider_resolvers.iter() { + if let Some(provider) = provider_resolver.resolve_provider(&config)? { + expression_resolver.add_provider(provider); } } } } Ok(AppState { - resolver: Arc::new(resolver), + expression_resolver: Arc::new(expression_resolver), }) } @@ -82,14 +89,15 @@ impl Factor for VariablesFactor { _builders: &mut InstanceBuilders, ) -> anyhow::Result { let component_id = ctx.app_component().id().to_string(); - let resolver = ctx.app_state().resolver.clone(); + let expression_resolver = ctx.app_state().expression_resolver.clone(); Ok(InstanceState { component_id, - resolver, + expression_resolver, }) } } +/// The runtime configuration for the variables factor. #[derive(Deserialize)] #[serde(transparent)] pub struct RuntimeConfig { @@ -101,17 +109,17 @@ impl FactorRuntimeConfig for RuntimeConfig { } pub struct AppState { - resolver: Arc, + expression_resolver: Arc, } pub struct InstanceState { component_id: String, - resolver: Arc, + expression_resolver: Arc, } impl InstanceState { - pub fn resolver(&self) -> &Arc { - &self.resolver + pub fn expression_resolver(&self) -> &Arc { + &self.expression_resolver } } @@ -121,7 +129,7 @@ impl SelfInstanceBuilder for InstanceState {} impl variables::Host for InstanceState { async fn get(&mut self, key: String) -> Result { let key = spin_expressions::Key::new(&key).map_err(expressions_to_variables_err)?; - self.resolver + self.expression_resolver .resolve(&self.component_id, key) .await .map_err(expressions_to_variables_err) diff --git a/crates/factor-variables/src/provider.rs b/crates/factor-variables/src/provider.rs index 6a8795fb9..82821fb32 100644 --- a/crates/factor-variables/src/provider.rs +++ b/crates/factor-variables/src/provider.rs @@ -3,14 +3,14 @@ use spin_expressions::Provider; use spin_factors::anyhow; /// A trait for converting a runtime configuration into a variables provider. -pub trait MakeVariablesProvider: 'static { +pub trait ProviderResolver: 'static { /// Serialized configuration for the provider. type RuntimeConfig: DeserializeOwned; /// Create a variables provider from the given runtime configuration. /// /// Returns `Ok(None)` if the provider is not applicable to the given configuration. - fn make_provider( + fn resolve_provider( &self, runtime_config: &Self::RuntimeConfig, ) -> anyhow::Result>>; diff --git a/crates/factor-variables/src/spin_cli/env.rs b/crates/factor-variables/src/spin_cli/env.rs index ca4ad6fa8..31717923b 100644 --- a/crates/factor-variables/src/spin_cli/env.rs +++ b/crates/factor-variables/src/spin_cli/env.rs @@ -11,17 +11,17 @@ use spin_factors::anyhow::{self, Context as _}; use spin_world::async_trait; use tracing::{instrument, Level}; -use crate::MakeVariablesProvider; +use crate::ProviderResolver; use super::RuntimeConfig; /// Creator of a environment variables provider. pub struct EnvVariables; -impl MakeVariablesProvider for EnvVariables { +impl ProviderResolver for EnvVariables { type RuntimeConfig = RuntimeConfig; - fn make_provider( + fn resolve_provider( &self, runtime_config: &Self::RuntimeConfig, ) -> anyhow::Result>> { diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index 448625d8d..dd1b79bfd 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -9,9 +9,12 @@ pub use statik::StaticVariables; use serde::Deserialize; use statik::StaticVariablesProvider; +/// The runtime configuration for the variables factor used in the Spin CLI. #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] pub enum RuntimeConfig { + /// A static provider of variables. Static(StaticVariablesProvider), + /// An environment variable provider. Env(env::EnvVariablesConfig), } diff --git a/crates/factor-variables/src/spin_cli/statik.rs b/crates/factor-variables/src/spin_cli/statik.rs index 8e31cff12..a63694057 100644 --- a/crates/factor-variables/src/spin_cli/statik.rs +++ b/crates/factor-variables/src/spin_cli/statik.rs @@ -4,17 +4,17 @@ use serde::Deserialize; use spin_expressions::{async_trait::async_trait, Key, Provider}; use spin_factors::anyhow; -use crate::MakeVariablesProvider; +use crate::ProviderResolver; use super::RuntimeConfig; /// Creator of a static variables provider. pub struct StaticVariables; -impl MakeVariablesProvider for StaticVariables { +impl ProviderResolver for StaticVariables { type RuntimeConfig = RuntimeConfig; - fn make_provider( + fn resolve_provider( &self, runtime_config: &Self::RuntimeConfig, ) -> anyhow::Result>> { diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index 734c9fa21..aaf7ceb51 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -1,4 +1,4 @@ -use spin_factor_variables::spin_cli::{RuntimeConfig, StaticVariables}; +use spin_factor_variables::spin_cli::{EnvVariables, RuntimeConfig, StaticVariables}; use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; @@ -30,13 +30,15 @@ async fn static_provider_works() -> anyhow::Result<()> { let mut factors = TestFactors { variables: VariablesFactor::default(), }; - factors.variables.add_provider_type(StaticVariables)?; + factors.variables.add_provider_resolver(StaticVariables)?; + // The env provider will be ignored since there's no configuration for it. + factors.variables.add_provider_resolver(EnvVariables)?; let env = test_env(); let state = env.build_instance_state(factors).await?; let val = state .variables - .resolver() + .expression_resolver() .resolve("test-component", "baz".try_into().unwrap()) .await?; assert_eq!(val, ""); diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index fb92b509a..3c8602855 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -61,7 +61,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { key_value: KeyValueFactor::new(key_value_resolver), }; - factors.variables.add_provider_type(StaticVariables)?; + factors.variables.add_provider_resolver(StaticVariables)?; let locked = spin_loader::from_file( "tests/smoke-app/spin.toml", @@ -83,7 +83,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { assert_eq!( state .variables - .resolver() + .expression_resolver() .resolve("smoke-app", "other".try_into().unwrap()) .await .unwrap(), From 87b55f1c96cc6eebae02b8ec2a9a7d98b3589594 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 23 Jul 2024 12:23:33 +0200 Subject: [PATCH 082/195] Test the actual host impl Signed-off-by: Ryan Levick --- crates/factor-outbound-http/tests/factor_test.rs | 2 +- crates/factor-outbound-networking/src/lib.rs | 2 +- .../tests/factor_test.rs | 2 +- crates/factor-outbound-pg/tests/factor_test.rs | 2 +- crates/factor-outbound-redis/tests/factor_test.rs | 2 +- crates/factor-variables/src/spin_cli/env.rs | 6 +++--- crates/factor-variables/src/spin_cli/mod.rs | 10 ++++++++-- crates/factor-variables/src/spin_cli/statik.rs | 6 +++--- crates/factor-variables/tests/factor_test.rs | 15 +++++++-------- crates/factors/tests/smoke.rs | 6 +++--- 10 files changed, 29 insertions(+), 24 deletions(-) diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index e1dc70bbc..d1761eda1 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -4,7 +4,7 @@ use anyhow::bail; use http::Request; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::VariablesFactor; +use spin_factor_variables::spin_cli::VariablesFactor; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index 9d7f04b2d..fbb973947 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -60,7 +60,7 @@ impl Factor for OutboundNetworkingFactor { .cloned() .context("missing component allowed hosts")?; let resolver = builders - .get_mut::()? + .get_mut::>()? .expression_resolver() .clone(); let allowed_hosts_future = async move { diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs index 0bd409b75..75e62a542 100644 --- a/crates/factor-outbound-networking/tests/factor_test.rs +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -1,5 +1,5 @@ use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::VariablesFactor; +use spin_factor_variables::spin_cli::VariablesFactor; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; diff --git a/crates/factor-outbound-pg/tests/factor_test.rs b/crates/factor-outbound-pg/tests/factor_test.rs index 07f47cc0c..87c42ac1c 100644 --- a/crates/factor-outbound-pg/tests/factor_test.rs +++ b/crates/factor-outbound-pg/tests/factor_test.rs @@ -2,7 +2,7 @@ use anyhow::{bail, Result}; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_pg::client::Client; use spin_factor_outbound_pg::OutboundPgFactor; -use spin_factor_variables::{StaticVariables, VariablesFactor}; +use spin_factor_variables::spin_cli::{StaticVariables, VariablesFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::async_trait; diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs index e0d42dd54..18bcfd261 100644 --- a/crates/factor-outbound-redis/tests/factor_test.rs +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -1,7 +1,7 @@ use anyhow::bail; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_redis::OutboundRedisFactor; -use spin_factor_variables::{StaticVariables, VariablesFactor}; +use spin_factor_variables::spin_cli::{StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; diff --git a/crates/factor-variables/src/spin_cli/env.rs b/crates/factor-variables/src/spin_cli/env.rs index 31717923b..8db82e1d6 100644 --- a/crates/factor-variables/src/spin_cli/env.rs +++ b/crates/factor-variables/src/spin_cli/env.rs @@ -13,19 +13,19 @@ use tracing::{instrument, Level}; use crate::ProviderResolver; -use super::RuntimeConfig; +use super::VariableProviderConfiguration; /// Creator of a environment variables provider. pub struct EnvVariables; impl ProviderResolver for EnvVariables { - type RuntimeConfig = RuntimeConfig; + type RuntimeConfig = VariableProviderConfiguration; fn resolve_provider( &self, runtime_config: &Self::RuntimeConfig, ) -> anyhow::Result>> { - let RuntimeConfig::Env(runtime_config) = runtime_config else { + let VariableProviderConfiguration::Env(runtime_config) = runtime_config else { return Ok(None); }; Ok(Some(Box::new(EnvVariablesProvider::new( diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index dd1b79bfd..5e6a2bd28 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -9,12 +9,18 @@ pub use statik::StaticVariables; use serde::Deserialize; use statik::StaticVariablesProvider; -/// The runtime configuration for the variables factor used in the Spin CLI. +/// A runtime configuration used in the Spin CLI for one type of variable provider. #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] -pub enum RuntimeConfig { +pub enum VariableProviderConfiguration { /// A static provider of variables. Static(StaticVariablesProvider), /// An environment variable provider. Env(env::EnvVariablesConfig), } + +/// The runtime configuration for the variables factor used in the Spin CLI. +pub type RuntimeConfig = super::RuntimeConfig; + +/// The variables factor used in the Spin CLI. +pub type VariablesFactor = super::VariablesFactor; diff --git a/crates/factor-variables/src/spin_cli/statik.rs b/crates/factor-variables/src/spin_cli/statik.rs index a63694057..a34756526 100644 --- a/crates/factor-variables/src/spin_cli/statik.rs +++ b/crates/factor-variables/src/spin_cli/statik.rs @@ -6,19 +6,19 @@ use spin_factors::anyhow; use crate::ProviderResolver; -use super::RuntimeConfig; +use super::VariableProviderConfiguration; /// Creator of a static variables provider. pub struct StaticVariables; impl ProviderResolver for StaticVariables { - type RuntimeConfig = RuntimeConfig; + type RuntimeConfig = VariableProviderConfiguration; fn resolve_provider( &self, runtime_config: &Self::RuntimeConfig, ) -> anyhow::Result>> { - let RuntimeConfig::Static(config) = runtime_config else { + let VariableProviderConfiguration::Static(config) = runtime_config else { return Ok(None); }; Ok(Some(Box::new(config.clone()) as _)) diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index aaf7ceb51..80bf480d1 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -1,11 +1,14 @@ -use spin_factor_variables::spin_cli::{EnvVariables, RuntimeConfig, StaticVariables}; +use spin_factor_variables::spin_cli::{ + EnvVariables, StaticVariables, VariableProviderConfiguration, +}; use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; +use spin_world::v2::variables::Host; #[derive(RuntimeFactors)] struct TestFactors { - variables: VariablesFactor, + variables: VariablesFactor, } fn test_env() -> TestEnvironment { @@ -35,12 +38,8 @@ async fn static_provider_works() -> anyhow::Result<()> { factors.variables.add_provider_resolver(EnvVariables)?; let env = test_env(); - let state = env.build_instance_state(factors).await?; - let val = state - .variables - .expression_resolver() - .resolve("test-component", "baz".try_into().unwrap()) - .await?; + let mut state = env.build_instance_state(factors).await?; + let val = state.variables.get("baz".try_into().unwrap()).await?; assert_eq!(val, ""); Ok(()) } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 3c8602855..475663249 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -11,7 +11,7 @@ use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::{StaticVariables, VariablesFactor}; +use spin_factor_variables::{spin_cli::StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; use wasmtime_wasi_http::WasiHttpView; @@ -19,7 +19,7 @@ use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] struct Factors { wasi: WasiFactor, - variables: VariablesFactor, + variables: VariablesFactor, outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, key_value: KeyValueFactor, @@ -142,7 +142,7 @@ struct TestSource; impl RuntimeConfigSource for TestSource { fn factor_config_keys(&self) -> impl IntoIterator { - [spin_factor_variables::RuntimeConfig::KEY] + [spin_factor_variables::RuntimeConfig::<()>::KEY] } fn get_factor_config( From 732f24e8242c8793b5cb4719a3c78aa8d604c8ad Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 23 Jul 2024 17:01:09 +0200 Subject: [PATCH 083/195] Check for uniquness of types Signed-off-by: Ryan Levick --- crates/factors-derive/src/lib.rs | 15 +++++++++++++-- crates/factors/src/lib.rs | 2 ++ crates/factors/tests/smoke.rs | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index c65f47485..6a9905b24 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -81,6 +81,17 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &mut self, linker: &mut #wasmtime::component::Linker, ) -> #Result<()> { + let factor_type_ids = [#( + (stringify!(#factor_types), #TypeId::of::<(<#factor_types as #Factor>::InstanceBuilder, <#factor_types as #Factor>::AppState)>()), + )*]; + + let mut unique = ::std::collections::HashSet::new(); + for (name, type_id) in factor_type_ids { + if !unique.insert(type_id) { + return Err(#Error::DuplicateFactorTypes(name.to_owned())); + } + } + #( #Factor::init::( &mut self.#factor_names, @@ -175,9 +186,9 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { fn instance_builder_mut( builders: &mut Self::InstanceBuilders, ) -> Option> { - let type_id = #TypeId::of::(); + let type_id = #TypeId::of::<(F::InstanceBuilder, F::AppState)>(); #( - if type_id == #TypeId::of::<#factor_types>() { + if type_id == #TypeId::of::<(<#factor_types as #Factor>::InstanceBuilder, <#factor_types as #Factor>::AppState)>() { return Some( builders.#factor_names.as_mut().map(|builder| { ::downcast_mut(builder).unwrap() diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 69f5c20b8..0b88b911e 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -22,6 +22,8 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum Error { + #[error("two or more factors share the same type: {0}")] + DuplicateFactorTypes(String), #[error("factor dependency ordering error: {0}")] DependencyOrderingError(String), #[error("{factor}::InstanceBuilder::build failed: {source}")] diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 475663249..091dd3326 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -74,7 +74,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; let mut linker = wasmtime::component::Linker::new(&engine); - factors.init(&mut linker).unwrap(); + factors.init(&mut linker)?; let configured_app = factors.configure_app(app, TestSource)?; let builders = factors.prepare(&configured_app, "smoke-app")?; From 29febb04b41cb171b79dd5eaf22372e1703d92b5 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 24 Jul 2024 11:25:37 +0200 Subject: [PATCH 084/195] WIP code crimes Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/lib.rs | 25 ++-- crates/factor-sqlite/src/runtime_config.rs | 26 +--- .../factor-sqlite/src/runtime_config/spin.rs | 53 +++++-- crates/factor-sqlite/tests/factor.rs | 135 ++++++++++++++---- crates/factors-derive/src/lib.rs | 7 +- crates/factors-test/src/lib.rs | 46 +++--- crates/factors/src/factor.rs | 11 +- crates/factors/src/lib.rs | 9 +- crates/factors/src/runtime_config.rs | 92 +----------- 9 files changed, 198 insertions(+), 206 deletions(-) diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 1c270b9a4..4306dc0dd 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -7,29 +7,29 @@ use std::sync::Arc; use host::InstanceState; use async_trait::async_trait; -use runtime_config::RuntimeConfigResolver; +use runtime_config::DefaultLabelResolver; use spin_factors::{anyhow, Factor}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; -pub struct SqliteFactor { - runtime_config_resolver: Arc, +pub struct SqliteFactor { + runtime_config_resolver: Arc, } -impl SqliteFactor { +impl SqliteFactor { /// Create a new `SqliteFactor` /// /// Takes a `runtime_config_resolver` that can resolve a runtime configuration into a connection pool. - pub fn new(runtime_config_resolver: R) -> Self { + pub fn new(runtime_config_resolver: impl DefaultLabelResolver + 'static) -> Self { Self { runtime_config_resolver: Arc::new(runtime_config_resolver), } } } -impl Factor for SqliteFactor { - type RuntimeConfig = runtime_config::RuntimeConfig; +impl Factor for SqliteFactor { + type RuntimeConfig = runtime_config::RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; @@ -46,13 +46,10 @@ impl Factor for SqliteFactor { &self, mut ctx: spin_factors::ConfigureAppContext, ) -> anyhow::Result { - let mut connection_pools = HashMap::new(); - if let Some(runtime_config) = ctx.take_runtime_config() { - for (database_label, config) in runtime_config.store_configs { - let pool = self.runtime_config_resolver.get_pool(config)?; - connection_pools.insert(database_label, pool); - } - } + let connection_pools = ctx + .take_runtime_config() + .map(|r| r.pools) + .unwrap_or_default(); let allowed_databases = ctx .app() diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index 10e6f8e72..3e5e5ac82 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -3,29 +3,17 @@ pub mod spin; use std::{collections::HashMap, sync::Arc}; -use serde::{de::DeserializeOwned, Deserialize}; -use spin_factors::{anyhow, FactorRuntimeConfig}; - use crate::ConnectionPool; -#[derive(Deserialize)] -#[serde(transparent)] -pub struct RuntimeConfig { - pub store_configs: HashMap, -} - -impl FactorRuntimeConfig for RuntimeConfig { - const KEY: &'static str = "sqlite_database"; +/// A runtime configuration for SQLite databases. +/// +/// Maps database labels to connection pools. +pub struct RuntimeConfig { + pub pools: HashMap>, } -/// Resolves some piece of runtime configuration to a connection pool -pub trait RuntimeConfigResolver: Send + Sync { - type Config: DeserializeOwned; - - /// Get a connection pool for a given config. - /// - fn get_pool(&self, config: Self::Config) -> anyhow::Result>; - +/// Resolves a label to a default connection pool. +pub trait DefaultLabelResolver: Send + Sync { /// If there is no runtime configuration for a given database label, return a default connection pool. /// /// If `Option::None` is returned, the database is not allowed. diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 003120661..dd72cfe40 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -10,10 +10,9 @@ use spin_factors::anyhow::{self, Context as _}; use spin_world::v2::sqlite as v2; use tokio::sync::OnceCell; +use super::DefaultLabelResolver; use crate::{Connection, ConnectionPool, SimpleConnectionPool}; -use super::RuntimeConfigResolver; - /// Spin's default handling of the runtime configuration for SQLite databases. /// /// This type implements the [`RuntimeConfigResolver`] trait and provides a way to @@ -48,20 +47,32 @@ impl SpinSqliteRuntimeConfig { local_database_dir, } } -} -#[derive(Deserialize)] -pub struct RuntimeConfig { - #[serde(rename = "type")] - pub type_: String, - #[serde(flatten)] - pub config: toml::Table, -} - -impl RuntimeConfigResolver for SpinSqliteRuntimeConfig { - type Config = RuntimeConfig; + /// Get the runtime configuration for SQLite databases from a TOML table. + /// + /// Expects table to be in the format: + /// ````toml + /// [sqlite_database.$database-label] + /// type = "$database-type" + /// ... extra type specific configuration ... + /// ``` + pub fn config_from_table>( + &self, + table: &T, + ) -> anyhow::Result> { + let Some(table) = table.get("sqlite_database") else { + return Ok(None); + }; + let config: std::collections::HashMap = table.clone().try_into()?; + let pools = config + .into_iter() + .map(|(k, v)| Ok((k, self.get_pool(v)?))) + .collect::>()?; + Ok(Some(super::RuntimeConfig { pools })) + } - fn get_pool(&self, config: RuntimeConfig) -> anyhow::Result> { + /// Get a connection pool for a given runtime configuration. + pub fn get_pool(&self, config: RuntimeConfig) -> anyhow::Result> { let database_kind = config.type_.as_str(); let pool = match database_kind { "spin" => { @@ -76,7 +87,21 @@ impl RuntimeConfigResolver for SpinSqliteRuntimeConfig { }; Ok(Arc::new(pool)) } +} + +pub trait GetKey { + fn get(&self, key: &str) -> Option<&toml::Value>; +} + +#[derive(Deserialize)] +pub struct RuntimeConfig { + #[serde(rename = "type")] + pub type_: String, + #[serde(flatten)] + pub config: toml::Table, +} +impl DefaultLabelResolver for SpinSqliteRuntimeConfig { fn default(&self, label: &str) -> Option> { // Only default the database labeled "default". if label != "default" { diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index 6897e015b..613023607 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -1,20 +1,23 @@ -use std::{collections::HashSet, sync::Arc}; +use std::{any::TypeId, cell::RefCell, collections::HashSet, sync::Arc}; -use factor_sqlite::{runtime_config::spin::RuntimeConfig, SqliteFactor}; +use factor_sqlite::{ + runtime_config::spin::{GetKey, SpinSqliteRuntimeConfig}, + SqliteFactor, +}; use spin_factors::{ anyhow::{self, bail}, - RuntimeFactors, + Factor, RuntimeConfigSource, RuntimeFactors, }; use spin_factors_test::{toml, TestEnvironment}; #[derive(RuntimeFactors)] struct TestFactors { - sqlite: SqliteFactor, + sqlite: SqliteFactor, } #[tokio::test] async fn sqlite_works() -> anyhow::Result<()> { - let test_resolver = RuntimeConfigResolver::new(Some("default")); + let test_resolver = DefaultLabelResolver::new(Some("default")); let factors = TestFactors { sqlite: SqliteFactor::new(test_resolver), }; @@ -23,7 +26,7 @@ async fn sqlite_works() -> anyhow::Result<()> { source = "does-not-exist.wasm" sqlite_databases = ["default"] }); - let state = env.build_instance_state(factors).await?; + let state = env.build_instance_state(factors, ()).await?; assert_eq!( state.sqlite.allowed_databases(), @@ -35,7 +38,7 @@ async fn sqlite_works() -> anyhow::Result<()> { #[tokio::test] async fn errors_when_non_configured_database_used() -> anyhow::Result<()> { - let test_resolver = RuntimeConfigResolver::new(None); + let test_resolver = DefaultLabelResolver::new(None); let factors = TestFactors { sqlite: SqliteFactor::new(test_resolver), }; @@ -44,7 +47,7 @@ async fn errors_when_non_configured_database_used() -> anyhow::Result<()> { source = "does-not-exist.wasm" sqlite_databases = ["foo"] }); - let Err(err) = env.build_instance_state(factors).await else { + let Err(err) = env.build_instance_state(factors, ()).await else { bail!("Expected build_instance_state to error but it did not"); }; @@ -57,50 +60,128 @@ async fn errors_when_non_configured_database_used() -> anyhow::Result<()> { #[tokio::test] async fn no_error_when_database_is_configured() -> anyhow::Result<()> { - let test_resolver = RuntimeConfigResolver::new(None); + let test_resolver = DefaultLabelResolver::new(None); let factors = TestFactors { sqlite: SqliteFactor::new(test_resolver), }; - let mut env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::default_manifest_extend(toml! { [component.test-component] source = "does-not-exist.wasm" sqlite_databases = ["foo"] }); - env.runtime_config = toml! { + let runtime_config = toml! { [sqlite_database.foo] - type = "sqlite" + type = "spin" }; - if let Err(e) = env.build_instance_state(factors).await { + let sqlite_config = SpinSqliteRuntimeConfig::new("/".into(), "/".into()); + if let Err(e) = env + .build_instance_state( + factors, + TomlRuntimeSource::new(&runtime_config, sqlite_config), + ) + .await + { bail!("Expected build_instance_state to succeed but it errored: {e}"); } Ok(()) } -/// Will return an `InvalidConnectionPool` for all runtime configured databases and the supplied default database. -struct RuntimeConfigResolver { - default: Option, +struct TomlRuntimeSource<'a> { + table: TomlKeyTracker<'a>, + sqlite_config: SpinSqliteRuntimeConfig, } -impl RuntimeConfigResolver { - fn new(default: Option<&str>) -> Self { +impl<'a> TomlRuntimeSource<'a> { + fn new(table: &'a toml::Table, sqlite_config: SpinSqliteRuntimeConfig) -> Self { Self { - default: default.map(Into::into), + table: TomlKeyTracker::new(table), + sqlite_config, } } } -impl factor_sqlite::runtime_config::RuntimeConfigResolver for RuntimeConfigResolver { - type Config = RuntimeConfig; - - fn get_pool( +impl<'a> RuntimeConfigSource for TomlRuntimeSource<'a> { + fn get_factor_config( &self, - config: RuntimeConfig, - ) -> anyhow::Result> { - let _ = config; - Ok(Arc::new(InvalidConnectionPool)) + ) -> anyhow::Result> { + if TypeId::of::() == TypeId::of::() { + let Some(config) = self.sqlite_config.config_from_table(&self.table)? else { + return Ok(None); + }; + type_cast::(config).map(Some) + } else { + Ok(None) + } + } +} + +struct TomlKeyTracker<'a> { + unused_keys: RefCell>, + table: &'a toml::Table, +} + +impl<'a> TomlKeyTracker<'a> { + fn new(table: &'a toml::Table) -> Self { + Self { + unused_keys: RefCell::new(table.keys().map(String::as_str).collect()), + table, + } + } + + fn validate_all_keys_used(self) -> Result<(), spin_factors::Error> { + if !self.unused_keys.borrow().is_empty() { + return Err(spin_factors::Error::RuntimeConfigUnusedKeys { + keys: self + .unused_keys + .borrow() + .iter() + .map(|s| (*s).to_owned()) + .collect(), + }); + } + Ok(()) + } +} + +impl GetKey for TomlKeyTracker<'_> { + fn get(&self, key: &str) -> Option<&toml::Value> { + self.unused_keys.borrow_mut().remove(key); + self.table.get(key) + } +} + +impl AsRef for TomlKeyTracker<'_> { + fn as_ref(&self) -> &toml::Table { + self.table + } +} + +/// Casts a concrete configuration type to a generic one. +/// +/// Will panic if the types are not the same. +fn type_cast( + config: F1::RuntimeConfig, +) -> Result { + assert_eq!(TypeId::of::(), TypeId::of::()); + let boxed = >::downcast::(Box::new(config)).unwrap(); + Ok(*boxed) +} + +/// Will return an `InvalidConnectionPool` for the supplied default database. +struct DefaultLabelResolver { + default: Option, +} + +impl DefaultLabelResolver { + fn new(default: Option<&str>) -> Self { + Self { + default: default.map(Into::into), + } } +} +impl factor_sqlite::runtime_config::DefaultLabelResolver for DefaultLabelResolver { fn default(&self, label: &str) -> Option> { let Some(default) = &self.default else { return None; diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 6a9905b24..99abf4cde 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -68,7 +68,6 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let Error = quote!(#factors_path::Error); let Factor = quote!(#factors_path::Factor); let ConfiguredApp = quote!(#factors_path::ConfiguredApp); - let RuntimeConfigTracker = quote!(#factors_path::__internal::RuntimeConfigTracker); let FactorInstanceBuilder = quote!(#factors_path::FactorInstanceBuilder); Ok(quote! { @@ -111,12 +110,11 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { fn configure_app( &self, app: #factors_path::App, - runtime_config: impl #factors_path::RuntimeConfigSource + mut runtime_config: impl #factors_path::RuntimeConfigSource ) -> #Result<#ConfiguredApp> { let mut app_state = #app_state_name { #( #factor_names: None, )* }; - let mut runtime_config_tracker = #RuntimeConfigTracker::new(runtime_config); #( app_state.#factor_names = Some( #Factor::configure_app( @@ -124,12 +122,11 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #factors_path::ConfigureAppContext::::new( &app, &app_state, - &mut runtime_config_tracker, + &mut runtime_config, )?, ).map_err(#Error::factor_configure_app_error::<#factor_types>)? ); )* - runtime_config_tracker.validate_all_keys_used()?; Ok(#ConfiguredApp::new(app, app_state)) } diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 7e9c78232..f4d2f4fca 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -1,7 +1,6 @@ use spin_app::locked::LockedApp; use spin_factors::{ anyhow::{self, Context}, - serde::de::DeserializeOwned, wasmtime::{component::Linker, Config, Engine}, App, RuntimeConfigSource, RuntimeFactors, }; @@ -13,8 +12,6 @@ pub use toml::toml; pub struct TestEnvironment { /// The `spin.toml` manifest. pub manifest: toml::Table, - /// The runtime config. - pub runtime_config: toml::Table, } impl Default for TestEnvironment { @@ -30,10 +27,7 @@ impl Default for TestEnvironment { [component.empty] source = "does-not-exist.wasm" }; - Self { - manifest, - runtime_config: Default::default(), - } + Self { manifest } } } @@ -53,9 +47,10 @@ impl TestEnvironment { /// Starting from a new _uninitialized_ [`RuntimeFactors`], run through the /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`] /// for the last component defined in the manifest. - pub async fn build_instance_state( - &self, + pub async fn build_instance_state<'a, T: RuntimeFactors, C: RuntimeConfigSource + 'a>( + &'a self, mut factors: T, + runtime_config: C, ) -> anyhow::Result { let mut linker = Self::new_linker::(); factors.init(&mut linker)?; @@ -65,7 +60,6 @@ impl TestEnvironment { .await .context("failed to build locked app")?; let app = App::new("test-app", locked_app); - let runtime_config = TomlRuntimeConfig(&self.runtime_config); let configured_app = factors.configure_app(app, runtime_config)?; let component = @@ -92,19 +86,19 @@ impl TestEnvironment { } } -/// A [`RuntimeConfigSource`] that reads from a TOML table. -pub struct TomlRuntimeConfig<'a>(&'a toml::Table); - -impl RuntimeConfigSource for TomlRuntimeConfig<'_> { - fn factor_config_keys(&self) -> impl IntoIterator { - self.0.keys().map(|key| key.as_str()) - } - - fn get_factor_config(&self, key: &str) -> anyhow::Result> { - let Some(val) = self.0.get(key) else { - return Ok(None); - }; - let config = val.clone().try_into()?; - Ok(Some(config)) - } -} +// / A [`RuntimeConfigSource`] that reads from a TOML table. +// pub struct TomlRuntimeConfig<'a>(&'a toml::Table); + +// impl RuntimeConfigSource for TomlRuntimeConfig<'_> { +// fn factor_config_keys(&self) -> impl IntoIterator { +// self.0.keys().map(|key| key.as_str()) +// } + +// fn get_factor_config(&self, key: &str) -> anyhow::Result> { +// let Some(val) = self.0.get(key) else { +// return Ok(None); +// }; +// let config = val.clone().try_into()?; +// Ok(Some(config)) +// } +// } diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 72c57371f..536be7864 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -3,8 +3,8 @@ use std::any::Any; use wasmtime::component::{Linker, ResourceTable}; use crate::{ - prepare::FactorInstanceBuilder, runtime_config::RuntimeConfigTracker, App, Error, - FactorRuntimeConfig, InstanceBuilders, PrepareContext, RuntimeConfigSource, RuntimeFactors, + prepare::FactorInstanceBuilder, App, Error, InstanceBuilders, PrepareContext, + RuntimeConfigSource, RuntimeFactors, }; /// A contained (i.e., "factored") piece of runtime functionality. @@ -13,7 +13,7 @@ pub trait Factor: Any + Sized { /// /// Runtime configuration allows for user-provided customization of the /// factor's behavior on a per-app basis. - type RuntimeConfig: FactorRuntimeConfig; + type RuntimeConfig; /// The application state of this factor. /// @@ -140,9 +140,10 @@ impl<'a, T: RuntimeFactors, F: Factor> ConfigureAppContext<'a, T, F> { pub fn new( app: &'a App, app_state: &'a T::AppState, - runtime_config_tracker: &mut RuntimeConfigTracker, + runtime_config: &mut S, ) -> crate::Result { - let runtime_config = runtime_config_tracker.get_config::()?; + // TODO: fix error + let runtime_config = runtime_config.get_factor_config::().unwrap(); Ok(Self { app, app_state, diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 0b88b911e..6d58cdfe6 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -13,7 +13,7 @@ pub use spin_factors_derive::RuntimeFactors; pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, FactorInstanceState, InitContext}, prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext, SelfInstanceBuilder}, - runtime_config::{FactorRuntimeConfig, RuntimeConfigSource}, + runtime_config::RuntimeConfigSource, runtime_factors::{RuntimeFactors, RuntimeFactorsInstanceState}, }; @@ -63,7 +63,7 @@ impl Error { Self::NoSuchFactor(std::any::type_name::()) } - fn runtime_config_reused_key(key: impl Into) -> Self { + pub fn runtime_config_reused_key(key: impl Into) -> Self { Self::RuntimeConfigReusedKey { factor: std::any::type_name::(), key: key.into(), @@ -96,8 +96,3 @@ impl Error { Self::FactorBuildError { factor, source } } } - -#[doc(hidden)] -pub mod __internal { - pub use crate::runtime_config::RuntimeConfigTracker; -} diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index 67ed29c50..15c3c7564 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -1,103 +1,17 @@ -use std::collections::HashSet; - -use serde::de::DeserializeOwned; - -use crate::{Error, Factor}; - -pub const NO_RUNTIME_CONFIG: &str = ""; - -/// FactorRuntimeConfig represents an application's runtime configuration. -/// -/// Runtime configuration is partitioned, with each partition being the -/// responsibility of exactly one [`crate::Factor`]. If configuration needs -/// to be shared between Factors, one Factor can be selected as the owner -/// and the others will have a dependency relationship with that owner. -pub trait FactorRuntimeConfig: DeserializeOwned { - /// The key used to identify this runtime configuration in a [`RuntimeConfigSource`]. - const KEY: &'static str; -} - -impl FactorRuntimeConfig for () { - const KEY: &'static str = NO_RUNTIME_CONFIG; -} +use crate::Factor; /// The source of runtime configuration for a Factor. pub trait RuntimeConfigSource { - /// Returns an iterator of factor config keys available in this source. - /// - /// Should only include keys that have been positively provided and that - /// haven't already been parsed by the runtime. A runtime may treat - /// unrecognized keys as a warning or error. - fn factor_config_keys(&self) -> impl IntoIterator; - /// Returns deserialized runtime config of the given type for the given /// factor config key. /// /// Returns Ok(None) if no configuration is available for the given key. /// Returns Err if configuration is available but deserialization fails. - fn get_factor_config(&self, key: &str) -> anyhow::Result>; + fn get_factor_config(&self) -> anyhow::Result>; } impl RuntimeConfigSource for () { - fn get_factor_config( - &self, - _factor_config_key: &str, - ) -> anyhow::Result> { + fn get_factor_config(&self) -> anyhow::Result> { Ok(None) } - - fn factor_config_keys(&self) -> impl IntoIterator { - std::iter::empty() - } -} - -/// Tracks runtime configuration keys used by the runtime. -/// -/// This ensures that the runtime config source does not have any unused keys. -#[doc(hidden)] -pub struct RuntimeConfigTracker { - source: S, - used_keys: HashSet<&'static str>, - unused_keys: HashSet, -} - -impl RuntimeConfigTracker { - #[doc(hidden)] - pub fn new(source: S) -> Self { - let unused_keys = source - .factor_config_keys() - .into_iter() - .map(ToOwned::to_owned) - .collect(); - Self { - source, - used_keys: Default::default(), - unused_keys, - } - } - - #[doc(hidden)] - pub fn validate_all_keys_used(self) -> crate::Result<()> { - if !self.unused_keys.is_empty() { - return Err(Error::RuntimeConfigUnusedKeys { - keys: self.unused_keys.into_iter().collect(), - }); - } - Ok(()) - } - - /// Get the runtime configuration for a factor. - pub(crate) fn get_config(&mut self) -> crate::Result> { - let key = F::RuntimeConfig::KEY; - if key == NO_RUNTIME_CONFIG { - return Ok(None); - } - if !self.used_keys.insert(key) { - return Err(Error::runtime_config_reused_key::(key)); - } - self.unused_keys.remove(key); - self.source - .get_factor_config::(key) - .map_err(Error::RuntimeConfigSource) - } } From 703ebe31a630bd4c151785c4e3fb83bd5e99596a Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 24 Jul 2024 16:33:41 +0200 Subject: [PATCH 085/195] More refactorings Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/lib.rs | 20 +++++++++++++------ crates/factor-sqlite/src/runtime_config.rs | 8 -------- .../factor-sqlite/src/runtime_config/spin.rs | 3 +-- crates/factor-sqlite/tests/factor.rs | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 4306dc0dd..10c4b2325 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -7,23 +7,23 @@ use std::sync::Arc; use host::InstanceState; use async_trait::async_trait; -use runtime_config::DefaultLabelResolver; use spin_factors::{anyhow, Factor}; use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; pub struct SqliteFactor { - runtime_config_resolver: Arc, + default_label_resolver: Arc, } impl SqliteFactor { /// Create a new `SqliteFactor` /// - /// Takes a `runtime_config_resolver` that can resolve a runtime configuration into a connection pool. - pub fn new(runtime_config_resolver: impl DefaultLabelResolver + 'static) -> Self { + /// Takes a `default_label_resolver` for how to handle when a database label doesn't + /// have a corresponding runtime configuration. + pub fn new(default_label_resolver: impl DefaultLabelResolver + 'static) -> Self { Self { - runtime_config_resolver: Arc::new(runtime_config_resolver), + default_label_resolver: Arc::new(default_label_resolver), } } } @@ -67,7 +67,7 @@ impl Factor for SqliteFactor { )) }) .collect::>>()?; - let resolver = self.runtime_config_resolver.clone(); + let resolver = self.default_label_resolver.clone(); let get_connection_pool: host::ConnectionPoolGetter = Arc::new(move |label| { connection_pools .get(label) @@ -136,6 +136,14 @@ fn ensure_allowed_databases_are_configured( pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); +/// Resolves a label to a default connection pool. +pub trait DefaultLabelResolver: Send + Sync { + /// If there is no runtime configuration for a given database label, return a default connection pool. + /// + /// If `Option::None` is returned, the database is not allowed. + fn default(&self, label: &str) -> Option>; +} + pub struct AppState { /// A map from component id to a set of allowed database labels. allowed_databases: HashMap>>, diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index 3e5e5ac82..ca13c4c02 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -11,11 +11,3 @@ use crate::ConnectionPool; pub struct RuntimeConfig { pub pools: HashMap>, } - -/// Resolves a label to a default connection pool. -pub trait DefaultLabelResolver: Send + Sync { - /// If there is no runtime configuration for a given database label, return a default connection pool. - /// - /// If `Option::None` is returned, the database is not allowed. - fn default(&self, label: &str) -> Option>; -} diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index dd72cfe40..515aeb788 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -10,8 +10,7 @@ use spin_factors::anyhow::{self, Context as _}; use spin_world::v2::sqlite as v2; use tokio::sync::OnceCell; -use super::DefaultLabelResolver; -use crate::{Connection, ConnectionPool, SimpleConnectionPool}; +use crate::{Connection, ConnectionPool, DefaultLabelResolver, SimpleConnectionPool}; /// Spin's default handling of the runtime configuration for SQLite databases. /// diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index 613023607..2ba819708 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -181,7 +181,7 @@ impl DefaultLabelResolver { } } -impl factor_sqlite::runtime_config::DefaultLabelResolver for DefaultLabelResolver { +impl factor_sqlite::DefaultLabelResolver for DefaultLabelResolver { fn default(&self, label: &str) -> Option> { let Some(default) = &self.default else { return None; From e9b32bdc267721b939bc382f0f05f7d2106ed377 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 26 Jul 2024 13:58:25 +0200 Subject: [PATCH 086/195] Implement @lann's suggestion Signed-off-by: Ryan Levick --- Cargo.lock | 8 ++--- crates/factor-sqlite/tests/factor.rs | 42 ++++++++-------------- crates/factors-derive/src/lib.rs | 50 +++++++++++++++++++++++++-- crates/factors-test/src/lib.rs | 25 ++++---------- crates/factors/src/factor.rs | 11 +++--- crates/factors/src/lib.rs | 4 ++- crates/factors/src/runtime_config.rs | 40 ++++++++++++++++----- crates/factors/src/runtime_factors.rs | 4 ++- 8 files changed, 116 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dae53727..7a1bfd987 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8689,9 +8689,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -8713,9 +8713,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index 2ba819708..ffc743b94 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -1,4 +1,4 @@ -use std::{any::TypeId, cell::RefCell, collections::HashSet, sync::Arc}; +use std::{cell::RefCell, collections::HashSet, sync::Arc}; use factor_sqlite::{ runtime_config::spin::{GetKey, SpinSqliteRuntimeConfig}, @@ -6,7 +6,7 @@ use factor_sqlite::{ }; use spin_factors::{ anyhow::{self, bail}, - Factor, RuntimeConfigSource, RuntimeFactors, + Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, }; use spin_factors_test::{toml, TestEnvironment}; @@ -77,7 +77,7 @@ async fn no_error_when_database_is_configured() -> anyhow::Result<()> { if let Err(e) = env .build_instance_state( factors, - TomlRuntimeSource::new(&runtime_config, sqlite_config), + Foo(TomlRuntimeSource::new(&runtime_config, sqlite_config)), ) .await { @@ -101,18 +101,17 @@ impl<'a> TomlRuntimeSource<'a> { } } -impl<'a> RuntimeConfigSource for TomlRuntimeSource<'a> { - fn get_factor_config( - &self, - ) -> anyhow::Result> { - if TypeId::of::() == TypeId::of::() { - let Some(config) = self.sqlite_config.config_from_table(&self.table)? else { - return Ok(None); - }; - type_cast::(config).map(Some) - } else { - Ok(None) - } +impl FactorRuntimeConfigSource for TomlRuntimeSource<'_> { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result::RuntimeConfig>> { + self.sqlite_config.config_from_table(&self.table) + } +} + +impl RuntimeConfigSourceFinalizer for TomlRuntimeSource<'_> { + fn finalize(&mut self) -> anyhow::Result<()> { + Ok(self.table.validate_all_keys_used().unwrap()) } } @@ -129,7 +128,7 @@ impl<'a> TomlKeyTracker<'a> { } } - fn validate_all_keys_used(self) -> Result<(), spin_factors::Error> { + fn validate_all_keys_used(&self) -> anyhow::Result<(), spin_factors::Error> { if !self.unused_keys.borrow().is_empty() { return Err(spin_factors::Error::RuntimeConfigUnusedKeys { keys: self @@ -157,17 +156,6 @@ impl AsRef for TomlKeyTracker<'_> { } } -/// Casts a concrete configuration type to a generic one. -/// -/// Will panic if the types are not the same. -fn type_cast( - config: F1::RuntimeConfig, -) -> Result { - assert_eq!(TypeId::of::(), TypeId::of::()); - let boxed = >::downcast::(Box::new(config)).unwrap(); - Ok(*boxed) -} - /// Will return an `InvalidConnectionPool` for the supplied default database. struct DefaultLabelResolver { default: Option, diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 99abf4cde..87e07ba6b 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -27,6 +27,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { let app_state_name = format_ident!("{name}AppState"); let builders_name = format_ident!("{name}InstanceBuilders"); let state_name = format_ident!("{name}InstanceState"); + let runtime_config_name = format_ident!("{name}RuntimeConfig"); if !input.generics.params.is_empty() { return Err(Error::new_spanned( @@ -75,6 +76,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { type AppState = #app_state_name; type InstanceBuilders = #builders_name; type InstanceState = #state_name; + type RuntimeConfig = #runtime_config_name; fn init + Send + 'static>( &mut self, @@ -110,11 +112,12 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { fn configure_app( &self, app: #factors_path::App, - mut runtime_config: impl #factors_path::RuntimeConfigSource + mut runtime_config: impl #factors_path::RuntimeConfigSource, ) -> #Result<#ConfiguredApp> { let mut app_state = #app_state_name { #( #factor_names: None, )* }; + let mut configs = runtime_config.get_factor_configs().map_err(#Error::RuntimeConfigSource)?; #( app_state.#factor_names = Some( #Factor::configure_app( @@ -122,7 +125,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #factors_path::ConfigureAppContext::::new( &app, &app_state, - &mut runtime_config, + &mut configs, )?, ).map_err(#Error::factor_configure_app_error::<#factor_types>)? ); @@ -242,5 +245,48 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { self } } + + #vis struct #runtime_config_name { + #( + pub #factor_names: Option<<#factor_types as #Factor>::RuntimeConfig>, + )* + } + + /// TODO: figure out how to make this work + struct Foo(T); + + impl #factors_path::RuntimeConfigSource<#runtime_config_name> for Foo + where T: #(#factors_path::FactorRuntimeConfigSource<#factor_types> +)* #factors_path::RuntimeConfigSourceFinalizer + { + fn get_factor_configs(&mut self) -> anyhow::Result<#runtime_config_name> { + #( + let #factor_names = >::get_runtime_config(&mut self.0)?; + )* + self.0.finalize()?; + Ok(#runtime_config_name { + #( + #factor_names, + )* + }) + } + } + impl #factors_path::RuntimeConfigSource<#runtime_config_name> for () { + fn get_factor_configs(&mut self) -> anyhow::Result<#runtime_config_name> { + Ok(#runtime_config_name { + #( + #factor_names: None, + )* + }) + } + } + + #( + impl #factors_path::FactorRuntimeConfigSource<#factor_types> for #runtime_config_name { + fn get_runtime_config(&mut self) -> anyhow::Result::RuntimeConfig>> { + Ok(self.#factor_names.take()) + } + } + )* + }) } diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index f4d2f4fca..0f12b599d 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -47,10 +47,14 @@ impl TestEnvironment { /// Starting from a new _uninitialized_ [`RuntimeFactors`], run through the /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`] /// for the last component defined in the manifest. - pub async fn build_instance_state<'a, T: RuntimeFactors, C: RuntimeConfigSource + 'a>( + pub async fn build_instance_state< + 'a, + T: RuntimeFactors, + S: RuntimeConfigSource + 'a, + >( &'a self, mut factors: T, - runtime_config: C, + runtime_config: S, ) -> anyhow::Result { let mut linker = Self::new_linker::(); factors.init(&mut linker)?; @@ -85,20 +89,3 @@ impl TestEnvironment { spin_loader::from_file(&path, FilesMountStrategy::Direct, None).await } } - -// / A [`RuntimeConfigSource`] that reads from a TOML table. -// pub struct TomlRuntimeConfig<'a>(&'a toml::Table); - -// impl RuntimeConfigSource for TomlRuntimeConfig<'_> { -// fn factor_config_keys(&self) -> impl IntoIterator { -// self.0.keys().map(|key| key.as_str()) -// } - -// fn get_factor_config(&self, key: &str) -> anyhow::Result> { -// let Some(val) = self.0.get(key) else { -// return Ok(None); -// }; -// let config = val.clone().try_into()?; -// Ok(Some(config)) -// } -// } diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 536be7864..39be75e59 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -3,8 +3,8 @@ use std::any::Any; use wasmtime::component::{Linker, ResourceTable}; use crate::{ - prepare::FactorInstanceBuilder, App, Error, InstanceBuilders, PrepareContext, - RuntimeConfigSource, RuntimeFactors, + prepare::FactorInstanceBuilder, runtime_config::FactorRuntimeConfigSource, App, Error, + InstanceBuilders, PrepareContext, RuntimeFactors, }; /// A contained (i.e., "factored") piece of runtime functionality. @@ -137,13 +137,14 @@ pub struct ConfigureAppContext<'a, T: RuntimeFactors, F: Factor> { impl<'a, T: RuntimeFactors, F: Factor> ConfigureAppContext<'a, T, F> { #[doc(hidden)] - pub fn new( + pub fn new>( app: &'a App, app_state: &'a T::AppState, runtime_config: &mut S, ) -> crate::Result { - // TODO: fix error - let runtime_config = runtime_config.get_factor_config::().unwrap(); + let runtime_config = runtime_config + .get_runtime_config() + .map_err(Error::factor_configure_app_error::)?; Ok(Self { app, app_state, diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 6d58cdfe6..5af5a525c 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -13,7 +13,9 @@ pub use spin_factors_derive::RuntimeFactors; pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, FactorInstanceState, InitContext}, prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext, SelfInstanceBuilder}, - runtime_config::RuntimeConfigSource, + runtime_config::{ + FactorRuntimeConfigSource, RuntimeConfigSource, RuntimeConfigSourceFinalizer, + }, runtime_factors::{RuntimeFactors, RuntimeFactorsInstanceState}, }; diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index 15c3c7564..bda6a753c 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -1,17 +1,39 @@ use crate::Factor; -/// The source of runtime configuration for a Factor. -pub trait RuntimeConfigSource { - /// Returns deserialized runtime config of the given type for the given - /// factor config key. +/// The source of runtime configuration for a [`RuntimeFactors`][crate::RuntimeFactors]. +pub trait RuntimeConfigSource { + /// Get the runtime configuration for all the factors. /// - /// Returns Ok(None) if no configuration is available for the given key. - /// Returns Err if configuration is available but deserialization fails. - fn get_factor_config(&self) -> anyhow::Result>; + /// This will be called once per call to [`RuntimeFactors::configure_app`][crate::RuntimeFactors::configure_app]. + fn get_factor_configs(&mut self) -> anyhow::Result; } -impl RuntimeConfigSource for () { - fn get_factor_config(&self) -> anyhow::Result> { +impl RuntimeConfigSource<()> for () { + fn get_factor_configs(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} + +/// The source of runtime configuration for a particular [`Factor`]. +pub trait FactorRuntimeConfigSource { + /// Get the runtime configuration for the factor. + fn get_runtime_config(&mut self) -> anyhow::Result>; +} + +impl FactorRuntimeConfigSource for () { + fn get_runtime_config(&mut self) -> anyhow::Result::RuntimeConfig>> { Ok(None) } } + +/// Run some finalization logic on a [`RuntimeConfigSource`]. +pub trait RuntimeConfigSourceFinalizer { + /// Finalize the runtime config source. + fn finalize(&mut self) -> anyhow::Result<()>; +} + +impl RuntimeConfigSourceFinalizer for () { + fn finalize(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 78154b7eb..ad0203918 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -37,6 +37,8 @@ pub trait RuntimeFactors: Sized + 'static { type InstanceState: RuntimeFactorsInstanceState; /// The collection of all the `InstanceBuilder`s of the factors. type InstanceBuilders; + /// TODO + type RuntimeConfig; /// Initialize the factors with the given linker. /// @@ -51,7 +53,7 @@ pub trait RuntimeFactors: Sized + 'static { fn configure_app( &self, app: App, - runtime_config: impl RuntimeConfigSource, + runtime_config: impl RuntimeConfigSource, ) -> crate::Result>; /// Prepare the factors' instance state builders. From 92e74b3bd3e60466aea901d0c83c0d91afcd098b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 26 Jul 2024 16:26:57 +0200 Subject: [PATCH 087/195] Separate out turning source into config Signed-off-by: Ryan Levick --- crates/factor-sqlite/tests/factor.rs | 10 ++++++++- crates/factors-derive/src/lib.rs | 30 +++++++++++++-------------- crates/factors-test/src/lib.rs | 20 ++++++++++-------- crates/factors/src/lib.rs | 4 +--- crates/factors/src/runtime_config.rs | 14 ------------- crates/factors/src/runtime_factors.rs | 4 ++-- 6 files changed, 37 insertions(+), 45 deletions(-) diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index ffc743b94..a33adcfc9 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -77,7 +77,7 @@ async fn no_error_when_database_is_configured() -> anyhow::Result<()> { if let Err(e) = env .build_instance_state( factors, - Foo(TomlRuntimeSource::new(&runtime_config, sqlite_config)), + TomlRuntimeSource::new(&runtime_config, sqlite_config), ) .await { @@ -115,6 +115,14 @@ impl RuntimeConfigSourceFinalizer for TomlRuntimeSource<'_> { } } +impl TryFrom> for TestFactorsRuntimeConfig { + type Error = anyhow::Error; + + fn try_from(value: TomlRuntimeSource<'_>) -> Result { + Self::from_source(value) + } +} + struct TomlKeyTracker<'a> { unused_keys: RefCell>, table: &'a toml::Table, diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 87e07ba6b..5b2d38568 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -112,12 +112,11 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { fn configure_app( &self, app: #factors_path::App, - mut runtime_config: impl #factors_path::RuntimeConfigSource, + mut runtime_config: Self::RuntimeConfig, ) -> #Result<#ConfiguredApp> { let mut app_state = #app_state_name { #( #factor_names: None, )* }; - let mut configs = runtime_config.get_factor_configs().map_err(#Error::RuntimeConfigSource)?; #( app_state.#factor_names = Some( #Factor::configure_app( @@ -125,7 +124,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #factors_path::ConfigureAppContext::::new( &app, &app_state, - &mut configs, + &mut runtime_config, )?, ).map_err(#Error::factor_configure_app_error::<#factor_types>)? ); @@ -252,17 +251,15 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { )* } - /// TODO: figure out how to make this work - struct Foo(T); - - impl #factors_path::RuntimeConfigSource<#runtime_config_name> for Foo - where T: #(#factors_path::FactorRuntimeConfigSource<#factor_types> +)* #factors_path::RuntimeConfigSourceFinalizer - { - fn get_factor_configs(&mut self) -> anyhow::Result<#runtime_config_name> { + impl #runtime_config_name { + /// Get the runtime configuration from the given source. + pub fn from_source(mut source: T) -> anyhow::Result + where T: #(#factors_path::FactorRuntimeConfigSource<#factor_types> +)* #factors_path::RuntimeConfigSourceFinalizer + { #( - let #factor_names = >::get_runtime_config(&mut self.0)?; + let #factor_names = >::get_runtime_config(&mut source)?; )* - self.0.finalize()?; + source.finalize()?; Ok(#runtime_config_name { #( #factor_names, @@ -270,13 +267,14 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { }) } } - impl #factors_path::RuntimeConfigSource<#runtime_config_name> for () { - fn get_factor_configs(&mut self) -> anyhow::Result<#runtime_config_name> { - Ok(#runtime_config_name { + + impl From<()> for #runtime_config_name { + fn from(_: ()) -> Self { + #runtime_config_name { #( #factor_names: None, )* - }) + } } } diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 0f12b599d..13437b0d4 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -2,7 +2,7 @@ use spin_app::locked::LockedApp; use spin_factors::{ anyhow::{self, Context}, wasmtime::{component::Linker, Config, Engine}, - App, RuntimeConfigSource, RuntimeFactors, + App, RuntimeFactors, }; use spin_loader::FilesMountStrategy; @@ -47,15 +47,16 @@ impl TestEnvironment { /// Starting from a new _uninitialized_ [`RuntimeFactors`], run through the /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`] /// for the last component defined in the manifest. - pub async fn build_instance_state< - 'a, - T: RuntimeFactors, - S: RuntimeConfigSource + 'a, - >( + pub async fn build_instance_state<'a, T, C, E>( &'a self, mut factors: T, - runtime_config: S, - ) -> anyhow::Result { + runtime_config: C, + ) -> anyhow::Result + where + T: RuntimeFactors, + C: TryInto, + E: Into, + { let mut linker = Self::new_linker::(); factors.init(&mut linker)?; @@ -64,7 +65,8 @@ impl TestEnvironment { .await .context("failed to build locked app")?; let app = App::new("test-app", locked_app); - let configured_app = factors.configure_app(app, runtime_config)?; + let configured_app = + factors.configure_app(app, runtime_config.try_into().map_err(|e| e.into())?)?; let component = configured_app.app().components().last().context( diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 5af5a525c..3d20b8072 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -13,9 +13,7 @@ pub use spin_factors_derive::RuntimeFactors; pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, FactorInstanceState, InitContext}, prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext, SelfInstanceBuilder}, - runtime_config::{ - FactorRuntimeConfigSource, RuntimeConfigSource, RuntimeConfigSourceFinalizer, - }, + runtime_config::{FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer}, runtime_factors::{RuntimeFactors, RuntimeFactorsInstanceState}, }; diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index bda6a753c..e63a4db50 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -1,19 +1,5 @@ use crate::Factor; -/// The source of runtime configuration for a [`RuntimeFactors`][crate::RuntimeFactors]. -pub trait RuntimeConfigSource { - /// Get the runtime configuration for all the factors. - /// - /// This will be called once per call to [`RuntimeFactors::configure_app`][crate::RuntimeFactors::configure_app]. - fn get_factor_configs(&mut self) -> anyhow::Result; -} - -impl RuntimeConfigSource<()> for () { - fn get_factor_configs(&mut self) -> anyhow::Result<()> { - Ok(()) - } -} - /// The source of runtime configuration for a particular [`Factor`]. pub trait FactorRuntimeConfigSource { /// Get the runtime configuration for the factor. diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index ad0203918..14d29960f 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -1,6 +1,6 @@ use wasmtime::component::{Linker, ResourceTable}; -use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor, RuntimeConfigSource}; +use crate::{factor::FactorInstanceState, App, ConfiguredApp, Factor}; /// A collection of `Factor`s that are initialized and configured together. /// @@ -53,7 +53,7 @@ pub trait RuntimeFactors: Sized + 'static { fn configure_app( &self, app: App, - runtime_config: impl RuntimeConfigSource, + runtime_config: Self::RuntimeConfig, ) -> crate::Result>; /// Prepare the factors' instance state builders. From fdb8f66a142e0908deaf0c924aeffae33fd32af6 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 26 Jul 2024 16:39:48 +0200 Subject: [PATCH 088/195] Move generic toml handling to factors crate Signed-off-by: Ryan Levick --- .../factor-sqlite/src/runtime_config/spin.rs | 11 ++-- crates/factor-sqlite/tests/factor.rs | 49 ++---------------- crates/factors/Cargo.toml | 3 +- crates/factors/src/lib.rs | 2 +- crates/factors/src/runtime_config.rs | 2 + crates/factors/src/runtime_config/toml.rs | 50 +++++++++++++++++++ 6 files changed, 63 insertions(+), 54 deletions(-) create mode 100644 crates/factors/src/runtime_config/toml.rs diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 515aeb788..9dc6a23e0 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -6,7 +6,10 @@ use std::{ }; use serde::Deserialize; -use spin_factors::anyhow::{self, Context as _}; +use spin_factors::{ + anyhow::{self, Context as _}, + runtime_config::toml::GetTomlValue, +}; use spin_world::v2::sqlite as v2; use tokio::sync::OnceCell; @@ -55,7 +58,7 @@ impl SpinSqliteRuntimeConfig { /// type = "$database-type" /// ... extra type specific configuration ... /// ``` - pub fn config_from_table>( + pub fn config_from_table>( &self, table: &T, ) -> anyhow::Result> { @@ -88,10 +91,6 @@ impl SpinSqliteRuntimeConfig { } } -pub trait GetKey { - fn get(&self, key: &str) -> Option<&toml::Value>; -} - #[derive(Deserialize)] pub struct RuntimeConfig { #[serde(rename = "type")] diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor.rs index a33adcfc9..690dff009 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor.rs @@ -1,11 +1,9 @@ -use std::{cell::RefCell, collections::HashSet, sync::Arc}; +use std::{collections::HashSet, sync::Arc}; -use factor_sqlite::{ - runtime_config::spin::{GetKey, SpinSqliteRuntimeConfig}, - SqliteFactor, -}; +use factor_sqlite::{runtime_config::spin::SpinSqliteRuntimeConfig, SqliteFactor}; use spin_factors::{ anyhow::{self, bail}, + runtime_config::toml::TomlKeyTracker, Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, }; use spin_factors_test::{toml, TestEnvironment}; @@ -123,47 +121,6 @@ impl TryFrom> for TestFactorsRuntimeConfig { } } -struct TomlKeyTracker<'a> { - unused_keys: RefCell>, - table: &'a toml::Table, -} - -impl<'a> TomlKeyTracker<'a> { - fn new(table: &'a toml::Table) -> Self { - Self { - unused_keys: RefCell::new(table.keys().map(String::as_str).collect()), - table, - } - } - - fn validate_all_keys_used(&self) -> anyhow::Result<(), spin_factors::Error> { - if !self.unused_keys.borrow().is_empty() { - return Err(spin_factors::Error::RuntimeConfigUnusedKeys { - keys: self - .unused_keys - .borrow() - .iter() - .map(|s| (*s).to_owned()) - .collect(), - }); - } - Ok(()) - } -} - -impl GetKey for TomlKeyTracker<'_> { - fn get(&self, key: &str) -> Option<&toml::Value> { - self.unused_keys.borrow_mut().remove(key); - self.table.get(key) - } -} - -impl AsRef for TomlKeyTracker<'_> { - fn as_ref(&self) -> &toml::Table { - self.table - } -} - /// Will return an `InvalidConnectionPool` for the supplied default database. struct DefaultLabelResolver { default: Option, diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 3efbb2e02..71b97694e 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -10,6 +10,8 @@ serde = "1.0" spin-app = { path = "../app" } spin-factors-derive = { path = "../factors-derive" } thiserror = "1.0" +# TODO: make this optional and behind a feature flag +toml = "0.8" tracing = { workspace = true } wasmtime = { workspace = true } @@ -28,7 +30,6 @@ spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } spin-loader = { path = "../loader" } tokio = { version = "1", features = ["macros", "rt", "sync"] } -toml = "0.8" wasmtime-wasi-http = { workspace = true } [build-dependencies] diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index 3d20b8072..ce0666825 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -1,6 +1,6 @@ mod factor; mod prepare; -mod runtime_config; +pub mod runtime_config; mod runtime_factors; pub use anyhow; diff --git a/crates/factors/src/runtime_config.rs b/crates/factors/src/runtime_config.rs index e63a4db50..a11fc4583 100644 --- a/crates/factors/src/runtime_config.rs +++ b/crates/factors/src/runtime_config.rs @@ -1,3 +1,5 @@ +pub mod toml; + use crate::Factor; /// The source of runtime configuration for a particular [`Factor`]. diff --git a/crates/factors/src/runtime_config/toml.rs b/crates/factors/src/runtime_config/toml.rs new file mode 100644 index 000000000..15e801a47 --- /dev/null +++ b/crates/factors/src/runtime_config/toml.rs @@ -0,0 +1,50 @@ +//! Helpers for reading runtime configuration from a TOML file. + +use std::{cell::RefCell, collections::HashSet}; + +/// A trait for getting a TOML value by key. +pub trait GetTomlValue { + fn get(&self, key: &str) -> Option<&toml::Value>; +} + +/// A helper for tracking which keys have been used in a TOML table. +pub struct TomlKeyTracker<'a> { + unused_keys: RefCell>, + table: &'a toml::Table, +} + +impl<'a> TomlKeyTracker<'a> { + pub fn new(table: &'a toml::Table) -> Self { + Self { + unused_keys: RefCell::new(table.keys().map(String::as_str).collect()), + table, + } + } + + pub fn validate_all_keys_used(&self) -> crate::Result<()> { + if !self.unused_keys.borrow().is_empty() { + return Err(crate::Error::RuntimeConfigUnusedKeys { + keys: self + .unused_keys + .borrow() + .iter() + .map(|s| (*s).to_owned()) + .collect(), + }); + } + Ok(()) + } +} + +impl GetTomlValue for TomlKeyTracker<'_> { + fn get(&self, key: &str) -> Option<&toml::Value> { + self.unused_keys.borrow_mut().remove(key); + self.table.get(key) + } +} + +impl AsRef for TomlKeyTracker<'_> { + fn as_ref(&self) -> &toml::Table { + self.table + } +} From dada2fb7996c07ea95fc6f73b4e48792786da892 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 26 Jul 2024 17:08:56 +0200 Subject: [PATCH 089/195] Change factor-variables over Signed-off-by: Ryan Levick --- Cargo.lock | 1 + crates/factor-key-value/src/runtime_config.rs | 6 +-- crates/factor-key-value/tests/test.rs | 5 +- crates/factor-variables/src/lib.rs | 8 +-- crates/factor-variables/tests/factor_test.rs | 52 +++++++++++++++---- crates/factors-test/Cargo.toml | 1 + 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a1bfd987..b13a54569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7763,6 +7763,7 @@ dependencies = [ name = "spin-factors-test" version = "2.7.0-pre0" dependencies = [ + "serde 1.0.197", "spin-app", "spin-factors", "spin-factors-derive", diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index 74fddeb54..02f411abd 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use serde::{de::DeserializeOwned, Deserialize}; -use spin_factors::{anyhow, FactorRuntimeConfig}; +use spin_factors::anyhow; use spin_key_value::StoreManager; /// Runtime configuration for all key value stores. @@ -12,10 +12,6 @@ pub struct RuntimeConfig { pub store_configs: HashMap, } -impl FactorRuntimeConfig for RuntimeConfig { - const KEY: &'static str = "key_value_store"; -} - /// Resolves some piece of runtime configuration to a key value store manager. pub trait RuntimeConfigResolver: Send + Sync { /// The type of configuration that this resolver can handle. diff --git a/crates/factor-key-value/tests/test.rs b/crates/factor-key-value/tests/test.rs index fd2e74b48..73c5ca33f 100644 --- a/crates/factor-key-value/tests/test.rs +++ b/crates/factor-key-value/tests/test.rs @@ -70,10 +70,7 @@ async fn run_test_with_config_and_stores_for_label( source = "does-not-exist.wasm" key_value_stores = labels_clone }); - if let Some(runtime_config) = runtime_config { - env.runtime_config.extend(runtime_config); - } - let state = env.build_instance_state(factors).await?; + let state = env.build_instance_state(factors, runtime_config).await?; assert_eq!( labels, state.key_value.allowed_stores().iter().collect::>() diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index ffa7b0787..5568ec360 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use serde::{de::DeserializeOwned, Deserialize}; use spin_expressions::ProviderResolver as ExpressionResolver; use spin_factors::{ - anyhow, ConfigureAppContext, Factor, FactorRuntimeConfig, InitContext, InstanceBuilders, - PrepareContext, RuntimeFactors, SelfInstanceBuilder, + anyhow, ConfigureAppContext, Factor, InitContext, InstanceBuilders, PrepareContext, + RuntimeFactors, SelfInstanceBuilder, }; use spin_world::{async_trait, v1, v2::variables}; @@ -104,10 +104,6 @@ pub struct RuntimeConfig { provider_configs: Vec, } -impl FactorRuntimeConfig for RuntimeConfig { - const KEY: &'static str = "variable_provider"; -} - pub struct AppState { expression_resolver: Arc, } diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index 80bf480d1..7557f1882 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -2,7 +2,9 @@ use spin_factor_variables::spin_cli::{ EnvVariables, StaticVariables, VariableProviderConfiguration, }; use spin_factor_variables::VariablesFactor; -use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors::{ + anyhow, Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, +}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::v2::variables::Host; @@ -12,24 +14,23 @@ struct TestFactors { } fn test_env() -> TestEnvironment { - let mut env = TestEnvironment::default_manifest_extend(toml! { + TestEnvironment::default_manifest_extend(toml! { [variables] foo = { required = true } [component.test-component] source = "does-not-exist.wasm" variables = { baz = "<{{ foo }}>" } - }); - env.runtime_config = toml! { - [[variable_provider]] - type = "static" - values = { foo = "bar" } - }; - env + }) } #[tokio::test] async fn static_provider_works() -> anyhow::Result<()> { + let runtime_config = toml! { + [[variable_provider]] + type = "static" + values = { foo = "bar" } + }; let mut factors = TestFactors { variables: VariablesFactor::default(), }; @@ -38,8 +39,39 @@ async fn static_provider_works() -> anyhow::Result<()> { factors.variables.add_provider_resolver(EnvVariables)?; let env = test_env(); - let mut state = env.build_instance_state(factors).await?; + let mut state = env + .build_instance_state(factors, TomlConfig(runtime_config)) + .await?; let val = state.variables.get("baz".try_into().unwrap()).await?; assert_eq!(val, ""); Ok(()) } + +struct TomlConfig(toml::Table); + +impl TryFrom for TestFactorsRuntimeConfig { + type Error = anyhow::Error; + + fn try_from(value: TomlConfig) -> Result { + Self::from_source(value) + } +} + +impl FactorRuntimeConfigSource> for TomlConfig { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result< + Option< as Factor>::RuntimeConfig>, + > { + let Some(table) = self.0.get("variable_provider") else { + return Ok(None); + }; + Ok(Some(table.clone().try_into()?)) + } +} + +impl RuntimeConfigSourceFinalizer for TomlConfig { + fn finalize(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/crates/factors-test/Cargo.toml b/crates/factors-test/Cargo.toml index 23e606bde..1516b851a 100644 --- a/crates/factors-test/Cargo.toml +++ b/crates/factors-test/Cargo.toml @@ -5,6 +5,7 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] +serde = "1.0" spin-app = { path = "../app" } spin-factors = { path = "../factors" } spin-factors-derive = { path = "../factors-derive", features = ["expander"] } From 2e72b683967918f54dccdf21782f8397838f809b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 26 Jul 2024 17:19:29 +0200 Subject: [PATCH 090/195] Move key-value-store Signed-off-by: Ryan Levick --- crates/factor-key-value/tests/test.rs | 48 +++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/crates/factor-key-value/tests/test.rs b/crates/factor-key-value/tests/test.rs index 73c5ca33f..aed6bb998 100644 --- a/crates/factor-key-value/tests/test.rs +++ b/crates/factor-key-value/tests/test.rs @@ -4,7 +4,9 @@ use spin_factor_key_value::{ }; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; -use spin_factors::RuntimeFactors; +use spin_factors::{ + Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, +}; use spin_factors_test::{toml, TestEnvironment}; use std::collections::HashSet; @@ -41,7 +43,7 @@ async fn default_key_value_works() -> anyhow::Result<()> { source = "does-not-exist.wasm" key_value_stores = ["default"] }); - let state = env.build_instance_state(factors).await?; + let state = env.build_instance_state(factors, ()).await?; assert_eq!( state.key_value.allowed_stores(), @@ -65,12 +67,14 @@ async fn run_test_with_config_and_stores_for_label( key_value: KeyValueFactor::new(test_resolver), }; let labels_clone = labels.clone(); - let mut env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::default_manifest_extend(toml! { [component.test-component] source = "does-not-exist.wasm" key_value_stores = labels_clone }); - let state = env.build_instance_state(factors, runtime_config).await?; + let state = env + .build_instance_state(factors, TomlConfig(runtime_config)) + .await?; assert_eq!( labels, state.key_value.allowed_stores().iter().collect::>() @@ -203,13 +207,14 @@ async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { let factors = TestFactors { key_value: KeyValueFactor::new(test_resolver), }; - let mut env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::default_manifest_extend(toml! { [component.test-component] source = "does-not-exist.wasm" key_value_stores = ["custom"] }); - env.runtime_config.extend(runtime_config); - let state = env.build_instance_state(factors).await?; + let state = env + .build_instance_state(factors, TomlConfig(Some(runtime_config))) + .await?; assert_eq!( state.key_value.allowed_stores(), @@ -219,3 +224,32 @@ async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { assert!(tmp_dir.path().exists()); Ok(()) } + +struct TomlConfig(Option); + +impl TryFrom for TestFactorsRuntimeConfig { + type Error = anyhow::Error; + + fn try_from(value: TomlConfig) -> Result { + Self::from_source(value) + } +} + +impl FactorRuntimeConfigSource> for TomlConfig { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result< + Option< as Factor>::RuntimeConfig>, + > { + let Some(table) = self.0.as_ref().and_then(|t| t.get("key_value_store")) else { + return Ok(None); + }; + Ok(Some(table.clone().try_into()?)) + } +} + +impl RuntimeConfigSourceFinalizer for TomlConfig { + fn finalize(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} From 4a3daec6a95df9cc47b9f98e7fb54f0b61278e1b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 26 Jul 2024 17:26:25 +0200 Subject: [PATCH 091/195] The rest of the factor tests Signed-off-by: Ryan Levick --- crates/factor-llm/tests/factor.rs | 2 +- crates/factor-outbound-http/tests/factor_test.rs | 2 +- .../factor-outbound-networking/tests/factor_test.rs | 13 ++++++++----- crates/factor-outbound-pg/tests/factor_test.rs | 10 +++++----- crates/factor-outbound-redis/tests/factor_test.rs | 2 +- crates/factor-wasi/tests/factor_test.rs | 2 +- crates/factors/tests/smoke.rs | 2 +- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/crates/factor-llm/tests/factor.rs b/crates/factor-llm/tests/factor.rs index 5337ff25f..a05b68aa3 100644 --- a/crates/factor-llm/tests/factor.rs +++ b/crates/factor-llm/tests/factor.rs @@ -48,7 +48,7 @@ async fn llm_works() -> anyhow::Result<()> { source = "does-not-exist.wasm" ai_models = ["llama2-chat"] }); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; assert_eq!( &*state.llm.allowed_models, &["llama2-chat".to_owned()] diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index d1761eda1..d5479ee1c 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -37,7 +37,7 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { http: OutboundHttpFactor, }; let env = test_env(); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); let req = Request::get("https://denied.test").body(Default::default())?; diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs index 75e62a542..950fd9a3a 100644 --- a/crates/factor-outbound-networking/tests/factor_test.rs +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -29,7 +29,7 @@ async fn configures_wasi_socket_addr_check() -> anyhow::Result<()> { }; let env = test_env(); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let mut wasi = WasiFactor::get_wasi_impl(&mut state).unwrap(); let network_resource = wasi.instance_network()?; @@ -62,10 +62,13 @@ async fn wasi_factor_is_optional() -> anyhow::Result<()> { networking: OutboundNetworkingFactor, } TestEnvironment::default() - .build_instance_state(WithoutWasi { - variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, - }) + .build_instance_state( + WithoutWasi { + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + }, + (), + ) .await?; Ok(()) } diff --git a/crates/factor-outbound-pg/tests/factor_test.rs b/crates/factor-outbound-pg/tests/factor_test.rs index 87c42ac1c..89d89f924 100644 --- a/crates/factor-outbound-pg/tests/factor_test.rs +++ b/crates/factor-outbound-pg/tests/factor_test.rs @@ -24,7 +24,7 @@ fn factors() -> Result { networking: OutboundNetworkingFactor, pg: OutboundPgFactor::::new(), }; - f.variables.add_provider_type(StaticVariables)?; + f.variables.add_provider_resolver(StaticVariables)?; Ok(f) } @@ -43,7 +43,7 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { [component.test-component] source = "does-not-exist.wasm" }); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let res = state .pg @@ -61,7 +61,7 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { async fn allowed_host_succeeds() -> anyhow::Result<()> { let factors = factors()?; let env = test_env(); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let res = state .pg @@ -78,7 +78,7 @@ async fn allowed_host_succeeds() -> anyhow::Result<()> { async fn exercise_execute() -> anyhow::Result<()> { let factors = factors()?; let env = test_env(); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let connection = state .pg @@ -97,7 +97,7 @@ async fn exercise_execute() -> anyhow::Result<()> { async fn exercise_query() -> anyhow::Result<()> { let factors = factors()?; let env = test_env(); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let connection = state .pg diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs index 18bcfd261..89b8f64da 100644 --- a/crates/factor-outbound-redis/tests/factor_test.rs +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -41,7 +41,7 @@ async fn no_outbound_hosts_fails() -> anyhow::Result<()> { }, ..Default::default() }; - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let connection = state .redis .open("redis://redis.test:8080".to_string()) diff --git a/crates/factor-wasi/tests/factor_test.rs b/crates/factor-wasi/tests/factor_test.rs index 3f7084336..d6019393f 100644 --- a/crates/factor-wasi/tests/factor_test.rs +++ b/crates/factor-wasi/tests/factor_test.rs @@ -22,7 +22,7 @@ async fn environment_works() -> anyhow::Result<()> { wasi: WasiFactor::new(DummyFilesMounter), }; let env = test_env(); - let mut state = env.build_instance_state(factors).await?; + let mut state = env.build_instance_state(factors, ()).await?; let mut wasi = WasiFactor::get_wasi_impl(&mut state).unwrap(); let val = wasi .get_environment()? diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 091dd3326..f6206995c 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -13,7 +13,7 @@ use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::{spin_cli::StaticVariables, VariablesFactor}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; -use spin_factors::{FactorRuntimeConfig, RuntimeConfigSource, RuntimeFactors}; +use spin_factors::RuntimeFactors; use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] From ecd85e04bb02e4d2c136eb27204ce7ac2574f23e Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 26 Jul 2024 17:26:52 +0200 Subject: [PATCH 092/195] Smoke test Signed-off-by: Ryan Levick --- crates/factors/src/runtime_factors.rs | 2 +- crates/factors/tests/smoke.rs | 92 +++++++++++++++++++++------ 2 files changed, 72 insertions(+), 22 deletions(-) diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 14d29960f..124005d32 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -37,7 +37,7 @@ pub trait RuntimeFactors: Sized + 'static { type InstanceState: RuntimeFactorsInstanceState; /// The collection of all the `InstanceBuilder`s of the factors. type InstanceBuilders; - /// TODO + /// The runtime configuration of all the factors. type RuntimeConfig; /// Initialize the factors with the given linker. diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index f6206995c..205775ca1 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -11,15 +11,20 @@ use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::{spin_cli::StaticVariables, VariablesFactor}; +use spin_factor_variables::{ + spin_cli::{StaticVariables, VariableProviderConfiguration}, + VariablesFactor, +}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; -use spin_factors::RuntimeFactors; +use spin_factors::{ + Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, +}; use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] struct Factors { wasi: WasiFactor, - variables: VariablesFactor, + variables: VariablesFactor, outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, key_value: KeyValueFactor, @@ -76,7 +81,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { factors.init(&mut linker)?; - let configured_app = factors.configure_app(app, TestSource)?; + let configured_app = factors.configure_app(app, TestSource.try_into()?)?; let builders = factors.prepare(&configured_app, "smoke-app")?; let state = factors.build_instance_state(builders)?; @@ -140,29 +145,74 @@ async fn smoke_test_works() -> anyhow::Result<()> { struct TestSource; -impl RuntimeConfigSource for TestSource { - fn factor_config_keys(&self) -> impl IntoIterator { - [spin_factor_variables::RuntimeConfig::<()>::KEY] +impl TryFrom for FactorsRuntimeConfig { + type Error = anyhow::Error; + + fn try_from(value: TestSource) -> Result { + Self::from_source(value) + } +} + +impl FactorRuntimeConfigSource> for TestSource { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result< + Option< as Factor>::RuntimeConfig>, + > { + let config = toml::toml! { + [other] + type = "redis" + url = "redis://localhost:6379" + }; + + Ok(Some(config.try_into()?)) } +} - fn get_factor_config( - &self, - key: &str, - ) -> anyhow::Result> { - let Some(table) = toml::toml! { +impl FactorRuntimeConfigSource> for TestSource { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result< + Option< as Factor>::RuntimeConfig>, + > { + let config = toml::toml! { [[variable_provider]] type = "static" [variable_provider.values] foo = "bar" - - [key_value_store.other] - type = "redis" - url = "redis://localhost:6379" } - .remove(key) else { - return Ok(None); - }; - let config = table.try_into()?; - Ok(Some(config)) + .remove("variable_provider") + .unwrap(); + Ok(Some(config.try_into()?)) + } +} + +impl FactorRuntimeConfigSource for TestSource { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result::RuntimeConfig>> { + Ok(None) + } +} + +impl FactorRuntimeConfigSource for TestSource { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result::RuntimeConfig>> { + Ok(None) + } +} + +impl FactorRuntimeConfigSource for TestSource { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result::RuntimeConfig>> { + Ok(None) + } +} + +impl RuntimeConfigSourceFinalizer for TestSource { + fn finalize(&mut self) -> anyhow::Result<()> { + Ok(()) } } From b7af6d9e1fbcb73e7c8221cb6253622525cec55f Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 26 Jul 2024 13:42:52 -0400 Subject: [PATCH 093/195] factors: Update AzureCosmosKeyValueRuntimeConfig Signed-off-by: Lann Martin --- crates/factor-key-value-azure/src/lib.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/factor-key-value-azure/src/lib.rs b/crates/factor-key-value-azure/src/lib.rs index 1e13cac35..949bfb186 100644 --- a/crates/factor-key-value-azure/src/lib.rs +++ b/crates/factor-key-value-azure/src/lib.rs @@ -1,6 +1,8 @@ use serde::Deserialize; use spin_factor_key_value::MakeKeyValueStore; -use spin_key_value_azure::KeyValueAzureCosmos; +use spin_key_value_azure::{ + KeyValueAzureCosmos, KeyValueAzureCosmosAuthOptions, KeyValueAzureCosmosRuntimeConfigOptions, +}; /// A key-value store that uses Azure Cosmos as the backend. pub struct AzureKeyValueStore; @@ -9,7 +11,7 @@ pub struct AzureKeyValueStore; #[derive(Deserialize)] pub struct AzureCosmosKeyValueRuntimeConfig { /// The authorization token for the Azure Cosmos DB account. - key: String, + key: Option, /// The Azure Cosmos DB account name. account: String, /// The Azure Cosmos DB database. @@ -30,11 +32,17 @@ impl MakeKeyValueStore for AzureKeyValueStore { &self, runtime_config: Self::RuntimeConfig, ) -> anyhow::Result { + let auth_options = match runtime_config.key { + Some(key) => KeyValueAzureCosmosAuthOptions::RuntimeConfigValues( + KeyValueAzureCosmosRuntimeConfigOptions::new(key), + ), + None => KeyValueAzureCosmosAuthOptions::Environmental, + }; KeyValueAzureCosmos::new( - runtime_config.key, runtime_config.account, runtime_config.database, runtime_config.container, + auth_options, ) } } From a10730cb0f61fb7331c62168459c5dc43d0724bd Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 26 Jul 2024 14:02:24 -0400 Subject: [PATCH 094/195] factors: Update spin-factors-test's TestEnvironment Signed-off-by: Lann Martin --- Cargo.lock | 2 - crates/core/tests/integration_test.rs | 2 +- .../tests/{test.rs => factor_test.rs} | 53 +++++++------- .../tests/{factor.rs => factor_test.rs} | 6 +- crates/factor-outbound-http/Cargo.toml | 1 - .../factor-outbound-http/tests/factor_test.rs | 19 ++--- .../tests/factor_test.rs | 32 +++----- .../factor-outbound-pg/tests/factor_test.rs | 31 +++----- crates/factor-outbound-redis/Cargo.toml | 1 - .../tests/factor_test.rs | 39 ++++------ .../tests/{factor.rs => factor_test.rs} | 33 ++++----- crates/factor-variables/src/spin_cli/env.rs | 4 +- crates/factor-variables/tests/factor_test.rs | 36 ++++----- crates/factor-wasi/tests/factor_test.rs | 17 ++--- crates/factors-derive/src/lib.rs | 25 +------ crates/factors-test/src/lib.rs | 73 ++++++++++--------- crates/factors/src/factor.rs | 10 +-- crates/factors/src/runtime_factors.rs | 2 +- 18 files changed, 160 insertions(+), 226 deletions(-) rename crates/factor-key-value/tests/{test.rs => factor_test.rs} (87%) rename crates/factor-llm/tests/{factor.rs => factor_test.rs} (96%) rename crates/factor-sqlite/tests/{factor.rs => factor_test.rs} (84%) diff --git a/Cargo.lock b/Cargo.lock index 12c87ed71..d9769241d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7648,7 +7648,6 @@ dependencies = [ "http 1.1.0", "spin-factor-outbound-networking", "spin-factor-variables", - "spin-factor-wasi", "spin-factors", "spin-factors-test", "spin-world", @@ -7702,7 +7701,6 @@ dependencies = [ "spin-core", "spin-factor-outbound-networking", "spin-factor-variables", - "spin-factor-wasi", "spin-factors", "spin-factors-test", "spin-world", diff --git a/crates/core/tests/integration_test.rs b/crates/core/tests/integration_test.rs index 97dd5e644..da328b2c5 100644 --- a/crates/core/tests/integration_test.rs +++ b/crates/core/tests/integration_test.rs @@ -138,7 +138,7 @@ async fn run_test( }] }))?; let app = App::new("test-app", locked); - let configured_app = factors.configure_app(app, ())?; + let configured_app = factors.configure_app(app, Default::default())?; let mut builders = factors.prepare(&configured_app, "test-component")?; // FIXME: it is unfortunate that we have to unwrap here... builders.wasi.as_mut().unwrap().args(args); diff --git a/crates/factor-key-value/tests/test.rs b/crates/factor-key-value/tests/factor_test.rs similarity index 87% rename from crates/factor-key-value/tests/test.rs rename to crates/factor-key-value/tests/factor_test.rs index aed6bb998..c0c24eac0 100644 --- a/crates/factor-key-value/tests/test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -38,12 +38,12 @@ async fn default_key_value_works() -> anyhow::Result<()> { let factors = TestFactors { key_value: KeyValueFactor::new(test_resolver), }; - let env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::new(factors).extend_manifest(toml! { [component.test-component] source = "does-not-exist.wasm" key_value_stores = ["default"] }); - let state = env.build_instance_state(factors, ()).await?; + let state = env.build_instance_state().await?; assert_eq!( state.key_value.allowed_stores(), @@ -67,14 +67,14 @@ async fn run_test_with_config_and_stores_for_label( key_value: KeyValueFactor::new(test_resolver), }; let labels_clone = labels.clone(); - let env = TestEnvironment::default_manifest_extend(toml! { - [component.test-component] - source = "does-not-exist.wasm" - key_value_stores = labels_clone - }); - let state = env - .build_instance_state(factors, TomlConfig(runtime_config)) - .await?; + let env = TestEnvironment::new(factors) + .extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + key_value_stores = labels_clone + }) + .runtime_config(TomlConfig(runtime_config))?; + let state = env.build_instance_state().await?; assert_eq!( labels, state.key_value.allowed_stores().iter().collect::>() @@ -192,29 +192,28 @@ async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { #[tokio::test] async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { let tmp_dir = tempdir::TempDir::new("example")?; - let runtime_config = toml::toml! { - [key_value_store.custom] - type = "spin" - path = "custom.db" - - [key_value_store.custom] - type = "redis" - url = "redis://localhost:6379" - }; let mut test_resolver = DelegatingRuntimeConfigResolver::new(); test_resolver.add_store_type(RedisKeyValueStore)?; test_resolver.add_store_type(SpinKeyValueStore::new(tmp_dir.path().to_owned()))?; let factors = TestFactors { key_value: KeyValueFactor::new(test_resolver), }; - let env = TestEnvironment::default_manifest_extend(toml! { - [component.test-component] - source = "does-not-exist.wasm" - key_value_stores = ["custom"] - }); - let state = env - .build_instance_state(factors, TomlConfig(Some(runtime_config))) - .await?; + let env = TestEnvironment::new(factors) + .extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + key_value_stores = ["custom"] + }) + .runtime_config(TomlConfig(Some(toml::toml! { + [key_value_store.custom] + type = "spin" + path = "custom.db" + + [key_value_store.custom] + type = "redis" + url = "redis://localhost:6379" + })))?; + let state = env.build_instance_state().await?; assert_eq!( state.key_value.allowed_stores(), diff --git a/crates/factor-llm/tests/factor.rs b/crates/factor-llm/tests/factor_test.rs similarity index 96% rename from crates/factor-llm/tests/factor.rs rename to crates/factor-llm/tests/factor_test.rs index a05b68aa3..4504238b5 100644 --- a/crates/factor-llm/tests/factor.rs +++ b/crates/factor-llm/tests/factor_test.rs @@ -42,13 +42,13 @@ async fn llm_works() -> anyhow::Result<()> { }) as _ }), }; - - let env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::new(factors).extend_manifest(toml! { [component.test-component] source = "does-not-exist.wasm" ai_models = ["llama2-chat"] }); - let mut state = env.build_instance_state(factors, ()).await?; + let mut state = env.build_instance_state().await?; + assert_eq!( &*state.llm.allowed_models, &["llama2-chat".to_owned()] diff --git a/crates/factor-outbound-http/Cargo.toml b/crates/factor-outbound-http/Cargo.toml index 7cf2044d8..b9c5943e4 100644 --- a/crates/factor-outbound-http/Cargo.toml +++ b/crates/factor-outbound-http/Cargo.toml @@ -8,7 +8,6 @@ edition = { workspace = true } anyhow = "1.0" http = "1.1.0" spin-factor-outbound-networking = { path = "../factor-outbound-networking" } -spin-factor-wasi = { path = "../factor-wasi" } spin-factors = { path = "../factors" } spin-world = { path = "../world" } tracing = { workspace = true } diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index d5479ee1c..90394f3a0 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -5,7 +5,6 @@ use http::Request; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::spin_cli::VariablesFactor; -use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use wasmtime_wasi_http::{ @@ -14,30 +13,24 @@ use wasmtime_wasi_http::{ #[derive(RuntimeFactors)] struct TestFactors { - wasi: WasiFactor, variables: VariablesFactor, networking: OutboundNetworkingFactor, http: OutboundHttpFactor, } -fn test_env() -> TestEnvironment { - TestEnvironment::default_manifest_extend(toml! { - [component.test-component] - source = "does-not-exist.wasm" - allowed_outbound_hosts = ["http://allowed.test"] - }) -} - #[tokio::test] async fn disallowed_host_fails() -> anyhow::Result<()> { let factors = TestFactors { - wasi: WasiFactor::new(DummyFilesMounter), variables: VariablesFactor::default(), networking: OutboundNetworkingFactor, http: OutboundHttpFactor, }; - let env = test_env(); - let mut state = env.build_instance_state(factors, ()).await?; + let env = TestEnvironment::new(factors).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + allowed_outbound_hosts = ["http://allowed.test"] + }); + let mut state = env.build_instance_state().await?; let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); let req = Request::get("https://denied.test").body(Default::default())?; diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs index 950fd9a3a..8d85af094 100644 --- a/crates/factor-outbound-networking/tests/factor_test.rs +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -12,14 +12,6 @@ struct TestFactors { networking: OutboundNetworkingFactor, } -fn test_env() -> TestEnvironment { - TestEnvironment::default_manifest_extend(toml! { - [component.test-component] - source = "does-not-exist.wasm" - allowed_outbound_hosts = ["*://192.0.2.1:12345"] - }) -} - #[tokio::test] async fn configures_wasi_socket_addr_check() -> anyhow::Result<()> { let factors = TestFactors { @@ -27,9 +19,12 @@ async fn configures_wasi_socket_addr_check() -> anyhow::Result<()> { variables: VariablesFactor::default(), networking: OutboundNetworkingFactor, }; - - let env = test_env(); - let mut state = env.build_instance_state(factors, ()).await?; + let env = TestEnvironment::new(factors).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + allowed_outbound_hosts = ["*://192.0.2.1:12345"] + }); + let mut state = env.build_instance_state().await?; let mut wasi = WasiFactor::get_wasi_impl(&mut state).unwrap(); let network_resource = wasi.instance_network()?; @@ -61,14 +56,11 @@ async fn wasi_factor_is_optional() -> anyhow::Result<()> { variables: VariablesFactor, networking: OutboundNetworkingFactor, } - TestEnvironment::default() - .build_instance_state( - WithoutWasi { - variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, - }, - (), - ) - .await?; + TestEnvironment::new(WithoutWasi { + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + }) + .build_instance_state() + .await?; Ok(()) } diff --git a/crates/factor-outbound-pg/tests/factor_test.rs b/crates/factor-outbound-pg/tests/factor_test.rs index 89d89f924..77fa257bb 100644 --- a/crates/factor-outbound-pg/tests/factor_test.rs +++ b/crates/factor-outbound-pg/tests/factor_test.rs @@ -2,7 +2,7 @@ use anyhow::{bail, Result}; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_pg::client::Client; use spin_factor_outbound_pg::OutboundPgFactor; -use spin_factor_variables::spin_cli::{StaticVariables, VariablesFactor}; +use spin_factor_variables::spin_cli::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::async_trait; @@ -18,18 +18,16 @@ struct TestFactors { pg: OutboundPgFactor, } -fn factors() -> Result { - let mut f = TestFactors { +fn factors() -> TestFactors { + TestFactors { variables: VariablesFactor::default(), networking: OutboundNetworkingFactor, pg: OutboundPgFactor::::new(), - }; - f.variables.add_provider_resolver(StaticVariables)?; - Ok(f) + } } -fn test_env() -> TestEnvironment { - TestEnvironment::default_manifest_extend(toml! { +fn test_env() -> TestEnvironment { + TestEnvironment::new(factors()).extend_manifest(toml! { [component.test-component] source = "does-not-exist.wasm" allowed_outbound_hosts = ["postgres://*:*"] @@ -38,12 +36,11 @@ fn test_env() -> TestEnvironment { #[tokio::test] async fn disallowed_host_fails() -> anyhow::Result<()> { - let factors = factors()?; - let env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::new(factors()).extend_manifest(toml! { [component.test-component] source = "does-not-exist.wasm" }); - let mut state = env.build_instance_state(factors, ()).await?; + let mut state = env.build_instance_state().await?; let res = state .pg @@ -59,9 +56,7 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { #[tokio::test] async fn allowed_host_succeeds() -> anyhow::Result<()> { - let factors = factors()?; - let env = test_env(); - let mut state = env.build_instance_state(factors, ()).await?; + let mut state = test_env().build_instance_state().await?; let res = state .pg @@ -76,9 +71,7 @@ async fn allowed_host_succeeds() -> anyhow::Result<()> { #[tokio::test] async fn exercise_execute() -> anyhow::Result<()> { - let factors = factors()?; - let env = test_env(); - let mut state = env.build_instance_state(factors, ()).await?; + let mut state = test_env().build_instance_state().await?; let connection = state .pg @@ -95,9 +88,7 @@ async fn exercise_execute() -> anyhow::Result<()> { #[tokio::test] async fn exercise_query() -> anyhow::Result<()> { - let factors = factors()?; - let env = test_env(); - let mut state = env.build_instance_state(factors, ()).await?; + let mut state = test_env().build_instance_state().await?; let connection = state .pg diff --git a/crates/factor-outbound-redis/Cargo.toml b/crates/factor-outbound-redis/Cargo.toml index 330e070a4..0dffe6ea4 100644 --- a/crates/factor-outbound-redis/Cargo.toml +++ b/crates/factor-outbound-redis/Cargo.toml @@ -19,7 +19,6 @@ redis = { version = "0.21", features = ["tokio-comp", "tokio-native-tls-comp", " spin-factors-test = { path = "../factors-test" } tokio = { version = "1", features = ["macros", "rt"] } spin-factor-variables = { path = "../factor-variables" } -spin-factor-wasi = { path = "../factor-wasi" } # wasmtime-wasi-http = { workspace = true } [lints] diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs index 89b8f64da..dc6578a33 100644 --- a/crates/factor-outbound-redis/tests/factor_test.rs +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -1,47 +1,34 @@ use anyhow::bail; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_redis::OutboundRedisFactor; -use spin_factor_variables::spin_cli::{StaticVariables, VariablesFactor}; -use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; +use spin_factor_variables::spin_cli::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::v2::redis::{Error, HostConnection}; #[derive(RuntimeFactors)] struct TestFactors { - wasi: WasiFactor, variables: VariablesFactor, networking: OutboundNetworkingFactor, redis: OutboundRedisFactor, } -fn get_test_factors() -> TestFactors { - TestFactors { - wasi: WasiFactor::new(DummyFilesMounter), +#[tokio::test] +async fn no_outbound_hosts_fails() -> anyhow::Result<()> { + let factors = TestFactors { variables: VariablesFactor::default(), networking: OutboundNetworkingFactor, redis: OutboundRedisFactor, - } -} - -#[tokio::test] -async fn no_outbound_hosts_fails() -> anyhow::Result<()> { - let mut factors = get_test_factors(); - - factors.variables.add_provider_resolver(StaticVariables)?; - - let env = TestEnvironment { - manifest: toml! { - spin_manifest_version = 2 - application.name = "test-app" - [[trigger.test]] - - [component.test-component] - source = "does-not-exist.wasm" - }, - ..Default::default() }; - let mut state = env.build_instance_state(factors, ()).await?; + let env = TestEnvironment::new(factors).extend_manifest(toml! { + spin_manifest_version = 2 + application.name = "test-app" + [[trigger.test]] + + [component.test-component] + source = "does-not-exist.wasm" + }); + let mut state = env.build_instance_state().await?; let connection = state .redis .open("redis://redis.test:8080".to_string()) diff --git a/crates/factor-sqlite/tests/factor.rs b/crates/factor-sqlite/tests/factor_test.rs similarity index 84% rename from crates/factor-sqlite/tests/factor.rs rename to crates/factor-sqlite/tests/factor_test.rs index 690dff009..4699dab78 100644 --- a/crates/factor-sqlite/tests/factor.rs +++ b/crates/factor-sqlite/tests/factor_test.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, sync::Arc}; use factor_sqlite::{runtime_config::spin::SpinSqliteRuntimeConfig, SqliteFactor}; use spin_factors::{ - anyhow::{self, bail}, + anyhow::{self, bail, Context}, runtime_config::toml::TomlKeyTracker, Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, }; @@ -19,12 +19,12 @@ async fn sqlite_works() -> anyhow::Result<()> { let factors = TestFactors { sqlite: SqliteFactor::new(test_resolver), }; - let env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::new(factors).extend_manifest(toml! { [component.test-component] source = "does-not-exist.wasm" sqlite_databases = ["default"] }); - let state = env.build_instance_state(factors, ()).await?; + let state = env.build_instance_state().await?; assert_eq!( state.sqlite.allowed_databases(), @@ -40,12 +40,12 @@ async fn errors_when_non_configured_database_used() -> anyhow::Result<()> { let factors = TestFactors { sqlite: SqliteFactor::new(test_resolver), }; - let env = TestEnvironment::default_manifest_extend(toml! { + let env = TestEnvironment::new(factors).extend_manifest(toml! { [component.test-component] source = "does-not-exist.wasm" sqlite_databases = ["foo"] }); - let Err(err) = env.build_instance_state(factors, ()).await else { + let Err(err) = env.build_instance_state().await else { bail!("Expected build_instance_state to error but it did not"); }; @@ -62,26 +62,21 @@ async fn no_error_when_database_is_configured() -> anyhow::Result<()> { let factors = TestFactors { sqlite: SqliteFactor::new(test_resolver), }; - let env = TestEnvironment::default_manifest_extend(toml! { - [component.test-component] - source = "does-not-exist.wasm" - sqlite_databases = ["foo"] - }); let runtime_config = toml! { [sqlite_database.foo] type = "spin" }; let sqlite_config = SpinSqliteRuntimeConfig::new("/".into(), "/".into()); - if let Err(e) = env - .build_instance_state( - factors, - TomlRuntimeSource::new(&runtime_config, sqlite_config), - ) + let env = TestEnvironment::new(factors) + .extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + sqlite_databases = ["foo"] + }) + .runtime_config(TomlRuntimeSource::new(&runtime_config, sqlite_config))?; + env.build_instance_state() .await - { - bail!("Expected build_instance_state to succeed but it errored: {e}"); - } - + .context("build_instance_state failed")?; Ok(()) } diff --git a/crates/factor-variables/src/spin_cli/env.rs b/crates/factor-variables/src/spin_cli/env.rs index 8db82e1d6..779f54d85 100644 --- a/crates/factor-variables/src/spin_cli/env.rs +++ b/crates/factor-variables/src/spin_cli/env.rs @@ -52,10 +52,12 @@ pub struct EnvVariablesConfig { const DEFAULT_ENV_PREFIX: &str = "SPIN_VARIABLE"; +type EnvFetcherFn = Box Result + Send + Sync>; + /// A config Provider that uses environment variables. pub struct EnvVariablesProvider { prefix: Option, - env_fetcher: Box Result + Send + Sync>, + env_fetcher: EnvFetcherFn, dotenv_path: Option, dotenv_cache: OnceLock>, } diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index 7557f1882..cc2735574 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -13,24 +13,8 @@ struct TestFactors { variables: VariablesFactor, } -fn test_env() -> TestEnvironment { - TestEnvironment::default_manifest_extend(toml! { - [variables] - foo = { required = true } - - [component.test-component] - source = "does-not-exist.wasm" - variables = { baz = "<{{ foo }}>" } - }) -} - #[tokio::test] async fn static_provider_works() -> anyhow::Result<()> { - let runtime_config = toml! { - [[variable_provider]] - type = "static" - values = { foo = "bar" } - }; let mut factors = TestFactors { variables: VariablesFactor::default(), }; @@ -38,10 +22,22 @@ async fn static_provider_works() -> anyhow::Result<()> { // The env provider will be ignored since there's no configuration for it. factors.variables.add_provider_resolver(EnvVariables)?; - let env = test_env(); - let mut state = env - .build_instance_state(factors, TomlConfig(runtime_config)) - .await?; + let env = TestEnvironment::new(factors) + .extend_manifest(toml! { + [variables] + foo = { required = true } + + [component.test-component] + source = "does-not-exist.wasm" + variables = { baz = "<{{ foo }}>" } + }) + .runtime_config(TomlConfig(toml! { + [[variable_provider]] + type = "static" + values = { foo = "bar" } + }))?; + + let mut state = env.build_instance_state().await?; let val = state.variables.get("baz".try_into().unwrap()).await?; assert_eq!(val, ""); Ok(()) diff --git a/crates/factor-wasi/tests/factor_test.rs b/crates/factor-wasi/tests/factor_test.rs index d6019393f..e5f1c7f29 100644 --- a/crates/factor-wasi/tests/factor_test.rs +++ b/crates/factor-wasi/tests/factor_test.rs @@ -8,22 +8,19 @@ struct TestFactors { wasi: WasiFactor, } -fn test_env() -> TestEnvironment { - TestEnvironment::default_manifest_extend(toml! { - [component.test-component] - source = "does-not-exist.wasm" - environment = { FOO = "bar" } - }) -} - #[tokio::test] async fn environment_works() -> anyhow::Result<()> { let factors = TestFactors { wasi: WasiFactor::new(DummyFilesMounter), }; - let env = test_env(); - let mut state = env.build_instance_state(factors, ()).await?; + let env = TestEnvironment::new(factors).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + environment = { FOO = "bar" } + }); + let mut state = env.build_instance_state().await?; let mut wasi = WasiFactor::get_wasi_impl(&mut state).unwrap(); + let val = wasi .get_environment()? .into_iter() diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 5b2d38568..80b750d46 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -112,7 +112,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { fn configure_app( &self, app: #factors_path::App, - mut runtime_config: Self::RuntimeConfig, + runtime_config: Self::RuntimeConfig, ) -> #Result<#ConfiguredApp> { let mut app_state = #app_state_name { #( #factor_names: None, )* @@ -124,7 +124,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #factors_path::ConfigureAppContext::::new( &app, &app_state, - &mut runtime_config, + runtime_config.#factor_names, )?, ).map_err(#Error::factor_configure_app_error::<#factor_types>)? ); @@ -245,6 +245,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } } + #[derive(Default)] #vis struct #runtime_config_name { #( pub #factor_names: Option<<#factor_types as #Factor>::RuntimeConfig>, @@ -253,6 +254,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { impl #runtime_config_name { /// Get the runtime configuration from the given source. + #[allow(dead_code)] pub fn from_source(mut source: T) -> anyhow::Result where T: #(#factors_path::FactorRuntimeConfigSource<#factor_types> +)* #factors_path::RuntimeConfigSourceFinalizer { @@ -267,24 +269,5 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { }) } } - - impl From<()> for #runtime_config_name { - fn from(_: ()) -> Self { - #runtime_config_name { - #( - #factor_names: None, - )* - } - } - } - - #( - impl #factors_path::FactorRuntimeConfigSource<#factor_types> for #runtime_config_name { - fn get_runtime_config(&mut self) -> anyhow::Result::RuntimeConfig>> { - Ok(self.#factor_names.take()) - } - } - )* - }) } diff --git a/crates/factors-test/src/lib.rs b/crates/factors-test/src/lib.rs index 13437b0d4..344153fbe 100644 --- a/crates/factors-test/src/lib.rs +++ b/crates/factors-test/src/lib.rs @@ -9,13 +9,26 @@ use spin_loader::FilesMountStrategy; pub use toml::toml; /// A test environment for building [`RuntimeFactors`] instances. -pub struct TestEnvironment { +pub struct TestEnvironment { + /// The RuntimeFactors under test. + pub factors: T, /// The `spin.toml` manifest. pub manifest: toml::Table, + /// Runtime configuration for the factors. + pub runtime_config: T::RuntimeConfig, } -impl Default for TestEnvironment { - fn default() -> Self { +impl TestEnvironment { + /// Creates a new test environment by initializing the given + /// [`RuntimeFactors`]. + pub fn new(mut factors: T) -> Self { + let engine = Engine::new(Config::new().async_support(true)) + .expect("wasmtime engine failed to initialize"); + let mut linker = Linker::::new(&engine); + factors + .init(&mut linker) + .expect("RuntimeFactors::init failed"); + let manifest = toml! { spin_manifest_version = 2 @@ -27,60 +40,54 @@ impl Default for TestEnvironment { [component.empty] source = "does-not-exist.wasm" }; - Self { manifest } + Self { + factors, + manifest, + runtime_config: Default::default(), + } } -} -impl TestEnvironment { - /// Builds a TestEnvironment by extending a default manifest with the given - /// manifest TOML. + /// Extends the manifest with the given TOML. /// /// The default manifest includes boilerplate like the /// `spin_manifest_version` and `[application]` section, so you typically /// need to pass only a `[component.test-component]` section. - pub fn default_manifest_extend(manifest_merge: toml::Table) -> Self { - let mut env = Self::default(); - env.manifest.extend(manifest_merge); - env + pub fn extend_manifest(mut self, manifest_merge: toml::Table) -> Self { + self.manifest.extend(manifest_merge); + self } - /// Starting from a new _uninitialized_ [`RuntimeFactors`], run through the - /// [`Factor`]s' lifecycle(s) to build a [`RuntimeFactors::InstanceState`] - /// for the last component defined in the manifest. - pub async fn build_instance_state<'a, T, C, E>( - &'a self, - mut factors: T, - runtime_config: C, - ) -> anyhow::Result + /// Sets the runtime config. + pub fn runtime_config(mut self, runtime_config: C) -> anyhow::Result where - T: RuntimeFactors, C: TryInto, E: Into, { - let mut linker = Self::new_linker::(); - factors.init(&mut linker)?; + self.runtime_config = runtime_config + .try_into() + .map_err(Into::into) + .context("failed to build runtime config")?; + Ok(self) + } + /// Run through the [`Factor`]s' lifecycle(s) to build a + /// [`RuntimeFactors::InstanceState`] for the last component defined in the + /// manifest. + pub async fn build_instance_state(self) -> anyhow::Result { let locked_app = self .build_locked_app() .await .context("failed to build locked app")?; let app = App::new("test-app", locked_app); - let configured_app = - factors.configure_app(app, runtime_config.try_into().map_err(|e| e.into())?)?; + let configured_app = self.factors.configure_app(app, self.runtime_config)?; let component = configured_app.app().components().last().context( "expected configured app to have at least one component, but it did not", )?; - let builders = factors.prepare(&configured_app, component.id())?; + let builders = self.factors.prepare(&configured_app, component.id())?; - Ok(factors.build_instance_state(builders)?) - } - - pub fn new_linker() -> Linker { - let engine = Engine::new(Config::new().async_support(true)) - .expect("wasmtime engine failed to initialize"); - Linker::::new(&engine) + Ok(self.factors.build_instance_state(builders)?) } pub async fn build_locked_app(&self) -> anyhow::Result { diff --git a/crates/factors/src/factor.rs b/crates/factors/src/factor.rs index 39be75e59..06e8a996d 100644 --- a/crates/factors/src/factor.rs +++ b/crates/factors/src/factor.rs @@ -3,8 +3,7 @@ use std::any::Any; use wasmtime::component::{Linker, ResourceTable}; use crate::{ - prepare::FactorInstanceBuilder, runtime_config::FactorRuntimeConfigSource, App, Error, - InstanceBuilders, PrepareContext, RuntimeFactors, + prepare::FactorInstanceBuilder, App, Error, InstanceBuilders, PrepareContext, RuntimeFactors, }; /// A contained (i.e., "factored") piece of runtime functionality. @@ -137,14 +136,11 @@ pub struct ConfigureAppContext<'a, T: RuntimeFactors, F: Factor> { impl<'a, T: RuntimeFactors, F: Factor> ConfigureAppContext<'a, T, F> { #[doc(hidden)] - pub fn new>( + pub fn new( app: &'a App, app_state: &'a T::AppState, - runtime_config: &mut S, + runtime_config: Option, ) -> crate::Result { - let runtime_config = runtime_config - .get_runtime_config() - .map_err(Error::factor_configure_app_error::)?; Ok(Self { app, app_state, diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index 124005d32..a0ff6dd68 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -38,7 +38,7 @@ pub trait RuntimeFactors: Sized + 'static { /// The collection of all the `InstanceBuilder`s of the factors. type InstanceBuilders; /// The runtime configuration of all the factors. - type RuntimeConfig; + type RuntimeConfig: Default; /// Initialize the factors with the given linker. /// From d3619b3474541994c8c58c7d17f55c5f0ece4a6b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 29 Jul 2024 13:09:42 +0200 Subject: [PATCH 095/195] Make KeyValue factor non-generic Signed-off-by: Ryan Levick --- .../src/delegating_resolver.rs | 88 +++++++++++++------ crates/factor-key-value/src/lib.rs | 58 ++++++------ crates/factor-key-value/src/runtime_config.rs | 29 +++--- crates/factor-key-value/src/store.rs | 2 +- crates/factor-key-value/tests/factor_test.rs | 80 +++++++++-------- crates/factors/tests/smoke.rs | 31 +++---- crates/key-value/Cargo.toml | 2 +- 7 files changed, 161 insertions(+), 129 deletions(-) diff --git a/crates/factor-key-value/src/delegating_resolver.rs b/crates/factor-key-value/src/delegating_resolver.rs index 89ecc2e1d..b4120a76b 100644 --- a/crates/factor-key-value/src/delegating_resolver.rs +++ b/crates/factor-key-value/src/delegating_resolver.rs @@ -1,16 +1,20 @@ -use crate::runtime_config::RuntimeConfigResolver; -use crate::store::{store_from_toml_fn, MakeKeyValueStore, StoreFromToml}; +use crate::{ + store::{store_from_toml_fn, MakeKeyValueStore, StoreFromToml}, + DefaultLabelResolver, RuntimeConfig, +}; +use anyhow::Context; use serde::{Deserialize, Serialize}; use spin_key_value::StoreManager; use std::{collections::HashMap, sync::Arc}; -/// A RuntimeConfigResolver that delegates to the appropriate key-value store -/// for a given label. +/// Converts from toml based runtime configuration into a [`RuntimeConfig`]. /// -/// The store types are registered with the resolver using `add_store_type`. The -/// default store for a label is registered using `add_default_store`. +/// Also acts as [`DefaultLabelResolver`]. +/// +/// The various store types (i.e., the "type" field in the toml field) are registered with the +/// resolver using `add_store_type`. The default store for a label is registered using `add_default_store`. #[derive(Default)] -pub struct DelegatingRuntimeConfigResolver { +pub struct RuntimeConfigResolver { /// A map of store types to a function that returns the appropriate store /// manager from runtime config TOML. store_types: HashMap<&'static str, StoreFromToml>, @@ -18,18 +22,25 @@ pub struct DelegatingRuntimeConfigResolver { defaults: HashMap<&'static str, StoreConfig>, } -impl DelegatingRuntimeConfigResolver { - /// Create a new DelegatingRuntimeConfigResolver. +impl RuntimeConfigResolver { + /// Create a new RuntimeConfigResolver. pub fn new() -> Self { - Self::default() + ::default() } + /// Adds a default store configuration for a label. + /// + /// Users must ensure that the store type for `config` has been registered with + /// the resolver using [`Self::register_store_type`]. pub fn add_default_store(&mut self, label: &'static str, config: StoreConfig) { self.defaults.insert(label, config); } - /// Adds a store type to the resolver. - pub fn add_store_type(&mut self, store_type: T) -> anyhow::Result<()> { + /// Registers a store type to the resolver. + pub fn register_store_type( + &mut self, + store_type: T, + ) -> anyhow::Result<()> { if self .store_types .insert(T::RUNTIME_CONFIG_TYPE, store_from_toml_fn(store_type)) @@ -42,26 +53,49 @@ impl DelegatingRuntimeConfigResolver { } Ok(()) } -} -impl RuntimeConfigResolver for DelegatingRuntimeConfigResolver { - type Config = StoreConfig; - - fn get_store(&self, config: StoreConfig) -> anyhow::Result> { - let store_kind = config.type_.as_str(); - let store_from_toml = self - .store_types - .get(store_kind) - .ok_or_else(|| anyhow::anyhow!("unknown store kind: {}", store_kind))?; - store_from_toml(config.config) + /// Resolves a toml table into a runtime config. + pub fn resolve_from_toml( + &self, + table: &Option, + ) -> anyhow::Result> { + let Some(table) = table.as_ref().and_then(|t| t.get("key_value_store")) else { + return Ok(None); + }; + let mut store_configs = HashMap::new(); + for (label, config) in table + .as_table() + .context("expected a 'key_value_store' to contain toml table")? + { + let config: StoreConfig = config.clone().try_into()?; + let store_manager = self.store_manager_from_config(config)?; + store_configs.insert(label.clone(), store_manager); + } + Ok(Some(RuntimeConfig { + store_managers: store_configs, + })) } - /// Get the default store manager for the given label. + /// Given a [`StoreConfig`], returns a store manager. /// - /// Returns None if no default store is registered for the label. - fn default_store(&self, label: &str) -> Option> { + /// Errors if there is no [`MakeKeyValueStore`] registered for the store config's type + /// or if the store manager cannot be created from the config. + fn store_manager_from_config( + &self, + config: StoreConfig, + ) -> anyhow::Result> { + let config_type = config.type_.as_str(); + let maker = self.store_types.get(config_type).with_context(|| { + format!("the store type '{config_type}' was not registered with the config resolver") + })?; + maker(config.config) + } +} + +impl DefaultLabelResolver for RuntimeConfigResolver { + fn default(&self, label: &str) -> Option> { let config = self.defaults.get(label)?; - self.get_store(config.clone()).ok() + Some(self.store_manager_from_config(config.clone()).unwrap()) } } diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index f83b417ab..ea27fa0ce 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -1,7 +1,7 @@ pub mod delegating_resolver; mod runtime_config; mod store; -pub use delegating_resolver::{DelegatingRuntimeConfigResolver, StoreConfig}; +pub use delegating_resolver::{RuntimeConfigResolver, StoreConfig}; use std::{ collections::{HashMap, HashSet}, @@ -9,7 +9,6 @@ use std::{ }; use anyhow::ensure; -use runtime_config::{RuntimeConfig, RuntimeConfigResolver}; use spin_factors::{ ConfigureAppContext, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, @@ -18,27 +17,29 @@ use spin_key_value::{ CachingStoreManager, DefaultManagerGetter, DelegatingStoreManager, KeyValueDispatch, StoreManager, KEY_VALUE_STORES_KEY, }; + +pub use runtime_config::RuntimeConfig; pub use store::MakeKeyValueStore; /// A factor that provides key-value storage. -pub struct KeyValueFactor { - /// Resolves runtime configuration into store managers. - runtime_config_resolver: Arc, +pub struct KeyValueFactor { + default_label_resolver: Arc, } -impl KeyValueFactor { +impl KeyValueFactor { /// Create a new KeyValueFactor. /// - /// The `runtime_config_resolver` is used to resolve runtime configuration into store managers. - pub fn new(runtime_config_resolver: R) -> Self { + /// The `default_label_resolver` is used to resolver store managers for labels that + /// are not defined in the runtime configuration. + pub fn new(default_label_resolver: impl DefaultLabelResolver + 'static) -> Self { Self { - runtime_config_resolver: Arc::new(runtime_config_resolver), + default_label_resolver: Arc::new(default_label_resolver), } } } -impl Factor for KeyValueFactor { - type RuntimeConfig = RuntimeConfig; +impl Factor for KeyValueFactor { + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceBuilder; @@ -52,23 +53,10 @@ impl Factor for KeyValueFactor { &self, mut ctx: ConfigureAppContext, ) -> anyhow::Result { - // Build store manager from runtime config - let mut store_managers: HashMap> = HashMap::new(); - if let Some(runtime_config) = ctx.take_runtime_config() { - for (store_label, config) in runtime_config.store_configs { - if let std::collections::hash_map::Entry::Vacant(e) = - store_managers.entry(store_label) - { - // Only add manager for labels that are not already configured. Runtime config - // takes top-down precedence. - let store = self.runtime_config_resolver.get_store(config)?; - e.insert(store); - } - } - } - let resolver_clone = self.runtime_config_resolver.clone(); + let store_managers = ctx.take_runtime_config().unwrap_or_default(); + let default_label_resolver = self.default_label_resolver.clone(); let default_fn: DefaultManagerGetter = - Arc::new(move |label| resolver_clone.default_store(label)); + Arc::new(move |label| default_label_resolver.default(label)); let delegating_manager = DelegatingStoreManager::new(store_managers, default_fn); let caching_manager = CachingStoreManager::new(delegating_manager); @@ -87,7 +75,7 @@ impl Factor for KeyValueFactor { // TODO: port nicer errors from KeyValueComponent (via error type?) ensure!( store_manager_manager.is_defined(label) - || self.runtime_config_resolver.default_store(label).is_some(), + || self.default_label_resolver.default(label).is_some(), "unknown key_value_stores label {label:?} for component {component_id:?}" ); } @@ -159,3 +147,17 @@ impl FactorInstanceBuilder for InstanceBuilder { Ok(dispatch) } } + +/// Resolves a label to a default [`StoreManager`]. +pub trait DefaultLabelResolver: Send + Sync { + /// If there is no runtime configuration for a given store label, return a default store manager. + /// + /// If `Option::None` is returned, the store is not allowed. + fn default(&self, label: &str) -> Option>; +} + +impl DefaultLabelResolver for Arc { + fn default(&self, label: &str) -> Option> { + self.as_ref().default(label) + } +} diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index 02f411abd..6959ff858 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -1,28 +1,19 @@ use std::{collections::HashMap, sync::Arc}; -use serde::{de::DeserializeOwned, Deserialize}; -use spin_factors::anyhow; use spin_key_value::StoreManager; /// Runtime configuration for all key value stores. -#[derive(Deserialize)] -#[serde(transparent)] -pub struct RuntimeConfig { - /// Map of store names to store configurations. - pub store_configs: HashMap, +#[derive(Default)] +pub struct RuntimeConfig { + /// Map of store names to store managers. + pub store_managers: HashMap>, } -/// Resolves some piece of runtime configuration to a key value store manager. -pub trait RuntimeConfigResolver: Send + Sync { - /// The type of configuration that this resolver can handle. - type Config: DeserializeOwned; +impl IntoIterator for RuntimeConfig { + type Item = (String, Arc); + type IntoIter = std::collections::hash_map::IntoIter>; - /// Get a store manager for a given config. - fn get_store(&self, config: Self::Config) -> anyhow::Result>; - - /// Returns a default store manager for a given label. Should only be called - /// if there is no runtime configuration for the label. - /// - /// If `Option::None` is returned, the database is not allowed. - fn default_store(&self, label: &str) -> Option>; + fn into_iter(self) -> Self::IntoIter { + self.store_managers.into_iter() + } } diff --git a/crates/factor-key-value/src/store.rs b/crates/factor-key-value/src/store.rs index 24c5e95bb..4dcf4e3b0 100644 --- a/crates/factor-key-value/src/store.rs +++ b/crates/factor-key-value/src/store.rs @@ -4,7 +4,7 @@ use anyhow::Context; use serde::de::DeserializeOwned; use spin_key_value::StoreManager; -/// Defines the construction of a key value store. +/// Defines the construction of a key value store from a serializable runtime config. pub trait MakeKeyValueStore: 'static + Send + Sync { /// Unique type identifier for the store. const RUNTIME_CONFIG_TYPE: &'static str; diff --git a/crates/factor-key-value/tests/factor_test.rs b/crates/factor-key-value/tests/factor_test.rs index c0c24eac0..246ee0266 100644 --- a/crates/factor-key-value/tests/factor_test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -1,24 +1,21 @@ use anyhow::Context; use spin_factor_key_value::{ - DelegatingRuntimeConfigResolver, KeyValueFactor, MakeKeyValueStore, StoreConfig, + KeyValueFactor, MakeKeyValueStore, RuntimeConfig, RuntimeConfigResolver, StoreConfig, }; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; -use spin_factors::{ - Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, -}; +use spin_factors::{FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; -use std::collections::HashSet; +use std::{collections::HashSet, sync::Arc}; #[derive(RuntimeFactors)] struct TestFactors { - key_value: KeyValueFactor, + key_value: KeyValueFactor, } -fn default_key_value_resolver( -) -> anyhow::Result<(DelegatingRuntimeConfigResolver, tempdir::TempDir)> { - let mut test_resolver = DelegatingRuntimeConfigResolver::new(); - test_resolver.add_store_type(SpinKeyValueStore::new( +fn default_key_value_resolver() -> anyhow::Result<(RuntimeConfigResolver, tempdir::TempDir)> { + let mut test_resolver = RuntimeConfigResolver::new(); + test_resolver.register_store_type(SpinKeyValueStore::new( std::env::current_dir().context("failed to get current directory")?, ))?; let tmp_dir = tempdir::TempDir::new("example")?; @@ -59,12 +56,13 @@ async fn run_test_with_config_and_stores_for_label( store_types: Vec, labels: Vec<&str>, ) -> anyhow::Result<()> { - let mut test_resolver = DelegatingRuntimeConfigResolver::new(); + let mut test_resolver = RuntimeConfigResolver::new(); for store_type in store_types { - test_resolver.add_store_type(store_type)?; + test_resolver.register_store_type(store_type)?; } + let test_resolver = Arc::new(test_resolver); let factors = TestFactors { - key_value: KeyValueFactor::new(test_resolver), + key_value: KeyValueFactor::new(test_resolver.clone()), }; let labels_clone = labels.clone(); let env = TestEnvironment::new(factors) @@ -73,7 +71,7 @@ async fn run_test_with_config_and_stores_for_label( source = "does-not-exist.wasm" key_value_stores = labels_clone }) - .runtime_config(TomlConfig(runtime_config))?; + .runtime_config(TomlConfig::new(test_resolver, runtime_config))?; let state = env.build_instance_state().await?; assert_eq!( labels, @@ -192,11 +190,12 @@ async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { #[tokio::test] async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { let tmp_dir = tempdir::TempDir::new("example")?; - let mut test_resolver = DelegatingRuntimeConfigResolver::new(); - test_resolver.add_store_type(RedisKeyValueStore)?; - test_resolver.add_store_type(SpinKeyValueStore::new(tmp_dir.path().to_owned()))?; + let mut test_resolver = RuntimeConfigResolver::new(); + test_resolver.register_store_type(RedisKeyValueStore)?; + test_resolver.register_store_type(SpinKeyValueStore::new(tmp_dir.path().to_owned()))?; + let test_resolver = Arc::new(test_resolver); let factors = TestFactors { - key_value: KeyValueFactor::new(test_resolver), + key_value: KeyValueFactor::new(test_resolver.clone()), }; let env = TestEnvironment::new(factors) .extend_manifest(toml! { @@ -204,15 +203,18 @@ async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { source = "does-not-exist.wasm" key_value_stores = ["custom"] }) - .runtime_config(TomlConfig(Some(toml::toml! { - [key_value_store.custom] - type = "spin" - path = "custom.db" - - [key_value_store.custom] - type = "redis" - url = "redis://localhost:6379" - })))?; + .runtime_config(TomlConfig::new( + test_resolver, + Some(toml::toml! { + [key_value_store.custom] + type = "spin" + path = "custom.db" + + [key_value_store.custom] + type = "redis" + url = "redis://localhost:6379" + }), + ))?; let state = env.build_instance_state().await?; assert_eq!( @@ -224,7 +226,16 @@ async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { Ok(()) } -struct TomlConfig(Option); +struct TomlConfig { + resolver: Arc, + toml: Option, +} + +impl TomlConfig { + fn new(resolver: Arc, toml: Option) -> Self { + Self { resolver, toml } + } +} impl TryFrom for TestFactorsRuntimeConfig { type Error = anyhow::Error; @@ -234,16 +245,9 @@ impl TryFrom for TestFactorsRuntimeConfig { } } -impl FactorRuntimeConfigSource> for TomlConfig { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result< - Option< as Factor>::RuntimeConfig>, - > { - let Some(table) = self.0.as_ref().and_then(|t| t.get("key_value_store")) else { - return Ok(None); - }; - Ok(Some(table.clone().try_into()?)) +impl FactorRuntimeConfigSource for TomlConfig { + fn get_runtime_config(&mut self) -> anyhow::Result> { + self.resolver.resolve_from_toml(&self.toml) } } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 205775ca1..94105b729 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,10 +1,10 @@ -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use anyhow::Context; use http_body_util::BodyExt; use spin_app::App; use spin_factor_key_value::{ - delegating_resolver::{DelegatingRuntimeConfigResolver, StoreConfig}, + delegating_resolver::{RuntimeConfigResolver, StoreConfig}, KeyValueFactor, MakeKeyValueStore, }; use spin_factor_key_value_redis::RedisKeyValueStore; @@ -27,7 +27,7 @@ struct Factors { variables: VariablesFactor, outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, - key_value: KeyValueFactor, + key_value: KeyValueFactor, } struct Data { @@ -43,7 +43,7 @@ impl AsMut for Data { #[tokio::test(flavor = "multi_thread")] async fn smoke_test_works() -> anyhow::Result<()> { - let mut key_value_resolver = DelegatingRuntimeConfigResolver::default(); + let mut key_value_resolver = RuntimeConfigResolver::default(); let default_config = SpinKeyValueRuntimeConfig::default(Some(PathBuf::from("tests/smoke-app/.spin"))); key_value_resolver.add_default_store( @@ -53,17 +53,18 @@ async fn smoke_test_works() -> anyhow::Result<()> { config: toml::value::Table::try_from(default_config)?, }, ); - key_value_resolver.add_store_type(SpinKeyValueStore::new( + key_value_resolver.register_store_type(SpinKeyValueStore::new( std::env::current_dir().context("failed to get current directory")?, ))?; - key_value_resolver.add_store_type(RedisKeyValueStore)?; + key_value_resolver.register_store_type(RedisKeyValueStore)?; + let key_value_resolver = Arc::new(key_value_resolver); let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), variables: VariablesFactor::default(), outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, - key_value: KeyValueFactor::new(key_value_resolver), + key_value: KeyValueFactor::new(key_value_resolver.clone()), }; factors.variables.add_provider_resolver(StaticVariables)?; @@ -81,7 +82,8 @@ async fn smoke_test_works() -> anyhow::Result<()> { factors.init(&mut linker)?; - let configured_app = factors.configure_app(app, TestSource.try_into()?)?; + let source = TestSource { key_value_resolver }; + let configured_app = factors.configure_app(app, source.try_into()?)?; let builders = factors.prepare(&configured_app, "smoke-app")?; let state = factors.build_instance_state(builders)?; @@ -143,7 +145,9 @@ async fn smoke_test_works() -> anyhow::Result<()> { Ok(()) } -struct TestSource; +struct TestSource { + key_value_resolver: Arc, +} impl TryFrom for FactorsRuntimeConfig { type Error = anyhow::Error; @@ -153,19 +157,16 @@ impl TryFrom for FactorsRuntimeConfig { } } -impl FactorRuntimeConfigSource> for TestSource { +impl FactorRuntimeConfigSource for TestSource { fn get_runtime_config( &mut self, - ) -> anyhow::Result< - Option< as Factor>::RuntimeConfig>, - > { + ) -> anyhow::Result::RuntimeConfig>> { let config = toml::toml! { [other] type = "redis" url = "redis://localhost:6379" }; - - Ok(Some(config.try_into()?)) + self.key_value_resolver.resolve_from_toml(&Some(config)) } } diff --git a/crates/key-value/Cargo.toml b/crates/key-value/Cargo.toml index fa8a44073..e5b53aec5 100644 --- a/crates/key-value/Cargo.toml +++ b/crates/key-value/Cargo.toml @@ -9,7 +9,7 @@ doctest = false [dependencies] anyhow = "1.0" -tokio = { version = "1", features = ["macros", "sync"] } +tokio = { version = "1", features = ["macros", "sync", "rt"] } spin-app = { path = "../app" } spin-core = { path = "../core" } spin-world = { path = "../world" } From c4d6f456d51419296a6cae05722d66639dfbd7f6 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 29 Jul 2024 13:52:43 +0200 Subject: [PATCH 096/195] Dedicated Spin CLI module Signed-off-by: Ryan Levick --- crates/factor-key-value-azure/src/lib.rs | 2 +- crates/factor-key-value-redis/src/lib.rs | 2 +- crates/factor-key-value-spin/src/lib.rs | 6 +-- crates/factor-key-value/src/lib.rs | 6 +-- crates/factor-key-value/src/runtime_config.rs | 2 + .../spin.rs} | 40 ++++++++++++++++--- crates/factor-key-value/src/store.rs | 35 ---------------- crates/factor-key-value/tests/factor_test.rs | 3 +- crates/factors/tests/smoke.rs | 4 +- 9 files changed, 47 insertions(+), 53 deletions(-) rename crates/factor-key-value/src/{delegating_resolver.rs => runtime_config/spin.rs} (73%) delete mode 100644 crates/factor-key-value/src/store.rs diff --git a/crates/factor-key-value-azure/src/lib.rs b/crates/factor-key-value-azure/src/lib.rs index 949bfb186..812a1df07 100644 --- a/crates/factor-key-value-azure/src/lib.rs +++ b/crates/factor-key-value-azure/src/lib.rs @@ -1,5 +1,5 @@ use serde::Deserialize; -use spin_factor_key_value::MakeKeyValueStore; +use spin_factor_key_value::runtime_config::spin::MakeKeyValueStore; use spin_key_value_azure::{ KeyValueAzureCosmos, KeyValueAzureCosmosAuthOptions, KeyValueAzureCosmosRuntimeConfigOptions, }; diff --git a/crates/factor-key-value-redis/src/lib.rs b/crates/factor-key-value-redis/src/lib.rs index 39fadd26f..6bea163e1 100644 --- a/crates/factor-key-value-redis/src/lib.rs +++ b/crates/factor-key-value-redis/src/lib.rs @@ -1,5 +1,5 @@ use serde::Deserialize; -use spin_factor_key_value::MakeKeyValueStore; +use spin_factor_key_value::runtime_config::spin::MakeKeyValueStore; use spin_key_value_redis::KeyValueRedis; /// A key-value store that uses Redis as the backend. diff --git a/crates/factor-key-value-spin/src/lib.rs b/crates/factor-key-value-spin/src/lib.rs index 7c107b4e7..38e9ef442 100644 --- a/crates/factor-key-value-spin/src/lib.rs +++ b/crates/factor-key-value-spin/src/lib.rs @@ -5,7 +5,7 @@ use std::{ use anyhow::Context; use serde::{Deserialize, Serialize}; -use spin_factor_key_value::MakeKeyValueStore; +use spin_factor_key_value::runtime_config::spin::MakeKeyValueStore; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; /// A key-value store that uses SQLite as the backend. @@ -32,10 +32,10 @@ impl SpinKeyValueRuntimeConfig { /// The default filename for the SQLite database. const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; - /// Create a new runtime configuration with the given state directory. + /// Create a new runtime configuration with the given directory. /// /// If the database directory is None, the database is in-memory. - /// If the database directory is Some, the database is stored in a file in the state directory. + /// If the database directory is Some, the database is stored in a file in the given directory. pub fn default(default_database_dir: Option) -> Self { let path = default_database_dir.map(|dir| dir.join(Self::DEFAULT_SPIN_STORE_FILENAME)); Self { path } diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index ea27fa0ce..54bec6242 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -1,7 +1,4 @@ -pub mod delegating_resolver; -mod runtime_config; -mod store; -pub use delegating_resolver::{RuntimeConfigResolver, StoreConfig}; +pub mod runtime_config; use std::{ collections::{HashMap, HashSet}, @@ -19,7 +16,6 @@ use spin_key_value::{ }; pub use runtime_config::RuntimeConfig; -pub use store::MakeKeyValueStore; /// A factor that provides key-value storage. pub struct KeyValueFactor { diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index 6959ff858..e2b3c9ff9 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -1,3 +1,5 @@ +pub mod spin; + use std::{collections::HashMap, sync::Arc}; use spin_key_value::StoreManager; diff --git a/crates/factor-key-value/src/delegating_resolver.rs b/crates/factor-key-value/src/runtime_config/spin.rs similarity index 73% rename from crates/factor-key-value/src/delegating_resolver.rs rename to crates/factor-key-value/src/runtime_config/spin.rs index b4120a76b..36b5266d8 100644 --- a/crates/factor-key-value/src/delegating_resolver.rs +++ b/crates/factor-key-value/src/runtime_config/spin.rs @@ -1,12 +1,42 @@ -use crate::{ - store::{store_from_toml_fn, MakeKeyValueStore, StoreFromToml}, - DefaultLabelResolver, RuntimeConfig, -}; -use anyhow::Context; +//! Runtime configuration implementation used by Spin CLI. + +use crate::{DefaultLabelResolver, RuntimeConfig}; +use anyhow::Context as _; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use spin_key_value::StoreManager; use std::{collections::HashMap, sync::Arc}; +/// Defines the construction of a key value store from a serialized runtime config. +pub trait MakeKeyValueStore: 'static + Send + Sync { + /// Unique type identifier for the store. + const RUNTIME_CONFIG_TYPE: &'static str; + /// Runtime configuration for the store. + type RuntimeConfig: DeserializeOwned; + /// The store manager for the store. + type StoreManager: StoreManager; + + /// Creates a new store manager from the runtime configuration. + fn make_store(&self, runtime_config: Self::RuntimeConfig) + -> anyhow::Result; +} + +/// A function that creates a store manager from a TOML table. +type StoreFromToml = + Box anyhow::Result> + Send + Sync>; + +/// Creates a `StoreFromToml` function from a `MakeKeyValueStore` implementation. +fn store_from_toml_fn(provider_type: T) -> StoreFromToml { + Box::new(move |table| { + let runtime_config: T::RuntimeConfig = + table.try_into().context("could not parse runtime config")?; + let provider = provider_type + .make_store(runtime_config) + .context("could not make store")?; + Ok(Arc::new(provider)) + }) +} + /// Converts from toml based runtime configuration into a [`RuntimeConfig`]. /// /// Also acts as [`DefaultLabelResolver`]. diff --git a/crates/factor-key-value/src/store.rs b/crates/factor-key-value/src/store.rs deleted file mode 100644 index 4dcf4e3b0..000000000 --- a/crates/factor-key-value/src/store.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::sync::Arc; - -use anyhow::Context; -use serde::de::DeserializeOwned; -use spin_key_value::StoreManager; - -/// Defines the construction of a key value store from a serializable runtime config. -pub trait MakeKeyValueStore: 'static + Send + Sync { - /// Unique type identifier for the store. - const RUNTIME_CONFIG_TYPE: &'static str; - /// Runtime configuration for the store. - type RuntimeConfig: DeserializeOwned; - /// The store manager for the store. - type StoreManager: StoreManager; - - /// Creates a new store manager from the runtime configuration. - fn make_store(&self, runtime_config: Self::RuntimeConfig) - -> anyhow::Result; -} - -/// A function that creates a store manager from a TOML table. -pub(crate) type StoreFromToml = - Box anyhow::Result> + Send + Sync>; - -/// Creates a `StoreFromToml` function from a `MakeKeyValueStore` implementation. -pub(crate) fn store_from_toml_fn(provider_type: T) -> StoreFromToml { - Box::new(move |table| { - let runtime_config: T::RuntimeConfig = - table.try_into().context("could not parse runtime config")?; - let provider = provider_type - .make_store(runtime_config) - .context("could not make store")?; - Ok(Arc::new(provider)) - }) -} diff --git a/crates/factor-key-value/tests/factor_test.rs b/crates/factor-key-value/tests/factor_test.rs index 246ee0266..978160601 100644 --- a/crates/factor-key-value/tests/factor_test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -1,6 +1,7 @@ use anyhow::Context; use spin_factor_key_value::{ - KeyValueFactor, MakeKeyValueStore, RuntimeConfig, RuntimeConfigResolver, StoreConfig, + runtime_config::spin::{MakeKeyValueStore, RuntimeConfigResolver, StoreConfig}, + KeyValueFactor, RuntimeConfig, }; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 94105b729..37ed6e10f 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -4,8 +4,8 @@ use anyhow::Context; use http_body_util::BodyExt; use spin_app::App; use spin_factor_key_value::{ - delegating_resolver::{RuntimeConfigResolver, StoreConfig}, - KeyValueFactor, MakeKeyValueStore, + runtime_config::spin::{MakeKeyValueStore, RuntimeConfigResolver, StoreConfig}, + KeyValueFactor, }; use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; From bf2e257a282940d6091b7de8885b8ca853875ef3 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 29 Jul 2024 14:45:51 +0200 Subject: [PATCH 097/195] Make the variables factor non-generic Signed-off-by: Ryan Levick --- .../factor-outbound-http/tests/factor_test.rs | 2 +- crates/factor-outbound-networking/src/lib.rs | 2 +- .../tests/factor_test.rs | 2 +- .../factor-outbound-pg/tests/factor_test.rs | 2 +- .../tests/factor_test.rs | 2 +- crates/factor-variables/src/lib.rs | 53 ++++--------------- crates/factor-variables/src/provider.rs | 17 ------ crates/factor-variables/src/runtime_config.rs | 16 ++++++ crates/factor-variables/src/spin_cli/env.rs | 27 +--------- crates/factor-variables/src/spin_cli/mod.rs | 37 ++++++++++--- .../factor-variables/src/spin_cli/statik.rs | 21 -------- crates/factor-variables/tests/factor_test.rs | 36 ++++++------- crates/factors/tests/smoke.rs | 22 +++----- 13 files changed, 84 insertions(+), 155 deletions(-) delete mode 100644 crates/factor-variables/src/provider.rs create mode 100644 crates/factor-variables/src/runtime_config.rs diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index 90394f3a0..9abee5f57 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -4,7 +4,7 @@ use anyhow::bail; use http::Request; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::spin_cli::VariablesFactor; +use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use wasmtime_wasi_http::{ diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index fbb973947..9d7f04b2d 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -60,7 +60,7 @@ impl Factor for OutboundNetworkingFactor { .cloned() .context("missing component allowed hosts")?; let resolver = builders - .get_mut::>()? + .get_mut::()? .expression_resolver() .clone(); let allowed_hosts_future = async move { diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs index 8d85af094..693ac6600 100644 --- a/crates/factor-outbound-networking/tests/factor_test.rs +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -1,5 +1,5 @@ use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::spin_cli::VariablesFactor; +use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; diff --git a/crates/factor-outbound-pg/tests/factor_test.rs b/crates/factor-outbound-pg/tests/factor_test.rs index 77fa257bb..e189b9d2a 100644 --- a/crates/factor-outbound-pg/tests/factor_test.rs +++ b/crates/factor-outbound-pg/tests/factor_test.rs @@ -2,7 +2,7 @@ use anyhow::{bail, Result}; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_pg::client::Client; use spin_factor_outbound_pg::OutboundPgFactor; -use spin_factor_variables::spin_cli::VariablesFactor; +use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::async_trait; diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs index dc6578a33..14c1ee24f 100644 --- a/crates/factor-outbound-redis/tests/factor_test.rs +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -1,7 +1,7 @@ use anyhow::bail; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_redis::OutboundRedisFactor; -use spin_factor_variables::spin_cli::VariablesFactor; +use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::v2::redis::{Error, HostConnection}; diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 5568ec360..554f89cbb 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -1,9 +1,9 @@ -pub mod provider; +pub mod runtime_config; pub mod spin_cli; use std::sync::Arc; -use serde::{de::DeserializeOwned, Deserialize}; +use runtime_config::RuntimeConfig; use spin_expressions::ProviderResolver as ExpressionResolver; use spin_factors::{ anyhow, ConfigureAppContext, Factor, InitContext, InstanceBuilders, PrepareContext, @@ -11,39 +11,19 @@ use spin_factors::{ }; use spin_world::{async_trait, v1, v2::variables}; -pub use provider::ProviderResolver; - /// A factor for providing variables to components. -/// -/// The factor is generic over the type of runtime configuration used to configure the providers. -pub struct VariablesFactor { - provider_resolvers: Vec>>, +pub struct VariablesFactor { + _priv: (), } -impl Default for VariablesFactor { +impl Default for VariablesFactor { fn default() -> Self { - Self { - provider_resolvers: Default::default(), - } - } -} - -impl VariablesFactor { - /// Adds a provider resolver to the factor. - /// - /// Each added provider will be called in order with the runtime configuration. This order - /// will be the order in which the providers are called to resolve variables. - pub fn add_provider_resolver>( - &mut self, - provider_type: T, - ) -> anyhow::Result<()> { - self.provider_resolvers.push(Box::new(provider_type)); - Ok(()) + Self { _priv: () } } } -impl Factor for VariablesFactor { - type RuntimeConfig = RuntimeConfig; +impl Factor for VariablesFactor { + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; @@ -68,14 +48,8 @@ impl Factor for VariablesFactor { )?; } - if let Some(runtime_config) = ctx.take_runtime_config() { - for config in runtime_config.provider_configs { - for provider_resolver in self.provider_resolvers.iter() { - if let Some(provider) = provider_resolver.resolve_provider(&config)? { - expression_resolver.add_provider(provider); - } - } - } + for provider in ctx.take_runtime_config().unwrap_or_default() { + expression_resolver.add_provider(provider); } Ok(AppState { @@ -97,13 +71,6 @@ impl Factor for VariablesFactor { } } -/// The runtime configuration for the variables factor. -#[derive(Deserialize)] -#[serde(transparent)] -pub struct RuntimeConfig { - provider_configs: Vec, -} - pub struct AppState { expression_resolver: Arc, } diff --git a/crates/factor-variables/src/provider.rs b/crates/factor-variables/src/provider.rs deleted file mode 100644 index 82821fb32..000000000 --- a/crates/factor-variables/src/provider.rs +++ /dev/null @@ -1,17 +0,0 @@ -use serde::de::DeserializeOwned; -use spin_expressions::Provider; -use spin_factors::anyhow; - -/// A trait for converting a runtime configuration into a variables provider. -pub trait ProviderResolver: 'static { - /// Serialized configuration for the provider. - type RuntimeConfig: DeserializeOwned; - - /// Create a variables provider from the given runtime configuration. - /// - /// Returns `Ok(None)` if the provider is not applicable to the given configuration. - fn resolve_provider( - &self, - runtime_config: &Self::RuntimeConfig, - ) -> anyhow::Result>>; -} diff --git a/crates/factor-variables/src/runtime_config.rs b/crates/factor-variables/src/runtime_config.rs new file mode 100644 index 000000000..aaa7eb8da --- /dev/null +++ b/crates/factor-variables/src/runtime_config.rs @@ -0,0 +1,16 @@ +use spin_expressions::Provider; + +/// The runtime configuration for the variables factor. +#[derive(Default)] +pub struct RuntimeConfig { + pub providers: Vec>, +} + +impl IntoIterator for RuntimeConfig { + type Item = Box; + type IntoIter = std::vec::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.providers.into_iter() + } +} diff --git a/crates/factor-variables/src/spin_cli/env.rs b/crates/factor-variables/src/spin_cli/env.rs index 779f54d85..4e9d3ccce 100644 --- a/crates/factor-variables/src/spin_cli/env.rs +++ b/crates/factor-variables/src/spin_cli/env.rs @@ -11,31 +11,6 @@ use spin_factors::anyhow::{self, Context as _}; use spin_world::async_trait; use tracing::{instrument, Level}; -use crate::ProviderResolver; - -use super::VariableProviderConfiguration; - -/// Creator of a environment variables provider. -pub struct EnvVariables; - -impl ProviderResolver for EnvVariables { - type RuntimeConfig = VariableProviderConfiguration; - - fn resolve_provider( - &self, - runtime_config: &Self::RuntimeConfig, - ) -> anyhow::Result>> { - let VariableProviderConfiguration::Env(runtime_config) = runtime_config else { - return Ok(None); - }; - Ok(Some(Box::new(EnvVariablesProvider::new( - runtime_config.prefix.clone(), - |key| std::env::var(key), - runtime_config.dotenv_path.clone(), - )))) - } -} - /// Configuration for the environment variables provider. #[derive(Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] @@ -88,7 +63,7 @@ impl EnvVariablesProvider { let prefix = self .prefix .clone() - .unwrap_or(DEFAULT_ENV_PREFIX.to_string()); + .unwrap_or_else(|| DEFAULT_ENV_PREFIX.to_string()); let upper_key = key.as_ref().to_ascii_uppercase(); let env_key = format!("{prefix}_{upper_key}"); diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index 5e6a2bd28..13c1cbb23 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -3,12 +3,13 @@ mod env; mod statik; -pub use env::EnvVariables; -pub use statik::StaticVariables; - use serde::Deserialize; +use spin_expressions::Provider; +use spin_factors::anyhow; use statik::StaticVariablesProvider; +use crate::runtime_config::RuntimeConfig; + /// A runtime configuration used in the Spin CLI for one type of variable provider. #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] @@ -19,8 +20,30 @@ pub enum VariableProviderConfiguration { Env(env::EnvVariablesConfig), } -/// The runtime configuration for the variables factor used in the Spin CLI. -pub type RuntimeConfig = super::RuntimeConfig; +impl VariableProviderConfiguration { + /// Returns the provider for the configuration. + pub fn into_provider(self) -> Box { + match self { + VariableProviderConfiguration::Static(provider) => Box::new(provider), + VariableProviderConfiguration::Env(config) => Box::new(env::EnvVariablesProvider::new( + config.prefix, + |s| std::env::var(s), + config.dotenv_path, + )), + } + } +} -/// The variables factor used in the Spin CLI. -pub type VariablesFactor = super::VariablesFactor; +/// Resolves a runtime configuration for the variables factor from a TOML table. +pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result> { + let Some(array) = table.get("variable_provider") else { + return Ok(None); + }; + + let provider_configs: Vec = array.clone().try_into()?; + let providers = provider_configs + .into_iter() + .map(VariableProviderConfiguration::into_provider) + .collect(); + Ok(Some(RuntimeConfig { providers })) +} diff --git a/crates/factor-variables/src/spin_cli/statik.rs b/crates/factor-variables/src/spin_cli/statik.rs index a34756526..3c024eabf 100644 --- a/crates/factor-variables/src/spin_cli/statik.rs +++ b/crates/factor-variables/src/spin_cli/statik.rs @@ -4,27 +4,6 @@ use serde::Deserialize; use spin_expressions::{async_trait::async_trait, Key, Provider}; use spin_factors::anyhow; -use crate::ProviderResolver; - -use super::VariableProviderConfiguration; - -/// Creator of a static variables provider. -pub struct StaticVariables; - -impl ProviderResolver for StaticVariables { - type RuntimeConfig = VariableProviderConfiguration; - - fn resolve_provider( - &self, - runtime_config: &Self::RuntimeConfig, - ) -> anyhow::Result>> { - let VariableProviderConfiguration::Static(config) = runtime_config else { - return Ok(None); - }; - Ok(Some(Box::new(config.clone()) as _)) - } -} - /// A variables provider that reads variables from an static map. #[derive(Debug, Deserialize, Clone)] pub struct StaticVariablesProvider { diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index cc2735574..2e17fc96c 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -1,7 +1,4 @@ -use spin_factor_variables::spin_cli::{ - EnvVariables, StaticVariables, VariableProviderConfiguration, -}; -use spin_factor_variables::VariablesFactor; +use spin_factor_variables::{spin_cli, VariablesFactor}; use spin_factors::{ anyhow, Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, }; @@ -10,18 +7,14 @@ use spin_world::v2::variables::Host; #[derive(RuntimeFactors)] struct TestFactors { - variables: VariablesFactor, + variables: VariablesFactor, } #[tokio::test] async fn static_provider_works() -> anyhow::Result<()> { - let mut factors = TestFactors { + let factors = TestFactors { variables: VariablesFactor::default(), }; - factors.variables.add_provider_resolver(StaticVariables)?; - // The env provider will be ignored since there's no configuration for it. - factors.variables.add_provider_resolver(EnvVariables)?; - let env = TestEnvironment::new(factors) .extend_manifest(toml! { [variables] @@ -31,7 +24,7 @@ async fn static_provider_works() -> anyhow::Result<()> { source = "does-not-exist.wasm" variables = { baz = "<{{ foo }}>" } }) - .runtime_config(TomlConfig(toml! { + .runtime_config(TomlConfig::new(toml! { [[variable_provider]] type = "static" values = { foo = "bar" } @@ -43,7 +36,15 @@ async fn static_provider_works() -> anyhow::Result<()> { Ok(()) } -struct TomlConfig(toml::Table); +struct TomlConfig { + table: toml::Table, +} + +impl TomlConfig { + fn new(table: toml::Table) -> Self { + Self { table } + } +} impl TryFrom for TestFactorsRuntimeConfig { type Error = anyhow::Error; @@ -53,16 +54,11 @@ impl TryFrom for TestFactorsRuntimeConfig { } } -impl FactorRuntimeConfigSource> for TomlConfig { +impl FactorRuntimeConfigSource for TomlConfig { fn get_runtime_config( &mut self, - ) -> anyhow::Result< - Option< as Factor>::RuntimeConfig>, - > { - let Some(table) = self.0.get("variable_provider") else { - return Ok(None); - }; - Ok(Some(table.clone().try_into()?)) + ) -> anyhow::Result::RuntimeConfig>> { + spin_cli::runtime_config_from_toml(&self.table) } } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 205775ca1..ec46d8489 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -11,10 +11,7 @@ use spin_factor_key_value_redis::RedisKeyValueStore; use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::{ - spin_cli::{StaticVariables, VariableProviderConfiguration}, - VariablesFactor, -}; +use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{ Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, @@ -24,7 +21,7 @@ use wasmtime_wasi_http::WasiHttpView; #[derive(RuntimeFactors)] struct Factors { wasi: WasiFactor, - variables: VariablesFactor, + variables: VariablesFactor, outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, key_value: KeyValueFactor, @@ -66,8 +63,6 @@ async fn smoke_test_works() -> anyhow::Result<()> { key_value: KeyValueFactor::new(key_value_resolver), }; - factors.variables.add_provider_resolver(StaticVariables)?; - let locked = spin_loader::from_file( "tests/smoke-app/spin.toml", spin_loader::FilesMountStrategy::Direct, @@ -169,21 +164,16 @@ impl FactorRuntimeConfigSource> } } -impl FactorRuntimeConfigSource> for TestSource { +impl FactorRuntimeConfigSource for TestSource { fn get_runtime_config( &mut self, - ) -> anyhow::Result< - Option< as Factor>::RuntimeConfig>, - > { - let config = toml::toml! { + ) -> anyhow::Result::RuntimeConfig>> { + spin_factor_variables::spin_cli::runtime_config_from_toml(&toml::toml! { [[variable_provider]] type = "static" [variable_provider.values] foo = "bar" - } - .remove("variable_provider") - .unwrap(); - Ok(Some(config.try_into()?)) + }) } } From 1a97874c09ad38f58980f629e46030901c93f9b3 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 29 Jul 2024 14:50:13 +0200 Subject: [PATCH 098/195] Move host impls to their own module Signed-off-by: Ryan Levick --- crates/factor-variables/src/host.rs | 46 +++++++++++++++++++++ crates/factor-variables/src/lib.rs | 44 +------------------- crates/factor-variables/src/spin_cli/mod.rs | 28 ++++++------- 3 files changed, 61 insertions(+), 57 deletions(-) create mode 100644 crates/factor-variables/src/host.rs diff --git a/crates/factor-variables/src/host.rs b/crates/factor-variables/src/host.rs new file mode 100644 index 000000000..aa1d70f34 --- /dev/null +++ b/crates/factor-variables/src/host.rs @@ -0,0 +1,46 @@ +use spin_factors::anyhow; +use spin_world::{async_trait, v1, v2::variables}; + +use crate::InstanceState; + +#[async_trait] +impl variables::Host for InstanceState { + async fn get(&mut self, key: String) -> Result { + let key = spin_expressions::Key::new(&key).map_err(expressions_to_variables_err)?; + self.expression_resolver + .resolve(&self.component_id, key) + .await + .map_err(expressions_to_variables_err) + } + + fn convert_error(&mut self, error: variables::Error) -> anyhow::Result { + Ok(error) + } +} + +#[async_trait] +impl v1::config::Host for InstanceState { + async fn get_config(&mut self, key: String) -> Result { + ::get(self, key) + .await + .map_err(|err| match err { + variables::Error::InvalidName(msg) => v1::config::Error::InvalidKey(msg), + variables::Error::Undefined(msg) => v1::config::Error::Provider(msg), + other => v1::config::Error::Other(format!("{other}")), + }) + } + + fn convert_error(&mut self, err: v1::config::Error) -> anyhow::Result { + Ok(err) + } +} + +fn expressions_to_variables_err(err: spin_expressions::Error) -> variables::Error { + use spin_expressions::Error; + match err { + Error::InvalidName(msg) => variables::Error::InvalidName(msg), + Error::Undefined(msg) => variables::Error::Undefined(msg), + Error::Provider(err) => variables::Error::Provider(err.to_string()), + other => variables::Error::Other(format!("{other}")), + } +} diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 554f89cbb..21e95b701 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -1,3 +1,4 @@ +mod host; pub mod runtime_config; pub mod spin_cli; @@ -9,7 +10,6 @@ use spin_factors::{ anyhow, ConfigureAppContext, Factor, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; -use spin_world::{async_trait, v1, v2::variables}; /// A factor for providing variables to components. pub struct VariablesFactor { @@ -87,45 +87,3 @@ impl InstanceState { } impl SelfInstanceBuilder for InstanceState {} - -#[async_trait] -impl variables::Host for InstanceState { - async fn get(&mut self, key: String) -> Result { - let key = spin_expressions::Key::new(&key).map_err(expressions_to_variables_err)?; - self.expression_resolver - .resolve(&self.component_id, key) - .await - .map_err(expressions_to_variables_err) - } - - fn convert_error(&mut self, error: variables::Error) -> anyhow::Result { - Ok(error) - } -} - -#[async_trait] -impl v1::config::Host for InstanceState { - async fn get_config(&mut self, key: String) -> Result { - ::get(self, key) - .await - .map_err(|err| match err { - variables::Error::InvalidName(msg) => v1::config::Error::InvalidKey(msg), - variables::Error::Undefined(msg) => v1::config::Error::Provider(msg), - other => v1::config::Error::Other(format!("{other}")), - }) - } - - fn convert_error(&mut self, err: v1::config::Error) -> anyhow::Result { - Ok(err) - } -} - -fn expressions_to_variables_err(err: spin_expressions::Error) -> variables::Error { - use spin_expressions::Error; - match err { - Error::InvalidName(msg) => variables::Error::InvalidName(msg), - Error::Undefined(msg) => variables::Error::Undefined(msg), - Error::Provider(err) => variables::Error::Provider(err.to_string()), - other => variables::Error::Other(format!("{other}")), - } -} diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index 13c1cbb23..bf21a944d 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -10,6 +10,20 @@ use statik::StaticVariablesProvider; use crate::runtime_config::RuntimeConfig; +/// Resolves a runtime configuration for the variables factor from a TOML table. +pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result> { + let Some(array) = table.get("variable_provider") else { + return Ok(None); + }; + + let provider_configs: Vec = array.clone().try_into()?; + let providers = provider_configs + .into_iter() + .map(VariableProviderConfiguration::into_provider) + .collect(); + Ok(Some(RuntimeConfig { providers })) +} + /// A runtime configuration used in the Spin CLI for one type of variable provider. #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] @@ -33,17 +47,3 @@ impl VariableProviderConfiguration { } } } - -/// Resolves a runtime configuration for the variables factor from a TOML table. -pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result> { - let Some(array) = table.get("variable_provider") else { - return Ok(None); - }; - - let provider_configs: Vec = array.clone().try_into()?; - let providers = provider_configs - .into_iter() - .map(VariableProviderConfiguration::into_provider) - .collect(); - Ok(Some(RuntimeConfig { providers })) -} From 8fed0976601c00f7634c1256e15deace3ec3b066 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 29 Jul 2024 15:29:05 +0200 Subject: [PATCH 099/195] Default support for env variables Signed-off-by: Ryan Levick --- crates/factor-variables/src/lib.rs | 7 +----- crates/factor-variables/src/spin_cli/env.rs | 17 +++++++++++--- crates/factor-variables/src/spin_cli/mod.rs | 23 +++++++++++-------- .../factor-variables/src/spin_cli/statik.rs | 2 +- crates/factor-variables/tests/factor_test.rs | 4 ++-- crates/factors/tests/smoke.rs | 1 + 6 files changed, 33 insertions(+), 21 deletions(-) diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 21e95b701..9ea892a2b 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -12,16 +12,11 @@ use spin_factors::{ }; /// A factor for providing variables to components. +#[derive(Default)] pub struct VariablesFactor { _priv: (), } -impl Default for VariablesFactor { - fn default() -> Self { - Self { _priv: () } - } -} - impl Factor for VariablesFactor { type RuntimeConfig = RuntimeConfig; type AppState = AppState; diff --git a/crates/factor-variables/src/spin_cli/env.rs b/crates/factor-variables/src/spin_cli/env.rs index 4e9d3ccce..781140a1b 100644 --- a/crates/factor-variables/src/spin_cli/env.rs +++ b/crates/factor-variables/src/spin_cli/env.rs @@ -29,7 +29,7 @@ const DEFAULT_ENV_PREFIX: &str = "SPIN_VARIABLE"; type EnvFetcherFn = Box Result + Send + Sync>; -/// A config Provider that uses environment variables. +/// A [`Provider`] that uses environment variables. pub struct EnvVariablesProvider { prefix: Option, env_fetcher: EnvFetcherFn, @@ -37,14 +37,25 @@ pub struct EnvVariablesProvider { dotenv_cache: OnceLock>, } +impl Default for EnvVariablesProvider { + fn default() -> Self { + Self { + prefix: None, + env_fetcher: Box::new(|s| std::env::var(s)), + dotenv_path: Some(".env".into()), + dotenv_cache: Default::default(), + } + } +} + impl EnvVariablesProvider { /// Creates a new EnvProvider. /// /// * `prefix` - The string prefix to use to distinguish an environment variable that should be used. - /// If not set, the default prefix is used. + /// If not set, the default prefix is used. /// * `env_fetcher` - The function to use to fetch an environment variable. /// * `dotenv_path` - The path to the .env file to load environment variables from. If not set, - /// no .env file is loaded. + /// no .env file is loaded. pub fn new( prefix: Option>, env_fetcher: impl Fn(&str) -> Result + Send + Sync + 'static, diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index bf21a944d..2b6a48c48 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -3,25 +3,30 @@ mod env; mod statik; +pub use env::*; +pub use statik::*; + use serde::Deserialize; use spin_expressions::Provider; use spin_factors::anyhow; -use statik::StaticVariablesProvider; use crate::runtime_config::RuntimeConfig; /// Resolves a runtime configuration for the variables factor from a TOML table. -pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result> { +pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result { + // Always include the environment variable provider. + let mut providers = vec![Box::new(EnvVariablesProvider::default()) as _]; let Some(array) = table.get("variable_provider") else { - return Ok(None); + return Ok(RuntimeConfig { providers }); }; let provider_configs: Vec = array.clone().try_into()?; - let providers = provider_configs - .into_iter() - .map(VariableProviderConfiguration::into_provider) - .collect(); - Ok(Some(RuntimeConfig { providers })) + providers.extend( + provider_configs + .into_iter() + .map(VariableProviderConfiguration::into_provider), + ); + Ok(RuntimeConfig { providers }) } /// A runtime configuration used in the Spin CLI for one type of variable provider. @@ -31,7 +36,7 @@ pub enum VariableProviderConfiguration { /// A static provider of variables. Static(StaticVariablesProvider), /// An environment variable provider. - Env(env::EnvVariablesConfig), + Env(EnvVariablesConfig), } impl VariableProviderConfiguration { diff --git a/crates/factor-variables/src/spin_cli/statik.rs b/crates/factor-variables/src/spin_cli/statik.rs index 3c024eabf..d596c3e56 100644 --- a/crates/factor-variables/src/spin_cli/statik.rs +++ b/crates/factor-variables/src/spin_cli/statik.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use spin_expressions::{async_trait::async_trait, Key, Provider}; use spin_factors::anyhow; -/// A variables provider that reads variables from an static map. +/// A [`Provider`] that reads variables from an static map. #[derive(Debug, Deserialize, Clone)] pub struct StaticVariablesProvider { values: Arc>, diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index 2e17fc96c..1ce403000 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -10,7 +10,7 @@ struct TestFactors { variables: VariablesFactor, } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn static_provider_works() -> anyhow::Result<()> { let factors = TestFactors { variables: VariablesFactor::default(), @@ -58,7 +58,7 @@ impl FactorRuntimeConfigSource for TomlConfig { fn get_runtime_config( &mut self, ) -> anyhow::Result::RuntimeConfig>> { - spin_cli::runtime_config_from_toml(&self.table) + spin_cli::runtime_config_from_toml(&self.table).map(Some) } } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index ec46d8489..6beb96f50 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -174,6 +174,7 @@ impl FactorRuntimeConfigSource for TestSource { [variable_provider.values] foo = "bar" }) + .map(Some) } } From afebc8efe8d2c4bd94271d82b0fbcb47506306b7 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 30 Jul 2024 11:40:40 +0200 Subject: [PATCH 100/195] PR feedback Signed-off-by: Ryan Levick --- crates/factor-key-value/src/lib.rs | 2 +- crates/factor-key-value/src/runtime_config.rs | 9 ++++++++- .../factor-key-value/src/runtime_config/spin.rs | 16 ++++++---------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/factor-key-value/src/lib.rs b/crates/factor-key-value/src/lib.rs index 54bec6242..1e71bb86a 100644 --- a/crates/factor-key-value/src/lib.rs +++ b/crates/factor-key-value/src/lib.rs @@ -25,7 +25,7 @@ pub struct KeyValueFactor { impl KeyValueFactor { /// Create a new KeyValueFactor. /// - /// The `default_label_resolver` is used to resolver store managers for labels that + /// The `default_label_resolver` is used to resolve store managers for labels that /// are not defined in the runtime configuration. pub fn new(default_label_resolver: impl DefaultLabelResolver + 'static) -> Self { Self { diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index e2b3c9ff9..d5895e75a 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -8,7 +8,14 @@ use spin_key_value::StoreManager; #[derive(Default)] pub struct RuntimeConfig { /// Map of store names to store managers. - pub store_managers: HashMap>, + store_managers: HashMap>, +} + +impl RuntimeConfig { + /// Adds a store manager for the store with the given label to the runtime configuration. + pub fn add_store_manager(&mut self, label: String, store_manager: Arc) { + self.store_managers.insert(label, store_manager); + } } impl IntoIterator for RuntimeConfig { diff --git a/crates/factor-key-value/src/runtime_config/spin.rs b/crates/factor-key-value/src/runtime_config/spin.rs index 36b5266d8..20c218447 100644 --- a/crates/factor-key-value/src/runtime_config/spin.rs +++ b/crates/factor-key-value/src/runtime_config/spin.rs @@ -92,18 +92,14 @@ impl RuntimeConfigResolver { let Some(table) = table.as_ref().and_then(|t| t.get("key_value_store")) else { return Ok(None); }; - let mut store_configs = HashMap::new(); - for (label, config) in table - .as_table() - .context("expected a 'key_value_store' to contain toml table")? - { - let config: StoreConfig = config.clone().try_into()?; + let table: HashMap = table.clone().try_into()?; + + let mut runtime_config = RuntimeConfig::default(); + for (label, config) in table { let store_manager = self.store_manager_from_config(config)?; - store_configs.insert(label.clone(), store_manager); + runtime_config.add_store_manager(label.clone(), store_manager); } - Ok(Some(RuntimeConfig { - store_managers: store_configs, - })) + Ok(Some(runtime_config)) } /// Given a [`StoreConfig`], returns a store manager. From 9b850824fbdf367dffb9a8439a0b65b27291da25 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 29 Jul 2024 15:12:25 +0200 Subject: [PATCH 101/195] Add vault variable provider Signed-off-by: Ryan Levick --- Cargo.lock | 1 + crates/factor-variables/Cargo.toml | 1 + crates/factor-variables/src/spin_cli/mod.rs | 5 ++ crates/factor-variables/src/spin_cli/vault.rs | 55 +++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 crates/factor-variables/src/spin_cli/vault.rs diff --git a/Cargo.lock b/Cargo.lock index d9769241d..496b7ad6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7722,6 +7722,7 @@ dependencies = [ "tokio", "toml 0.8.14", "tracing", + "vaultrs", ] [[package]] diff --git a/crates/factor-variables/Cargo.toml b/crates/factor-variables/Cargo.toml index ac7370d70..85f0687b1 100644 --- a/crates/factor-variables/Cargo.toml +++ b/crates/factor-variables/Cargo.toml @@ -13,6 +13,7 @@ spin-world = { path = "../world" } tokio = { version = "1", features = ["rt-multi-thread"] } toml = "0.8" tracing = { workspace = true } +vaultrs = "0.6.2" [dev-dependencies] spin-factors-test = { path = "../factors-test" } diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index 2b6a48c48..49ea1261c 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -2,9 +2,11 @@ mod env; mod statik; +mod vault; pub use env::*; pub use statik::*; +pub use vault::*; use serde::Deserialize; use spin_expressions::Provider; @@ -35,6 +37,8 @@ pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result Box::new(provider), } } } diff --git a/crates/factor-variables/src/spin_cli/vault.rs b/crates/factor-variables/src/spin_cli/vault.rs new file mode 100644 index 000000000..9008ce3c5 --- /dev/null +++ b/crates/factor-variables/src/spin_cli/vault.rs @@ -0,0 +1,55 @@ +use serde::{Deserialize, Serialize}; +use spin_expressions::async_trait::async_trait; +use spin_factors::anyhow::{self, Context as _}; +use tracing::{instrument, Level}; +use vaultrs::{ + client::{VaultClient, VaultClientSettingsBuilder}, + error::ClientError, + kv2, +}; + +use spin_expressions::{Key, Provider}; + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +/// A config Provider that uses HashiCorp Vault. +pub struct VaultVariablesProvider { + /// The URL of the Vault server. + url: String, + /// The token to authenticate with. + token: String, + /// The mount point of the KV engine. + mount: String, + /// The optional prefix to use for all keys. + #[serde(default)] + prefix: Option, +} + +#[async_trait] +impl Provider for VaultVariablesProvider { + #[instrument(name = "spin_variables.get_from_vault", skip(self), err(level = Level::INFO), fields(otel.kind = "client"))] + async fn get(&self, key: &Key) -> anyhow::Result> { + let client = VaultClient::new( + VaultClientSettingsBuilder::default() + .address(&self.url) + .token(&self.token) + .build()?, + )?; + let path = match &self.prefix { + Some(prefix) => format!("{}/{}", prefix, key.as_str()), + None => key.as_str().to_string(), + }; + + #[derive(Deserialize, Serialize)] + struct Secret { + value: String, + } + match kv2::read::(&client, &self.mount, &path).await { + Ok(secret) => Ok(Some(secret.value)), + // Vault doesn't have this entry so pass along the chain + Err(ClientError::APIError { code: 404, .. }) => Ok(None), + // Other Vault error so bail rather than looking elsewhere + Err(e) => Err(e).context("Failed to check Vault for config"), + } + } +} From f9782c41f50e89e6ba977570c75eda19e3aa264a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 29 Jul 2024 17:48:53 -0400 Subject: [PATCH 102/195] core: Change AsMut to AsState Signed-off-by: Lann Martin --- crates/core/build.rs | 6 ++++++ crates/core/src/lib.rs | 2 +- crates/core/src/store.rs | 19 ++++++++++++++++--- crates/core/tests/integration_test.rs | 12 ++++++------ 4 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 crates/core/build.rs diff --git a/crates/core/build.rs b/crates/core/build.rs new file mode 100644 index 000000000..c96556b06 --- /dev/null +++ b/crates/core/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + // Enable spin-factors-derive to emit expanded macro output. + let out_dir = std::env::var("OUT_DIR").unwrap(); + println!("cargo:rustc-env=SPIN_FACTORS_DERIVE_EXPAND_DIR={out_dir}"); +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index cd02404ba..411d96c51 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -26,7 +26,7 @@ pub use wasmtime::{ Instance as ModuleInstance, Module, Trap, }; -pub use store::{Store, StoreBuilder}; +pub use store::{AsState, Store, StoreBuilder}; /// The default [`EngineBuilder::epoch_tick_interval`]. pub const DEFAULT_EPOCH_TICK_INTERVAL: Duration = Duration::from_millis(10); diff --git a/crates/core/src/store.rs b/crates/core/src/store.rs index b2b950b5f..7ad7168b5 100644 --- a/crates/core/src/store.rs +++ b/crates/core/src/store.rs @@ -97,11 +97,11 @@ impl StoreBuilder { /// /// The `T` parameter must provide access to a [`State`] via `impl /// AsMut`. - pub fn build>(self, mut data: T) -> Result> { - data.as_mut().store_limits = self.store_limits; + pub fn build(self, mut data: T) -> Result> { + data.as_state().store_limits = self.store_limits; let mut inner = wasmtime::Store::new(&self.engine, data); - inner.limiter_async(|data| &mut data.as_mut().store_limits); + inner.limiter_async(|data| &mut data.as_state().store_limits); // With epoch interruption enabled, there must be _some_ deadline set // or execution will trap immediately. Since this is a delta, we need @@ -115,3 +115,16 @@ impl StoreBuilder { }) } } + +/// For consumers that need to use a type other than [`State`] as the [`Store`] +/// `data`, this trait must be implemented for that type. +pub trait AsState { + /// Gives access to the inner [`State`]. + fn as_state(&mut self) -> &mut State; +} + +impl AsState for State { + fn as_state(&mut self) -> &mut State { + self + } +} diff --git a/crates/core/tests/integration_test.rs b/crates/core/tests/integration_test.rs index da328b2c5..8db91940d 100644 --- a/crates/core/tests/integration_test.rs +++ b/crates/core/tests/integration_test.rs @@ -5,9 +5,9 @@ use std::{ use anyhow::Context; use serde_json::json; -use spin_core::{Component, Config, Engine, State, Store, StoreBuilder, Trap}; +use spin_core::{AsState, Component, Config, Engine, State, Store, StoreBuilder, Trap}; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; -use spin_factors::{App, RuntimeFactors}; +use spin_factors::{App, AsInstanceState, RuntimeFactors}; use spin_locked_app::locked::LockedApp; use tokio::{fs, io::AsyncWrite}; use wasmtime_wasi::I32Exit; @@ -93,14 +93,14 @@ struct TestState { factors: TestFactorsInstanceState, } -impl AsMut for TestState { - fn as_mut(&mut self) -> &mut State { +impl AsState for TestState { + fn as_state(&mut self) -> &mut State { &mut self.core } } -impl AsMut for TestState { - fn as_mut(&mut self) -> &mut TestFactorsInstanceState { +impl AsInstanceState for TestState { + fn as_instance_state(&mut self) -> &mut TestFactorsInstanceState { &mut self.factors } } From 5e0d72979ce2b699f0ced58e469c4331aa9c5431 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 29 Jul 2024 17:54:38 -0400 Subject: [PATCH 103/195] triggers: Replace AsMut with AsInstanceState Signed-off-by: Lann Martin --- crates/factors-derive/src/lib.rs | 10 +++++----- crates/factors/src/lib.rs | 2 +- crates/factors/src/runtime_factors.rs | 8 ++++++-- crates/factors/tests/smoke.rs | 10 ++++++---- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 80b750d46..921196c8b 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -78,7 +78,7 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { type InstanceState = #state_name; type RuntimeConfig = #runtime_config_name; - fn init + Send + 'static>( + fn init + Send + 'static>( &mut self, linker: &mut #wasmtime::component::Linker, ) -> #Result<()> { @@ -98,9 +98,9 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { &mut self.#factor_names, #factors_path::InitContext::::new( linker, - |data| &mut data.as_mut().#factor_names, + |data| &mut data.as_instance_state().#factor_names, |data| { - let state = data.as_mut(); + let state = data.as_instance_state(); (&mut state.#factor_names, &mut state.__table) }, ) @@ -239,8 +239,8 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { } } - impl AsMut<#state_name> for #state_name { - fn as_mut(&mut self) -> &mut Self { + impl #factors_path::AsInstanceState<#state_name> for #state_name { + fn as_instance_state(&mut self) -> &mut Self { self } } diff --git a/crates/factors/src/lib.rs b/crates/factors/src/lib.rs index ce0666825..d73435f21 100644 --- a/crates/factors/src/lib.rs +++ b/crates/factors/src/lib.rs @@ -14,7 +14,7 @@ pub use crate::{ factor::{ConfigureAppContext, ConfiguredApp, Factor, FactorInstanceState, InitContext}, prepare::{FactorInstanceBuilder, InstanceBuilders, PrepareContext, SelfInstanceBuilder}, runtime_config::{FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer}, - runtime_factors::{RuntimeFactors, RuntimeFactorsInstanceState}, + runtime_factors::{AsInstanceState, RuntimeFactors, RuntimeFactorsInstanceState}, }; /// Result wrapper type defaulting to use [`Error`]. diff --git a/crates/factors/src/runtime_factors.rs b/crates/factors/src/runtime_factors.rs index a0ff6dd68..e18a66c17 100644 --- a/crates/factors/src/runtime_factors.rs +++ b/crates/factors/src/runtime_factors.rs @@ -44,7 +44,7 @@ pub trait RuntimeFactors: Sized + 'static { /// /// Each factor's `init` is called in turn. Must be called once before /// [`RuntimeFactors::prepare`]. - fn init + Send + 'static>( + fn init + Send + 'static>( &mut self, linker: &mut Linker, ) -> crate::Result<()>; @@ -84,7 +84,7 @@ pub trait RuntimeFactors: Sized + 'static { /// Get the state of a particular Factor from the overall InstanceState /// /// Implemented by `#[derive(RuntimeFactors)]` -pub trait RuntimeFactorsInstanceState: AsMut + Send + 'static { +pub trait RuntimeFactorsInstanceState: AsInstanceState + Send + 'static { fn get_with_table( &mut self, ) -> Option<(&mut FactorInstanceState, &mut ResourceTable)>; @@ -97,3 +97,7 @@ pub trait RuntimeFactorsInstanceState: AsMut + Send + 'static { fn table_mut(&mut self) -> &mut ResourceTable; } + +pub trait AsInstanceState { + fn as_instance_state(&mut self) -> &mut T; +} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index f23a3d15d..7cbbb24b3 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -14,7 +14,8 @@ use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; use spin_factors::{ - Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors, + AsInstanceState, Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, + RuntimeFactors, }; use wasmtime_wasi_http::WasiHttpView; @@ -32,8 +33,8 @@ struct Data { _other_data: usize, } -impl AsMut for Data { - fn as_mut(&mut self) -> &mut FactorsInstanceState { +impl AsInstanceState for Data { + fn as_instance_state(&mut self) -> &mut FactorsInstanceState { &mut self.factors_instance_state } } @@ -116,7 +117,8 @@ async fn smoke_test_works() -> anyhow::Result<()> { // Invoke handler let req = http::Request::get("/").body(Default::default()).unwrap(); - let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(store.data_mut().as_mut()).unwrap(); + let mut wasi_http = + OutboundHttpFactor::get_wasi_http_impl(store.data_mut().as_instance_state()).unwrap(); let request = wasi_http.new_incoming_request(req)?; let (response_tx, response_rx) = tokio::sync::oneshot::channel(); let response = wasi_http.new_response_outparam(response_tx)?; From 3782a36ba677bf8affeaf4d87d93cd76791341d6 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 30 Jul 2024 09:41:46 -0400 Subject: [PATCH 104/195] Add spin-factors-executor Signed-off-by: Lann Martin --- Cargo.lock | 13 ++ crates/core/src/lib.rs | 3 +- crates/factors-executor/Cargo.toml | 23 +++ crates/factors-executor/src/lib.rs | 223 +++++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 crates/factors-executor/Cargo.toml create mode 100644 crates/factors-executor/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index d9769241d..cb5949fac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7776,6 +7776,19 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "spin-factors-executor" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "spin-app", + "spin-core", + "spin-factor-wasi", + "spin-factors", + "spin-factors-test", + "tokio", +] + [[package]] name = "spin-factors-test" version = "2.7.0-pre0" diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 411d96c51..e586c1bcb 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -16,13 +16,12 @@ use std::{path::PathBuf, time::Duration}; use anyhow::Result; use crossbeam_channel::Sender; use tracing::instrument; -use wasmtime::component::{InstancePre, Linker}; use wasmtime::{InstanceAllocationStrategy, PoolingAllocationConfig}; pub use async_trait::async_trait; pub use wasmtime::{ self, - component::{Component, Instance}, + component::{Component, Instance, InstancePre, Linker}, Instance as ModuleInstance, Module, Trap, }; diff --git a/crates/factors-executor/Cargo.toml b/crates/factors-executor/Cargo.toml new file mode 100644 index 000000000..14500c624 --- /dev/null +++ b/crates/factors-executor/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "spin-factors-executor" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = "1" +spin-app = { path = "../app" } +spin-core = { path = "../core" } +spin-factors = { path = "../factors" } + +[dev-dependencies] +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/factors-executor/src/lib.rs b/crates/factors-executor/src/lib.rs new file mode 100644 index 000000000..c624a02f0 --- /dev/null +++ b/crates/factors-executor/src/lib.rs @@ -0,0 +1,223 @@ +use std::collections::HashMap; + +use anyhow::Context; +use spin_app::{App, AppComponent}; +use spin_core::Component; +use spin_factors::{AsInstanceState, ConfiguredApp, RuntimeFactors, RuntimeFactorsInstanceState}; + +/// A FactorsExecutor manages execution of a Spin app. +/// +/// `Factors` is the executor's [`RuntimeFactors`]. `ExecutorInstanceState` +/// holds any other per-instance state needed by the caller. +pub struct FactorsExecutor { + factors: T, + core_engine: spin_core::Engine>, + configured_app: ConfiguredApp, + // Maps component IDs -> InstancePres + component_instance_pres: HashMap>, +} + +type InstancePre = + spin_core::InstancePre::InstanceState, U>>; + +impl FactorsExecutor { + /// Constructs a new executor. + pub fn new( + core_config: &spin_core::Config, + mut factors: T, + app: App, + mut component_loader: impl ComponentLoader, + runtime_config: T::RuntimeConfig, + ) -> anyhow::Result { + let core_engine = { + let mut builder = + spin_core::Engine::builder(core_config).context("failed to initialize engine")?; + factors + .init(builder.linker()) + .context("failed to initialize factors")?; + builder.build() + }; + + let configured_app = factors + .configure_app(app, runtime_config) + .context("failed to configure app")?; + + let component_instance_pres = configured_app + .app() + .components() + .map(|app_component| { + let component = + component_loader.load_component(core_engine.as_ref(), &app_component)?; + let instance_pre = core_engine.instantiate_pre(&component)?; + Ok((app_component.id().to_string(), instance_pre)) + }) + .collect::>>()?; + + Ok(Self { + factors, + core_engine, + configured_app, + component_instance_pres, + }) + } + + /// Returns an instance builder for the given component ID. + pub fn prepare(&mut self, component_id: &str) -> anyhow::Result> { + let app_component = self + .configured_app + .app() + .get_component(component_id) + .with_context(|| format!("no such component {component_id:?}"))?; + let instance_pre = self.component_instance_pres.get(component_id).unwrap(); + let factor_builders = self.factors.prepare(&self.configured_app, component_id)?; + let store_builder = self.core_engine.store_builder(); + Ok(FactorsInstanceBuilder { + store_builder, + factor_builders, + instance_pre, + app_component, + factors: &self.factors, + }) + } +} + +/// A ComponentLoader is responsible for loading Wasmtime [`Component`]s. +pub trait ComponentLoader { + /// Loads a [`Component`] for the given [`AppComponent`]. + fn load_component( + &mut self, + engine: &spin_core::wasmtime::Engine, + component: &AppComponent, + ) -> anyhow::Result; +} + +/// A FactorsInstanceBuilder manages the instantiation of a Spin component +/// instance. +pub struct FactorsInstanceBuilder<'a, T: RuntimeFactors, U> { + app_component: AppComponent<'a>, + store_builder: spin_core::StoreBuilder, + factor_builders: T::InstanceBuilders, + instance_pre: &'a InstancePre, + factors: &'a T, +} + +impl<'a, T: RuntimeFactors, U: Send> FactorsInstanceBuilder<'a, T, U> { + /// Returns the app component for the instance. + pub fn app_component(&self) -> &AppComponent { + &self.app_component + } + + /// Returns the store builder for the instance. + pub fn store_builder(&mut self) -> &mut spin_core::StoreBuilder { + &mut self.store_builder + } + + /// Returns the factor instance builders for the instance. + pub fn factor_builders(&mut self) -> &mut T::InstanceBuilders { + &mut self.factor_builders + } + + /// Instantiates the instance with the given executor instance state + pub async fn instantiate( + self, + executor_instance_state: U, + ) -> anyhow::Result<( + spin_core::Instance, + spin_core::Store>, + )> { + let instance_state = InstanceState { + core: Default::default(), + factors: self.factors.build_instance_state(self.factor_builders)?, + executor: executor_instance_state, + }; + let mut store = self.store_builder.build(instance_state)?; + let instance = self.instance_pre.instantiate_async(&mut store).await?; + Ok((instance, store)) + } +} + +/// InstanceState is the [`spin_core::Store`] `data` for an instance. +pub struct InstanceState { + core: spin_core::State, + factors: FactorsState, + executor: ExecutorInstanceState, +} + +impl InstanceState { + /// Provides access to the `ExecutorInstanceState`. + pub fn executor_instance_state(&mut self) -> &mut U { + &mut self.executor + } +} + +impl spin_core::AsState for InstanceState { + fn as_state(&mut self) -> &mut spin_core::State { + &mut self.core + } +} + +impl AsInstanceState for InstanceState { + fn as_instance_state(&mut self) -> &mut T { + &mut self.factors + } +} + +#[cfg(test)] +mod tests { + use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; + use spin_factors::RuntimeFactors; + use spin_factors_test::TestEnvironment; + + use super::*; + + #[derive(RuntimeFactors)] + struct TestFactors { + wasi: WasiFactor, + } + + #[tokio::test] + async fn instance_builder_works() -> anyhow::Result<()> { + let factors = TestFactors { + wasi: WasiFactor::new(DummyFilesMounter), + }; + let env = TestEnvironment::new(factors); + let locked = env.build_locked_app().await?; + let app = App::new("test-app", locked); + + let mut executor = FactorsExecutor::new( + &Default::default(), + env.factors, + app, + DummyComponentLoader, + Default::default(), + )?; + + let mut instance_builder = executor.prepare("empty")?; + + assert_eq!(instance_builder.app_component().id(), "empty"); + + instance_builder.store_builder().max_memory_size(1_000_000); + + instance_builder + .factor_builders() + .wasi + .as_mut() + .unwrap() + .args(["foo"]); + + let (_instance, _store) = instance_builder.instantiate(()).await?; + Ok(()) + } + + struct DummyComponentLoader; + + impl ComponentLoader for DummyComponentLoader { + fn load_component( + &mut self, + engine: &spin_core::wasmtime::Engine, + _component: &AppComponent, + ) -> anyhow::Result { + Component::new(engine, "(component)") + } + } +} From fd54685a670aee51820b71cf30a7177395124e0e Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 30 Jul 2024 14:26:03 -0400 Subject: [PATCH 105/195] factors: Add SpinFilesMounter Signed-off-by: Lann Martin --- crates/factor-wasi/Cargo.toml | 1 + crates/factor-wasi/src/lib.rs | 3 ++- crates/factor-wasi/src/spin.rs | 48 ++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 crates/factor-wasi/src/spin.rs diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index 6fb2dfc8e..b35bfc04e 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -7,6 +7,7 @@ edition = { workspace = true } [dependencies] async-trait = "0.1" cap-primitives = "3.0.0" +spin-common = { path = "../common" } spin-factors = { path = "../factors" } tokio = { version = "1" } wasmtime = { workspace = true } diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index 2579b15e6..b7cc8a90f 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,3 +1,4 @@ +pub mod spin; mod wasi_2023_10_18; mod wasi_2023_11_10; @@ -122,7 +123,7 @@ impl Factor for WasiFactor { } } -pub trait FilesMounter { +pub trait FilesMounter: Send + Sync { fn mount_files( &self, app_component: &AppComponent, diff --git a/crates/factor-wasi/src/spin.rs b/crates/factor-wasi/src/spin.rs new file mode 100644 index 000000000..25de63b5e --- /dev/null +++ b/crates/factor-wasi/src/spin.rs @@ -0,0 +1,48 @@ +use std::path::PathBuf; + +use spin_common::{ui::quoted_path, url::parse_file_url}; +use spin_factors::anyhow::{ensure, Context}; + +use crate::FilesMounter; + +pub struct SpinFilesMounter { + working_dir: PathBuf, + allow_transient_writes: bool, +} + +impl SpinFilesMounter { + pub fn new(working_dir: impl Into, allow_transient_writes: bool) -> Self { + Self { + working_dir: working_dir.into(), + allow_transient_writes, + } + } +} + +impl FilesMounter for SpinFilesMounter { + fn mount_files( + &self, + app_component: &spin_factors::AppComponent, + mut ctx: crate::MountFilesContext, + ) -> spin_factors::anyhow::Result<()> { + for content_dir in app_component.files() { + let source_uri = content_dir + .content + .source + .as_deref() + .with_context(|| format!("Missing 'source' on files mount {content_dir:?}"))?; + let source_path = self.working_dir.join(parse_file_url(source_uri)?); + ensure!( + source_path.is_dir(), + "SpinFilesMounter only supports directory mounts; {} is not a directory", + quoted_path(&source_path), + ); + let guest_path = &content_dir.path; + let guest_path = guest_path + .to_str() + .with_context(|| format!("guest path {guest_path:?} not valid UTF-8"))?; + ctx.preopened_dir(source_path, guest_path, self.allow_transient_writes)?; + } + Ok(()) + } +} From ee56891a2509e4f2bca91e1574bef082bec2cc67 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 11:30:06 -0400 Subject: [PATCH 106/195] core: Add Store data/_mut methods Signed-off-by: Lann Martin --- crates/core/src/store.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/core/src/store.rs b/crates/core/src/store.rs index 7ad7168b5..9afbd3cfd 100644 --- a/crates/core/src/store.rs +++ b/crates/core/src/store.rs @@ -38,6 +38,16 @@ impl Store { }; self.inner.set_epoch_deadline(ticks); } + + /// Provides access to the inner [`wasmtime::Store`]'s data. + pub fn data(&self) -> &T { + self.inner.data() + } + + /// Provides access to the inner [`wasmtime::Store`]'s data. + pub fn data_mut(&mut self) -> &mut T { + self.inner.data_mut() + } } impl AsRef> for Store { From 1a294dd595eb40e36639c0d443dbcc03c6105822 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 11:30:27 -0400 Subject: [PATCH 107/195] app: Add App::trigger_configs Signed-off-by: Lann Martin --- crates/app/src/lib.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 22d97ceba..e3aa54ac3 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -101,7 +101,7 @@ impl App { } /// Returns the trigger metadata for a specific trigger type. - pub fn get_trigger_metadata<'this, T: Deserialize<'this> + Default>( + pub fn get_trigger_metadata<'this, T: Deserialize<'this>>( &'this self, trigger_type: &str, ) -> Result> { @@ -140,6 +140,20 @@ impl App { .filter(move |trigger| trigger.locked.trigger_type == trigger_type) } + /// Returns an iterator of trigger IDs and deserialized trigger configs for + /// the given `trigger_type`. + pub fn trigger_configs<'a, T: Deserialize<'a>>( + &'a self, + trigger_type: &'a str, + ) -> Result> { + self.triggers_with_type(trigger_type) + .map(|trigger| { + let config = trigger.typed_config::()?; + Ok((trigger.id(), config)) + }) + .collect::>>() + } + /// Checks that the application does not have any host requirements /// outside the supported set. The error case returns a comma-separated /// list of unmet requirements. @@ -215,12 +229,12 @@ pub struct AppTrigger<'a> { impl<'a> AppTrigger<'a> { /// Returns this trigger's app-unique ID. - pub fn id(&self) -> &str { + pub fn id(&self) -> &'a str { &self.locked.id } /// Returns the Trigger's type. - pub fn trigger_type(&self) -> &str { + pub fn trigger_type(&self) -> &'a str { &self.locked.trigger_type } From 4eb86ebefabb322c0400a2b3ebc54cbf19d1b9af Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 11:33:42 -0400 Subject: [PATCH 108/195] factors: Use getters for RuntimeFactors::InstanceBuilder fields Signed-off-by: Lann Martin --- crates/core/tests/integration_test.rs | 3 +-- crates/factors-derive/src/lib.rs | 11 ++++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/core/tests/integration_test.rs b/crates/core/tests/integration_test.rs index 8db91940d..ad47912b4 100644 --- a/crates/core/tests/integration_test.rs +++ b/crates/core/tests/integration_test.rs @@ -140,8 +140,7 @@ async fn run_test( let app = App::new("test-app", locked); let configured_app = factors.configure_app(app, Default::default())?; let mut builders = factors.prepare(&configured_app, "test-component")?; - // FIXME: it is unfortunate that we have to unwrap here... - builders.wasi.as_mut().unwrap().args(args); + builders.wasi().args(args); let instance_state = factors.build_instance_state(builders)?; let state = TestState { core: State::default(), diff --git a/crates/factors-derive/src/lib.rs b/crates/factors-derive/src/lib.rs index 921196c8b..df9792448 100644 --- a/crates/factors-derive/src/lib.rs +++ b/crates/factors-derive/src/lib.rs @@ -207,7 +207,16 @@ fn expand_factors(input: &DeriveInput) -> syn::Result { #vis struct #builders_name { #( - pub #factor_names: Option<<#factor_types as #Factor>::InstanceBuilder>, + #factor_names: Option<<#factor_types as #Factor>::InstanceBuilder>, + )* + } + + #[allow(dead_code)] + impl #builders_name { + #( + pub fn #factor_names(&mut self) -> &mut <#factor_types as #Factor>::InstanceBuilder { + self.#factor_names.as_mut().unwrap() + } )* } From 29f4f1ee4bb0aefa843154a57a67df1c61f7b26b Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 11:35:07 -0400 Subject: [PATCH 109/195] factors-executor: Add intermediate FactorsExecutorApp type Signed-off-by: Lann Martin --- crates/factors-executor/src/lib.rs | 186 ++++++++++++++++++++--------- 1 file changed, 132 insertions(+), 54 deletions(-) diff --git a/crates/factors-executor/src/lib.rs b/crates/factors-executor/src/lib.rs index c624a02f0..2d1c00330 100644 --- a/crates/factors-executor/src/lib.rs +++ b/crates/factors-executor/src/lib.rs @@ -10,85 +10,154 @@ use spin_factors::{AsInstanceState, ConfiguredApp, RuntimeFactors, RuntimeFactor /// `Factors` is the executor's [`RuntimeFactors`]. `ExecutorInstanceState` /// holds any other per-instance state needed by the caller. pub struct FactorsExecutor { - factors: T, core_engine: spin_core::Engine>, - configured_app: ConfiguredApp, - // Maps component IDs -> InstancePres - component_instance_pres: HashMap>, + factors: T, + hooks: Vec>>, } -type InstancePre = - spin_core::InstancePre::InstanceState, U>>; - impl FactorsExecutor { /// Constructs a new executor. pub fn new( - core_config: &spin_core::Config, + mut core_engine_builder: spin_core::EngineBuilder< + InstanceState<::InstanceState, U>, + >, mut factors: T, - app: App, - mut component_loader: impl ComponentLoader, - runtime_config: T::RuntimeConfig, ) -> anyhow::Result { - let core_engine = { - let mut builder = - spin_core::Engine::builder(core_config).context("failed to initialize engine")?; - factors - .init(builder.linker()) - .context("failed to initialize factors")?; - builder.build() - }; + factors + .init(core_engine_builder.linker()) + .context("failed to initialize factors")?; + Ok(Self { + factors, + core_engine: core_engine_builder.build(), + hooks: Default::default(), + }) + } - let configured_app = factors + /// Adds the given [`ExecutorHooks`] to this executor. + /// + /// Hooks are run in the order they are added. + pub fn add_hooks(&mut self, hooks: impl ExecutorHooks + 'static) { + self.hooks.push(Box::new(hooks)); + } + + /// Loads a [`FactorsApp`] with this executor. + pub fn load_app( + mut self, + app: App, + runtime_config: T::RuntimeConfig, + mut component_loader: impl ComponentLoader, + ) -> anyhow::Result> { + let configured_app = self + .factors .configure_app(app, runtime_config) .context("failed to configure app")?; + for hooks in &mut self.hooks { + hooks.configure_app(&configured_app)?; + } + let component_instance_pres = configured_app .app() .components() .map(|app_component| { let component = - component_loader.load_component(core_engine.as_ref(), &app_component)?; - let instance_pre = core_engine.instantiate_pre(&component)?; + component_loader.load_component(self.core_engine.as_ref(), &app_component)?; + let instance_pre = self.core_engine.instantiate_pre(&component)?; Ok((app_component.id().to_string(), instance_pre)) }) .collect::>>()?; - Ok(Self { - factors, - core_engine, + Ok(FactorsExecutorApp { + executor: self, configured_app, component_instance_pres, }) } +} + +pub trait ExecutorHooks: Send + Sync { + /// Configure app hooks run immediately after [`RuntimeFactors::configure_app`]. + fn configure_app(&mut self, configured_app: &ConfiguredApp) -> anyhow::Result<()> { + let _ = configured_app; + Ok(()) + } + + /// Prepare instance hooks run immediately before [`FactorsExecutor::prepare`] returns. + fn prepare_instance(&self, builder: &mut FactorsInstanceBuilder) -> anyhow::Result<()> { + let _ = builder; + Ok(()) + } +} + +/// A ComponentLoader is responsible for loading Wasmtime [`Component`]s. +pub trait ComponentLoader { + /// Loads a [`Component`] for the given [`AppComponent`]. + fn load_component( + &mut self, + engine: &spin_core::wasmtime::Engine, + component: &AppComponent, + ) -> anyhow::Result; +} + +type InstancePre = + spin_core::InstancePre::InstanceState, U>>; + +/// A FactorsExecutorApp represents a loaded Spin app, ready for instantiation. +pub struct FactorsExecutorApp { + executor: FactorsExecutor, + configured_app: ConfiguredApp, + // Maps component IDs -> InstancePres + component_instance_pres: HashMap>, +} + +impl FactorsExecutorApp { + pub fn engine(&self) -> &spin_core::Engine> { + &self.executor.core_engine + } + + pub fn app(&self) -> &App { + self.configured_app.app() + } + + pub fn get_component(&self, component_id: &str) -> anyhow::Result<&Component> { + let instance_pre = self + .component_instance_pres + .get(component_id) + .with_context(|| format!("no such component {component_id:?}"))?; + Ok(instance_pre.component()) + } /// Returns an instance builder for the given component ID. - pub fn prepare(&mut self, component_id: &str) -> anyhow::Result> { + pub fn prepare(&self, component_id: &str) -> anyhow::Result> { let app_component = self .configured_app .app() .get_component(component_id) .with_context(|| format!("no such component {component_id:?}"))?; + let instance_pre = self.component_instance_pres.get(component_id).unwrap(); - let factor_builders = self.factors.prepare(&self.configured_app, component_id)?; - let store_builder = self.core_engine.store_builder(); - Ok(FactorsInstanceBuilder { + + let factor_builders = self + .executor + .factors + .prepare(&self.configured_app, component_id)?; + + let store_builder = self.executor.core_engine.store_builder(); + + let mut builder = FactorsInstanceBuilder { store_builder, factor_builders, instance_pre, app_component, - factors: &self.factors, - }) - } -} + factors: &self.executor.factors, + }; -/// A ComponentLoader is responsible for loading Wasmtime [`Component`]s. -pub trait ComponentLoader { - /// Loads a [`Component`] for the given [`AppComponent`]. - fn load_component( - &mut self, - engine: &spin_core::wasmtime::Engine, - component: &AppComponent, - ) -> anyhow::Result; + for hooks in &self.executor.hooks { + hooks.prepare_instance(&mut builder)?; + } + + Ok(builder) + } } /// A FactorsInstanceBuilder manages the instantiation of a Spin component @@ -101,7 +170,7 @@ pub struct FactorsInstanceBuilder<'a, T: RuntimeFactors, U> { factors: &'a T, } -impl<'a, T: RuntimeFactors, U: Send> FactorsInstanceBuilder<'a, T, U> { +impl<'a, T: RuntimeFactors, U> FactorsInstanceBuilder<'a, T, U> { /// Returns the app component for the instance. pub fn app_component(&self) -> &AppComponent { &self.app_component @@ -116,7 +185,9 @@ impl<'a, T: RuntimeFactors, U: Send> FactorsInstanceBuilder<'a, T, U> { pub fn factor_builders(&mut self) -> &mut T::InstanceBuilders { &mut self.factor_builders } +} +impl<'a, T: RuntimeFactors, U: Send> FactorsInstanceBuilder<'a, T, U> { /// Instantiates the instance with the given executor instance state pub async fn instantiate( self, @@ -137,14 +208,24 @@ impl<'a, T: RuntimeFactors, U: Send> FactorsInstanceBuilder<'a, T, U> { } /// InstanceState is the [`spin_core::Store`] `data` for an instance. -pub struct InstanceState { +pub struct InstanceState { core: spin_core::State, - factors: FactorsState, - executor: ExecutorInstanceState, + factors: T, + executor: U, } impl InstanceState { - /// Provides access to the `ExecutorInstanceState`. + /// Provides access to the [`spin_core::State`]. + pub fn core_state(&self) -> &spin_core::State { + &self.core + } + + /// Provides access to the [`RuntimeFactors::InstanceState`]. + pub fn factors_instance_state(&mut self) -> &mut T { + &mut self.factors + } + + /// Provides access to the `Self::ExecutorInstanceState`. pub fn executor_instance_state(&mut self) -> &mut U { &mut self.executor } @@ -184,15 +265,12 @@ mod tests { let locked = env.build_locked_app().await?; let app = App::new("test-app", locked); - let mut executor = FactorsExecutor::new( - &Default::default(), - env.factors, - app, - DummyComponentLoader, - Default::default(), - )?; + let engine_builder = spin_core::Engine::builder(&Default::default())?; + let executor = FactorsExecutor::new(engine_builder, env.factors)?; + + let factors_app = executor.load_app(app, Default::default(), DummyComponentLoader)?; - let mut instance_builder = executor.prepare("empty")?; + let mut instance_builder = factors_app.prepare("empty")?; assert_eq!(instance_builder.app_component().id(), "empty"); From 195f64fe257be3f422f4b005967ee70486877f60 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 11:35:40 -0400 Subject: [PATCH 110/195] factors: Fix spin_factor_outbound_http exports Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/lib.rs | 4 ++-- crates/factor-outbound-http/src/wasi_2023_10_18.rs | 13 ++++++++++++- crates/factor-outbound-http/src/wasi_2023_11_10.rs | 13 ++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index e17babf2e..ed01d137e 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -1,7 +1,7 @@ mod spin; mod wasi; -mod wasi_2023_10_18; -mod wasi_2023_11_10; +pub mod wasi_2023_10_18; +pub mod wasi_2023_11_10; use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; use spin_factors::{ diff --git a/crates/factor-outbound-http/src/wasi_2023_10_18.rs b/crates/factor-outbound-http/src/wasi_2023_10_18.rs index 92a9cd4cc..891853ec6 100644 --- a/crates/factor-outbound-http/src/wasi_2023_10_18.rs +++ b/crates/factor-outbound-http/src/wasi_2023_10_18.rs @@ -17,6 +17,10 @@ mod bindings { interfaces: r#" include wasi:http/proxy@0.2.0-rc-2023-10-18; "#, + async: { + // Only need async exports + only_imports: [], + }, with: { "wasi:io/poll/pollable": latest::io::poll::Pollable, "wasi:io/streams/input-stream": latest::io::streams::InputStream, @@ -41,6 +45,12 @@ mod wasi { pub use super::bindings::wasi::{http0_2_0_rc_2023_10_18 as http, io0_2_0_rc_2023_10_18 as io}; } +pub mod exports { + pub mod wasi { + pub use super::super::bindings::exports::wasi::http0_2_0_rc_2023_10_18 as http; + } +} + use wasi::http::types::{ Error as HttpError, Fields, FutureIncomingResponse, FutureTrailers, Headers, IncomingBody, IncomingRequest, IncomingResponse, Method, OutgoingBody, OutgoingRequest, OutgoingResponse, @@ -51,8 +61,9 @@ use wasi::io::streams::{InputStream, OutputStream}; use crate::wasi::WasiHttpImplInner; -pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> +pub(crate) fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where + T: Send, F: Fn(&mut T) -> WasiHttpImpl + Send + Sync + Copy + 'static, { wasi::http::types::add_to_linker_get_host(linker, closure)?; diff --git a/crates/factor-outbound-http/src/wasi_2023_11_10.rs b/crates/factor-outbound-http/src/wasi_2023_11_10.rs index 439003d15..0a878cfcf 100644 --- a/crates/factor-outbound-http/src/wasi_2023_11_10.rs +++ b/crates/factor-outbound-http/src/wasi_2023_11_10.rs @@ -20,6 +20,10 @@ mod bindings { interfaces: r#" include wasi:http/proxy@0.2.0-rc-2023-11-10; "#, + async: { + // Only need async exports + only_imports: [], + }, with: { "wasi:io/poll/pollable": latest::io::poll::Pollable, "wasi:io/streams/input-stream": latest::io::streams::InputStream, @@ -45,6 +49,12 @@ mod wasi { pub use super::bindings::wasi::{http0_2_0_rc_2023_11_10 as http, io0_2_0_rc_2023_11_10 as io}; } +pub mod exports { + pub mod wasi { + pub use super::super::bindings::exports::wasi::http0_2_0_rc_2023_11_10 as http; + } +} + use wasi::http::types::{ DnsErrorPayload, ErrorCode as HttpErrorCode, FieldSizePayload, Fields, FutureIncomingResponse, FutureTrailers, HeaderError, Headers, IncomingBody, IncomingRequest, IncomingResponse, Method, @@ -56,8 +66,9 @@ use wasi::io::streams::{Error as IoError, InputStream, OutputStream}; use crate::wasi::WasiHttpImplInner; -pub fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> +pub(crate) fn add_to_linker(linker: &mut Linker, closure: F) -> Result<()> where + T: Send, F: Fn(&mut T) -> WasiHttpImpl + Send + Sync + Copy + 'static, { wasi::http::types::add_to_linker_get_host(linker, closure)?; From c33c153d4c10a6d509d70a6333ec49c1292dbb41 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 11:36:21 -0400 Subject: [PATCH 111/195] Fix a couple of lints Signed-off-by: Lann Martin --- crates/factor-variables/src/spin_cli/mod.rs | 2 +- crates/http/src/trigger.rs | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index 49ea1261c..f03a12472 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -17,7 +17,7 @@ use crate::runtime_config::RuntimeConfig; /// Resolves a runtime configuration for the variables factor from a TOML table. pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result { // Always include the environment variable provider. - let mut providers = vec![Box::new(EnvVariablesProvider::default()) as _]; + let mut providers = vec![Box::::default() as _]; let Some(array) = table.get("variable_provider") else { return Ok(RuntimeConfig { providers }); }; diff --git a/crates/http/src/trigger.rs b/crates/http/src/trigger.rs index ca53b3354..37a030ed0 100644 --- a/crates/http/src/trigger.rs +++ b/crates/http/src/trigger.rs @@ -1,8 +1,4 @@ use serde::{Deserialize, Serialize}; -use spin_locked_app::MetadataKey; - -/// Http trigger metadata key -pub const METADATA_KEY: MetadataKey = MetadataKey::new("trigger"); #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] From 982de7112792ce9962fd5f5fbf6ee21a3746673a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 11:37:08 -0400 Subject: [PATCH 112/195] Add spin-trigger2 and spin-trigger-http2 crates These temporary copies to ease refactoring. Signed-off-by: Lann Martin --- Cargo.lock | 76 +++- crates/trigger-http2/Cargo.toml | 48 ++ crates/trigger-http2/src/handler.rs | 411 ++++++++++++++++++ crates/trigger-http2/src/instrument.rs | 93 ++++ crates/trigger-http2/src/lib.rs | 394 +++++++++++++++++ crates/trigger-http2/src/server.rs | 386 ++++++++++++++++ crates/trigger-http2/src/tls.rs | 134 ++++++ crates/trigger-http2/src/wagi.rs | 135 ++++++ .../trigger-http2/testdata/invalid-cert.pem | 20 + .../testdata/invalid-private-key.pem | 5 + crates/trigger-http2/testdata/valid-cert.pem | 21 + .../testdata/valid-private-key.pem | 5 + crates/trigger2/Cargo.toml | 30 ++ crates/trigger2/src/cli.rs | 317 ++++++++++++++ crates/trigger2/src/cli/launch_metadata.rs | 83 ++++ crates/trigger2/src/factors.rs | 18 + crates/trigger2/src/lib.rs | 66 +++ crates/trigger2/src/stdio.rs | 331 ++++++++++++++ run-factors-tests.sh | 2 +- 19 files changed, 2573 insertions(+), 2 deletions(-) create mode 100644 crates/trigger-http2/Cargo.toml create mode 100644 crates/trigger-http2/src/handler.rs create mode 100644 crates/trigger-http2/src/instrument.rs create mode 100644 crates/trigger-http2/src/lib.rs create mode 100644 crates/trigger-http2/src/server.rs create mode 100644 crates/trigger-http2/src/tls.rs create mode 100644 crates/trigger-http2/src/wagi.rs create mode 100644 crates/trigger-http2/testdata/invalid-cert.pem create mode 100644 crates/trigger-http2/testdata/invalid-private-key.pem create mode 100644 crates/trigger-http2/testdata/valid-cert.pem create mode 100644 crates/trigger-http2/testdata/valid-private-key.pem create mode 100644 crates/trigger2/Cargo.toml create mode 100644 crates/trigger2/src/cli.rs create mode 100644 crates/trigger2/src/cli/launch_metadata.rs create mode 100644 crates/trigger2/src/factors.rs create mode 100644 crates/trigger2/src/lib.rs create mode 100644 crates/trigger2/src/stdio.rs diff --git a/Cargo.lock b/Cargo.lock index cffb1d41d..6580b5082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6836,6 +6836,16 @@ dependencies = [ "regex", ] +[[package]] +name = "sanitize-filename" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ed72fbaf78e6f2d41744923916966c4fbe3d7c74e3037a8ee482f1115572603" +dependencies = [ + "lazy_static 1.4.0", + "regex", +] + [[package]] name = "saturating" version = "0.1.0" @@ -7731,6 +7741,7 @@ version = "2.7.0-pre0" dependencies = [ "async-trait", "cap-primitives 3.0.0", + "spin-common", "spin-factors", "spin-factors-test", "tokio", @@ -8217,7 +8228,7 @@ dependencies = [ "outbound-redis", "rustls-pemfile 2.1.2", "rustls-pki-types", - "sanitize-filename", + "sanitize-filename 0.4.0", "serde 1.0.197", "serde_json", "spin-app", @@ -8297,6 +8308,47 @@ dependencies = [ "webpki-roots 0.26.1", ] +[[package]] +name = "spin-trigger-http2" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "clap 3.2.25", + "futures", + "futures-util", + "http 1.1.0", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "indexmap 1.9.3", + "percent-encoding", + "rustls 0.22.4", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "serde 1.0.197", + "serde_json", + "spin-app", + "spin-core", + "spin-factor-outbound-http", + "spin-factor-wasi", + "spin-http", + "spin-outbound-networking", + "spin-telemetry", + "spin-trigger2", + "spin-world", + "terminal", + "tls-listener", + "tokio", + "tokio-rustls 0.25.0", + "tracing", + "url", + "wasmtime", + "wasmtime-wasi", + "wasmtime-wasi-http", + "webpki-roots 0.26.1", +] + [[package]] name = "spin-trigger-redis" version = "2.7.0-pre0" @@ -8318,6 +8370,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "spin-trigger2" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "clap 3.2.25", + "ctrlc", + "futures", + "sanitize-filename 0.5.0", + "serde 1.0.197", + "serde_json", + "spin-app", + "spin-common", + "spin-core", + "spin-factor-wasi", + "spin-factors", + "spin-factors-executor", + "spin-telemetry", + "tokio", + "tracing", +] + [[package]] name = "spin-variables" version = "2.7.0-pre0" diff --git a/crates/trigger-http2/Cargo.toml b/crates/trigger-http2/Cargo.toml new file mode 100644 index 000000000..0abb81436 --- /dev/null +++ b/crates/trigger-http2/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "spin-trigger-http2" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[lib] +doctest = false + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +clap = "3" +futures = "0.3" +futures-util = "0.3.8" +http = "1.0.0" +hyper = { workspace = true } +hyper-util = { version = "0.1.2", features = ["tokio"] } +http-body-util = { workspace = true } +indexmap = "1" +percent-encoding = "2" +rustls = { version = "0.22.4" } +rustls-pemfile = "2.1.2" +rustls-pki-types = "1.7" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" +spin-app = { path = "../app" } +spin-core = { path = "../core" } +spin-factor-outbound-http = { path = "../factor-outbound-http" } +spin-factor-wasi = { path = "../factor-wasi" } +spin-http = { path = "../http" } +spin-outbound-networking = { path = "../outbound-networking" } +spin-telemetry = { path = "../telemetry" } +spin-trigger2 = { path = "../trigger2" } +spin-world = { path = "../world" } +terminal = { path = "../terminal" } +tls-listener = { version = "0.10.0", features = ["rustls"] } +tokio = { version = "1.23", features = ["full"] } +tokio-rustls = { version = "0.25.0" } +url = "2.4.1" +tracing = { workspace = true } +wasmtime = { workspace = true } +wasmtime-wasi = { workspace = true } +wasmtime-wasi-http = { workspace = true } +webpki-roots = { version = "0.26.0" } + +[lints] +workspace = true diff --git a/crates/trigger-http2/src/handler.rs b/crates/trigger-http2/src/handler.rs new file mode 100644 index 000000000..1631c5e16 --- /dev/null +++ b/crates/trigger-http2/src/handler.rs @@ -0,0 +1,411 @@ +use std::{net::SocketAddr, str, str::FromStr}; + +use anyhow::{anyhow, Context, Result}; +use futures::TryFutureExt; +use http::{HeaderName, HeaderValue}; +use http_body_util::BodyExt; +use hyper::{Request, Response}; +use spin_core::{Component, Instance}; +use spin_factor_outbound_http::wasi_2023_10_18::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_10_18; +use spin_factor_outbound_http::wasi_2023_11_10::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_11_10; +use spin_http::body; +use spin_http::routes::RouteMatch; +use spin_world::v1::http_types; +use tokio::{sync::oneshot, task}; +use tracing::{instrument, Instrument, Level}; +use wasmtime_wasi_http::{body::HyperIncomingBody as Body, proxy::Proxy, WasiHttpView}; + +use crate::{server::HttpExecutor, Store, TriggerInstanceBuilder}; + +#[derive(Clone)] +pub struct HttpHandlerExecutor { + pub handler_type: HandlerType, +} + +impl HttpExecutor for HttpHandlerExecutor { + #[instrument(name = "spin_trigger_http.execute_wasm", skip_all, err(level = Level::INFO), fields(otel.name = format!("execute_wasm_component {}", route_match.component_id())))] + async fn execute( + &self, + instance_builder: TriggerInstanceBuilder<'_>, + route_match: &RouteMatch, + req: Request, + client_addr: SocketAddr, + ) -> Result> { + let component_id = route_match.component_id(); + + tracing::trace!("Executing request using the Spin executor for component {component_id}"); + + let (instance, store) = instance_builder.instantiate(()).await?; + + let resp = match self.handler_type { + HandlerType::Spin => self + .execute_spin(store, instance, route_match, req, client_addr) + .await + .map_err(contextualise_err)?, + _ => { + self.execute_wasi(store, instance, route_match, req, client_addr) + .await? + } + }; + + tracing::info!( + "Request finished, sending response with status code {}", + resp.status() + ); + Ok(resp) + } +} + +impl HttpHandlerExecutor { + pub async fn execute_spin( + &self, + mut store: Store, + instance: Instance, + route_match: &RouteMatch, + req: Request, + client_addr: SocketAddr, + ) -> Result> { + let headers = Self::headers(&req, route_match, client_addr)?; + let func = instance + .exports(&mut store) + .instance("fermyon:spin/inbound-http") + // Safe since we have already checked that this instance exists + .expect("no fermyon:spin/inbound-http found") + .typed_func::<(http_types::Request,), (http_types::Response,)>("handle-request")?; + + let (parts, body) = req.into_parts(); + let bytes = body.collect().await?.to_bytes().to_vec(); + + let method = if let Some(method) = Self::method(&parts.method) { + method + } else { + return Ok(Response::builder() + .status(http::StatusCode::METHOD_NOT_ALLOWED) + .body(body::empty())?); + }; + + // Preparing to remove the params field. We are leaving it in place for now + // to avoid breaking the ABI, but no longer pass or accept values in it. + // https://github.com/fermyon/spin/issues/663 + let params = vec![]; + + let uri = match parts.uri.path_and_query() { + Some(u) => u.to_string(), + None => parts.uri.to_string(), + }; + + let req = http_types::Request { + method, + uri, + headers, + params, + body: Some(bytes), + }; + + let (resp,) = func.call_async(&mut store, (req,)).await?; + + if resp.status < 100 || resp.status > 600 { + tracing::error!("malformed HTTP status code"); + return Ok(Response::builder() + .status(http::StatusCode::INTERNAL_SERVER_ERROR) + .body(body::empty())?); + }; + + let mut response = http::Response::builder().status(resp.status); + if let Some(headers) = response.headers_mut() { + Self::append_headers(headers, resp.headers)?; + } + + let body = match resp.body { + Some(b) => body::full(b.into()), + None => body::empty(), + }; + + Ok(response.body(body)?) + } + + fn method(m: &http::Method) -> Option { + Some(match *m { + http::Method::GET => http_types::Method::Get, + http::Method::POST => http_types::Method::Post, + http::Method::PUT => http_types::Method::Put, + http::Method::DELETE => http_types::Method::Delete, + http::Method::PATCH => http_types::Method::Patch, + http::Method::HEAD => http_types::Method::Head, + http::Method::OPTIONS => http_types::Method::Options, + _ => return None, + }) + } + + async fn execute_wasi( + &self, + mut store: Store, + instance: Instance, + route_match: &RouteMatch, + mut req: Request, + client_addr: SocketAddr, + ) -> anyhow::Result> { + let headers = Self::headers(&req, route_match, client_addr)?; + req.headers_mut().clear(); + req.headers_mut() + .extend(headers.into_iter().filter_map(|(n, v)| { + let Ok(name) = n.parse::() else { + return None; + }; + let Ok(value) = HeaderValue::from_bytes(v.as_bytes()) else { + return None; + }; + Some((name, value)) + })); + + let mut wasi_http = spin_factor_outbound_http::OutboundHttpFactor::get_wasi_http_impl( + store.data_mut().factors_instance_state(), + ) + .context("missing OutboundHttpFactor")?; + + let request = wasi_http.new_incoming_request(req)?; + + let (response_tx, response_rx) = oneshot::channel(); + let response = wasi_http.new_response_outparam(response_tx)?; + + drop(wasi_http); + + enum Handler { + Latest(Proxy), + Handler2023_11_10(IncomingHandler2023_11_10), + Handler2023_10_18(IncomingHandler2023_10_18), + } + + let handler = + { + let mut exports = instance.exports(&mut store); + match self.handler_type { + HandlerType::Wasi2023_10_18 => { + let mut instance = exports + .instance(WASI_HTTP_EXPORT_2023_10_18) + .ok_or_else(|| { + anyhow!("export of `{WASI_HTTP_EXPORT_2023_10_18}` not an instance") + })?; + Handler::Handler2023_10_18(IncomingHandler2023_10_18::new(&mut instance)?) + } + HandlerType::Wasi2023_11_10 => { + let mut instance = exports + .instance(WASI_HTTP_EXPORT_2023_11_10) + .ok_or_else(|| { + anyhow!("export of `{WASI_HTTP_EXPORT_2023_11_10}` not an instance") + })?; + Handler::Handler2023_11_10(IncomingHandler2023_11_10::new(&mut instance)?) + } + HandlerType::Wasi0_2 => { + drop(exports); + Handler::Latest(Proxy::new(&mut store, &instance)?) + } + HandlerType::Spin => panic!("should have used execute_spin instead"), + } + }; + + let span = tracing::debug_span!("execute_wasi"); + let handle = task::spawn( + async move { + let result = match handler { + Handler::Latest(proxy) => { + proxy + .wasi_http_incoming_handler() + .call_handle(&mut store, request, response) + .instrument(span) + .await + } + Handler::Handler2023_10_18(handler) => { + handler + .call_handle(&mut store, request, response) + .instrument(span) + .await + } + Handler::Handler2023_11_10(handler) => { + handler + .call_handle(&mut store, request, response) + .instrument(span) + .await + } + }; + + tracing::trace!( + "wasi-http memory consumed: {}", + store.data().core_state().memory_consumed() + ); + + result + } + .in_current_span(), + ); + + match response_rx.await { + Ok(response) => { + task::spawn( + async move { + handle + .await + .context("guest invocation panicked")? + .context("guest invocation failed")?; + + Ok(()) + } + .map_err(|e: anyhow::Error| { + tracing::warn!("component error after response: {e:?}"); + }), + ); + + Ok(response.context("guest failed to produce a response")?) + } + + Err(_) => { + handle + .await + .context("guest invocation panicked")? + .context("guest invocation failed")?; + + Err(anyhow!( + "guest failed to produce a response prior to returning" + )) + } + } + } + + fn headers( + req: &Request, + route_match: &RouteMatch, + client_addr: SocketAddr, + ) -> Result> { + let mut res = Vec::new(); + for (name, value) in req + .headers() + .iter() + .map(|(name, value)| (name.to_string(), std::str::from_utf8(value.as_bytes()))) + { + let value = value?.to_string(); + res.push((name, value)); + } + + let default_host = http::HeaderValue::from_str("localhost")?; + let host = std::str::from_utf8( + req.headers() + .get("host") + .unwrap_or(&default_host) + .as_bytes(), + )?; + + // Set the environment information (path info, base path, etc) as headers. + // In the future, we might want to have this information in a context + // object as opposed to headers. + for (keys, val) in + crate::server::compute_default_headers(req.uri(), host, route_match, client_addr)? + { + res.push((Self::prepare_header_key(&keys[0]), val)); + } + + Ok(res) + } + + fn prepare_header_key(key: &str) -> String { + key.replace('_', "-").to_ascii_lowercase() + } + + fn append_headers(res: &mut http::HeaderMap, src: Option>) -> Result<()> { + if let Some(src) = src { + for (k, v) in src.iter() { + res.insert( + http::header::HeaderName::from_str(k)?, + http::header::HeaderValue::from_str(v)?, + ); + } + }; + + Ok(()) + } +} + +/// Whether this handler uses the custom Spin http handler interface for wasi-http +#[derive(Copy, Clone)] +pub enum HandlerType { + Spin, + Wasi0_2, + Wasi2023_11_10, + Wasi2023_10_18, +} + +const WASI_HTTP_EXPORT_2023_10_18: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-10-18"; +const WASI_HTTP_EXPORT_2023_11_10: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-11-10"; +const WASI_HTTP_EXPORT_0_2_0: &str = "wasi:http/incoming-handler@0.2.0"; + +impl HandlerType { + /// Determine the handler type from the exports of a component + pub fn from_component( + engine: impl AsRef, + component: &Component, + ) -> Result { + let mut handler_ty = None; + + let mut set = |ty: HandlerType| { + if handler_ty.is_none() { + handler_ty = Some(ty); + Ok(()) + } else { + Err(anyhow!( + "component exports multiple different handlers but \ + it's expected to export only one" + )) + } + }; + let ty = component.component_type(); + for (name, _) in ty.exports(engine.as_ref()) { + match name { + WASI_HTTP_EXPORT_2023_10_18 => set(HandlerType::Wasi2023_10_18)?, + WASI_HTTP_EXPORT_2023_11_10 => set(HandlerType::Wasi2023_11_10)?, + WASI_HTTP_EXPORT_0_2_0 => set(HandlerType::Wasi0_2)?, + "fermyon:spin/inbound-http" => set(HandlerType::Spin)?, + _ => {} + } + } + + handler_ty.ok_or_else(|| { + anyhow!( + "Expected component to either export `{WASI_HTTP_EXPORT_2023_10_18}`, \ + `{WASI_HTTP_EXPORT_2023_11_10}`, `{WASI_HTTP_EXPORT_0_2_0}`, \ + or `fermyon:spin/inbound-http` but it exported none of those" + ) + }) + } +} + +fn contextualise_err(e: anyhow::Error) -> anyhow::Error { + if e.to_string() + .contains("failed to find function export `canonical_abi_free`") + { + e.context( + "component is not compatible with Spin executor - should this use the Wagi executor?", + ) + } else { + e + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_spin_header_keys() { + assert_eq!( + HttpHandlerExecutor::prepare_header_key("SPIN_FULL_URL"), + "spin-full-url".to_string() + ); + assert_eq!( + HttpHandlerExecutor::prepare_header_key("SPIN_PATH_INFO"), + "spin-path-info".to_string() + ); + assert_eq!( + HttpHandlerExecutor::prepare_header_key("SPIN_RAW_COMPONENT_ROUTE"), + "spin-raw-component-route".to_string() + ); + } +} diff --git a/crates/trigger-http2/src/instrument.rs b/crates/trigger-http2/src/instrument.rs new file mode 100644 index 000000000..aa5b3e0e0 --- /dev/null +++ b/crates/trigger-http2/src/instrument.rs @@ -0,0 +1,93 @@ +use anyhow::Result; +use http::Response; +use tracing::Level; +use wasmtime_wasi_http::body::HyperIncomingBody; + +/// Create a span for an HTTP request. +macro_rules! http_span { + ($request:tt, $addr:tt) => { + tracing::info_span!( + "spin_trigger_http.handle_http_request", + "otel.kind" = "server", + "http.request.method" = %$request.method(), + "network.peer.address" = %$addr.ip(), + "network.peer.port" = %$addr.port(), + "network.protocol.name" = "http", + "url.path" = $request.uri().path(), + "url.query" = $request.uri().query().unwrap_or(""), + "url.scheme" = $request.uri().scheme_str().unwrap_or(""), + "client.address" = $request.headers().get("x-forwarded-for").and_then(|val| val.to_str().ok()), + // Recorded later + "error.type" = ::tracing::field::Empty, + "http.response.status_code" = ::tracing::field::Empty, + "http.route" = ::tracing::field::Empty, + "otel.name" = ::tracing::field::Empty, + ) + }; +} + +pub(crate) use http_span; + +/// Finish setting attributes on the HTTP span. +pub(crate) fn finalize_http_span( + response: Result>, + method: String, +) -> Result> { + let span = tracing::Span::current(); + match response { + Ok(response) => { + let matched_route = response.extensions().get::(); + // Set otel.name and http.route + if let Some(MatchedRoute { route }) = matched_route { + span.record("http.route", route); + span.record("otel.name", format!("{method} {route}")); + } else { + span.record("otel.name", method); + } + + // Set status code + span.record("http.response.status_code", response.status().as_u16()); + + Ok(response) + } + Err(err) => { + instrument_error(&err); + span.record("http.response.status_code", 500); + span.record("otel.name", method); + Err(err) + } + } +} + +/// Marks the current span as errored. +pub(crate) fn instrument_error(err: &anyhow::Error) { + let span = tracing::Span::current(); + tracing::event!(target:module_path!(), Level::INFO, error = %err); + span.record("error.type", format!("{:?}", err)); +} + +/// MatchedRoute is used as a response extension to track the route that was matched for OTel +/// tracing purposes. +#[derive(Clone)] +pub struct MatchedRoute { + pub route: String, +} + +impl MatchedRoute { + pub fn set_response_extension( + resp: &mut Response, + route: impl Into, + ) { + resp.extensions_mut().insert(MatchedRoute { + route: route.into(), + }); + } + + pub fn with_response_extension( + mut resp: Response, + route: impl Into, + ) -> Response { + Self::set_response_extension(&mut resp, route); + resp + } +} diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs new file mode 100644 index 000000000..f2b9c26b5 --- /dev/null +++ b/crates/trigger-http2/src/lib.rs @@ -0,0 +1,394 @@ +//! Implementation for the Spin HTTP engine. + +mod handler; +mod instrument; +mod server; +mod tls; +mod wagi; + +use std::{ + collections::HashMap, + error::Error, + net::{Ipv4Addr, SocketAddr, ToSocketAddrs}, + path::PathBuf, + sync::Arc, +}; + +use anyhow::{bail, Context}; +use clap::Args; +use serde::Deserialize; +use spin_app::App; +use spin_http::{config::HttpTriggerConfig, routes::Router}; +use spin_trigger2::Trigger; +use tokio::net::TcpListener; +use wasmtime_wasi_http::bindings::wasi::http::types::ErrorCode; + +use server::HttpServer; + +pub use tls::TlsConfig; + +pub(crate) type TriggerApp = spin_trigger2::TriggerApp; +pub(crate) type TriggerInstanceBuilder<'a> = spin_trigger2::TriggerInstanceBuilder<'a, HttpTrigger>; +pub(crate) type Store = spin_trigger2::Store; + +#[derive(Args)] +pub struct CliArgs { + /// IP address and port to listen on + #[clap(long = "listen", env = "SPIN_HTTP_LISTEN_ADDR", default_value = "127.0.0.1:3000", value_parser = parse_listen_addr)] + pub address: SocketAddr, + + /// The path to the certificate to use for https, if this is not set, normal http will be used. The cert should be in PEM format + #[clap(long, env = "SPIN_TLS_CERT", requires = "tls-key")] + pub tls_cert: Option, + + /// The path to the certificate key to use for https, if this is not set, normal http will be used. The key should be in PKCS#8 format + #[clap(long, env = "SPIN_TLS_KEY", requires = "tls-cert")] + pub tls_key: Option, +} + +impl CliArgs { + fn into_tls_config(self) -> Option { + match (self.tls_cert, self.tls_key) { + (Some(cert_path), Some(key_path)) => Some(TlsConfig { + cert_path, + key_path, + }), + (None, None) => None, + _ => unreachable!(), + } + } +} + +pub(crate) type InstanceState = (); + +/// The Spin HTTP trigger. +pub struct HttpTrigger { + listen_addr: SocketAddr, + tls_config: Option, + router: Router, + // Component ID -> component trigger config + component_trigger_configs: HashMap, +} + +impl Trigger for HttpTrigger { + const TYPE: &'static str = "http"; + + type CliArgs = CliArgs; + type InstanceState = InstanceState; + + fn new(cli_args: Self::CliArgs, app: &spin_app::App) -> anyhow::Result { + Self::validate_app(app)?; + + let component_trigger_configs = HashMap::from_iter( + app.trigger_configs::("http")? + .into_iter() + .map(|(_, config)| (config.component.clone(), config)), + ); + + let component_routes = component_trigger_configs + .iter() + .map(|(component_id, config)| (component_id.as_str(), &config.route)); + let (router, duplicate_routes) = Router::build("/", component_routes)?; + if !duplicate_routes.is_empty() { + tracing::error!( + "The following component routes are duplicates and will never be used:" + ); + for dup in &duplicate_routes { + tracing::error!( + " {}: {} (duplicate of {})", + dup.replaced_id, + dup.route(), + dup.effective_id, + ); + } + } + tracing::trace!( + "Constructed router: {:?}", + router.routes().collect::>() + ); + + Ok(Self { + listen_addr: cli_args.address, + tls_config: cli_args.into_tls_config(), + router, + component_trigger_configs, + }) + } + + async fn run(self, trigger_app: TriggerApp) -> anyhow::Result<()> { + let Self { + listen_addr, + tls_config, + router, + component_trigger_configs, + } = self; + + let listener = TcpListener::bind(listen_addr) + .await + .with_context(|| format!("Unable to listen on {listen_addr}"))?; + + let server = Arc::new(HttpServer::new( + listen_addr, + trigger_app, + router, + component_trigger_configs, + )?); + + if let Some(tls_config) = tls_config { + server.serve_tls(listener, tls_config).await? + } else { + server.serve(listener).await? + }; + + Ok(()) + } + + fn supported_host_requirements() -> Vec<&'static str> { + vec![spin_app::locked::SERVICE_CHAINING_KEY] + } +} + +impl HttpTrigger { + fn validate_app(app: &App) -> anyhow::Result<()> { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct TriggerMetadata { + base: Option, + } + if let Some(TriggerMetadata { base: Some(base) }) = app.get_trigger_metadata("http")? { + if base == "/" { + tracing::warn!("This application has the deprecated trigger 'base' set to the default value '/'. This may be an error in the future!"); + } else { + bail!("This application is using the deprecated trigger 'base' field. The base must be prepended to each [[trigger.http]]'s 'route'.") + } + } + Ok(()) + } +} + +fn parse_listen_addr(addr: &str) -> anyhow::Result { + let addrs: Vec = addr.to_socket_addrs()?.collect(); + // Prefer 127.0.0.1 over e.g. [::1] because CHANGE IS HARD + if let Some(addr) = addrs + .iter() + .find(|addr| addr.is_ipv4() && addr.ip() == Ipv4Addr::LOCALHOST) + { + return Ok(*addr); + } + // Otherwise, take the first addr (OS preference) + addrs.into_iter().next().context("couldn't resolve address") +} + +#[derive(Debug, PartialEq)] +enum NotFoundRouteKind { + Normal(String), + WellKnown, +} + +/// Translate a [`hyper::Error`] to a wasi-http `ErrorCode` in the context of a request. +pub fn hyper_request_error(err: hyper::Error) -> ErrorCode { + // If there's a source, we might be able to extract a wasi-http error from it. + if let Some(cause) = err.source() { + if let Some(err) = cause.downcast_ref::() { + return err.clone(); + } + } + + tracing::warn!("hyper request error: {err:?}"); + + ErrorCode::HttpProtocolError +} + +pub fn dns_error(rcode: String, info_code: u16) -> ErrorCode { + ErrorCode::DnsError(wasmtime_wasi_http::bindings::http::types::DnsErrorPayload { + rcode: Some(rcode), + info_code: Some(info_code), + }) +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use http::Request; + + use super::{server::*, *}; + + #[test] + fn test_default_headers() -> Result<()> { + let scheme = "https"; + let host = "fermyon.dev"; + let trigger_route = "/foo/..."; + let component_path = "/foo"; + let path_info = "/bar"; + let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); + + let req_uri = format!( + "{}://{}{}{}?key1=value1&key2=value2", + scheme, host, component_path, path_info + ); + + let req = http::Request::builder() + .method("POST") + .uri(req_uri) + .body("")?; + + let (router, _) = Router::build("/", [("DUMMY", &trigger_route.into())])?; + let route_match = router.route("/foo/bar")?; + + let default_headers = compute_default_headers(req.uri(), host, &route_match, client_addr)?; + + assert_eq!( + search(&FULL_URL, &default_headers).unwrap(), + "https://fermyon.dev/foo/bar?key1=value1&key2=value2".to_string() + ); + assert_eq!( + search(&PATH_INFO, &default_headers).unwrap(), + "/bar".to_string() + ); + assert_eq!( + search(&MATCHED_ROUTE, &default_headers).unwrap(), + "/foo/...".to_string() + ); + assert_eq!( + search(&BASE_PATH, &default_headers).unwrap(), + "/".to_string() + ); + assert_eq!( + search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo/...".to_string() + ); + assert_eq!( + search(&COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo".to_string() + ); + assert_eq!( + search(&CLIENT_ADDR, &default_headers).unwrap(), + "127.0.0.1:8777".to_string() + ); + + Ok(()) + } + + #[test] + fn test_default_headers_with_named_wildcards() -> Result<()> { + let scheme = "https"; + let host = "fermyon.dev"; + let trigger_route = "/foo/:userid/..."; + let component_path = "/foo"; + let path_info = "/bar"; + let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); + + let req_uri = format!( + "{}://{}{}/42{}?key1=value1&key2=value2", + scheme, host, component_path, path_info + ); + + let req = http::Request::builder() + .method("POST") + .uri(req_uri) + .body("")?; + + let (router, _) = Router::build("/", [("DUMMY", &trigger_route.into())])?; + let route_match = router.route("/foo/42/bar")?; + + let default_headers = compute_default_headers(req.uri(), host, &route_match, client_addr)?; + + // TODO: we currently replace the scheme with HTTP. When TLS is supported, this should be fixed. + assert_eq!( + search(&FULL_URL, &default_headers).unwrap(), + "https://fermyon.dev/foo/42/bar?key1=value1&key2=value2".to_string() + ); + assert_eq!( + search(&PATH_INFO, &default_headers).unwrap(), + "/bar".to_string() + ); + assert_eq!( + search(&MATCHED_ROUTE, &default_headers).unwrap(), + "/foo/:userid/...".to_string() + ); + assert_eq!( + search(&BASE_PATH, &default_headers).unwrap(), + "/".to_string() + ); + assert_eq!( + search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo/:userid/...".to_string() + ); + assert_eq!( + search(&COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo/:userid".to_string() + ); + assert_eq!( + search(&CLIENT_ADDR, &default_headers).unwrap(), + "127.0.0.1:8777".to_string() + ); + + assert_eq!( + search( + &["SPIN_PATH_MATCH_USERID", "X_PATH_MATCH_USERID"], + &default_headers + ) + .unwrap(), + "42".to_string() + ); + + Ok(()) + } + + fn search(keys: &[&str; 2], headers: &[([String; 2], String)]) -> Option { + let mut res: Option = None; + for (k, v) in headers { + if k[0] == keys[0] && k[1] == keys[1] { + res = Some(v.clone()); + } + } + + res + } + + #[test] + fn parse_listen_addr_prefers_ipv4() { + let addr = parse_listen_addr("localhost:12345").unwrap(); + assert_eq!(addr.ip(), Ipv4Addr::LOCALHOST); + assert_eq!(addr.port(), 12345); + } + + #[test] + fn forbidden_headers_are_removed() { + let mut req = Request::get("http://test.spin.internal") + .header("Host", "test.spin.internal") + .header("accept", "text/plain") + .body(Default::default()) + .unwrap(); + + strip_forbidden_headers(&mut req); + + assert_eq!(1, req.headers().len()); + assert!(req.headers().get("Host").is_none()); + + let mut req = Request::get("http://test.spin.internal") + .header("Host", "test.spin.internal:1234") + .header("accept", "text/plain") + .body(Default::default()) + .unwrap(); + + strip_forbidden_headers(&mut req); + + assert_eq!(1, req.headers().len()); + assert!(req.headers().get("Host").is_none()); + } + + #[test] + fn non_forbidden_headers_are_not_removed() { + let mut req = Request::get("http://test.example.com") + .header("Host", "test.example.org") + .header("accept", "text/plain") + .body(Default::default()) + .unwrap(); + + strip_forbidden_headers(&mut req); + + assert_eq!(2, req.headers().len()); + assert!(req.headers().get("Host").is_some()); + } +} diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs new file mode 100644 index 000000000..c4f4a881a --- /dev/null +++ b/crates/trigger-http2/src/server.rs @@ -0,0 +1,386 @@ +use std::{collections::HashMap, future::Future, io::IsTerminal, net::SocketAddr, sync::Arc}; + +use http::{uri::Scheme, Request, Response, StatusCode, Uri}; +use http_body_util::BodyExt; +use hyper::{ + body::{Bytes, Incoming}, + server::conn::http1, + service::service_fn, +}; +use hyper_util::rt::TokioIo; +use spin_app::{APP_DESCRIPTION_KEY, APP_NAME_KEY}; +use spin_http::{ + app_info::AppInfo, + body, + config::{HttpExecutorType, HttpTriggerConfig}, + routes::{RouteMatch, Router}, +}; +use spin_outbound_networking::is_service_chaining_host; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::TcpListener, + task, +}; +use tracing::Instrument; +use wasmtime_wasi_http::body::{HyperIncomingBody as Body, HyperOutgoingBody}; + +use crate::{ + handler::{HandlerType, HttpHandlerExecutor}, + instrument::{finalize_http_span, http_span, instrument_error, MatchedRoute}, + wagi::WagiHttpExecutor, + NotFoundRouteKind, TlsConfig, TriggerApp, TriggerInstanceBuilder, +}; + +pub struct HttpServer { + listen_addr: SocketAddr, + trigger_app: TriggerApp, + router: Router, + // Component ID -> component trigger config + component_trigger_configs: HashMap, + // Component ID -> handler type + component_handler_types: HashMap, +} + +impl HttpServer { + pub fn new( + listen_addr: SocketAddr, + trigger_app: TriggerApp, + router: Router, + component_trigger_configs: HashMap, + ) -> anyhow::Result { + let component_handler_types = component_trigger_configs + .keys() + .map(|component_id| { + let component = trigger_app.get_component(component_id)?; + let handler_type = HandlerType::from_component(trigger_app.engine(), component)?; + Ok((component_id.clone(), handler_type)) + }) + .collect::>()?; + Ok(Self { + listen_addr, + trigger_app, + router, + component_trigger_configs, + component_handler_types, + }) + } + + pub async fn serve(self: Arc, listener: TcpListener) -> anyhow::Result<()> { + self.print_startup_msgs("http", &listener)?; + loop { + let (stream, client_addr) = listener.accept().await?; + self.clone().serve_connection(stream, client_addr); + } + } + + pub async fn serve_tls( + self: Arc, + listener: TcpListener, + tls_config: TlsConfig, + ) -> anyhow::Result<()> { + self.print_startup_msgs("https", &listener)?; + let acceptor = tls_config.server_config()?; + loop { + let (stream, client_addr) = listener.accept().await?; + match acceptor.accept(stream).await { + Ok(stream) => self.clone().serve_connection(stream, client_addr), + Err(err) => tracing::error!(?err, "Failed to start TLS session"), + } + } + } + + /// Handles incoming requests using an HTTP executor. + pub async fn handle( + &self, + mut req: Request, + scheme: Scheme, + client_addr: SocketAddr, + ) -> anyhow::Result> { + set_req_uri(&mut req, scheme, self.listen_addr)?; + strip_forbidden_headers(&mut req); + + spin_telemetry::extract_trace_context(&req); + + tracing::info!("Processing request on URI {}", req.uri()); + + let path = req.uri().path().to_string(); + + // Handle well-known spin paths + if let Some(well_known) = path.strip_prefix(spin_http::WELL_KNOWN_PREFIX) { + return match well_known { + "health" => Ok(MatchedRoute::with_response_extension( + Response::new(body::full(Bytes::from_static(b"OK"))), + path, + )), + "info" => self.app_info(path), + _ => Self::not_found(NotFoundRouteKind::WellKnown), + }; + } + + let app_id = self + .trigger_app + .app() + .get_metadata(APP_NAME_KEY)? + .unwrap_or_else(|| "".into()); + + // Route to app component + match self.router.route(&path) { + Ok(route_match) => { + let component_id = route_match.component_id(); + + spin_telemetry::metrics::monotonic_counter!( + spin.request_count = 1, + trigger_type = "http", + app_id = app_id, + component_id = component_id + ); + + let instance_builder = self.trigger_app.prepare(component_id)?; + let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); + let handler_type = self.component_handler_types.get(component_id).unwrap(); + let executor = trigger_config + .executor + .as_ref() + .unwrap_or(&HttpExecutorType::Http); + + let res = match executor { + HttpExecutorType::Http => { + HttpHandlerExecutor { + handler_type: *handler_type, + } + .execute(instance_builder, &route_match, req, client_addr) + .await + } + HttpExecutorType::Wagi(wagi_config) => { + let executor = WagiHttpExecutor { + wagi_config: wagi_config.clone(), + }; + executor + .execute(instance_builder, &route_match, req, client_addr) + .await + } + }; + match res { + Ok(res) => Ok(MatchedRoute::with_response_extension( + res, + route_match.raw_route(), + )), + Err(err) => { + tracing::error!("Error processing request: {err:?}"); + instrument_error(&err); + Self::internal_error(None, route_match.raw_route()) + } + } + } + Err(_) => Self::not_found(NotFoundRouteKind::Normal(path.to_string())), + } + } + + /// Returns spin status information. + fn app_info(&self, route: String) -> anyhow::Result> { + let info = AppInfo::new(self.trigger_app.app()); + let body = serde_json::to_vec_pretty(&info)?; + Ok(MatchedRoute::with_response_extension( + Response::builder() + .header("content-type", "application/json") + .body(body::full(body.into()))?, + route, + )) + } + + /// Creates an HTTP 500 response. + fn internal_error( + body: Option<&str>, + route: impl Into, + ) -> anyhow::Result> { + let body = match body { + Some(body) => body::full(Bytes::copy_from_slice(body.as_bytes())), + None => body::empty(), + }; + + Ok(MatchedRoute::with_response_extension( + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(body)?, + route, + )) + } + + /// Creates an HTTP 404 response. + fn not_found(kind: NotFoundRouteKind) -> anyhow::Result> { + use std::sync::atomic::{AtomicBool, Ordering}; + static SHOWN_GENERIC_404_WARNING: AtomicBool = AtomicBool::new(false); + if let NotFoundRouteKind::Normal(route) = kind { + if !SHOWN_GENERIC_404_WARNING.fetch_or(true, Ordering::Relaxed) + && std::io::stderr().is_terminal() + { + terminal::warn!("Request to {route} matched no pattern, and received a generic 404 response. To serve a more informative 404 page, add a catch-all (/...) route."); + } + } + Ok(Response::builder() + .status(StatusCode::NOT_FOUND) + .body(body::empty())?) + } + + fn serve_connection( + self: Arc, + stream: S, + client_addr: SocketAddr, + ) { + task::spawn(async move { + if let Err(err) = http1::Builder::new() + .keep_alive(true) + .serve_connection( + TokioIo::new(stream), + service_fn(move |request| { + self.clone().instrumented_service_fn(client_addr, request) + }), + ) + .await + { + tracing::warn!("Error serving HTTP connection: {err:?}"); + } + }); + } + + async fn instrumented_service_fn( + self: Arc, + client_addr: SocketAddr, + request: Request, + ) -> anyhow::Result> { + let span = http_span!(request, client_addr); + let method = request.method().to_string(); + async { + let result = self + .handle( + request.map(|body: Incoming| { + body.map_err(wasmtime_wasi_http::hyper_response_error) + .boxed() + }), + Scheme::HTTP, + client_addr, + ) + .await; + finalize_http_span(result, method) + } + .instrument(span) + .await + } + + fn print_startup_msgs(&self, scheme: &str, listener: &TcpListener) -> anyhow::Result<()> { + let local_addr = listener.local_addr()?; + let base_url = format!("{scheme}://{local_addr:?}"); + terminal::step!("\nServing", "{base_url}"); + tracing::info!("Serving {base_url}"); + + println!("Available Routes:"); + for (route, component_id) in self.router.routes() { + println!(" {}: {}{}", component_id, base_url, route); + if let Some(component) = self.trigger_app.app().get_component(component_id) { + if let Some(description) = component.get_metadata(APP_DESCRIPTION_KEY)? { + println!(" {}", description); + } + } + } + Ok(()) + } +} + +/// The incoming request's scheme and authority +/// +/// The incoming request's URI is relative to the server, so we need to set the scheme and authority +fn set_req_uri(req: &mut Request, scheme: Scheme, addr: SocketAddr) -> anyhow::Result<()> { + let uri = req.uri().clone(); + let mut parts = uri.into_parts(); + let authority = format!("{}:{}", addr.ip(), addr.port()).parse().unwrap(); + parts.scheme = Some(scheme); + parts.authority = Some(authority); + *req.uri_mut() = Uri::from_parts(parts).unwrap(); + Ok(()) +} + +pub fn strip_forbidden_headers(req: &mut Request) { + let headers = req.headers_mut(); + if let Some(host_header) = headers.get("Host") { + if let Ok(host) = host_header.to_str() { + if is_service_chaining_host(host) { + headers.remove("Host"); + } + } + } +} + +// We need to make the following pieces of information available to both executors. +// While the values we set are identical, the way they are passed to the +// modules is going to be different, so each executor must must use the info +// in its standardized way (environment variables for the Wagi executor, and custom headers +// for the Spin HTTP executor). +pub const FULL_URL: [&str; 2] = ["SPIN_FULL_URL", "X_FULL_URL"]; +pub const PATH_INFO: [&str; 2] = ["SPIN_PATH_INFO", "PATH_INFO"]; +pub const MATCHED_ROUTE: [&str; 2] = ["SPIN_MATCHED_ROUTE", "X_MATCHED_ROUTE"]; +pub const COMPONENT_ROUTE: [&str; 2] = ["SPIN_COMPONENT_ROUTE", "X_COMPONENT_ROUTE"]; +pub const RAW_COMPONENT_ROUTE: [&str; 2] = ["SPIN_RAW_COMPONENT_ROUTE", "X_RAW_COMPONENT_ROUTE"]; +pub const BASE_PATH: [&str; 2] = ["SPIN_BASE_PATH", "X_BASE_PATH"]; +pub const CLIENT_ADDR: [&str; 2] = ["SPIN_CLIENT_ADDR", "X_CLIENT_ADDR"]; + +pub(crate) fn compute_default_headers( + uri: &Uri, + host: &str, + route_match: &RouteMatch, + client_addr: SocketAddr, +) -> anyhow::Result> { + fn owned(strs: &[&'static str; 2]) -> [String; 2] { + [strs[0].to_owned(), strs[1].to_owned()] + } + + let owned_full_url: [String; 2] = owned(&FULL_URL); + let owned_path_info: [String; 2] = owned(&PATH_INFO); + let owned_matched_route: [String; 2] = owned(&MATCHED_ROUTE); + let owned_component_route: [String; 2] = owned(&COMPONENT_ROUTE); + let owned_raw_component_route: [String; 2] = owned(&RAW_COMPONENT_ROUTE); + let owned_base_path: [String; 2] = owned(&BASE_PATH); + let owned_client_addr: [String; 2] = owned(&CLIENT_ADDR); + + let mut res = vec![]; + let abs_path = uri + .path_and_query() + .expect("cannot get path and query") + .as_str(); + + let path_info = route_match.trailing_wildcard(); + + let scheme = uri.scheme_str().unwrap_or("http"); + + let full_url = format!("{}://{}{}", scheme, host, abs_path); + + res.push((owned_path_info, path_info)); + res.push((owned_full_url, full_url)); + res.push((owned_matched_route, route_match.based_route().to_string())); + + res.push((owned_base_path, "/".to_string())); + res.push(( + owned_raw_component_route, + route_match.raw_route().to_string(), + )); + res.push((owned_component_route, route_match.raw_route_or_prefix())); + res.push((owned_client_addr, client_addr.to_string())); + + for (wild_name, wild_value) in route_match.named_wildcards() { + let wild_header = format!("SPIN_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer + let wild_wagi_header = format!("X_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer + res.push(([wild_header, wild_wagi_header], wild_value.clone())); + } + + Ok(res) +} + +/// An HTTP executor. +pub(crate) trait HttpExecutor: Clone + Send + Sync + 'static { + fn execute( + &self, + instance_builder: TriggerInstanceBuilder, + route_match: &RouteMatch, + req: Request, + client_addr: SocketAddr, + ) -> impl Future>>; +} diff --git a/crates/trigger-http2/src/tls.rs b/crates/trigger-http2/src/tls.rs new file mode 100644 index 000000000..0f75eaac8 --- /dev/null +++ b/crates/trigger-http2/src/tls.rs @@ -0,0 +1,134 @@ +use rustls_pemfile::private_key; +use std::{ + fs, io, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio_rustls::{rustls, TlsAcceptor}; + +/// TLS configuration for the server. +#[derive(Clone)] +pub struct TlsConfig { + /// Path to TLS certificate. + pub cert_path: PathBuf, + /// Path to TLS key. + pub key_path: PathBuf, +} + +impl TlsConfig { + // Creates a TLS acceptor from server config. + pub(super) fn server_config(&self) -> anyhow::Result { + let certs = load_certs(&self.cert_path)?; + let private_key = load_key(&self.key_path)?; + + let cfg = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, private_key) + .map_err(|e| anyhow::anyhow!("{}", e))?; + + Ok(Arc::new(cfg).into()) + } +} + +// load_certs parse and return the certs from the provided file +fn load_certs( + path: impl AsRef, +) -> io::Result>> { + rustls_pemfile::certs(&mut io::BufReader::new(fs::File::open(path).map_err( + |err| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("failed to read cert file {:?}", err), + ) + }, + )?)) + .collect() +} + +// parse and return the first private key from the provided file +fn load_key(path: impl AsRef) -> io::Result> { + private_key(&mut io::BufReader::new(fs::File::open(path).map_err( + |err| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("failed to read private key file {:?}", err), + ) + }, + )?)) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid private key")) + .transpose() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "private key file contains no private keys", + ) + })? +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_datadir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata") + } + + #[test] + fn test_read_non_existing_cert() { + let path = test_datadir().join("non-existing-file.pem"); + + let certs = load_certs(path); + assert!(certs.is_err()); + assert_eq!(certs.err().unwrap().to_string(), "failed to read cert file Os { code: 2, kind: NotFound, message: \"No such file or directory\" }"); + } + + #[test] + fn test_read_invalid_cert() { + let path = test_datadir().join("invalid-cert.pem"); + + let certs = load_certs(path); + assert!(certs.is_err()); + assert_eq!( + certs.err().unwrap().to_string(), + "section end \"-----END CERTIFICATE-----\" missing" + ); + } + + #[test] + fn test_read_valid_cert() { + let path = test_datadir().join("valid-cert.pem"); + + let certs = load_certs(path); + assert!(certs.is_ok()); + assert_eq!(certs.unwrap().len(), 2); + } + + #[test] + fn test_read_non_existing_private_key() { + let mut path = test_datadir(); + path.push("non-existing-file.pem"); + + let keys = load_key(path); + assert!(keys.is_err()); + assert_eq!(keys.err().unwrap().to_string(), "failed to read private key file Os { code: 2, kind: NotFound, message: \"No such file or directory\" }"); + } + + #[test] + fn test_read_invalid_private_key() { + let mut path = test_datadir(); + path.push("invalid-private-key.pem"); + + let keys = load_key(path); + assert!(keys.is_err()); + assert_eq!(keys.err().unwrap().to_string(), "invalid private key"); + } + + #[test] + fn test_read_valid_private_key() { + let mut path = test_datadir(); + path.push("valid-private-key.pem"); + + let keys = load_key(path); + assert!(keys.is_ok()); + } +} diff --git a/crates/trigger-http2/src/wagi.rs b/crates/trigger-http2/src/wagi.rs new file mode 100644 index 000000000..7a0a3701a --- /dev/null +++ b/crates/trigger-http2/src/wagi.rs @@ -0,0 +1,135 @@ +use std::{io::Cursor, net::SocketAddr}; + +use anyhow::{anyhow, ensure, Context, Result}; +use http_body_util::BodyExt; +use hyper::{Request, Response}; +use spin_http::{config::WagiTriggerConfig, routes::RouteMatch, wagi}; +use tracing::{instrument, Level}; +use wasmtime_wasi::pipe::MemoryOutputPipe; +use wasmtime_wasi_http::body::HyperIncomingBody as Body; + +use crate::{server::HttpExecutor, TriggerInstanceBuilder}; + +#[derive(Clone)] +pub struct WagiHttpExecutor { + pub wagi_config: WagiTriggerConfig, +} + +impl HttpExecutor for WagiHttpExecutor { + #[instrument(name = "spin_trigger_http.execute_wagi", skip_all, err(level = Level::INFO), fields(otel.name = format!("execute_wagi_component {}", route_match.component_id())))] + async fn execute( + &self, + mut instance_builder: TriggerInstanceBuilder<'_>, + route_match: &RouteMatch, + req: Request, + client_addr: SocketAddr, + ) -> Result> { + let component = route_match.component_id(); + + tracing::trace!( + "Executing request using the Wagi executor for component {}", + component + ); + + let uri_path = req.uri().path(); + + // Build the argv array by starting with the config for `argv` and substituting in + // script name and args where appropriate. + let script_name = uri_path.to_string(); + let args = req.uri().query().unwrap_or_default().replace('&', " "); + let argv = self + .wagi_config + .argv + .clone() + .replace("${SCRIPT_NAME}", &script_name) + .replace("${ARGS}", &args); + + let (parts, body) = req.into_parts(); + + let body = body.collect().await?.to_bytes().to_vec(); + let len = body.len(); + + // TODO + // The default host and TLS fields are currently hard-coded. + let mut headers = + wagi::build_headers(route_match, &parts, len, client_addr, "default_host", false); + + let default_host = http::HeaderValue::from_str("localhost")?; + let host = std::str::from_utf8( + parts + .headers + .get("host") + .unwrap_or(&default_host) + .as_bytes(), + )?; + + // Add the default Spin headers. + // This sets the current environment variables Wagi expects (such as + // `PATH_INFO`, or `X_FULL_URL`). + // Note that this overrides any existing headers previously set by Wagi. + for (keys, val) in + crate::server::compute_default_headers(&parts.uri, host, route_match, client_addr)? + { + headers.insert(keys[1].to_string(), val); + } + + let stdout = MemoryOutputPipe::new(usize::MAX); + + let wasi_builder = instance_builder.factor_builders().wasi(); + + // Set up Wagi environment + wasi_builder.args(argv.split(' ')); + wasi_builder.env(headers); + wasi_builder.stdin_pipe(Cursor::new(body)); + wasi_builder.stdout(stdout.clone()); + + let (instance, mut store) = instance_builder.instantiate(()).await?; + + let start = instance + .get_func(&mut store, &self.wagi_config.entrypoint) + .ok_or_else(|| { + anyhow::anyhow!( + "No such function '{}' in {}", + self.wagi_config.entrypoint, + component + ) + })?; + tracing::trace!("Calling Wasm entry point"); + start + .call_async(&mut store, &[], &mut []) + .await + .or_else(ignore_successful_proc_exit_trap) + .with_context(|| { + anyhow!( + "invoking {} for component {component}", + self.wagi_config.entrypoint + ) + })?; + tracing::info!("Module execution complete"); + + // Drop the store so we're left with a unique reference to `stdout`: + drop(store); + + let stdout = stdout.try_into_inner().unwrap(); + ensure!( + !stdout.is_empty(), + "The {component:?} component is configured to use the WAGI executor \ + but did not write to stdout. Check the `executor` in spin.toml." + ); + + wagi::compose_response(&stdout) + } +} + +fn ignore_successful_proc_exit_trap(guest_err: anyhow::Error) -> Result<()> { + match guest_err + .root_cause() + .downcast_ref::() + { + Some(trap) => match trap.0 { + 0 => Ok(()), + _ => Err(guest_err), + }, + None => Err(guest_err), + } +} diff --git a/crates/trigger-http2/testdata/invalid-cert.pem b/crates/trigger-http2/testdata/invalid-cert.pem new file mode 100644 index 000000000..f1a952b9c --- /dev/null +++ b/crates/trigger-http2/testdata/invalid-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIBkjCCATegAwIBAgIIEOURVvWgx1AwCgYIKoZIzj0EAwIwIzEhMB8GA1UEAwwY +azNzLWNsaWVudC1jYUAxNzE3NzgwNTIwMB4XDTI0MDYwNzE3MTUyMFoXDTI1MDYw +NzE3MTUyMFowMDEXMBUGA1UEChMOc3lzdGVtOm1hc3RlcnMxFTATBgNVBAMTDHN5 +c3RlbTphZG1pbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFGE/CVuauj8kmde +i4AagSJ5GYgGnL0eF55ItiXrKSjMmsIf/N8EyeamxQfWPKVk/1xhH7cS9GcQgNe6 +XrRvmLyjSDBGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAf +BgNVHSMEGDAWgBRpihySeW3DafmU1cw6LMnQCQDD4jAKBggqhkjOPQQDAgNJADBG +AiEA/db1wb4mVrqJVctqbPU9xd0bXzJx7cBDzpWgPP9ISfkCIQDNyuskAkXvUMHH +F73/GJnh8Bt2H38qyzThM8nlR9v1eQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3MtY2xp +ZW50LWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw +WjAjMSEwHwYDVQQDDBhrM3MtY2xpZW50LWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO +PQIBBggqhkjOPQMBBwNCAASozciE0YGl8ak3G0Ll1riwXSScfpK0QRle/cFizdlA +HgDowBssBcla0/2a/eWabxqTPzsZH0cVhL7Tialoj8GNo0IwQDAOBgNVHQ8BAf8E +BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUaYocknltw2n5lNXMOizJ +0AkAw+IwCgYIKoZIzj0EAwIDRwAwRAIgR8YcLA8cH4qAMDRPDsJqLaw4GJFkgjwV +TCrMgyUxSvACIBwyklgm7mgHcC5WM9CqmliAGZJyV0xRPZBK01POrNf0 diff --git a/crates/trigger-http2/testdata/invalid-private-key.pem b/crates/trigger-http2/testdata/invalid-private-key.pem new file mode 100644 index 000000000..39d7e59ee --- /dev/null +++ b/crates/trigger-http2/testdata/invalid-private-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIA+FBtmKJbd8wBGOWeuJQfHiCKjjXF8ywEPrvj8S1N3VoAoGCCqGSM49 +AwEHoUQDQgAEUYT8JW5q6PySZ16LgBqBInkZiAacvR4Xnki2JespKMyawh/83wTJ +5qbFB9Y8pWT/XGEftxL0ZxCA17petG+YvA== +-----END EC PRIVATE KEY- \ No newline at end of file diff --git a/crates/trigger-http2/testdata/valid-cert.pem b/crates/trigger-http2/testdata/valid-cert.pem new file mode 100644 index 000000000..e75166d0e --- /dev/null +++ b/crates/trigger-http2/testdata/valid-cert.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIBkjCCATegAwIBAgIIEOURVvWgx1AwCgYIKoZIzj0EAwIwIzEhMB8GA1UEAwwY +azNzLWNsaWVudC1jYUAxNzE3NzgwNTIwMB4XDTI0MDYwNzE3MTUyMFoXDTI1MDYw +NzE3MTUyMFowMDEXMBUGA1UEChMOc3lzdGVtOm1hc3RlcnMxFTATBgNVBAMTDHN5 +c3RlbTphZG1pbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFGE/CVuauj8kmde +i4AagSJ5GYgGnL0eF55ItiXrKSjMmsIf/N8EyeamxQfWPKVk/1xhH7cS9GcQgNe6 +XrRvmLyjSDBGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAf +BgNVHSMEGDAWgBRpihySeW3DafmU1cw6LMnQCQDD4jAKBggqhkjOPQQDAgNJADBG +AiEA/db1wb4mVrqJVctqbPU9xd0bXzJx7cBDzpWgPP9ISfkCIQDNyuskAkXvUMHH +F73/GJnh8Bt2H38qyzThM8nlR9v1eQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3MtY2xp +ZW50LWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw +WjAjMSEwHwYDVQQDDBhrM3MtY2xpZW50LWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO +PQIBBggqhkjOPQMBBwNCAASozciE0YGl8ak3G0Ll1riwXSScfpK0QRle/cFizdlA +HgDowBssBcla0/2a/eWabxqTPzsZH0cVhL7Tialoj8GNo0IwQDAOBgNVHQ8BAf8E +BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUaYocknltw2n5lNXMOizJ +0AkAw+IwCgYIKoZIzj0EAwIDRwAwRAIgR8YcLA8cH4qAMDRPDsJqLaw4GJFkgjwV +TCrMgyUxSvACIBwyklgm7mgHcC5WM9CqmliAGZJyV0xRPZBK01POrNf0 +-----END CERTIFICATE----- \ No newline at end of file diff --git a/crates/trigger-http2/testdata/valid-private-key.pem b/crates/trigger-http2/testdata/valid-private-key.pem new file mode 100644 index 000000000..2820fbed2 --- /dev/null +++ b/crates/trigger-http2/testdata/valid-private-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIA+FBtmKJbd8wBGOWeuJQfHiCKjjXF8ywEPrvj8S1N3VoAoGCCqGSM49 +AwEHoUQDQgAEUYT8JW5q6PySZ16LgBqBInkZiAacvR4Xnki2JespKMyawh/83wTJ +5qbFB9Y8pWT/XGEftxL0ZxCA17petG+YvA== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml new file mode 100644 index 000000000..a978ce987 --- /dev/null +++ b/crates/trigger2/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "spin-trigger2" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = "1" +clap = { version = "3.1.18", features = ["derive", "env"] } +ctrlc = { version = "3.2", features = ["termination"] } +futures = "0.3" +sanitize-filename = "0.5" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +spin-app = { path = "../app" } +spin-common = { path = "../common" } +spin-core = { path = "../core" } +spin-factor-wasi = { path = "../factor-wasi" } +spin-factors = { path = "../factors" } +spin-factors-executor = { path = "../factors-executor" } +spin-telemetry = { path = "../telemetry" } +tokio = { version = "1.23", features = ["fs"] } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs new file mode 100644 index 000000000..b3c9ea1c0 --- /dev/null +++ b/crates/trigger2/src/cli.rs @@ -0,0 +1,317 @@ +mod launch_metadata; + +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::{Args, IntoApp, Parser}; +use spin_app::App; +use spin_common::ui::quoted_path; +use spin_common::url::parse_file_url; +use spin_common::{arg_parser::parse_kv, sloth}; +use spin_factors_executor::{ComponentLoader, FactorsExecutor}; + +use crate::factors::TriggerFactors; +use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; +use crate::Trigger; +pub use launch_metadata::LaunchMetadata; + +pub const APP_LOG_DIR: &str = "APP_LOG_DIR"; +pub const DISABLE_WASMTIME_CACHE: &str = "DISABLE_WASMTIME_CACHE"; +pub const FOLLOW_LOG_OPT: &str = "FOLLOW_ID"; +pub const WASMTIME_CACHE_FILE: &str = "WASMTIME_CACHE_FILE"; +pub const RUNTIME_CONFIG_FILE: &str = "RUNTIME_CONFIG_FILE"; + +// Set by `spin up` +pub const SPIN_LOCKED_URL: &str = "SPIN_LOCKED_URL"; +pub const SPIN_LOCAL_APP_DIR: &str = "SPIN_LOCAL_APP_DIR"; +pub const SPIN_WORKING_DIR: &str = "SPIN_WORKING_DIR"; + +/// A command that runs a TriggerExecutor. +#[derive(Parser, Debug)] +#[clap( + usage = "spin [COMMAND] [OPTIONS]", + next_help_heading = help_heading::() +)] +pub struct FactorsTriggerCommand { + /// Log directory for the stdout and stderr of components. Setting to + /// the empty string disables logging to disk. + #[clap( + name = APP_LOG_DIR, + short = 'L', + long = "log-dir", + env = "SPIN_LOG_DIR", + )] + pub log: Option, + + /// Disable Wasmtime cache. + #[clap( + name = DISABLE_WASMTIME_CACHE, + long = "disable-cache", + env = DISABLE_WASMTIME_CACHE, + conflicts_with = WASMTIME_CACHE_FILE, + takes_value = false, + )] + pub disable_cache: bool, + + /// Wasmtime cache configuration file. + #[clap( + name = WASMTIME_CACHE_FILE, + long = "cache", + env = WASMTIME_CACHE_FILE, + conflicts_with = DISABLE_WASMTIME_CACHE, + )] + pub cache: Option, + + /// Disable Wasmtime's pooling instance allocator. + #[clap(long = "disable-pooling")] + pub disable_pooling: bool, + + /// Print output to stdout/stderr only for given component(s) + #[clap( + name = FOLLOW_LOG_OPT, + long = "follow", + multiple_occurrences = true, + )] + pub follow_components: Vec, + + /// Silence all component output to stdout/stderr + #[clap( + long = "quiet", + short = 'q', + aliases = &["sh", "shush"], + conflicts_with = FOLLOW_LOG_OPT, + )] + pub silence_component_logs: bool, + + /// Set the static assets of the components in the temporary directory as writable. + #[clap(long = "allow-transient-write")] + pub allow_transient_write: bool, + + /// Configuration file for config providers and wasmtime config. + #[clap( + name = RUNTIME_CONFIG_FILE, + long = "runtime-config-file", + env = RUNTIME_CONFIG_FILE, + )] + pub runtime_config_file: Option, + + /// Set the application state directory path. This is used in the default + /// locations for logs, key value stores, etc. + /// + /// For local apps, this defaults to `.spin/` relative to the `spin.toml` file. + /// For remote apps, this has no default (unset). + /// Passing an empty value forces the value to be unset. + #[clap(long)] + pub state_dir: Option, + + #[clap(flatten)] + pub trigger_args: T::CliArgs, + + /// Set a key/value pair (key=value) in the application's + /// default store. Any existing value will be overwritten. + /// Can be used multiple times. + #[clap(long = "key-value", parse(try_from_str = parse_kv))] + key_values: Vec<(String, String)>, + + /// Run a SQLite statement such as a migration against the default database. + /// To run from a file, prefix the filename with @ e.g. spin up --sqlite @migration.sql + #[clap(long = "sqlite")] + sqlite_statements: Vec, + + #[clap(long = "help-args-only", hide = true)] + pub help_args_only: bool, + + #[clap(long = "launch-metadata-only", hide = true)] + pub launch_metadata_only: bool, +} + +/// An empty implementation of clap::Args to be used as TriggerExecutor::RunConfig +/// for executors that do not need additional CLI args. +#[derive(Args)] +pub struct NoArgs; + +impl FactorsTriggerCommand { + /// Create a new TriggerExecutorBuilder from this TriggerExecutorCommand. + pub async fn run(self) -> Result<()> { + // Handle --help-args-only + if self.help_args_only { + Self::command() + .disable_help_flag(true) + .help_template("{all-args}") + .print_long_help()?; + return Ok(()); + } + + // Handle --launch-metadata-only + if self.launch_metadata_only { + let lm = LaunchMetadata::infer::(); + let json = serde_json::to_string_pretty(&lm)?; + eprintln!("{json}"); + return Ok(()); + } + + // Required env vars + let working_dir = std::env::var(SPIN_WORKING_DIR).context(SPIN_WORKING_DIR)?; + let locked_url = std::env::var(SPIN_LOCKED_URL).context(SPIN_LOCKED_URL)?; + + let follow_components = self.follow_components(); + + // Load App + let app = { + let path = parse_file_url(&locked_url)?; + let contents = std::fs::read(&path) + .with_context(|| format!("failed to read manifest at {}", quoted_path(&path)))?; + let locked = + serde_json::from_slice(&contents).context("failed to parse app lock file JSON")?; + App::new(locked_url, locked) + }; + + // Validate required host features + if let Err(unmet) = app.ensure_needs_only(&T::supported_host_requirements()) { + anyhow::bail!("This application requires the following features that are not available in this version of the '{}' trigger: {unmet}", T::TYPE); + } + + let mut trigger = T::new(self.trigger_args, &app)?; + + let mut core_engine_builder = { + let mut config = spin_core::Config::default(); + + // Apply --cache / --disable-cache + if !self.disable_cache { + config.enable_cache(&self.cache)?; + } + + if self.disable_pooling { + config.disable_pooling(); + } + + trigger.update_core_config(&mut config)?; + + spin_core::Engine::builder(&config)? + }; + trigger.add_to_linker(core_engine_builder.linker())?; + + let factors = TriggerFactors::new(working_dir, self.allow_transient_write); + + // TODO: move these into Factor methods/constructors + // let init_data = crate::HostComponentInitData::new( + // &*self.key_values, + // &*self.sqlite_statements, + // LLmOptions { use_gpu: true }, + // ); + + // TODO: component loader + struct TodoComponentLoader; + impl ComponentLoader for TodoComponentLoader { + fn load_component( + &mut self, + _engine: &spin_core::wasmtime::Engine, + _component: &spin_factors::AppComponent, + ) -> anyhow::Result { + todo!() + } + } + + let mut executor = FactorsExecutor::new(core_engine_builder, factors)?; + + // TODO: integrate with runtime config + let log_dir = self.log.clone(); + executor.add_hooks(StdioLoggingExecutorHooks::new(follow_components, log_dir)); + // TODO: + // builder.hooks(SummariseRuntimeConfigHook::new(&self.runtime_config_file)); + // builder.hooks(KeyValuePersistenceMessageHook); + // builder.hooks(SqlitePersistenceMessageHook); + + let configured_app = { + let _sloth_guard = warn_if_wasm_build_slothful(); + executor.load_app( + app, + Default::default(), // TODO runtime config + TodoComponentLoader, + )? + }; + + // TODO: Construct factors + let run_fut = trigger.run(configured_app); + + let (abortable, abort_handle) = futures::future::abortable(run_fut); + ctrlc::set_handler(move || abort_handle.abort())?; + match abortable.await { + Ok(Ok(())) => { + tracing::info!("Trigger executor shut down: exiting"); + Ok(()) + } + Ok(Err(err)) => { + tracing::error!("Trigger executor failed"); + Err(err) + } + Err(_aborted) => { + tracing::info!("User requested shutdown: exiting"); + Ok(()) + } + } + } + + fn follow_components(&self) -> FollowComponents { + if self.silence_component_logs { + FollowComponents::None + } else if self.follow_components.is_empty() { + FollowComponents::All + } else { + let followed = self.follow_components.clone().into_iter().collect(); + FollowComponents::Named(followed) + } + } +} + +const SLOTH_WARNING_DELAY_MILLIS: u64 = 1250; + +fn warn_if_wasm_build_slothful() -> sloth::SlothGuard { + #[cfg(debug_assertions)] + let message = "\ + This is a debug build - preparing Wasm modules might take a few seconds\n\ + If you're experiencing long startup times please switch to the release build"; + + #[cfg(not(debug_assertions))] + let message = "Preparing Wasm modules is taking a few seconds..."; + + sloth::warn_if_slothful(SLOTH_WARNING_DELAY_MILLIS, format!("{message}\n")) +} + +fn help_heading() -> Option<&'static str> { + if T::TYPE == help::HelpArgsOnlyTrigger::TYPE { + Some("TRIGGER OPTIONS") + } else { + let heading = format!("{} TRIGGER OPTIONS", T::TYPE.to_uppercase()); + let as_str = Box::new(heading).leak(); + Some(as_str) + } +} + +pub mod help { + use super::*; + + /// Null object to support --help-args-only in the absence of + /// a `spin.toml` file. + pub struct HelpArgsOnlyTrigger; + + impl Trigger for HelpArgsOnlyTrigger { + const TYPE: &'static str = "help-args-only"; + type CliArgs = NoArgs; + type InstanceState = (); + + fn new(_cli_args: Self::CliArgs, _app: &App) -> anyhow::Result { + Ok(Self) + } + + async fn run( + self, + _configured_app: spin_factors_executor::FactorsExecutorApp< + TriggerFactors, + Self::InstanceState, + >, + ) -> anyhow::Result<()> { + Ok(()) + } + } +} diff --git a/crates/trigger2/src/cli/launch_metadata.rs b/crates/trigger2/src/cli/launch_metadata.rs new file mode 100644 index 000000000..be78e3634 --- /dev/null +++ b/crates/trigger2/src/cli/launch_metadata.rs @@ -0,0 +1,83 @@ +use clap::CommandFactory; +use serde::{Deserialize, Serialize}; +use std::ffi::OsString; + +use crate::{cli::FactorsTriggerCommand, Trigger}; + +/// Contains information about the trigger flags (and potentially +/// in future configuration) that a consumer (such as `spin up`) +/// can query using `--launch-metadata-only`. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct LaunchMetadata { + all_flags: Vec, +} + +// This assumes no triggers that want to participate in multi-trigger +// use positional arguments. This is a restriction we'll have to make +// anyway: suppose triggers A and B both take one positional arg, and +// the user writes `spin up 123 456` - which value would go to which trigger? +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +struct LaunchFlag { + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + short: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + long: Option, +} + +impl LaunchMetadata { + pub fn infer() -> Self { + let all_flags: Vec<_> = FactorsTriggerCommand::::command() + .get_arguments() + .map(LaunchFlag::infer) + .collect(); + + LaunchMetadata { all_flags } + } + + pub fn matches<'a>(&self, groups: &[Vec<&'a OsString>]) -> Vec<&'a OsString> { + let mut matches = vec![]; + + for group in groups { + if group.is_empty() { + continue; + } + if self.is_match(group[0]) { + matches.extend(group); + } + } + + matches + } + + fn is_match(&self, arg: &OsString) -> bool { + self.all_flags.iter().any(|f| f.is_match(arg)) + } + + pub fn is_group_match(&self, group: &[&OsString]) -> bool { + if group.is_empty() { + false + } else { + self.all_flags.iter().any(|f| f.is_match(group[0])) + } + } +} + +impl LaunchFlag { + fn infer(arg: &clap::Arg) -> Self { + Self { + long: arg.get_long().map(|s| format!("--{s}")), + short: arg.get_short().map(|ch| format!("-{ch}")), + } + } + + fn is_match(&self, candidate: &OsString) -> bool { + let Some(s) = candidate.to_str() else { + return false; + }; + let candidate = Some(s.to_owned()); + + candidate == self.long || candidate == self.short + } +} diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs new file mode 100644 index 000000000..b433883da --- /dev/null +++ b/crates/trigger2/src/factors.rs @@ -0,0 +1,18 @@ +use std::path::PathBuf; + +use spin_factor_wasi::{spin::SpinFilesMounter, WasiFactor}; +use spin_factors::RuntimeFactors; + +#[derive(RuntimeFactors)] +pub struct TriggerFactors { + pub wasi: WasiFactor, +} + +impl TriggerFactors { + pub fn new(working_dir: impl Into, allow_transient_writes: bool) -> Self { + let files_mounter = SpinFilesMounter::new(working_dir, allow_transient_writes); + Self { + wasi: WasiFactor::new(files_mounter), + } + } +} diff --git a/crates/trigger2/src/lib.rs b/crates/trigger2/src/lib.rs new file mode 100644 index 000000000..095d30e3b --- /dev/null +++ b/crates/trigger2/src/lib.rs @@ -0,0 +1,66 @@ +use std::future::Future; + +use clap::Args; +use factors::{TriggerFactors, TriggerFactorsInstanceState}; +use spin_app::App; +use spin_core::Linker; +use spin_factors_executor::{FactorsExecutorApp, FactorsInstanceBuilder}; + +pub mod cli; +mod factors; +mod stdio; + +/// Type alias for a [`FactorsConfiguredApp`] specialized to a [`Trigger`]. +pub type TriggerApp = FactorsExecutorApp::InstanceState>; + +pub type TriggerInstanceBuilder<'a, T> = + FactorsInstanceBuilder<'a, TriggerFactors, ::InstanceState>; + +pub type Store = spin_core::Store>; + +type TriggerInstanceState = spin_factors_executor::InstanceState< + TriggerFactorsInstanceState, + ::InstanceState, +>; + +pub trait Trigger: Sized + Send { + const TYPE: &'static str; + + type CliArgs: Args; + type InstanceState: Send + 'static; + + /// Constructs a new trigger. + fn new(cli_args: Self::CliArgs, app: &App) -> anyhow::Result; + + /// Update the [`spin_core::Config`] for this trigger. + /// + /// !!!Warning!!! This is unsupported; many configurations are likely to + /// cause errors or unexpected behavior, especially in future versions. + #[doc(hidden)] + fn update_core_config(&mut self, config: &mut spin_core::Config) -> anyhow::Result<()> { + let _ = config; + Ok(()) + } + + /// Update the [`Linker`] for this trigger. + fn add_to_linker( + &mut self, + linker: &mut Linker>, + ) -> anyhow::Result<()> { + let _ = linker; + Ok(()) + } + + /// Run this trigger. + fn run( + self, + configured_app: TriggerApp, + ) -> impl Future> + Send; + + /// Returns a list of host requirements supported by this trigger specifically. + /// + /// See [`App::ensure_needs_only`]. + fn supported_host_requirements() -> Vec<&'static str> { + Vec::new() + } +} diff --git a/crates/trigger2/src/stdio.rs b/crates/trigger2/src/stdio.rs new file mode 100644 index 000000000..fe62c6393 --- /dev/null +++ b/crates/trigger2/src/stdio.rs @@ -0,0 +1,331 @@ +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + task::Poll, +}; + +use anyhow::{Context, Result}; +use spin_common::ui::quoted_path; +use spin_factors_executor::ExecutorHooks; +use tokio::io::AsyncWrite; + +use crate::factors::TriggerFactors; + +/// Which components should have their logs followed on stdout/stderr. +#[derive(Clone, Debug)] +pub enum FollowComponents { + /// No components should have their logs followed. + None, + /// Only the specified components should have their logs followed. + Named(HashSet), + /// All components should have their logs followed. + All, +} + +impl FollowComponents { + /// Whether a given component should have its logs followed on stdout/stderr. + pub fn should_follow(&self, component_id: &str) -> bool { + match self { + Self::None => false, + Self::All => true, + Self::Named(ids) => ids.contains(component_id), + } + } +} + +impl Default for FollowComponents { + fn default() -> Self { + Self::None + } +} + +/// Implements TriggerHooks, writing logs to a log file and (optionally) stderr +pub struct StdioLoggingExecutorHooks { + follow_components: FollowComponents, + log_dir: Option, +} + +impl StdioLoggingExecutorHooks { + pub fn new(follow_components: FollowComponents, log_dir: Option) -> Self { + Self { + follow_components, + log_dir, + } + } + + fn component_stdio_writer( + &self, + component_id: &str, + log_suffix: &str, + log_dir: Option<&Path>, + ) -> Result { + let sanitized_component_id = sanitize_filename::sanitize(component_id); + let log_path = log_dir + .map(|log_dir| log_dir.join(format!("{sanitized_component_id}_{log_suffix}.txt",))); + let log_path = log_path.as_deref(); + + let follow = self.follow_components.should_follow(component_id); + match log_path { + Some(log_path) => ComponentStdioWriter::new_forward(log_path, follow) + .with_context(|| format!("Failed to open log file {}", quoted_path(log_path))), + None => ComponentStdioWriter::new_inherit(), + } + } + + fn validate_follows(&self, app: &spin_app::App) -> anyhow::Result<()> { + match &self.follow_components { + FollowComponents::Named(names) => { + let component_ids: HashSet<_> = + app.components().map(|c| c.id().to_owned()).collect(); + let unknown_names: Vec<_> = names.difference(&component_ids).collect(); + if unknown_names.is_empty() { + Ok(()) + } else { + let unknown_list = bullet_list(&unknown_names); + let actual_list = bullet_list(&component_ids); + let message = anyhow::anyhow!("The following component(s) specified in --follow do not exist in the application:\n{unknown_list}\nThe following components exist:\n{actual_list}"); + Err(message) + } + } + _ => Ok(()), + } + } +} + +impl ExecutorHooks for StdioLoggingExecutorHooks { + fn configure_app( + &mut self, + configured_app: &spin_factors::ConfiguredApp, + ) -> anyhow::Result<()> { + self.validate_follows(configured_app.app())?; + if let Some(dir) = &self.log_dir { + // Ensure log dir exists if set + std::fs::create_dir_all(dir) + .with_context(|| format!("Failed to create log dir {}", quoted_path(dir)))?; + + println!("Logging component stdio to {}", quoted_path(dir.join(""))) + } + Ok(()) + } + + fn prepare_instance( + &self, + builder: &mut spin_factors_executor::FactorsInstanceBuilder, + ) -> anyhow::Result<()> { + let component_id = builder.app_component().id().to_string(); + let wasi_builder = builder.factor_builders().wasi(); + wasi_builder.stdout_pipe(self.component_stdio_writer( + &component_id, + "stdout", + self.log_dir.as_deref(), + )?); + wasi_builder.stderr_pipe(self.component_stdio_writer( + &component_id, + "stderr", + self.log_dir.as_deref(), + )?); + Ok(()) + } +} + +/// ComponentStdioWriter forwards output to a log file, (optionally) stderr, and (optionally) to a +/// tracing compatibility layer. +pub struct ComponentStdioWriter { + inner: ComponentStdioWriterInner, +} + +enum ComponentStdioWriterInner { + /// Inherit stdout/stderr from the parent process. + Inherit, + /// Forward stdout/stderr to a file in addition to the inherited stdout/stderr. + Forward { + sync_file: std::fs::File, + async_file: tokio::fs::File, + state: ComponentStdioWriterState, + follow: bool, + }, +} + +#[derive(Debug)] +enum ComponentStdioWriterState { + File, + Follow(std::ops::Range), +} + +impl ComponentStdioWriter { + fn new_forward(log_path: &Path, follow: bool) -> anyhow::Result { + let sync_file = std::fs::File::options() + .create(true) + .append(true) + .open(log_path)?; + + let async_file = sync_file + .try_clone() + .context("could not get async file handle")? + .into(); + + Ok(Self { + inner: ComponentStdioWriterInner::Forward { + sync_file, + async_file, + state: ComponentStdioWriterState::File, + follow, + }, + }) + } + + fn new_inherit() -> anyhow::Result { + Ok(Self { + inner: ComponentStdioWriterInner::Inherit, + }) + } +} + +impl AsyncWrite for ComponentStdioWriter { + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = self.get_mut(); + + loop { + match &mut this.inner { + ComponentStdioWriterInner::Inherit => { + let written = futures::ready!( + std::pin::Pin::new(&mut tokio::io::stderr()).poll_write(cx, buf) + ); + let written = match written { + Ok(w) => w, + Err(e) => return Poll::Ready(Err(e)), + }; + return Poll::Ready(Ok(written)); + } + ComponentStdioWriterInner::Forward { + async_file, + state, + follow, + .. + } => match &state { + ComponentStdioWriterState::File => { + let written = + futures::ready!(std::pin::Pin::new(async_file).poll_write(cx, buf)); + let written = match written { + Ok(w) => w, + Err(e) => return Poll::Ready(Err(e)), + }; + if *follow { + *state = ComponentStdioWriterState::Follow(0..written); + } else { + return Poll::Ready(Ok(written)); + } + } + ComponentStdioWriterState::Follow(range) => { + let written = futures::ready!(std::pin::Pin::new(&mut tokio::io::stderr()) + .poll_write(cx, &buf[range.clone()])); + let written = match written { + Ok(w) => w, + Err(e) => return Poll::Ready(Err(e)), + }; + if range.start + written >= range.end { + let end = range.end; + *state = ComponentStdioWriterState::File; + return Poll::Ready(Ok(end)); + } else { + *state = ComponentStdioWriterState::Follow( + (range.start + written)..range.end, + ); + }; + } + }, + } + } + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + + match &mut this.inner { + ComponentStdioWriterInner::Inherit => { + std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) + } + ComponentStdioWriterInner::Forward { + async_file, state, .. + } => match state { + ComponentStdioWriterState::File => std::pin::Pin::new(async_file).poll_flush(cx), + ComponentStdioWriterState::Follow(_) => { + std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) + } + }, + } + } + + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + + match &mut this.inner { + ComponentStdioWriterInner::Inherit => { + std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) + } + ComponentStdioWriterInner::Forward { + async_file, state, .. + } => match state { + ComponentStdioWriterState::File => std::pin::Pin::new(async_file).poll_shutdown(cx), + ComponentStdioWriterState::Follow(_) => { + std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) + } + }, + } + } +} + +impl std::io::Write for ComponentStdioWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + spin_telemetry::logs::handle_app_log(buf); + + match &mut self.inner { + ComponentStdioWriterInner::Inherit => { + std::io::stderr().write_all(buf)?; + Ok(buf.len()) + } + ComponentStdioWriterInner::Forward { + sync_file, follow, .. + } => { + let written = sync_file.write(buf)?; + if *follow { + std::io::stderr().write_all(&buf[..written])?; + } + Ok(written) + } + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match &mut self.inner { + ComponentStdioWriterInner::Inherit => std::io::stderr().flush(), + ComponentStdioWriterInner::Forward { + sync_file, follow, .. + } => { + sync_file.flush()?; + if *follow { + std::io::stderr().flush()?; + } + Ok(()) + } + } + } +} + +fn bullet_list(items: impl IntoIterator) -> String { + items + .into_iter() + .map(|item| format!(" - {item}")) + .collect::>() + .join("\n") +} diff --git a/run-factors-tests.sh b/run-factors-tests.sh index 497a0f319..3b9ca4e42 100755 --- a/run-factors-tests.sh +++ b/run-factors-tests.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # TODO(factors): Remove after enabling CI for factors branch -cargo test -p '*factor*' +cargo test -p '*factor*' -p spin-trigger2 -p spin-trigger-http2 From ae28895da2b3bade60b93e626c881c868500688a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 14:24:57 -0400 Subject: [PATCH 113/195] factors: Split HttpHandlerExecutor into variants Signed-off-by: Lann Martin --- crates/trigger-http2/src/handler.rs | 411 ------------------------- crates/trigger-http2/src/headers.rs | 158 ++++++++++ crates/trigger-http2/src/instrument.rs | 21 +- crates/trigger-http2/src/lib.rs | 9 +- crates/trigger-http2/src/server.rs | 159 +++++----- crates/trigger-http2/src/spin.rs | 107 +++++++ crates/trigger-http2/src/wagi.rs | 6 +- crates/trigger-http2/src/wasi.rs | 166 ++++++++++ 8 files changed, 525 insertions(+), 512 deletions(-) delete mode 100644 crates/trigger-http2/src/handler.rs create mode 100644 crates/trigger-http2/src/headers.rs create mode 100644 crates/trigger-http2/src/spin.rs create mode 100644 crates/trigger-http2/src/wasi.rs diff --git a/crates/trigger-http2/src/handler.rs b/crates/trigger-http2/src/handler.rs deleted file mode 100644 index 1631c5e16..000000000 --- a/crates/trigger-http2/src/handler.rs +++ /dev/null @@ -1,411 +0,0 @@ -use std::{net::SocketAddr, str, str::FromStr}; - -use anyhow::{anyhow, Context, Result}; -use futures::TryFutureExt; -use http::{HeaderName, HeaderValue}; -use http_body_util::BodyExt; -use hyper::{Request, Response}; -use spin_core::{Component, Instance}; -use spin_factor_outbound_http::wasi_2023_10_18::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_10_18; -use spin_factor_outbound_http::wasi_2023_11_10::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_11_10; -use spin_http::body; -use spin_http::routes::RouteMatch; -use spin_world::v1::http_types; -use tokio::{sync::oneshot, task}; -use tracing::{instrument, Instrument, Level}; -use wasmtime_wasi_http::{body::HyperIncomingBody as Body, proxy::Proxy, WasiHttpView}; - -use crate::{server::HttpExecutor, Store, TriggerInstanceBuilder}; - -#[derive(Clone)] -pub struct HttpHandlerExecutor { - pub handler_type: HandlerType, -} - -impl HttpExecutor for HttpHandlerExecutor { - #[instrument(name = "spin_trigger_http.execute_wasm", skip_all, err(level = Level::INFO), fields(otel.name = format!("execute_wasm_component {}", route_match.component_id())))] - async fn execute( - &self, - instance_builder: TriggerInstanceBuilder<'_>, - route_match: &RouteMatch, - req: Request, - client_addr: SocketAddr, - ) -> Result> { - let component_id = route_match.component_id(); - - tracing::trace!("Executing request using the Spin executor for component {component_id}"); - - let (instance, store) = instance_builder.instantiate(()).await?; - - let resp = match self.handler_type { - HandlerType::Spin => self - .execute_spin(store, instance, route_match, req, client_addr) - .await - .map_err(contextualise_err)?, - _ => { - self.execute_wasi(store, instance, route_match, req, client_addr) - .await? - } - }; - - tracing::info!( - "Request finished, sending response with status code {}", - resp.status() - ); - Ok(resp) - } -} - -impl HttpHandlerExecutor { - pub async fn execute_spin( - &self, - mut store: Store, - instance: Instance, - route_match: &RouteMatch, - req: Request, - client_addr: SocketAddr, - ) -> Result> { - let headers = Self::headers(&req, route_match, client_addr)?; - let func = instance - .exports(&mut store) - .instance("fermyon:spin/inbound-http") - // Safe since we have already checked that this instance exists - .expect("no fermyon:spin/inbound-http found") - .typed_func::<(http_types::Request,), (http_types::Response,)>("handle-request")?; - - let (parts, body) = req.into_parts(); - let bytes = body.collect().await?.to_bytes().to_vec(); - - let method = if let Some(method) = Self::method(&parts.method) { - method - } else { - return Ok(Response::builder() - .status(http::StatusCode::METHOD_NOT_ALLOWED) - .body(body::empty())?); - }; - - // Preparing to remove the params field. We are leaving it in place for now - // to avoid breaking the ABI, but no longer pass or accept values in it. - // https://github.com/fermyon/spin/issues/663 - let params = vec![]; - - let uri = match parts.uri.path_and_query() { - Some(u) => u.to_string(), - None => parts.uri.to_string(), - }; - - let req = http_types::Request { - method, - uri, - headers, - params, - body: Some(bytes), - }; - - let (resp,) = func.call_async(&mut store, (req,)).await?; - - if resp.status < 100 || resp.status > 600 { - tracing::error!("malformed HTTP status code"); - return Ok(Response::builder() - .status(http::StatusCode::INTERNAL_SERVER_ERROR) - .body(body::empty())?); - }; - - let mut response = http::Response::builder().status(resp.status); - if let Some(headers) = response.headers_mut() { - Self::append_headers(headers, resp.headers)?; - } - - let body = match resp.body { - Some(b) => body::full(b.into()), - None => body::empty(), - }; - - Ok(response.body(body)?) - } - - fn method(m: &http::Method) -> Option { - Some(match *m { - http::Method::GET => http_types::Method::Get, - http::Method::POST => http_types::Method::Post, - http::Method::PUT => http_types::Method::Put, - http::Method::DELETE => http_types::Method::Delete, - http::Method::PATCH => http_types::Method::Patch, - http::Method::HEAD => http_types::Method::Head, - http::Method::OPTIONS => http_types::Method::Options, - _ => return None, - }) - } - - async fn execute_wasi( - &self, - mut store: Store, - instance: Instance, - route_match: &RouteMatch, - mut req: Request, - client_addr: SocketAddr, - ) -> anyhow::Result> { - let headers = Self::headers(&req, route_match, client_addr)?; - req.headers_mut().clear(); - req.headers_mut() - .extend(headers.into_iter().filter_map(|(n, v)| { - let Ok(name) = n.parse::() else { - return None; - }; - let Ok(value) = HeaderValue::from_bytes(v.as_bytes()) else { - return None; - }; - Some((name, value)) - })); - - let mut wasi_http = spin_factor_outbound_http::OutboundHttpFactor::get_wasi_http_impl( - store.data_mut().factors_instance_state(), - ) - .context("missing OutboundHttpFactor")?; - - let request = wasi_http.new_incoming_request(req)?; - - let (response_tx, response_rx) = oneshot::channel(); - let response = wasi_http.new_response_outparam(response_tx)?; - - drop(wasi_http); - - enum Handler { - Latest(Proxy), - Handler2023_11_10(IncomingHandler2023_11_10), - Handler2023_10_18(IncomingHandler2023_10_18), - } - - let handler = - { - let mut exports = instance.exports(&mut store); - match self.handler_type { - HandlerType::Wasi2023_10_18 => { - let mut instance = exports - .instance(WASI_HTTP_EXPORT_2023_10_18) - .ok_or_else(|| { - anyhow!("export of `{WASI_HTTP_EXPORT_2023_10_18}` not an instance") - })?; - Handler::Handler2023_10_18(IncomingHandler2023_10_18::new(&mut instance)?) - } - HandlerType::Wasi2023_11_10 => { - let mut instance = exports - .instance(WASI_HTTP_EXPORT_2023_11_10) - .ok_or_else(|| { - anyhow!("export of `{WASI_HTTP_EXPORT_2023_11_10}` not an instance") - })?; - Handler::Handler2023_11_10(IncomingHandler2023_11_10::new(&mut instance)?) - } - HandlerType::Wasi0_2 => { - drop(exports); - Handler::Latest(Proxy::new(&mut store, &instance)?) - } - HandlerType::Spin => panic!("should have used execute_spin instead"), - } - }; - - let span = tracing::debug_span!("execute_wasi"); - let handle = task::spawn( - async move { - let result = match handler { - Handler::Latest(proxy) => { - proxy - .wasi_http_incoming_handler() - .call_handle(&mut store, request, response) - .instrument(span) - .await - } - Handler::Handler2023_10_18(handler) => { - handler - .call_handle(&mut store, request, response) - .instrument(span) - .await - } - Handler::Handler2023_11_10(handler) => { - handler - .call_handle(&mut store, request, response) - .instrument(span) - .await - } - }; - - tracing::trace!( - "wasi-http memory consumed: {}", - store.data().core_state().memory_consumed() - ); - - result - } - .in_current_span(), - ); - - match response_rx.await { - Ok(response) => { - task::spawn( - async move { - handle - .await - .context("guest invocation panicked")? - .context("guest invocation failed")?; - - Ok(()) - } - .map_err(|e: anyhow::Error| { - tracing::warn!("component error after response: {e:?}"); - }), - ); - - Ok(response.context("guest failed to produce a response")?) - } - - Err(_) => { - handle - .await - .context("guest invocation panicked")? - .context("guest invocation failed")?; - - Err(anyhow!( - "guest failed to produce a response prior to returning" - )) - } - } - } - - fn headers( - req: &Request, - route_match: &RouteMatch, - client_addr: SocketAddr, - ) -> Result> { - let mut res = Vec::new(); - for (name, value) in req - .headers() - .iter() - .map(|(name, value)| (name.to_string(), std::str::from_utf8(value.as_bytes()))) - { - let value = value?.to_string(); - res.push((name, value)); - } - - let default_host = http::HeaderValue::from_str("localhost")?; - let host = std::str::from_utf8( - req.headers() - .get("host") - .unwrap_or(&default_host) - .as_bytes(), - )?; - - // Set the environment information (path info, base path, etc) as headers. - // In the future, we might want to have this information in a context - // object as opposed to headers. - for (keys, val) in - crate::server::compute_default_headers(req.uri(), host, route_match, client_addr)? - { - res.push((Self::prepare_header_key(&keys[0]), val)); - } - - Ok(res) - } - - fn prepare_header_key(key: &str) -> String { - key.replace('_', "-").to_ascii_lowercase() - } - - fn append_headers(res: &mut http::HeaderMap, src: Option>) -> Result<()> { - if let Some(src) = src { - for (k, v) in src.iter() { - res.insert( - http::header::HeaderName::from_str(k)?, - http::header::HeaderValue::from_str(v)?, - ); - } - }; - - Ok(()) - } -} - -/// Whether this handler uses the custom Spin http handler interface for wasi-http -#[derive(Copy, Clone)] -pub enum HandlerType { - Spin, - Wasi0_2, - Wasi2023_11_10, - Wasi2023_10_18, -} - -const WASI_HTTP_EXPORT_2023_10_18: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-10-18"; -const WASI_HTTP_EXPORT_2023_11_10: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-11-10"; -const WASI_HTTP_EXPORT_0_2_0: &str = "wasi:http/incoming-handler@0.2.0"; - -impl HandlerType { - /// Determine the handler type from the exports of a component - pub fn from_component( - engine: impl AsRef, - component: &Component, - ) -> Result { - let mut handler_ty = None; - - let mut set = |ty: HandlerType| { - if handler_ty.is_none() { - handler_ty = Some(ty); - Ok(()) - } else { - Err(anyhow!( - "component exports multiple different handlers but \ - it's expected to export only one" - )) - } - }; - let ty = component.component_type(); - for (name, _) in ty.exports(engine.as_ref()) { - match name { - WASI_HTTP_EXPORT_2023_10_18 => set(HandlerType::Wasi2023_10_18)?, - WASI_HTTP_EXPORT_2023_11_10 => set(HandlerType::Wasi2023_11_10)?, - WASI_HTTP_EXPORT_0_2_0 => set(HandlerType::Wasi0_2)?, - "fermyon:spin/inbound-http" => set(HandlerType::Spin)?, - _ => {} - } - } - - handler_ty.ok_or_else(|| { - anyhow!( - "Expected component to either export `{WASI_HTTP_EXPORT_2023_10_18}`, \ - `{WASI_HTTP_EXPORT_2023_11_10}`, `{WASI_HTTP_EXPORT_0_2_0}`, \ - or `fermyon:spin/inbound-http` but it exported none of those" - ) - }) - } -} - -fn contextualise_err(e: anyhow::Error) -> anyhow::Error { - if e.to_string() - .contains("failed to find function export `canonical_abi_free`") - { - e.context( - "component is not compatible with Spin executor - should this use the Wagi executor?", - ) - } else { - e - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_spin_header_keys() { - assert_eq!( - HttpHandlerExecutor::prepare_header_key("SPIN_FULL_URL"), - "spin-full-url".to_string() - ); - assert_eq!( - HttpHandlerExecutor::prepare_header_key("SPIN_PATH_INFO"), - "spin-path-info".to_string() - ); - assert_eq!( - HttpHandlerExecutor::prepare_header_key("SPIN_RAW_COMPONENT_ROUTE"), - "spin-raw-component-route".to_string() - ); - } -} diff --git a/crates/trigger-http2/src/headers.rs b/crates/trigger-http2/src/headers.rs new file mode 100644 index 000000000..d9a2843c9 --- /dev/null +++ b/crates/trigger-http2/src/headers.rs @@ -0,0 +1,158 @@ +use std::{net::SocketAddr, str, str::FromStr}; + +use anyhow::Result; +use http::Uri; +use hyper::Request; +use spin_http::routes::RouteMatch; +use spin_outbound_networking::is_service_chaining_host; + +use crate::Body; + +// We need to make the following pieces of information available to both executors. +// While the values we set are identical, the way they are passed to the +// modules is going to be different, so each executor must must use the info +// in its standardized way (environment variables for the Wagi executor, and custom headers +// for the Spin HTTP executor). +pub const FULL_URL: [&str; 2] = ["SPIN_FULL_URL", "X_FULL_URL"]; +pub const PATH_INFO: [&str; 2] = ["SPIN_PATH_INFO", "PATH_INFO"]; +pub const MATCHED_ROUTE: [&str; 2] = ["SPIN_MATCHED_ROUTE", "X_MATCHED_ROUTE"]; +pub const COMPONENT_ROUTE: [&str; 2] = ["SPIN_COMPONENT_ROUTE", "X_COMPONENT_ROUTE"]; +pub const RAW_COMPONENT_ROUTE: [&str; 2] = ["SPIN_RAW_COMPONENT_ROUTE", "X_RAW_COMPONENT_ROUTE"]; +pub const BASE_PATH: [&str; 2] = ["SPIN_BASE_PATH", "X_BASE_PATH"]; +pub const CLIENT_ADDR: [&str; 2] = ["SPIN_CLIENT_ADDR", "X_CLIENT_ADDR"]; + +pub fn compute_default_headers( + uri: &Uri, + host: &str, + route_match: &RouteMatch, + client_addr: SocketAddr, +) -> anyhow::Result> { + fn owned(strs: &[&'static str; 2]) -> [String; 2] { + [strs[0].to_owned(), strs[1].to_owned()] + } + + let owned_full_url: [String; 2] = owned(&FULL_URL); + let owned_path_info: [String; 2] = owned(&PATH_INFO); + let owned_matched_route: [String; 2] = owned(&MATCHED_ROUTE); + let owned_component_route: [String; 2] = owned(&COMPONENT_ROUTE); + let owned_raw_component_route: [String; 2] = owned(&RAW_COMPONENT_ROUTE); + let owned_base_path: [String; 2] = owned(&BASE_PATH); + let owned_client_addr: [String; 2] = owned(&CLIENT_ADDR); + + let mut res = vec![]; + let abs_path = uri + .path_and_query() + .expect("cannot get path and query") + .as_str(); + + let path_info = route_match.trailing_wildcard(); + + let scheme = uri.scheme_str().unwrap_or("http"); + + let full_url = format!("{}://{}{}", scheme, host, abs_path); + + res.push((owned_path_info, path_info)); + res.push((owned_full_url, full_url)); + res.push((owned_matched_route, route_match.based_route().to_string())); + + res.push((owned_base_path, "/".to_string())); + res.push(( + owned_raw_component_route, + route_match.raw_route().to_string(), + )); + res.push((owned_component_route, route_match.raw_route_or_prefix())); + res.push((owned_client_addr, client_addr.to_string())); + + for (wild_name, wild_value) in route_match.named_wildcards() { + let wild_header = format!("SPIN_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer + let wild_wagi_header = format!("X_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer + res.push(([wild_header, wild_wagi_header], wild_value.clone())); + } + + Ok(res) +} + +pub fn strip_forbidden_headers(req: &mut Request) { + let headers = req.headers_mut(); + if let Some(host_header) = headers.get("Host") { + if let Ok(host) = host_header.to_str() { + if is_service_chaining_host(host) { + headers.remove("Host"); + } + } + } +} + +pub fn prepare_request_headers( + req: &Request, + route_match: &RouteMatch, + client_addr: SocketAddr, +) -> Result> { + let mut res = Vec::new(); + for (name, value) in req + .headers() + .iter() + .map(|(name, value)| (name.to_string(), std::str::from_utf8(value.as_bytes()))) + { + let value = value?.to_string(); + res.push((name, value)); + } + + let default_host = http::HeaderValue::from_str("localhost")?; + let host = std::str::from_utf8( + req.headers() + .get("host") + .unwrap_or(&default_host) + .as_bytes(), + )?; + + // Set the environment information (path info, base path, etc) as headers. + // In the future, we might want to have this information in a context + // object as opposed to headers. + for (keys, val) in compute_default_headers(req.uri(), host, route_match, client_addr)? { + res.push((prepare_header_key(&keys[0]), val)); + } + + Ok(res) +} + +pub fn append_headers( + map: &mut http::HeaderMap, + headers: Option>, +) -> Result<()> { + if let Some(src) = headers { + for (k, v) in src.iter() { + map.insert( + http::header::HeaderName::from_str(k)?, + http::header::HeaderValue::from_str(v)?, + ); + } + }; + + Ok(()) +} + +fn prepare_header_key(key: &str) -> String { + key.replace('_', "-").to_ascii_lowercase() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_spin_header_keys() { + assert_eq!( + prepare_header_key("SPIN_FULL_URL"), + "spin-full-url".to_string() + ); + assert_eq!( + prepare_header_key("SPIN_PATH_INFO"), + "spin-path-info".to_string() + ); + assert_eq!( + prepare_header_key("SPIN_RAW_COMPONENT_ROUTE"), + "spin-raw-component-route".to_string() + ); + } +} diff --git a/crates/trigger-http2/src/instrument.rs b/crates/trigger-http2/src/instrument.rs index aa5b3e0e0..89c5aa69d 100644 --- a/crates/trigger-http2/src/instrument.rs +++ b/crates/trigger-http2/src/instrument.rs @@ -1,7 +1,8 @@ use anyhow::Result; use http::Response; use tracing::Level; -use wasmtime_wasi_http::body::HyperIncomingBody; + +use crate::Body; /// Create a span for an HTTP request. macro_rules! http_span { @@ -30,12 +31,17 @@ pub(crate) use http_span; /// Finish setting attributes on the HTTP span. pub(crate) fn finalize_http_span( - response: Result>, + response: Result>, method: String, -) -> Result> { +) -> Result> { let span = tracing::Span::current(); match response { Ok(response) => { + tracing::info!( + "Request finished, sending response with status code {}", + response.status() + ); + let matched_route = response.extensions().get::(); // Set otel.name and http.route if let Some(MatchedRoute { route }) = matched_route { @@ -74,19 +80,16 @@ pub struct MatchedRoute { } impl MatchedRoute { - pub fn set_response_extension( - resp: &mut Response, - route: impl Into, - ) { + pub fn set_response_extension(resp: &mut Response, route: impl Into) { resp.extensions_mut().insert(MatchedRoute { route: route.into(), }); } pub fn with_response_extension( - mut resp: Response, + mut resp: Response, route: impl Into, - ) -> Response { + ) -> Response { Self::set_response_extension(&mut resp, route); resp } diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index f2b9c26b5..86d00ace1 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -1,10 +1,12 @@ //! Implementation for the Spin HTTP engine. -mod handler; +mod headers; mod instrument; mod server; +mod spin; mod tls; mod wagi; +mod wasi; use std::{ collections::HashMap, @@ -27,9 +29,10 @@ use server::HttpServer; pub use tls::TlsConfig; +pub(crate) use wasmtime_wasi_http::body::HyperIncomingBody as Body; + pub(crate) type TriggerApp = spin_trigger2::TriggerApp; pub(crate) type TriggerInstanceBuilder<'a> = spin_trigger2::TriggerInstanceBuilder<'a, HttpTrigger>; -pub(crate) type Store = spin_trigger2::Store; #[derive(Args)] pub struct CliArgs { @@ -211,7 +214,7 @@ mod tests { use anyhow::Result; use http::Request; - use super::{server::*, *}; + use super::{headers::*, *}; #[test] fn test_default_headers() -> Result<()> { diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index c4f4a881a..cc7c80a9e 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -15,20 +15,22 @@ use spin_http::{ config::{HttpExecutorType, HttpTriggerConfig}, routes::{RouteMatch, Router}, }; -use spin_outbound_networking::is_service_chaining_host; use tokio::{ io::{AsyncRead, AsyncWrite}, net::TcpListener, task, }; use tracing::Instrument; -use wasmtime_wasi_http::body::{HyperIncomingBody as Body, HyperOutgoingBody}; +use wasmtime::component::Component; +use wasmtime_wasi_http::body::HyperOutgoingBody; use crate::{ - handler::{HandlerType, HttpHandlerExecutor}, + headers::strip_forbidden_headers, instrument::{finalize_http_span, http_span, instrument_error, MatchedRoute}, + spin::SpinHttpExecutor, wagi::WagiHttpExecutor, - NotFoundRouteKind, TlsConfig, TriggerApp, TriggerInstanceBuilder, + wasi::WasiHttpExecutor, + Body, NotFoundRouteKind, TlsConfig, TriggerApp, TriggerInstanceBuilder, }; pub struct HttpServer { @@ -144,13 +146,22 @@ impl HttpServer { .unwrap_or(&HttpExecutorType::Http); let res = match executor { - HttpExecutorType::Http => { - HttpHandlerExecutor { - handler_type: *handler_type, + HttpExecutorType::Http => match handler_type { + HandlerType::Spin => { + SpinHttpExecutor + .execute(instance_builder, &route_match, req, client_addr) + .await } - .execute(instance_builder, &route_match, req, client_addr) - .await - } + HandlerType::Wasi0_2 + | HandlerType::Wasi2023_11_10 + | HandlerType::Wasi2023_10_18 => { + WasiHttpExecutor { + handler_type: *handler_type, + } + .execute(instance_builder, &route_match, req, client_addr) + .await + } + }, HttpExecutorType::Wagi(wagi_config) => { let executor = WagiHttpExecutor { wagi_config: wagi_config.clone(), @@ -299,81 +310,6 @@ fn set_req_uri(req: &mut Request, scheme: Scheme, addr: SocketAddr) -> any Ok(()) } -pub fn strip_forbidden_headers(req: &mut Request) { - let headers = req.headers_mut(); - if let Some(host_header) = headers.get("Host") { - if let Ok(host) = host_header.to_str() { - if is_service_chaining_host(host) { - headers.remove("Host"); - } - } - } -} - -// We need to make the following pieces of information available to both executors. -// While the values we set are identical, the way they are passed to the -// modules is going to be different, so each executor must must use the info -// in its standardized way (environment variables for the Wagi executor, and custom headers -// for the Spin HTTP executor). -pub const FULL_URL: [&str; 2] = ["SPIN_FULL_URL", "X_FULL_URL"]; -pub const PATH_INFO: [&str; 2] = ["SPIN_PATH_INFO", "PATH_INFO"]; -pub const MATCHED_ROUTE: [&str; 2] = ["SPIN_MATCHED_ROUTE", "X_MATCHED_ROUTE"]; -pub const COMPONENT_ROUTE: [&str; 2] = ["SPIN_COMPONENT_ROUTE", "X_COMPONENT_ROUTE"]; -pub const RAW_COMPONENT_ROUTE: [&str; 2] = ["SPIN_RAW_COMPONENT_ROUTE", "X_RAW_COMPONENT_ROUTE"]; -pub const BASE_PATH: [&str; 2] = ["SPIN_BASE_PATH", "X_BASE_PATH"]; -pub const CLIENT_ADDR: [&str; 2] = ["SPIN_CLIENT_ADDR", "X_CLIENT_ADDR"]; - -pub(crate) fn compute_default_headers( - uri: &Uri, - host: &str, - route_match: &RouteMatch, - client_addr: SocketAddr, -) -> anyhow::Result> { - fn owned(strs: &[&'static str; 2]) -> [String; 2] { - [strs[0].to_owned(), strs[1].to_owned()] - } - - let owned_full_url: [String; 2] = owned(&FULL_URL); - let owned_path_info: [String; 2] = owned(&PATH_INFO); - let owned_matched_route: [String; 2] = owned(&MATCHED_ROUTE); - let owned_component_route: [String; 2] = owned(&COMPONENT_ROUTE); - let owned_raw_component_route: [String; 2] = owned(&RAW_COMPONENT_ROUTE); - let owned_base_path: [String; 2] = owned(&BASE_PATH); - let owned_client_addr: [String; 2] = owned(&CLIENT_ADDR); - - let mut res = vec![]; - let abs_path = uri - .path_and_query() - .expect("cannot get path and query") - .as_str(); - - let path_info = route_match.trailing_wildcard(); - - let scheme = uri.scheme_str().unwrap_or("http"); - - let full_url = format!("{}://{}{}", scheme, host, abs_path); - - res.push((owned_path_info, path_info)); - res.push((owned_full_url, full_url)); - res.push((owned_matched_route, route_match.based_route().to_string())); - - res.push((owned_base_path, "/".to_string())); - res.push(( - owned_raw_component_route, - route_match.raw_route().to_string(), - )); - res.push((owned_component_route, route_match.raw_route_or_prefix())); - res.push((owned_client_addr, client_addr.to_string())); - - for (wild_name, wild_value) in route_match.named_wildcards() { - let wild_header = format!("SPIN_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer - let wild_wagi_header = format!("X_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer - res.push(([wild_header, wild_wagi_header], wild_value.clone())); - } - - Ok(res) -} - /// An HTTP executor. pub(crate) trait HttpExecutor: Clone + Send + Sync + 'static { fn execute( @@ -384,3 +320,56 @@ pub(crate) trait HttpExecutor: Clone + Send + Sync + 'static { client_addr: SocketAddr, ) -> impl Future>>; } + +/// Whether this handler uses the custom Spin http handler interface for wasi-http +#[derive(Copy, Clone)] +pub enum HandlerType { + Spin, + Wasi0_2, + Wasi2023_11_10, + Wasi2023_10_18, +} + +pub const WASI_HTTP_EXPORT_2023_10_18: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-10-18"; +pub const WASI_HTTP_EXPORT_2023_11_10: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-11-10"; +pub const WASI_HTTP_EXPORT_0_2_0: &str = "wasi:http/incoming-handler@0.2.0"; + +impl HandlerType { + /// Determine the handler type from the exports of a component + pub fn from_component( + engine: impl AsRef, + component: &Component, + ) -> anyhow::Result { + let mut handler_ty = None; + + let mut set = |ty: HandlerType| { + if handler_ty.is_none() { + handler_ty = Some(ty); + Ok(()) + } else { + Err(anyhow::anyhow!( + "component exports multiple different handlers but \ + it's expected to export only one" + )) + } + }; + let ty = component.component_type(); + for (name, _) in ty.exports(engine.as_ref()) { + match name { + WASI_HTTP_EXPORT_2023_10_18 => set(HandlerType::Wasi2023_10_18)?, + WASI_HTTP_EXPORT_2023_11_10 => set(HandlerType::Wasi2023_11_10)?, + WASI_HTTP_EXPORT_0_2_0 => set(HandlerType::Wasi0_2)?, + "fermyon:spin/inbound-http" => set(HandlerType::Spin)?, + _ => {} + } + } + + handler_ty.ok_or_else(|| { + anyhow::anyhow!( + "Expected component to either export `{WASI_HTTP_EXPORT_2023_10_18}`, \ + `{WASI_HTTP_EXPORT_2023_11_10}`, `{WASI_HTTP_EXPORT_0_2_0}`, \ + or `fermyon:spin/inbound-http` but it exported none of those" + ) + }) + } +} diff --git a/crates/trigger-http2/src/spin.rs b/crates/trigger-http2/src/spin.rs new file mode 100644 index 000000000..10c854b20 --- /dev/null +++ b/crates/trigger-http2/src/spin.rs @@ -0,0 +1,107 @@ +use std::net::SocketAddr; + +use anyhow::Result; +use http_body_util::BodyExt; +use hyper::{Request, Response}; +use spin_http::body; +use spin_http::routes::RouteMatch; +use spin_world::v1::http_types; +use tracing::{instrument, Level}; + +use crate::{ + headers::{append_headers, prepare_request_headers}, + server::HttpExecutor, + Body, TriggerInstanceBuilder, +}; + +/// An [`HttpExecutor`] that uses the `fermyon:spin/inbound-http` interface. +#[derive(Clone)] +pub struct SpinHttpExecutor; + +impl HttpExecutor for SpinHttpExecutor { + #[instrument(name = "spin_trigger_http.execute_wasm", skip_all, err(level = Level::INFO), fields(otel.name = format!("execute_wasm_component {}", route_match.component_id())))] + async fn execute( + &self, + instance_builder: TriggerInstanceBuilder<'_>, + route_match: &RouteMatch, + req: Request, + client_addr: SocketAddr, + ) -> Result> { + let component_id = route_match.component_id(); + + tracing::trace!("Executing request using the Spin executor for component {component_id}"); + + let (instance, mut store) = instance_builder.instantiate(()).await?; + + let headers = prepare_request_headers(&req, route_match, client_addr)?; + let func = instance + .exports(&mut store) + .instance("fermyon:spin/inbound-http") + // Safe since we have already checked that this instance exists + .expect("no fermyon:spin/inbound-http found") + .typed_func::<(http_types::Request,), (http_types::Response,)>("handle-request")?; + + let (parts, body) = req.into_parts(); + let bytes = body.collect().await?.to_bytes().to_vec(); + + let method = if let Some(method) = convert_method(&parts.method) { + method + } else { + return Ok(Response::builder() + .status(http::StatusCode::METHOD_NOT_ALLOWED) + .body(body::empty())?); + }; + + // Preparing to remove the params field. We are leaving it in place for now + // to avoid breaking the ABI, but no longer pass or accept values in it. + // https://github.com/fermyon/spin/issues/663 + let params = vec![]; + + let uri = match parts.uri.path_and_query() { + Some(u) => u.to_string(), + None => parts.uri.to_string(), + }; + + let req = http_types::Request { + method, + uri, + headers, + params, + body: Some(bytes), + }; + + let (resp,) = func.call_async(&mut store, (req,)).await?; + + if resp.status < 100 || resp.status > 600 { + tracing::error!("malformed HTTP status code"); + return Ok(Response::builder() + .status(http::StatusCode::INTERNAL_SERVER_ERROR) + .body(body::empty())?); + }; + + let mut response = http::Response::builder().status(resp.status); + if let Some(headers) = response.headers_mut() { + append_headers(headers, resp.headers)?; + } + + let body = match resp.body { + Some(b) => body::full(b.into()), + None => body::empty(), + }; + + Ok(response.body(body)?) + } +} + +fn convert_method(m: &http::Method) -> Option { + Some(match *m { + http::Method::GET => http_types::Method::Get, + http::Method::POST => http_types::Method::Post, + http::Method::PUT => http_types::Method::Put, + http::Method::DELETE => http_types::Method::Delete, + http::Method::PATCH => http_types::Method::Patch, + http::Method::HEAD => http_types::Method::Head, + http::Method::OPTIONS => http_types::Method::Options, + _ => return None, + }) +} diff --git a/crates/trigger-http2/src/wagi.rs b/crates/trigger-http2/src/wagi.rs index 7a0a3701a..c0a28485f 100644 --- a/crates/trigger-http2/src/wagi.rs +++ b/crates/trigger-http2/src/wagi.rs @@ -8,7 +8,7 @@ use tracing::{instrument, Level}; use wasmtime_wasi::pipe::MemoryOutputPipe; use wasmtime_wasi_http::body::HyperIncomingBody as Body; -use crate::{server::HttpExecutor, TriggerInstanceBuilder}; +use crate::{headers::compute_default_headers, server::HttpExecutor, TriggerInstanceBuilder}; #[derive(Clone)] pub struct WagiHttpExecutor { @@ -67,9 +67,7 @@ impl HttpExecutor for WagiHttpExecutor { // This sets the current environment variables Wagi expects (such as // `PATH_INFO`, or `X_FULL_URL`). // Note that this overrides any existing headers previously set by Wagi. - for (keys, val) in - crate::server::compute_default_headers(&parts.uri, host, route_match, client_addr)? - { + for (keys, val) in compute_default_headers(&parts.uri, host, route_match, client_addr)? { headers.insert(keys[1].to_string(), val); } diff --git a/crates/trigger-http2/src/wasi.rs b/crates/trigger-http2/src/wasi.rs new file mode 100644 index 000000000..05d6b33b7 --- /dev/null +++ b/crates/trigger-http2/src/wasi.rs @@ -0,0 +1,166 @@ +use std::net::SocketAddr; + +use anyhow::{anyhow, Context, Result}; +use futures::TryFutureExt; +use http::{HeaderName, HeaderValue}; +use hyper::{Request, Response}; +use spin_factor_outbound_http::wasi_2023_10_18::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_10_18; +use spin_factor_outbound_http::wasi_2023_11_10::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_11_10; +use spin_http::routes::RouteMatch; +use tokio::{sync::oneshot, task}; +use tracing::{instrument, Instrument, Level}; +use wasmtime_wasi_http::{body::HyperIncomingBody as Body, proxy::Proxy, WasiHttpView}; + +use crate::{ + headers::prepare_request_headers, + server::{HandlerType, HttpExecutor, WASI_HTTP_EXPORT_2023_10_18, WASI_HTTP_EXPORT_2023_11_10}, + TriggerInstanceBuilder, +}; + +/// An [`HttpExecutor`] that uses the `wasi:http/incoming-handler` interface. +#[derive(Clone)] +pub struct WasiHttpExecutor { + pub handler_type: HandlerType, +} + +impl HttpExecutor for WasiHttpExecutor { + #[instrument(name = "spin_trigger_http.execute_wasm", skip_all, err(level = Level::INFO), fields(otel.name = format!("execute_wasm_component {}", route_match.component_id())))] + async fn execute( + &self, + instance_builder: TriggerInstanceBuilder<'_>, + route_match: &RouteMatch, + mut req: Request, + client_addr: SocketAddr, + ) -> Result> { + let component_id = route_match.component_id(); + + tracing::trace!("Executing request using the Wasi executor for component {component_id}"); + + let (instance, mut store) = instance_builder.instantiate(()).await?; + + let headers = prepare_request_headers(&req, route_match, client_addr)?; + req.headers_mut().clear(); + req.headers_mut() + .extend(headers.into_iter().filter_map(|(n, v)| { + let Ok(name) = n.parse::() else { + return None; + }; + let Ok(value) = HeaderValue::from_bytes(v.as_bytes()) else { + return None; + }; + Some((name, value)) + })); + + let mut wasi_http = spin_factor_outbound_http::OutboundHttpFactor::get_wasi_http_impl( + store.data_mut().factors_instance_state(), + ) + .context("missing OutboundHttpFactor")?; + + let request = wasi_http.new_incoming_request(req)?; + + let (response_tx, response_rx) = oneshot::channel(); + let response = wasi_http.new_response_outparam(response_tx)?; + + drop(wasi_http); + + enum Handler { + Latest(Proxy), + Handler2023_11_10(IncomingHandler2023_11_10), + Handler2023_10_18(IncomingHandler2023_10_18), + } + + let handler = + { + let mut exports = instance.exports(&mut store); + match self.handler_type { + HandlerType::Wasi2023_10_18 => { + let mut instance = exports + .instance(WASI_HTTP_EXPORT_2023_10_18) + .ok_or_else(|| { + anyhow!("export of `{WASI_HTTP_EXPORT_2023_10_18}` not an instance") + })?; + Handler::Handler2023_10_18(IncomingHandler2023_10_18::new(&mut instance)?) + } + HandlerType::Wasi2023_11_10 => { + let mut instance = exports + .instance(WASI_HTTP_EXPORT_2023_11_10) + .ok_or_else(|| { + anyhow!("export of `{WASI_HTTP_EXPORT_2023_11_10}` not an instance") + })?; + Handler::Handler2023_11_10(IncomingHandler2023_11_10::new(&mut instance)?) + } + HandlerType::Wasi0_2 => { + drop(exports); + Handler::Latest(Proxy::new(&mut store, &instance)?) + } + HandlerType::Spin => panic!("should have used execute_spin instead"), + } + }; + + let span = tracing::debug_span!("execute_wasi"); + let handle = task::spawn( + async move { + let result = match handler { + Handler::Latest(proxy) => { + proxy + .wasi_http_incoming_handler() + .call_handle(&mut store, request, response) + .instrument(span) + .await + } + Handler::Handler2023_10_18(handler) => { + handler + .call_handle(&mut store, request, response) + .instrument(span) + .await + } + Handler::Handler2023_11_10(handler) => { + handler + .call_handle(&mut store, request, response) + .instrument(span) + .await + } + }; + + tracing::trace!( + "wasi-http memory consumed: {}", + store.data().core_state().memory_consumed() + ); + + result + } + .in_current_span(), + ); + + match response_rx.await { + Ok(response) => { + task::spawn( + async move { + handle + .await + .context("guest invocation panicked")? + .context("guest invocation failed")?; + + Ok(()) + } + .map_err(|e: anyhow::Error| { + tracing::warn!("component error after response: {e:?}"); + }), + ); + + Ok(response.context("guest failed to produce a response")?) + } + + Err(_) => { + handle + .await + .context("guest invocation panicked")? + .context("guest invocation failed")?; + + Err(anyhow!( + "guest failed to produce a response prior to returning" + )) + } + } + } +} From 824d3eb34ba6281982e6d0e0cda7881bfff8ae57 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 1 Aug 2024 15:54:45 -0400 Subject: [PATCH 114/195] factor-outbound-networking: Add support for client TLS Signed-off-by: Lann Martin --- Cargo.lock | 76 +++++ crates/factor-outbound-networking/Cargo.toml | 16 + crates/factor-outbound-networking/src/lib.rs | 30 +- .../src/runtime_config.rs | 250 ++++++++++++++++ .../src/runtime_config/spin.rs | 273 ++++++++++++++++++ .../testdata/invalid-cert.pem | 20 ++ .../testdata/invalid-private-key.pem | 5 + .../testdata/valid-cert.pem | 21 ++ .../testdata/valid-private-key.pem | 5 + .../factor-sqlite/src/runtime_config/spin.rs | 15 +- crates/factors/src/runtime_config/toml.rs | 6 + 11 files changed, 708 insertions(+), 9 deletions(-) create mode 100644 crates/factor-outbound-networking/src/runtime_config.rs create mode 100644 crates/factor-outbound-networking/src/runtime_config/spin.rs create mode 100644 crates/factor-outbound-networking/testdata/invalid-cert.pem create mode 100644 crates/factor-outbound-networking/testdata/invalid-private-key.pem create mode 100644 crates/factor-outbound-networking/testdata/valid-cert.pem create mode 100644 crates/factor-outbound-networking/testdata/valid-private-key.pem diff --git a/Cargo.lock b/Cargo.lock index 6580b5082..a40322f9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -517,6 +517,33 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "aws-lc-rs" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "axum" version = "0.6.20" @@ -753,12 +780,15 @@ dependencies = [ "itertools 0.12.1", "lazy_static 1.4.0", "lazycell", + "log", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", "syn 2.0.58", + "which", ] [[package]] @@ -4661,6 +4691,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + [[package]] name = "monostate" version = "0.1.11" @@ -6736,6 +6772,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls" +version = "0.23.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebbbdb961df0ad3f2652da8f3fdc4b36122f568f968f45ad3316f26c025c677b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.102.2", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" version = "0.7.0" @@ -6790,6 +6841,7 @@ version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -7671,16 +7723,26 @@ dependencies = [ name = "spin-factor-outbound-networking" version = "2.7.0-pre0" dependencies = [ + "anyhow", "futures-util", + "http 1.1.0", "ipnet", + "rustls 0.23.7", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "serde 1.0.197", "spin-factor-variables", "spin-factor-wasi", "spin-factors", "spin-factors-test", "spin-outbound-networking", + "spin-serde", + "tempfile", "tokio", + "toml 0.8.14", "tracing", "wasmtime-wasi", + "webpki-roots 0.26.1", ] [[package]] @@ -11153,6 +11215,20 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] [[package]] name = "zip" diff --git a/crates/factor-outbound-networking/Cargo.toml b/crates/factor-outbound-networking/Cargo.toml index c24284d13..e232d06ec 100644 --- a/crates/factor-outbound-networking/Cargo.toml +++ b/crates/factor-outbound-networking/Cargo.toml @@ -5,19 +5,35 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] +anyhow = "1" futures-util = "0.3" +http = "1.1.0" ipnet = "2.9.0" +rustls = "0.23" +rustls-pemfile = { version = "2.1.2", optional = true } +rustls-pki-types = "1.7.0" +serde = { version = "1", features = ["derive"] } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } spin-factors = { path = "../factors" } # TODO: merge with this crate spin-outbound-networking = { path = "../outbound-networking" } +spin-serde = { path = "../serde" } tracing = { workspace = true } +webpki-roots = "0.26" [dev-dependencies] spin-factors-test = { path = "../factors-test" } +tempfile = "3.10.1" tokio = { version = "1", features = ["macros", "rt"] } +toml = "0.8" wasmtime-wasi = { workspace = true } +[features] +default = ["spin-cli"] +# Includes the runtime configuration handling used by the Spin CLI +spin-cli = [ + "dep:rustls-pemfile", +] [lints] workspace = true diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index 9d7f04b2d..fdcb1ffed 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -1,9 +1,12 @@ +pub mod runtime_config; + use std::{collections::HashMap, sync::Arc}; use futures_util::{ future::{BoxFuture, Shared}, FutureExt, }; +use runtime_config::RuntimeConfig; use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{SocketAddrUse, WasiFactor}; use spin_factors::{ @@ -15,18 +18,20 @@ use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; pub use spin_outbound_networking::OutboundUrl; +pub use runtime_config::ComponentTlsConfigs; + pub type SharedFutureResult = Shared, Arc>>>; pub struct OutboundNetworkingFactor; impl Factor for OutboundNetworkingFactor { - type RuntimeConfig = (); + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceBuilder; fn configure_app( &self, - ctx: ConfigureAppContext, + mut ctx: ConfigureAppContext, ) -> anyhow::Result { // Extract allowed_outbound_hosts for all components let component_allowed_hosts = ctx @@ -43,8 +48,16 @@ impl Factor for OutboundNetworkingFactor { )) }) .collect::>()?; + + let runtime_config = match ctx.take_runtime_config() { + Some(cfg) => cfg, + // The default RuntimeConfig provides default TLS client configs + None => RuntimeConfig::new([])?, + }; + Ok(AppState { component_allowed_hosts, + runtime_config, }) } @@ -100,18 +113,27 @@ impl Factor for OutboundNetworkingFactor { Err(Error::NoSuchFactor(_)) => (), // no WasiFactor to configure; that's OK Err(err) => return Err(err.into()), } + + let component_tls_configs = ctx + .app_state() + .runtime_config + .get_component_tls_configs(ctx.app_component().id()); + Ok(InstanceBuilder { allowed_hosts_future, + component_tls_configs, }) } } pub struct AppState { component_allowed_hosts: HashMap>, + runtime_config: RuntimeConfig, } pub struct InstanceBuilder { allowed_hosts_future: SharedFutureResult, + component_tls_configs: ComponentTlsConfigs, } impl InstanceBuilder { @@ -120,6 +142,10 @@ impl InstanceBuilder { allowed_hosts_future: self.allowed_hosts_future.clone(), } } + + pub fn component_tls_configs(&self) -> &ComponentTlsConfigs { + &self.component_tls_configs + } } impl FactorInstanceBuilder for InstanceBuilder { diff --git a/crates/factor-outbound-networking/src/runtime_config.rs b/crates/factor-outbound-networking/src/runtime_config.rs new file mode 100644 index 000000000..711b33731 --- /dev/null +++ b/crates/factor-outbound-networking/src/runtime_config.rs @@ -0,0 +1,250 @@ +#[cfg(feature = "spin-cli")] +pub mod spin; + +use std::{collections::HashMap, str::FromStr, sync::Arc}; + +use anyhow::{ensure, Context}; +use rustls::{ClientConfig, RootCertStore}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; + +/// Runtime configuration for outbound networking. +#[derive(Debug)] +pub struct RuntimeConfig { + // Maps component ID -> HostClientConfigs + component_host_client_configs: HashMap, + default_client_config: Arc, +} + +// Maps host authority -> ClientConfig +type HostClientConfigs = Arc>>; + +impl RuntimeConfig { + pub fn new(tls_configs: impl IntoIterator) -> anyhow::Result { + let mut component_host_client_configs = HashMap::::new(); + for tls_config in tls_configs { + ensure!( + !tls_config.components.is_empty(), + "client TLS 'components' list may not be empty" + ); + ensure!( + !tls_config.hosts.is_empty(), + "client TLS 'hosts' list may not be empty" + ); + let client_config = Arc::new(tls_config.to_client_config()?); + for component in &tls_config.components { + let host_configs = component_host_client_configs + .entry(component.clone()) + .or_default(); + for host in &tls_config.hosts { + validate_host(host)?; + // First matching (component, host) pair wins + Arc::get_mut(host_configs) + .unwrap() + .entry(host.clone()) + .or_insert_with(|| client_config.clone()); + } + } + } + + let default_client_config = Arc::new(TlsConfig::default().to_client_config()?); + + Ok(Self { + component_host_client_configs, + default_client_config, + }) + } + + /// Returns [`ComponentTlsConfigs`] for the given component. + pub fn get_component_tls_configs(&self, component_id: &str) -> ComponentTlsConfigs { + let host_client_configs = self + .component_host_client_configs + .get(component_id) + .cloned(); + ComponentTlsConfigs { + host_client_configs, + default_client_config: self.default_client_config.clone(), + } + } + + /// Returns a [`ClientConfig`] for the given component and host authority. + /// + /// This is a convenience method, equivalent to: + /// `.get_client_config(component_id).get_client_config(host)` + pub fn get_client_config(&self, component_id: &str, host: &str) -> Arc { + let component_config = self.get_component_tls_configs(component_id); + component_config.get_client_config(host).clone() + } +} + +pub(crate) fn validate_host(host: &str) -> anyhow::Result<()> { + // Validate hostname + http::uri::Authority::from_str(host).with_context(|| format!("invalid TLS 'host' {host:?}"))?; + if host.contains(':') { + anyhow::bail!("invalid TLS 'host' {host:?}; ports not currently supported"); + } + Ok(()) +} + +#[derive(Clone)] +pub struct ComponentTlsConfigs { + host_client_configs: Option, + default_client_config: Arc, +} + +impl ComponentTlsConfigs { + /// Returns a [`ClientConfig`] for the given host authority. + pub fn get_client_config(&self, host: &str) -> &Arc { + self.host_client_configs + .as_ref() + .and_then(|configs| configs.get(host)) + .unwrap_or(&self.default_client_config) + } +} + +#[derive(Debug)] +pub struct ClientCertConfig { + cert_chain: Vec>, + key_der: PrivateKeyDer<'static>, +} + +#[derive(Debug)] +pub struct TlsConfig { + pub components: Vec, + pub hosts: Vec, + pub root_certificates: Vec>, + pub use_webpki_roots: bool, + pub client_cert: Option, +} + +impl Default for TlsConfig { + fn default() -> Self { + Self { + components: vec![], + hosts: vec![], + root_certificates: vec![], + // Use webpki roots by default + use_webpki_roots: true, + client_cert: None, + } + } +} + +impl TlsConfig { + fn to_client_config(&self) -> anyhow::Result { + let mut root_store = RootCertStore::empty(); + if self.use_webpki_roots { + root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + } + for ca in &self.root_certificates { + root_store.add(ca.clone())?; + } + + let builder = ClientConfig::builder().with_root_certificates(root_store); + + if let Some(ClientCertConfig { + cert_chain, + key_der, + }) = &self.client_cert + { + Ok(builder.with_client_auth_cert(cert_chain.clone(), key_der.clone_key())?) + } else { + Ok(builder.with_no_client_auth()) + } + } +} + +#[cfg(test)] +mod tests { + use std::{io::BufReader, path::Path}; + + use anyhow::Context; + + use super::*; + + #[test] + fn test_empty_config() -> anyhow::Result<()> { + let runtime_config = RuntimeConfig::new([])?; + // Just make sure the default path doesn't panic + runtime_config.get_client_config("foo", "bar"); + Ok(()) + } + + #[test] + fn test_minimal_config() -> anyhow::Result<()> { + let runtime_config = RuntimeConfig::new([TlsConfig { + components: vec!["test-component".into()], + hosts: vec!["test-host".into()], + root_certificates: vec![], + use_webpki_roots: false, + client_cert: None, + }])?; + let client_config = runtime_config.get_client_config("test-component", "test-host"); + // Check that we didn't just get the default + let default_config = runtime_config.get_client_config("other_component", "test-host"); + assert!(!Arc::ptr_eq(&client_config, &default_config)); + Ok(()) + } + + #[test] + fn test_maximal_config() -> anyhow::Result<()> { + let test_certs = test_certs()?; + let test_key = test_key()?; + let runtime_config = RuntimeConfig::new([TlsConfig { + components: vec!["test-component".into()], + hosts: vec!["test-host".into()], + root_certificates: vec![test_certs[0].clone()], + use_webpki_roots: false, + client_cert: Some(ClientCertConfig { + cert_chain: test_certs, + key_der: test_key, + }), + }])?; + let client_config = runtime_config.get_client_config("test-component", "test-host"); + assert!(client_config.client_auth_cert_resolver.has_certs()); + Ok(()) + } + + #[test] + fn test_config_overrides() -> anyhow::Result<()> { + let test_certs = test_certs()?; + let test_key = test_key()?; + let runtime_config = RuntimeConfig::new([ + TlsConfig { + components: vec!["test-component1".into()], + hosts: vec!["test-host".into()], + client_cert: Some(ClientCertConfig { + cert_chain: test_certs, + key_der: test_key, + }), + ..Default::default() + }, + TlsConfig { + components: vec!["test-component1".into(), "test-component2".into()], + hosts: vec!["test-host".into()], + ..Default::default() + }, + ])?; + // First match wins + let client_config1 = runtime_config.get_client_config("test-component1", "test-host"); + assert!(client_config1.client_auth_cert_resolver.has_certs()); + + // Correctly select by differing component ID + let client_config2 = runtime_config.get_client_config("test-component-2", "test-host"); + assert!(!client_config2.client_auth_cert_resolver.has_certs()); + Ok(()) + } + + const TESTDATA_DIR: &'static str = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata"); + + fn test_certs() -> anyhow::Result>> { + let file = std::fs::File::open(Path::new(TESTDATA_DIR).join("valid-cert.pem"))?; + rustls_pemfile::certs(&mut BufReader::new(file)) + .map(|res| res.map_err(Into::into)) + .collect() + } + + fn test_key() -> anyhow::Result> { + let file = std::fs::File::open(Path::new(TESTDATA_DIR).join("valid-private-key.pem"))?; + rustls_pemfile::private_key(&mut BufReader::new(file))?.context("no private key") + } +} diff --git a/crates/factor-outbound-networking/src/runtime_config/spin.rs b/crates/factor-outbound-networking/src/runtime_config/spin.rs new file mode 100644 index 000000000..dc524d2a6 --- /dev/null +++ b/crates/factor-outbound-networking/src/runtime_config/spin.rs @@ -0,0 +1,273 @@ +use anyhow::{bail, ensure, Context}; +use serde::{Deserialize, Deserializer}; +use spin_factors::runtime_config::toml::GetTomlValue; +use std::io; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use super::{validate_host, TlsConfig}; + +pub struct SpinTlsRuntimeConfig { + runtime_config_dir: PathBuf, +} + +impl SpinTlsRuntimeConfig { + pub fn new(runtime_config_dir: impl Into) -> Self { + Self { + runtime_config_dir: runtime_config_dir.into(), + } + } + + /// Get the runtime configuration for client TLS from a TOML table. + /// + /// Expects table to be in the format: + /// ````toml + /// [[client_tls]] + /// component_ids = ["example-component"] + /// hosts = ["example.com"] + /// ca_use_webpki_roots = true + /// ca_roots_file = "path/to/roots.crt" + /// client_cert_file = "path/to/client.crt" + /// client_private_key_file = "path/to/client.key" + /// ``` + pub fn config_from_table( + &self, + table: &T, + ) -> anyhow::Result> { + let Some(tls_configs) = self.tls_configs_from_table(table)? else { + return Ok(None); + }; + let runtime_config = super::RuntimeConfig::new(tls_configs)?; + Ok(Some(runtime_config)) + } + + fn tls_configs_from_table( + &self, + table: &T, + ) -> anyhow::Result>> { + let Some(array) = table.get("client_tls") else { + return Ok(None); + }; + let toml_configs: Vec = array.clone().try_into()?; + + let tls_configs = toml_configs + .into_iter() + .map(|toml_config| self.load_tls_config(toml_config)) + .collect::>>()?; + Ok(Some(tls_configs)) + } + + fn load_tls_config(&self, toml_config: RuntimeConfigToml) -> anyhow::Result { + let RuntimeConfigToml { + component_ids, + hosts, + ca_use_webpki_roots, + ca_roots_file, + client_cert_file, + client_private_key_file, + } = toml_config; + ensure!( + !component_ids.is_empty(), + "[[client_tls]] 'component_ids' list may not be empty" + ); + ensure!( + !hosts.is_empty(), + "[[client_tls]] 'hosts' list may not be empty" + ); + + let components = component_ids.into_iter().map(Into::into).collect(); + + let hosts = hosts + .iter() + .map(|host| { + let host: &str = host; + host.parse() + .map_err(|err| anyhow::anyhow!("invalid host {host:?}: {err:?}")) + }) + .collect::>>()?; + + let use_webpki_roots = if let Some(ca_use_webpki_roots) = ca_use_webpki_roots { + ca_use_webpki_roots + } else { + // Use webpki roots by default *unless* explicit roots were given + ca_roots_file.is_none() + }; + + let root_certificates = ca_roots_file + .map(|path| self.load_certs(path)) + .transpose()? + .unwrap_or_default(); + + let client_cert = match (client_cert_file, client_private_key_file) { + (Some(cert_path), Some(key_path)) => Some(super::ClientCertConfig { + cert_chain: self.load_certs(cert_path)?, + key_der: self.load_key(key_path)?, + }), + (None, None) => None, + (Some(_), None) => bail!("client_cert_file specified without client_private_key_file"), + (None, Some(_)) => bail!("client_private_key_file specified without client_cert_file"), + }; + + Ok(TlsConfig { + components, + hosts, + root_certificates, + use_webpki_roots, + client_cert, + }) + } + + // Parse certs from the provided file + fn load_certs( + &self, + path: impl AsRef, + ) -> io::Result>> { + let path = self.runtime_config_dir.join(path); + rustls_pemfile::certs(&mut io::BufReader::new(fs::File::open(path).map_err( + |err| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("failed to read cert file {:?}", err), + ) + }, + )?)) + .collect::>>>() + } + + // Parse a private key from the provided file + fn load_key( + &self, + path: impl AsRef, + ) -> anyhow::Result> { + let path = self.runtime_config_dir.join(path); + rustls_pemfile::private_key(&mut io::BufReader::new( + fs::File::open(path).context("loading private key")?, + )) + .map_err(|_| anyhow::anyhow!("invalid input")) + .transpose() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "private key file contains no private keys", + ) + })? + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeConfigToml { + component_ids: Vec, + #[serde(deserialize_with = "deserialize_hosts")] + hosts: Vec, + ca_use_webpki_roots: Option, + ca_roots_file: Option, + client_cert_file: Option, + client_private_key_file: Option, +} + +fn deserialize_hosts<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let hosts = Vec::::deserialize(deserializer)?; + for host in &hosts { + validate_host(host).map_err(serde::de::Error::custom)?; + } + Ok(hosts) +} + +#[cfg(test)] +mod tests { + use super::*; + + const TESTDATA_DIR: &'static str = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata"); + + #[test] + fn test_min_config() -> anyhow::Result<()> { + let config = SpinTlsRuntimeConfig::new("/doesnt-matter"); + + let tls_configs = config + .tls_configs_from_table(&toml::toml! { + [[client_tls]] + component_ids = ["test-component"] + hosts = ["test-host"] + + })? + .context("missing config section")?; + assert_eq!(tls_configs.len(), 1); + + assert_eq!(tls_configs[0].components, ["test-component"]); + assert_eq!(tls_configs[0].hosts[0].as_str(), "test-host"); + assert!(tls_configs[0].use_webpki_roots); + Ok(()) + } + + #[test] + fn test_max_config() -> anyhow::Result<()> { + let config = SpinTlsRuntimeConfig::new(TESTDATA_DIR); + + let tls_configs = config + .tls_configs_from_table(&toml::toml! { + [[client_tls]] + component_ids = ["test-component"] + hosts = ["test-host"] + ca_use_webpki_roots = true + ca_roots_file = "valid-cert.pem" + client_cert_file = "valid-cert.pem" + client_private_key_file = "valid-private-key.pem" + })? + .context("missing config section")?; + assert_eq!(tls_configs.len(), 1); + + assert!(tls_configs[0].use_webpki_roots); + assert_eq!(tls_configs[0].root_certificates.len(), 2); + assert!(tls_configs[0].client_cert.is_some()); + Ok(()) + } + + #[test] + fn test_use_webpki_roots_default_with_explicit_roots() -> anyhow::Result<()> { + let config = SpinTlsRuntimeConfig::new(TESTDATA_DIR); + + let tls_configs = config + .tls_configs_from_table(&toml::toml! { + [[client_tls]] + component_ids = ["test-component"] + hosts = ["test-host"] + ca_roots_file = "valid-cert.pem" + })? + .context("missing config section")?; + + assert!(!tls_configs[0].use_webpki_roots); + Ok(()) + } + + #[test] + fn test_invalid_cert() { + let config = SpinTlsRuntimeConfig::new(TESTDATA_DIR); + + config + .tls_configs_from_table(&toml::toml! { + [[client_tls]] + component_ids = ["test-component"] + hosts = ["test-host"] + ca_roots_file = "invalid-cert.pem" + }) + .unwrap_err(); + } + + #[test] + fn test_invalid_private_key() { + let config = SpinTlsRuntimeConfig::new(TESTDATA_DIR); + + config + .tls_configs_from_table(&toml::toml! { + [[client_tls]] + component_ids = ["test-component"] + hosts = ["test-host"] + client_cert_file = "valid-cert.pem" + client_private_key_file = "invalid-key.pem" + }) + .unwrap_err(); + } +} diff --git a/crates/factor-outbound-networking/testdata/invalid-cert.pem b/crates/factor-outbound-networking/testdata/invalid-cert.pem new file mode 100644 index 000000000..f1a952b9c --- /dev/null +++ b/crates/factor-outbound-networking/testdata/invalid-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIBkjCCATegAwIBAgIIEOURVvWgx1AwCgYIKoZIzj0EAwIwIzEhMB8GA1UEAwwY +azNzLWNsaWVudC1jYUAxNzE3NzgwNTIwMB4XDTI0MDYwNzE3MTUyMFoXDTI1MDYw +NzE3MTUyMFowMDEXMBUGA1UEChMOc3lzdGVtOm1hc3RlcnMxFTATBgNVBAMTDHN5 +c3RlbTphZG1pbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFGE/CVuauj8kmde +i4AagSJ5GYgGnL0eF55ItiXrKSjMmsIf/N8EyeamxQfWPKVk/1xhH7cS9GcQgNe6 +XrRvmLyjSDBGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAf +BgNVHSMEGDAWgBRpihySeW3DafmU1cw6LMnQCQDD4jAKBggqhkjOPQQDAgNJADBG +AiEA/db1wb4mVrqJVctqbPU9xd0bXzJx7cBDzpWgPP9ISfkCIQDNyuskAkXvUMHH +F73/GJnh8Bt2H38qyzThM8nlR9v1eQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3MtY2xp +ZW50LWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw +WjAjMSEwHwYDVQQDDBhrM3MtY2xpZW50LWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO +PQIBBggqhkjOPQMBBwNCAASozciE0YGl8ak3G0Ll1riwXSScfpK0QRle/cFizdlA +HgDowBssBcla0/2a/eWabxqTPzsZH0cVhL7Tialoj8GNo0IwQDAOBgNVHQ8BAf8E +BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUaYocknltw2n5lNXMOizJ +0AkAw+IwCgYIKoZIzj0EAwIDRwAwRAIgR8YcLA8cH4qAMDRPDsJqLaw4GJFkgjwV +TCrMgyUxSvACIBwyklgm7mgHcC5WM9CqmliAGZJyV0xRPZBK01POrNf0 diff --git a/crates/factor-outbound-networking/testdata/invalid-private-key.pem b/crates/factor-outbound-networking/testdata/invalid-private-key.pem new file mode 100644 index 000000000..39d7e59ee --- /dev/null +++ b/crates/factor-outbound-networking/testdata/invalid-private-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIA+FBtmKJbd8wBGOWeuJQfHiCKjjXF8ywEPrvj8S1N3VoAoGCCqGSM49 +AwEHoUQDQgAEUYT8JW5q6PySZ16LgBqBInkZiAacvR4Xnki2JespKMyawh/83wTJ +5qbFB9Y8pWT/XGEftxL0ZxCA17petG+YvA== +-----END EC PRIVATE KEY- \ No newline at end of file diff --git a/crates/factor-outbound-networking/testdata/valid-cert.pem b/crates/factor-outbound-networking/testdata/valid-cert.pem new file mode 100644 index 000000000..e75166d0e --- /dev/null +++ b/crates/factor-outbound-networking/testdata/valid-cert.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIBkjCCATegAwIBAgIIEOURVvWgx1AwCgYIKoZIzj0EAwIwIzEhMB8GA1UEAwwY +azNzLWNsaWVudC1jYUAxNzE3NzgwNTIwMB4XDTI0MDYwNzE3MTUyMFoXDTI1MDYw +NzE3MTUyMFowMDEXMBUGA1UEChMOc3lzdGVtOm1hc3RlcnMxFTATBgNVBAMTDHN5 +c3RlbTphZG1pbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFGE/CVuauj8kmde +i4AagSJ5GYgGnL0eF55ItiXrKSjMmsIf/N8EyeamxQfWPKVk/1xhH7cS9GcQgNe6 +XrRvmLyjSDBGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAf +BgNVHSMEGDAWgBRpihySeW3DafmU1cw6LMnQCQDD4jAKBggqhkjOPQQDAgNJADBG +AiEA/db1wb4mVrqJVctqbPU9xd0bXzJx7cBDzpWgPP9ISfkCIQDNyuskAkXvUMHH +F73/GJnh8Bt2H38qyzThM8nlR9v1eQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3MtY2xp +ZW50LWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw +WjAjMSEwHwYDVQQDDBhrM3MtY2xpZW50LWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO +PQIBBggqhkjOPQMBBwNCAASozciE0YGl8ak3G0Ll1riwXSScfpK0QRle/cFizdlA +HgDowBssBcla0/2a/eWabxqTPzsZH0cVhL7Tialoj8GNo0IwQDAOBgNVHQ8BAf8E +BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUaYocknltw2n5lNXMOizJ +0AkAw+IwCgYIKoZIzj0EAwIDRwAwRAIgR8YcLA8cH4qAMDRPDsJqLaw4GJFkgjwV +TCrMgyUxSvACIBwyklgm7mgHcC5WM9CqmliAGZJyV0xRPZBK01POrNf0 +-----END CERTIFICATE----- \ No newline at end of file diff --git a/crates/factor-outbound-networking/testdata/valid-private-key.pem b/crates/factor-outbound-networking/testdata/valid-private-key.pem new file mode 100644 index 000000000..2820fbed2 --- /dev/null +++ b/crates/factor-outbound-networking/testdata/valid-private-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIA+FBtmKJbd8wBGOWeuJQfHiCKjjXF8ywEPrvj8S1N3VoAoGCCqGSM49 +AwEHoUQDQgAEUYT8JW5q6PySZ16LgBqBInkZiAacvR4Xnki2JespKMyawh/83wTJ +5qbFB9Y8pWT/XGEftxL0ZxCA17petG+YvA== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 9dc6a23e0..bc8eff92a 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -28,13 +28,14 @@ impl SpinSqliteRuntimeConfig { /// Create a new `SpinSqliteRuntimeConfig` /// /// This takes as arguments: - /// * the directory to use as the default location for SQLite databases. Usually this - /// will be the path to the `.spin` state directory. - /// * the *absolute* path to the directory from which relative paths to local SQLite - /// databases are resolved. (this should most likely be the path to the runtime-config - /// file or the current working dir). + /// * the directory to use as the default location for SQLite databases. + /// Usually this will be the path to the `.spin` state directory. + /// * the *absolute* path to the directory from which relative paths to + /// local SQLite databases are resolved. (this should most likely be the + /// path to the runtime-config file or the current working dir). /// - /// Panics if either `default_database_dir` or `local_database_dir` are not absolute paths. + /// Panics if either `default_database_dir` or `local_database_dir` are not + /// absolute paths. pub fn new(default_database_dir: PathBuf, local_database_dir: PathBuf) -> Self { assert!( default_database_dir.is_absolute(), @@ -58,7 +59,7 @@ impl SpinSqliteRuntimeConfig { /// type = "$database-type" /// ... extra type specific configuration ... /// ``` - pub fn config_from_table>( + pub fn config_from_table( &self, table: &T, ) -> anyhow::Result> { diff --git a/crates/factors/src/runtime_config/toml.rs b/crates/factors/src/runtime_config/toml.rs index 15e801a47..42231a148 100644 --- a/crates/factors/src/runtime_config/toml.rs +++ b/crates/factors/src/runtime_config/toml.rs @@ -7,6 +7,12 @@ pub trait GetTomlValue { fn get(&self, key: &str) -> Option<&toml::Value>; } +impl GetTomlValue for toml::Table { + fn get(&self, key: &str) -> Option<&toml::Value> { + self.get(key) + } +} + /// A helper for tracking which keys have been used in a TOML table. pub struct TomlKeyTracker<'a> { unused_keys: RefCell>, From c7a7319538fb8ea02e548496cec6bc2cb1284e4c Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 2 Aug 2024 10:53:31 -0400 Subject: [PATCH 115/195] Constify some test paths Signed-off-by: Lann Martin --- crates/trigger-http2/src/tls.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/trigger-http2/src/tls.rs b/crates/trigger-http2/src/tls.rs index 0f75eaac8..39f6ca3d3 100644 --- a/crates/trigger-http2/src/tls.rs +++ b/crates/trigger-http2/src/tls.rs @@ -69,13 +69,11 @@ fn load_key(path: impl AsRef) -> io::Result PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata") - } + const TESTDATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata"); #[test] fn test_read_non_existing_cert() { - let path = test_datadir().join("non-existing-file.pem"); + let path = Path::new(TESTDATA_DIR).join("non-existing-file.pem"); let certs = load_certs(path); assert!(certs.is_err()); @@ -84,7 +82,7 @@ mod tests { #[test] fn test_read_invalid_cert() { - let path = test_datadir().join("invalid-cert.pem"); + let path = Path::new(TESTDATA_DIR).join("invalid-cert.pem"); let certs = load_certs(path); assert!(certs.is_err()); @@ -96,7 +94,7 @@ mod tests { #[test] fn test_read_valid_cert() { - let path = test_datadir().join("valid-cert.pem"); + let path = Path::new(TESTDATA_DIR).join("valid-cert.pem"); let certs = load_certs(path); assert!(certs.is_ok()); @@ -105,8 +103,7 @@ mod tests { #[test] fn test_read_non_existing_private_key() { - let mut path = test_datadir(); - path.push("non-existing-file.pem"); + let path = Path::new(TESTDATA_DIR).join("non-existing-file.pem"); let keys = load_key(path); assert!(keys.is_err()); @@ -115,8 +112,7 @@ mod tests { #[test] fn test_read_invalid_private_key() { - let mut path = test_datadir(); - path.push("invalid-private-key.pem"); + let path = Path::new(TESTDATA_DIR).join("invalid-private-key.pem"); let keys = load_key(path); assert!(keys.is_err()); @@ -125,8 +121,7 @@ mod tests { #[test] fn test_read_valid_private_key() { - let mut path = test_datadir(); - path.push("valid-private-key.pem"); + let path = Path::new(TESTDATA_DIR).join("valid-private-key.pem"); let keys = load_key(path); assert!(keys.is_ok()); From 2c172ca3bd42b9a22ad4dfb4b2dc97cc1ce47e10 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 12 Aug 2024 10:19:56 +0200 Subject: [PATCH 116/195] SQLite make sure connection is not used across instances Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/host.rs | 26 +++++---- crates/factor-sqlite/src/lib.rs | 55 ++++++++++--------- crates/factor-sqlite/src/runtime_config.rs | 6 +- .../factor-sqlite/src/runtime_config/spin.rs | 26 +++++---- crates/factor-sqlite/tests/factor_test.rs | 6 +- 5 files changed, 65 insertions(+), 54 deletions(-) diff --git a/crates/factor-sqlite/src/host.rs b/crates/factor-sqlite/src/host.rs index e12817625..8911c37c4 100644 --- a/crates/factor-sqlite/src/host.rs +++ b/crates/factor-sqlite/src/host.rs @@ -8,12 +8,12 @@ use spin_factors::{anyhow, SelfInstanceBuilder}; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; -use crate::{Connection, ConnectionPool}; +use crate::{Connection, ConnectionCreator}; pub struct InstanceState { allowed_databases: Arc>, - connections: table::Table>, - get_pool: ConnectionPoolGetter, + connections: table::Table>, + get_connection_creator: ConnectionCreatorGetter, } impl InstanceState { @@ -22,25 +22,29 @@ impl InstanceState { } } -/// A function that takes a database label and returns a connection pool, if one exists. -pub type ConnectionPoolGetter = Arc Option> + Send + Sync>; +/// A function that takes a database label and returns a connection creator, if one exists. +pub type ConnectionCreatorGetter = + Arc Option> + Send + Sync>; impl InstanceState { /// Create a new `InstanceState` /// - /// Takes the list of allowed databases, and a function for getting a connection pool given a database label. - pub fn new(allowed_databases: Arc>, get_pool: ConnectionPoolGetter) -> Self { + /// Takes the list of allowed databases, and a function for getting a connection creator given a database label. + pub fn new( + allowed_databases: Arc>, + get_connection_creator: ConnectionCreatorGetter, + ) -> Self { Self { allowed_databases, connections: table::Table::new(256), - get_pool, + get_connection_creator, } } fn get_connection( &self, connection: Resource, - ) -> Result<&Arc, v2::Error> { + ) -> Result<&Box, v2::Error> { self.connections .get(connection.rep()) .ok_or(v2::Error::InvalidConnection) @@ -61,9 +65,9 @@ impl v2::HostConnection for InstanceState { if !self.allowed_databases.contains(&database) { return Err(v2::Error::AccessDenied); } - (self.get_pool)(&database) + (self.get_connection_creator)(&database) .ok_or(v2::Error::NoSuchDatabase)? - .get_connection() + .create_connection() .await .and_then(|conn| { self.connections diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index 10c4b2325..e154fc81e 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -46,9 +46,9 @@ impl Factor for SqliteFactor { &self, mut ctx: spin_factors::ConfigureAppContext, ) -> anyhow::Result { - let connection_pools = ctx + let connection_creators = ctx .take_runtime_config() - .map(|r| r.pools) + .map(|r| r.connection_creators) .unwrap_or_default(); let allowed_databases = ctx @@ -68,20 +68,20 @@ impl Factor for SqliteFactor { }) .collect::>>()?; let resolver = self.default_label_resolver.clone(); - let get_connection_pool: host::ConnectionPoolGetter = Arc::new(move |label| { - connection_pools + let get_connection_creator: host::ConnectionCreatorGetter = Arc::new(move |label| { + connection_creators .get(label) .cloned() .or_else(|| resolver.default(label)) }); ensure_allowed_databases_are_configured(&allowed_databases, |label| { - get_connection_pool(label).is_some() + get_connection_creator(label).is_some() })?; Ok(AppState { allowed_databases, - get_connection_pool, + get_connection_creator, }) } @@ -96,8 +96,11 @@ impl Factor for SqliteFactor { .get(ctx.app_component().id()) .cloned() .unwrap_or_default(); - let get_connection_pool = ctx.app_state().get_connection_pool.clone(); - Ok(InstanceState::new(allowed_databases, get_connection_pool)) + let get_connection_creator = ctx.app_state().get_connection_creator.clone(); + Ok(InstanceState::new( + allowed_databases, + get_connection_creator, + )) } } @@ -136,45 +139,47 @@ fn ensure_allowed_databases_are_configured( pub const ALLOWED_DATABASES_KEY: MetadataKey> = MetadataKey::new("databases"); -/// Resolves a label to a default connection pool. +/// Resolves a label to a default connection creator. pub trait DefaultLabelResolver: Send + Sync { - /// If there is no runtime configuration for a given database label, return a default connection pool. + /// If there is no runtime configuration for a given database label, return a default connection creator. /// /// If `Option::None` is returned, the database is not allowed. - fn default(&self, label: &str) -> Option>; + fn default(&self, label: &str) -> Option>; } pub struct AppState { /// A map from component id to a set of allowed database labels. allowed_databases: HashMap>>, - /// A function for mapping from database name to a connection pool - get_connection_pool: host::ConnectionPoolGetter, + /// A function for mapping from database name to a connection creator. + get_connection_creator: host::ConnectionCreatorGetter, } -/// A pool of connections for a particular SQLite database +/// A creator of a connections for a particular SQLite database. #[async_trait] -pub trait ConnectionPool: Send + Sync { - /// Get a `Connection` from the pool - async fn get_connection(&self) -> Result, v2::Error>; +pub trait ConnectionCreator: Send + Sync { + /// Get a *new* [`Connection`] + /// + /// The connection should be a new connection, not a reused one. + async fn create_connection(&self) -> Result, v2::Error>; } -/// A simple [`ConnectionPool`] that always creates a new connection. -pub struct SimpleConnectionPool( - Box anyhow::Result> + Send + Sync>, +/// A simple [`ConnectionCreator`] that delegates to a function. +pub struct FunctionConnectionCreator( + Box anyhow::Result> + Send + Sync>, ); -impl SimpleConnectionPool { - /// Create a new `SimpleConnectionPool` with the given connection factory. +impl FunctionConnectionCreator { + /// Create a new [`FunctionConnectionCreator`] with the given connection factory function. pub fn new( - factory: impl Fn() -> anyhow::Result> + Send + Sync + 'static, + factory: impl Fn() -> anyhow::Result> + Send + Sync + 'static, ) -> Self { Self(Box::new(factory)) } } #[async_trait::async_trait] -impl ConnectionPool for SimpleConnectionPool { - async fn get_connection(&self) -> Result, v2::Error> { +impl ConnectionCreator for FunctionConnectionCreator { + async fn create_connection(&self) -> Result, v2::Error> { (self.0)().map_err(|_| v2::Error::InvalidConnection) } } diff --git a/crates/factor-sqlite/src/runtime_config.rs b/crates/factor-sqlite/src/runtime_config.rs index ca13c4c02..10eb8e871 100644 --- a/crates/factor-sqlite/src/runtime_config.rs +++ b/crates/factor-sqlite/src/runtime_config.rs @@ -3,11 +3,11 @@ pub mod spin; use std::{collections::HashMap, sync::Arc}; -use crate::ConnectionPool; +use crate::ConnectionCreator; /// A runtime configuration for SQLite databases. /// -/// Maps database labels to connection pools. +/// Maps database labels to connection creators. pub struct RuntimeConfig { - pub pools: HashMap>, + pub connection_creators: HashMap>, } diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 9dc6a23e0..48a4faaa3 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -13,7 +13,7 @@ use spin_factors::{ use spin_world::v2::sqlite as v2; use tokio::sync::OnceCell; -use crate::{Connection, ConnectionPool, DefaultLabelResolver, SimpleConnectionPool}; +use crate::{Connection, ConnectionCreator, DefaultLabelResolver, FunctionConnectionCreator}; /// Spin's default handling of the runtime configuration for SQLite databases. /// @@ -70,11 +70,13 @@ impl SpinSqliteRuntimeConfig { .into_iter() .map(|(k, v)| Ok((k, self.get_pool(v)?))) .collect::>()?; - Ok(Some(super::RuntimeConfig { pools })) + Ok(Some(super::RuntimeConfig { + connection_creators: pools, + })) } /// Get a connection pool for a given runtime configuration. - pub fn get_pool(&self, config: RuntimeConfig) -> anyhow::Result> { + pub fn get_pool(&self, config: RuntimeConfig) -> anyhow::Result> { let database_kind = config.type_.as_str(); let pool = match database_kind { "spin" => { @@ -100,7 +102,7 @@ pub struct RuntimeConfig { } impl DefaultLabelResolver for SpinSqliteRuntimeConfig { - fn default(&self, label: &str) -> Option> { + fn default(&self, label: &str) -> Option> { // Only default the database labeled "default". if label != "default" { return None; @@ -110,9 +112,9 @@ impl DefaultLabelResolver for SpinSqliteRuntimeConfig { let factory = move || { let location = spin_sqlite_inproc::InProcDatabaseLocation::Path(path.clone()); let connection = spin_sqlite_inproc::InProcConnection::new(location)?; - Ok(Arc::new(connection) as _) + Ok(Box::new(connection) as _) }; - let pool = SimpleConnectionPool::new(factory); + let pool = FunctionConnectionCreator::new(factory); Some(Arc::new(pool)) } } @@ -199,7 +201,7 @@ impl LocalDatabase { /// Create a new connection pool for a local database. /// /// `base_dir` is the base directory path from which `path` is resolved if it is a relative path. - fn pool(self, base_dir: &Path) -> anyhow::Result { + fn pool(self, base_dir: &Path) -> anyhow::Result { let location = match self.path { Some(path) => { let path = resolve_relative_path(&path, base_dir); @@ -213,9 +215,9 @@ impl LocalDatabase { }; let factory = move || { let connection = spin_sqlite_inproc::InProcConnection::new(location.clone())?; - Ok(Arc::new(connection) as _) + Ok(Box::new(connection) as _) }; - Ok(SimpleConnectionPool::new(factory)) + Ok(FunctionConnectionCreator::new(factory)) } } @@ -239,7 +241,7 @@ pub struct LibSqlDatabase { impl LibSqlDatabase { /// Create a new connection pool for a libSQL database. - fn pool(self) -> anyhow::Result { + fn pool(self) -> anyhow::Result { let url = check_url(&self.url) .with_context(|| { format!( @@ -250,9 +252,9 @@ impl LibSqlDatabase { .to_owned(); let factory = move || { let connection = LibSqlConnection::new(url.clone(), self.token.clone()); - Ok(Arc::new(connection) as _) + Ok(Box::new(connection) as _) }; - Ok(SimpleConnectionPool::new(factory)) + Ok(FunctionConnectionCreator::new(factory)) } } diff --git a/crates/factor-sqlite/tests/factor_test.rs b/crates/factor-sqlite/tests/factor_test.rs index 4699dab78..e84c6e9c0 100644 --- a/crates/factor-sqlite/tests/factor_test.rs +++ b/crates/factor-sqlite/tests/factor_test.rs @@ -130,7 +130,7 @@ impl DefaultLabelResolver { } impl factor_sqlite::DefaultLabelResolver for DefaultLabelResolver { - fn default(&self, label: &str) -> Option> { + fn default(&self, label: &str) -> Option> { let Some(default) = &self.default else { return None; }; @@ -142,8 +142,8 @@ impl factor_sqlite::DefaultLabelResolver for DefaultLabelResolver { struct InvalidConnectionPool; #[async_trait::async_trait] -impl factor_sqlite::ConnectionPool for InvalidConnectionPool { - async fn get_connection( +impl factor_sqlite::ConnectionCreator for InvalidConnectionPool { + async fn create_connection( &self, ) -> Result, spin_world::v2::sqlite::Error> { Err(spin_world::v2::sqlite::Error::InvalidConnection) From 1ce0b7c0e3dc7cb52010bd52af9adaf4a30df0b2 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 12 Aug 2024 10:29:57 +0200 Subject: [PATCH 117/195] Impl ConnectionCreator for functions Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/lib.rs | 21 +++------- .../factor-sqlite/src/runtime_config/spin.rs | 41 ++++++++++--------- crates/factor-sqlite/tests/factor_test.rs | 12 +++--- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index e154fc81e..e9a80e646 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -163,24 +163,13 @@ pub trait ConnectionCreator: Send + Sync { async fn create_connection(&self) -> Result, v2::Error>; } -/// A simple [`ConnectionCreator`] that delegates to a function. -pub struct FunctionConnectionCreator( - Box anyhow::Result> + Send + Sync>, -); - -impl FunctionConnectionCreator { - /// Create a new [`FunctionConnectionCreator`] with the given connection factory function. - pub fn new( - factory: impl Fn() -> anyhow::Result> + Send + Sync + 'static, - ) -> Self { - Self(Box::new(factory)) - } -} - #[async_trait::async_trait] -impl ConnectionCreator for FunctionConnectionCreator { +impl ConnectionCreator for F +where + F: Fn() -> anyhow::Result> + Send + Sync + 'static, +{ async fn create_connection(&self) -> Result, v2::Error> { - (self.0)().map_err(|_| v2::Error::InvalidConnection) + (self)().map_err(|_| v2::Error::InvalidConnection) } } diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 48a4faaa3..7bc81404a 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -13,7 +13,7 @@ use spin_factors::{ use spin_world::v2::sqlite as v2; use tokio::sync::OnceCell; -use crate::{Connection, ConnectionCreator, DefaultLabelResolver, FunctionConnectionCreator}; +use crate::{Connection, ConnectionCreator, DefaultLabelResolver}; /// Spin's default handling of the runtime configuration for SQLite databases. /// @@ -66,30 +66,34 @@ impl SpinSqliteRuntimeConfig { return Ok(None); }; let config: std::collections::HashMap = table.clone().try_into()?; - let pools = config + let connection_creators = config .into_iter() - .map(|(k, v)| Ok((k, self.get_pool(v)?))) + .map(|(k, v)| Ok((k, self.get_connection_creator(v)?))) .collect::>()?; Ok(Some(super::RuntimeConfig { - connection_creators: pools, + connection_creators, })) } - /// Get a connection pool for a given runtime configuration. - pub fn get_pool(&self, config: RuntimeConfig) -> anyhow::Result> { + /// Get a connection creator for a given runtime configuration. + pub fn get_connection_creator( + &self, + config: RuntimeConfig, + ) -> anyhow::Result> { let database_kind = config.type_.as_str(); - let pool = match database_kind { + match database_kind { "spin" => { let config: LocalDatabase = config.config.try_into()?; - config.pool(&self.local_database_dir)? + Ok(Arc::new( + config.connection_creator(&self.local_database_dir)?, + )) } "libsql" => { let config: LibSqlDatabase = config.config.try_into()?; - config.pool()? + Ok(Arc::new(config.connection_creator()?)) } _ => anyhow::bail!("Unknown database kind: {database_kind}"), - }; - Ok(Arc::new(pool)) + } } } @@ -114,8 +118,7 @@ impl DefaultLabelResolver for SpinSqliteRuntimeConfig { let connection = spin_sqlite_inproc::InProcConnection::new(location)?; Ok(Box::new(connection) as _) }; - let pool = FunctionConnectionCreator::new(factory); - Some(Arc::new(pool)) + Some(Arc::new(factory)) } } @@ -198,10 +201,10 @@ pub struct LocalDatabase { } impl LocalDatabase { - /// Create a new connection pool for a local database. + /// Get a new connection creator for a local database. /// /// `base_dir` is the base directory path from which `path` is resolved if it is a relative path. - fn pool(self, base_dir: &Path) -> anyhow::Result { + fn connection_creator(self, base_dir: &Path) -> anyhow::Result { let location = match self.path { Some(path) => { let path = resolve_relative_path(&path, base_dir); @@ -217,7 +220,7 @@ impl LocalDatabase { let connection = spin_sqlite_inproc::InProcConnection::new(location.clone())?; Ok(Box::new(connection) as _) }; - Ok(FunctionConnectionCreator::new(factory)) + Ok(factory) } } @@ -240,8 +243,8 @@ pub struct LibSqlDatabase { } impl LibSqlDatabase { - /// Create a new connection pool for a libSQL database. - fn pool(self) -> anyhow::Result { + /// Get a new connection creator for a libSQL database. + fn connection_creator(self) -> anyhow::Result { let url = check_url(&self.url) .with_context(|| { format!( @@ -254,7 +257,7 @@ impl LibSqlDatabase { let connection = LibSqlConnection::new(url.clone(), self.token.clone()); Ok(Box::new(connection) as _) }; - Ok(FunctionConnectionCreator::new(factory)) + Ok(factory) } } diff --git a/crates/factor-sqlite/tests/factor_test.rs b/crates/factor-sqlite/tests/factor_test.rs index e84c6e9c0..8fb73c3c4 100644 --- a/crates/factor-sqlite/tests/factor_test.rs +++ b/crates/factor-sqlite/tests/factor_test.rs @@ -116,7 +116,7 @@ impl TryFrom> for TestFactorsRuntimeConfig { } } -/// Will return an `InvalidConnectionPool` for the supplied default database. +/// Will return an `InvalidConnectionCreator` for the supplied default database. struct DefaultLabelResolver { default: Option, } @@ -134,18 +134,18 @@ impl factor_sqlite::DefaultLabelResolver for DefaultLabelResolver { let Some(default) = &self.default else { return None; }; - (default == label).then_some(Arc::new(InvalidConnectionPool)) + (default == label).then_some(Arc::new(InvalidConnectionCreator)) } } -/// A connection pool that always returns an error. -struct InvalidConnectionPool; +/// A connection creator that always returns an error. +struct InvalidConnectionCreator; #[async_trait::async_trait] -impl factor_sqlite::ConnectionCreator for InvalidConnectionPool { +impl factor_sqlite::ConnectionCreator for InvalidConnectionCreator { async fn create_connection( &self, - ) -> Result, spin_world::v2::sqlite::Error> { + ) -> Result, spin_world::v2::sqlite::Error> { Err(spin_world::v2::sqlite::Error::InvalidConnection) } } From 34f66ea4a8ac9a12fe76bc3f81aeeef7cde4c32a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Aug 2024 13:14:04 -0400 Subject: [PATCH 118/195] Address review feedback Signed-off-by: Lann Martin --- .../src/runtime_config.rs | 31 ++++++++++++++---- .../src/runtime_config/spin.rs | 32 +++++++++++-------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/crates/factor-outbound-networking/src/runtime_config.rs b/crates/factor-outbound-networking/src/runtime_config.rs index 711b33731..9ea5d7d8f 100644 --- a/crates/factor-outbound-networking/src/runtime_config.rs +++ b/crates/factor-outbound-networking/src/runtime_config.rs @@ -10,8 +10,9 @@ use rustls_pki_types::{CertificateDer, PrivateKeyDer}; /// Runtime configuration for outbound networking. #[derive(Debug)] pub struct RuntimeConfig { - // Maps component ID -> HostClientConfigs + /// Maps component ID -> HostClientConfigs component_host_client_configs: HashMap, + /// The default [`ClientConfig`] for a host if one is not explicitly configured for it. default_client_config: Arc, } @@ -19,6 +20,9 @@ pub struct RuntimeConfig { type HostClientConfigs = Arc>>; impl RuntimeConfig { + /// Returns runtime config with the given list of [`TlsConfig`]s. The first + /// [`TlsConfig`] to match an outgoing request (based on + /// [`TlsConfig::components`] and [`TlsConfig::hosts`]) will be used. pub fn new(tls_configs: impl IntoIterator) -> anyhow::Result { let mut component_host_client_configs = HashMap::::new(); for tls_config in tls_configs { @@ -30,7 +34,11 @@ impl RuntimeConfig { !tls_config.hosts.is_empty(), "client TLS 'hosts' list may not be empty" ); - let client_config = Arc::new(tls_config.to_client_config()?); + let client_config = Arc::new( + tls_config + .to_client_config() + .context("error building TLS client config")?, + ); for component in &tls_config.components { let host_configs = component_host_client_configs .entry(component.clone()) @@ -78,13 +86,16 @@ impl RuntimeConfig { pub(crate) fn validate_host(host: &str) -> anyhow::Result<()> { // Validate hostname - http::uri::Authority::from_str(host).with_context(|| format!("invalid TLS 'host' {host:?}"))?; - if host.contains(':') { - anyhow::bail!("invalid TLS 'host' {host:?}; ports not currently supported"); - } + let authority = http::uri::Authority::from_str(host) + .with_context(|| format!("invalid TLS 'host' {host:?}"))?; + ensure!( + authority.port().is_none(), + "invalid TLS 'host' {host:?}; ports not currently supported" + ); Ok(()) } +/// TLS configurations for a specific component. #[derive(Clone)] pub struct ComponentTlsConfigs { host_client_configs: Option, @@ -107,12 +118,20 @@ pub struct ClientCertConfig { key_der: PrivateKeyDer<'static>, } +/// TLS configuration for one or more component(s) and host(s). #[derive(Debug)] pub struct TlsConfig { + /// The component(s) this configuration applies to. pub components: Vec, + /// The host(s) this configuration applies to. pub hosts: Vec, + /// A set of CA certs that should be considered valid roots. pub root_certificates: Vec>, + /// If true, the "standard" CA certs defined by `webpki-roots` crate will be + /// considered valid roots in addition to `root_certificates`. pub use_webpki_roots: bool, + /// A certificate and private key to be used as the client certificate for + /// "mutual TLS" (mTLS). pub client_cert: Option, } diff --git a/crates/factor-outbound-networking/src/runtime_config/spin.rs b/crates/factor-outbound-networking/src/runtime_config/spin.rs index dc524d2a6..580d0e6c7 100644 --- a/crates/factor-outbound-networking/src/runtime_config/spin.rs +++ b/crates/factor-outbound-networking/src/runtime_config/spin.rs @@ -9,11 +9,16 @@ use std::{ use super::{validate_host, TlsConfig}; +/// Spin's default handling of the runtime configuration for outbound TLS. pub struct SpinTlsRuntimeConfig { runtime_config_dir: PathBuf, } impl SpinTlsRuntimeConfig { + /// Creates a new `SpinTlsRuntimeConfig`. + /// + /// The given `runtime_config_dir` will be used as the root to resolve any + /// relative paths. pub fn new(runtime_config_dir: impl Into) -> Self { Self { runtime_config_dir: runtime_config_dir.into(), @@ -55,7 +60,8 @@ impl SpinTlsRuntimeConfig { let tls_configs = toml_configs .into_iter() .map(|toml_config| self.load_tls_config(toml_config)) - .collect::>>()?; + .collect::>>() + .context("failed to parse TLS configs from TOML")?; Ok(Some(tls_configs)) } @@ -82,7 +88,6 @@ impl SpinTlsRuntimeConfig { let hosts = hosts .iter() .map(|host| { - let host: &str = host; host.parse() .map_err(|err| anyhow::anyhow!("invalid host {host:?}: {err:?}")) }) @@ -133,7 +138,7 @@ impl SpinTlsRuntimeConfig { ) }, )?)) - .collect::>>>() + .collect() } // Parse a private key from the provided file @@ -142,17 +147,16 @@ impl SpinTlsRuntimeConfig { path: impl AsRef, ) -> anyhow::Result> { let path = self.runtime_config_dir.join(path); - rustls_pemfile::private_key(&mut io::BufReader::new( - fs::File::open(path).context("loading private key")?, - )) - .map_err(|_| anyhow::anyhow!("invalid input")) - .transpose() - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "private key file contains no private keys", - ) - })? + let file = fs::File::open(&path) + .with_context(|| format!("failed to read private key from '{}'", path.display()))?; + Ok(rustls_pemfile::private_key(&mut io::BufReader::new(file)) + .with_context(|| format!("failed to parse private key from '{}'", path.display()))? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("private key file '{}' contains no private keys", path.display()), + ) + })?) } } From 79d1556ec0fcfdce22c65936138d8dbeff153f5e Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 14 Aug 2024 10:25:16 +0200 Subject: [PATCH 119/195] Add basic runtime config support to trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 1 + crates/trigger2/Cargo.toml | 1 + crates/trigger2/src/cli.rs | 26 +++++++++++++----- crates/trigger2/src/lib.rs | 1 + crates/trigger2/src/runtime_config.rs | 39 +++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 crates/trigger2/src/runtime_config.rs diff --git a/Cargo.lock b/Cargo.lock index a40322f9d..f15daa101 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8451,6 +8451,7 @@ dependencies = [ "spin-factors-executor", "spin-telemetry", "tokio", + "toml 0.8.14", "tracing", ] diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index a978ce987..f4f0ad3e7 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -24,6 +24,7 @@ spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } tokio = { version = "1.23", features = ["fs"] } +toml = "0.8" tracing = { workspace = true } [lints] diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index b3c9ea1c0..53b483eca 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -11,6 +11,7 @@ use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; use crate::factors::TriggerFactors; +use crate::runtime_config::RuntimeConfigSource; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; use crate::Trigger; pub use launch_metadata::LaunchMetadata; @@ -214,7 +215,6 @@ impl FactorsTriggerCommand { let mut executor = FactorsExecutor::new(core_engine_builder, factors)?; - // TODO: integrate with runtime config let log_dir = self.log.clone(); executor.add_hooks(StdioLoggingExecutorHooks::new(follow_components, log_dir)); // TODO: @@ -222,13 +222,27 @@ impl FactorsTriggerCommand { // builder.hooks(KeyValuePersistenceMessageHook); // builder.hooks(SqlitePersistenceMessageHook); + let runtime_config = match &self.runtime_config_file { + Some(path) => { + let file = std::fs::read_to_string(path).with_context(|| { + format!("failed to read runtime config file {}", quoted_path(path)) + })?; + let toml = toml::from_str(&file).with_context(|| { + format!( + "failed to parse runtime config file {} as toml", + quoted_path(path) + ) + })?; + + let source = RuntimeConfigSource::new(&toml); + source.try_into().context("error parsing runtime config")? + } + None => Default::default(), + }; + let configured_app = { let _sloth_guard = warn_if_wasm_build_slothful(); - executor.load_app( - app, - Default::default(), // TODO runtime config - TodoComponentLoader, - )? + executor.load_app(app, runtime_config, TodoComponentLoader)? }; // TODO: Construct factors diff --git a/crates/trigger2/src/lib.rs b/crates/trigger2/src/lib.rs index 095d30e3b..1e3934bc8 100644 --- a/crates/trigger2/src/lib.rs +++ b/crates/trigger2/src/lib.rs @@ -8,6 +8,7 @@ use spin_factors_executor::{FactorsExecutorApp, FactorsInstanceBuilder}; pub mod cli; mod factors; +mod runtime_config; mod stdio; /// Type alias for a [`FactorsConfiguredApp`] specialized to a [`Trigger`]. diff --git a/crates/trigger2/src/runtime_config.rs b/crates/trigger2/src/runtime_config.rs new file mode 100644 index 000000000..6fbba4d00 --- /dev/null +++ b/crates/trigger2/src/runtime_config.rs @@ -0,0 +1,39 @@ +use spin_factor_wasi::WasiFactor; +use spin_factors::{ + runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, +}; + +use crate::factors::TriggerFactorsRuntimeConfig; + +/// A runtime configuration source for the [`TriggerFactors`][crate::TriggerFactors]. +pub struct RuntimeConfigSource<'a> { + table: TomlKeyTracker<'a>, +} + +impl<'a> RuntimeConfigSource<'a> { + pub fn new(table: &'a toml::Table) -> Self { + Self { + table: TomlKeyTracker::new(table), + } + } +} + +impl RuntimeConfigSourceFinalizer for RuntimeConfigSource<'_> { + fn finalize(&mut self) -> anyhow::Result<()> { + Ok(self.table.validate_all_keys_used()?) + } +} + +impl FactorRuntimeConfigSource for RuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(None) + } +} + +impl TryFrom> for TriggerFactorsRuntimeConfig { + type Error = anyhow::Error; + + fn try_from(value: RuntimeConfigSource<'_>) -> Result { + Self::from_source(value) + } +} From d76469fe636eab5e058fa8505ac6d71e3d1bbc88 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 14 Aug 2024 17:05:42 +0200 Subject: [PATCH 120/195] Configure key-value for trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 4 ++ crates/factor-key-value-azure/src/lib.rs | 11 +++- crates/factor-key-value-redis/src/lib.rs | 11 +++- crates/factor-key-value/src/runtime_config.rs | 2 +- .../src/runtime_config/spin.rs | 10 ++-- crates/factor-key-value/tests/factor_test.rs | 2 +- crates/factors/tests/smoke.rs | 2 +- crates/trigger2/Cargo.toml | 4 ++ crates/trigger2/src/cli.rs | 50 ++++++++++-------- crates/trigger2/src/factors.rs | 9 +++- crates/trigger2/src/runtime_config.rs | 51 ++++++++++++++++++- 11 files changed, 123 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f15daa101..e7fa1a216 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8446,6 +8446,10 @@ dependencies = [ "spin-app", "spin-common", "spin-core", + "spin-factor-key-value", + "spin-factor-key-value-azure", + "spin-factor-key-value-redis", + "spin-factor-key-value-spin", "spin-factor-wasi", "spin-factors", "spin-factors-executor", diff --git a/crates/factor-key-value-azure/src/lib.rs b/crates/factor-key-value-azure/src/lib.rs index 812a1df07..bf5e8067a 100644 --- a/crates/factor-key-value-azure/src/lib.rs +++ b/crates/factor-key-value-azure/src/lib.rs @@ -5,7 +5,16 @@ use spin_key_value_azure::{ }; /// A key-value store that uses Azure Cosmos as the backend. -pub struct AzureKeyValueStore; +pub struct AzureKeyValueStore { + _priv: (), +} + +impl AzureKeyValueStore { + /// Creates a new `AzureKeyValueStore`. + pub fn new() -> Self { + Self { _priv: () } + } +} /// Runtime configuration for the Azure Cosmos key-value store. #[derive(Deserialize)] diff --git a/crates/factor-key-value-redis/src/lib.rs b/crates/factor-key-value-redis/src/lib.rs index 6bea163e1..f8da8a980 100644 --- a/crates/factor-key-value-redis/src/lib.rs +++ b/crates/factor-key-value-redis/src/lib.rs @@ -3,7 +3,16 @@ use spin_factor_key_value::runtime_config::spin::MakeKeyValueStore; use spin_key_value_redis::KeyValueRedis; /// A key-value store that uses Redis as the backend. -pub struct RedisKeyValueStore; +pub struct RedisKeyValueStore { + _priv: (), +} + +impl RedisKeyValueStore { + /// Creates a new `RedisKeyValueStore`. + pub fn new() -> Self { + Self { _priv: () } + } +} /// Runtime configuration for the Redis key-value store. #[derive(Deserialize)] diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index d5895e75a..97e2a3ece 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -5,7 +5,7 @@ use std::{collections::HashMap, sync::Arc}; use spin_key_value::StoreManager; /// Runtime configuration for all key value stores. -#[derive(Default)] +#[derive(Default, Clone)] pub struct RuntimeConfig { /// Map of store names to store managers. store_managers: HashMap>, diff --git a/crates/factor-key-value/src/runtime_config/spin.rs b/crates/factor-key-value/src/runtime_config/spin.rs index 20c218447..3f3e487f5 100644 --- a/crates/factor-key-value/src/runtime_config/spin.rs +++ b/crates/factor-key-value/src/runtime_config/spin.rs @@ -23,11 +23,11 @@ pub trait MakeKeyValueStore: 'static + Send + Sync { /// A function that creates a store manager from a TOML table. type StoreFromToml = - Box anyhow::Result> + Send + Sync>; + Arc anyhow::Result> + Send + Sync>; /// Creates a `StoreFromToml` function from a `MakeKeyValueStore` implementation. fn store_from_toml_fn(provider_type: T) -> StoreFromToml { - Box::new(move |table| { + Arc::new(move |table| { let runtime_config: T::RuntimeConfig = table.try_into().context("could not parse runtime config")?; let provider = provider_type @@ -43,7 +43,7 @@ fn store_from_toml_fn(provider_type: T) -> StoreFromToml { /// /// The various store types (i.e., the "type" field in the toml field) are registered with the /// resolver using `add_store_type`. The default store for a label is registered using `add_default_store`. -#[derive(Default)] +#[derive(Default, Clone)] pub struct RuntimeConfigResolver { /// A map of store types to a function that returns the appropriate store /// manager from runtime config TOML. @@ -87,9 +87,9 @@ impl RuntimeConfigResolver { /// Resolves a toml table into a runtime config. pub fn resolve_from_toml( &self, - table: &Option, + table: Option<&toml::Table>, ) -> anyhow::Result> { - let Some(table) = table.as_ref().and_then(|t| t.get("key_value_store")) else { + let Some(table) = table.and_then(|t| t.get("key_value_store")) else { return Ok(None); }; let table: HashMap = table.clone().try_into()?; diff --git a/crates/factor-key-value/tests/factor_test.rs b/crates/factor-key-value/tests/factor_test.rs index 978160601..57c13b719 100644 --- a/crates/factor-key-value/tests/factor_test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -248,7 +248,7 @@ impl TryFrom for TestFactorsRuntimeConfig { impl FactorRuntimeConfigSource for TomlConfig { fn get_runtime_config(&mut self) -> anyhow::Result> { - self.resolver.resolve_from_toml(&self.toml) + self.resolver.resolve_from_toml(self.toml.as_ref()) } } diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 7cbbb24b3..75435414e 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -163,7 +163,7 @@ impl FactorRuntimeConfigSource for TestSource { type = "redis" url = "redis://localhost:6379" }; - self.key_value_resolver.resolve_from_toml(&Some(config)) + self.key_value_resolver.resolve_from_toml(Some(&config)) } } diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index f4f0ad3e7..7ba90f9dc 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -20,6 +20,10 @@ spin-app = { path = "../app" } spin-common = { path = "../common" } spin-core = { path = "../core" } spin-factor-wasi = { path = "../factor-wasi" } +spin-factor-key-value = { path = "../factor-key-value" } +spin-factor-key-value-spin = { path = "../factor-key-value-spin" } +spin-factor-key-value-redis = { path = "../factor-key-value-redis" } +spin-factor-key-value-azure = { path = "../factor-key-value-azure" } spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index 53b483eca..d54d19dd4 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -10,8 +10,8 @@ use spin_common::url::parse_file_url; use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; -use crate::factors::TriggerFactors; -use crate::runtime_config::RuntimeConfigSource; +use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; +use crate::runtime_config::{key_value_resolver, RuntimeConfigSource}; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; use crate::Trigger; pub use launch_metadata::LaunchMetadata; @@ -21,6 +21,7 @@ pub const DISABLE_WASMTIME_CACHE: &str = "DISABLE_WASMTIME_CACHE"; pub const FOLLOW_LOG_OPT: &str = "FOLLOW_ID"; pub const WASMTIME_CACHE_FILE: &str = "WASMTIME_CACHE_FILE"; pub const RUNTIME_CONFIG_FILE: &str = "RUNTIME_CONFIG_FILE"; +pub const DEFAULT_STATE_DIR: &str = ".spin"; // Set by `spin up` pub const SPIN_LOCKED_URL: &str = "SPIN_LOCKED_URL"; @@ -192,7 +193,32 @@ impl FactorsTriggerCommand { }; trigger.add_to_linker(core_engine_builder.linker())?; - let factors = TriggerFactors::new(working_dir, self.allow_transient_write); + let (runtime_config, key_value_resolver) = self + .runtime_config_file + .as_ref() + .map(|path| { + let file = std::fs::read_to_string(path).with_context(|| { + format!("failed to read runtime config file {}", quoted_path(path)) + })?; + let toml = toml::from_str(&file).with_context(|| { + format!( + "failed to parse runtime config file {} as toml", + quoted_path(path) + ) + })?; + + let key_value_resolver = key_value_resolver(PathBuf::from( + self.state_dir.unwrap_or_else(|| DEFAULT_STATE_DIR.into()), + )); + let source: TriggerFactorsRuntimeConfig = + RuntimeConfigSource::new(&toml, &key_value_resolver).try_into()?; + anyhow::Ok((source, key_value_resolver)) + }) + .transpose()? + .unwrap_or_default(); + + let factors = + TriggerFactors::new(working_dir, self.allow_transient_write, key_value_resolver); // TODO: move these into Factor methods/constructors // let init_data = crate::HostComponentInitData::new( @@ -222,24 +248,6 @@ impl FactorsTriggerCommand { // builder.hooks(KeyValuePersistenceMessageHook); // builder.hooks(SqlitePersistenceMessageHook); - let runtime_config = match &self.runtime_config_file { - Some(path) => { - let file = std::fs::read_to_string(path).with_context(|| { - format!("failed to read runtime config file {}", quoted_path(path)) - })?; - let toml = toml::from_str(&file).with_context(|| { - format!( - "failed to parse runtime config file {} as toml", - quoted_path(path) - ) - })?; - - let source = RuntimeConfigSource::new(&toml); - source.try_into().context("error parsing runtime config")? - } - None => Default::default(), - }; - let configured_app = { let _sloth_guard = warn_if_wasm_build_slothful(); executor.load_app(app, runtime_config, TodoComponentLoader)? diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index b433883da..f323c8964 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -1,18 +1,25 @@ use std::path::PathBuf; +use spin_factor_key_value::{DefaultLabelResolver, KeyValueFactor}; use spin_factor_wasi::{spin::SpinFilesMounter, WasiFactor}; use spin_factors::RuntimeFactors; #[derive(RuntimeFactors)] pub struct TriggerFactors { pub wasi: WasiFactor, + pub key_value: KeyValueFactor, } impl TriggerFactors { - pub fn new(working_dir: impl Into, allow_transient_writes: bool) -> Self { + pub fn new( + working_dir: impl Into, + allow_transient_writes: bool, + default_key_value_label_resolver: impl DefaultLabelResolver + 'static, + ) -> Self { let files_mounter = SpinFilesMounter::new(working_dir, allow_transient_writes); Self { wasi: WasiFactor::new(files_mounter), + key_value: KeyValueFactor::new(default_key_value_label_resolver), } } } diff --git a/crates/trigger2/src/runtime_config.rs b/crates/trigger2/src/runtime_config.rs index 6fbba4d00..44dd2c6cd 100644 --- a/crates/trigger2/src/runtime_config.rs +++ b/crates/trigger2/src/runtime_config.rs @@ -1,3 +1,6 @@ +use std::path::PathBuf; + +use spin_factor_key_value::{self as key_value, KeyValueFactor}; use spin_factor_wasi::WasiFactor; use spin_factors::{ runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, @@ -8,12 +11,17 @@ use crate::factors::TriggerFactorsRuntimeConfig; /// A runtime configuration source for the [`TriggerFactors`][crate::TriggerFactors]. pub struct RuntimeConfigSource<'a> { table: TomlKeyTracker<'a>, + pub key_value: &'a key_value::runtime_config::spin::RuntimeConfigResolver, } impl<'a> RuntimeConfigSource<'a> { - pub fn new(table: &'a toml::Table) -> Self { + pub fn new( + table: &'a toml::Table, + key_value: &'a key_value::runtime_config::spin::RuntimeConfigResolver, + ) -> Self { Self { table: TomlKeyTracker::new(table), + key_value, } } } @@ -30,6 +38,14 @@ impl FactorRuntimeConfigSource for RuntimeConfigSource<'_> { } } +impl FactorRuntimeConfigSource for RuntimeConfigSource<'_> { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result> { + self.key_value.resolve_from_toml(Some(self.table.as_ref())) + } +} + impl TryFrom> for TriggerFactorsRuntimeConfig { type Error = anyhow::Error; @@ -37,3 +53,36 @@ impl TryFrom> for TriggerFactorsRuntimeConfig { Self::from_source(value) } } + +const DEFAULT_SPIN_STORE_FILENAME: &str = "sqlite_key_value.db"; + +/// The key-value runtime configuration resolver used by the trigger. +/// +/// Takes a base path for the local store. +pub fn key_value_resolver( + local_store_base_path: PathBuf, +) -> spin_factor_key_value::runtime_config::spin::RuntimeConfigResolver { + let mut key_value = key_value::runtime_config::spin::RuntimeConfigResolver::new(); + key_value.add_default_store( + "default", + spin_factor_key_value::runtime_config::spin::StoreConfig { + type_: "spin".to_owned(), + config: toml::toml! { + path = DEFAULT_SPIN_STORE_FILENAME + }, + }, + ); + // Unwraps are safe because the store types are known to not overlap. + key_value + .register_store_type(spin_factor_key_value_spin::SpinKeyValueStore::new( + local_store_base_path, + )) + .unwrap(); + key_value + .register_store_type(spin_factor_key_value_redis::RedisKeyValueStore::new()) + .unwrap(); + key_value + .register_store_type(spin_factor_key_value_azure::AzureKeyValueStore::new()) + .unwrap(); + key_value +} From e42b24947a32e1245cbb04dba3bfcdcb37b18508 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 14 Aug 2024 19:27:18 +0200 Subject: [PATCH 121/195] Move runtime config to its own crate Signed-off-by: Ryan Levick --- Cargo.lock | 19 ++- crates/factor-key-value/tests/factor_test.rs | 6 +- crates/factors/tests/smoke.rs | 2 +- crates/runtime-config/Cargo.toml | 22 ++++ crates/runtime-config/src/lib.rs | 127 +++++++++++++++++++ crates/trigger2/Cargo.toml | 5 +- crates/trigger2/src/cli.rs | 46 +++---- crates/trigger2/src/factors.rs | 9 ++ crates/trigger2/src/lib.rs | 1 - crates/trigger2/src/runtime_config.rs | 88 ------------- 10 files changed, 193 insertions(+), 132 deletions(-) create mode 100644 crates/runtime-config/Cargo.toml create mode 100644 crates/runtime-config/src/lib.rs delete mode 100644 crates/trigger2/src/runtime_config.rs diff --git a/Cargo.lock b/Cargo.lock index e7fa1a216..ec7e38c55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8147,6 +8147,20 @@ dependencies = [ "url", ] +[[package]] +name = "spin-runtime-config" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "spin-factor-key-value", + "spin-factor-key-value-azure", + "spin-factor-key-value-redis", + "spin-factor-key-value-spin", + "spin-factor-wasi", + "spin-factors", + "toml 0.8.14", +] + [[package]] name = "spin-serde" version = "2.7.0-pre0" @@ -8447,15 +8461,12 @@ dependencies = [ "spin-common", "spin-core", "spin-factor-key-value", - "spin-factor-key-value-azure", - "spin-factor-key-value-redis", - "spin-factor-key-value-spin", "spin-factor-wasi", "spin-factors", "spin-factors-executor", + "spin-runtime-config", "spin-telemetry", "tokio", - "toml 0.8.14", "tracing", ] diff --git a/crates/factor-key-value/tests/factor_test.rs b/crates/factor-key-value/tests/factor_test.rs index 57c13b719..9412b9e51 100644 --- a/crates/factor-key-value/tests/factor_test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -91,7 +91,7 @@ async fn overridden_default_key_value_works() -> anyhow::Result<()> { }; run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![RedisKeyValueStore], + vec![RedisKeyValueStore::new()], vec!["default"], ) .await @@ -163,7 +163,7 @@ async fn custom_redis_key_value_works() -> anyhow::Result<()> { }; run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![RedisKeyValueStore], + vec![RedisKeyValueStore::new()], vec!["custom"], ) .await @@ -192,7 +192,7 @@ async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { let tmp_dir = tempdir::TempDir::new("example")?; let mut test_resolver = RuntimeConfigResolver::new(); - test_resolver.register_store_type(RedisKeyValueStore)?; + test_resolver.register_store_type(RedisKeyValueStore::new())?; test_resolver.register_store_type(SpinKeyValueStore::new(tmp_dir.path().to_owned()))?; let test_resolver = Arc::new(test_resolver); let factors = TestFactors { diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 75435414e..9925c59b6 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -54,7 +54,7 @@ async fn smoke_test_works() -> anyhow::Result<()> { key_value_resolver.register_store_type(SpinKeyValueStore::new( std::env::current_dir().context("failed to get current directory")?, ))?; - key_value_resolver.register_store_type(RedisKeyValueStore)?; + key_value_resolver.register_store_type(RedisKeyValueStore::new())?; let key_value_resolver = Arc::new(key_value_resolver); let mut factors = Factors { diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml new file mode 100644 index 000000000..db71453f9 --- /dev/null +++ b/crates/runtime-config/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "spin-runtime-config" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +spin-factors = { path = "../factors" } +spin-factor-key-value = { path = "../factor-key-value" } +spin-factor-key-value-spin = { path = "../factor-key-value-spin" } +spin-factor-key-value-redis = { path = "../factor-key-value-redis" } +spin-factor-key-value-azure = { path = "../factor-key-value-azure" } +spin-factor-wasi = { path = "../factor-wasi" } +toml = "0.8" + +[lints] +workspace = true diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs new file mode 100644 index 000000000..28f62b354 --- /dev/null +++ b/crates/runtime-config/src/lib.rs @@ -0,0 +1,127 @@ +use std::path::{Path, PathBuf}; + +use anyhow::Context as _; +use spin_factor_key_value::runtime_config::spin::{self as key_value, MakeKeyValueStore}; +use spin_factor_key_value::KeyValueFactor; +use spin_factor_wasi::WasiFactor; +use spin_factors::{ + runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, +}; + +pub const DEFAULT_STATE_DIR: &str = ".spin"; + +/// A runtime configuration which has been resolved from a runtime config source. +/// +/// Includes other pieces of configuration that are used to resolve the runtime configuration. +#[derive(Default)] +pub struct ResolvedRuntimeConfig { + /// The resolved runtime configuration. + pub runtime_config: T, + /// The resolver used to resolve key-value stores from runtime configuration. + pub key_value_resolver: key_value::RuntimeConfigResolver, +} + +impl ResolvedRuntimeConfig +where + T: for<'a> TryFrom>, + for<'a> >>::Error: Into, +{ + /// Creates a new resolved runtime configuration from a runtime config source TOML file. + pub fn from_file(runtime_config_path: &Path, state_dir: Option<&str>) -> anyhow::Result { + let key_value_resolver = key_value_resolver(PathBuf::from( + state_dir.unwrap_or_else(|| DEFAULT_STATE_DIR.into()), + )); + + let file = std::fs::read_to_string(runtime_config_path).with_context(|| { + format!( + "failed to read runtime config file '{}'", + runtime_config_path.display() + ) + })?; + let toml = toml::from_str(&file).with_context(|| { + format!( + "failed to parse runtime config file '{}' as toml", + runtime_config_path.display() + ) + })?; + let runtime_config: T = TomlRuntimeConfigSource::new(&toml, &key_value_resolver) + .try_into() + .map_err(Into::into)?; + + Ok(Self { + runtime_config, + key_value_resolver, + }) + } +} + +/// The TOML based runtime configuration source Spin CLI. +pub struct TomlRuntimeConfigSource<'a> { + table: TomlKeyTracker<'a>, + key_value: &'a key_value::RuntimeConfigResolver, +} + +impl<'a> TomlRuntimeConfigSource<'a> { + pub fn new(table: &'a toml::Table, key_value: &'a key_value::RuntimeConfigResolver) -> Self { + Self { + table: TomlKeyTracker::new(table), + key_value, + } + } +} + +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(None) + } +} + +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result> { + self.key_value.resolve_from_toml(Some(self.table.as_ref())) + } +} + +impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_> { + fn finalize(&mut self) -> anyhow::Result<()> { + Ok(self.table.validate_all_keys_used()?) + } +} + +const DEFAULT_SPIN_STORE_FILENAME: &str = "sqlite_key_value.db"; + +/// The key-value runtime configuration resolver used by the trigger. +/// +/// Takes a base path for the local store. +pub fn key_value_resolver(local_store_base_path: PathBuf) -> key_value::RuntimeConfigResolver { + let mut key_value = key_value::RuntimeConfigResolver::new(); + + // Register the supported store types. + // Unwraps are safe because the store types are known to not overlap. + key_value + .register_store_type(spin_factor_key_value_spin::SpinKeyValueStore::new( + local_store_base_path, + )) + .unwrap(); + key_value + .register_store_type(spin_factor_key_value_redis::RedisKeyValueStore::new()) + .unwrap(); + key_value + .register_store_type(spin_factor_key_value_azure::AzureKeyValueStore::new()) + .unwrap(); + + // Add handling of "default" store. + key_value.add_default_store( + "default", + key_value::StoreConfig { + type_: spin_factor_key_value_spin::SpinKeyValueStore::RUNTIME_CONFIG_TYPE.to_owned(), + config: toml::toml! { + path = DEFAULT_SPIN_STORE_FILENAME + }, + }, + ); + + key_value +} diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 7ba90f9dc..47c6b48ce 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -13,6 +13,7 @@ anyhow = "1" clap = { version = "3.1.18", features = ["derive", "env"] } ctrlc = { version = "3.2", features = ["termination"] } futures = "0.3" +spin-runtime-config = { path = "../runtime-config" } sanitize-filename = "0.5" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -21,14 +22,10 @@ spin-common = { path = "../common" } spin-core = { path = "../core" } spin-factor-wasi = { path = "../factor-wasi" } spin-factor-key-value = { path = "../factor-key-value" } -spin-factor-key-value-spin = { path = "../factor-key-value-spin" } -spin-factor-key-value-redis = { path = "../factor-key-value-redis" } -spin-factor-key-value-azure = { path = "../factor-key-value-azure" } spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } tokio = { version = "1.23", features = ["fs"] } -toml = "0.8" tracing = { workspace = true } [lints] diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index d54d19dd4..11c59a680 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -9,9 +9,9 @@ use spin_common::ui::quoted_path; use spin_common::url::parse_file_url; use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; +use spin_runtime_config::ResolvedRuntimeConfig; -use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; -use crate::runtime_config::{key_value_resolver, RuntimeConfigSource}; +use crate::factors::TriggerFactors; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; use crate::Trigger; pub use launch_metadata::LaunchMetadata; @@ -21,7 +21,6 @@ pub const DISABLE_WASMTIME_CACHE: &str = "DISABLE_WASMTIME_CACHE"; pub const FOLLOW_LOG_OPT: &str = "FOLLOW_ID"; pub const WASMTIME_CACHE_FILE: &str = "WASMTIME_CACHE_FILE"; pub const RUNTIME_CONFIG_FILE: &str = "RUNTIME_CONFIG_FILE"; -pub const DEFAULT_STATE_DIR: &str = ".spin"; // Set by `spin up` pub const SPIN_LOCKED_URL: &str = "SPIN_LOCKED_URL"; @@ -193,32 +192,18 @@ impl FactorsTriggerCommand { }; trigger.add_to_linker(core_engine_builder.linker())?; - let (runtime_config, key_value_resolver) = self - .runtime_config_file - .as_ref() - .map(|path| { - let file = std::fs::read_to_string(path).with_context(|| { - format!("failed to read runtime config file {}", quoted_path(path)) - })?; - let toml = toml::from_str(&file).with_context(|| { - format!( - "failed to parse runtime config file {} as toml", - quoted_path(path) - ) - })?; - - let key_value_resolver = key_value_resolver(PathBuf::from( - self.state_dir.unwrap_or_else(|| DEFAULT_STATE_DIR.into()), - )); - let source: TriggerFactorsRuntimeConfig = - RuntimeConfigSource::new(&toml, &key_value_resolver).try_into()?; - anyhow::Ok((source, key_value_resolver)) - }) - .transpose()? - .unwrap_or_default(); - - let factors = - TriggerFactors::new(working_dir, self.allow_transient_write, key_value_resolver); + let runtime_config = match &self.runtime_config_file { + Some(runtime_config_path) => { + ResolvedRuntimeConfig::from_file(runtime_config_path, self.state_dir.as_deref())? + } + None => ResolvedRuntimeConfig::default(), + }; + + let factors = TriggerFactors::new( + working_dir, + self.allow_transient_write, + runtime_config.key_value_resolver, + ); // TODO: move these into Factor methods/constructors // let init_data = crate::HostComponentInitData::new( @@ -250,10 +235,9 @@ impl FactorsTriggerCommand { let configured_app = { let _sloth_guard = warn_if_wasm_build_slothful(); - executor.load_app(app, runtime_config, TodoComponentLoader)? + executor.load_app(app, runtime_config.runtime_config, TodoComponentLoader)? }; - // TODO: Construct factors let run_fut = trigger.run(configured_app); let (abortable, abort_handle) = futures::future::abortable(run_fut); diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index f323c8964..cbb197aab 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use spin_factor_key_value::{DefaultLabelResolver, KeyValueFactor}; use spin_factor_wasi::{spin::SpinFilesMounter, WasiFactor}; use spin_factors::RuntimeFactors; +use spin_runtime_config::TomlRuntimeConfigSource; #[derive(RuntimeFactors)] pub struct TriggerFactors { @@ -23,3 +24,11 @@ impl TriggerFactors { } } } + +impl TryFrom> for TriggerFactorsRuntimeConfig { + type Error = anyhow::Error; + + fn try_from(value: TomlRuntimeConfigSource<'_>) -> Result { + Self::from_source(value) + } +} diff --git a/crates/trigger2/src/lib.rs b/crates/trigger2/src/lib.rs index 1e3934bc8..095d30e3b 100644 --- a/crates/trigger2/src/lib.rs +++ b/crates/trigger2/src/lib.rs @@ -8,7 +8,6 @@ use spin_factors_executor::{FactorsExecutorApp, FactorsInstanceBuilder}; pub mod cli; mod factors; -mod runtime_config; mod stdio; /// Type alias for a [`FactorsConfiguredApp`] specialized to a [`Trigger`]. diff --git a/crates/trigger2/src/runtime_config.rs b/crates/trigger2/src/runtime_config.rs deleted file mode 100644 index 44dd2c6cd..000000000 --- a/crates/trigger2/src/runtime_config.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::path::PathBuf; - -use spin_factor_key_value::{self as key_value, KeyValueFactor}; -use spin_factor_wasi::WasiFactor; -use spin_factors::{ - runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, -}; - -use crate::factors::TriggerFactorsRuntimeConfig; - -/// A runtime configuration source for the [`TriggerFactors`][crate::TriggerFactors]. -pub struct RuntimeConfigSource<'a> { - table: TomlKeyTracker<'a>, - pub key_value: &'a key_value::runtime_config::spin::RuntimeConfigResolver, -} - -impl<'a> RuntimeConfigSource<'a> { - pub fn new( - table: &'a toml::Table, - key_value: &'a key_value::runtime_config::spin::RuntimeConfigResolver, - ) -> Self { - Self { - table: TomlKeyTracker::new(table), - key_value, - } - } -} - -impl RuntimeConfigSourceFinalizer for RuntimeConfigSource<'_> { - fn finalize(&mut self) -> anyhow::Result<()> { - Ok(self.table.validate_all_keys_used()?) - } -} - -impl FactorRuntimeConfigSource for RuntimeConfigSource<'_> { - fn get_runtime_config(&mut self) -> anyhow::Result> { - Ok(None) - } -} - -impl FactorRuntimeConfigSource for RuntimeConfigSource<'_> { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result> { - self.key_value.resolve_from_toml(Some(self.table.as_ref())) - } -} - -impl TryFrom> for TriggerFactorsRuntimeConfig { - type Error = anyhow::Error; - - fn try_from(value: RuntimeConfigSource<'_>) -> Result { - Self::from_source(value) - } -} - -const DEFAULT_SPIN_STORE_FILENAME: &str = "sqlite_key_value.db"; - -/// The key-value runtime configuration resolver used by the trigger. -/// -/// Takes a base path for the local store. -pub fn key_value_resolver( - local_store_base_path: PathBuf, -) -> spin_factor_key_value::runtime_config::spin::RuntimeConfigResolver { - let mut key_value = key_value::runtime_config::spin::RuntimeConfigResolver::new(); - key_value.add_default_store( - "default", - spin_factor_key_value::runtime_config::spin::StoreConfig { - type_: "spin".to_owned(), - config: toml::toml! { - path = DEFAULT_SPIN_STORE_FILENAME - }, - }, - ); - // Unwraps are safe because the store types are known to not overlap. - key_value - .register_store_type(spin_factor_key_value_spin::SpinKeyValueStore::new( - local_store_base_path, - )) - .unwrap(); - key_value - .register_store_type(spin_factor_key_value_redis::RedisKeyValueStore::new()) - .unwrap(); - key_value - .register_store_type(spin_factor_key_value_azure::AzureKeyValueStore::new()) - .unwrap(); - key_value -} From 75a314eb74ed7092a0cf3e81b50da2ae2160c152 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 15 Aug 2024 10:03:45 +0200 Subject: [PATCH 122/195] Set initial key-values Signed-off-by: Ryan Levick --- crates/runtime-config/src/lib.rs | 31 +++++++++++++++++++++++++++---- crates/trigger2/src/cli.rs | 11 +++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 28f62b354..3e61a048b 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -2,12 +2,13 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; use spin_factor_key_value::runtime_config::spin::{self as key_value, MakeKeyValueStore}; -use spin_factor_key_value::KeyValueFactor; +use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; use spin_factor_wasi::WasiFactor; use spin_factors::{ runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, }; +/// The default state directory for the trigger. pub const DEFAULT_STATE_DIR: &str = ".spin"; /// A runtime configuration which has been resolved from a runtime config source. @@ -53,6 +54,27 @@ where key_value_resolver, }) } + + /// Set initial key-value pairs supplied in the CLI arguments in the default store. + pub async fn set_initial_key_values( + &self, + initial_key_values: impl IntoIterator, + ) -> anyhow::Result<()> { + let store = self + .key_value_resolver + .default(DEFAULT_KEY_VALUE_STORE_LABEL) + .expect("trigger was misconfigured and lacks a default store") + .get(DEFAULT_KEY_VALUE_STORE_LABEL) + .await + .expect("trigger was misconfigured and lacks a default store"); + for (key, value) in initial_key_values { + store + .set(key, value.as_bytes()) + .await + .context("failed to set key-value pair")?; + } + Ok(()) + } } /// The TOML based runtime configuration source Spin CLI. @@ -90,7 +112,8 @@ impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_> { } } -const DEFAULT_SPIN_STORE_FILENAME: &str = "sqlite_key_value.db"; +const DEFAULT_KEY_VALUE_STORE_FILENAME: &str = "sqlite_key_value.db"; +const DEFAULT_KEY_VALUE_STORE_LABEL: &str = "default"; /// The key-value runtime configuration resolver used by the trigger. /// @@ -114,11 +137,11 @@ pub fn key_value_resolver(local_store_base_path: PathBuf) -> key_value::RuntimeC // Add handling of "default" store. key_value.add_default_store( - "default", + DEFAULT_KEY_VALUE_STORE_LABEL, key_value::StoreConfig { type_: spin_factor_key_value_spin::SpinKeyValueStore::RUNTIME_CONFIG_TYPE.to_owned(), config: toml::toml! { - path = DEFAULT_SPIN_STORE_FILENAME + path = DEFAULT_KEY_VALUE_STORE_FILENAME }, }, ); diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index 11c59a680..0c72480b3 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -11,7 +11,7 @@ use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; use spin_runtime_config::ResolvedRuntimeConfig; -use crate::factors::TriggerFactors; +use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; use crate::Trigger; pub use launch_metadata::LaunchMetadata; @@ -194,11 +194,18 @@ impl FactorsTriggerCommand { let runtime_config = match &self.runtime_config_file { Some(runtime_config_path) => { - ResolvedRuntimeConfig::from_file(runtime_config_path, self.state_dir.as_deref())? + ResolvedRuntimeConfig::::from_file( + runtime_config_path, + self.state_dir.as_deref(), + )? } None => ResolvedRuntimeConfig::default(), }; + runtime_config + .set_initial_key_values(&self.key_values) + .await?; + let factors = TriggerFactors::new( working_dir, self.allow_transient_write, From 111e5d786c152576a63fffe820d01dc30817cfb9 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 2 Aug 2024 17:27:35 -0400 Subject: [PATCH 123/195] factors: Implement outbound HTTP request handling Signed-off-by: Lann Martin --- Cargo.lock | 41 ++- crates/factor-outbound-http/Cargo.toml | 8 +- crates/factor-outbound-http/src/lib.rs | 96 +++++- crates/factor-outbound-http/src/wasi.rs | 294 +++++++++++++++++- .../factor-outbound-http/tests/factor_test.rs | 10 +- crates/factor-outbound-networking/src/lib.rs | 15 +- crates/trigger-http2/src/lib.rs | 2 +- crates/trigger-http2/src/outbound_http.rs | 64 ++++ crates/trigger-http2/src/server.rs | 139 +++++---- crates/trigger-http2/src/tls.rs | 2 + crates/trigger2/Cargo.toml | 2 + crates/trigger2/src/factors.rs | 6 + 12 files changed, 574 insertions(+), 105 deletions(-) create mode 100644 crates/trigger-http2/src/outbound_http.rs diff --git a/Cargo.lock b/Cargo.lock index ec7e38c55..36441eec3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3488,9 +3488,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.2.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -3572,7 +3572,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.2.0", + "hyper 1.4.1", "hyper-util", "native-tls", "tokio", @@ -3591,7 +3591,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.2.0", + "hyper 1.4.1", "pin-project-lite", "socket2 0.5.6", "tokio", @@ -6519,7 +6519,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.4.1", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -7500,7 +7500,7 @@ dependencies = [ "glob", "hex", "http-body-util", - "hyper 1.2.0", + "hyper 1.4.1", "hyper-util", "indicatif 0.17.8", "is-terminal", @@ -7708,14 +7708,20 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "http 1.1.0", + "http-body-util", + "hyper 1.4.1", + "rustls 0.23.7", "spin-factor-outbound-networking", "spin-factor-variables", "spin-factors", "spin-factors-test", "spin-world", + "terminal", "tokio", + "tokio-rustls 0.26.0", "tracing", "wasmtime", + "wasmtime-wasi", "wasmtime-wasi-http", ] @@ -7883,7 +7889,7 @@ dependencies = [ "anyhow", "http 1.1.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.4.1", "indexmap 1.9.3", "percent-encoding", "routefinder", @@ -8272,7 +8278,7 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "http 1.1.0", - "hyper 1.2.0", + "hyper 1.4.1", "serde 1.0.197", "serde_json", "spin-app", @@ -8352,7 +8358,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.4.1", "hyper-util", "indexmap 1.9.3", "num_cpus", @@ -8395,7 +8401,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.4.1", "hyper-util", "indexmap 1.9.3", "percent-encoding", @@ -8461,6 +8467,8 @@ dependencies = [ "spin-common", "spin-core", "spin-factor-key-value", + "spin-factor-outbound-http", + "spin-factor-outbound-networking", "spin-factor-wasi", "spin-factors", "spin-factors-executor", @@ -9086,6 +9094,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.7", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-socks" version = "0.5.1" @@ -10422,7 +10441,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.4.1", "rustls 0.22.4", "tokio", "tokio-rustls 0.25.0", diff --git a/crates/factor-outbound-http/Cargo.toml b/crates/factor-outbound-http/Cargo.toml index b9c5943e4..cee32e483 100644 --- a/crates/factor-outbound-http/Cargo.toml +++ b/crates/factor-outbound-http/Cargo.toml @@ -7,17 +7,23 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" http = "1.1.0" +http-body-util = "0.1" +hyper = "1.4.1" +rustls = "0.23" spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factors = { path = "../factors" } spin-world = { path = "../world" } +terminal = { path = "../terminal" } +tokio = { version = "1", features = ["macros", "rt"] } +tokio-rustls = "0.26" tracing = { workspace = true } wasmtime = { workspace = true } +wasmtime-wasi = { workspace = true } wasmtime-wasi-http = { workspace = true } [dev-dependencies] spin-factor-variables = { path = "../factor-variables" } spin-factors-test = { path = "../factors-test" } -tokio = { version = "1", features = ["macros", "rt"] } [lints] workspace = true diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index ed01d137e..1e365e253 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -3,13 +3,26 @@ mod wasi; pub mod wasi_2023_10_18; pub mod wasi_2023_11_10; -use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; +use anyhow::Context; +use http::{ + uri::{Authority, Parts, PathAndQuery, Scheme}, + HeaderValue, Uri, +}; +use spin_factor_outbound_networking::{ + ComponentTlsConfigs, OutboundAllowedHosts, OutboundNetworkingFactor, +}; use spin_factors::{ anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; use wasmtime_wasi_http::WasiHttpCtx; +pub use wasmtime_wasi_http::{ + body::HyperOutgoingBody, + types::{HostFutureIncomingResponse, OutgoingRequestConfig}, + HttpResult, +}; + pub struct OutboundHttpFactor; impl Factor for OutboundHttpFactor { @@ -38,19 +51,90 @@ impl Factor for OutboundHttpFactor { _ctx: PrepareContext, builders: &mut InstanceBuilders, ) -> anyhow::Result { - let allowed_hosts = builders - .get_mut::()? - .allowed_hosts(); + let outbound_networking = builders.get_mut::()?; + let allowed_hosts = outbound_networking.allowed_hosts(); + let component_tls_configs = outbound_networking.component_tls_configs().clone(); Ok(InstanceState { - allowed_hosts, wasi_http_ctx: WasiHttpCtx::new(), + allowed_hosts, + component_tls_configs, + request_interceptor: None, }) } } pub struct InstanceState { - allowed_hosts: OutboundAllowedHosts, wasi_http_ctx: WasiHttpCtx, + allowed_hosts: OutboundAllowedHosts, + component_tls_configs: ComponentTlsConfigs, + request_interceptor: Option>, +} + +impl InstanceState { + pub fn set_request_interceptor( + &mut self, + interceptor: impl OutboundHttpInterceptor + 'static, + ) -> anyhow::Result<()> { + if self.request_interceptor.is_some() { + anyhow::bail!("set_request_interceptor can only be called once"); + } + self.request_interceptor = Some(Box::new(interceptor)); + Ok(()) + } } impl SelfInstanceBuilder for InstanceState {} + +pub type Request = http::Request; + +/// SelfRequestOrigin indicates the base URI to use for "self" requests. +/// +/// This is meant to be set on [`Request::extensions_mut`] in appropriate +/// contexts such as an incoming request handler. +#[derive(Clone, Debug)] +pub struct SelfRequestOrigin { + pub scheme: Scheme, + pub authority: Authority, +} + +impl SelfRequestOrigin { + pub fn from_uri(uri: &Uri) -> anyhow::Result { + Ok(Self { + scheme: uri.scheme().context("URI missing scheme")?.clone(), + authority: uri.authority().context("URI missing authority")?.clone(), + }) + } + + fn into_uri(self, path_and_query: Option) -> Uri { + let mut parts = Parts::default(); + parts.scheme = Some(self.scheme); + parts.authority = Some(self.authority); + parts.path_and_query = path_and_query; + Uri::from_parts(parts).unwrap() + } + + fn use_tls(&self) -> bool { + self.scheme == Scheme::HTTPS + } + + fn host_header(&self) -> HeaderValue { + HeaderValue::from_str(self.authority.as_str()).unwrap() + } +} + +pub trait OutboundHttpInterceptor: Send + Sync { + /// Intercept an outgoing HTTP request. + /// + /// If this method returns `None`, the (possibly updated) request and config + /// will be passed on to the default outgoing request handler. + /// + /// If this method returns `Some(...)`, the inner result will be returned as + /// the result of the request, bypassing the default handler. + fn intercept(&self, intercepted: Intercepted) + -> Option>; +} + +pub struct Intercepted<'a> { + pub request: &'a mut Request, + pub config: &'a mut OutgoingRequestConfig, +} diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index f5b99497e..0402193ac 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -1,10 +1,25 @@ -use http::Request; +use std::{error::Error, sync::Arc}; + +use anyhow::Context; +use http::{header::HOST, uri::Authority, Request, Uri}; +use http_body_util::BodyExt; +use rustls::ClientConfig; +use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundUrl}; use spin_factors::{wasmtime::component::ResourceTable, RuntimeFactorsInstanceState}; +use tokio::{net::TcpStream, time::timeout}; +use tracing::Instrument; use wasmtime_wasi_http::{ - bindings::http::types::ErrorCode, WasiHttpCtx, WasiHttpImpl, WasiHttpView, + bindings::http::types::ErrorCode, + body::HyperOutgoingBody, + io::TokioIo, + types::{HostFutureIncomingResponse, IncomingResponse}, + WasiHttpCtx, WasiHttpImpl, WasiHttpView, }; -use crate::{wasi_2023_10_18, wasi_2023_11_10, OutboundHttpFactor}; +use crate::{ + wasi_2023_10_18, wasi_2023_11_10, InstanceState, Intercepted, OutboundHttpFactor, + SelfRequestOrigin, +}; pub(crate) fn add_to_linker( ctx: &mut spin_factors::InitContext, @@ -18,10 +33,7 @@ pub(crate) fn add_to_linker( let get_data_with_table = ctx.get_data_with_table_fn(); let closure = type_annotate(move |data| { let (state, table) = get_data_with_table(data); - WasiHttpImpl(WasiHttpImplInner { - ctx: &mut state.wasi_http_ctx, - table, - }) + WasiHttpImpl(WasiHttpImplInner { state, table }) }); let linker = ctx.linker(); wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker_get_host(linker, closure)?; @@ -38,21 +50,18 @@ impl OutboundHttpFactor { runtime_instance_state: &mut impl RuntimeFactorsInstanceState, ) -> Option> { let (state, table) = runtime_instance_state.get_with_table::()?; - Some(WasiHttpImpl(WasiHttpImplInner { - ctx: &mut state.wasi_http_ctx, - table, - })) + Some(WasiHttpImpl(WasiHttpImplInner { state, table })) } } pub(crate) struct WasiHttpImplInner<'a> { - ctx: &'a mut WasiHttpCtx, + state: &'a mut InstanceState, table: &'a mut ResourceTable, } impl<'a> WasiHttpView for WasiHttpImplInner<'a> { fn ctx(&mut self) -> &mut WasiHttpCtx { - self.ctx + &mut self.state.wasi_http_ctx } fn table(&mut self) -> &mut ResourceTable { @@ -61,10 +70,261 @@ impl<'a> WasiHttpView for WasiHttpImplInner<'a> { fn send_request( &mut self, - _request: Request, - _config: wasmtime_wasi_http::types::OutgoingRequestConfig, + mut request: Request, + mut config: wasmtime_wasi_http::types::OutgoingRequestConfig, ) -> wasmtime_wasi_http::HttpResult { - // TODO: port implementation from spin-trigger-http - Err(ErrorCode::HttpRequestDenied.into()) + if let Some(interceptor) = &self.state.request_interceptor { + let intercepted = Intercepted { + request: &mut request, + config: &mut config, + }; + if let Some(res) = interceptor.intercept(intercepted) { + return res; + } + } + + let host = request.uri().host().unwrap_or_default(); + let tls_client_config = self + .state + .component_tls_configs + .get_client_config(host) + .clone(); + + Ok(HostFutureIncomingResponse::Pending( + wasmtime_wasi::runtime::spawn( + send_request_impl( + request, + config, + self.state.allowed_hosts.clone(), + tls_client_config, + ) + .in_current_span(), + ), + )) } } + +async fn send_request_impl( + mut request: Request, + mut config: wasmtime_wasi_http::types::OutgoingRequestConfig, + allowed_hosts: OutboundAllowedHosts, + tls_client_config: Arc, +) -> anyhow::Result> { + let allowed_hosts = allowed_hosts.resolve().await?; + + let is_relative_url = request.uri().authority().is_none(); + if is_relative_url { + if !allowed_hosts.allows_relative_url(&["http", "https"]) { + return handle_not_allowed(request.uri(), true); + } + + let origin = request + .extensions() + .get::() + .cloned() + .context("cannot send relative outbound request; no 'origin' set by host")?; + + config.use_tls = origin.use_tls(); + + request.headers_mut().insert(HOST, origin.host_header()); + + let path_and_query = request.uri().path_and_query().cloned(); + *request.uri_mut() = origin.into_uri(path_and_query); + } else { + let outbound_url = OutboundUrl::parse(request.uri().to_string(), "https") + .map_err(|_| ErrorCode::HttpRequestUriInvalid)?; + if !allowed_hosts.allows(&outbound_url) { + return handle_not_allowed(request.uri(), false); + } + } + + if let Some(authority) = request.uri().authority() { + let current_span = tracing::Span::current(); + current_span.record("server.address", authority.host()); + if let Some(port) = authority.port() { + current_span.record("server.port", port.as_u16()); + } + } + + Ok(send_request_handler(request, config, tls_client_config).await) +} + +// TODO(factors): Move to some callback on spin-factor-outbound-networking (?) +fn handle_not_allowed( + uri: &Uri, + is_relative: bool, +) -> anyhow::Result> { + tracing::error!("Destination not allowed: {uri}"); + let host = if is_relative { + terminal::warn!("A component tried to make a HTTP request to the same component but it does not have permission."); + "self".to_string() + } else { + let host = format!( + "{scheme}://{authority}", + scheme = uri.scheme_str().unwrap_or_default(), + authority = uri.authority().map(Authority::as_str).unwrap_or_default() + ); + terminal::warn!("A component tried to make a HTTP request to non-allowed host '{host}'."); + host + }; + eprintln!("To allow requests, add 'allowed_outbound_hosts = [\"{host}\"]' to the manifest component section."); + Err(ErrorCode::HttpRequestDenied.into()) +} + +/// This is a fork of wasmtime_wasi_http::default_send_request_handler function +/// forked from bytecodealliance/wasmtime commit-sha 29a76b68200fcfa69c8fb18ce6c850754279a05b +/// This fork provides the ability to configure client cert auth for mTLS +async fn send_request_handler( + mut request: http::Request, + wasmtime_wasi_http::types::OutgoingRequestConfig { + use_tls, + connect_timeout, + first_byte_timeout, + between_bytes_timeout, + }: wasmtime_wasi_http::types::OutgoingRequestConfig, + tls_client_config: Arc, +) -> Result { + let authority_str = if let Some(authority) = request.uri().authority() { + if authority.port().is_some() { + authority.to_string() + } else { + let port = if use_tls { 443 } else { 80 }; + format!("{}:{port}", authority) + } + } else { + return Err(ErrorCode::HttpRequestUriInvalid); + }; + + let tcp_stream = timeout(connect_timeout, TcpStream::connect(&authority_str)) + .await + .map_err(|_| ErrorCode::ConnectionTimeout)? + .map_err(|err| match err.kind() { + std::io::ErrorKind::AddrNotAvailable => { + dns_error("address not available".to_string(), 0) + } + _ => { + if err + .to_string() + .starts_with("failed to lookup address information") + { + dns_error("address not available".to_string(), 0) + } else { + ErrorCode::ConnectionRefused + } + } + })?; + + let (mut sender, worker) = if use_tls { + #[cfg(any(target_arch = "riscv64", target_arch = "s390x"))] + { + return Err(ErrorCode::InternalError(Some( + "unsupported architecture for SSL".to_string(), + ))); + } + + #[cfg(not(any(target_arch = "riscv64", target_arch = "s390x")))] + { + use rustls::pki_types::ServerName; + let connector = tokio_rustls::TlsConnector::from(tls_client_config); + let mut parts = authority_str.split(':'); + let host = parts.next().unwrap_or(&authority_str); + let domain = ServerName::try_from(host) + .map_err(|e| { + tracing::warn!("dns lookup error: {e:?}"); + dns_error("invalid dns name".to_string(), 0) + })? + .to_owned(); + let stream = connector.connect(domain, tcp_stream).await.map_err(|e| { + tracing::warn!("tls protocol error: {e:?}"); + ErrorCode::TlsProtocolError + })?; + let stream = TokioIo::new(stream); + + let (sender, conn) = timeout( + connect_timeout, + hyper::client::conn::http1::handshake(stream), + ) + .await + .map_err(|_| ErrorCode::ConnectionTimeout)? + .map_err(hyper_request_error)?; + + let worker = wasmtime_wasi::runtime::spawn(async move { + match conn.await { + Ok(()) => {} + // TODO: shouldn't throw away this error and ideally should + // surface somewhere. + Err(e) => tracing::warn!("dropping error {e}"), + } + }); + + (sender, worker) + } + } else { + let tcp_stream = TokioIo::new(tcp_stream); + let (sender, conn) = timeout( + connect_timeout, + // TODO: we should plumb the builder through the http context, and use it here + hyper::client::conn::http1::handshake(tcp_stream), + ) + .await + .map_err(|_| ErrorCode::ConnectionTimeout)? + .map_err(hyper_request_error)?; + + let worker = wasmtime_wasi::runtime::spawn(async move { + match conn.await { + Ok(()) => {} + // TODO: same as above, shouldn't throw this error away. + Err(e) => tracing::warn!("dropping error {e}"), + } + }); + + (sender, worker) + }; + + // at this point, the request contains the scheme and the authority, but + // the http packet should only include those if addressing a proxy, so + // remove them here, since SendRequest::send_request does not do it for us + *request.uri_mut() = http::Uri::builder() + .path_and_query( + request + .uri() + .path_and_query() + .map(|p| p.as_str()) + .unwrap_or("/"), + ) + .build() + .expect("comes from valid request"); + + let resp = timeout(first_byte_timeout, sender.send_request(request)) + .await + .map_err(|_| ErrorCode::ConnectionReadTimeout)? + .map_err(hyper_request_error)? + .map(|body| body.map_err(hyper_request_error).boxed()); + + Ok(wasmtime_wasi_http::types::IncomingResponse { + resp, + worker: Some(worker), + between_bytes_timeout, + }) +} + +/// Translate a [`hyper::Error`] to a wasi-http `ErrorCode` in the context of a request. +fn hyper_request_error(err: hyper::Error) -> ErrorCode { + // If there's a source, we might be able to extract a wasi-http error from it. + if let Some(cause) = err.source() { + if let Some(err) = cause.downcast_ref::() { + return err.clone(); + } + } + + tracing::warn!("hyper request error: {err:?}"); + + ErrorCode::HttpProtocolError +} + +fn dns_error(rcode: String, info_code: u16) -> ErrorCode { + ErrorCode::DnsError(wasmtime_wasi_http::bindings::http::types::DnsErrorPayload { + rcode: Some(rcode), + info_code: Some(info_code), + }) +} diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index 9abee5f57..7633aa4ba 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -7,6 +7,7 @@ use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; +use wasmtime_wasi::Subscribe; use wasmtime_wasi_http::{ bindings::http::types::ErrorCode, types::OutgoingRequestConfig, WasiHttpView, }; @@ -34,11 +35,12 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); let req = Request::get("https://denied.test").body(Default::default())?; - let res = wasi_http.send_request(req, test_request_config()); - let Err(err) = res else { - bail!("expected Err, got Ok"); + let mut future_resp = wasi_http.send_request(req, test_request_config())?; + future_resp.ready().await; + match future_resp.unwrap_ready() { + Ok(_) => bail!("expected Err, got Ok"), + Err(err) => assert!(matches!(err.downcast()?, ErrorCode::HttpRequestDenied)), }; - assert!(matches!(err.downcast()?, ErrorCode::HttpRequestDenied)); Ok(()) } diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index fdcb1ffed..abf270bb0 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -157,11 +157,19 @@ impl FactorInstanceBuilder for InstanceBuilder { } // TODO: Refactor w/ spin-outbound-networking crate to simplify +#[derive(Clone)] pub struct OutboundAllowedHosts { allowed_hosts_future: SharedFutureResult, } impl OutboundAllowedHosts { + pub async fn resolve(&self) -> anyhow::Result> { + self.allowed_hosts_future.clone().await.map_err(|err| { + // TODO: better way to handle this? + anyhow::Error::msg(err) + }) + } + pub async fn allows(&self, url: &OutboundUrl) -> anyhow::Result { Ok(self.resolve().await?.allows(url)) } @@ -174,11 +182,4 @@ impl OutboundAllowedHosts { &allowed_hosts, )) } - - async fn resolve(&self) -> anyhow::Result> { - self.allowed_hosts_future.clone().await.map_err(|err| { - // TODO: better way to handle this? - anyhow::Error::msg(err) - }) - } } diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index 86d00ace1..81bbc43db 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -7,6 +7,7 @@ mod spin; mod tls; mod wagi; mod wasi; +mod outbound_http; use std::{ collections::HashMap, @@ -296,7 +297,6 @@ mod tests { let default_headers = compute_default_headers(req.uri(), host, &route_match, client_addr)?; - // TODO: we currently replace the scheme with HTTP. When TLS is supported, this should be fixed. assert_eq!( search(&FULL_URL, &default_headers).unwrap(), "https://fermyon.dev/foo/42/bar?key1=value1&key2=value2".to_string() diff --git a/crates/trigger-http2/src/outbound_http.rs b/crates/trigger-http2/src/outbound_http.rs new file mode 100644 index 000000000..5baaf9792 --- /dev/null +++ b/crates/trigger-http2/src/outbound_http.rs @@ -0,0 +1,64 @@ +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::Arc, +}; + +use spin_factor_outbound_http::{HostFutureIncomingResponse, SelfRequestOrigin}; +use spin_http::routes::RouteMatch; +use spin_outbound_networking::parse_service_chaining_target; +use wasmtime_wasi_http::types::IncomingResponse; + +use crate::server::HttpServer; + +pub struct OutboundHttpInterceptor { + server: Arc, + origin: SelfRequestOrigin, +} + +impl OutboundHttpInterceptor { + pub fn new(server: Arc, origin: SelfRequestOrigin) -> Self { + Self { server, origin } + } +} + +const CHAINED_CLIENT_ADDR: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); + +impl spin_factor_outbound_http::OutboundHttpInterceptor for OutboundHttpInterceptor { + fn intercept( + &self, + intercepted: spin_factor_outbound_http::Intercepted, + ) -> Option> + { + let uri = intercepted.request.uri(); + + // Handle service chaining requests + if let Some(component_id) = parse_service_chaining_target(uri) { + // TODO: look at the rest of chain_request + let route_match = RouteMatch::synthetic(&component_id, uri.path()); + let req = std::mem::take(intercepted.request); + let between_bytes_timeout = intercepted.config.between_bytes_timeout; + let server = self.server.clone(); + let resp_fut = async move { + match server + .handle_trigger_route(req, route_match, CHAINED_CLIENT_ADDR) + .await + { + Ok(resp) => Ok(Ok(IncomingResponse { + resp, + between_bytes_timeout, + worker: None, + })), + Err(e) => Err(wasmtime::Error::msg(e)), + } + }; + let resp = HostFutureIncomingResponse::pending(wasmtime_wasi::runtime::spawn(resp_fut)); + Some(Ok(resp)) + } else { + intercepted + .request + .extensions_mut() + .insert(self.origin.clone()); + None + } + } +} diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index cc7c80a9e..8e346369b 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, future::Future, io::IsTerminal, net::SocketAddr, sync::Arc}; +use anyhow::Context; use http::{uri::Scheme, Request, Response, StatusCode, Uri}; use http_body_util::BodyExt; use hyper::{ @@ -9,6 +10,7 @@ use hyper::{ }; use hyper_util::rt::TokioIo; use spin_app::{APP_DESCRIPTION_KEY, APP_NAME_KEY}; +use spin_factor_outbound_http::SelfRequestOrigin; use spin_http::{ app_info::AppInfo, body, @@ -27,6 +29,7 @@ use wasmtime_wasi_http::body::HyperOutgoingBody; use crate::{ headers::strip_forbidden_headers, instrument::{finalize_http_span, http_span, instrument_error, MatchedRoute}, + outbound_http::OutboundHttpInterceptor, spin::SpinHttpExecutor, wagi::WagiHttpExecutor, wasi::WasiHttpExecutor, @@ -92,8 +95,8 @@ impl HttpServer { } /// Handles incoming requests using an HTTP executor. - pub async fn handle( - &self, + async fn handle( + self: &Arc, mut req: Request, scheme: Scheme, client_addr: SocketAddr, @@ -119,71 +122,91 @@ impl HttpServer { }; } + match self.router.route(&path) { + Ok(route_match) => { + self.handle_trigger_route(req, route_match, client_addr) + .await + } + Err(_) => Self::not_found(NotFoundRouteKind::Normal(path.to_string())), + } + } + + pub async fn handle_trigger_route( + self: &Arc, + req: Request, + route_match: RouteMatch, + client_addr: SocketAddr, + ) -> anyhow::Result> { let app_id = self .trigger_app .app() .get_metadata(APP_NAME_KEY)? .unwrap_or_else(|| "".into()); - // Route to app component - match self.router.route(&path) { - Ok(route_match) => { - let component_id = route_match.component_id(); - - spin_telemetry::metrics::monotonic_counter!( - spin.request_count = 1, - trigger_type = "http", - app_id = app_id, - component_id = component_id - ); - - let instance_builder = self.trigger_app.prepare(component_id)?; - let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); - let handler_type = self.component_handler_types.get(component_id).unwrap(); - let executor = trigger_config - .executor - .as_ref() - .unwrap_or(&HttpExecutorType::Http); - - let res = match executor { - HttpExecutorType::Http => match handler_type { - HandlerType::Spin => { - SpinHttpExecutor - .execute(instance_builder, &route_match, req, client_addr) - .await - } - HandlerType::Wasi0_2 - | HandlerType::Wasi2023_11_10 - | HandlerType::Wasi2023_10_18 => { - WasiHttpExecutor { - handler_type: *handler_type, - } - .execute(instance_builder, &route_match, req, client_addr) - .await - } - }, - HttpExecutorType::Wagi(wagi_config) => { - let executor = WagiHttpExecutor { - wagi_config: wagi_config.clone(), - }; - executor - .execute(instance_builder, &route_match, req, client_addr) - .await - } - }; - match res { - Ok(res) => Ok(MatchedRoute::with_response_extension( - res, - route_match.raw_route(), - )), - Err(err) => { - tracing::error!("Error processing request: {err:?}"); - instrument_error(&err); - Self::internal_error(None, route_match.raw_route()) + let component_id = route_match.component_id(); + + spin_telemetry::metrics::monotonic_counter!( + spin.request_count = 1, + trigger_type = "http", + app_id = app_id, + component_id = component_id + ); + + let mut instance_builder = self.trigger_app.prepare(component_id)?; + + // Set up outbound HTTP request origin and service chaining + let uri = req.uri(); + let origin = SelfRequestOrigin::from_uri(uri) + .with_context(|| format!("invalid request URI {uri:?}"))?; + instance_builder + .factor_builders() + .outbound_http() + .set_request_interceptor(OutboundHttpInterceptor::new(self.clone(), origin))?; + + // Prepare HTTP executor + let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); + let handler_type = self.component_handler_types.get(component_id).unwrap(); + let executor = trigger_config + .executor + .as_ref() + .unwrap_or(&HttpExecutorType::Http); + + let res = match executor { + HttpExecutorType::Http => match handler_type { + HandlerType::Spin => { + SpinHttpExecutor + .execute(instance_builder, &route_match, req, client_addr) + .await + } + HandlerType::Wasi0_2 + | HandlerType::Wasi2023_11_10 + | HandlerType::Wasi2023_10_18 => { + WasiHttpExecutor { + handler_type: *handler_type, } + .execute(instance_builder, &route_match, req, client_addr) + .await } + }, + HttpExecutorType::Wagi(wagi_config) => { + let executor = WagiHttpExecutor { + wagi_config: wagi_config.clone(), + }; + executor + .execute(instance_builder, &route_match, req, client_addr) + .await + } + }; + match res { + Ok(res) => Ok(MatchedRoute::with_response_extension( + res, + route_match.raw_route(), + )), + Err(err) => { + tracing::error!("Error processing request: {err:?}"); + instrument_error(&err); + Self::internal_error(None, route_match.raw_route()) } - Err(_) => Self::not_found(NotFoundRouteKind::Normal(path.to_string())), } } diff --git a/crates/trigger-http2/src/tls.rs b/crates/trigger-http2/src/tls.rs index 39f6ca3d3..0888feed0 100644 --- a/crates/trigger-http2/src/tls.rs +++ b/crates/trigger-http2/src/tls.rs @@ -6,6 +6,8 @@ use std::{ }; use tokio_rustls::{rustls, TlsAcceptor}; +// TODO: dedupe with spin-factor-outbound-networking (spin-tls crate?) + /// TLS configuration for the server. #[derive(Clone)] pub struct TlsConfig { diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 47c6b48ce..1342cc02a 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -20,6 +20,8 @@ serde_json = "1" spin-app = { path = "../app" } spin-common = { path = "../common" } spin-core = { path = "../core" } +spin-factor-outbound-http = { path = "../factor-outbound-http" } +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-wasi = { path = "../factor-wasi" } spin-factor-key-value = { path = "../factor-key-value" } spin-factors = { path = "../factors" } diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index cbb197aab..aa1a0f315 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -1,6 +1,8 @@ use std::path::PathBuf; use spin_factor_key_value::{DefaultLabelResolver, KeyValueFactor}; +use spin_factor_outbound_http::OutboundHttpFactor; +use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_wasi::{spin::SpinFilesMounter, WasiFactor}; use spin_factors::RuntimeFactors; use spin_runtime_config::TomlRuntimeConfigSource; @@ -9,6 +11,8 @@ use spin_runtime_config::TomlRuntimeConfigSource; pub struct TriggerFactors { pub wasi: WasiFactor, pub key_value: KeyValueFactor, + pub outbound_networking: OutboundNetworkingFactor, + pub outbound_http: OutboundHttpFactor, } impl TriggerFactors { @@ -21,6 +25,8 @@ impl TriggerFactors { Self { wasi: WasiFactor::new(files_mounter), key_value: KeyValueFactor::new(default_key_value_label_resolver), + outbound_networking: OutboundNetworkingFactor, + outbound_http: OutboundHttpFactor, } } } From f2ba90eca4cf47f88778483fa3a3a9bf41d46f98 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 16 Aug 2024 10:39:45 -0400 Subject: [PATCH 124/195] Address review feedback Signed-off-by: Lann Martin --- Cargo.lock | 2 + crates/factor-outbound-http/src/lib.rs | 33 ++++++--- crates/factor-outbound-http/src/wasi.rs | 17 ++--- .../factor-outbound-http/tests/factor_test.rs | 70 +++++++++++++++---- crates/runtime-config/Cargo.toml | 2 + crates/runtime-config/src/lib.rs | 49 +++++++++---- crates/trigger-http2/src/lib.rs | 2 +- crates/trigger-http2/src/outbound_http.rs | 25 ++++--- 8 files changed, 142 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 36441eec3..4b5828045 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8162,6 +8162,8 @@ dependencies = [ "spin-factor-key-value-azure", "spin-factor-key-value-redis", "spin-factor-key-value-spin", + "spin-factor-outbound-http", + "spin-factor-outbound-networking", "spin-factor-wasi", "spin-factors", "toml 0.8.14", diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 1e365e253..78d05848d 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -71,6 +71,9 @@ pub struct InstanceState { } impl InstanceState { + /// Sets a [`OutboundHttpInterceptor`] for this instance. + /// + /// Returns an error if it has already been called for this instance. pub fn set_request_interceptor( &mut self, interceptor: impl OutboundHttpInterceptor + 'static, @@ -122,19 +125,31 @@ impl SelfRequestOrigin { } } +/// An outbound HTTP request interceptor to be used with +/// [`InstanceState::set_request_interceptor`]. pub trait OutboundHttpInterceptor: Send + Sync { /// Intercept an outgoing HTTP request. /// - /// If this method returns `None`, the (possibly updated) request and config - /// will be passed on to the default outgoing request handler. + /// If this method returns [`InterceptedResponse::Continue`], the (possibly + /// updated) request and config will be passed on to the default outgoing + /// request handler. /// - /// If this method returns `Some(...)`, the inner result will be returned as - /// the result of the request, bypassing the default handler. - fn intercept(&self, intercepted: Intercepted) - -> Option>; + /// If this method returns [`InterceptedResponse::Intercepted`], the inner + /// result will be returned as the result of the request, bypassing the + /// default handler. + fn intercept( + &self, + request: &mut Request, + config: &mut OutgoingRequestConfig, + ) -> InterceptOutcome; } -pub struct Intercepted<'a> { - pub request: &'a mut Request, - pub config: &'a mut OutgoingRequestConfig, +/// The type returned by an [`OutboundHttpInterceptor`]. +pub enum InterceptOutcome { + /// The intercepted request will be passed on to the default outgoing + /// request handler. + Continue, + /// The given result will be returned as the result of the intercepted + /// request, bypassing the default handler. + Complete(HttpResult), } diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index 0402193ac..c1427b0f1 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -17,7 +17,7 @@ use wasmtime_wasi_http::{ }; use crate::{ - wasi_2023_10_18, wasi_2023_11_10, InstanceState, Intercepted, OutboundHttpFactor, + wasi_2023_10_18, wasi_2023_11_10, InstanceState, InterceptOutcome, OutboundHttpFactor, SelfRequestOrigin, }; @@ -74,12 +74,9 @@ impl<'a> WasiHttpView for WasiHttpImplInner<'a> { mut config: wasmtime_wasi_http::types::OutgoingRequestConfig, ) -> wasmtime_wasi_http::HttpResult { if let Some(interceptor) = &self.state.request_interceptor { - let intercepted = Intercepted { - request: &mut request, - config: &mut config, - }; - if let Some(res) = interceptor.intercept(intercepted) { - return res; + match interceptor.intercept(&mut request, &mut config) { + InterceptOutcome::Continue => (), + InterceptOutcome::Complete(res) => return res, } } @@ -155,9 +152,9 @@ fn handle_not_allowed( is_relative: bool, ) -> anyhow::Result> { tracing::error!("Destination not allowed: {uri}"); - let host = if is_relative { + let allowed_host_example = if is_relative { terminal::warn!("A component tried to make a HTTP request to the same component but it does not have permission."); - "self".to_string() + "http://self".to_string() } else { let host = format!( "{scheme}://{authority}", @@ -167,7 +164,7 @@ fn handle_not_allowed( terminal::warn!("A component tried to make a HTTP request to non-allowed host '{host}'."); host }; - eprintln!("To allow requests, add 'allowed_outbound_hosts = [\"{host}\"]' to the manifest component section."); + eprintln!("To allow requests, add 'allowed_outbound_hosts = [\"{allowed_host_example}\"]' to the manifest component section."); Err(ErrorCode::HttpRequestDenied.into()) } diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index 7633aa4ba..5f03d2ca2 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -1,8 +1,8 @@ use std::time::Duration; use anyhow::bail; -use http::Request; -use spin_factor_outbound_http::OutboundHttpFactor; +use http::{Request, Uri}; +use spin_factor_outbound_http::{OutboundHttpFactor, SelfRequestOrigin}; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; @@ -20,18 +20,46 @@ struct TestFactors { } #[tokio::test] -async fn disallowed_host_fails() -> anyhow::Result<()> { - let factors = TestFactors { - variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, - http: OutboundHttpFactor, +async fn allowed_host_is_allowed() -> anyhow::Result<()> { + let mut state = test_instance_state("https://*").await?; + let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); + + // [100::] is an IPv6 "black hole", which should always fail + let req = Request::get("https://[100::1]:443").body(Default::default())?; + let mut future_resp = wasi_http.send_request(req, test_request_config())?; + future_resp.ready().await; + + // We don't want to make an actual network request, so treat "connection refused" as success + match future_resp.unwrap_ready().unwrap() { + Ok(_) => bail!("expected Err, got Ok"), + Err(err) => assert!(matches!(err, ErrorCode::ConnectionRefused), "{err:?}"), }; - let env = TestEnvironment::new(factors).extend_manifest(toml! { - [component.test-component] - source = "does-not-exist.wasm" - allowed_outbound_hosts = ["http://allowed.test"] - }); - let mut state = env.build_instance_state().await?; + Ok(()) +} + +#[tokio::test] +async fn self_request_smoke_test() -> anyhow::Result<()> { + let mut state = test_instance_state("http://self").await?; + let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); + + let mut req = Request::get("/self-request").body(Default::default())?; + let origin = Uri::from_static("http://[100::1]"); + req.extensions_mut() + .insert(SelfRequestOrigin::from_uri(&origin).unwrap()); + let mut future_resp = wasi_http.send_request(req, test_request_config())?; + future_resp.ready().await; + + // We don't want to make an actual network request, so treat "connection refused" as success + match future_resp.unwrap_ready().unwrap() { + Ok(_) => bail!("expected Err, got Ok"), + Err(err) => assert!(matches!(err, ErrorCode::ConnectionRefused), "{err:?}"), + }; + Ok(()) +} + +#[tokio::test] +async fn disallowed_host_fails() -> anyhow::Result<()> { + let mut state = test_instance_state("https://allowed.test").await?; let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); let req = Request::get("https://denied.test").body(Default::default())?; @@ -44,6 +72,22 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { Ok(()) } +async fn test_instance_state( + allowed_outbound_hosts: &str, +) -> anyhow::Result { + let factors = TestFactors { + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + http: OutboundHttpFactor, + }; + let env = TestEnvironment::new(factors).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + allowed_outbound_hosts = [allowed_outbound_hosts] + }); + env.build_instance_state().await +} + fn test_request_config() -> OutgoingRequestConfig { OutgoingRequestConfig { use_tls: false, diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index db71453f9..a5b66b5f3 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -15,6 +15,8 @@ spin-factor-key-value = { path = "../factor-key-value" } spin-factor-key-value-spin = { path = "../factor-key-value-spin" } spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-key-value-azure = { path = "../factor-key-value-azure" } +spin-factor-outbound-http = { path = "../factor-outbound-http" } +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-wasi = { path = "../factor-wasi" } toml = "0.8" diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 3e61a048b..eeeb02600 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -3,6 +3,9 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; use spin_factor_key_value::runtime_config::spin::{self as key_value, MakeKeyValueStore}; use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; +use spin_factor_outbound_http::OutboundHttpFactor; +use spin_factor_outbound_networking::runtime_config::spin::SpinTlsRuntimeConfig; +use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_wasi::WasiFactor; use spin_factors::{ runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, @@ -29,9 +32,9 @@ where { /// Creates a new resolved runtime configuration from a runtime config source TOML file. pub fn from_file(runtime_config_path: &Path, state_dir: Option<&str>) -> anyhow::Result { - let key_value_resolver = key_value_resolver(PathBuf::from( - state_dir.unwrap_or_else(|| DEFAULT_STATE_DIR.into()), - )); + let key_value_resolver = + key_value_resolver(PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR))); + let tls_resolver = SpinTlsRuntimeConfig::new(runtime_config_path); let file = std::fs::read_to_string(runtime_config_path).with_context(|| { format!( @@ -45,9 +48,10 @@ where runtime_config_path.display() ) })?; - let runtime_config: T = TomlRuntimeConfigSource::new(&toml, &key_value_resolver) - .try_into() - .map_err(Into::into)?; + let runtime_config: T = + TomlRuntimeConfigSource::new(&toml, &key_value_resolver, &tls_resolver) + .try_into() + .map_err(Into::into)?; Ok(Self { runtime_config, @@ -81,28 +85,49 @@ where pub struct TomlRuntimeConfigSource<'a> { table: TomlKeyTracker<'a>, key_value: &'a key_value::RuntimeConfigResolver, + tls: &'a SpinTlsRuntimeConfig, } impl<'a> TomlRuntimeConfigSource<'a> { - pub fn new(table: &'a toml::Table, key_value: &'a key_value::RuntimeConfigResolver) -> Self { + pub fn new( + table: &'a toml::Table, + key_value: &'a key_value::RuntimeConfigResolver, + tls: &'a SpinTlsRuntimeConfig, + ) -> Self { Self { table: TomlKeyTracker::new(table), key_value, + tls, } } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result> { + self.key_value.resolve_from_toml(Some(self.table.as_ref())) + } +} + +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config( + &mut self, + ) -> anyhow::Result::RuntimeConfig>> + { + self.tls.config_from_table(self.table.as_ref()) + } +} + impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result> { - self.key_value.resolve_from_toml(Some(self.table.as_ref())) +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(None) } } diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index 81bbc43db..856468916 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -2,12 +2,12 @@ mod headers; mod instrument; +mod outbound_http; mod server; mod spin; mod tls; mod wagi; mod wasi; -mod outbound_http; use std::{ collections::HashMap, diff --git a/crates/trigger-http2/src/outbound_http.rs b/crates/trigger-http2/src/outbound_http.rs index 5baaf9792..ac99a1e7f 100644 --- a/crates/trigger-http2/src/outbound_http.rs +++ b/crates/trigger-http2/src/outbound_http.rs @@ -3,7 +3,9 @@ use std::{ sync::Arc, }; -use spin_factor_outbound_http::{HostFutureIncomingResponse, SelfRequestOrigin}; +use spin_factor_outbound_http::{ + HostFutureIncomingResponse, InterceptOutcome, OutgoingRequestConfig, Request, SelfRequestOrigin, +}; use spin_http::routes::RouteMatch; use spin_outbound_networking::parse_service_chaining_target; use wasmtime_wasi_http::types::IncomingResponse; @@ -26,17 +28,17 @@ const CHAINED_CLIENT_ADDR: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new impl spin_factor_outbound_http::OutboundHttpInterceptor for OutboundHttpInterceptor { fn intercept( &self, - intercepted: spin_factor_outbound_http::Intercepted, - ) -> Option> - { - let uri = intercepted.request.uri(); + request: &mut Request, + config: &mut OutgoingRequestConfig, + ) -> InterceptOutcome { + let uri = request.uri(); // Handle service chaining requests if let Some(component_id) = parse_service_chaining_target(uri) { // TODO: look at the rest of chain_request let route_match = RouteMatch::synthetic(&component_id, uri.path()); - let req = std::mem::take(intercepted.request); - let between_bytes_timeout = intercepted.config.between_bytes_timeout; + let req = std::mem::take(request); + let between_bytes_timeout = config.between_bytes_timeout; let server = self.server.clone(); let resp_fut = async move { match server @@ -52,13 +54,10 @@ impl spin_factor_outbound_http::OutboundHttpInterceptor for OutboundHttpIntercep } }; let resp = HostFutureIncomingResponse::pending(wasmtime_wasi::runtime::spawn(resp_fut)); - Some(Ok(resp)) + InterceptOutcome::Complete(Ok(resp)) } else { - intercepted - .request - .extensions_mut() - .insert(self.origin.clone()); - None + request.extensions_mut().insert(self.origin.clone()); + InterceptOutcome::Continue } } } From 5bf5cea18774f4bb4a1cd58caf9ebd60ac324e0f Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 19 Aug 2024 11:09:27 +0200 Subject: [PATCH 125/195] Change how incoming-request.authority is set. Always rely on the `Host` header for authority incoming-reques.authority. To determine where `self` requests should be routed, explicitly pass a the listening address. Signed-off-by: Ryan Levick --- crates/trigger-http2/src/lib.rs | 13 ++++-- crates/trigger-http2/src/outbound_http.rs | 3 +- crates/trigger-http2/src/server.rs | 53 +++++++++++++++++------ 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index 856468916..d36b70d6b 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -67,7 +67,8 @@ pub(crate) type InstanceState = (); /// The Spin HTTP trigger. pub struct HttpTrigger { - listen_addr: SocketAddr, + /// The address the server will listen on. + addr_to_bind: SocketAddr, tls_config: Option, router: Router, // Component ID -> component trigger config @@ -112,7 +113,7 @@ impl Trigger for HttpTrigger { ); Ok(Self { - listen_addr: cli_args.address, + addr_to_bind: cli_args.address, tls_config: cli_args.into_tls_config(), router, component_trigger_configs, @@ -121,7 +122,7 @@ impl Trigger for HttpTrigger { async fn run(self, trigger_app: TriggerApp) -> anyhow::Result<()> { let Self { - listen_addr, + addr_to_bind: listen_addr, tls_config, router, component_trigger_configs, @@ -131,6 +132,12 @@ impl Trigger for HttpTrigger { .await .with_context(|| format!("Unable to listen on {listen_addr}"))?; + // Get the address the server is actually listening on + // We can't use `self.listen_addr` because it might not + // be fully resolved (e.g, port 0). + let listen_addr = listener + .local_addr() + .context("failed to retrieve address server is listening on")?; let server = Arc::new(HttpServer::new( listen_addr, trigger_app, diff --git a/crates/trigger-http2/src/outbound_http.rs b/crates/trigger-http2/src/outbound_http.rs index ac99a1e7f..623eb6b69 100644 --- a/crates/trigger-http2/src/outbound_http.rs +++ b/crates/trigger-http2/src/outbound_http.rs @@ -3,6 +3,7 @@ use std::{ sync::Arc, }; +use http::uri::Scheme; use spin_factor_outbound_http::{ HostFutureIncomingResponse, InterceptOutcome, OutgoingRequestConfig, Request, SelfRequestOrigin, }; @@ -42,7 +43,7 @@ impl spin_factor_outbound_http::OutboundHttpInterceptor for OutboundHttpIntercep let server = self.server.clone(); let resp_fut = async move { match server - .handle_trigger_route(req, route_match, CHAINED_CLIENT_ADDR) + .handle_trigger_route(req, route_match, Scheme::HTTP, CHAINED_CLIENT_ADDR) .await { Ok(resp) => Ok(Ok(IncomingResponse { diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index 8e346369b..1474c9ddd 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -37,6 +37,7 @@ use crate::{ }; pub struct HttpServer { + /// The address the server is listening on. listen_addr: SocketAddr, trigger_app: TriggerApp, router: Router, @@ -74,7 +75,8 @@ impl HttpServer { self.print_startup_msgs("http", &listener)?; loop { let (stream, client_addr) = listener.accept().await?; - self.clone().serve_connection(stream, client_addr); + self.clone() + .serve_connection(stream, Scheme::HTTP, client_addr); } } @@ -88,7 +90,9 @@ impl HttpServer { loop { let (stream, client_addr) = listener.accept().await?; match acceptor.accept(stream).await { - Ok(stream) => self.clone().serve_connection(stream, client_addr), + Ok(stream) => self + .clone() + .serve_connection(stream, Scheme::HTTPS, client_addr), Err(err) => tracing::error!(?err, "Failed to start TLS session"), } } @@ -98,10 +102,10 @@ impl HttpServer { async fn handle( self: &Arc, mut req: Request, - scheme: Scheme, + server_scheme: Scheme, client_addr: SocketAddr, ) -> anyhow::Result> { - set_req_uri(&mut req, scheme, self.listen_addr)?; + set_req_uri(&mut req, server_scheme.clone())?; strip_forbidden_headers(&mut req); spin_telemetry::extract_trace_context(&req); @@ -124,7 +128,7 @@ impl HttpServer { match self.router.route(&path) { Ok(route_match) => { - self.handle_trigger_route(req, route_match, client_addr) + self.handle_trigger_route(req, route_match, server_scheme, client_addr) .await } Err(_) => Self::not_found(NotFoundRouteKind::Normal(path.to_string())), @@ -135,6 +139,7 @@ impl HttpServer { self: &Arc, req: Request, route_match: RouteMatch, + server_scheme: Scheme, client_addr: SocketAddr, ) -> anyhow::Result> { let app_id = self @@ -155,9 +160,15 @@ impl HttpServer { let mut instance_builder = self.trigger_app.prepare(component_id)?; // Set up outbound HTTP request origin and service chaining - let uri = req.uri(); - let origin = SelfRequestOrigin::from_uri(uri) - .with_context(|| format!("invalid request URI {uri:?}"))?; + let origin = SelfRequestOrigin { + scheme: server_scheme, + authority: self.listen_addr.to_string().parse().with_context(|| { + format!( + "server address '{}' is not a valid authority", + self.listen_addr + ) + })?, + }; instance_builder .factor_builders() .outbound_http() @@ -259,6 +270,7 @@ impl HttpServer { fn serve_connection( self: Arc, stream: S, + server_scheme: Scheme, client_addr: SocketAddr, ) { task::spawn(async move { @@ -267,7 +279,11 @@ impl HttpServer { .serve_connection( TokioIo::new(stream), service_fn(move |request| { - self.clone().instrumented_service_fn(client_addr, request) + self.clone().instrumented_service_fn( + server_scheme.clone(), + client_addr, + request, + ) }), ) .await @@ -279,6 +295,7 @@ impl HttpServer { async fn instrumented_service_fn( self: Arc, + server_scheme: Scheme, client_addr: SocketAddr, request: Request, ) -> anyhow::Result> { @@ -291,7 +308,7 @@ impl HttpServer { body.map_err(wasmtime_wasi_http::hyper_response_error) .boxed() }), - Scheme::HTTP, + server_scheme, client_addr, ) .await; @@ -322,11 +339,21 @@ impl HttpServer { /// The incoming request's scheme and authority /// -/// The incoming request's URI is relative to the server, so we need to set the scheme and authority -fn set_req_uri(req: &mut Request, scheme: Scheme, addr: SocketAddr) -> anyhow::Result<()> { +/// The incoming request's URI is relative to the server, so we need to set the scheme and authority. +/// The `Host` header is used to set the authority. This function will error if no `Host` header is +/// present or if it is not parsable as an `Authority`. +fn set_req_uri(req: &mut Request, scheme: Scheme) -> anyhow::Result<()> { let uri = req.uri().clone(); let mut parts = uri.into_parts(); - let authority = format!("{}:{}", addr.ip(), addr.port()).parse().unwrap(); + let headers = req.headers(); + let host_header = headers + .get(http::header::HOST) + .context("missing 'Host' header")? + .to_str() + .context("'Host' header is not valid UTF-8")?; + let authority = host_header + .parse() + .context("'Host' header contains an invalid authority")?; parts.scheme = Some(scheme); parts.authority = Some(authority); *req.uri_mut() = Uri::from_parts(parts).unwrap(); From af04bc2a2a0685f5d34f7afbd22bc190e0e2c084 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 19 Aug 2024 11:20:04 +0200 Subject: [PATCH 126/195] Move uri munging inside handle_trigger_route Signed-off-by: Ryan Levick --- crates/trigger-http2/src/server.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index 1474c9ddd..72ffe9e0f 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -99,21 +99,23 @@ impl HttpServer { } /// Handles incoming requests using an HTTP executor. + /// + /// This method handles well known paths and routes requests to the handler when the router + /// matches the requests path. async fn handle( self: &Arc, mut req: Request, server_scheme: Scheme, client_addr: SocketAddr, ) -> anyhow::Result> { - set_req_uri(&mut req, server_scheme.clone())?; strip_forbidden_headers(&mut req); spin_telemetry::extract_trace_context(&req); - tracing::info!("Processing request on URI {}", req.uri()); - let path = req.uri().path().to_string(); + tracing::info!("Processing request on path '{path}'"); + // Handle well-known spin paths if let Some(well_known) = path.strip_prefix(spin_http::WELL_KNOWN_PREFIX) { return match well_known { @@ -135,13 +137,15 @@ impl HttpServer { } } + /// Handles a successful route match. pub async fn handle_trigger_route( self: &Arc, - req: Request, + mut req: Request, route_match: RouteMatch, server_scheme: Scheme, client_addr: SocketAddr, ) -> anyhow::Result> { + set_req_uri(&mut req, server_scheme.clone())?; let app_id = self .trigger_app .app() From 12cc792f96351a4dc259046f5c35364ff204fc08 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 16 Aug 2024 18:31:46 +0200 Subject: [PATCH 127/195] Integrate SQLite factor into trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 38 +++++----- crates/factor-sqlite/Cargo.toml | 2 +- crates/factor-sqlite/src/lib.rs | 4 +- .../factor-sqlite/src/runtime_config/spin.rs | 14 ++-- crates/factor-sqlite/tests/factor_test.rs | 21 +++--- crates/runtime-config/Cargo.toml | 1 + crates/runtime-config/src/lib.rs | 70 ++++++++++++++++--- crates/trigger2/Cargo.toml | 1 + crates/trigger2/src/cli.rs | 3 +- crates/trigger2/src/factors.rs | 8 ++- 10 files changed, 112 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b5828045..e773786f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2498,24 +2498,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "factor-sqlite" -version = "2.7.0-pre0" -dependencies = [ - "async-trait", - "serde 1.0.197", - "spin-factors", - "spin-factors-test", - "spin-locked-app", - "spin-sqlite", - "spin-sqlite-inproc", - "spin-sqlite-libsql", - "spin-world", - "table", - "tokio", - "toml 0.8.14", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -7787,6 +7769,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "spin-factor-sqlite" +version = "2.7.0-pre0" +dependencies = [ + "async-trait", + "serde 1.0.197", + "spin-factors", + "spin-factors-test", + "spin-locked-app", + "spin-sqlite", + "spin-sqlite-inproc", + "spin-sqlite-libsql", + "spin-world", + "table", + "tokio", + "toml 0.8.14", +] + [[package]] name = "spin-factor-variables" version = "2.7.0-pre0" @@ -8164,6 +8164,7 @@ dependencies = [ "spin-factor-key-value-spin", "spin-factor-outbound-http", "spin-factor-outbound-networking", + "spin-factor-sqlite", "spin-factor-wasi", "spin-factors", "toml 0.8.14", @@ -8471,6 +8472,7 @@ dependencies = [ "spin-factor-key-value", "spin-factor-outbound-http", "spin-factor-outbound-networking", + "spin-factor-sqlite", "spin-factor-wasi", "spin-factors", "spin-factors-executor", diff --git a/crates/factor-sqlite/Cargo.toml b/crates/factor-sqlite/Cargo.toml index 24442bd33..3b45dcf3c 100644 --- a/crates/factor-sqlite/Cargo.toml +++ b/crates/factor-sqlite/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "factor-sqlite" +name = "spin-factor-sqlite" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/crates/factor-sqlite/src/lib.rs b/crates/factor-sqlite/src/lib.rs index e9a80e646..4d90a1c64 100644 --- a/crates/factor-sqlite/src/lib.rs +++ b/crates/factor-sqlite/src/lib.rs @@ -12,6 +12,8 @@ use spin_locked_app::MetadataKey; use spin_world::v1::sqlite as v1; use spin_world::v2::sqlite as v2; +pub use runtime_config::RuntimeConfig; + pub struct SqliteFactor { default_label_resolver: Arc, } @@ -29,7 +31,7 @@ impl SqliteFactor { } impl Factor for SqliteFactor { - type RuntimeConfig = runtime_config::RuntimeConfig; + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index bf14cf04b..3bc2452f0 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -15,16 +15,16 @@ use tokio::sync::OnceCell; use crate::{Connection, ConnectionCreator, DefaultLabelResolver}; -/// Spin's default handling of the runtime configuration for SQLite databases. +/// Spin's default resolution of runtime configuration for SQLite databases. /// -/// This type implements the [`RuntimeConfigResolver`] trait and provides a way to -/// opt into the default behavior of Spin's SQLite database handling. -pub struct SpinSqliteRuntimeConfig { +/// This type implements how Spin CLI's SQLite implementation is configured +/// through the runtime config toml as well as the behavior of the "default" label. +pub struct RuntimeConfigResolver { default_database_dir: PathBuf, local_database_dir: PathBuf, } -impl SpinSqliteRuntimeConfig { +impl RuntimeConfigResolver { /// Create a new `SpinSqliteRuntimeConfig` /// /// This takes as arguments: @@ -59,7 +59,7 @@ impl SpinSqliteRuntimeConfig { /// type = "$database-type" /// ... extra type specific configuration ... /// ``` - pub fn config_from_table( + pub fn resolve_from_toml( &self, table: &T, ) -> anyhow::Result> { @@ -106,7 +106,7 @@ pub struct RuntimeConfig { pub config: toml::Table, } -impl DefaultLabelResolver for SpinSqliteRuntimeConfig { +impl DefaultLabelResolver for RuntimeConfigResolver { fn default(&self, label: &str) -> Option> { // Only default the database labeled "default". if label != "default" { diff --git a/crates/factor-sqlite/tests/factor_test.rs b/crates/factor-sqlite/tests/factor_test.rs index 8fb73c3c4..5b9972c9e 100644 --- a/crates/factor-sqlite/tests/factor_test.rs +++ b/crates/factor-sqlite/tests/factor_test.rs @@ -1,6 +1,6 @@ use std::{collections::HashSet, sync::Arc}; -use factor_sqlite::{runtime_config::spin::SpinSqliteRuntimeConfig, SqliteFactor}; +use spin_factor_sqlite::{runtime_config::spin::RuntimeConfigResolver, SqliteFactor}; use spin_factors::{ anyhow::{self, bail, Context}, runtime_config::toml::TomlKeyTracker, @@ -66,7 +66,7 @@ async fn no_error_when_database_is_configured() -> anyhow::Result<()> { [sqlite_database.foo] type = "spin" }; - let sqlite_config = SpinSqliteRuntimeConfig::new("/".into(), "/".into()); + let sqlite_config = RuntimeConfigResolver::new("/".into(), "/".into()); let env = TestEnvironment::new(factors) .extend_manifest(toml! { [component.test-component] @@ -82,14 +82,14 @@ async fn no_error_when_database_is_configured() -> anyhow::Result<()> { struct TomlRuntimeSource<'a> { table: TomlKeyTracker<'a>, - sqlite_config: SpinSqliteRuntimeConfig, + runtime_config_resolver: RuntimeConfigResolver, } impl<'a> TomlRuntimeSource<'a> { - fn new(table: &'a toml::Table, sqlite_config: SpinSqliteRuntimeConfig) -> Self { + fn new(table: &'a toml::Table, runtime_config_resolver: RuntimeConfigResolver) -> Self { Self { table: TomlKeyTracker::new(table), - sqlite_config, + runtime_config_resolver, } } } @@ -98,7 +98,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeSource<'_> { fn get_runtime_config( &mut self, ) -> anyhow::Result::RuntimeConfig>> { - self.sqlite_config.config_from_table(&self.table) + self.runtime_config_resolver.resolve_from_toml(&self.table) } } @@ -129,8 +129,8 @@ impl DefaultLabelResolver { } } -impl factor_sqlite::DefaultLabelResolver for DefaultLabelResolver { - fn default(&self, label: &str) -> Option> { +impl spin_factor_sqlite::DefaultLabelResolver for DefaultLabelResolver { + fn default(&self, label: &str) -> Option> { let Some(default) = &self.default else { return None; }; @@ -142,10 +142,11 @@ impl factor_sqlite::DefaultLabelResolver for DefaultLabelResolver { struct InvalidConnectionCreator; #[async_trait::async_trait] -impl factor_sqlite::ConnectionCreator for InvalidConnectionCreator { +impl spin_factor_sqlite::ConnectionCreator for InvalidConnectionCreator { async fn create_connection( &self, - ) -> Result, spin_world::v2::sqlite::Error> { + ) -> Result, spin_world::v2::sqlite::Error> + { Err(spin_world::v2::sqlite::Error::InvalidConnection) } } diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index a5b66b5f3..7a52c5128 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -17,6 +17,7 @@ spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-key-value-azure = { path = "../factor-key-value-azure" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-sqlite = { path = "../factor-sqlite" } spin-factor-wasi = { path = "../factor-wasi" } toml = "0.8" diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index eeeb02600..0ab315d4b 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -6,6 +6,8 @@ use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::runtime_config::spin::SpinTlsRuntimeConfig; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_sqlite::runtime_config::spin as sqlite; +use spin_factor_sqlite::SqliteFactor; use spin_factor_wasi::WasiFactor; use spin_factors::{ runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, @@ -17,12 +19,13 @@ pub const DEFAULT_STATE_DIR: &str = ".spin"; /// A runtime configuration which has been resolved from a runtime config source. /// /// Includes other pieces of configuration that are used to resolve the runtime configuration. -#[derive(Default)] pub struct ResolvedRuntimeConfig { /// The resolved runtime configuration. pub runtime_config: T, /// The resolver used to resolve key-value stores from runtime configuration. pub key_value_resolver: key_value::RuntimeConfigResolver, + /// The resolver used to resolve sqlite databases from runtime configuration. + pub sqlite_resolver: sqlite::RuntimeConfigResolver, } impl ResolvedRuntimeConfig @@ -32,9 +35,12 @@ where { /// Creates a new resolved runtime configuration from a runtime config source TOML file. pub fn from_file(runtime_config_path: &Path, state_dir: Option<&str>) -> anyhow::Result { - let key_value_resolver = - key_value_resolver(PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR))); let tls_resolver = SpinTlsRuntimeConfig::new(runtime_config_path); + let key_value_config_resolver = + key_value_config_resolver(PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR))); + + let sqlite_config_resolver = + sqlite_config_resolver(state_dir).context("failed to resolve sqlite runtime config")?; let file = std::fs::read_to_string(runtime_config_path).with_context(|| { format!( @@ -48,14 +54,19 @@ where runtime_config_path.display() ) })?; - let runtime_config: T = - TomlRuntimeConfigSource::new(&toml, &key_value_resolver, &tls_resolver) - .try_into() - .map_err(Into::into)?; + let runtime_config: T = TomlRuntimeConfigSource::new( + &toml, + &key_value_config_resolver, + &tls_resolver, + &sqlite_config_resolver, + ) + .try_into() + .map_err(Into::into)?; Ok(Self { runtime_config, - key_value_resolver, + key_value_resolver: key_value_config_resolver, + sqlite_resolver: sqlite_config_resolver, }) } @@ -81,11 +92,23 @@ where } } +impl ResolvedRuntimeConfig { + pub fn default(state_dir: Option<&str>) -> Self { + Self { + sqlite_resolver: sqlite_config_resolver(state_dir) + .expect("failed to resolve sqlite runtime config"), + key_value_resolver: Default::default(), + runtime_config: Default::default(), + } + } +} + /// The TOML based runtime configuration source Spin CLI. pub struct TomlRuntimeConfigSource<'a> { table: TomlKeyTracker<'a>, key_value: &'a key_value::RuntimeConfigResolver, tls: &'a SpinTlsRuntimeConfig, + sqlite: &'a sqlite::RuntimeConfigResolver, } impl<'a> TomlRuntimeConfigSource<'a> { @@ -93,11 +116,13 @@ impl<'a> TomlRuntimeConfigSource<'a> { table: &'a toml::Table, key_value: &'a key_value::RuntimeConfigResolver, tls: &'a SpinTlsRuntimeConfig, + sqlite: &'a sqlite::RuntimeConfigResolver, ) -> Self { Self { table: TomlKeyTracker::new(table), key_value, tls, + sqlite, } } } @@ -131,6 +156,12 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<' } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + self.sqlite.resolve_from_toml(self.table.as_ref()) + } +} + impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_> { fn finalize(&mut self) -> anyhow::Result<()> { Ok(self.table.validate_all_keys_used()?) @@ -140,10 +171,12 @@ impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_> { const DEFAULT_KEY_VALUE_STORE_FILENAME: &str = "sqlite_key_value.db"; const DEFAULT_KEY_VALUE_STORE_LABEL: &str = "default"; -/// The key-value runtime configuration resolver used by the trigger. +/// The key-value runtime configuration resolver. /// /// Takes a base path for the local store. -pub fn key_value_resolver(local_store_base_path: PathBuf) -> key_value::RuntimeConfigResolver { +pub fn key_value_config_resolver( + local_store_base_path: PathBuf, +) -> key_value::RuntimeConfigResolver { let mut key_value = key_value::RuntimeConfigResolver::new(); // Register the supported store types. @@ -173,3 +206,20 @@ pub fn key_value_resolver(local_store_base_path: PathBuf) -> key_value::RuntimeC key_value } + +/// The sqlite runtime configuration resolver. +/// +/// Takes a base path to the state directory. +fn sqlite_config_resolver( + state_dir: Option<&str>, +) -> anyhow::Result { + let default_database_dir = PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR)); + let default_database_dir = std::path::absolute(default_database_dir) + .context("failed to make default database directory absolute")?; + let local_database_dir = + std::env::current_dir().context("failed to get current working directory")?; + Ok(sqlite::RuntimeConfigResolver::new( + default_database_dir, + local_database_dir, + )) +} diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 1342cc02a..656f51035 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -24,6 +24,7 @@ spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-wasi = { path = "../factor-wasi" } spin-factor-key-value = { path = "../factor-key-value" } +spin-factor-sqlite = { path = "../factor-sqlite" } spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index 0c72480b3..321f1d56d 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -199,7 +199,7 @@ impl FactorsTriggerCommand { self.state_dir.as_deref(), )? } - None => ResolvedRuntimeConfig::default(), + None => ResolvedRuntimeConfig::default(self.state_dir.as_deref()), }; runtime_config @@ -210,6 +210,7 @@ impl FactorsTriggerCommand { working_dir, self.allow_transient_write, runtime_config.key_value_resolver, + runtime_config.sqlite_resolver, ); // TODO: move these into Factor methods/constructors diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index aa1a0f315..973471894 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; -use spin_factor_key_value::{DefaultLabelResolver, KeyValueFactor}; +use spin_factor_key_value::KeyValueFactor; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_sqlite::SqliteFactor; use spin_factor_wasi::{spin::SpinFilesMounter, WasiFactor}; use spin_factors::RuntimeFactors; use spin_runtime_config::TomlRuntimeConfigSource; @@ -13,13 +14,15 @@ pub struct TriggerFactors { pub key_value: KeyValueFactor, pub outbound_networking: OutboundNetworkingFactor, pub outbound_http: OutboundHttpFactor, + pub sqlite: SqliteFactor, } impl TriggerFactors { pub fn new( working_dir: impl Into, allow_transient_writes: bool, - default_key_value_label_resolver: impl DefaultLabelResolver + 'static, + default_key_value_label_resolver: impl spin_factor_key_value::DefaultLabelResolver + 'static, + default_sqlite_label_resolver: impl spin_factor_sqlite::DefaultLabelResolver + 'static, ) -> Self { let files_mounter = SpinFilesMounter::new(working_dir, allow_transient_writes); Self { @@ -27,6 +30,7 @@ impl TriggerFactors { key_value: KeyValueFactor::new(default_key_value_label_resolver), outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, + sqlite: SqliteFactor::new(default_sqlite_label_resolver), } } } From 3cafa2a82621c9967deac6861231675c06f6b5a3 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 19 Aug 2024 13:39:08 +0200 Subject: [PATCH 128/195] Don't require absolute paths in sqlite runtime config resolver Signed-off-by: Ryan Levick --- crates/factor-sqlite/src/runtime_config/spin.rs | 13 +------------ crates/runtime-config/src/lib.rs | 2 -- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 3bc2452f0..a98387df1 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -30,21 +30,10 @@ impl RuntimeConfigResolver { /// This takes as arguments: /// * the directory to use as the default location for SQLite databases. /// Usually this will be the path to the `.spin` state directory. - /// * the *absolute* path to the directory from which relative paths to + /// * the path to the directory from which relative paths to /// local SQLite databases are resolved. (this should most likely be the /// path to the runtime-config file or the current working dir). - /// - /// Panics if either `default_database_dir` or `local_database_dir` are not - /// absolute paths. pub fn new(default_database_dir: PathBuf, local_database_dir: PathBuf) -> Self { - assert!( - default_database_dir.is_absolute(), - "default_database_dir must be an absolute path" - ); - assert!( - local_database_dir.is_absolute(), - "local_database_dir must be an absolute path" - ); Self { default_database_dir, local_database_dir, diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 0ab315d4b..12f40c750 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -214,8 +214,6 @@ fn sqlite_config_resolver( state_dir: Option<&str>, ) -> anyhow::Result { let default_database_dir = PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR)); - let default_database_dir = std::path::absolute(default_database_dir) - .context("failed to make default database directory absolute")?; let local_database_dir = std::env::current_dir().context("failed to get current working directory")?; Ok(sqlite::RuntimeConfigResolver::new( From 5a036c3f9b995b95a37a5c06d7a72decfcfef51b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 19 Aug 2024 15:09:57 +0200 Subject: [PATCH 129/195] Add Azure key-vault variable provider to factors Signed-off-by: Ryan Levick --- Cargo.lock | 30 +++- crates/factor-variables/Cargo.toml | 3 + .../src/spin_cli/azure_key_vault.rs | 157 ++++++++++++++++++ crates/factor-variables/src/spin_cli/mod.rs | 24 ++- 4 files changed, 199 insertions(+), 15 deletions(-) create mode 100644 crates/factor-variables/src/spin_cli/azure_key_vault.rs diff --git a/Cargo.lock b/Cargo.lock index e773786f2..eb6ee5342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -619,7 +619,7 @@ dependencies = [ [[package]] name = "azure_core" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "base64 0.22.0", @@ -647,10 +647,10 @@ dependencies = [ [[package]] name = "azure_data_cosmos" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", - "azure_core 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", + "azure_core 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", "bytes", "futures", "serde 1.0.197", @@ -686,12 +686,12 @@ dependencies = [ [[package]] name = "azure_identity" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-lock 3.3.0", "async-process 2.2.2", "async-trait", - "azure_core 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", + "azure_core 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", "futures", "oauth2", "pin-project", @@ -717,6 +717,19 @@ dependencies = [ "time", ] +[[package]] +name = "azure_security_keyvault" +version = "0.20.0" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +dependencies = [ + "async-trait", + "azure_core 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", + "futures", + "serde 1.0.197", + "serde_json", + "time", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -7791,6 +7804,9 @@ dependencies = [ name = "spin-factor-variables" version = "2.7.0-pre0" dependencies = [ + "azure_core 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", + "azure_identity 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", + "azure_security_keyvault 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", "dotenvy", "serde 1.0.197", "spin-expressions", @@ -7921,7 +7937,7 @@ version = "2.7.0-pre0" dependencies = [ "anyhow", "azure_data_cosmos", - "azure_identity 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", + "azure_identity 0.20.0 (git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c)", "futures", "serde 1.0.197", "spin-core", @@ -8490,7 +8506,7 @@ dependencies = [ "async-trait", "azure_core 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "azure_identity 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", - "azure_security_keyvault", + "azure_security_keyvault 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "dotenvy", "once_cell", "serde 1.0.197", diff --git a/crates/factor-variables/Cargo.toml b/crates/factor-variables/Cargo.toml index 85f0687b1..60e0f507b 100644 --- a/crates/factor-variables/Cargo.toml +++ b/crates/factor-variables/Cargo.toml @@ -5,6 +5,9 @@ authors = { workspace = true } edition = { workspace = true } [dependencies] +azure_security_keyvault = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } +azure_core = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } +azure_identity = { git = "https://github.com/azure/azure-sdk-for-rust", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } dotenvy = "0.15" serde = { version = "1.0", features = ["rc"] } spin-expressions = { path = "../expressions" } diff --git a/crates/factor-variables/src/spin_cli/azure_key_vault.rs b/crates/factor-variables/src/spin_cli/azure_key_vault.rs new file mode 100644 index 000000000..a832fa536 --- /dev/null +++ b/crates/factor-variables/src/spin_cli/azure_key_vault.rs @@ -0,0 +1,157 @@ +use std::sync::Arc; + +use anyhow::Context as _; +use azure_core::{auth::TokenCredential, Url}; +use azure_security_keyvault::SecretClient; +use serde::Deserialize; +use spin_expressions::{Key, Provider}; +use spin_factors::anyhow; +use spin_world::async_trait; +use tracing::{instrument, Level}; + +/// Azure KeyVault runtime config literal options for authentication +/// +/// Some of these fields are optional. Whether they are set determines whether environmental variables +/// will be used to resolve the information instead. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AzureKeyVaultVariablesConfig { + pub vault_url: String, + pub client_id: Option, + pub client_secret: Option, + pub tenant_id: Option, + pub authority_host: Option, +} + +#[derive(Debug, Copy, Clone, Deserialize, Default)] +pub enum AzureAuthorityHost { + #[default] + AzurePublicCloud, + AzureChina, + AzureGermany, + AzureGovernment, +} + +impl TryFrom for AzureKeyVaultAuthOptions { + type Error = anyhow::Error; + + fn try_from(value: AzureKeyVaultVariablesConfig) -> Result { + match (value.client_id, value.tenant_id, value.client_secret) { + (Some(client_id), Some(tenant_id), Some(client_secret)) => Ok( + AzureKeyVaultAuthOptions::RuntimeConfigValues{ + client_id, + client_secret, + tenant_id, + authority_host: value.authority_host.unwrap_or_default(), + } + ), + (None, None, None) => Ok(AzureKeyVaultAuthOptions::Environmental), + _ => anyhow::bail!("The current runtime config specifies some but not all of the Azure KeyVault 'client_id', 'client_secret', and 'tenant_id' values. Provide the missing values to authenticate to Azure KeyVault with the given service principal, or remove all these values to authenticate using ambient authentication (e.g. env vars, Azure CLI, Managed Identity, Workload Identity).") + } + } +} + +/// Azure Cosmos Key / Value enumeration for the possible authentication options +#[derive(Clone, Debug)] +pub enum AzureKeyVaultAuthOptions { + /// Runtime Config values indicates the service principal credentials have been supplied + RuntimeConfigValues { + client_id: String, + client_secret: String, + tenant_id: String, + authority_host: AzureAuthorityHost, + }, + /// Environmental indicates that the environment variables of the process should be used to + /// create the TokenCredential for the Cosmos client. This will use the Azure Rust SDK's + /// DefaultCredentialChain to derive the TokenCredential based on what environment variables + /// have been set. + /// + /// Service Principal with client secret: + /// - `AZURE_TENANT_ID`: ID of the service principal's Azure tenant. + /// - `AZURE_CLIENT_ID`: the service principal's client ID. + /// - `AZURE_CLIENT_SECRET`: one of the service principal's secrets. + /// + /// Service Principal with certificate: + /// - `AZURE_TENANT_ID`: ID of the service principal's Azure tenant. + /// - `AZURE_CLIENT_ID`: the service principal's client ID. + /// - `AZURE_CLIENT_CERTIFICATE_PATH`: path to a PEM or PKCS12 certificate file including the private key. + /// - `AZURE_CLIENT_CERTIFICATE_PASSWORD`: (optional) password for the certificate file. + /// + /// Workload Identity (Kubernetes, injected by the Workload Identity mutating webhook): + /// - `AZURE_TENANT_ID`: ID of the service principal's Azure tenant. + /// - `AZURE_CLIENT_ID`: the service principal's client ID. + /// - `AZURE_FEDERATED_TOKEN_FILE`: TokenFilePath is the path of a file containing a Kubernetes service account token. + /// + /// Managed Identity (User Assigned or System Assigned identities): + /// - `AZURE_CLIENT_ID`: (optional) if using a user assigned identity, this will be the client ID of the identity. + /// + /// Azure CLI: + /// - `AZURE_TENANT_ID`: (optional) use a specific tenant via the Azure CLI. + /// + /// Common across each: + /// - `AZURE_AUTHORITY_HOST`: (optional) the host for the identity provider. For example, for Azure public cloud the host defaults to "https://login.microsoftonline.com". + /// See also: https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/identity/README.md + Environmental, +} + +/// A provider that fetches variables from Azure Key Vault. +#[derive(Debug)] +pub struct AzureKeyVaultProvider { + secret_client: SecretClient, +} + +impl AzureKeyVaultProvider { + pub fn create( + vault_url: impl Into, + auth_options: AzureKeyVaultAuthOptions, + ) -> anyhow::Result { + let http_client = azure_core::new_http_client(); + let token_credential = match auth_options { + AzureKeyVaultAuthOptions::RuntimeConfigValues { + client_id, + client_secret, + tenant_id, + authority_host, + } => { + let credential = azure_identity::ClientSecretCredential::new( + http_client, + authority_host.into(), + tenant_id, + client_id, + client_secret, + ); + Arc::new(credential) as Arc + } + AzureKeyVaultAuthOptions::Environmental => azure_identity::create_default_credential()?, + }; + + Ok(Self { + secret_client: SecretClient::new(&vault_url.into(), token_credential)?, + }) + } +} + +#[async_trait] +impl Provider for AzureKeyVaultProvider { + #[instrument(name = "spin_variables.get_from_azure_key_vault", skip(self), err(level = Level::INFO), fields(otel.kind = "client"))] + async fn get(&self, key: &Key) -> anyhow::Result> { + let secret = self + .secret_client + .get(key.as_str()) + .await + .context("Failed to read variable from Azure Key Vault")?; + Ok(Some(secret.value)) + } +} + +impl From for Url { + fn from(value: AzureAuthorityHost) -> Self { + let url = match value { + AzureAuthorityHost::AzureChina => "https://login.chinacloudapi.cn/", + AzureAuthorityHost::AzureGovernment => "https://login.microsoftonline.us/", + AzureAuthorityHost::AzureGermany => "https://login.microsoftonline.de/", + AzureAuthorityHost::AzurePublicCloud => "https://login.microsoftonline.com/", + }; + Url::parse(url).unwrap() + } +} diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index f03a12472..f86c52d4a 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -1,9 +1,11 @@ //! The runtime configuration for the variables factor used in the Spin CLI. +mod azure_key_vault; mod env; mod statik; mod vault; +pub use azure_key_vault::*; pub use env::*; pub use statik::*; pub use vault::*; @@ -23,11 +25,11 @@ pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result = array.clone().try_into()?; - providers.extend( - provider_configs - .into_iter() - .map(VariableProviderConfiguration::into_provider), - ); + let new_providers = provider_configs + .into_iter() + .map(VariableProviderConfiguration::into_provider) + .collect::>>()?; + providers.extend(new_providers); Ok(RuntimeConfig { providers }) } @@ -35,6 +37,8 @@ pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result Box { - match self { + pub fn into_provider(self) -> anyhow::Result> { + let provider: Box = match self { VariableProviderConfiguration::Static(provider) => Box::new(provider), VariableProviderConfiguration::Env(config) => Box::new(env::EnvVariablesProvider::new( config.prefix, @@ -54,6 +58,10 @@ impl VariableProviderConfiguration { config.dotenv_path, )), VariableProviderConfiguration::Vault(provider) => Box::new(provider), - } + VariableProviderConfiguration::AzureKeyVault(config) => Box::new( + AzureKeyVaultProvider::create(config.vault_url.clone(), config.try_into()?)?, + ), + }; + Ok(provider) } } From d0f1873c639a724983353a1af70fd941989f93ba Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 19 Aug 2024 10:26:39 -0400 Subject: [PATCH 130/195] factors: Get `spin up` working Signed-off-by: Lann Martin --- Cargo.lock | 13 +++++-------- Cargo.toml | 19 ++++++++----------- crates/loader/Cargo.toml | 1 - crates/oci/Cargo.toml | 2 +- crates/runtime-config/Cargo.toml | 1 + crates/runtime-config/src/lib.rs | 11 ++++++++++- crates/trigger2/Cargo.toml | 2 ++ crates/trigger2/src/cli.rs | 29 ++++++++++++++++++++++------- crates/trigger2/src/factors.rs | 3 +++ src/bin/spin.rs | 16 ++++++++-------- src/commands/up.rs | 2 +- 11 files changed, 61 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e773786f2..666c85605 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7491,9 +7491,6 @@ dependencies = [ "levenshtein", "nix 0.24.3", "openssl", - "outbound-http", - "outbound-mqtt", - "outbound-redis", "path-absolutize", "rand 0.8.5", "redis 0.24.0", @@ -7520,10 +7517,8 @@ dependencies = [ "spin-plugins", "spin-telemetry", "spin-templates", - "spin-trigger", - "spin-trigger-http", - "spin-trigger-redis", - "spin-variables", + "spin-trigger-http2", + "spin-trigger2", "subprocess", "tempfile", "terminal", @@ -8027,7 +8022,6 @@ dependencies = [ "itertools 0.10.5", "lazy_static 1.4.0", "mime_guess", - "outbound-http", "path-absolutize", "regex", "reqwest 0.11.27", @@ -8165,6 +8159,7 @@ dependencies = [ "spin-factor-outbound-http", "spin-factor-outbound-networking", "spin-factor-sqlite", + "spin-factor-variables", "spin-factor-wasi", "spin-factors", "toml 0.8.14", @@ -8468,11 +8463,13 @@ dependencies = [ "serde_json", "spin-app", "spin-common", + "spin-componentize", "spin-core", "spin-factor-key-value", "spin-factor-outbound-http", "spin-factor-outbound-networking", "spin-factor-sqlite", + "spin-factor-variables", "spin-factor-wasi", "spin-factors", "spin-factors-executor", diff --git a/Cargo.toml b/Cargo.toml index 8497b1e1c..b69d2c0a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,9 +35,6 @@ itertools = "0.11.0" lazy_static = "1.4.0" levenshtein = "1.0.5" nix = { version = "0.24", features = ["signal"] } -outbound-http = { path = "crates/outbound-http" } -outbound-redis = { path = "crates/outbound-redis" } -outbound-mqtt = { path = "crates/outbound-mqtt" } spin-key-value = { path = "crates/key-value" } spin-key-value-sqlite = { path = "crates/key-value-sqlite" } path-absolutize = "3.0.11" @@ -65,10 +62,9 @@ spin-telemetry = { path = "crates/telemetry", features = [ "tracing-log-compat", ] } spin-templates = { path = "crates/templates" } -spin-trigger = { path = "crates/trigger" } -spin-trigger-http = { path = "crates/trigger-http" } -spin-trigger-redis = { path = "crates/trigger-redis" } -spin-variables = { path = "crates/variables" } +spin-trigger2 = { path = "crates/trigger2" } +spin-trigger-http2 = { path = "crates/trigger-http2" } +# TODO: spin-trigger-redis = { path = "crates/trigger-redis" } tempfile = "3.8.0" tokio = { version = "1.23", features = ["full"] } @@ -114,12 +110,13 @@ vergen = { version = "^8.2.1", default-features = false, features = [ wit-component = "0.19.0" [features] -default = ["llm"] +# TODO(factors): default = ["llm"] all-tests = ["extern-dependencies-tests"] extern-dependencies-tests = [] -llm = ["spin-trigger-http/llm"] -llm-metal = ["llm", "spin-trigger-http/llm-metal"] -llm-cublas = ["llm", "spin-trigger-http/llm-cublas"] +# TODO(factors): +# llm = ["spin-trigger-http/llm"] +# llm-metal = ["llm", "spin-trigger-http/llm-metal"] +# llm-cublas = ["llm", "spin-trigger-http/llm-cublas"] [workspace] members = [ diff --git a/crates/loader/Cargo.toml b/crates/loader/Cargo.toml index 9173ec0eb..74680ca8d 100644 --- a/crates/loader/Cargo.toml +++ b/crates/loader/Cargo.toml @@ -16,7 +16,6 @@ indexmap = { version = "1" } itertools = "0.10.3" lazy_static = "1.4.0" mime_guess = { version = "2.0" } -outbound-http = { path = "../outbound-http", default-features = false } spin-outbound-networking = { path = "../outbound-networking" } path-absolutize = { version = "3.0.11", features = ["use_unix_paths_on_wasm"] } regex = "1.5.4" diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml index 54e3e03e1..8dcbd55b7 100644 --- a/crates/oci/Cargo.toml +++ b/crates/oci/Cargo.toml @@ -6,7 +6,7 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" -async-compression = "0.4.3" +async-compression = { version = "0.4.3", features = ["gzip", "tokio"] } # Fork with nested async-std dependency bumped to satisfy Windows build; branch/revision is protected async-tar = { git = "https://github.com/vdice/async-tar", rev = "71e037f9652971e7a55b412a8e47a37b06f9c29d" } base64 = "0.21" diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index 7a52c5128..989793a87 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -18,6 +18,7 @@ spin-factor-key-value-azure = { path = "../factor-key-value-azure" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-sqlite = { path = "../factor-sqlite" } +spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } toml = "0.8" diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 12f40c750..61624133b 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -7,6 +7,7 @@ use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::runtime_config::spin::SpinTlsRuntimeConfig; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_sqlite::runtime_config::spin as sqlite; +use spin_factor_variables::{spin_cli as variables, VariablesFactor}; use spin_factor_sqlite::SqliteFactor; use spin_factor_wasi::WasiFactor; use spin_factors::{ @@ -97,7 +98,9 @@ impl ResolvedRuntimeConfig { Self { sqlite_resolver: sqlite_config_resolver(state_dir) .expect("failed to resolve sqlite runtime config"), - key_value_resolver: Default::default(), + key_value_resolver: key_value_config_resolver(PathBuf::from( + state_dir.unwrap_or(DEFAULT_STATE_DIR), + )), runtime_config: Default::default(), } } @@ -144,6 +147,12 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSo } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result::RuntimeConfig>> { + Ok(Some(variables::runtime_config_from_toml(self.table.as_ref())?)) + } +} + impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 656f51035..ab955f5d5 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -19,9 +19,11 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" spin-app = { path = "../app" } spin-common = { path = "../common" } +spin-componentize = { path = "../componentize" } spin-core = { path = "../core" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-sqlite = { path = "../factor-sqlite" } diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index 321f1d56d..db095fd45 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -220,15 +220,30 @@ impl FactorsTriggerCommand { // LLmOptions { use_gpu: true }, // ); - // TODO: component loader - struct TodoComponentLoader; - impl ComponentLoader for TodoComponentLoader { + // TODO: port the rest of the component loader logic + struct SimpleComponentLoader; + impl ComponentLoader for SimpleComponentLoader { fn load_component( &mut self, - _engine: &spin_core::wasmtime::Engine, - _component: &spin_factors::AppComponent, + engine: &spin_core::wasmtime::Engine, + component: &spin_factors::AppComponent, ) -> anyhow::Result { - todo!() + let source = component + .source() + .content + .source + .as_ref() + .context("LockedComponentSource missing source field")?; + let path = parse_file_url(source)?; + let bytes = std::fs::read(&path).with_context(|| { + format!( + "failed to read component source from disk at path {}", + quoted_path(&path) + ) + })?; + let component = spin_componentize::componentize_if_necessary(&bytes)?; + spin_core::Component::new(engine, component.as_ref()) + .with_context(|| format!("loading module {}", quoted_path(&path))) } } @@ -243,7 +258,7 @@ impl FactorsTriggerCommand { let configured_app = { let _sloth_guard = warn_if_wasm_build_slothful(); - executor.load_app(app, runtime_config.runtime_config, TodoComponentLoader)? + executor.load_app(app, runtime_config.runtime_config, SimpleComponentLoader)? }; let run_fut = trigger.run(configured_app); diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index 973471894..c1bd26a39 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -4,6 +4,7 @@ use spin_factor_key_value::KeyValueFactor; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_sqlite::SqliteFactor; +use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{spin::SpinFilesMounter, WasiFactor}; use spin_factors::RuntimeFactors; use spin_runtime_config::TomlRuntimeConfigSource; @@ -11,6 +12,7 @@ use spin_runtime_config::TomlRuntimeConfigSource; #[derive(RuntimeFactors)] pub struct TriggerFactors { pub wasi: WasiFactor, + pub variables: VariablesFactor, pub key_value: KeyValueFactor, pub outbound_networking: OutboundNetworkingFactor, pub outbound_http: OutboundHttpFactor, @@ -27,6 +29,7 @@ impl TriggerFactors { let files_mounter = SpinFilesMounter::new(working_dir, allow_transient_writes); Self { wasi: WasiFactor::new(files_mounter), + variables: VariablesFactor::default(), key_value: KeyValueFactor::new(default_key_value_label_resolver), outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, diff --git a/src/bin/spin.rs b/src/bin/spin.rs index 481f48d0c..42e6539dc 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -15,10 +15,10 @@ use spin_cli::commands::{ watch::WatchCommand, }; use spin_cli::{build_info::*, subprocess::ExitStatusError}; -use spin_trigger::cli::help::HelpArgsOnlyTrigger; -use spin_trigger::cli::TriggerExecutorCommand; -use spin_trigger_http::HttpTrigger; -use spin_trigger_redis::RedisTrigger; +use spin_trigger2::cli::help::HelpArgsOnlyTrigger; +use spin_trigger2::cli::FactorsTriggerCommand; +use spin_trigger_http2::HttpTrigger; +// TODO(factors): use spin_trigger_redis::RedisTrigger; #[tokio::main] async fn main() { @@ -136,10 +136,10 @@ enum SpinApp { #[derive(Subcommand)] enum TriggerCommands { - Http(TriggerExecutorCommand), - Redis(TriggerExecutorCommand), + Http(FactorsTriggerCommand), + // TODO(factors): Redis(TriggerExecutorCommand), #[clap(name = spin_cli::HELP_ARGS_ONLY_TRIGGER_TYPE, hide = true)] - HelpArgsOnly(TriggerExecutorCommand), + HelpArgsOnly(FactorsTriggerCommand), } impl SpinApp { @@ -155,7 +155,7 @@ impl SpinApp { Self::Registry(cmd) => cmd.run().await, Self::Build(cmd) => cmd.run().await, Self::Trigger(TriggerCommands::Http(cmd)) => cmd.run().await, - Self::Trigger(TriggerCommands::Redis(cmd)) => cmd.run().await, + // TODO(factors): Self::Trigger(TriggerCommands::Redis(cmd)) => cmd.run().await, Self::Trigger(TriggerCommands::HelpArgsOnly(cmd)) => cmd.run().await, Self::Plugins(cmd) => cmd.run().await, Self::External(cmd) => execute_external_subcommand(cmd, app).await, diff --git a/src/commands/up.rs b/src/commands/up.rs index 87f5d61cf..1b8afd106 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -15,7 +15,7 @@ use spin_app::locked::LockedApp; use spin_common::ui::quoted_path; use spin_loader::FilesMountStrategy; use spin_oci::OciLoader; -use spin_trigger::cli::{LaunchMetadata, SPIN_LOCAL_APP_DIR, SPIN_LOCKED_URL, SPIN_WORKING_DIR}; +use spin_trigger2::cli::{LaunchMetadata, SPIN_LOCAL_APP_DIR, SPIN_LOCKED_URL, SPIN_WORKING_DIR}; use tempfile::TempDir; use crate::opts::*; From fdec60b9fdd2cc98b6e6cc1fe4ff3aec643b277c Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 19 Aug 2024 16:41:25 +0200 Subject: [PATCH 131/195] PR Feedback Signed-off-by: Ryan Levick --- crates/factor-outbound-http/src/lib.rs | 12 ++++++++++++ crates/trigger-http2/src/lib.rs | 9 ++++++--- crates/trigger-http2/src/server.rs | 19 ++++++++++--------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 78d05848d..7db483bb0 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -3,6 +3,8 @@ mod wasi; pub mod wasi_2023_10_18; pub mod wasi_2023_11_10; +use std::net::SocketAddr; + use anyhow::Context; use http::{ uri::{Authority, Parts, PathAndQuery, Scheme}, @@ -101,6 +103,16 @@ pub struct SelfRequestOrigin { } impl SelfRequestOrigin { + pub fn create(scheme: Scheme, addr: &SocketAddr) -> anyhow::Result { + Ok(SelfRequestOrigin { + scheme, + authority: addr + .to_string() + .parse() + .with_context(|| format!("address '{addr}' is not a valid authority"))?, + }) + } + pub fn from_uri(uri: &Uri) -> anyhow::Result { Ok(Self { scheme: uri.scheme().context("URI missing scheme")?.clone(), diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index d36b70d6b..abb731982 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -68,7 +68,10 @@ pub(crate) type InstanceState = (); /// The Spin HTTP trigger. pub struct HttpTrigger { /// The address the server will listen on. - addr_to_bind: SocketAddr, + /// + /// Note that this might not be the actual socket address that ends up being bound to. + /// If the port is set to 0, the actual address will be determined by the OS. + listen_addr: SocketAddr, tls_config: Option, router: Router, // Component ID -> component trigger config @@ -113,7 +116,7 @@ impl Trigger for HttpTrigger { ); Ok(Self { - addr_to_bind: cli_args.address, + listen_addr: cli_args.address, tls_config: cli_args.into_tls_config(), router, component_trigger_configs, @@ -122,7 +125,7 @@ impl Trigger for HttpTrigger { async fn run(self, trigger_app: TriggerApp) -> anyhow::Result<()> { let Self { - addr_to_bind: listen_addr, + listen_addr, tls_config, router, component_trigger_configs, diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index 72ffe9e0f..396a1870b 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -164,15 +164,7 @@ impl HttpServer { let mut instance_builder = self.trigger_app.prepare(component_id)?; // Set up outbound HTTP request origin and service chaining - let origin = SelfRequestOrigin { - scheme: server_scheme, - authority: self.listen_addr.to_string().parse().with_context(|| { - format!( - "server address '{}' is not a valid authority", - self.listen_addr - ) - })?, - }; + let origin = SelfRequestOrigin::create(server_scheme, &self.listen_addr)?; instance_builder .factor_builders() .outbound_http() @@ -358,6 +350,15 @@ fn set_req_uri(req: &mut Request, scheme: Scheme) -> anyhow::Result<()> { let authority = host_header .parse() .context("'Host' header contains an invalid authority")?; + // Ensure that if `req.authority` is set, it matches what was in the `Host` header + // https://github.com/hyperium/hyper/issues/1612 + if let Some(a) = parts.authority.as_ref() { + if a != &authority { + return Err(anyhow::anyhow!( + "authority in 'Host' header does not match authority in URI" + )); + } + } parts.scheme = Some(scheme); parts.authority = Some(authority); *req.uri_mut() = Uri::from_parts(parts).unwrap(); From 1b25410a884f84c1009bf840125c6216398aae92 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 19 Aug 2024 17:15:38 +0200 Subject: [PATCH 132/195] Integrate outbound-redis factor into trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 2 ++ crates/factor-outbound-redis/src/lib.rs | 12 +++++++++++- .../factor-outbound-redis/tests/factor_test.rs | 2 +- crates/runtime-config/Cargo.toml | 1 + crates/runtime-config/src/lib.rs | 17 ++++++++++++++--- crates/trigger2/Cargo.toml | 1 + crates/trigger2/src/factors.rs | 3 +++ 7 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 666c85605..d98a72b59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8158,6 +8158,7 @@ dependencies = [ "spin-factor-key-value-spin", "spin-factor-outbound-http", "spin-factor-outbound-networking", + "spin-factor-outbound-redis", "spin-factor-sqlite", "spin-factor-variables", "spin-factor-wasi", @@ -8468,6 +8469,7 @@ dependencies = [ "spin-factor-key-value", "spin-factor-outbound-http", "spin-factor-outbound-networking", + "spin-factor-outbound-redis", "spin-factor-sqlite", "spin-factor-variables", "spin-factor-wasi", diff --git a/crates/factor-outbound-redis/src/lib.rs b/crates/factor-outbound-redis/src/lib.rs index bcd983091..c0ff5924d 100644 --- a/crates/factor-outbound-redis/src/lib.rs +++ b/crates/factor-outbound-redis/src/lib.rs @@ -6,7 +6,17 @@ use spin_factors::{ anyhow, ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, }; -pub struct OutboundRedisFactor; + +/// The [`Factor`] for `fermyon:spin/outbound-redis`. +pub struct OutboundRedisFactor { + _priv: (), +} + +impl OutboundRedisFactor { + pub fn new() -> Self { + Self { _priv: () } + } +} impl Factor for OutboundRedisFactor { type RuntimeConfig = (); diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs index 14c1ee24f..70a09f0be 100644 --- a/crates/factor-outbound-redis/tests/factor_test.rs +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -18,7 +18,7 @@ async fn no_outbound_hosts_fails() -> anyhow::Result<()> { let factors = TestFactors { variables: VariablesFactor::default(), networking: OutboundNetworkingFactor, - redis: OutboundRedisFactor, + redis: OutboundRedisFactor::new(), }; let env = TestEnvironment::new(factors).extend_manifest(toml! { spin_manifest_version = 2 diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index 989793a87..e67129a5e 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -17,6 +17,7 @@ spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-key-value-azure = { path = "../factor-key-value-azure" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-sqlite = { path = "../factor-sqlite" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 61624133b..d4df03c73 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -6,9 +6,10 @@ use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::runtime_config::spin::SpinTlsRuntimeConfig; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_outbound_redis::OutboundRedisFactor; use spin_factor_sqlite::runtime_config::spin as sqlite; -use spin_factor_variables::{spin_cli as variables, VariablesFactor}; use spin_factor_sqlite::SqliteFactor; +use spin_factor_variables::{spin_cli as variables, VariablesFactor}; use spin_factor_wasi::WasiFactor; use spin_factors::{ runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, @@ -148,8 +149,18 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSo } impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { - fn get_runtime_config(&mut self) -> anyhow::Result::RuntimeConfig>> { - Ok(Some(variables::runtime_config_from_toml(self.table.as_ref())?)) + fn get_runtime_config( + &mut self, + ) -> anyhow::Result::RuntimeConfig>> { + Ok(Some(variables::runtime_config_from_toml( + self.table.as_ref(), + )?)) + } +} + +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(None) } } diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index ab955f5d5..70a7588ac 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -27,6 +27,7 @@ spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-sqlite = { path = "../factor-sqlite" } +spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index c1bd26a39..897151863 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use spin_factor_key_value::KeyValueFactor; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_outbound_redis::OutboundRedisFactor; use spin_factor_sqlite::SqliteFactor; use spin_factor_variables::VariablesFactor; use spin_factor_wasi::{spin::SpinFilesMounter, WasiFactor}; @@ -17,6 +18,7 @@ pub struct TriggerFactors { pub outbound_networking: OutboundNetworkingFactor, pub outbound_http: OutboundHttpFactor, pub sqlite: SqliteFactor, + pub redis: OutboundRedisFactor, } impl TriggerFactors { @@ -34,6 +36,7 @@ impl TriggerFactors { outbound_networking: OutboundNetworkingFactor, outbound_http: OutboundHttpFactor, sqlite: SqliteFactor::new(default_sqlite_label_resolver), + redis: OutboundRedisFactor::new(), } } } From 6fea86018e6ff573ae4ce83f19047054856fa217 Mon Sep 17 00:00:00 2001 From: Caleb Schoepp Date: Mon, 19 Aug 2024 09:10:35 -0600 Subject: [PATCH 133/195] factors: Add spin-factor-outbound-mysql Signed-off-by: Caleb Schoepp Co-authored-by: Karthik Ganeshram --- Cargo.lock | 21 + crates/factor-outbound-mysql/Cargo.toml | 32 ++ crates/factor-outbound-mysql/src/host.rs | 402 ++++++++++++++++++ crates/factor-outbound-mysql/src/lib.rs | 57 +++ crates/factor-outbound-mysql/src/old-lib.rs | 442 ++++++++++++++++++++ 5 files changed, 954 insertions(+) create mode 100644 crates/factor-outbound-mysql/Cargo.toml create mode 100644 crates/factor-outbound-mysql/src/host.rs create mode 100644 crates/factor-outbound-mysql/src/lib.rs create mode 100644 crates/factor-outbound-mysql/src/old-lib.rs diff --git a/Cargo.lock b/Cargo.lock index e773786f2..f510e50ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7707,6 +7707,27 @@ dependencies = [ "wasmtime-wasi-http", ] +[[package]] +name = "spin-factor-outbound-mysql" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "flate2", + "mysql_async", + "mysql_common", + "spin-app", + "spin-core", + "spin-expressions", + "spin-factor-outbound-networking", + "spin-factors", + "spin-outbound-networking", + "spin-world", + "table", + "tokio", + "tracing", + "url", +] + [[package]] name = "spin-factor-outbound-networking" version = "2.7.0-pre0" diff --git a/crates/factor-outbound-mysql/Cargo.toml b/crates/factor-outbound-mysql/Cargo.toml new file mode 100644 index 000000000..2dd206dac --- /dev/null +++ b/crates/factor-outbound-mysql/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "spin-factor-outbound-mysql" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[lib] +doctest = false + +[dependencies] +anyhow = "1.0" +flate2 = "1.0.17" +# Removing default features for mysql_async to remove flate2/zlib feature +mysql_async = { version = "0.33.0", default-features = false, features = [ + "native-tls-tls", +] } +# Removing default features for mysql_common to remove flate2/zlib feature +mysql_common = { version = "0.31.0", default-features = false } +spin-app = { path = "../app" } +spin-core = { path = "../core" } +spin-expressions = { path = "../expressions" } +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factors = { path = "../factors"} +spin-outbound-networking = { path = "../outbound-networking" } +spin-world = { path = "../world" } +table = { path = "../table" } +tokio = { version = "1", features = ["rt-multi-thread"] } +tracing = { version = "0.1", features = ["log"] } +url = "2.3.1" + +[lints] +workspace = true diff --git a/crates/factor-outbound-mysql/src/host.rs b/crates/factor-outbound-mysql/src/host.rs new file mode 100644 index 000000000..0a0e1b35a --- /dev/null +++ b/crates/factor-outbound-mysql/src/host.rs @@ -0,0 +1,402 @@ +use anyhow::Result; +use mysql_async::{consts::ColumnType, from_value_opt, prelude::*, Opts, OptsBuilder, SslOpts}; +use spin_core::async_trait; +use spin_core::wasmtime::component::Resource; +use spin_world::v1::mysql as v1; +use spin_world::v2::mysql::{self as v2, Connection}; +use spin_world::v2::rdbms_types as v2_types; +use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue}; +use std::sync::Arc; +use tracing::{instrument, Level}; +use url::Url; + +use crate::InstanceState; + +impl InstanceState { + async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { + self.connections + .push( + build_conn(address) + .await + .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, + ) + .map_err(|_| v2::Error::ConnectionFailed("too many connections".into())) + .map(Resource::new_own) + } + + async fn get_conn( + &mut self, + connection: Resource, + ) -> Result<&mut mysql_async::Conn, v2::Error> { + self.connections + .get_mut(connection.rep()) + .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) + } + + async fn is_address_allowed(&self, address: &str) -> Result { + self.allowed_hosts.check_url(address, "mysql").await + } +} + +impl v2::Host for InstanceState {} + +#[async_trait] +impl v2::HostConnection for InstanceState { + #[instrument(name = "spin_outbound_mysql.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql"))] + async fn open(&mut self, address: String) -> Result, v2::Error> { + if !self + .is_address_allowed(&address) + .await + .map_err(|e| v2::Error::Other(e.to_string()))? + { + return Err(v2::Error::ConnectionFailed(format!( + "address {address} is not permitted" + ))); + } + self.open_connection(&address).await + } + + #[instrument(name = "spin_outbound_mysql.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] + async fn execute( + &mut self, + connection: Resource, + statement: String, + params: Vec, + ) -> Result<(), v2::Error> { + let db_params = params.into_iter().map(to_sql_parameter).collect::>(); + let parameters = mysql_async::Params::Positional(db_params); + + self.get_conn(connection) + .await? + .exec_batch(&statement, &[parameters]) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + Ok(()) + } + + #[instrument(name = "spin_outbound_mysql.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] + async fn query( + &mut self, + connection: Resource, + statement: String, + params: Vec, + ) -> Result { + let db_params = params.into_iter().map(to_sql_parameter).collect::>(); + let parameters = mysql_async::Params::Positional(db_params); + + let mut query_result = self + .get_conn(connection) + .await? + .exec_iter(&statement, parameters) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + // We have to get these before collect() destroys them + let columns = convert_columns(query_result.columns()); + + match query_result.collect::().await { + Err(e) => Err(v2::Error::Other(e.to_string())), + Ok(result_set) => { + let rows = result_set + .into_iter() + .map(|row| convert_row(row, &columns)) + .collect::, _>>()?; + + Ok(v2_types::RowSet { columns, rows }) + } + } + } + + fn drop(&mut self, connection: Resource) -> Result<()> { + self.connections.remove(connection.rep()); + Ok(()) + } +} + +impl v2_types::Host for InstanceState { + fn convert_error(&mut self, error: v2::Error) -> Result { + Ok(error) + } +} + +/// Delegate a function call to the v2::HostConnection implementation +macro_rules! delegate { + ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ + if !$self.is_address_allowed(&$address).await.map_err(|e| v2::Error::Other(e.to_string()))? { + return Err(v1::MysqlError::ConnectionFailed(format!( + "address {} is not permitted", $address + ))); + } + let connection = match $self.open_connection(&$address).await { + Ok(c) => c, + Err(e) => return Err(e.into()), + }; + ::$name($self, connection, $($arg),*) + .await + .map_err(Into::into) + }}; +} + +#[async_trait] +impl v1::Host for InstanceState { + async fn execute( + &mut self, + address: String, + statement: String, + params: Vec, + ) -> Result<(), v1::MysqlError> { + delegate!(self.execute( + address, + statement, + params.into_iter().map(Into::into).collect() + )) + } + + async fn query( + &mut self, + address: String, + statement: String, + params: Vec, + ) -> Result { + delegate!(self.query( + address, + statement, + params.into_iter().map(Into::into).collect() + )) + .map(Into::into) + } + + fn convert_mysql_error(&mut self, error: v1::MysqlError) -> Result { + Ok(error) + } +} + +fn to_sql_parameter(value: ParameterValue) -> mysql_async::Value { + match value { + ParameterValue::Boolean(v) => mysql_async::Value::from(v), + ParameterValue::Int32(v) => mysql_async::Value::from(v), + ParameterValue::Int64(v) => mysql_async::Value::from(v), + ParameterValue::Int8(v) => mysql_async::Value::from(v), + ParameterValue::Int16(v) => mysql_async::Value::from(v), + ParameterValue::Floating32(v) => mysql_async::Value::from(v), + ParameterValue::Floating64(v) => mysql_async::Value::from(v), + ParameterValue::Uint8(v) => mysql_async::Value::from(v), + ParameterValue::Uint16(v) => mysql_async::Value::from(v), + ParameterValue::Uint32(v) => mysql_async::Value::from(v), + ParameterValue::Uint64(v) => mysql_async::Value::from(v), + ParameterValue::Str(v) => mysql_async::Value::from(v), + ParameterValue::Binary(v) => mysql_async::Value::from(v), + ParameterValue::DbNull => mysql_async::Value::NULL, + } +} + +fn convert_columns(columns: Option>) -> Vec { + match columns { + Some(columns) => columns.iter().map(convert_column).collect(), + None => vec![], + } +} + +fn convert_column(column: &mysql_async::Column) -> Column { + let name = column.name_str().into_owned(); + let data_type = convert_data_type(column); + + Column { name, data_type } +} + +fn convert_data_type(column: &mysql_async::Column) -> DbDataType { + let column_type = column.column_type(); + + if column_type.is_numeric_type() { + convert_numeric_type(column) + } else if column_type.is_character_type() { + convert_character_type(column) + } else { + DbDataType::Other + } +} + +fn convert_character_type(column: &mysql_async::Column) -> DbDataType { + match (column.column_type(), is_binary(column)) { + (ColumnType::MYSQL_TYPE_BLOB, false) => DbDataType::Str, // TEXT type + (ColumnType::MYSQL_TYPE_BLOB, _) => DbDataType::Binary, + (ColumnType::MYSQL_TYPE_LONG_BLOB, _) => DbDataType::Binary, + (ColumnType::MYSQL_TYPE_MEDIUM_BLOB, _) => DbDataType::Binary, + (ColumnType::MYSQL_TYPE_STRING, true) => DbDataType::Binary, // BINARY type + (ColumnType::MYSQL_TYPE_STRING, _) => DbDataType::Str, + (ColumnType::MYSQL_TYPE_VAR_STRING, true) => DbDataType::Binary, // VARBINARY type + (ColumnType::MYSQL_TYPE_VAR_STRING, _) => DbDataType::Str, + (_, _) => DbDataType::Other, + } +} + +fn convert_numeric_type(column: &mysql_async::Column) -> DbDataType { + match (column.column_type(), is_signed(column)) { + (ColumnType::MYSQL_TYPE_DOUBLE, _) => DbDataType::Floating64, + (ColumnType::MYSQL_TYPE_FLOAT, _) => DbDataType::Floating32, + (ColumnType::MYSQL_TYPE_INT24, true) => DbDataType::Int32, + (ColumnType::MYSQL_TYPE_INT24, false) => DbDataType::Uint32, + (ColumnType::MYSQL_TYPE_LONG, true) => DbDataType::Int32, + (ColumnType::MYSQL_TYPE_LONG, false) => DbDataType::Uint32, + (ColumnType::MYSQL_TYPE_LONGLONG, true) => DbDataType::Int64, + (ColumnType::MYSQL_TYPE_LONGLONG, false) => DbDataType::Uint64, + (ColumnType::MYSQL_TYPE_SHORT, true) => DbDataType::Int16, + (ColumnType::MYSQL_TYPE_SHORT, false) => DbDataType::Uint16, + (ColumnType::MYSQL_TYPE_TINY, true) => DbDataType::Int8, + (ColumnType::MYSQL_TYPE_TINY, false) => DbDataType::Uint8, + (_, _) => DbDataType::Other, + } +} + +fn is_signed(column: &mysql_async::Column) -> bool { + !column + .flags() + .contains(mysql_async::consts::ColumnFlags::UNSIGNED_FLAG) +} + +fn is_binary(column: &mysql_async::Column) -> bool { + column + .flags() + .contains(mysql_async::consts::ColumnFlags::BINARY_FLAG) +} + +fn convert_row(mut row: mysql_async::Row, columns: &[Column]) -> Result, v2::Error> { + let mut result = Vec::with_capacity(row.len()); + for index in 0..row.len() { + result.push(convert_entry(&mut row, index, columns)?); + } + Ok(result) +} + +fn convert_entry( + row: &mut mysql_async::Row, + index: usize, + columns: &[Column], +) -> Result { + match (row.take(index), columns.get(index)) { + (None, _) => Ok(DbValue::DbNull), // TODO: is this right or is this an "index out of range" thing + (_, None) => Err(v2::Error::Other(format!( + "Can't get column at index {}", + index + ))), + (Some(mysql_async::Value::NULL), _) => Ok(DbValue::DbNull), + (Some(value), Some(column)) => convert_value(value, column), + } +} + +fn convert_value(value: mysql_async::Value, column: &Column) -> Result { + match column.data_type { + DbDataType::Binary => convert_value_to::>(value).map(DbValue::Binary), + DbDataType::Boolean => convert_value_to::(value).map(DbValue::Boolean), + DbDataType::Floating32 => convert_value_to::(value).map(DbValue::Floating32), + DbDataType::Floating64 => convert_value_to::(value).map(DbValue::Floating64), + DbDataType::Int8 => convert_value_to::(value).map(DbValue::Int8), + DbDataType::Int16 => convert_value_to::(value).map(DbValue::Int16), + DbDataType::Int32 => convert_value_to::(value).map(DbValue::Int32), + DbDataType::Int64 => convert_value_to::(value).map(DbValue::Int64), + DbDataType::Str => convert_value_to::(value).map(DbValue::Str), + DbDataType::Uint8 => convert_value_to::(value).map(DbValue::Uint8), + DbDataType::Uint16 => convert_value_to::(value).map(DbValue::Uint16), + DbDataType::Uint32 => convert_value_to::(value).map(DbValue::Uint32), + DbDataType::Uint64 => convert_value_to::(value).map(DbValue::Uint64), + DbDataType::Other => Err(v2::Error::ValueConversionFailed(format!( + "Cannot convert value {:?} in column {} data type {:?}", + value, column.name, column.data_type + ))), + } +} + +async fn build_conn(address: &str) -> Result { + tracing::debug!("Build new connection: {}", address); + + let opts = build_opts(address)?; + + let connection_pool = mysql_async::Pool::new(opts); + + connection_pool.get_conn().await +} + +fn is_ssl_param(s: &str) -> bool { + ["ssl-mode", "sslmode"].contains(&s.to_lowercase().as_str()) +} + +/// The mysql_async crate blows up if you pass it an SSL parameter and doesn't support SSL opts properly. This function +/// is a workaround to manually set SSL opts if the user requests them. +/// +/// We only support ssl-mode in the query as per +/// https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-connp-props-security.html#cj-conn-prop_sslMode. +/// +/// An issue has been filed in the upstream repository https://github.com/blackbeam/mysql_async/issues/225. +fn build_opts(address: &str) -> Result { + let url = Url::parse(address)?; + + let use_ssl = url + .query_pairs() + .any(|(k, v)| is_ssl_param(&k) && v.to_lowercase() != "disabled"); + + let query_without_ssl: Vec<(_, _)> = url + .query_pairs() + .filter(|(k, _v)| !is_ssl_param(k)) + .collect(); + let mut cleaned_url = url.clone(); + cleaned_url.set_query(None); + cleaned_url + .query_pairs_mut() + .extend_pairs(query_without_ssl); + + Ok(OptsBuilder::from_opts(cleaned_url.as_str()) + .ssl_opts(if use_ssl { + Some(SslOpts::default()) + } else { + None + }) + .into()) +} + +fn convert_value_to(value: mysql_async::Value) -> Result { + from_value_opt::(value).map_err(|e| v2::Error::ValueConversionFailed(format!("{}", e))) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_mysql_address_without_ssl_mode() { + assert!(build_opts("mysql://myuser:password@127.0.0.1/db") + .unwrap() + .ssl_opts() + .is_none()) + } + + #[test] + fn test_mysql_address_with_ssl_mode_disabled() { + assert!( + build_opts("mysql://myuser:password@127.0.0.1/db?ssl-mode=DISABLED") + .unwrap() + .ssl_opts() + .is_none() + ) + } + + #[test] + fn test_mysql_address_with_ssl_mode_verify_ca() { + assert!( + build_opts("mysql://myuser:password@127.0.0.1/db?sslMode=VERIFY_CA") + .unwrap() + .ssl_opts() + .is_some() + ) + } + + #[test] + fn test_mysql_address_with_more_to_query() { + let address = "mysql://myuser:password@127.0.0.1/db?SsLmOdE=VERIFY_CA&pool_max=10"; + assert!(build_opts(address).unwrap().ssl_opts().is_some()); + assert_eq!( + build_opts(address).unwrap().pool_opts().constraints().max(), + 10 + ) + } +} diff --git a/crates/factor-outbound-mysql/src/lib.rs b/crates/factor-outbound-mysql/src/lib.rs new file mode 100644 index 000000000..20f9d83b7 --- /dev/null +++ b/crates/factor-outbound-mysql/src/lib.rs @@ -0,0 +1,57 @@ +mod host; + +use anyhow::{Context, Result}; +use mysql_async::{consts::ColumnType, from_value_opt, prelude::*, Opts, OptsBuilder, SslOpts}; +use spin_core::async_trait; +use spin_core::wasmtime::component::Resource; +use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; +use spin_factors::{Factor, InitContext, RuntimeFactors, SelfInstanceBuilder}; +use spin_world::v1::mysql as v1; +use spin_world::v2::mysql::{self as v2, Connection}; +use spin_world::v2::rdbms_types as v2_types; +use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue}; +use std::sync::Arc; +use tracing::{instrument, Level}; +use url::Url; + +pub struct OutboundMysqlFactor {} + +impl Factor for OutboundMysqlFactor { + type RuntimeConfig = (); + type AppState = (); + type InstanceBuilder = InstanceState; + + fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { + ctx.link_bindings(v1::add_to_linker)?; + ctx.link_bindings(v2::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + ctx: spin_factors::ConfigureAppContext, + ) -> anyhow::Result { + Ok(()) + } + + fn prepare( + &self, + ctx: spin_factors::PrepareContext, + builders: &mut spin_factors::InstanceBuilders, + ) -> anyhow::Result { + let allowed_hosts = builders + .get_mut::()? + .allowed_hosts(); + Ok(InstanceState { + allowed_hosts, + connections: Default::default(), + }) + } +} + +pub struct InstanceState { + allowed_hosts: OutboundAllowedHosts, + connections: table::Table, +} + +impl SelfInstanceBuilder for InstanceState {} diff --git a/crates/factor-outbound-mysql/src/old-lib.rs b/crates/factor-outbound-mysql/src/old-lib.rs new file mode 100644 index 000000000..78f369f47 --- /dev/null +++ b/crates/factor-outbound-mysql/src/old-lib.rs @@ -0,0 +1,442 @@ +use anyhow::{Context, Result}; +use mysql_async::{consts::ColumnType, from_value_opt, prelude::*, Opts, OptsBuilder, SslOpts}; +use spin_app::DynamicHostComponent; +use spin_core::wasmtime::component::Resource; +use spin_core::{async_trait, HostComponent}; +use spin_world::v1::mysql as v1; +use spin_world::v2::mysql::{self as v2, Connection}; +use spin_world::v2::rdbms_types as v2_types; +use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue}; +use std::sync::Arc; +use tracing::{instrument, Level}; +use url::Url; + +/// A simple implementation to support outbound mysql connection +pub struct OutboundMysqlComponent { + pub resolver: spin_expressions::SharedPreparedResolver, +} + +#[derive(Default)] +pub struct OutboundMysql { + allowed_hosts: spin_outbound_networking::AllowedHostsConfig, + pub connections: table::Table, +} + +impl OutboundMysql { + async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { + self.connections + .push( + build_conn(address) + .await + .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, + ) + .map_err(|_| v2::Error::ConnectionFailed("too many connections".into())) + .map(Resource::new_own) + } + + async fn get_conn( + &mut self, + connection: Resource, + ) -> Result<&mut mysql_async::Conn, v2::Error> { + self.connections + .get_mut(connection.rep()) + .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) + } + + fn is_address_allowed(&self, address: &str) -> bool { + spin_outbound_networking::check_url(address, "mysql", &self.allowed_hosts) + } +} + +impl HostComponent for OutboundMysqlComponent { + type Data = OutboundMysql; + + fn add_to_linker( + linker: &mut spin_core::Linker, + get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, + ) -> anyhow::Result<()> { + v2::add_to_linker(linker, get)?; + v1::add_to_linker(linker, get) + } + + fn build_data(&self) -> Self::Data { + Default::default() + } +} + +impl DynamicHostComponent for OutboundMysqlComponent { + fn update_data( + &self, + data: &mut Self::Data, + component: &spin_app::AppComponent, + ) -> anyhow::Result<()> { + let hosts = component + .get_metadata(spin_outbound_networking::ALLOWED_HOSTS_KEY)? + .unwrap_or_default(); + data.allowed_hosts = spin_outbound_networking::AllowedHostsConfig::parse( + &hosts, + self.resolver.get().unwrap(), + ) + .context("`allowed_outbound_hosts` contained an invalid url")?; + Ok(()) + } +} + +impl v2::Host for OutboundMysql {} + +#[async_trait] +impl v2::HostConnection for OutboundMysql { + #[instrument(name = "spin_outbound_mysql.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql"))] + async fn open(&mut self, address: String) -> Result, v2::Error> { + if !self.is_address_allowed(&address) { + return Err(v2::Error::ConnectionFailed(format!( + "address {address} is not permitted" + ))); + } + self.open_connection(&address).await + } + + #[instrument(name = "spin_outbound_mysql.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] + async fn execute( + &mut self, + connection: Resource, + statement: String, + params: Vec, + ) -> Result<(), v2::Error> { + let db_params = params.into_iter().map(to_sql_parameter).collect::>(); + let parameters = mysql_async::Params::Positional(db_params); + + self.get_conn(connection) + .await? + .exec_batch(&statement, &[parameters]) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + Ok(()) + } + + #[instrument(name = "spin_outbound_mysql.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] + async fn query( + &mut self, + connection: Resource, + statement: String, + params: Vec, + ) -> Result { + let db_params = params.into_iter().map(to_sql_parameter).collect::>(); + let parameters = mysql_async::Params::Positional(db_params); + + let mut query_result = self + .get_conn(connection) + .await? + .exec_iter(&statement, parameters) + .await + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; + + // We have to get these before collect() destroys them + let columns = convert_columns(query_result.columns()); + + match query_result.collect::().await { + Err(e) => Err(v2::Error::Other(e.to_string())), + Ok(result_set) => { + let rows = result_set + .into_iter() + .map(|row| convert_row(row, &columns)) + .collect::, _>>()?; + + Ok(v2_types::RowSet { columns, rows }) + } + } + } + + fn drop(&mut self, connection: Resource) -> Result<()> { + self.connections.remove(connection.rep()); + Ok(()) + } +} + +impl v2_types::Host for OutboundMysql { + fn convert_error(&mut self, error: v2::Error) -> Result { + Ok(error) + } +} + +/// Delegate a function call to the v2::HostConnection implementation +macro_rules! delegate { + ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ + if !$self.is_address_allowed(&$address) { + return Err(v1::MysqlError::ConnectionFailed(format!( + "address {} is not permitted", $address + ))); + } + let connection = match $self.open_connection(&$address).await { + Ok(c) => c, + Err(e) => return Err(e.into()), + }; + ::$name($self, connection, $($arg),*) + .await + .map_err(Into::into) + }}; +} + +#[async_trait] +impl v1::Host for OutboundMysql { + async fn execute( + &mut self, + address: String, + statement: String, + params: Vec, + ) -> Result<(), v1::MysqlError> { + delegate!(self.execute( + address, + statement, + params.into_iter().map(Into::into).collect() + )) + } + + async fn query( + &mut self, + address: String, + statement: String, + params: Vec, + ) -> Result { + delegate!(self.query( + address, + statement, + params.into_iter().map(Into::into).collect() + )) + .map(Into::into) + } + + fn convert_mysql_error(&mut self, error: v1::MysqlError) -> Result { + Ok(error) + } +} + +fn to_sql_parameter(value: ParameterValue) -> mysql_async::Value { + match value { + ParameterValue::Boolean(v) => mysql_async::Value::from(v), + ParameterValue::Int32(v) => mysql_async::Value::from(v), + ParameterValue::Int64(v) => mysql_async::Value::from(v), + ParameterValue::Int8(v) => mysql_async::Value::from(v), + ParameterValue::Int16(v) => mysql_async::Value::from(v), + ParameterValue::Floating32(v) => mysql_async::Value::from(v), + ParameterValue::Floating64(v) => mysql_async::Value::from(v), + ParameterValue::Uint8(v) => mysql_async::Value::from(v), + ParameterValue::Uint16(v) => mysql_async::Value::from(v), + ParameterValue::Uint32(v) => mysql_async::Value::from(v), + ParameterValue::Uint64(v) => mysql_async::Value::from(v), + ParameterValue::Str(v) => mysql_async::Value::from(v), + ParameterValue::Binary(v) => mysql_async::Value::from(v), + ParameterValue::DbNull => mysql_async::Value::NULL, + } +} + +fn convert_columns(columns: Option>) -> Vec { + match columns { + Some(columns) => columns.iter().map(convert_column).collect(), + None => vec![], + } +} + +fn convert_column(column: &mysql_async::Column) -> Column { + let name = column.name_str().into_owned(); + let data_type = convert_data_type(column); + + Column { name, data_type } +} + +fn convert_data_type(column: &mysql_async::Column) -> DbDataType { + let column_type = column.column_type(); + + if column_type.is_numeric_type() { + convert_numeric_type(column) + } else if column_type.is_character_type() { + convert_character_type(column) + } else { + DbDataType::Other + } +} + +fn convert_character_type(column: &mysql_async::Column) -> DbDataType { + match (column.column_type(), is_binary(column)) { + (ColumnType::MYSQL_TYPE_BLOB, false) => DbDataType::Str, // TEXT type + (ColumnType::MYSQL_TYPE_BLOB, _) => DbDataType::Binary, + (ColumnType::MYSQL_TYPE_LONG_BLOB, _) => DbDataType::Binary, + (ColumnType::MYSQL_TYPE_MEDIUM_BLOB, _) => DbDataType::Binary, + (ColumnType::MYSQL_TYPE_STRING, true) => DbDataType::Binary, // BINARY type + (ColumnType::MYSQL_TYPE_STRING, _) => DbDataType::Str, + (ColumnType::MYSQL_TYPE_VAR_STRING, true) => DbDataType::Binary, // VARBINARY type + (ColumnType::MYSQL_TYPE_VAR_STRING, _) => DbDataType::Str, + (_, _) => DbDataType::Other, + } +} + +fn convert_numeric_type(column: &mysql_async::Column) -> DbDataType { + match (column.column_type(), is_signed(column)) { + (ColumnType::MYSQL_TYPE_DOUBLE, _) => DbDataType::Floating64, + (ColumnType::MYSQL_TYPE_FLOAT, _) => DbDataType::Floating32, + (ColumnType::MYSQL_TYPE_INT24, true) => DbDataType::Int32, + (ColumnType::MYSQL_TYPE_INT24, false) => DbDataType::Uint32, + (ColumnType::MYSQL_TYPE_LONG, true) => DbDataType::Int32, + (ColumnType::MYSQL_TYPE_LONG, false) => DbDataType::Uint32, + (ColumnType::MYSQL_TYPE_LONGLONG, true) => DbDataType::Int64, + (ColumnType::MYSQL_TYPE_LONGLONG, false) => DbDataType::Uint64, + (ColumnType::MYSQL_TYPE_SHORT, true) => DbDataType::Int16, + (ColumnType::MYSQL_TYPE_SHORT, false) => DbDataType::Uint16, + (ColumnType::MYSQL_TYPE_TINY, true) => DbDataType::Int8, + (ColumnType::MYSQL_TYPE_TINY, false) => DbDataType::Uint8, + (_, _) => DbDataType::Other, + } +} + +fn is_signed(column: &mysql_async::Column) -> bool { + !column + .flags() + .contains(mysql_async::consts::ColumnFlags::UNSIGNED_FLAG) +} + +fn is_binary(column: &mysql_async::Column) -> bool { + column + .flags() + .contains(mysql_async::consts::ColumnFlags::BINARY_FLAG) +} + +fn convert_row(mut row: mysql_async::Row, columns: &[Column]) -> Result, v2::Error> { + let mut result = Vec::with_capacity(row.len()); + for index in 0..row.len() { + result.push(convert_entry(&mut row, index, columns)?); + } + Ok(result) +} + +fn convert_entry( + row: &mut mysql_async::Row, + index: usize, + columns: &[Column], +) -> Result { + match (row.take(index), columns.get(index)) { + (None, _) => Ok(DbValue::DbNull), // TODO: is this right or is this an "index out of range" thing + (_, None) => Err(v2::Error::Other(format!( + "Can't get column at index {}", + index + ))), + (Some(mysql_async::Value::NULL), _) => Ok(DbValue::DbNull), + (Some(value), Some(column)) => convert_value(value, column), + } +} + +fn convert_value(value: mysql_async::Value, column: &Column) -> Result { + match column.data_type { + DbDataType::Binary => convert_value_to::>(value).map(DbValue::Binary), + DbDataType::Boolean => convert_value_to::(value).map(DbValue::Boolean), + DbDataType::Floating32 => convert_value_to::(value).map(DbValue::Floating32), + DbDataType::Floating64 => convert_value_to::(value).map(DbValue::Floating64), + DbDataType::Int8 => convert_value_to::(value).map(DbValue::Int8), + DbDataType::Int16 => convert_value_to::(value).map(DbValue::Int16), + DbDataType::Int32 => convert_value_to::(value).map(DbValue::Int32), + DbDataType::Int64 => convert_value_to::(value).map(DbValue::Int64), + DbDataType::Str => convert_value_to::(value).map(DbValue::Str), + DbDataType::Uint8 => convert_value_to::(value).map(DbValue::Uint8), + DbDataType::Uint16 => convert_value_to::(value).map(DbValue::Uint16), + DbDataType::Uint32 => convert_value_to::(value).map(DbValue::Uint32), + DbDataType::Uint64 => convert_value_to::(value).map(DbValue::Uint64), + DbDataType::Other => Err(v2::Error::ValueConversionFailed(format!( + "Cannot convert value {:?} in column {} data type {:?}", + value, column.name, column.data_type + ))), + } +} + +async fn build_conn(address: &str) -> Result { + tracing::debug!("Build new connection: {}", address); + + let opts = build_opts(address)?; + + let connection_pool = mysql_async::Pool::new(opts); + + connection_pool.get_conn().await +} + +fn is_ssl_param(s: &str) -> bool { + ["ssl-mode", "sslmode"].contains(&s.to_lowercase().as_str()) +} + +/// The mysql_async crate blows up if you pass it an SSL parameter and doesn't support SSL opts properly. This function +/// is a workaround to manually set SSL opts if the user requests them. +/// +/// We only support ssl-mode in the query as per +/// https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-connp-props-security.html#cj-conn-prop_sslMode. +/// +/// An issue has been filed in the upstream repository https://github.com/blackbeam/mysql_async/issues/225. +fn build_opts(address: &str) -> Result { + let url = Url::parse(address)?; + + let use_ssl = url + .query_pairs() + .any(|(k, v)| is_ssl_param(&k) && v.to_lowercase() != "disabled"); + + let query_without_ssl: Vec<(_, _)> = url + .query_pairs() + .filter(|(k, _v)| !is_ssl_param(k)) + .collect(); + let mut cleaned_url = url.clone(); + cleaned_url.set_query(None); + cleaned_url + .query_pairs_mut() + .extend_pairs(query_without_ssl); + + Ok(OptsBuilder::from_opts(cleaned_url.as_str()) + .ssl_opts(if use_ssl { + Some(SslOpts::default()) + } else { + None + }) + .into()) +} + +fn convert_value_to(value: mysql_async::Value) -> Result { + from_value_opt::(value).map_err(|e| v2::Error::ValueConversionFailed(format!("{}", e))) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_mysql_address_without_ssl_mode() { + assert!(build_opts("mysql://myuser:password@127.0.0.1/db") + .unwrap() + .ssl_opts() + .is_none()) + } + + #[test] + fn test_mysql_address_with_ssl_mode_disabled() { + assert!( + build_opts("mysql://myuser:password@127.0.0.1/db?ssl-mode=DISABLED") + .unwrap() + .ssl_opts() + .is_none() + ) + } + + #[test] + fn test_mysql_address_with_ssl_mode_verify_ca() { + assert!( + build_opts("mysql://myuser:password@127.0.0.1/db?sslMode=VERIFY_CA") + .unwrap() + .ssl_opts() + .is_some() + ) + } + + #[test] + fn test_mysql_address_with_more_to_query() { + let address = "mysql://myuser:password@127.0.0.1/db?SsLmOdE=VERIFY_CA&pool_max=10"; + assert!(build_opts(address).unwrap().ssl_opts().is_some()); + assert_eq!( + build_opts(address).unwrap().pool_opts().constraints().max(), + 10 + ) + } +} From c7620e0c786cc7f20162d28fa8144047c0eb5b45 Mon Sep 17 00:00:00 2001 From: Caleb Schoepp Date: Mon, 19 Aug 2024 10:32:10 -0600 Subject: [PATCH 134/195] factors: Make spin-factor-outbound-mysql generic across clients and write tests Signed-off-by: Caleb Schoepp Co-authored-by: Karthik Ganeshram Signed-off-by: Caleb Schoepp --- Cargo.lock | 2 + crates/factor-outbound-mysql/Cargo.toml | 4 + .../src/{old-lib.rs => client.rs} | 208 +++---------- crates/factor-outbound-mysql/src/host.rs | 293 ++---------------- crates/factor-outbound-mysql/src/lib.rs | 46 +-- .../tests/factor_test.rs | 135 ++++++++ 6 files changed, 224 insertions(+), 464 deletions(-) rename crates/factor-outbound-mysql/src/{old-lib.rs => client.rs} (63%) create mode 100644 crates/factor-outbound-mysql/tests/factor_test.rs diff --git a/Cargo.lock b/Cargo.lock index f510e50ba..afff4aea6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7719,7 +7719,9 @@ dependencies = [ "spin-core", "spin-expressions", "spin-factor-outbound-networking", + "spin-factor-variables", "spin-factors", + "spin-factors-test", "spin-outbound-networking", "spin-world", "table", diff --git a/crates/factor-outbound-mysql/Cargo.toml b/crates/factor-outbound-mysql/Cargo.toml index 2dd206dac..da9324eab 100644 --- a/crates/factor-outbound-mysql/Cargo.toml +++ b/crates/factor-outbound-mysql/Cargo.toml @@ -28,5 +28,9 @@ tokio = { version = "1", features = ["rt-multi-thread"] } tracing = { version = "0.1", features = ["log"] } url = "2.3.1" +[dev-dependencies] +spin-factor-variables = { path = "../factor-variables" } +spin-factors-test = { path = "../factors-test" } + [lints] workspace = true diff --git a/crates/factor-outbound-mysql/src/old-lib.rs b/crates/factor-outbound-mysql/src/client.rs similarity index 63% rename from crates/factor-outbound-mysql/src/old-lib.rs rename to crates/factor-outbound-mysql/src/client.rs index 78f369f47..335074d68 100644 --- a/crates/factor-outbound-mysql/src/old-lib.rs +++ b/crates/factor-outbound-mysql/src/client.rs @@ -1,133 +1,72 @@ -use anyhow::{Context, Result}; -use mysql_async::{consts::ColumnType, from_value_opt, prelude::*, Opts, OptsBuilder, SslOpts}; -use spin_app::DynamicHostComponent; -use spin_core::wasmtime::component::Resource; -use spin_core::{async_trait, HostComponent}; -use spin_world::v1::mysql as v1; -use spin_world::v2::mysql::{self as v2, Connection}; -use spin_world::v2::rdbms_types as v2_types; -use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue}; use std::sync::Arc; -use tracing::{instrument, Level}; -use url::Url; - -/// A simple implementation to support outbound mysql connection -pub struct OutboundMysqlComponent { - pub resolver: spin_expressions::SharedPreparedResolver, -} -#[derive(Default)] -pub struct OutboundMysql { - allowed_hosts: spin_outbound_networking::AllowedHostsConfig, - pub connections: table::Table, -} +use anyhow::{anyhow, Result}; +use mysql_async::consts::ColumnType; +use mysql_async::prelude::{FromValue, Queryable as _}; +use mysql_async::{from_value_opt, Conn as MysqlClient, Opts, OptsBuilder, SslOpts}; +use spin_core::async_trait; +use spin_world::v2::mysql::{self as v2}; +use spin_world::v2::rdbms_types::{ + self as v2_types, Column, DbDataType, DbValue, ParameterValue, RowSet, +}; +use url::Url; -impl OutboundMysql { - async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { - self.connections - .push( - build_conn(address) - .await - .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, - ) - .map_err(|_| v2::Error::ConnectionFailed("too many connections".into())) - .map(Resource::new_own) - } +#[async_trait] +pub trait Client: Send + Sync + 'static { + async fn build_client(address: &str) -> Result + where + Self: Sized; - async fn get_conn( + async fn execute( &mut self, - connection: Resource, - ) -> Result<&mut mysql_async::Conn, v2::Error> { - self.connections - .get_mut(connection.rep()) - .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) - } + statement: String, + params: Vec, + ) -> Result<(), v2::Error>; - fn is_address_allowed(&self, address: &str) -> bool { - spin_outbound_networking::check_url(address, "mysql", &self.allowed_hosts) - } + async fn query( + &mut self, + statement: String, + params: Vec, + ) -> Result; } -impl HostComponent for OutboundMysqlComponent { - type Data = OutboundMysql; - - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - v2::add_to_linker(linker, get)?; - v1::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - Default::default() - } -} +#[async_trait] +impl Client for MysqlClient { + async fn build_client(address: &str) -> Result + where + Self: Sized, + { + tracing::debug!("Build new connection: {}", address); -impl DynamicHostComponent for OutboundMysqlComponent { - fn update_data( - &self, - data: &mut Self::Data, - component: &spin_app::AppComponent, - ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(spin_outbound_networking::ALLOWED_HOSTS_KEY)? - .unwrap_or_default(); - data.allowed_hosts = spin_outbound_networking::AllowedHostsConfig::parse( - &hosts, - self.resolver.get().unwrap(), - ) - .context("`allowed_outbound_hosts` contained an invalid url")?; - Ok(()) - } -} + let opts = build_opts(address)?; -impl v2::Host for OutboundMysql {} + let connection_pool = mysql_async::Pool::new(opts); -#[async_trait] -impl v2::HostConnection for OutboundMysql { - #[instrument(name = "spin_outbound_mysql.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql"))] - async fn open(&mut self, address: String) -> Result, v2::Error> { - if !self.is_address_allowed(&address) { - return Err(v2::Error::ConnectionFailed(format!( - "address {address} is not permitted" - ))); - } - self.open_connection(&address).await + connection_pool.get_conn().await.map_err(|e| anyhow!(e)) } - #[instrument(name = "spin_outbound_mysql.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] async fn execute( &mut self, - connection: Resource, statement: String, params: Vec, ) -> Result<(), v2::Error> { let db_params = params.into_iter().map(to_sql_parameter).collect::>(); let parameters = mysql_async::Params::Positional(db_params); - self.get_conn(connection) - .await? - .exec_batch(&statement, &[parameters]) + self.exec_batch(&statement, &[parameters]) .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - Ok(()) + .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e))) } - #[instrument(name = "spin_outbound_mysql.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] async fn query( &mut self, - connection: Resource, statement: String, params: Vec, - ) -> Result { + ) -> Result { let db_params = params.into_iter().map(to_sql_parameter).collect::>(); let parameters = mysql_async::Params::Positional(db_params); let mut query_result = self - .get_conn(connection) - .await? .exec_iter(&statement, parameters) .await .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; @@ -147,69 +86,6 @@ impl v2::HostConnection for OutboundMysql { } } } - - fn drop(&mut self, connection: Resource) -> Result<()> { - self.connections.remove(connection.rep()); - Ok(()) - } -} - -impl v2_types::Host for OutboundMysql { - fn convert_error(&mut self, error: v2::Error) -> Result { - Ok(error) - } -} - -/// Delegate a function call to the v2::HostConnection implementation -macro_rules! delegate { - ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ - if !$self.is_address_allowed(&$address) { - return Err(v1::MysqlError::ConnectionFailed(format!( - "address {} is not permitted", $address - ))); - } - let connection = match $self.open_connection(&$address).await { - Ok(c) => c, - Err(e) => return Err(e.into()), - }; - ::$name($self, connection, $($arg),*) - .await - .map_err(Into::into) - }}; -} - -#[async_trait] -impl v1::Host for OutboundMysql { - async fn execute( - &mut self, - address: String, - statement: String, - params: Vec, - ) -> Result<(), v1::MysqlError> { - delegate!(self.execute( - address, - statement, - params.into_iter().map(Into::into).collect() - )) - } - - async fn query( - &mut self, - address: String, - statement: String, - params: Vec, - ) -> Result { - delegate!(self.query( - address, - statement, - params.into_iter().map(Into::into).collect() - )) - .map(Into::into) - } - - fn convert_mysql_error(&mut self, error: v1::MysqlError) -> Result { - Ok(error) - } } fn to_sql_parameter(value: ParameterValue) -> mysql_async::Value { @@ -347,16 +223,6 @@ fn convert_value(value: mysql_async::Value, column: &Column) -> Result Result { - tracing::debug!("Build new connection: {}", address); - - let opts = build_opts(address)?; - - let connection_pool = mysql_async::Pool::new(opts); - - connection_pool.get_conn().await -} - fn is_ssl_param(s: &str) -> bool { ["ssl-mode", "sslmode"].contains(&s.to_lowercase().as_str()) } diff --git a/crates/factor-outbound-mysql/src/host.rs b/crates/factor-outbound-mysql/src/host.rs index 0a0e1b35a..b28a340a8 100644 --- a/crates/factor-outbound-mysql/src/host.rs +++ b/crates/factor-outbound-mysql/src/host.rs @@ -1,22 +1,20 @@ use anyhow::Result; -use mysql_async::{consts::ColumnType, from_value_opt, prelude::*, Opts, OptsBuilder, SslOpts}; use spin_core::async_trait; use spin_core::wasmtime::component::Resource; use spin_world::v1::mysql as v1; use spin_world::v2::mysql::{self as v2, Connection}; use spin_world::v2::rdbms_types as v2_types; -use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue}; -use std::sync::Arc; +use spin_world::v2::rdbms_types::ParameterValue; use tracing::{instrument, Level}; -use url::Url; +use crate::client::Client; use crate::InstanceState; -impl InstanceState { +impl InstanceState { async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { self.connections .push( - build_conn(address) + C::build_client(address) .await .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, ) @@ -24,10 +22,7 @@ impl InstanceState { .map(Resource::new_own) } - async fn get_conn( - &mut self, - connection: Resource, - ) -> Result<&mut mysql_async::Conn, v2::Error> { + async fn get_client(&mut self, connection: Resource) -> Result<&mut C, v2::Error> { self.connections .get_mut(connection.rep()) .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) @@ -38,10 +33,11 @@ impl InstanceState { } } -impl v2::Host for InstanceState {} +#[async_trait] +impl v2::Host for InstanceState {} #[async_trait] -impl v2::HostConnection for InstanceState { +impl v2::HostConnection for InstanceState { #[instrument(name = "spin_outbound_mysql.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql"))] async fn open(&mut self, address: String) -> Result, v2::Error> { if !self @@ -63,16 +59,11 @@ impl v2::HostConnection for InstanceState { statement: String, params: Vec, ) -> Result<(), v2::Error> { - let db_params = params.into_iter().map(to_sql_parameter).collect::>(); - let parameters = mysql_async::Params::Positional(db_params); - - self.get_conn(connection) + Ok(self + .get_client(connection) .await? - .exec_batch(&statement, &[parameters]) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - Ok(()) + .execute(statement, params) + .await?) } #[instrument(name = "spin_outbound_mysql.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] @@ -82,30 +73,11 @@ impl v2::HostConnection for InstanceState { statement: String, params: Vec, ) -> Result { - let db_params = params.into_iter().map(to_sql_parameter).collect::>(); - let parameters = mysql_async::Params::Positional(db_params); - - let mut query_result = self - .get_conn(connection) + Ok(self + .get_client(connection) .await? - .exec_iter(&statement, parameters) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - // We have to get these before collect() destroys them - let columns = convert_columns(query_result.columns()); - - match query_result.collect::().await { - Err(e) => Err(v2::Error::Other(e.to_string())), - Ok(result_set) => { - let rows = result_set - .into_iter() - .map(|row| convert_row(row, &columns)) - .collect::, _>>()?; - - Ok(v2_types::RowSet { columns, rows }) - } - } + .query(statement, params) + .await?) } fn drop(&mut self, connection: Resource) -> Result<()> { @@ -114,7 +86,7 @@ impl v2::HostConnection for InstanceState { } } -impl v2_types::Host for InstanceState { +impl v2_types::Host for InstanceState { fn convert_error(&mut self, error: v2::Error) -> Result { Ok(error) } @@ -139,7 +111,7 @@ macro_rules! delegate { } #[async_trait] -impl v1::Host for InstanceState { +impl v1::Host for InstanceState { async fn execute( &mut self, address: String, @@ -171,232 +143,3 @@ impl v1::Host for InstanceState { Ok(error) } } - -fn to_sql_parameter(value: ParameterValue) -> mysql_async::Value { - match value { - ParameterValue::Boolean(v) => mysql_async::Value::from(v), - ParameterValue::Int32(v) => mysql_async::Value::from(v), - ParameterValue::Int64(v) => mysql_async::Value::from(v), - ParameterValue::Int8(v) => mysql_async::Value::from(v), - ParameterValue::Int16(v) => mysql_async::Value::from(v), - ParameterValue::Floating32(v) => mysql_async::Value::from(v), - ParameterValue::Floating64(v) => mysql_async::Value::from(v), - ParameterValue::Uint8(v) => mysql_async::Value::from(v), - ParameterValue::Uint16(v) => mysql_async::Value::from(v), - ParameterValue::Uint32(v) => mysql_async::Value::from(v), - ParameterValue::Uint64(v) => mysql_async::Value::from(v), - ParameterValue::Str(v) => mysql_async::Value::from(v), - ParameterValue::Binary(v) => mysql_async::Value::from(v), - ParameterValue::DbNull => mysql_async::Value::NULL, - } -} - -fn convert_columns(columns: Option>) -> Vec { - match columns { - Some(columns) => columns.iter().map(convert_column).collect(), - None => vec![], - } -} - -fn convert_column(column: &mysql_async::Column) -> Column { - let name = column.name_str().into_owned(); - let data_type = convert_data_type(column); - - Column { name, data_type } -} - -fn convert_data_type(column: &mysql_async::Column) -> DbDataType { - let column_type = column.column_type(); - - if column_type.is_numeric_type() { - convert_numeric_type(column) - } else if column_type.is_character_type() { - convert_character_type(column) - } else { - DbDataType::Other - } -} - -fn convert_character_type(column: &mysql_async::Column) -> DbDataType { - match (column.column_type(), is_binary(column)) { - (ColumnType::MYSQL_TYPE_BLOB, false) => DbDataType::Str, // TEXT type - (ColumnType::MYSQL_TYPE_BLOB, _) => DbDataType::Binary, - (ColumnType::MYSQL_TYPE_LONG_BLOB, _) => DbDataType::Binary, - (ColumnType::MYSQL_TYPE_MEDIUM_BLOB, _) => DbDataType::Binary, - (ColumnType::MYSQL_TYPE_STRING, true) => DbDataType::Binary, // BINARY type - (ColumnType::MYSQL_TYPE_STRING, _) => DbDataType::Str, - (ColumnType::MYSQL_TYPE_VAR_STRING, true) => DbDataType::Binary, // VARBINARY type - (ColumnType::MYSQL_TYPE_VAR_STRING, _) => DbDataType::Str, - (_, _) => DbDataType::Other, - } -} - -fn convert_numeric_type(column: &mysql_async::Column) -> DbDataType { - match (column.column_type(), is_signed(column)) { - (ColumnType::MYSQL_TYPE_DOUBLE, _) => DbDataType::Floating64, - (ColumnType::MYSQL_TYPE_FLOAT, _) => DbDataType::Floating32, - (ColumnType::MYSQL_TYPE_INT24, true) => DbDataType::Int32, - (ColumnType::MYSQL_TYPE_INT24, false) => DbDataType::Uint32, - (ColumnType::MYSQL_TYPE_LONG, true) => DbDataType::Int32, - (ColumnType::MYSQL_TYPE_LONG, false) => DbDataType::Uint32, - (ColumnType::MYSQL_TYPE_LONGLONG, true) => DbDataType::Int64, - (ColumnType::MYSQL_TYPE_LONGLONG, false) => DbDataType::Uint64, - (ColumnType::MYSQL_TYPE_SHORT, true) => DbDataType::Int16, - (ColumnType::MYSQL_TYPE_SHORT, false) => DbDataType::Uint16, - (ColumnType::MYSQL_TYPE_TINY, true) => DbDataType::Int8, - (ColumnType::MYSQL_TYPE_TINY, false) => DbDataType::Uint8, - (_, _) => DbDataType::Other, - } -} - -fn is_signed(column: &mysql_async::Column) -> bool { - !column - .flags() - .contains(mysql_async::consts::ColumnFlags::UNSIGNED_FLAG) -} - -fn is_binary(column: &mysql_async::Column) -> bool { - column - .flags() - .contains(mysql_async::consts::ColumnFlags::BINARY_FLAG) -} - -fn convert_row(mut row: mysql_async::Row, columns: &[Column]) -> Result, v2::Error> { - let mut result = Vec::with_capacity(row.len()); - for index in 0..row.len() { - result.push(convert_entry(&mut row, index, columns)?); - } - Ok(result) -} - -fn convert_entry( - row: &mut mysql_async::Row, - index: usize, - columns: &[Column], -) -> Result { - match (row.take(index), columns.get(index)) { - (None, _) => Ok(DbValue::DbNull), // TODO: is this right or is this an "index out of range" thing - (_, None) => Err(v2::Error::Other(format!( - "Can't get column at index {}", - index - ))), - (Some(mysql_async::Value::NULL), _) => Ok(DbValue::DbNull), - (Some(value), Some(column)) => convert_value(value, column), - } -} - -fn convert_value(value: mysql_async::Value, column: &Column) -> Result { - match column.data_type { - DbDataType::Binary => convert_value_to::>(value).map(DbValue::Binary), - DbDataType::Boolean => convert_value_to::(value).map(DbValue::Boolean), - DbDataType::Floating32 => convert_value_to::(value).map(DbValue::Floating32), - DbDataType::Floating64 => convert_value_to::(value).map(DbValue::Floating64), - DbDataType::Int8 => convert_value_to::(value).map(DbValue::Int8), - DbDataType::Int16 => convert_value_to::(value).map(DbValue::Int16), - DbDataType::Int32 => convert_value_to::(value).map(DbValue::Int32), - DbDataType::Int64 => convert_value_to::(value).map(DbValue::Int64), - DbDataType::Str => convert_value_to::(value).map(DbValue::Str), - DbDataType::Uint8 => convert_value_to::(value).map(DbValue::Uint8), - DbDataType::Uint16 => convert_value_to::(value).map(DbValue::Uint16), - DbDataType::Uint32 => convert_value_to::(value).map(DbValue::Uint32), - DbDataType::Uint64 => convert_value_to::(value).map(DbValue::Uint64), - DbDataType::Other => Err(v2::Error::ValueConversionFailed(format!( - "Cannot convert value {:?} in column {} data type {:?}", - value, column.name, column.data_type - ))), - } -} - -async fn build_conn(address: &str) -> Result { - tracing::debug!("Build new connection: {}", address); - - let opts = build_opts(address)?; - - let connection_pool = mysql_async::Pool::new(opts); - - connection_pool.get_conn().await -} - -fn is_ssl_param(s: &str) -> bool { - ["ssl-mode", "sslmode"].contains(&s.to_lowercase().as_str()) -} - -/// The mysql_async crate blows up if you pass it an SSL parameter and doesn't support SSL opts properly. This function -/// is a workaround to manually set SSL opts if the user requests them. -/// -/// We only support ssl-mode in the query as per -/// https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-connp-props-security.html#cj-conn-prop_sslMode. -/// -/// An issue has been filed in the upstream repository https://github.com/blackbeam/mysql_async/issues/225. -fn build_opts(address: &str) -> Result { - let url = Url::parse(address)?; - - let use_ssl = url - .query_pairs() - .any(|(k, v)| is_ssl_param(&k) && v.to_lowercase() != "disabled"); - - let query_without_ssl: Vec<(_, _)> = url - .query_pairs() - .filter(|(k, _v)| !is_ssl_param(k)) - .collect(); - let mut cleaned_url = url.clone(); - cleaned_url.set_query(None); - cleaned_url - .query_pairs_mut() - .extend_pairs(query_without_ssl); - - Ok(OptsBuilder::from_opts(cleaned_url.as_str()) - .ssl_opts(if use_ssl { - Some(SslOpts::default()) - } else { - None - }) - .into()) -} - -fn convert_value_to(value: mysql_async::Value) -> Result { - from_value_opt::(value).map_err(|e| v2::Error::ValueConversionFailed(format!("{}", e))) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_mysql_address_without_ssl_mode() { - assert!(build_opts("mysql://myuser:password@127.0.0.1/db") - .unwrap() - .ssl_opts() - .is_none()) - } - - #[test] - fn test_mysql_address_with_ssl_mode_disabled() { - assert!( - build_opts("mysql://myuser:password@127.0.0.1/db?ssl-mode=DISABLED") - .unwrap() - .ssl_opts() - .is_none() - ) - } - - #[test] - fn test_mysql_address_with_ssl_mode_verify_ca() { - assert!( - build_opts("mysql://myuser:password@127.0.0.1/db?sslMode=VERIFY_CA") - .unwrap() - .ssl_opts() - .is_some() - ) - } - - #[test] - fn test_mysql_address_with_more_to_query() { - let address = "mysql://myuser:password@127.0.0.1/db?SsLmOdE=VERIFY_CA&pool_max=10"; - assert!(build_opts(address).unwrap().ssl_opts().is_some()); - assert_eq!( - build_opts(address).unwrap().pool_opts().constraints().max(), - 10 - ) - } -} diff --git a/crates/factor-outbound-mysql/src/lib.rs b/crates/factor-outbound-mysql/src/lib.rs index 20f9d83b7..123620052 100644 --- a/crates/factor-outbound-mysql/src/lib.rs +++ b/crates/factor-outbound-mysql/src/lib.rs @@ -1,25 +1,21 @@ +pub mod client; mod host; -use anyhow::{Context, Result}; -use mysql_async::{consts::ColumnType, from_value_opt, prelude::*, Opts, OptsBuilder, SslOpts}; -use spin_core::async_trait; -use spin_core::wasmtime::component::Resource; +use client::Client; +use mysql_async::Conn as MysqlClient; use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundNetworkingFactor}; use spin_factors::{Factor, InitContext, RuntimeFactors, SelfInstanceBuilder}; use spin_world::v1::mysql as v1; -use spin_world::v2::mysql::{self as v2, Connection}; -use spin_world::v2::rdbms_types as v2_types; -use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue}; -use std::sync::Arc; -use tracing::{instrument, Level}; -use url::Url; +use spin_world::v2::mysql::{self as v2}; -pub struct OutboundMysqlFactor {} +pub struct OutboundMysqlFactor { + _phantom: std::marker::PhantomData, +} -impl Factor for OutboundMysqlFactor { +impl Factor for OutboundMysqlFactor { type RuntimeConfig = (); type AppState = (); - type InstanceBuilder = InstanceState; + type InstanceBuilder = InstanceState; fn init(&mut self, mut ctx: InitContext) -> anyhow::Result<()> { ctx.link_bindings(v1::add_to_linker)?; @@ -29,14 +25,14 @@ impl Factor for OutboundMysqlFactor { fn configure_app( &self, - ctx: spin_factors::ConfigureAppContext, + _ctx: spin_factors::ConfigureAppContext, ) -> anyhow::Result { Ok(()) } fn prepare( &self, - ctx: spin_factors::PrepareContext, + _ctx: spin_factors::PrepareContext, builders: &mut spin_factors::InstanceBuilders, ) -> anyhow::Result { let allowed_hosts = builders @@ -49,9 +45,23 @@ impl Factor for OutboundMysqlFactor { } } -pub struct InstanceState { +impl Default for OutboundMysqlFactor { + fn default() -> Self { + Self { + _phantom: Default::default(), + } + } +} + +impl OutboundMysqlFactor { + pub fn new() -> Self { + Self::default() + } +} + +pub struct InstanceState { allowed_hosts: OutboundAllowedHosts, - connections: table::Table, + connections: table::Table, } -impl SelfInstanceBuilder for InstanceState {} +impl SelfInstanceBuilder for InstanceState {} diff --git a/crates/factor-outbound-mysql/tests/factor_test.rs b/crates/factor-outbound-mysql/tests/factor_test.rs new file mode 100644 index 000000000..d392a0dff --- /dev/null +++ b/crates/factor-outbound-mysql/tests/factor_test.rs @@ -0,0 +1,135 @@ +use anyhow::{bail, Result}; +use spin_factor_outbound_mysql::client::Client; +use spin_factor_outbound_mysql::OutboundMysqlFactor; +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_variables::VariablesFactor; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use spin_world::async_trait; +use spin_world::v2::mysql::HostConnection; +use spin_world::v2::mysql::{self as v2}; +use spin_world::v2::rdbms_types::{ParameterValue, RowSet}; + +#[derive(RuntimeFactors)] +struct TestFactors { + variables: VariablesFactor, + networking: OutboundNetworkingFactor, + mysql: OutboundMysqlFactor, +} + +fn factors() -> TestFactors { + TestFactors { + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + mysql: OutboundMysqlFactor::::new(), + } +} + +fn test_env() -> TestEnvironment { + TestEnvironment::new(factors()).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + allowed_outbound_hosts = ["mysql://*:*"] + }) +} + +#[tokio::test] +async fn disallowed_host_fails() -> anyhow::Result<()> { + let env = TestEnvironment::new(factors()).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + }); + let mut state = env.build_instance_state().await?; + + let res = state + .mysql + .open("mysql://user:pass@mysql.test:3306/test".to_string()) + .await; + let Err(err) = res else { + bail!("expected Err, got Ok"); + }; + assert!(matches!(err, v2::Error::ConnectionFailed(_))); + + Ok(()) +} + +#[tokio::test] +async fn allowed_host_succeeds() -> anyhow::Result<()> { + let mut state = test_env().build_instance_state().await?; + + let res = state + .mysql + .open("mysql://user:pass@localhost:3306/test".to_string()) + .await; + let Ok(_) = res else { + bail!("expected Ok, got Err"); + }; + + Ok(()) +} + +#[tokio::test] +async fn exercise_execute() -> anyhow::Result<()> { + let mut state = test_env().build_instance_state().await?; + + let connection = state + .mysql + .open("mysql://user:pass@localhost:3306/test".to_string()) + .await?; + + state + .mysql + .execute(connection, "SELECT * FROM test".to_string(), vec![]) + .await?; + + Ok(()) +} + +#[tokio::test] +async fn exercise_query() -> anyhow::Result<()> { + let mut state = test_env().build_instance_state().await?; + + let connection = state + .mysql + .open("mysql://user:pass@localhost:3306/test".to_string()) + .await?; + + state + .mysql + .query(connection, "SELECT * FROM test".to_string(), vec![]) + .await?; + + Ok(()) +} + +// TODO: We can expand this mock to track calls and simulate return values +pub struct MockClient {} + +#[async_trait] +impl Client for MockClient { + async fn build_client(_address: &str) -> anyhow::Result + where + Self: Sized, + { + Ok(MockClient {}) + } + + async fn execute( + &mut self, + _statement: String, + _params: Vec, + ) -> Result<(), v2::Error> { + Ok(()) + } + + async fn query( + &mut self, + _statement: String, + _params: Vec, + ) -> Result { + Ok(RowSet { + columns: vec![], + rows: vec![], + }) + } +} From ec47aedb1a0c7e9eecfa7a06032c50551c93d588 Mon Sep 17 00:00:00 2001 From: karthik2804 Date: Mon, 5 Aug 2024 08:44:27 +0200 Subject: [PATCH 135/195] Add outbound MQTT factor Co-authored-by: rylev Signed-off-by: karthik2804 --- Cargo.lock | 17 +++ crates/factor-outbound-mqtt/Cargo.toml | 24 ++++ crates/factor-outbound-mqtt/src/host.rs | 131 ++++++++++++++++++ crates/factor-outbound-mqtt/src/lib.rs | 128 +++++++++++++++++ .../factor-outbound-mqtt/tests/factor_test.rs | 119 ++++++++++++++++ 5 files changed, 419 insertions(+) create mode 100644 crates/factor-outbound-mqtt/Cargo.toml create mode 100644 crates/factor-outbound-mqtt/src/host.rs create mode 100644 crates/factor-outbound-mqtt/src/lib.rs create mode 100644 crates/factor-outbound-mqtt/tests/factor_test.rs diff --git a/Cargo.lock b/Cargo.lock index e66bbf0d0..32d7e70fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7715,6 +7715,23 @@ dependencies = [ "wasmtime-wasi-http", ] +[[package]] +name = "spin-factor-outbound-mqtt" +version = "2.7.0-pre0" +dependencies = [ + "anyhow", + "rumqttc", + "spin-core", + "spin-factor-outbound-networking", + "spin-factor-variables", + "spin-factors", + "spin-factors-test", + "spin-world", + "table", + "tokio", + "tracing", +] + [[package]] name = "spin-factor-outbound-mysql" version = "2.7.0-pre0" diff --git a/crates/factor-outbound-mqtt/Cargo.toml b/crates/factor-outbound-mqtt/Cargo.toml new file mode 100644 index 000000000..76c44511e --- /dev/null +++ b/crates/factor-outbound-mqtt/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "spin-factor-outbound-mqtt" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = "1.0" +rumqttc = { version = "0.24", features = ["url"] } +spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factors = { path = "../factors" } +spin-core = { path = "../core" } +spin-world = { path = "../world" } +tracing = { workspace = true } +table = { path = "../table" } +tokio = { version = "1.0", features = ["sync"] } + +[dev-dependencies] +spin-factor-variables = { path = "../factor-variables" } +spin-factors-test = { path = "../factors-test" } +tokio = { version = "1", features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/factor-outbound-mqtt/src/host.rs b/crates/factor-outbound-mqtt/src/host.rs new file mode 100644 index 000000000..a6d0a1b0c --- /dev/null +++ b/crates/factor-outbound-mqtt/src/host.rs @@ -0,0 +1,131 @@ +use std::{sync::Arc, time::Duration}; + +use anyhow::Result; +use spin_core::{async_trait, wasmtime::component::Resource}; +use spin_factor_outbound_networking::OutboundAllowedHosts; +use spin_world::v2::mqtt::{self as v2, Connection, Error, Qos}; +use tracing::{instrument, Level}; + +pub type CreateClient = Arc< + dyn Fn(String, String, String, Duration) -> Result, Error> + Send + Sync, +>; + +pub struct InstanceState { + pub allowed_hosts: OutboundAllowedHosts, + pub connections: table::Table>, + pub create_client: CreateClient, +} + +impl InstanceState { + pub fn new(allowed_hosts: OutboundAllowedHosts, create_client: CreateClient) -> Self { + Self { + allowed_hosts, + create_client, + connections: table::Table::new(1024), + } + } +} + +#[async_trait] +pub trait MqttClient: Send + Sync { + async fn publish_bytes(&self, topic: String, qos: Qos, payload: Vec) -> Result<(), Error>; +} + +impl InstanceState { + async fn is_address_allowed(&self, address: &str) -> Result { + self.allowed_hosts.check_url(address, "mqtt").await + } + + async fn establish_connection( + &mut self, + address: String, + username: String, + password: String, + keep_alive_interval: Duration, + ) -> Result, Error> { + self.connections + .push((self.create_client)( + address, + username, + password, + keep_alive_interval, + )?) + .map(Resource::new_own) + .map_err(|_| Error::TooManyConnections) + } + + async fn get_conn(&self, connection: Resource) -> Result<&dyn MqttClient, Error> { + self.connections + .get(connection.rep()) + .ok_or(Error::Other( + "could not find connection for resource".into(), + )) + .map(|c| c.as_ref()) + } +} + +impl v2::Host for InstanceState { + fn convert_error(&mut self, error: Error) -> Result { + Ok(error) + } +} + +#[async_trait] +impl v2::HostConnection for InstanceState { + #[instrument(name = "spin_outbound_mqtt.open_connection", skip(self, password), err(level = Level::INFO), fields(otel.kind = "client"))] + async fn open( + &mut self, + address: String, + username: String, + password: String, + keep_alive_interval: u64, + ) -> Result, Error> { + if !self + .is_address_allowed(&address) + .await + .map_err(|e| v2::Error::Other(e.to_string()))? + { + return Err(v2::Error::ConnectionFailed(format!( + "address {address} is not permitted" + ))); + } + self.establish_connection( + address, + username, + password, + Duration::from_secs(keep_alive_interval), + ) + .await + } + + /// Publish a message to the MQTT broker. + /// + /// OTEL trace propagation is not directly supported in MQTT V3. You will need to embed the + /// current trace context into the payload yourself. + /// https://w3c.github.io/trace-context-mqtt/#mqtt-v3-recommendation. + #[instrument(name = "spin_outbound_mqtt.publish", skip(self, connection, payload), err(level = Level::INFO), + fields(otel.kind = "producer", otel.name = format!("{} publish", topic), messaging.operation = "publish", + messaging.system = "mqtt"))] + async fn publish( + &mut self, + connection: Resource, + topic: String, + payload: Vec, + qos: Qos, + ) -> Result<(), Error> { + let conn = self.get_conn(connection).await.map_err(other_error)?; + + conn.publish_bytes(topic, qos, payload).await?; + + Ok(()) + } + + fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { + self.connections.remove(connection.rep()); + Ok(()) + } +} + +pub fn other_error(e: impl std::fmt::Display) -> Error { + Error::Other(e.to_string()) +} diff --git a/crates/factor-outbound-mqtt/src/lib.rs b/crates/factor-outbound-mqtt/src/lib.rs new file mode 100644 index 000000000..db63318db --- /dev/null +++ b/crates/factor-outbound-mqtt/src/lib.rs @@ -0,0 +1,128 @@ +mod host; + +use std::time::Duration; + +use host::other_error; +use host::CreateClient; +use host::InstanceState; +use rumqttc::{AsyncClient, Event, Incoming, Outgoing, QoS}; +use spin_core::async_trait; +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factors::{ + ConfigureAppContext, Factor, InstanceBuilders, PrepareContext, RuntimeFactors, + SelfInstanceBuilder, +}; +use spin_world::v2::mqtt::{self as v2, Error, Qos}; +use tokio::sync::Mutex; + +pub use host::MqttClient; + +pub struct OutboundMqttFactor { + create_client: CreateClient, +} + +impl OutboundMqttFactor { + pub fn new(create_client: CreateClient) -> Self { + Self { create_client } + } +} + +impl Factor for OutboundMqttFactor { + type RuntimeConfig = (); + type AppState = (); + type InstanceBuilder = InstanceState; + + fn init( + &mut self, + mut ctx: spin_factors::InitContext, + ) -> anyhow::Result<()> { + ctx.link_bindings(spin_world::v2::mqtt::add_to_linker)?; + Ok(()) + } + + fn configure_app( + &self, + _ctx: ConfigureAppContext, + ) -> anyhow::Result { + Ok(()) + } + + fn prepare( + &self, + _ctx: PrepareContext, + builders: &mut InstanceBuilders, + ) -> anyhow::Result { + let allowed_hosts = builders + .get_mut::()? + .allowed_hosts(); + Ok(InstanceState::new( + allowed_hosts, + self.create_client.clone(), + )) + } +} + +impl SelfInstanceBuilder for InstanceState {} + +pub struct NetworkedMqttClient { + inner: rumqttc::AsyncClient, + event_loop: Mutex, +} + +const MQTT_CHANNEL_CAP: usize = 1000; + +impl NetworkedMqttClient { + pub fn create( + address: String, + username: String, + password: String, + keep_alive_interval: Duration, + ) -> Result { + let mut conn_opts = rumqttc::MqttOptions::parse_url(address).map_err(|e| { + tracing::error!("MQTT URL parse error: {e:?}"); + Error::InvalidAddress + })?; + conn_opts.set_credentials(username, password); + conn_opts.set_keep_alive(keep_alive_interval); + let (client, event_loop) = AsyncClient::new(conn_opts, MQTT_CHANNEL_CAP); + Ok(Self { + inner: client, + event_loop: Mutex::new(event_loop), + }) + } +} + +#[async_trait] +impl MqttClient for NetworkedMqttClient { + async fn publish_bytes(&self, topic: String, qos: Qos, payload: Vec) -> Result<(), Error> { + let qos = match qos { + Qos::AtMostOnce => rumqttc::QoS::AtMostOnce, + Qos::AtLeastOnce => rumqttc::QoS::AtLeastOnce, + Qos::ExactlyOnce => rumqttc::QoS::ExactlyOnce, + }; + // Message published to EventLoop (not MQTT Broker) + self.inner + .publish_bytes(topic, qos, false, payload.into()) + .await + .map_err(other_error)?; + + // Poll event loop until outgoing publish event is iterated over to send the message to MQTT broker or capture/throw error. + // We may revisit this later to manage long running connections, high throughput use cases and their issues in the connection pool. + let mut lock = self.event_loop.lock().await; + loop { + let event = lock + .poll() + .await + .map_err(|err| v2::Error::ConnectionFailed(err.to_string()))?; + + match (qos, event) { + (QoS::AtMostOnce, Event::Outgoing(Outgoing::Publish(_))) + | (QoS::AtLeastOnce, Event::Incoming(Incoming::PubAck(_))) + | (QoS::ExactlyOnce, Event::Incoming(Incoming::PubComp(_))) => break, + + (_, _) => continue, + } + } + Ok(()) + } +} diff --git a/crates/factor-outbound-mqtt/tests/factor_test.rs b/crates/factor-outbound-mqtt/tests/factor_test.rs new file mode 100644 index 000000000..178d17a0e --- /dev/null +++ b/crates/factor-outbound-mqtt/tests/factor_test.rs @@ -0,0 +1,119 @@ +use std::sync::Arc; + +use anyhow::{bail, Result}; +use spin_core::async_trait; +use spin_factor_outbound_mqtt::{MqttClient, OutboundMqttFactor}; +use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_variables::VariablesFactor; +use spin_factors::{anyhow, RuntimeFactors}; +use spin_factors_test::{toml, TestEnvironment}; +use spin_world::v2::mqtt::{self as v2, Error, HostConnection, Qos}; + +pub struct MockMqttClient {} + +#[async_trait] +impl MqttClient for MockMqttClient { + async fn publish_bytes( + &self, + _topic: String, + _qos: Qos, + _payload: Vec, + ) -> Result<(), Error> { + Ok(()) + } +} + +#[derive(RuntimeFactors)] +struct TestFactors { + variables: VariablesFactor, + networking: OutboundNetworkingFactor, + mqtt: OutboundMqttFactor, +} + +fn factors() -> TestFactors { + TestFactors { + variables: VariablesFactor::default(), + networking: OutboundNetworkingFactor, + mqtt: OutboundMqttFactor::new(Arc::new(|_, _, _, _| Ok(Box::new(MockMqttClient {})))), + } +} + +fn test_env() -> TestEnvironment { + TestEnvironment::new(factors()).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + allowed_outbound_hosts = ["mqtt://*:*"] + }) +} + +#[tokio::test] +async fn disallowed_host_fails() -> anyhow::Result<()> { + let env = TestEnvironment::new(factors()).extend_manifest(toml! { + [component.test-component] + source = "does-not-exist.wasm" + }); + let mut state = env.build_instance_state().await?; + + let res = state + .mqtt + .open( + "mqtt://mqtt.test:1883".to_string(), + "username".to_string(), + "password".to_string(), + 1, + ) + .await; + let Err(err) = res else { + bail!("expected Err, got Ok"); + }; + assert!(matches!(err, v2::Error::ConnectionFailed(_))); + + Ok(()) +} + +#[tokio::test] +async fn allowed_host_succeeds() -> anyhow::Result<()> { + let mut state = test_env().build_instance_state().await?; + + let res = state + .mqtt + .open( + "mqtt://mqtt.test:1883".to_string(), + "username".to_string(), + "password".to_string(), + 1, + ) + .await; + let Ok(_) = res else { + bail!("expected Ok, got Err"); + }; + + Ok(()) +} + +#[tokio::test] +async fn exercise_publish() -> anyhow::Result<()> { + let mut state = test_env().build_instance_state().await?; + + let res = state + .mqtt + .open( + "mqtt://mqtt.test:1883".to_string(), + "username".to_string(), + "password".to_string(), + 1, + ) + .await?; + + state + .mqtt + .publish( + res, + "message".to_string(), + b"test message".to_vec(), + Qos::ExactlyOnce, + ) + .await?; + + Ok(()) +} From 398cd3f0198a4c6949d5165ded17eb21259938b9 Mon Sep 17 00:00:00 2001 From: karthik2804 Date: Mon, 19 Aug 2024 15:22:40 +0200 Subject: [PATCH 136/195] use custom trait object instead of closure trait Signed-off-by: karthik2804 --- crates/factor-outbound-mqtt/Cargo.toml | 2 +- crates/factor-outbound-mqtt/src/host.rs | 28 ++++++++++--------- crates/factor-outbound-mqtt/src/lib.rs | 9 +++--- .../factor-outbound-mqtt/tests/factor_test.rs | 17 +++++++++-- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/crates/factor-outbound-mqtt/Cargo.toml b/crates/factor-outbound-mqtt/Cargo.toml index 76c44511e..95d7dce53 100644 --- a/crates/factor-outbound-mqtt/Cargo.toml +++ b/crates/factor-outbound-mqtt/Cargo.toml @@ -11,9 +11,9 @@ spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factors = { path = "../factors" } spin-core = { path = "../core" } spin-world = { path = "../world" } -tracing = { workspace = true } table = { path = "../table" } tokio = { version = "1.0", features = ["sync"] } +tracing = { workspace = true } [dev-dependencies] spin-factor-variables = { path = "../factor-variables" } diff --git a/crates/factor-outbound-mqtt/src/host.rs b/crates/factor-outbound-mqtt/src/host.rs index a6d0a1b0c..3cd22abbd 100644 --- a/crates/factor-outbound-mqtt/src/host.rs +++ b/crates/factor-outbound-mqtt/src/host.rs @@ -6,18 +6,25 @@ use spin_factor_outbound_networking::OutboundAllowedHosts; use spin_world::v2::mqtt::{self as v2, Connection, Error, Qos}; use tracing::{instrument, Level}; -pub type CreateClient = Arc< - dyn Fn(String, String, String, Duration) -> Result, Error> + Send + Sync, ->; +#[async_trait] +pub trait ClientCreator: Send + Sync { + fn create( + &self, + address: String, + username: String, + password: String, + keep_alive_interval: Duration, + ) -> Result, Error>; +} pub struct InstanceState { - pub allowed_hosts: OutboundAllowedHosts, - pub connections: table::Table>, - pub create_client: CreateClient, + allowed_hosts: OutboundAllowedHosts, + connections: table::Table>, + create_client: Arc, } impl InstanceState { - pub fn new(allowed_hosts: OutboundAllowedHosts, create_client: CreateClient) -> Self { + pub fn new(allowed_hosts: OutboundAllowedHosts, create_client: Arc) -> Self { Self { allowed_hosts, create_client, @@ -44,12 +51,7 @@ impl InstanceState { keep_alive_interval: Duration, ) -> Result, Error> { self.connections - .push((self.create_client)( - address, - username, - password, - keep_alive_interval, - )?) + .push((self.create_client).create(address, username, password, keep_alive_interval)?) .map(Resource::new_own) .map_err(|_| Error::TooManyConnections) } diff --git a/crates/factor-outbound-mqtt/src/lib.rs b/crates/factor-outbound-mqtt/src/lib.rs index db63318db..4816e12be 100644 --- a/crates/factor-outbound-mqtt/src/lib.rs +++ b/crates/factor-outbound-mqtt/src/lib.rs @@ -1,9 +1,9 @@ mod host; +use std::sync::Arc; use std::time::Duration; use host::other_error; -use host::CreateClient; use host::InstanceState; use rumqttc::{AsyncClient, Event, Incoming, Outgoing, QoS}; use spin_core::async_trait; @@ -15,14 +15,14 @@ use spin_factors::{ use spin_world::v2::mqtt::{self as v2, Error, Qos}; use tokio::sync::Mutex; -pub use host::MqttClient; +pub use host::{ClientCreator, MqttClient}; pub struct OutboundMqttFactor { - create_client: CreateClient, + create_client: Arc, } impl OutboundMqttFactor { - pub fn new(create_client: CreateClient) -> Self { + pub fn new(create_client: Arc) -> Self { Self { create_client } } } @@ -64,6 +64,7 @@ impl Factor for OutboundMqttFactor { impl SelfInstanceBuilder for InstanceState {} +// This is a concrete implementation of the MQTT client using rumqttc. pub struct NetworkedMqttClient { inner: rumqttc::AsyncClient, event_loop: Mutex, diff --git a/crates/factor-outbound-mqtt/tests/factor_test.rs b/crates/factor-outbound-mqtt/tests/factor_test.rs index 178d17a0e..1d88b3e57 100644 --- a/crates/factor-outbound-mqtt/tests/factor_test.rs +++ b/crates/factor-outbound-mqtt/tests/factor_test.rs @@ -1,8 +1,9 @@ use std::sync::Arc; +use std::time::Duration; use anyhow::{bail, Result}; use spin_core::async_trait; -use spin_factor_outbound_mqtt::{MqttClient, OutboundMqttFactor}; +use spin_factor_outbound_mqtt::{ClientCreator, MqttClient, OutboundMqttFactor}; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; use spin_factors::{anyhow, RuntimeFactors}; @@ -23,6 +24,18 @@ impl MqttClient for MockMqttClient { } } +impl ClientCreator for MockMqttClient { + fn create( + &self, + _address: String, + _username: String, + _password: String, + _keep_alive_interval: Duration, + ) -> Result, Error> { + Ok(Arc::new(MockMqttClient {})) + } +} + #[derive(RuntimeFactors)] struct TestFactors { variables: VariablesFactor, @@ -34,7 +47,7 @@ fn factors() -> TestFactors { TestFactors { variables: VariablesFactor::default(), networking: OutboundNetworkingFactor, - mqtt: OutboundMqttFactor::new(Arc::new(|_, _, _, _| Ok(Box::new(MockMqttClient {})))), + mqtt: OutboundMqttFactor::new(Arc::new(MockMqttClient {})), } } From c578fdd1a32aef0f02cc11e648e0a0f3e4137488 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 20 Aug 2024 17:50:58 +0200 Subject: [PATCH 137/195] Refactor http-trigger to be more reusable outside of main flow Signed-off-by: Ryan Levick --- crates/trigger-http2/src/lib.rs | 61 +++--- crates/trigger-http2/src/outbound_http.rs | 13 +- crates/trigger-http2/src/server.rs | 226 ++++++++++++++-------- 3 files changed, 180 insertions(+), 120 deletions(-) diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index abb731982..1ee4262c4 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -23,7 +23,6 @@ use serde::Deserialize; use spin_app::App; use spin_http::{config::HttpTriggerConfig, routes::Router}; use spin_trigger2::Trigger; -use tokio::net::TcpListener; use wasmtime_wasi_http::bindings::wasi::http::types::ErrorCode; use server::HttpServer; @@ -67,7 +66,7 @@ pub(crate) type InstanceState = (); /// The Spin HTTP trigger. pub struct HttpTrigger { - /// The address the server will listen on. + /// The address the server should listen on. /// /// Note that this might not be the actual socket address that ends up being bound to. /// If the port is set to 0, the actual address will be determined by the OS. @@ -85,6 +84,29 @@ impl Trigger for HttpTrigger { type InstanceState = InstanceState; fn new(cli_args: Self::CliArgs, app: &spin_app::App) -> anyhow::Result { + Self::new(app, cli_args.address, cli_args.into_tls_config()) + } + + async fn run(self, trigger_app: TriggerApp) -> anyhow::Result<()> { + let server = self.into_server(trigger_app)?; + + server.serve().await?; + + Ok(()) + } + + fn supported_host_requirements() -> Vec<&'static str> { + vec![spin_app::locked::SERVICE_CHAINING_KEY] + } +} + +impl HttpTrigger { + /// Create a new `HttpTrigger`. + pub fn new( + app: &spin_app::App, + listen_addr: SocketAddr, + tls_config: Option, + ) -> anyhow::Result { Self::validate_app(app)?; let component_trigger_configs = HashMap::from_iter( @@ -114,55 +136,32 @@ impl Trigger for HttpTrigger { "Constructed router: {:?}", router.routes().collect::>() ); - Ok(Self { - listen_addr: cli_args.address, - tls_config: cli_args.into_tls_config(), + listen_addr, + tls_config, router, component_trigger_configs, }) } - async fn run(self, trigger_app: TriggerApp) -> anyhow::Result<()> { + /// Turn this [`HttpTrigger`] into an [`HttpServer`]. + pub fn into_server(self, trigger_app: TriggerApp) -> anyhow::Result> { let Self { listen_addr, tls_config, router, component_trigger_configs, } = self; - - let listener = TcpListener::bind(listen_addr) - .await - .with_context(|| format!("Unable to listen on {listen_addr}"))?; - - // Get the address the server is actually listening on - // We can't use `self.listen_addr` because it might not - // be fully resolved (e.g, port 0). - let listen_addr = listener - .local_addr() - .context("failed to retrieve address server is listening on")?; let server = Arc::new(HttpServer::new( listen_addr, + tls_config, trigger_app, router, component_trigger_configs, )?); - - if let Some(tls_config) = tls_config { - server.serve_tls(listener, tls_config).await? - } else { - server.serve(listener).await? - }; - - Ok(()) + Ok(server) } - fn supported_host_requirements() -> Vec<&'static str> { - vec![spin_app::locked::SERVICE_CHAINING_KEY] - } -} - -impl HttpTrigger { fn validate_app(app: &App) -> anyhow::Result<()> { #[derive(Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/trigger-http2/src/outbound_http.rs b/crates/trigger-http2/src/outbound_http.rs index 623eb6b69..38219b8f7 100644 --- a/crates/trigger-http2/src/outbound_http.rs +++ b/crates/trigger-http2/src/outbound_http.rs @@ -11,16 +11,17 @@ use spin_http::routes::RouteMatch; use spin_outbound_networking::parse_service_chaining_target; use wasmtime_wasi_http::types::IncomingResponse; -use crate::server::HttpServer; +use crate::server::RequestHandler; +/// An outbound HTTP interceptor that handles service chaining requests. pub struct OutboundHttpInterceptor { - server: Arc, + handler: Arc, origin: SelfRequestOrigin, } impl OutboundHttpInterceptor { - pub fn new(server: Arc, origin: SelfRequestOrigin) -> Self { - Self { server, origin } + pub fn new(handler: Arc, origin: SelfRequestOrigin) -> Self { + Self { handler, origin } } } @@ -40,10 +41,10 @@ impl spin_factor_outbound_http::OutboundHttpInterceptor for OutboundHttpIntercep let route_match = RouteMatch::synthetic(&component_id, uri.path()); let req = std::mem::take(request); let between_bytes_timeout = config.between_bytes_timeout; - let server = self.server.clone(); + let server = self.handler.clone(); let resp_fut = async move { match server - .handle_trigger_route(req, route_match, Scheme::HTTP, CHAINED_CLIENT_ADDR) + .handle_trigger_route(req, &route_match, Scheme::HTTP, CHAINED_CLIENT_ADDR) .await { Ok(resp) => Ok(Ok(IncomingResponse { diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index 396a1870b..acea0eb9e 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -36,42 +36,52 @@ use crate::{ Body, NotFoundRouteKind, TlsConfig, TriggerApp, TriggerInstanceBuilder, }; +/// An HTTP server which runs Spin apps. pub struct HttpServer { - /// The address the server is listening on. - listen_addr: SocketAddr, - trigger_app: TriggerApp, + tls_config: Option, router: Router, - // Component ID -> component trigger config - component_trigger_configs: HashMap, - // Component ID -> handler type - component_handler_types: HashMap, + handler: Arc, } impl HttpServer { + /// Create a new [`HttpServer`]. pub fn new( listen_addr: SocketAddr, + tls_config: Option, trigger_app: TriggerApp, router: Router, component_trigger_configs: HashMap, ) -> anyhow::Result { - let component_handler_types = component_trigger_configs - .keys() - .map(|component_id| { - let component = trigger_app.get_component(component_id)?; - let handler_type = HandlerType::from_component(trigger_app.engine(), component)?; - Ok((component_id.clone(), handler_type)) - }) - .collect::>()?; Ok(Self { - listen_addr, - trigger_app, + tls_config, router, - component_trigger_configs, - component_handler_types, + handler: Arc::new(RequestHandler::new( + listen_addr, + trigger_app, + component_trigger_configs, + )?), }) } - pub async fn serve(self: Arc, listener: TcpListener) -> anyhow::Result<()> { + /// Serve incoming requests over the provided [`TcpListener`]. + pub async fn serve(self: Arc) -> anyhow::Result<()> { + let listener = TcpListener::bind(self.handler.listen_addr) + .await + .with_context(|| { + format!( + "Unable to listen on {listen_addr}", + listen_addr = self.handler.listen_addr + ) + })?; + if let Some(tls_config) = self.tls_config.clone() { + self.serve_https(listener, tls_config).await?; + } else { + self.serve_http(listener).await?; + } + Ok(()) + } + + async fn serve_http(self: Arc, listener: TcpListener) -> anyhow::Result<()> { self.print_startup_msgs("http", &listener)?; loop { let (stream, client_addr) = listener.accept().await?; @@ -80,7 +90,7 @@ impl HttpServer { } } - pub async fn serve_tls( + async fn serve_https( self: Arc, listener: TcpListener, tls_config: TlsConfig, @@ -140,70 +150,15 @@ impl HttpServer { /// Handles a successful route match. pub async fn handle_trigger_route( self: &Arc, - mut req: Request, + req: Request, route_match: RouteMatch, server_scheme: Scheme, client_addr: SocketAddr, ) -> anyhow::Result> { - set_req_uri(&mut req, server_scheme.clone())?; - let app_id = self - .trigger_app - .app() - .get_metadata(APP_NAME_KEY)? - .unwrap_or_else(|| "".into()); - - let component_id = route_match.component_id(); - - spin_telemetry::metrics::monotonic_counter!( - spin.request_count = 1, - trigger_type = "http", - app_id = app_id, - component_id = component_id - ); - - let mut instance_builder = self.trigger_app.prepare(component_id)?; - - // Set up outbound HTTP request origin and service chaining - let origin = SelfRequestOrigin::create(server_scheme, &self.listen_addr)?; - instance_builder - .factor_builders() - .outbound_http() - .set_request_interceptor(OutboundHttpInterceptor::new(self.clone(), origin))?; - - // Prepare HTTP executor - let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); - let handler_type = self.component_handler_types.get(component_id).unwrap(); - let executor = trigger_config - .executor - .as_ref() - .unwrap_or(&HttpExecutorType::Http); - - let res = match executor { - HttpExecutorType::Http => match handler_type { - HandlerType::Spin => { - SpinHttpExecutor - .execute(instance_builder, &route_match, req, client_addr) - .await - } - HandlerType::Wasi0_2 - | HandlerType::Wasi2023_11_10 - | HandlerType::Wasi2023_10_18 => { - WasiHttpExecutor { - handler_type: *handler_type, - } - .execute(instance_builder, &route_match, req, client_addr) - .await - } - }, - HttpExecutorType::Wagi(wagi_config) => { - let executor = WagiHttpExecutor { - wagi_config: wagi_config.clone(), - }; - executor - .execute(instance_builder, &route_match, req, client_addr) - .await - } - }; + let res = self + .handler + .handle_trigger_route(req, &route_match, server_scheme, client_addr) + .await; match res { Ok(res) => Ok(MatchedRoute::with_response_extension( res, @@ -219,7 +174,7 @@ impl HttpServer { /// Returns spin status information. fn app_info(&self, route: String) -> anyhow::Result> { - let info = AppInfo::new(self.trigger_app.app()); + let info = AppInfo::new(self.handler.trigger_app.app()); let body = serde_json::to_vec_pretty(&info)?; Ok(MatchedRoute::with_response_extension( Response::builder() @@ -323,7 +278,7 @@ impl HttpServer { println!("Available Routes:"); for (route, component_id) in self.router.routes() { println!(" {}: {}{}", component_id, base_url, route); - if let Some(component) = self.trigger_app.app().get_component(component_id) { + if let Some(component) = self.handler.trigger_app.app().get_component(component_id) { if let Some(description) = component.get_metadata(APP_DESCRIPTION_KEY)? { println!(" {}", description); } @@ -333,6 +288,111 @@ impl HttpServer { } } +/// Handles a routed HTTP trigger request. +pub struct RequestHandler { + /// The address the server is listening on. + pub(crate) listen_addr: SocketAddr, + /// The app being triggered. + trigger_app: TriggerApp, + // Component ID -> component trigger config + component_trigger_configs: HashMap, + // Component ID -> handler type + component_handler_types: HashMap, +} + +impl RequestHandler { + /// Create a new [`RequestHandler`] + pub fn new( + listen_addr: SocketAddr, + trigger_app: TriggerApp, + component_trigger_configs: HashMap, + ) -> anyhow::Result { + let component_handler_types = component_trigger_configs + .keys() + .map(|component_id| { + let component = trigger_app.get_component(component_id)?; + let handler_type = HandlerType::from_component(trigger_app.engine(), component)?; + Ok((component_id.clone(), handler_type)) + }) + .collect::>()?; + Ok(Self { + listen_addr, + trigger_app, + component_trigger_configs, + component_handler_types, + }) + } + + /// Handle a routed request. + pub async fn handle_trigger_route( + self: &Arc, + mut req: Request, + route_match: &RouteMatch, + server_scheme: Scheme, + client_addr: SocketAddr, + ) -> anyhow::Result> { + set_req_uri(&mut req, server_scheme.clone())?; + let app_id = self + .trigger_app + .app() + .get_metadata(APP_NAME_KEY)? + .unwrap_or_else(|| "".into()); + + let component_id = route_match.component_id(); + + spin_telemetry::metrics::monotonic_counter!( + spin.request_count = 1, + trigger_type = "http", + app_id = app_id, + component_id = component_id + ); + + let mut instance_builder = self.trigger_app.prepare(component_id)?; + + // Set up outbound HTTP request origin and service chaining + let origin = SelfRequestOrigin::create(server_scheme, &self.listen_addr)?; + instance_builder + .factor_builders() + .outbound_http() + .set_request_interceptor(OutboundHttpInterceptor::new(self.clone(), origin))?; + + // Prepare HTTP executor + let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); + let handler_type = self.component_handler_types.get(component_id).unwrap(); + let executor = trigger_config + .executor + .as_ref() + .unwrap_or(&HttpExecutorType::Http); + + match executor { + HttpExecutorType::Http => match handler_type { + HandlerType::Spin => { + SpinHttpExecutor + .execute(instance_builder, route_match, req, client_addr) + .await + } + HandlerType::Wasi0_2 + | HandlerType::Wasi2023_11_10 + | HandlerType::Wasi2023_10_18 => { + WasiHttpExecutor { + handler_type: *handler_type, + } + .execute(instance_builder, route_match, req, client_addr) + .await + } + }, + HttpExecutorType::Wagi(wagi_config) => { + let executor = WagiHttpExecutor { + wagi_config: wagi_config.clone(), + }; + executor + .execute(instance_builder, route_match, req, client_addr) + .await + } + } + } +} + /// The incoming request's scheme and authority /// /// The incoming request's URI is relative to the server, so we need to set the scheme and authority. From 6a67cefe0cdabcf550d9e0be0c92589defbcb88e Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 20 Aug 2024 19:07:48 +0200 Subject: [PATCH 138/195] Get things compiling Signed-off-by: Ryan Levick --- Cargo.lock | 6 +- crates/trigger-http2/src/lib.rs | 2 +- crates/trigger2/src/cli.rs | 218 ++++++++++++------ crates/trigger2/src/stdio.rs | 9 +- tests/conformance-tests/src/lib.rs | 9 +- tests/testing-framework/Cargo.toml | 6 +- .../src/runtimes/in_process_spin.rs | 70 +++--- 7 files changed, 199 insertions(+), 121 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aed3f025a..20e822f0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8811,10 +8811,12 @@ dependencies = [ "nix 0.26.4", "regex", "reqwest 0.12.4", + "spin-app", + "spin-factors-executor", "spin-http", "spin-loader", - "spin-trigger", - "spin-trigger-http", + "spin-trigger-http2", + "spin-trigger2", "temp-dir", "test-environment", "tokio", diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index 1ee4262c4..8263915b1 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -25,7 +25,7 @@ use spin_http::{config::HttpTriggerConfig, routes::Router}; use spin_trigger2::Trigger; use wasmtime_wasi_http::bindings::wasi::http::types::ErrorCode; -use server::HttpServer; +pub use server::HttpServer; pub use tls::TlsConfig; diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index db095fd45..2aab88adb 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -1,6 +1,7 @@ mod launch_metadata; -use std::path::PathBuf; +use std::future::Future; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use clap::{Args, IntoApp, Parser}; @@ -13,7 +14,7 @@ use spin_runtime_config::ResolvedRuntimeConfig; use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; -use crate::Trigger; +use crate::{Trigger, TriggerApp}; pub use launch_metadata::LaunchMetadata; pub const APP_LOG_DIR: &str = "APP_LOG_DIR"; @@ -172,43 +173,154 @@ impl FactorsTriggerCommand { anyhow::bail!("This application requires the following features that are not available in this version of the '{}' trigger: {unmet}", T::TYPE); } - let mut trigger = T::new(self.trigger_args, &app)?; + let trigger = T::new(self.trigger_args, &app)?; + let mut builder = TriggerAppBuilder::new(trigger, PathBuf::from(working_dir)); + let config = builder.engine_config(); - let mut core_engine_builder = { - let mut config = spin_core::Config::default(); + // Apply --cache / --disable-cache + if !self.disable_cache { + config.enable_cache(&self.cache)?; + } - // Apply --cache / --disable-cache - if !self.disable_cache { - config.enable_cache(&self.cache)?; - } + if self.disable_pooling { + config.disable_pooling(); + } - if self.disable_pooling { - config.disable_pooling(); + let run_fut = builder + .run( + app, + TriggerAppOptions { + runtime_config_file: self.runtime_config_file.as_deref(), + state_dir: self.state_dir.as_deref(), + initial_key_values: self.key_values, + allow_transient_write: self.allow_transient_write, + follow_components, + log_dir: self.log, + }, + ) + .await?; + + let (abortable, abort_handle) = futures::future::abortable(run_fut); + ctrlc::set_handler(move || abort_handle.abort())?; + match abortable.await { + Ok(Ok(())) => { + tracing::info!("Trigger executor shut down: exiting"); + Ok(()) + } + Ok(Err(err)) => { + tracing::error!("Trigger executor failed"); + Err(err) + } + Err(_aborted) => { + tracing::info!("User requested shutdown: exiting"); + Ok(()) } + } + } + + fn follow_components(&self) -> FollowComponents { + if self.silence_component_logs { + FollowComponents::None + } else if self.follow_components.is_empty() { + FollowComponents::All + } else { + let followed = self.follow_components.clone().into_iter().collect(); + FollowComponents::Named(followed) + } + } +} + +const SLOTH_WARNING_DELAY_MILLIS: u64 = 1250; + +fn warn_if_wasm_build_slothful() -> sloth::SlothGuard { + #[cfg(debug_assertions)] + let message = "\ + This is a debug build - preparing Wasm modules might take a few seconds\n\ + If you're experiencing long startup times please switch to the release build"; + + #[cfg(not(debug_assertions))] + let message = "Preparing Wasm modules is taking a few seconds..."; + + sloth::warn_if_slothful(SLOTH_WARNING_DELAY_MILLIS, format!("{message}\n")) +} + +fn help_heading() -> Option<&'static str> { + if T::TYPE == help::HelpArgsOnlyTrigger::TYPE { + Some("TRIGGER OPTIONS") + } else { + let heading = format!("{} TRIGGER OPTIONS", T::TYPE.to_uppercase()); + let as_str = Box::new(heading).leak(); + Some(as_str) + } +} + +/// A builder for a [`TriggerApp`]. +pub struct TriggerAppBuilder { + engine_config: spin_core::Config, + working_dir: PathBuf, + pub trigger: T, +} - trigger.update_core_config(&mut config)?; +/// Options for building a [`TriggerApp`]. +#[derive(Default)] +pub struct TriggerAppOptions<'a> { + /// Path to the runtime config file. + runtime_config_file: Option<&'a Path>, + /// Path to the state directory. + state_dir: Option<&'a str>, + /// Initial key/value pairs to set in the app's default store. + initial_key_values: Vec<(String, String)>, + /// Whether to allow transient writes to mounted files + allow_transient_write: bool, + /// Which components should have their logs followed. + follow_components: FollowComponents, + /// Log directory for component stdout/stderr. + log_dir: Option, +} + +impl TriggerAppBuilder { + pub fn new(trigger: T, working_dir: PathBuf) -> Self { + Self { + engine_config: spin_core::Config::default(), + working_dir, + trigger, + } + } + + pub fn engine_config(&mut self) -> &mut spin_core::Config { + &mut self.engine_config + } - spin_core::Engine::builder(&config)? + /// Build a [`TriggerApp`] from the given [`App`] and options. + pub async fn build( + &mut self, + app: App, + options: TriggerAppOptions<'_>, + ) -> anyhow::Result> { + let mut core_engine_builder = { + self.trigger.update_core_config(&mut self.engine_config)?; + + spin_core::Engine::builder(&self.engine_config)? }; - trigger.add_to_linker(core_engine_builder.linker())?; + self.trigger.add_to_linker(core_engine_builder.linker())?; - let runtime_config = match &self.runtime_config_file { + let runtime_config = match options.runtime_config_file { Some(runtime_config_path) => { ResolvedRuntimeConfig::::from_file( runtime_config_path, - self.state_dir.as_deref(), + options.state_dir, )? } - None => ResolvedRuntimeConfig::default(self.state_dir.as_deref()), + None => ResolvedRuntimeConfig::default(options.state_dir), }; runtime_config - .set_initial_key_values(&self.key_values) + .set_initial_key_values(&options.initial_key_values) .await?; let factors = TriggerFactors::new( - working_dir, - self.allow_transient_write, + self.working_dir.clone(), + options.allow_transient_write, runtime_config.key_value_resolver, runtime_config.sqlite_resolver, ); @@ -249,8 +361,10 @@ impl FactorsTriggerCommand { let mut executor = FactorsExecutor::new(core_engine_builder, factors)?; - let log_dir = self.log.clone(); - executor.add_hooks(StdioLoggingExecutorHooks::new(follow_components, log_dir)); + executor.add_hooks(StdioLoggingExecutorHooks::new( + options.follow_components, + options.log_dir, + )); // TODO: // builder.hooks(SummariseRuntimeConfigHook::new(&self.runtime_config_file)); // builder.hooks(KeyValuePersistenceMessageHook); @@ -261,59 +375,17 @@ impl FactorsTriggerCommand { executor.load_app(app, runtime_config.runtime_config, SimpleComponentLoader)? }; - let run_fut = trigger.run(configured_app); - - let (abortable, abort_handle) = futures::future::abortable(run_fut); - ctrlc::set_handler(move || abort_handle.abort())?; - match abortable.await { - Ok(Ok(())) => { - tracing::info!("Trigger executor shut down: exiting"); - Ok(()) - } - Ok(Err(err)) => { - tracing::error!("Trigger executor failed"); - Err(err) - } - Err(_aborted) => { - tracing::info!("User requested shutdown: exiting"); - Ok(()) - } - } - } - - fn follow_components(&self) -> FollowComponents { - if self.silence_component_logs { - FollowComponents::None - } else if self.follow_components.is_empty() { - FollowComponents::All - } else { - let followed = self.follow_components.clone().into_iter().collect(); - FollowComponents::Named(followed) - } + Ok(configured_app) } -} - -const SLOTH_WARNING_DELAY_MILLIS: u64 = 1250; - -fn warn_if_wasm_build_slothful() -> sloth::SlothGuard { - #[cfg(debug_assertions)] - let message = "\ - This is a debug build - preparing Wasm modules might take a few seconds\n\ - If you're experiencing long startup times please switch to the release build"; - - #[cfg(not(debug_assertions))] - let message = "Preparing Wasm modules is taking a few seconds..."; - - sloth::warn_if_slothful(SLOTH_WARNING_DELAY_MILLIS, format!("{message}\n")) -} -fn help_heading() -> Option<&'static str> { - if T::TYPE == help::HelpArgsOnlyTrigger::TYPE { - Some("TRIGGER OPTIONS") - } else { - let heading = format!("{} TRIGGER OPTIONS", T::TYPE.to_uppercase()); - let as_str = Box::new(heading).leak(); - Some(as_str) + /// Run the [`TriggerApp`] with the given [`App`] and options. + pub async fn run( + mut self, + app: App, + options: TriggerAppOptions<'_>, + ) -> anyhow::Result>> { + let configured_app = self.build(app, options).await?; + Ok(self.trigger.run(configured_app)) } } diff --git a/crates/trigger2/src/stdio.rs b/crates/trigger2/src/stdio.rs index fe62c6393..398d59f72 100644 --- a/crates/trigger2/src/stdio.rs +++ b/crates/trigger2/src/stdio.rs @@ -12,8 +12,9 @@ use tokio::io::AsyncWrite; use crate::factors::TriggerFactors; /// Which components should have their logs followed on stdout/stderr. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub enum FollowComponents { + #[default] /// No components should have their logs followed. None, /// Only the specified components should have their logs followed. @@ -33,12 +34,6 @@ impl FollowComponents { } } -impl Default for FollowComponents { - fn default() -> Self { - Self::None - } -} - /// Implements TriggerHooks, writing logs to a log file and (optionally) stderr pub struct StdioLoggingExecutorHooks { follow_components: FollowComponents, diff --git a/tests/conformance-tests/src/lib.rs b/tests/conformance-tests/src/lib.rs index a03084819..bed1488a4 100644 --- a/tests/conformance-tests/src/lib.rs +++ b/tests/conformance-tests/src/lib.rs @@ -1,4 +1,5 @@ use anyhow::Context as _; +use test_environment::http::Request; use testing_framework::runtimes::spin_cli::{SpinCli, SpinConfig}; /// Run a single conformance test against the supplied spin binary. @@ -56,9 +57,11 @@ pub fn run_test( let conformance_tests::config::Invocation::Http(mut invocation) = invocation; invocation.request.substitute_from_env(&mut env)?; let spin = env.runtime_mut(); - let actual = invocation - .request - .send(|request| spin.make_http_request(request))?; + let actual = invocation.request.send(|request| { + let request = + Request::full(request.method, request.path, request.headers, request.body); + spin.make_http_request(request) + })?; conformance_tests::assertions::assert_response(&invocation.response, &actual) .with_context(|| { diff --git a/tests/testing-framework/Cargo.toml b/tests/testing-framework/Cargo.toml index d247b4123..cc388246f 100644 --- a/tests/testing-framework/Cargo.toml +++ b/tests/testing-framework/Cargo.toml @@ -14,9 +14,11 @@ regex = "1.10.2" reqwest = { workspace = true } temp-dir = "0.1.11" test-environment = { workspace = true } -spin-trigger-http = { path = "../../crates/trigger-http" } +spin-factors-executor = { path = "../../crates/factors-executor" } +spin-app = { path = "../../crates/app" } +spin-trigger-http2 = { path = "../../crates/trigger-http2" } spin-http = { path = "../../crates/http" } -spin-trigger = { path = "../../crates/trigger" } +spin-trigger2 = { path = "../../crates/trigger2" } spin-loader = { path = "../../crates/loader" } toml = "0.8.6" tokio = "1.23" diff --git a/tests/testing-framework/src/runtimes/in_process_spin.rs b/tests/testing-framework/src/runtimes/in_process_spin.rs index 5574c72b5..16ad654af 100644 --- a/tests/testing-framework/src/runtimes/in_process_spin.rs +++ b/tests/testing-framework/src/runtimes/in_process_spin.rs @@ -1,6 +1,11 @@ //! The Spin runtime running in the same process as the test +use std::{path::PathBuf, sync::Arc}; + use anyhow::Context as _; +use spin_http::routes::RouteMatch; +use spin_trigger2::cli::{TriggerAppBuilder, TriggerAppOptions}; +use spin_trigger_http2::{HttpServer, HttpTrigger}; use test_environment::{ http::{Request, Response}, services::ServicesConfig, @@ -11,7 +16,7 @@ use test_environment::{ /// /// Use `runtimes::spin_cli::SpinCli` if you'd rather use Spin as a separate process pub struct InProcessSpin { - trigger: spin_trigger_http::HttpTrigger, + server: Arc, } impl InProcessSpin { @@ -32,31 +37,45 @@ impl InProcessSpin { } /// Create a new instance of Spin running in the same process as the tests - pub fn new(trigger: spin_trigger_http::HttpTrigger) -> Self { - Self { trigger } + pub fn new(server: Arc) -> Self { + Self { server } } /// Make an HTTP request to the Spin instance pub fn make_http_request(&self, req: Request<'_, &[u8]>) -> anyhow::Result { tokio::runtime::Runtime::new()?.block_on(async { let method: reqwest::Method = req.method.into(); - let req = http::request::Request::builder() + let mut builder = http::request::Request::builder() .method(method) - .uri(req.path) - // TODO(rylev): convert headers and body as well - .body(spin_http::body::empty()) - .unwrap(); + .uri(req.path); + + for (key, value) in req.headers { + builder = builder.header(*key, *value); + } + // TODO(rylev): convert body as well + let req = builder.body(spin_http::body::empty()).unwrap(); + let route_match = RouteMatch::synthetic("test", "/"); let response = self - .trigger - .handle( + .server + .handle_trigger_route( req, + route_match, http::uri::Scheme::HTTP, - (std::net::Ipv4Addr::LOCALHOST, 3000).into(), (std::net::Ipv4Addr::LOCALHOST, 7000).into(), ) .await?; use http_body_util::BodyExt; let status = response.status().as_u16(); + let headers = response + .headers() + .iter() + .map(|(k, v)| { + ( + k.as_str().to_owned(), + String::from_utf8(v.as_bytes().to_owned()).unwrap(), + ) + }) + .collect(); let body = response.into_body(); let chunks = body .collect() @@ -64,7 +83,7 @@ impl InProcessSpin { .context("could not get runtime test HTTP response")? .to_bytes() .to_vec(); - Ok(Response::full(status, Default::default(), chunks)) + Ok(Response::full(status, headers, chunks)) }) } } @@ -79,33 +98,18 @@ impl Runtime for InProcessSpin { async fn initialize_trigger( env: &mut TestEnvironment, ) -> anyhow::Result { - use spin_trigger::{ - loader::TriggerLoader, HostComponentInitData, RuntimeConfig, TriggerExecutorBuilder, - }; - use spin_trigger_http::HttpTrigger; - - // Create the locked app and write it to a file let locked_app = spin_loader::from_file( env.path().join("spin.toml"), spin_loader::FilesMountStrategy::Direct, None, ) .await?; - let json = locked_app.to_json()?; - std::fs::write(env.path().join("locked.json"), json)?; - // Create a loader and trigger builder - let loader = TriggerLoader::new(env.path().join(".working_dir"), false); - let mut builder = TriggerExecutorBuilder::::new(loader); - builder.hooks(spin_trigger::network::Network::default()); + let app = spin_app::App::new("my-app", locked_app); + let trigger = HttpTrigger::new(&app, "127.0.0.1:80".parse().unwrap(), None)?; + let mut builder = TriggerAppBuilder::new(trigger, PathBuf::from(".")); + let trigger_app = builder.build(app, TriggerAppOptions::default()).await?; + let server = builder.trigger.into_server(trigger_app)?; - // Build the trigger - let trigger = builder - .build( - format!("file:{}", env.path().join("locked.json").display()), - RuntimeConfig::default(), - HostComponentInitData::default(), - ) - .await?; - Ok(InProcessSpin::new(trigger)) + Ok(InProcessSpin::new(server)) } From e9a524891054bb4b1c5088e2fe87ed398df748c7 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 20 Aug 2024 21:20:37 +0200 Subject: [PATCH 139/195] Don't trap when outbound http request is not allowed Signed-off-by: Ryan Levick --- crates/factor-outbound-http/src/wasi.rs | 13 +++++-------- crates/factor-outbound-http/tests/factor_test.rs | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index c1427b0f1..9f0fffeeb 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -112,7 +112,7 @@ async fn send_request_impl( let is_relative_url = request.uri().authority().is_none(); if is_relative_url { if !allowed_hosts.allows_relative_url(&["http", "https"]) { - return handle_not_allowed(request.uri(), true); + return Ok(handle_not_allowed(request.uri(), true)); } let origin = request @@ -131,7 +131,7 @@ async fn send_request_impl( let outbound_url = OutboundUrl::parse(request.uri().to_string(), "https") .map_err(|_| ErrorCode::HttpRequestUriInvalid)?; if !allowed_hosts.allows(&outbound_url) { - return handle_not_allowed(request.uri(), false); + return Ok(handle_not_allowed(request.uri(), false)); } } @@ -147,11 +147,8 @@ async fn send_request_impl( } // TODO(factors): Move to some callback on spin-factor-outbound-networking (?) -fn handle_not_allowed( - uri: &Uri, - is_relative: bool, -) -> anyhow::Result> { - tracing::error!("Destination not allowed: {uri}"); +fn handle_not_allowed(uri: &Uri, is_relative: bool) -> Result { + tracing::error!("Destination not allowed!: {uri}"); let allowed_host_example = if is_relative { terminal::warn!("A component tried to make a HTTP request to the same component but it does not have permission."); "http://self".to_string() @@ -165,7 +162,7 @@ fn handle_not_allowed( host }; eprintln!("To allow requests, add 'allowed_outbound_hosts = [\"{allowed_host_example}\"]' to the manifest component section."); - Err(ErrorCode::HttpRequestDenied.into()) + Err(ErrorCode::HttpRequestDenied) } /// This is a fork of wasmtime_wasi_http::default_send_request_handler function diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index 5f03d2ca2..3de9ac7c1 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -65,9 +65,9 @@ async fn disallowed_host_fails() -> anyhow::Result<()> { let req = Request::get("https://denied.test").body(Default::default())?; let mut future_resp = wasi_http.send_request(req, test_request_config())?; future_resp.ready().await; - match future_resp.unwrap_ready() { + match future_resp.unwrap_ready().unwrap() { Ok(_) => bail!("expected Err, got Ok"), - Err(err) => assert!(matches!(err.downcast()?, ErrorCode::HttpRequestDenied)), + Err(err) => assert!(matches!(err, ErrorCode::HttpRequestDenied)), }; Ok(()) } From 32f6da45eb37d97c64d875f4adadedfe96736749 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 21 Aug 2024 12:52:22 +0200 Subject: [PATCH 140/195] Revert RequestHandler type refactoring Signed-off-by: Ryan Levick --- crates/trigger-http2/src/outbound_http.rs | 12 +- crates/trigger-http2/src/server.rs | 218 ++++++++---------- tests/conformance-tests/src/lib.rs | 9 +- tests/testing-framework/Cargo.toml | 6 +- .../src/runtimes/in_process_spin.rs | 5 +- 5 files changed, 104 insertions(+), 146 deletions(-) diff --git a/crates/trigger-http2/src/outbound_http.rs b/crates/trigger-http2/src/outbound_http.rs index 38219b8f7..45e63ab75 100644 --- a/crates/trigger-http2/src/outbound_http.rs +++ b/crates/trigger-http2/src/outbound_http.rs @@ -11,17 +11,17 @@ use spin_http::routes::RouteMatch; use spin_outbound_networking::parse_service_chaining_target; use wasmtime_wasi_http::types::IncomingResponse; -use crate::server::RequestHandler; +use crate::HttpServer; /// An outbound HTTP interceptor that handles service chaining requests. pub struct OutboundHttpInterceptor { - handler: Arc, + server: Arc, origin: SelfRequestOrigin, } impl OutboundHttpInterceptor { - pub fn new(handler: Arc, origin: SelfRequestOrigin) -> Self { - Self { handler, origin } + pub fn new(server: Arc, origin: SelfRequestOrigin) -> Self { + Self { server, origin } } } @@ -41,10 +41,10 @@ impl spin_factor_outbound_http::OutboundHttpInterceptor for OutboundHttpIntercep let route_match = RouteMatch::synthetic(&component_id, uri.path()); let req = std::mem::take(request); let between_bytes_timeout = config.between_bytes_timeout; - let server = self.handler.clone(); + let server = self.server.clone(); let resp_fut = async move { match server - .handle_trigger_route(req, &route_match, Scheme::HTTP, CHAINED_CLIENT_ADDR) + .handle_trigger_route(req, route_match, Scheme::HTTP, CHAINED_CLIENT_ADDR) .await { Ok(resp) => Ok(Ok(IncomingResponse { diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index acea0eb9e..c01327fda 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -38,9 +38,18 @@ use crate::{ /// An HTTP server which runs Spin apps. pub struct HttpServer { + /// The address the server is listening on. + listen_addr: SocketAddr, + /// The TLS configuration for the server. tls_config: Option, + /// Request router. router: Router, - handler: Arc, + /// The app being triggered. + trigger_app: TriggerApp, + // Component ID -> component trigger config + component_trigger_configs: HashMap, + // Component ID -> handler type + component_handler_types: HashMap, } impl HttpServer { @@ -52,27 +61,32 @@ impl HttpServer { router: Router, component_trigger_configs: HashMap, ) -> anyhow::Result { + let component_handler_types = component_trigger_configs + .keys() + .map(|component_id| { + let component = trigger_app.get_component(component_id)?; + let handler_type = HandlerType::from_component(trigger_app.engine(), component)?; + Ok((component_id.clone(), handler_type)) + }) + .collect::>()?; Ok(Self { + listen_addr, tls_config, router, - handler: Arc::new(RequestHandler::new( - listen_addr, - trigger_app, - component_trigger_configs, - )?), + trigger_app, + component_trigger_configs, + component_handler_types, }) } /// Serve incoming requests over the provided [`TcpListener`]. pub async fn serve(self: Arc) -> anyhow::Result<()> { - let listener = TcpListener::bind(self.handler.listen_addr) - .await - .with_context(|| { - format!( - "Unable to listen on {listen_addr}", - listen_addr = self.handler.listen_addr - ) - })?; + let listener = TcpListener::bind(self.listen_addr).await.with_context(|| { + format!( + "Unable to listen on {listen_addr}", + listen_addr = self.listen_addr + ) + })?; if let Some(tls_config) = self.tls_config.clone() { self.serve_https(listener, tls_config).await?; } else { @@ -112,7 +126,7 @@ impl HttpServer { /// /// This method handles well known paths and routes requests to the handler when the router /// matches the requests path. - async fn handle( + pub async fn handle( self: &Arc, mut req: Request, server_scheme: Scheme, @@ -150,15 +164,70 @@ impl HttpServer { /// Handles a successful route match. pub async fn handle_trigger_route( self: &Arc, - req: Request, + mut req: Request, route_match: RouteMatch, server_scheme: Scheme, client_addr: SocketAddr, ) -> anyhow::Result> { - let res = self - .handler - .handle_trigger_route(req, &route_match, server_scheme, client_addr) - .await; + set_req_uri(&mut req, server_scheme.clone())?; + let app_id = self + .trigger_app + .app() + .get_metadata(APP_NAME_KEY)? + .unwrap_or_else(|| "".into()); + + let component_id = route_match.component_id(); + + spin_telemetry::metrics::monotonic_counter!( + spin.request_count = 1, + trigger_type = "http", + app_id = app_id, + component_id = component_id + ); + + let mut instance_builder = self.trigger_app.prepare(component_id)?; + + // Set up outbound HTTP request origin and service chaining + let origin = SelfRequestOrigin::create(server_scheme, &self.listen_addr)?; + instance_builder + .factor_builders() + .outbound_http() + .set_request_interceptor(OutboundHttpInterceptor::new(self.clone(), origin))?; + + // Prepare HTTP executor + let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); + let handler_type = self.component_handler_types.get(component_id).unwrap(); + let executor = trigger_config + .executor + .as_ref() + .unwrap_or(&HttpExecutorType::Http); + + let res = match executor { + HttpExecutorType::Http => match handler_type { + HandlerType::Spin => { + SpinHttpExecutor + .execute(instance_builder, &route_match, req, client_addr) + .await + } + HandlerType::Wasi0_2 + | HandlerType::Wasi2023_11_10 + | HandlerType::Wasi2023_10_18 => { + WasiHttpExecutor { + handler_type: *handler_type, + } + .execute(instance_builder, &route_match, req, client_addr) + .await + } + }, + HttpExecutorType::Wagi(wagi_config) => { + let executor = WagiHttpExecutor { + wagi_config: wagi_config.clone(), + }; + executor + .execute(instance_builder, &route_match, req, client_addr) + .await + } + }; match res { Ok(res) => Ok(MatchedRoute::with_response_extension( res, @@ -174,7 +243,7 @@ impl HttpServer { /// Returns spin status information. fn app_info(&self, route: String) -> anyhow::Result> { - let info = AppInfo::new(self.handler.trigger_app.app()); + let info = AppInfo::new(self.trigger_app.app()); let body = serde_json::to_vec_pretty(&info)?; Ok(MatchedRoute::with_response_extension( Response::builder() @@ -278,7 +347,7 @@ impl HttpServer { println!("Available Routes:"); for (route, component_id) in self.router.routes() { println!(" {}: {}{}", component_id, base_url, route); - if let Some(component) = self.handler.trigger_app.app().get_component(component_id) { + if let Some(component) = self.trigger_app.app().get_component(component_id) { if let Some(description) = component.get_metadata(APP_DESCRIPTION_KEY)? { println!(" {}", description); } @@ -288,111 +357,6 @@ impl HttpServer { } } -/// Handles a routed HTTP trigger request. -pub struct RequestHandler { - /// The address the server is listening on. - pub(crate) listen_addr: SocketAddr, - /// The app being triggered. - trigger_app: TriggerApp, - // Component ID -> component trigger config - component_trigger_configs: HashMap, - // Component ID -> handler type - component_handler_types: HashMap, -} - -impl RequestHandler { - /// Create a new [`RequestHandler`] - pub fn new( - listen_addr: SocketAddr, - trigger_app: TriggerApp, - component_trigger_configs: HashMap, - ) -> anyhow::Result { - let component_handler_types = component_trigger_configs - .keys() - .map(|component_id| { - let component = trigger_app.get_component(component_id)?; - let handler_type = HandlerType::from_component(trigger_app.engine(), component)?; - Ok((component_id.clone(), handler_type)) - }) - .collect::>()?; - Ok(Self { - listen_addr, - trigger_app, - component_trigger_configs, - component_handler_types, - }) - } - - /// Handle a routed request. - pub async fn handle_trigger_route( - self: &Arc, - mut req: Request, - route_match: &RouteMatch, - server_scheme: Scheme, - client_addr: SocketAddr, - ) -> anyhow::Result> { - set_req_uri(&mut req, server_scheme.clone())?; - let app_id = self - .trigger_app - .app() - .get_metadata(APP_NAME_KEY)? - .unwrap_or_else(|| "".into()); - - let component_id = route_match.component_id(); - - spin_telemetry::metrics::monotonic_counter!( - spin.request_count = 1, - trigger_type = "http", - app_id = app_id, - component_id = component_id - ); - - let mut instance_builder = self.trigger_app.prepare(component_id)?; - - // Set up outbound HTTP request origin and service chaining - let origin = SelfRequestOrigin::create(server_scheme, &self.listen_addr)?; - instance_builder - .factor_builders() - .outbound_http() - .set_request_interceptor(OutboundHttpInterceptor::new(self.clone(), origin))?; - - // Prepare HTTP executor - let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); - let handler_type = self.component_handler_types.get(component_id).unwrap(); - let executor = trigger_config - .executor - .as_ref() - .unwrap_or(&HttpExecutorType::Http); - - match executor { - HttpExecutorType::Http => match handler_type { - HandlerType::Spin => { - SpinHttpExecutor - .execute(instance_builder, route_match, req, client_addr) - .await - } - HandlerType::Wasi0_2 - | HandlerType::Wasi2023_11_10 - | HandlerType::Wasi2023_10_18 => { - WasiHttpExecutor { - handler_type: *handler_type, - } - .execute(instance_builder, route_match, req, client_addr) - .await - } - }, - HttpExecutorType::Wagi(wagi_config) => { - let executor = WagiHttpExecutor { - wagi_config: wagi_config.clone(), - }; - executor - .execute(instance_builder, route_match, req, client_addr) - .await - } - } - } -} - /// The incoming request's scheme and authority /// /// The incoming request's URI is relative to the server, so we need to set the scheme and authority. diff --git a/tests/conformance-tests/src/lib.rs b/tests/conformance-tests/src/lib.rs index bed1488a4..a03084819 100644 --- a/tests/conformance-tests/src/lib.rs +++ b/tests/conformance-tests/src/lib.rs @@ -1,5 +1,4 @@ use anyhow::Context as _; -use test_environment::http::Request; use testing_framework::runtimes::spin_cli::{SpinCli, SpinConfig}; /// Run a single conformance test against the supplied spin binary. @@ -57,11 +56,9 @@ pub fn run_test( let conformance_tests::config::Invocation::Http(mut invocation) = invocation; invocation.request.substitute_from_env(&mut env)?; let spin = env.runtime_mut(); - let actual = invocation.request.send(|request| { - let request = - Request::full(request.method, request.path, request.headers, request.body); - spin.make_http_request(request) - })?; + let actual = invocation + .request + .send(|request| spin.make_http_request(request))?; conformance_tests::assertions::assert_response(&invocation.response, &actual) .with_context(|| { diff --git a/tests/testing-framework/Cargo.toml b/tests/testing-framework/Cargo.toml index cc388246f..2118f93b3 100644 --- a/tests/testing-framework/Cargo.toml +++ b/tests/testing-framework/Cargo.toml @@ -14,12 +14,12 @@ regex = "1.10.2" reqwest = { workspace = true } temp-dir = "0.1.11" test-environment = { workspace = true } -spin-factors-executor = { path = "../../crates/factors-executor" } spin-app = { path = "../../crates/app" } -spin-trigger-http2 = { path = "../../crates/trigger-http2" } +spin-factors-executor = { path = "../../crates/factors-executor" } spin-http = { path = "../../crates/http" } -spin-trigger2 = { path = "../../crates/trigger2" } spin-loader = { path = "../../crates/loader" } +spin-trigger2 = { path = "../../crates/trigger2" } +spin-trigger-http2 = { path = "../../crates/trigger-http2" } toml = "0.8.6" tokio = "1.23" wasmtime-wasi-http = { workspace = true } diff --git a/tests/testing-framework/src/runtimes/in_process_spin.rs b/tests/testing-framework/src/runtimes/in_process_spin.rs index 16ad654af..7580842a2 100644 --- a/tests/testing-framework/src/runtimes/in_process_spin.rs +++ b/tests/testing-framework/src/runtimes/in_process_spin.rs @@ -3,7 +3,6 @@ use std::{path::PathBuf, sync::Arc}; use anyhow::Context as _; -use spin_http::routes::RouteMatch; use spin_trigger2::cli::{TriggerAppBuilder, TriggerAppOptions}; use spin_trigger_http2::{HttpServer, HttpTrigger}; use test_environment::{ @@ -54,12 +53,10 @@ impl InProcessSpin { } // TODO(rylev): convert body as well let req = builder.body(spin_http::body::empty()).unwrap(); - let route_match = RouteMatch::synthetic("test", "/"); let response = self .server - .handle_trigger_route( + .handle( req, - route_match, http::uri::Scheme::HTTP, (std::net::Ipv4Addr::LOCALHOST, 7000).into(), ) From 4237a81e6eebf94578a5a1cc52449479f359bf9a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 11:00:08 -0400 Subject: [PATCH 141/195] factors: Update outbound networking Signed-off-by: Lann Martin --- Cargo.lock | 188 ++++++++++++------ crates/factor-outbound-http/src/lib.rs | 11 +- crates/factor-outbound-http/src/wasi.rs | 65 +++--- .../factor-outbound-http/tests/factor_test.rs | 4 +- .../factor-outbound-mqtt/tests/factor_test.rs | 2 +- .../tests/factor_test.rs | 2 +- crates/factor-outbound-networking/src/lib.rs | 90 ++++++--- .../src/runtime_config.rs | 2 +- .../src/runtime_config/spin.rs | 7 +- .../tests/factor_test.rs | 4 +- .../factor-outbound-pg/tests/factor_test.rs | 2 +- .../tests/factor_test.rs | 2 +- crates/factors/tests/smoke.rs | 4 +- crates/outbound-networking/src/lib.rs | 37 ++-- crates/trigger2/Cargo.toml | 1 + crates/trigger2/src/factors.rs | 30 ++- 16 files changed, 294 insertions(+), 157 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20e822f0e..860bb418d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -566,7 +566,7 @@ dependencies = [ "pin-project-lite", "rustversion", "serde 1.0.197", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", @@ -606,7 +606,7 @@ dependencies = [ "paste", "pin-project", "rand 0.8.5", - "reqwest 0.12.4", + "reqwest 0.12.7", "rustc_version", "serde 1.0.197", "serde_json", @@ -1266,7 +1266,7 @@ dependencies = [ "num-traits 0.2.18", "serde 1.0.197", "wasm-bindgen", - "windows-targets 0.52.4", + "windows-targets 0.52.6", ] [[package]] @@ -1505,7 +1505,7 @@ dependencies = [ "flate2", "json5", "libtest-mimic 0.7.3", - "reqwest 0.12.4", + "reqwest 0.12.7", "serde 1.0.197", "tar", "test-environment", @@ -3472,6 +3472,23 @@ dependencies = [ "webpki-roots 0.26.1", ] +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.4.1", + "hyper-util", + "rustls 0.23.7", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -4022,7 +4039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.4", + "windows-targets 0.52.6", ] [[package]] @@ -5034,7 +5051,7 @@ dependencies = [ "lazy_static 1.4.0", "olpc-cjson", "regex", - "reqwest 0.12.4", + "reqwest 0.12.7", "serde 1.0.197", "serde_json", "sha2", @@ -5058,7 +5075,7 @@ dependencies = [ "lazy_static 1.4.0", "olpc-cjson", "regex", - "reqwest 0.12.4", + "reqwest 0.12.7", "serde 1.0.197", "serde_json", "sha2", @@ -6420,8 +6437,8 @@ dependencies = [ "serde 1.0.197", "serde_json", "serde_urlencoded", - "sync_wrapper", - "system-configuration", + "sync_wrapper 0.1.2", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", @@ -6433,14 +6450,14 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots 0.25.4", - "winreg 0.50.0", + "winreg", ] [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.0", "bytes", @@ -6453,6 +6470,7 @@ dependencies = [ "http-body 1.0.0", "http-body-util", "hyper 1.4.1", + "hyper-rustls 0.27.2", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -6467,8 +6485,8 @@ dependencies = [ "serde 1.0.197", "serde_json", "serde_urlencoded", - "sync_wrapper", - "system-configuration", + "sync_wrapper 1.0.1", + "system-configuration 0.6.0", "tokio", "tokio-native-tls", "tokio-socks", @@ -6479,7 +6497,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg 0.52.0", + "windows-registry", ] [[package]] @@ -7446,7 +7464,7 @@ dependencies = [ "rand 0.8.5", "redis 0.24.0", "regex", - "reqwest 0.12.4", + "reqwest 0.12.7", "rpassword", "runtime-tests", "semver", @@ -8470,6 +8488,7 @@ dependencies = [ "spin-factors-executor", "spin-runtime-config", "spin-telemetry", + "terminal", "tokio", "tracing", ] @@ -8640,6 +8659,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.12.6" @@ -8660,7 +8688,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -8673,6 +8712,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-interface" version = "0.27.2" @@ -8794,7 +8843,7 @@ dependencies = [ "anyhow", "fslock", "regex", - "reqwest 0.12.4", + "reqwest 0.12.7", "temp-dir", "tokio", ] @@ -8810,7 +8859,7 @@ dependencies = [ "log", "nix 0.26.4", "regex", - "reqwest 0.12.4", + "reqwest 0.12.7", "spin-app", "spin-factors-executor", "spin-http", @@ -9724,7 +9773,7 @@ dependencies = [ "once_cell", "pathdiff", "ptree", - "reqwest 0.12.4", + "reqwest 0.12.7", "secrecy", "semver", "serde 1.0.197", @@ -10051,7 +10100,7 @@ dependencies = [ "anyhow", "dirs 5.0.1", "http 1.1.0", - "reqwest 0.12.4", + "reqwest 0.12.7", "semver", "serde 1.0.197", "serde_json", @@ -10747,7 +10796,37 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", ] [[package]] @@ -10774,7 +10853,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.6", ] [[package]] @@ -10809,17 +10888,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -10836,9 +10916,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -10854,9 +10934,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -10872,9 +10952,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -10890,9 +10976,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -10908,9 +10994,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -10926,9 +11012,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -10944,9 +11030,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -10976,16 +11062,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "winx" version = "0.36.3" diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 7db483bb0..3e72a3763 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -25,7 +25,16 @@ pub use wasmtime_wasi_http::{ HttpResult, }; -pub struct OutboundHttpFactor; +#[derive(Default)] +pub struct OutboundHttpFactor { + _priv: (), +} + +impl OutboundHttpFactor { + pub fn new() -> Self { + Self::default() + } +} impl Factor for OutboundHttpFactor { type RuntimeConfig = (); diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index 9f0fffeeb..8fb0a17d8 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -1,13 +1,13 @@ use std::{error::Error, sync::Arc}; use anyhow::Context; -use http::{header::HOST, uri::Authority, Request, Uri}; +use http::{header::HOST, Request}; use http_body_util::BodyExt; use rustls::ClientConfig; -use spin_factor_outbound_networking::{OutboundAllowedHosts, OutboundUrl}; +use spin_factor_outbound_networking::OutboundAllowedHosts; use spin_factors::{wasmtime::component::ResourceTable, RuntimeFactorsInstanceState}; use tokio::{net::TcpStream, time::timeout}; -use tracing::Instrument; +use tracing::{field::Empty, instrument, Instrument}; use wasmtime_wasi_http::{ bindings::http::types::ErrorCode, body::HyperOutgoingBody, @@ -68,6 +68,19 @@ impl<'a> WasiHttpView for WasiHttpImplInner<'a> { self.table } + #[instrument( + name = "spin_outbound_http.send_request", + skip_all, + fields( + otel.kind = "client", + url.full = %request.uri(), + http.request.method = %request.method(), + otel.name = %request.method(), + http.response.status_code = Empty, + server.address = Empty, + server.port = Empty, + ), + )] fn send_request( &mut self, mut request: Request, @@ -104,15 +117,24 @@ impl<'a> WasiHttpView for WasiHttpImplInner<'a> { async fn send_request_impl( mut request: Request, mut config: wasmtime_wasi_http::types::OutgoingRequestConfig, - allowed_hosts: OutboundAllowedHosts, + outbound_allowed_hosts: OutboundAllowedHosts, tls_client_config: Arc, ) -> anyhow::Result> { - let allowed_hosts = allowed_hosts.resolve().await?; - - let is_relative_url = request.uri().authority().is_none(); - if is_relative_url { + if request.uri().authority().is_some() { + // Absolute URI + let is_allowed = outbound_allowed_hosts + .check_url(&request.uri().to_string(), "https") + .await + .map_err(|_| ErrorCode::HttpRequestUriInvalid)?; + if !is_allowed { + return Ok(Err(ErrorCode::HttpRequestDenied)); + } + } else { + // Relative URI ("self" request) + let allowed_hosts = outbound_allowed_hosts.resolve().await?; if !allowed_hosts.allows_relative_url(&["http", "https"]) { - return Ok(handle_not_allowed(request.uri(), true)); + outbound_allowed_hosts.report_disallowed_host("http", "self"); + return Ok(Err(ErrorCode::HttpRequestDenied)); } let origin = request @@ -127,12 +149,6 @@ async fn send_request_impl( let path_and_query = request.uri().path_and_query().cloned(); *request.uri_mut() = origin.into_uri(path_and_query); - } else { - let outbound_url = OutboundUrl::parse(request.uri().to_string(), "https") - .map_err(|_| ErrorCode::HttpRequestUriInvalid)?; - if !allowed_hosts.allows(&outbound_url) { - return Ok(handle_not_allowed(request.uri(), false)); - } } if let Some(authority) = request.uri().authority() { @@ -146,25 +162,6 @@ async fn send_request_impl( Ok(send_request_handler(request, config, tls_client_config).await) } -// TODO(factors): Move to some callback on spin-factor-outbound-networking (?) -fn handle_not_allowed(uri: &Uri, is_relative: bool) -> Result { - tracing::error!("Destination not allowed!: {uri}"); - let allowed_host_example = if is_relative { - terminal::warn!("A component tried to make a HTTP request to the same component but it does not have permission."); - "http://self".to_string() - } else { - let host = format!( - "{scheme}://{authority}", - scheme = uri.scheme_str().unwrap_or_default(), - authority = uri.authority().map(Authority::as_str).unwrap_or_default() - ); - terminal::warn!("A component tried to make a HTTP request to non-allowed host '{host}'."); - host - }; - eprintln!("To allow requests, add 'allowed_outbound_hosts = [\"{allowed_host_example}\"]' to the manifest component section."); - Err(ErrorCode::HttpRequestDenied) -} - /// This is a fork of wasmtime_wasi_http::default_send_request_handler function /// forked from bytecodealliance/wasmtime commit-sha 29a76b68200fcfa69c8fb18ce6c850754279a05b /// This fork provides the ability to configure client cert auth for mTLS diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index 3de9ac7c1..caa1b65c0 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -77,8 +77,8 @@ async fn test_instance_state( ) -> anyhow::Result { let factors = TestFactors { variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, - http: OutboundHttpFactor, + networking: OutboundNetworkingFactor::new(), + http: OutboundHttpFactor::new(), }; let env = TestEnvironment::new(factors).extend_manifest(toml! { [component.test-component] diff --git a/crates/factor-outbound-mqtt/tests/factor_test.rs b/crates/factor-outbound-mqtt/tests/factor_test.rs index 1d88b3e57..e507d6520 100644 --- a/crates/factor-outbound-mqtt/tests/factor_test.rs +++ b/crates/factor-outbound-mqtt/tests/factor_test.rs @@ -46,7 +46,7 @@ struct TestFactors { fn factors() -> TestFactors { TestFactors { variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, + networking: OutboundNetworkingFactor::new(), mqtt: OutboundMqttFactor::new(Arc::new(MockMqttClient {})), } } diff --git a/crates/factor-outbound-mysql/tests/factor_test.rs b/crates/factor-outbound-mysql/tests/factor_test.rs index d392a0dff..b9261377b 100644 --- a/crates/factor-outbound-mysql/tests/factor_test.rs +++ b/crates/factor-outbound-mysql/tests/factor_test.rs @@ -20,7 +20,7 @@ struct TestFactors { fn factors() -> TestFactors { TestFactors { variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, + networking: OutboundNetworkingFactor::new(), mysql: OutboundMysqlFactor::::new(), } } diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index abf270bb0..f5e1c783e 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -22,7 +22,24 @@ pub use runtime_config::ComponentTlsConfigs; pub type SharedFutureResult = Shared, Arc>>>; -pub struct OutboundNetworkingFactor; +#[derive(Default)] +pub struct OutboundNetworkingFactor { + disallowed_host_callback: Option, +} + +pub type DisallowedHostCallback = fn(scheme: &str, authority: &str); + +impl OutboundNetworkingFactor { + pub fn new() -> Self { + Self::default() + } + + /// Sets a function to be called when a request is disallowed by an + /// instance's configured `allowed_outbound_hosts`. + pub fn set_disallowed_host_callback(&mut self, callback: DisallowedHostCallback) { + self.disallowed_host_callback = Some(callback); + } +} impl Factor for OutboundNetworkingFactor { type RuntimeConfig = RuntimeConfig; @@ -87,26 +104,24 @@ impl Factor for OutboundNetworkingFactor { match builders.get_mut::() { Ok(wasi_builder) => { // Update Wasi socket allowed ports - let hosts_future = allowed_hosts_future.clone(); + let allowed_hosts = OutboundAllowedHosts { + allowed_hosts_future: allowed_hosts_future.clone(), + disallowed_host_callback: self.disallowed_host_callback, + }; wasi_builder.outbound_socket_addr_check(move |addr, addr_use| { - let hosts_future = hosts_future.clone(); + let allowed_hosts = allowed_hosts.clone(); async move { - match hosts_future.await { - Ok(allowed_hosts) => { - // TODO: validate against existing spin-core behavior - let scheme = match addr_use { - SocketAddrUse::TcpBind => return false, - SocketAddrUse::TcpConnect => "tcp", - SocketAddrUse::UdpBind | SocketAddrUse::UdpConnect | SocketAddrUse::UdpOutgoingDatagram => "udp", - }; - spin_outbound_networking::check_url(&addr.to_string(),scheme, &allowed_hosts) - } - Err(err) => { - // TODO: should this trap (somehow)? - tracing::error!(%err, "allowed_outbound_hosts variable resolution failed"); - false - } - } + // TODO: validate against existing spin-core behavior + let scheme = match addr_use { + SocketAddrUse::TcpBind => return false, + SocketAddrUse::TcpConnect => "tcp", + SocketAddrUse::UdpBind | SocketAddrUse::UdpConnect | SocketAddrUse::UdpOutgoingDatagram => "udp", + }; + allowed_hosts.check_url(&addr.to_string(), scheme).await.unwrap_or_else(|err| { + // TODO: should this trap (somehow)? + tracing::error!(%err, "allowed_outbound_hosts variable resolution failed"); + false + }) } }); } @@ -122,6 +137,7 @@ impl Factor for OutboundNetworkingFactor { Ok(InstanceBuilder { allowed_hosts_future, component_tls_configs, + disallowed_host_callback: self.disallowed_host_callback, }) } } @@ -134,12 +150,14 @@ pub struct AppState { pub struct InstanceBuilder { allowed_hosts_future: SharedFutureResult, component_tls_configs: ComponentTlsConfigs, + disallowed_host_callback: Option, } impl InstanceBuilder { pub fn allowed_hosts(&self) -> OutboundAllowedHosts { OutboundAllowedHosts { allowed_hosts_future: self.allowed_hosts_future.clone(), + disallowed_host_callback: self.disallowed_host_callback, } } @@ -160,6 +178,7 @@ impl FactorInstanceBuilder for InstanceBuilder { #[derive(Clone)] pub struct OutboundAllowedHosts { allowed_hosts_future: SharedFutureResult, + disallowed_host_callback: Option, } impl OutboundAllowedHosts { @@ -170,16 +189,39 @@ impl OutboundAllowedHosts { }) } + /// Checks if the given URL is allowed by this component's + /// `allowed_outbound_hosts`. pub async fn allows(&self, url: &OutboundUrl) -> anyhow::Result { Ok(self.resolve().await?.allows(url)) } + /// Report that an outbound connection has been disallowed by e.g. + /// [`OutboundAllowedHosts::allows`] returning `false`. + /// + /// Calls the [`DisallowedHostCallback`] if set. + pub fn report_disallowed_host(&self, scheme: &str, authority: &str) { + if let Some(disallowed_host_callback) = self.disallowed_host_callback { + disallowed_host_callback(scheme, authority); + } + } + + /// Checks address against allowed hosts + /// + /// Calls the [`DisallowedHostCallback`] if set and URL is disallowed. pub async fn check_url(&self, url: &str, scheme: &str) -> anyhow::Result { + let Ok(url) = OutboundUrl::parse(url, scheme) else { + tracing::warn!( + "A component tried to make a request to a url that could not be parsed: {url}", + ); + return Ok(false); + }; + let allowed_hosts = self.resolve().await?; - Ok(spin_outbound_networking::check_url( - url, - scheme, - &allowed_hosts, - )) + + let is_allowed = allowed_hosts.allows(&url); + if !is_allowed { + self.report_disallowed_host(url.scheme(), &url.authority()); + } + Ok(is_allowed) } } diff --git a/crates/factor-outbound-networking/src/runtime_config.rs b/crates/factor-outbound-networking/src/runtime_config.rs index 9ea5d7d8f..fad06edd1 100644 --- a/crates/factor-outbound-networking/src/runtime_config.rs +++ b/crates/factor-outbound-networking/src/runtime_config.rs @@ -253,7 +253,7 @@ mod tests { Ok(()) } - const TESTDATA_DIR: &'static str = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata"); + const TESTDATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata"); fn test_certs() -> anyhow::Result>> { let file = std::fs::File::open(Path::new(TESTDATA_DIR).join("valid-cert.pem"))?; diff --git a/crates/factor-outbound-networking/src/runtime_config/spin.rs b/crates/factor-outbound-networking/src/runtime_config/spin.rs index 580d0e6c7..c8e2a3fc5 100644 --- a/crates/factor-outbound-networking/src/runtime_config/spin.rs +++ b/crates/factor-outbound-networking/src/runtime_config/spin.rs @@ -154,7 +154,10 @@ impl SpinTlsRuntimeConfig { .ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidInput, - format!("private key file '{}' contains no private keys", path.display()), + format!( + "private key file '{}' contains no private keys", + path.display() + ), ) })?) } @@ -184,7 +187,7 @@ fn deserialize_hosts<'de, D: Deserializer<'de>>(deserializer: D) -> Result anyhow::Result<()> { diff --git a/crates/factor-outbound-networking/tests/factor_test.rs b/crates/factor-outbound-networking/tests/factor_test.rs index 693ac6600..d4bd1a51c 100644 --- a/crates/factor-outbound-networking/tests/factor_test.rs +++ b/crates/factor-outbound-networking/tests/factor_test.rs @@ -17,7 +17,7 @@ async fn configures_wasi_socket_addr_check() -> anyhow::Result<()> { let factors = TestFactors { wasi: WasiFactor::new(DummyFilesMounter), variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, + networking: OutboundNetworkingFactor::new(), }; let env = TestEnvironment::new(factors).extend_manifest(toml! { [component.test-component] @@ -58,7 +58,7 @@ async fn wasi_factor_is_optional() -> anyhow::Result<()> { } TestEnvironment::new(WithoutWasi { variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, + networking: OutboundNetworkingFactor::new(), }) .build_instance_state() .await?; diff --git a/crates/factor-outbound-pg/tests/factor_test.rs b/crates/factor-outbound-pg/tests/factor_test.rs index e189b9d2a..b765d805f 100644 --- a/crates/factor-outbound-pg/tests/factor_test.rs +++ b/crates/factor-outbound-pg/tests/factor_test.rs @@ -21,7 +21,7 @@ struct TestFactors { fn factors() -> TestFactors { TestFactors { variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, + networking: OutboundNetworkingFactor::new(), pg: OutboundPgFactor::::new(), } } diff --git a/crates/factor-outbound-redis/tests/factor_test.rs b/crates/factor-outbound-redis/tests/factor_test.rs index 70a09f0be..6f2f7b051 100644 --- a/crates/factor-outbound-redis/tests/factor_test.rs +++ b/crates/factor-outbound-redis/tests/factor_test.rs @@ -17,7 +17,7 @@ struct TestFactors { async fn no_outbound_hosts_fails() -> anyhow::Result<()> { let factors = TestFactors { variables: VariablesFactor::default(), - networking: OutboundNetworkingFactor, + networking: OutboundNetworkingFactor::new(), redis: OutboundRedisFactor::new(), }; let env = TestEnvironment::new(factors).extend_manifest(toml! { diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 9925c59b6..3b31c701d 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -60,8 +60,8 @@ async fn smoke_test_works() -> anyhow::Result<()> { let mut factors = Factors { wasi: WasiFactor::new(DummyFilesMounter), variables: VariablesFactor::default(), - outbound_networking: OutboundNetworkingFactor, - outbound_http: OutboundHttpFactor, + outbound_networking: OutboundNetworkingFactor::new(), + outbound_http: OutboundHttpFactor::new(), key_value: KeyValueFactor::new(key_value_resolver.clone()), }; diff --git a/crates/outbound-networking/src/lib.rs b/crates/outbound-networking/src/lib.rs index c2c3095b8..aec793419 100644 --- a/crates/outbound-networking/src/lib.rs +++ b/crates/outbound-networking/src/lib.rs @@ -8,31 +8,6 @@ pub const ALLOWED_HOSTS_KEY: MetadataKey> = MetadataKey::new("allowe pub const SERVICE_CHAINING_DOMAIN: &str = "spin.internal"; pub const SERVICE_CHAINING_DOMAIN_SUFFIX: &str = ".spin.internal"; -/// Checks address against allowed hosts -/// -/// Emits several warnings -pub fn check_url(url: &str, scheme: &str, allowed_hosts: &AllowedHostsConfig) -> bool { - let Ok(url) = OutboundUrl::parse(url, scheme) else { - terminal::warn!( - "A component tried to make a request to an url that could not be parsed {url}.", - ); - return false; - }; - let is_allowed = allowed_hosts.allows(&url); - - if !is_allowed { - terminal::warn!("A component tried to make a request to non-allowed url '{url}'."); - let (scheme, host, port) = (url.scheme, url.host, url.port); - let msg = if let Some(port) = port { - format!("`allowed_outbound_hosts = [\"{scheme}://{host}:{port}\"]`") - } else { - format!("`allowed_outbound_hosts = [\"{scheme}://{host}:$PORT\"]` (where $PORT is the correct port number)") - }; - eprintln!("To allow requests, add {msg} to the manifest component section."); - } - is_allowed -} - /// An address is a url-like string that contains a host, a port, and an optional scheme #[derive(Eq, Debug, Clone)] pub struct AllowedHostConfig { @@ -431,6 +406,18 @@ impl OutboundUrl { original, }) } + + pub fn scheme(&self) -> &str { + &self.scheme + } + + pub fn authority(&self) -> String { + if let Some(port) = self.port { + format!("{}:{port}", self.host) + } else { + self.host.clone() + } + } } impl std::fmt::Display for OutboundUrl { diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 70a7588ac..a98909534 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -32,6 +32,7 @@ spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } tokio = { version = "1.23", features = ["fs"] } +terminal = { path = "../terminal" } tracing = { workspace = true } [lints] diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index 897151863..e4719e506 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -28,19 +28,41 @@ impl TriggerFactors { default_key_value_label_resolver: impl spin_factor_key_value::DefaultLabelResolver + 'static, default_sqlite_label_resolver: impl spin_factor_sqlite::DefaultLabelResolver + 'static, ) -> Self { - let files_mounter = SpinFilesMounter::new(working_dir, allow_transient_writes); Self { - wasi: WasiFactor::new(files_mounter), + wasi: wasi_factor(working_dir, allow_transient_writes), variables: VariablesFactor::default(), key_value: KeyValueFactor::new(default_key_value_label_resolver), - outbound_networking: OutboundNetworkingFactor, - outbound_http: OutboundHttpFactor, + outbound_networking: outbound_networking_factor(), + outbound_http: OutboundHttpFactor::new(), sqlite: SqliteFactor::new(default_sqlite_label_resolver), redis: OutboundRedisFactor::new(), } } } +fn wasi_factor(working_dir: impl Into, allow_transient_writes: bool) -> WasiFactor { + WasiFactor::new(SpinFilesMounter::new(working_dir, allow_transient_writes)) +} + +fn outbound_networking_factor() -> OutboundNetworkingFactor { + fn disallowed_host_callback(scheme: &str, authority: &str) { + let host_pattern = format!("{scheme}://{authority}"); + tracing::error!("Outbound network destination not allowed: {host_pattern}"); + if scheme.starts_with("http") && authority == "self" { + terminal::warn!("A component tried to make an HTTP request to its own app but it does not have permission."); + } else { + terminal::warn!( + "A component tried to make an outbound network connection to disallowed destination '{host_pattern}'." + ); + }; + eprintln!("To allow this request, add 'allowed_outbound_hosts = [\"{host_pattern}\"]' to the manifest component section."); + } + + let mut factor = OutboundNetworkingFactor::new(); + factor.set_disallowed_host_callback(disallowed_host_callback); + factor +} + impl TryFrom> for TriggerFactorsRuntimeConfig { type Error = anyhow::Error; From a8b4c8a05ec49f27e76d155948ed83e4395f8c56 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 11:36:35 -0400 Subject: [PATCH 142/195] factors: Refactor OutboundHttpFactor SelfRequestOrigin Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/lib.rs | 10 ++++++++++ crates/factor-outbound-http/src/wasi.rs | 7 +++---- crates/factor-outbound-http/tests/factor_test.rs | 9 ++++----- crates/trigger-http2/src/outbound_http.rs | 8 +++----- crates/trigger-http2/src/server.rs | 7 +++---- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index 3e72a3763..b0c18baf8 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -69,6 +69,7 @@ impl Factor for OutboundHttpFactor { wasi_http_ctx: WasiHttpCtx::new(), allowed_hosts, component_tls_configs, + self_request_origin: None, request_interceptor: None, }) } @@ -78,10 +79,19 @@ pub struct InstanceState { wasi_http_ctx: WasiHttpCtx, allowed_hosts: OutboundAllowedHosts, component_tls_configs: ComponentTlsConfigs, + self_request_origin: Option, request_interceptor: Option>, } impl InstanceState { + /// Sets the [`SelfRequestOrigin`] for this instance. + /// + /// This is used to handle outbound requests to relative URLs. If unset, + /// those requests will fail. + pub fn set_self_request_origin(&mut self, origin: SelfRequestOrigin) { + self.self_request_origin = Some(origin); + } + /// Sets a [`OutboundHttpInterceptor`] for this instance. /// /// Returns an error if it has already been called for this instance. diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index 8fb0a17d8..925c2f9f9 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -106,6 +106,7 @@ impl<'a> WasiHttpView for WasiHttpImplInner<'a> { request, config, self.state.allowed_hosts.clone(), + self.state.self_request_origin.clone(), tls_client_config, ) .in_current_span(), @@ -118,6 +119,7 @@ async fn send_request_impl( mut request: Request, mut config: wasmtime_wasi_http::types::OutgoingRequestConfig, outbound_allowed_hosts: OutboundAllowedHosts, + self_request_origin: Option, tls_client_config: Arc, ) -> anyhow::Result> { if request.uri().authority().is_some() { @@ -137,10 +139,7 @@ async fn send_request_impl( return Ok(Err(ErrorCode::HttpRequestDenied)); } - let origin = request - .extensions() - .get::() - .cloned() + let origin = self_request_origin .context("cannot send relative outbound request; no 'origin' set by host")?; config.use_tls = origin.use_tls(); diff --git a/crates/factor-outbound-http/tests/factor_test.rs b/crates/factor-outbound-http/tests/factor_test.rs index caa1b65c0..42bfe7c5a 100644 --- a/crates/factor-outbound-http/tests/factor_test.rs +++ b/crates/factor-outbound-http/tests/factor_test.rs @@ -40,12 +40,11 @@ async fn allowed_host_is_allowed() -> anyhow::Result<()> { #[tokio::test] async fn self_request_smoke_test() -> anyhow::Result<()> { let mut state = test_instance_state("http://self").await?; - let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); + let origin = SelfRequestOrigin::from_uri(&Uri::from_static("http://[100::1]"))?; + state.http.set_self_request_origin(origin); - let mut req = Request::get("/self-request").body(Default::default())?; - let origin = Uri::from_static("http://[100::1]"); - req.extensions_mut() - .insert(SelfRequestOrigin::from_uri(&origin).unwrap()); + let mut wasi_http = OutboundHttpFactor::get_wasi_http_impl(&mut state).unwrap(); + let req = Request::get("/self-request").body(Default::default())?; let mut future_resp = wasi_http.send_request(req, test_request_config())?; future_resp.ready().await; diff --git a/crates/trigger-http2/src/outbound_http.rs b/crates/trigger-http2/src/outbound_http.rs index 45e63ab75..a722bc315 100644 --- a/crates/trigger-http2/src/outbound_http.rs +++ b/crates/trigger-http2/src/outbound_http.rs @@ -5,7 +5,7 @@ use std::{ use http::uri::Scheme; use spin_factor_outbound_http::{ - HostFutureIncomingResponse, InterceptOutcome, OutgoingRequestConfig, Request, SelfRequestOrigin, + HostFutureIncomingResponse, InterceptOutcome, OutgoingRequestConfig, Request, }; use spin_http::routes::RouteMatch; use spin_outbound_networking::parse_service_chaining_target; @@ -16,12 +16,11 @@ use crate::HttpServer; /// An outbound HTTP interceptor that handles service chaining requests. pub struct OutboundHttpInterceptor { server: Arc, - origin: SelfRequestOrigin, } impl OutboundHttpInterceptor { - pub fn new(server: Arc, origin: SelfRequestOrigin) -> Self { - Self { server, origin } + pub fn new(server: Arc) -> Self { + Self { server } } } @@ -58,7 +57,6 @@ impl spin_factor_outbound_http::OutboundHttpInterceptor for OutboundHttpIntercep let resp = HostFutureIncomingResponse::pending(wasmtime_wasi::runtime::spawn(resp_fut)); InterceptOutcome::Complete(Ok(resp)) } else { - request.extensions_mut().insert(self.origin.clone()); InterceptOutcome::Continue } } diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index c01327fda..3923bc9c4 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -188,11 +188,10 @@ impl HttpServer { let mut instance_builder = self.trigger_app.prepare(component_id)?; // Set up outbound HTTP request origin and service chaining + let outbound_http = instance_builder.factor_builders().outbound_http(); let origin = SelfRequestOrigin::create(server_scheme, &self.listen_addr)?; - instance_builder - .factor_builders() - .outbound_http() - .set_request_interceptor(OutboundHttpInterceptor::new(self.clone(), origin))?; + outbound_http.set_self_request_origin(origin); + outbound_http.set_request_interceptor(OutboundHttpInterceptor::new(self.clone()))?; // Prepare HTTP executor let trigger_config = self.component_trigger_configs.get(component_id).unwrap(); From 5c5e794dbff66e0b09464e9ffc393985316498bb Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 21 Aug 2024 17:22:17 +0200 Subject: [PATCH 143/195] Switch back to the old sync version of Host{Output,Input}Stream. Signed-off-by: Ryan Levick --- Cargo.lock | 1 + crates/factor-wasi/Cargo.toml | 1 + crates/factor-wasi/src/io.rs | 126 ++++++++++++++++++++++++++++++++++ crates/factor-wasi/src/lib.rs | 39 +++++------ 4 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 crates/factor-wasi/src/io.rs diff --git a/Cargo.lock b/Cargo.lock index aed3f025a..ec45b0220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7796,6 +7796,7 @@ name = "spin-factor-wasi" version = "2.8.0-pre0" dependencies = [ "async-trait", + "bytes", "cap-primitives 3.0.0", "spin-common", "spin-factors", diff --git a/crates/factor-wasi/Cargo.toml b/crates/factor-wasi/Cargo.toml index b35bfc04e..6201ac272 100644 --- a/crates/factor-wasi/Cargo.toml +++ b/crates/factor-wasi/Cargo.toml @@ -6,6 +6,7 @@ edition = { workspace = true } [dependencies] async-trait = "0.1" +bytes = "1.0" cap-primitives = "3.0.0" spin-common = { path = "../common" } spin-factors = { path = "../factors" } diff --git a/crates/factor-wasi/src/io.rs b/crates/factor-wasi/src/io.rs new file mode 100644 index 000000000..f5dd0c2b5 --- /dev/null +++ b/crates/factor-wasi/src/io.rs @@ -0,0 +1,126 @@ +use std::io::{Read, Write}; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use spin_factors::anyhow; +use wasmtime_wasi::{ + HostInputStream, HostOutputStream, StdinStream, StdoutStream, StreamError, Subscribe, +}; + +/// A [`HostOutputStream`] that writes to a `Write` type. +/// +/// `StdinStream::stream` and `StdoutStream::new` can be called more than once in components +/// which are composed of multiple subcomponents, since each subcomponent will potentially want +/// its own handle. This means the streams need to be shareable. The easiest way to do that is +/// provide cloneable implementations of streams which operate synchronously. +/// +/// Note that this amounts to doing synchronous I/O in an asynchronous context, which we'd normally +/// prefer to avoid, but the properly asynchronous implementations Host{In|Out}putStream based on +/// `AsyncRead`/`AsyncWrite`` are quite hairy and probably not worth it for "normal" stdio streams in +/// Spin. If this does prove to be a performance bottleneck, though, we can certainly revisit it. +pub struct PipedWriteStream(Arc>); + +impl PipedWriteStream { + pub fn new(inner: T) -> Self { + Self(Arc::new(Mutex::new(inner))) + } +} + +impl Clone for PipedWriteStream { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl HostOutputStream for PipedWriteStream { + fn write(&mut self, bytes: bytes::Bytes) -> Result<(), StreamError> { + self.0 + .lock() + .unwrap() + .write_all(&bytes) + .map_err(|e| StreamError::LastOperationFailed(anyhow::anyhow!(e))) + } + + fn flush(&mut self) -> Result<(), StreamError> { + self.0 + .lock() + .unwrap() + .flush() + .map_err(|e| StreamError::LastOperationFailed(anyhow::anyhow!(e))) + } + + fn check_write(&mut self) -> Result { + Ok(1024 * 1024) + } +} + +impl StdoutStream for PipedWriteStream { + fn stream(&self) -> Box { + Box::new(self.clone()) + } + + fn isatty(&self) -> bool { + false + } +} + +#[async_trait] +impl Subscribe for PipedWriteStream { + async fn ready(&mut self) {} +} + +/// A [`HostInputStream`] that reads to a `Read` type. +/// +/// See [`PipedWriteStream`] for more information on why this is synchronous. +pub struct PipeReadStream { + buffer: Vec, + inner: Arc>, +} + +impl PipeReadStream { + pub fn new(inner: T) -> Self { + Self { + buffer: vec![0_u8; 64 * 1024], + inner: Arc::new(Mutex::new(inner)), + } + } +} + +impl Clone for PipeReadStream { + fn clone(&self) -> Self { + Self { + buffer: vec![0_u8; 64 * 1024], + inner: self.inner.clone(), + } + } +} + +impl HostInputStream for PipeReadStream { + fn read(&mut self, size: usize) -> wasmtime_wasi::StreamResult { + let size = size.min(self.buffer.len()); + + let count = self + .inner + .lock() + .unwrap() + .read(&mut self.buffer[..size]) + .map_err(|e| StreamError::LastOperationFailed(anyhow::anyhow!(e)))?; + + Ok(bytes::Bytes::copy_from_slice(&self.buffer[..count])) + } +} + +#[async_trait] +impl Subscribe for PipeReadStream { + async fn ready(&mut self) {} +} + +impl StdinStream for PipeReadStream { + fn stream(&self) -> Box { + Box::new(self.clone()) + } + + fn isatty(&self) -> bool { + false + } +} diff --git a/crates/factor-wasi/src/lib.rs b/crates/factor-wasi/src/lib.rs index b7cc8a90f..885f96a79 100644 --- a/crates/factor-wasi/src/lib.rs +++ b/crates/factor-wasi/src/lib.rs @@ -1,18 +1,23 @@ +mod io; pub mod spin; mod wasi_2023_10_18; mod wasi_2023_11_10; -use std::{future::Future, net::SocketAddr, path::Path}; +use std::{ + future::Future, + io::{Read, Write}, + net::SocketAddr, + path::Path, +}; +use io::{PipeReadStream, PipedWriteStream}; use spin_factors::{ anyhow, AppComponent, Factor, FactorInstanceBuilder, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, RuntimeFactorsInstanceState, }; -use tokio::io::{AsyncRead, AsyncWrite}; use wasmtime_wasi::{ - pipe::{AsyncReadStream, AsyncWriteStream}, - AsyncStdinStream, AsyncStdoutStream, DirPerms, FilePerms, ResourceTable, StdinStream, - StdoutStream, WasiCtx, WasiCtxBuilder, WasiImpl, WasiView, + DirPerms, FilePerms, ResourceTable, StdinStream, StdoutStream, WasiCtx, WasiCtxBuilder, + WasiImpl, WasiView, }; pub use wasmtime_wasi::SocketAddrUse; @@ -179,9 +184,9 @@ impl InstanceBuilder { self.ctx.stdin(stdin); } - /// Sets the WASI `stdin` descriptor to the given [`AsyncRead`]er. - pub fn stdin_pipe(&mut self, r: impl AsyncRead + Send + Unpin + 'static) { - self.stdin(AsyncStdinStream::new(AsyncReadStream::new(r))); + /// Sets the WASI `stdin` descriptor to the given [`Read`]er. + pub fn stdin_pipe(&mut self, r: impl Read + Send + Sync + Unpin + 'static) { + self.stdin(PipeReadStream::new(r)); } /// Sets the WASI `stdout` descriptor to the given [`StdoutStream`]. @@ -189,12 +194,9 @@ impl InstanceBuilder { self.ctx.stdout(stdout); } - /// Sets the WASI `stdout` descriptor to the given [`AsyncWrite`]r. - pub fn stdout_pipe(&mut self, w: impl AsyncWrite + Send + Unpin + 'static) { - self.stdout(AsyncStdoutStream::new(AsyncWriteStream::new( - 1024 * 1024, - w, - ))); + /// Sets the WASI `stdout` descriptor to the given [`Write`]r. + pub fn stdout_pipe(&mut self, w: impl Write + Send + Sync + Unpin + 'static) { + self.stdout(PipedWriteStream::new(w)); } /// Sets the WASI `stderr` descriptor to the given [`StdoutStream`]. @@ -202,12 +204,9 @@ impl InstanceBuilder { self.ctx.stderr(stderr); } - /// Sets the WASI `stderr` descriptor to the given [`AsyncWrite`]r. - pub fn stderr_pipe(&mut self, w: impl AsyncWrite + Send + Unpin + 'static) { - self.stderr(AsyncStdoutStream::new(AsyncWriteStream::new( - 1024 * 1024, - w, - ))); + /// Sets the WASI `stderr` descriptor to the given [`Write`]r. + pub fn stderr_pipe(&mut self, w: impl Write + Send + Sync + Unpin + 'static) { + self.stderr(PipedWriteStream::new(w)); } /// Appends the given strings to the WASI 'args'. From 6ee514d6d06a9db333bf59c42dd5a401e487fe39 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 21 Aug 2024 18:55:40 +0200 Subject: [PATCH 144/195] Integrate mqtt into trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 2 ++ crates/factor-outbound-mqtt/src/host.rs | 11 +------ crates/factor-outbound-mqtt/src/lib.rs | 42 ++++++++++++++++++++++++- crates/runtime-config/Cargo.toml | 1 + crates/runtime-config/src/lib.rs | 7 +++++ crates/trigger2/Cargo.toml | 7 +++-- crates/trigger2/src/factors.rs | 3 ++ 7 files changed, 59 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e09144361..4d54df716 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8151,6 +8151,7 @@ dependencies = [ "spin-factor-key-value-redis", "spin-factor-key-value-spin", "spin-factor-outbound-http", + "spin-factor-outbound-mqtt", "spin-factor-outbound-networking", "spin-factor-outbound-redis", "spin-factor-sqlite", @@ -8462,6 +8463,7 @@ dependencies = [ "spin-core", "spin-factor-key-value", "spin-factor-outbound-http", + "spin-factor-outbound-mqtt", "spin-factor-outbound-networking", "spin-factor-outbound-redis", "spin-factor-sqlite", diff --git a/crates/factor-outbound-mqtt/src/host.rs b/crates/factor-outbound-mqtt/src/host.rs index 3cd22abbd..460b888c1 100644 --- a/crates/factor-outbound-mqtt/src/host.rs +++ b/crates/factor-outbound-mqtt/src/host.rs @@ -6,16 +6,7 @@ use spin_factor_outbound_networking::OutboundAllowedHosts; use spin_world::v2::mqtt::{self as v2, Connection, Error, Qos}; use tracing::{instrument, Level}; -#[async_trait] -pub trait ClientCreator: Send + Sync { - fn create( - &self, - address: String, - username: String, - password: String, - keep_alive_interval: Duration, - ) -> Result, Error>; -} +use crate::ClientCreator; pub struct InstanceState { allowed_hosts: OutboundAllowedHosts, diff --git a/crates/factor-outbound-mqtt/src/lib.rs b/crates/factor-outbound-mqtt/src/lib.rs index 4816e12be..21e4242f9 100644 --- a/crates/factor-outbound-mqtt/src/lib.rs +++ b/crates/factor-outbound-mqtt/src/lib.rs @@ -15,7 +15,7 @@ use spin_factors::{ use spin_world::v2::mqtt::{self as v2, Error, Qos}; use tokio::sync::Mutex; -pub use host::{ClientCreator, MqttClient}; +pub use host::MqttClient; pub struct OutboundMqttFactor { create_client: Arc, @@ -73,6 +73,19 @@ pub struct NetworkedMqttClient { const MQTT_CHANNEL_CAP: usize = 1000; impl NetworkedMqttClient { + /// Create a [`ClientCreator`] that creates a [`NetworkedMqttClient`]. + pub fn creator() -> Arc { + Arc::new(|address, username, password, keep_alive_interval| { + Ok(Arc::new(NetworkedMqttClient::create( + address, + username, + password, + keep_alive_interval, + )?) as _) + }) + } + + /// Create a new [`NetworkedMqttClient`] with the given address, username, password, and keep alive interval. pub fn create( address: String, username: String, @@ -127,3 +140,30 @@ impl MqttClient for NetworkedMqttClient { Ok(()) } } + +/// A trait for creating MQTT client. +#[async_trait] +pub trait ClientCreator: Send + Sync { + fn create( + &self, + address: String, + username: String, + password: String, + keep_alive_interval: Duration, + ) -> Result, Error>; +} + +impl ClientCreator for F +where + F: Fn(String, String, String, Duration) -> Result, Error> + Send + Sync, +{ + fn create( + &self, + address: String, + username: String, + password: String, + keep_alive_interval: Duration, + ) -> Result, Error> { + self(address, username, password, keep_alive_interval) + } +} diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index e67129a5e..6954a6dac 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -16,6 +16,7 @@ spin-factor-key-value-spin = { path = "../factor-key-value-spin" } spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-key-value-azure = { path = "../factor-key-value-azure" } spin-factor-outbound-http = { path = "../factor-outbound-http" } +spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-sqlite = { path = "../factor-sqlite" } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index d4df03c73..b6dead06c 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -4,6 +4,7 @@ use anyhow::Context as _; use spin_factor_key_value::runtime_config::spin::{self as key_value, MakeKeyValueStore}; use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; use spin_factor_outbound_http::OutboundHttpFactor; +use spin_factor_outbound_mqtt::OutboundMqttFactor; use spin_factor_outbound_networking::runtime_config::spin::SpinTlsRuntimeConfig; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_redis::OutboundRedisFactor; @@ -176,6 +177,12 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<' } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(None) + } +} + impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { self.sqlite.resolve_from_toml(self.table.as_ref()) diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 70a7588ac..d6246eda8 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -21,13 +21,14 @@ spin-app = { path = "../app" } spin-common = { path = "../common" } spin-componentize = { path = "../componentize" } spin-core = { path = "../core" } +spin-factor-key-value = { path = "../factor-key-value" } spin-factor-outbound-http = { path = "../factor-outbound-http" } +spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-outbound-redis = { path = "../factor-outbound-redis" } +spin-factor-sqlite = { path = "../factor-sqlite" } spin-factor-variables = { path = "../factor-variables" } spin-factor-wasi = { path = "../factor-wasi" } -spin-factor-key-value = { path = "../factor-key-value" } -spin-factor-sqlite = { path = "../factor-sqlite" } -spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factors = { path = "../factors" } spin-factors-executor = { path = "../factors-executor" } spin-telemetry = { path = "../telemetry" } diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index 897151863..dd98bb474 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use spin_factor_key_value::KeyValueFactor; use spin_factor_outbound_http::OutboundHttpFactor; +use spin_factor_outbound_mqtt::{NetworkedMqttClient, OutboundMqttFactor}; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_redis::OutboundRedisFactor; use spin_factor_sqlite::SqliteFactor; @@ -19,6 +20,7 @@ pub struct TriggerFactors { pub outbound_http: OutboundHttpFactor, pub sqlite: SqliteFactor, pub redis: OutboundRedisFactor, + pub mqtt: OutboundMqttFactor, } impl TriggerFactors { @@ -37,6 +39,7 @@ impl TriggerFactors { outbound_http: OutboundHttpFactor, sqlite: SqliteFactor::new(default_sqlite_label_resolver), redis: OutboundRedisFactor::new(), + mqtt: OutboundMqttFactor::new(NetworkedMqttClient::creator()), } } } From f09aaffa6457eb1d7b2ff80b729c72dcde8fcd41 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 12:46:14 -0400 Subject: [PATCH 145/195] Address review feedback Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/spin.rs | 4 +- crates/factor-outbound-http/src/wasi.rs | 27 ++++--- crates/factor-outbound-networking/src/lib.rs | 81 +++++++++++--------- crates/trigger2/src/factors.rs | 4 +- 4 files changed, 64 insertions(+), 52 deletions(-) diff --git a/crates/factor-outbound-http/src/spin.rs b/crates/factor-outbound-http/src/spin.rs index 22ea723f2..c9dd6f186 100644 --- a/crates/factor-outbound-http/src/spin.rs +++ b/crates/factor-outbound-http/src/spin.rs @@ -1,4 +1,3 @@ -use spin_factor_outbound_networking::OutboundUrl; use spin_world::{ async_trait, v1::http, @@ -9,8 +8,7 @@ use spin_world::{ impl http::Host for crate::InstanceState { async fn send_request(&mut self, req: Request) -> Result { // FIXME(lann): This is all just a stub to test allowed_outbound_hosts - let outbound_url = OutboundUrl::parse(&req.uri, "https").or(Err(HttpError::InvalidUrl))?; - match self.allowed_hosts.allows(&outbound_url).await { + match self.allowed_hosts.check_url(&req.uri, "https").await { Ok(true) => (), _ => { return Err(HttpError::DestinationNotAllowed); diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index 925c2f9f9..e63f40b53 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -127,20 +127,24 @@ async fn send_request_impl( let is_allowed = outbound_allowed_hosts .check_url(&request.uri().to_string(), "https") .await - .map_err(|_| ErrorCode::HttpRequestUriInvalid)?; + .unwrap_or(false); if !is_allowed { return Ok(Err(ErrorCode::HttpRequestDenied)); } } else { // Relative URI ("self" request) - let allowed_hosts = outbound_allowed_hosts.resolve().await?; - if !allowed_hosts.allows_relative_url(&["http", "https"]) { - outbound_allowed_hosts.report_disallowed_host("http", "self"); + let is_allowed = outbound_allowed_hosts + .check_relative_url(&["http", "https"]) + .await + .unwrap_or(false); + if !is_allowed { return Ok(Err(ErrorCode::HttpRequestDenied)); } - let origin = self_request_origin - .context("cannot send relative outbound request; no 'origin' set by host")?; + let Some(origin) = self_request_origin else { + tracing::error!("Couldn't handle outbound HTTP request to relative URI; no origin set"); + return Ok(Err(ErrorCode::HttpRequestUriInvalid)); + }; config.use_tls = origin.use_tls(); @@ -150,12 +154,11 @@ async fn send_request_impl( *request.uri_mut() = origin.into_uri(path_and_query); } - if let Some(authority) = request.uri().authority() { - let current_span = tracing::Span::current(); - current_span.record("server.address", authority.host()); - if let Some(port) = authority.port() { - current_span.record("server.port", port.as_u16()); - } + let authority = request.uri().authority().context("authority not set")?; + let current_span = tracing::Span::current(); + current_span.record("server.address", authority.host()); + if let Some(port) = authority.port() { + current_span.record("server.port", port.as_u16()); } Ok(send_request_handler(request, config, tls_client_config).await) diff --git a/crates/factor-outbound-networking/src/lib.rs b/crates/factor-outbound-networking/src/lib.rs index f5e1c783e..7af5f34f1 100644 --- a/crates/factor-outbound-networking/src/lib.rs +++ b/crates/factor-outbound-networking/src/lib.rs @@ -24,20 +24,18 @@ pub type SharedFutureResult = Shared, Arc, + disallowed_host_handler: Option>, } -pub type DisallowedHostCallback = fn(scheme: &str, authority: &str); - impl OutboundNetworkingFactor { pub fn new() -> Self { Self::default() } - /// Sets a function to be called when a request is disallowed by an + /// Sets a handler to be called when a request is disallowed by an /// instance's configured `allowed_outbound_hosts`. - pub fn set_disallowed_host_callback(&mut self, callback: DisallowedHostCallback) { - self.disallowed_host_callback = Some(callback); + pub fn set_disallowed_host_handler(&mut self, handler: impl DisallowedHostHandler + 'static) { + self.disallowed_host_handler = Some(Arc::new(handler)); } } @@ -106,7 +104,7 @@ impl Factor for OutboundNetworkingFactor { // Update Wasi socket allowed ports let allowed_hosts = OutboundAllowedHosts { allowed_hosts_future: allowed_hosts_future.clone(), - disallowed_host_callback: self.disallowed_host_callback, + disallowed_host_handler: self.disallowed_host_handler.clone(), }; wasi_builder.outbound_socket_addr_check(move |addr, addr_use| { let allowed_hosts = allowed_hosts.clone(); @@ -137,7 +135,7 @@ impl Factor for OutboundNetworkingFactor { Ok(InstanceBuilder { allowed_hosts_future, component_tls_configs, - disallowed_host_callback: self.disallowed_host_callback, + disallowed_host_handler: self.disallowed_host_handler.clone(), }) } } @@ -150,14 +148,14 @@ pub struct AppState { pub struct InstanceBuilder { allowed_hosts_future: SharedFutureResult, component_tls_configs: ComponentTlsConfigs, - disallowed_host_callback: Option, + disallowed_host_handler: Option>, } impl InstanceBuilder { pub fn allowed_hosts(&self) -> OutboundAllowedHosts { OutboundAllowedHosts { allowed_hosts_future: self.allowed_hosts_future.clone(), - disallowed_host_callback: self.disallowed_host_callback, + disallowed_host_handler: self.disallowed_host_handler.clone(), } } @@ -178,33 +176,10 @@ impl FactorInstanceBuilder for InstanceBuilder { #[derive(Clone)] pub struct OutboundAllowedHosts { allowed_hosts_future: SharedFutureResult, - disallowed_host_callback: Option, + disallowed_host_handler: Option>, } impl OutboundAllowedHosts { - pub async fn resolve(&self) -> anyhow::Result> { - self.allowed_hosts_future.clone().await.map_err(|err| { - // TODO: better way to handle this? - anyhow::Error::msg(err) - }) - } - - /// Checks if the given URL is allowed by this component's - /// `allowed_outbound_hosts`. - pub async fn allows(&self, url: &OutboundUrl) -> anyhow::Result { - Ok(self.resolve().await?.allows(url)) - } - - /// Report that an outbound connection has been disallowed by e.g. - /// [`OutboundAllowedHosts::allows`] returning `false`. - /// - /// Calls the [`DisallowedHostCallback`] if set. - pub fn report_disallowed_host(&self, scheme: &str, authority: &str) { - if let Some(disallowed_host_callback) = self.disallowed_host_callback { - disallowed_host_callback(scheme, authority); - } - } - /// Checks address against allowed hosts /// /// Calls the [`DisallowedHostCallback`] if set and URL is disallowed. @@ -217,11 +192,47 @@ impl OutboundAllowedHosts { }; let allowed_hosts = self.resolve().await?; - let is_allowed = allowed_hosts.allows(&url); if !is_allowed { self.report_disallowed_host(url.scheme(), &url.authority()); } Ok(is_allowed) } + + /// Checks if allowed hosts permit relative requests + /// + /// Calls the [`DisallowedHostCallback`] if set and relative requests are + /// disallowed. + pub async fn check_relative_url(&self, schemes: &[&str]) -> anyhow::Result { + let allowed_hosts = self.resolve().await?; + let is_allowed = allowed_hosts.allows_relative_url(schemes); + if !is_allowed { + let scheme = schemes.first().unwrap_or(&""); + self.report_disallowed_host(scheme, "self"); + } + Ok(is_allowed) + } + + async fn resolve(&self) -> anyhow::Result> { + self.allowed_hosts_future.clone().await.map_err(|err| { + tracing::error!("Error resolving allowed_outbound_hosts variables: {err}"); + anyhow::Error::msg(err) + }) + } + + fn report_disallowed_host(&self, scheme: &str, authority: &str) { + if let Some(handler) = &self.disallowed_host_handler { + handler.handle_disallowed_host(scheme, authority); + } + } +} + +pub trait DisallowedHostHandler: Send + Sync { + fn handle_disallowed_host(&self, scheme: &str, authority: &str); +} + +impl DisallowedHostHandler for F { + fn handle_disallowed_host(&self, scheme: &str, authority: &str) { + self(scheme, authority); + } } diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index e4719e506..42b2dae13 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -45,7 +45,7 @@ fn wasi_factor(working_dir: impl Into, allow_transient_writes: bool) -> } fn outbound_networking_factor() -> OutboundNetworkingFactor { - fn disallowed_host_callback(scheme: &str, authority: &str) { + fn disallowed_host_handler(scheme: &str, authority: &str) { let host_pattern = format!("{scheme}://{authority}"); tracing::error!("Outbound network destination not allowed: {host_pattern}"); if scheme.starts_with("http") && authority == "self" { @@ -59,7 +59,7 @@ fn outbound_networking_factor() -> OutboundNetworkingFactor { } let mut factor = OutboundNetworkingFactor::new(); - factor.set_disallowed_host_callback(disallowed_host_callback); + factor.set_disallowed_host_handler(disallowed_host_handler); factor } From 29d2cac5b5641b2bfcc179107e628b9dc3946e58 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 14:09:11 -0400 Subject: [PATCH 146/195] factors: Implement spin outbound http Signed-off-by: Lann Martin --- Cargo.lock | 2 + crates/factor-outbound-http/Cargo.toml | 2 + crates/factor-outbound-http/src/lib.rs | 9 ++ crates/factor-outbound-http/src/spin.rs | 174 ++++++++++++++++++++++-- 4 files changed, 175 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61d790215..450782507 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7655,11 +7655,13 @@ dependencies = [ "http 1.1.0", "http-body-util", "hyper 1.4.1", + "reqwest 0.11.27", "rustls 0.23.7", "spin-factor-outbound-networking", "spin-factor-variables", "spin-factors", "spin-factors-test", + "spin-telemetry", "spin-world", "terminal", "tokio", diff --git a/crates/factor-outbound-http/Cargo.toml b/crates/factor-outbound-http/Cargo.toml index cee32e483..e62c6c52f 100644 --- a/crates/factor-outbound-http/Cargo.toml +++ b/crates/factor-outbound-http/Cargo.toml @@ -9,9 +9,11 @@ anyhow = "1.0" http = "1.1.0" http-body-util = "0.1" hyper = "1.4.1" +reqwest = { version = "0.11", features = ["gzip"] } rustls = "0.23" spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factors = { path = "../factors" } +spin-telemetry = { path = "../telemetry" } spin-world = { path = "../world" } terminal = { path = "../terminal" } tokio = { version = "1", features = ["macros", "rt"] } diff --git a/crates/factor-outbound-http/src/lib.rs b/crates/factor-outbound-http/src/lib.rs index b0c18baf8..739be2ab9 100644 --- a/crates/factor-outbound-http/src/lib.rs +++ b/crates/factor-outbound-http/src/lib.rs @@ -71,6 +71,7 @@ impl Factor for OutboundHttpFactor { component_tls_configs, self_request_origin: None, request_interceptor: None, + spin_http_client: None, }) } } @@ -81,6 +82,8 @@ pub struct InstanceState { component_tls_configs: ComponentTlsConfigs, self_request_origin: Option, request_interceptor: Option>, + // Connection-pooling client for 'fermyon:spin/http' interface + spin_http_client: Option, } impl InstanceState { @@ -156,6 +159,12 @@ impl SelfRequestOrigin { } } +impl std::fmt::Display for SelfRequestOrigin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}://{}", self.scheme, self.authority) + } +} + /// An outbound HTTP request interceptor to be used with /// [`InstanceState::set_request_interceptor`]. pub trait OutboundHttpInterceptor: Send + Sync { diff --git a/crates/factor-outbound-http/src/spin.rs b/crates/factor-outbound-http/src/spin.rs index c9dd6f186..32b69f522 100644 --- a/crates/factor-outbound-http/src/spin.rs +++ b/crates/factor-outbound-http/src/spin.rs @@ -1,24 +1,84 @@ use spin_world::{ async_trait, - v1::http, - v1::http_types::{self, HttpError, Request, Response}, + v1::{ + http as spin_http, + http_types::{self, HttpError, Method, Request, Response}, + }, }; +use tracing::{field::Empty, instrument, Level, Span}; #[async_trait] -impl http::Host for crate::InstanceState { +impl spin_http::Host for crate::InstanceState { + #[instrument(name = "spin_outbound_http.send_request", skip_all, err(level = Level::INFO), + fields(otel.kind = "client", url.full = Empty, http.request.method = Empty, + http.response.status_code = Empty, otel.name = Empty, server.address = Empty, server.port = Empty))] async fn send_request(&mut self, req: Request) -> Result { - // FIXME(lann): This is all just a stub to test allowed_outbound_hosts - match self.allowed_hosts.check_url(&req.uri, "https").await { - Ok(true) => (), - _ => { + let span = Span::current(); + record_request_fields(&span, &req); + + let uri = req.uri; + tracing::trace!("Sending outbound HTTP to {uri:?}"); + + let abs_url = if uri.starts_with('/') { + // Relative URI ("self" request) + let is_allowed = self + .allowed_hosts + .check_relative_url(&["http", "https"]) + .await + .unwrap_or(false); + if !is_allowed { return Err(HttpError::DestinationNotAllowed); } + + let Some(origin) = &self.self_request_origin else { + tracing::error!( + "Couldn't handle outbound HTTP request to relative URI; no origin set" + ); + return Err(HttpError::InvalidUrl); + }; + format!("{origin}{uri}") + } else { + // Absolute URI + let is_allowed = self + .allowed_hosts + .check_url(&uri, "https") + .await + .unwrap_or(false); + if !is_allowed { + return Err(HttpError::DestinationNotAllowed); + } + uri + }; + let req_url = reqwest::Url::parse(&abs_url).map_err(|_| HttpError::InvalidUrl)?; + + if !req.params.is_empty() { + tracing::warn!("HTTP params field is deprecated"); } - Ok(Response { - status: 200, - headers: None, - body: Some(b"test response".into()), - }) + + // Allow reuse of Client's internal connection pool for multiple requests + // in a single component execution + let client = self.spin_http_client.get_or_insert_with(Default::default); + + let mut req = { + let mut builder = client.request(reqwest_method(req.method), req_url); + for (key, val) in req.headers { + builder = builder.header(key, val); + } + builder + .body(req.body.unwrap_or_default()) + .build() + .map_err(|err| { + tracing::error!("Error building outbound request: {err}"); + HttpError::RuntimeError + })? + }; + spin_telemetry::inject_trace_context(req.headers_mut()); + + let resp = client.execute(req).await.map_err(log_reqwest_error)?; + + tracing::trace!("Returning response from outbound request to {abs_url}"); + span.record("http.response.status_code", resp.status().as_u16()); + response_from_reqwest(resp).await } } @@ -27,3 +87,93 @@ impl http_types::Host for crate::InstanceState { Ok(err) } } + +fn record_request_fields(span: &Span, req: &Request) { + let method = match req.method { + Method::Get => "GET", + Method::Post => "POST", + Method::Put => "PUT", + Method::Delete => "DELETE", + Method::Patch => "PATCH", + Method::Head => "HEAD", + Method::Options => "OPTIONS", + }; + span.record("otel.name", method) + .record("http.request.method", method) + .record("url.full", req.uri.clone()); + if let Ok(uri) = req.uri.parse::() { + if let Some(authority) = uri.authority() { + span.record("server.address", authority.host()); + if let Some(port) = authority.port() { + span.record("server.port", port.as_u16()); + } + } + } +} + +fn reqwest_method(m: Method) -> reqwest::Method { + match m { + Method::Get => reqwest::Method::GET, + Method::Post => reqwest::Method::POST, + Method::Put => reqwest::Method::PUT, + Method::Delete => reqwest::Method::DELETE, + Method::Patch => reqwest::Method::PATCH, + Method::Head => reqwest::Method::HEAD, + Method::Options => reqwest::Method::OPTIONS, + } +} + +fn log_reqwest_error(err: reqwest::Error) -> HttpError { + let error_desc = if err.is_timeout() { + "timeout error" + } else if err.is_connect() { + "connection error" + } else if err.is_body() || err.is_decode() { + "message body error" + } else if err.is_request() { + "request error" + } else { + "error" + }; + tracing::warn!( + "Outbound HTTP {}: URL {}, error detail {:?}", + error_desc, + err.url() + .map(|u| u.to_string()) + .unwrap_or_else(|| "".to_owned()), + err + ); + HttpError::RuntimeError +} + +async fn response_from_reqwest(res: reqwest::Response) -> Result { + let status = res.status().as_u16(); + + let headers = res + .headers() + .into_iter() + .map(|(key, val)| { + Ok(( + key.to_string(), + val.to_str() + .map_err(|_| { + tracing::error!("Non-ascii response header {key} = {val:?}"); + HttpError::RuntimeError + })? + .to_string(), + )) + }) + .collect::, _>>()?; + + let body = res + .bytes() + .await + .map_err(|_| HttpError::RuntimeError)? + .to_vec(); + + Ok(Response { + status, + headers: Some(headers), + body: Some(body), + }) +} From 951760c76ad767cbe1b308a3f95bffa05961ab40 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 14:09:32 -0400 Subject: [PATCH 147/195] factors: Re-add trace context injection to wasi outbound http Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/wasi.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index e63f40b53..b67565247 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -149,6 +149,7 @@ async fn send_request_impl( config.use_tls = origin.use_tls(); request.headers_mut().insert(HOST, origin.host_header()); + spin_telemetry::inject_trace_context(&mut request); let path_and_query = request.uri().path_and_query().cloned(); *request.uri_mut() = origin.into_uri(path_and_query); From e9da68a22040743a263d5e52bb9bb55c58a42217 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 15:25:46 -0400 Subject: [PATCH 148/195] factors: Invert if statement for consistency Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/spin.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/factor-outbound-http/src/spin.rs b/crates/factor-outbound-http/src/spin.rs index 32b69f522..633df727d 100644 --- a/crates/factor-outbound-http/src/spin.rs +++ b/crates/factor-outbound-http/src/spin.rs @@ -19,7 +19,18 @@ impl spin_http::Host for crate::InstanceState { let uri = req.uri; tracing::trace!("Sending outbound HTTP to {uri:?}"); - let abs_url = if uri.starts_with('/') { + let abs_url = if !uri.starts_with('/') { + // Absolute URI + let is_allowed = self + .allowed_hosts + .check_url(&uri, "https") + .await + .unwrap_or(false); + if !is_allowed { + return Err(HttpError::DestinationNotAllowed); + } + uri + } else { // Relative URI ("self" request) let is_allowed = self .allowed_hosts @@ -37,17 +48,6 @@ impl spin_http::Host for crate::InstanceState { return Err(HttpError::InvalidUrl); }; format!("{origin}{uri}") - } else { - // Absolute URI - let is_allowed = self - .allowed_hosts - .check_url(&uri, "https") - .await - .unwrap_or(false); - if !is_allowed { - return Err(HttpError::DestinationNotAllowed); - } - uri }; let req_url = reqwest::Url::parse(&abs_url).map_err(|_| HttpError::InvalidUrl)?; From 07a238af4b81865a529aa438d142a35bc8c9fcd2 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 15:40:22 -0400 Subject: [PATCH 149/195] factors: Fix some outbound http tests Signed-off-by: Lann Martin --- crates/factor-outbound-http/src/wasi.rs | 15 +++ examples/spin-timer/Cargo.lock | 94 +------------------ tests/integration.rs | 32 +++---- tests/runtime-tests/tests/llm/spin.toml | 2 +- .../internal-http-middle/src/lib.rs | 4 - .../components/outbound-http/src/lib.rs | 2 +- tests/testcases/http-smoke-test/spin.toml | 2 +- tests/testcases/key-value/spin.toml | 2 +- tests/testcases/otel-smoke-test/spin.toml | 2 +- .../outbound-http-to-same-app/spin.toml | 2 +- tests/testcases/simple-test/spin.toml | 2 +- tests/testcases/spin-inbound-http/spin.toml | 3 - tests/testcases/wagi-http/spin.toml | 3 - 13 files changed, 39 insertions(+), 126 deletions(-) diff --git a/crates/factor-outbound-http/src/wasi.rs b/crates/factor-outbound-http/src/wasi.rs index b67565247..8d49bad2a 100644 --- a/crates/factor-outbound-http/src/wasi.rs +++ b/crates/factor-outbound-http/src/wasi.rs @@ -86,6 +86,21 @@ impl<'a> WasiHttpView for WasiHttpImplInner<'a> { mut request: Request, mut config: wasmtime_wasi_http::types::OutgoingRequestConfig, ) -> wasmtime_wasi_http::HttpResult { + // wasmtime-wasi-http fills in scheme and authority for relative URLs + // (e.g. https://:443/), which makes them hard to reason about. + // Undo that here. + let uri = request.uri_mut(); + if uri + .authority() + .is_some_and(|authority| authority.host().is_empty()) + { + let mut builder = http::uri::Builder::new(); + if let Some(paq) = uri.path_and_query() { + builder = builder.path_and_query(paq.clone()); + } + *uri = builder.build().unwrap(); + } + if let Some(interceptor) = &self.state.request_interceptor { match interceptor.intercept(&mut request, &mut config) { InterceptOutcome::Continue => (), diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index b9d18f476..425b5c4b7 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -58,12 +58,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "allocator-api2" version = "0.2.16" @@ -3923,31 +3917,6 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -[[package]] -name = "ouroboros" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b7be5a8a3462b752f4be3ff2b2bf2f7f1d00834902e46be2a4d68b87b0573c" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645dcde5f119c2c454a92d0dfa271a2a3b205da92e4292a68ead4bdbfde1f33" -dependencies = [ - "heck 0.4.1", - "itertools 0.12.1", - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.48", -] - [[package]] name = "outbound-http" version = "2.8.0-pre0" @@ -4470,19 +4439,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "proc-macro2-diagnostics" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", - "version_check", - "yansi", -] - [[package]] name = "prost" version = "0.12.6" @@ -5643,10 +5599,8 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "async-trait", - "ouroboros", "serde 1.0.203", "serde_json", - "spin-core", "spin-locked-app", "spin-serde", "thiserror", @@ -5683,20 +5637,8 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "async-trait", - "bytes", - "cap-primitives", - "cap-std", - "http 1.1.0", - "io-extras", - "rustix 0.37.27", - "spin-telemetry", - "system-interface", - "tokio", "tracing", - "wasi-common", "wasmtime", - "wasmtime-wasi", - "wasmtime-wasi-http", ] [[package]] @@ -5814,7 +5756,6 @@ dependencies = [ "itertools 0.10.5", "lazy_static 1.4.0", "mime_guess", - "outbound-http", "path-absolutize", "regex", "reqwest 0.11.24", @@ -5844,7 +5785,6 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "async-trait", - "ouroboros", "serde 1.0.203", "serde_json", "spin-serde", @@ -6032,6 +5972,7 @@ dependencies = [ name = "spin-world" version = "2.8.0-pre0" dependencies = [ + "async-trait", "wasmtime", ] @@ -7132,33 +7073,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasi-common" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86fd41e1e26ff6af9451c6a332a5ce5f5283ca51e87d875cdd9a05305598ee3" -dependencies = [ - "anyhow", - "bitflags 2.4.2", - "cap-fs-ext", - "cap-rand", - "cap-std", - "cap-time-ext", - "fs-set-times", - "io-extras", - "io-lifetimes 2.0.3", - "log", - "once_cell", - "rustix 0.38.31", - "system-interface", - "thiserror", - "tokio", - "tracing", - "wasmtime", - "wiggle", - "windows-sys 0.52.0", -] - [[package]] name = "wasite" version = "0.1.0" @@ -8218,12 +8132,6 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "zbus" version = "3.15.2" diff --git a/tests/integration.rs b/tests/integration.rs index ee869aeef..2c4b8ad04 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -72,12 +72,12 @@ mod integration_tests { let spin = env.runtime_mut(); assert_spin_request( spin, - Request::new(Method::Get, "/test/hello"), + Request::new(Method::Get, "/hello"), Response::new_with_body(200, "I'm a teapot"), )?; assert_spin_request( spin, - Request::new(Method::Get, "/test/hello/wildcards/should/be/handled"), + Request::new(Method::Get, "/hello/wildcards/should/be/handled"), Response::new_with_body(200, "I'm a teapot"), )?; assert_spin_request( @@ -87,7 +87,7 @@ mod integration_tests { )?; assert_spin_request( spin, - Request::new(Method::Get, "/test/hello/test-placement"), + Request::new(Method::Get, "/hello/test-placement"), Response::new_with_body(200, "text for test"), ) }, @@ -183,7 +183,7 @@ mod integration_tests { let spin = env.runtime_mut(); assert_spin_request( spin, - Request::new(Method::Get, "/test/hello"), + Request::new(Method::Get, "/hello"), Response::new_with_body(200, "Hello, Fermyon!\n"), )?; @@ -368,13 +368,13 @@ Caused by: let spin = env.runtime_mut(); assert_spin_request( spin, - Request::new(Method::Get, "/test/outbound-allowed"), + Request::new(Method::Get, "/outbound-allowed"), Response::new_with_body(200, "Hello, Fermyon!\n"), )?; assert_spin_request( spin, - Request::new(Method::Get, "/test/outbound-not-allowed"), + Request::new(Method::Get, "/outbound-not-allowed"), Response::new_with_body( 500, "Error::UnexpectedError(\"ErrorCode::HttpRequestDenied\")", @@ -421,14 +421,14 @@ Caused by: Response::new_with_body(expected_status, expected_body), ) }; - ensure_success("/test/hello", 200, "I'm a teapot")?; + ensure_success("/hello", 200, "I'm a teapot")?; ensure_success( - "/test/hello/wildcards/should/be/handled", + "/hello/wildcards/should/be/handled", 200, "I'm a teapot", )?; ensure_success("/thisshouldfail", 404, "")?; - ensure_success("/test/hello/test-placement", 200, "text for test")?; + ensure_success("/hello/test-placement", 200, "text for test")?; Ok(()) }, )?; @@ -1255,14 +1255,14 @@ route = "/..." let spin = env.runtime_mut(); assert_spin_request( spin, - Request::full(Method::Get, "/base/echo", &[], Some("Echo...")), + Request::full(Method::Get, "/echo", &[], Some("Echo...")), Response::new_with_body(200, "Echo..."), )?; assert_spin_request( spin, Request::full( Method::Get, - "/base/assert-headers?k=v", + "/assert-headers?k=v", &[("X-Custom-Foo", "bar")], Some(r#"{"x-custom-foo": "bar"}"#), ), @@ -1288,16 +1288,16 @@ route = "/..." let spin = env.runtime_mut(); assert_spin_request( spin, - Request::full(Method::Get, "/base/echo", &[], Some("Echo...")), + Request::full(Method::Get, "/echo", &[], Some("Echo...")), Response::new_with_body(200, "Echo..."), )?; assert_spin_request( spin, Request::full( Method::Get, - "/base/assert-args?x=y", + "/assert-args?x=y", &[], - Some(r#"["/base/assert-args", "x=y"]"#), + Some(r#"["/assert-args", "x=y"]"#), ), Response::new(200), )?; @@ -1305,7 +1305,7 @@ route = "/..." spin, Request::full( Method::Get, - "/base/assert-env", + "/assert-env", &[("X-Custom-Foo", "bar")], Some(r#"{"HTTP_X_CUSTOM_FOO": "bar"}"#), ), @@ -1464,7 +1464,7 @@ route = "/..." spin, Request::full( Method::Get, - "/test/outbound-allowed/hello", + "/outbound-allowed/hello", &[("Host", "google.com")], Some(""), ), diff --git a/tests/runtime-tests/tests/llm/spin.toml b/tests/runtime-tests/tests/llm/spin.toml index 9a1e18e1f..bee13ef4a 100644 --- a/tests/runtime-tests/tests/llm/spin.toml +++ b/tests/runtime-tests/tests/llm/spin.toml @@ -2,7 +2,7 @@ spin_manifest_version = "1" authors = ["Ryan Levick "] description = "" name = "ai" -trigger = { type = "http", base = "/" } +trigger = { type = "http" } version = "0.1.0" [[component]] diff --git a/tests/test-components/components/internal-http-middle/src/lib.rs b/tests/test-components/components/internal-http-middle/src/lib.rs index 32160ab55..6095422f0 100644 --- a/tests/test-components/components/internal-http-middle/src/lib.rs +++ b/tests/test-components/components/internal-http-middle/src/lib.rs @@ -16,10 +16,6 @@ async fn handle_middle_impl(req: Request) -> Result { .header("spin-path-info") .and_then(|v| v.as_str()); let inbound_rel_path = ensure_some!(inbound_rel_path); - let inbound_base = req - .header("spin-base-path") - .and_then(|v| v.as_str()); - ensure_eq!("/", ensure_some!(inbound_base)); let out_req = spin_sdk::http::Request::builder() .uri("https://back.spin.internal/hello/from/middle") diff --git a/tests/test-components/components/outbound-http/src/lib.rs b/tests/test-components/components/outbound-http/src/lib.rs index 5799ac667..d9155ab84 100644 --- a/tests/test-components/components/outbound-http/src/lib.rs +++ b/tests/test-components/components/outbound-http/src/lib.rs @@ -10,7 +10,7 @@ async fn send_outbound(_req: Request) -> Result { let mut res: http::Response = spin_sdk::http::send( http::Request::builder() .method("GET") - .uri("/test/hello") + .uri("/hello") .body(())?, ) .await?; diff --git a/tests/testcases/http-smoke-test/spin.toml b/tests/testcases/http-smoke-test/spin.toml index b12a0e28d..c0340f9e2 100644 --- a/tests/testcases/http-smoke-test/spin.toml +++ b/tests/testcases/http-smoke-test/spin.toml @@ -2,7 +2,7 @@ spin_version = "1" authors = ["Fermyon Engineering "] description = "A simple application that returns hello and goodbye." name = "head-rust-sdk-http" -trigger = { type = "http", base = "/test" } +trigger = { type = "http" } version = "1.0.0" [variables] diff --git a/tests/testcases/key-value/spin.toml b/tests/testcases/key-value/spin.toml index d24146717..3202eb21d 100644 --- a/tests/testcases/key-value/spin.toml +++ b/tests/testcases/key-value/spin.toml @@ -2,7 +2,7 @@ spin_version = "1" authors = ["Fermyon Engineering "] description = "A simple application that exercises key/value storage." name = "key-value" -trigger = { type = "http", base = "/test" } +trigger = { type = "http" } version = "1.0.0" [[component]] diff --git a/tests/testcases/otel-smoke-test/spin.toml b/tests/testcases/otel-smoke-test/spin.toml index c5911038b..a4eb09f67 100644 --- a/tests/testcases/otel-smoke-test/spin.toml +++ b/tests/testcases/otel-smoke-test/spin.toml @@ -2,7 +2,7 @@ spin_version = "1" authors = ["Fermyon Engineering "] description = "A simple application that returns hello and goodbye." name = "head-rust-sdk-http" -trigger = { type = "http", base = "/test" } +trigger = { type = "http" } version = "1.0.0" [[component]] diff --git a/tests/testcases/outbound-http-to-same-app/spin.toml b/tests/testcases/outbound-http-to-same-app/spin.toml index 90e517f55..8f241d7e6 100644 --- a/tests/testcases/outbound-http-to-same-app/spin.toml +++ b/tests/testcases/outbound-http-to-same-app/spin.toml @@ -2,7 +2,7 @@ spin_version = "1" authors = ["Fermyon Engineering "] description = "An application that demonstates a component making an outbound http request to another component in the same application." name = "local-outbound-http" -trigger = { type = "http", base = "/test" } +trigger = { type = "http" } version = "1.0.0" [[component]] diff --git a/tests/testcases/simple-test/spin.toml b/tests/testcases/simple-test/spin.toml index e069317e1..26fd2fa70 100644 --- a/tests/testcases/simple-test/spin.toml +++ b/tests/testcases/simple-test/spin.toml @@ -2,7 +2,7 @@ spin_version = "1" authors = ["Fermyon Engineering "] description = "A simple application that returns hello and goodbye." name = "spin-hello-world" -trigger = { type = "http", base = "/test" } +trigger = { type = "http" } version = "1.0.0" [variables] diff --git a/tests/testcases/spin-inbound-http/spin.toml b/tests/testcases/spin-inbound-http/spin.toml index 5c018b71e..a97853d34 100644 --- a/tests/testcases/spin-inbound-http/spin.toml +++ b/tests/testcases/spin-inbound-http/spin.toml @@ -6,9 +6,6 @@ description = "Test using the spin inbound-http interface." name = "spin-inbound-http" version = "1.0.0" -[application.trigger.http] -base = "/base" - [[trigger.http]] route = "/..." [trigger.http.component] diff --git a/tests/testcases/wagi-http/spin.toml b/tests/testcases/wagi-http/spin.toml index 871d681b4..5e3f02812 100644 --- a/tests/testcases/wagi-http/spin.toml +++ b/tests/testcases/wagi-http/spin.toml @@ -6,9 +6,6 @@ description = "Test using WAGI HTTP." name = "wagi-http" version = "1.0.0" -[application.trigger.http] -base = "/base" - [[trigger.http]] route = "/..." executor = { type = "wagi" } From bbf0a360e773c6d6cf05ec908cd75f2b34810087 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 22 Aug 2024 14:51:09 +0200 Subject: [PATCH 150/195] Integrate PG into trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 2 ++ crates/runtime-config/Cargo.toml | 1 + crates/runtime-config/src/lib.rs | 7 +++++++ crates/trigger2/Cargo.toml | 1 + crates/trigger2/src/factors.rs | 3 +++ 5 files changed, 14 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f52bf1699..734e5c3fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8173,6 +8173,7 @@ dependencies = [ "spin-factor-outbound-http", "spin-factor-outbound-mqtt", "spin-factor-outbound-networking", + "spin-factor-outbound-pg", "spin-factor-outbound-redis", "spin-factor-sqlite", "spin-factor-variables", @@ -8485,6 +8486,7 @@ dependencies = [ "spin-factor-outbound-http", "spin-factor-outbound-mqtt", "spin-factor-outbound-networking", + "spin-factor-outbound-pg", "spin-factor-outbound-redis", "spin-factor-sqlite", "spin-factor-variables", diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index 6954a6dac..a863b1edd 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -18,6 +18,7 @@ spin-factor-key-value-azure = { path = "../factor-key-value-azure" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-outbound-pg = { path = "../factor-outbound-pg" } spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-sqlite = { path = "../factor-sqlite" } spin-factor-variables = { path = "../factor-variables" } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index b6dead06c..04149d34a 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -7,6 +7,7 @@ use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::OutboundMqttFactor; use spin_factor_outbound_networking::runtime_config::spin::SpinTlsRuntimeConfig; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_outbound_pg::OutboundPgFactor; use spin_factor_outbound_redis::OutboundRedisFactor; use spin_factor_sqlite::runtime_config::spin as sqlite; use spin_factor_sqlite::SqliteFactor; @@ -159,6 +160,12 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(None) + } +} + impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 4127c2258..b11c598bd 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -25,6 +25,7 @@ spin-factor-key-value = { path = "../factor-key-value" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } +spin-factor-outbound-pg = { path = "../factor-outbound-pg" } spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-sqlite = { path = "../factor-sqlite" } spin-factor-variables = { path = "../factor-variables" } diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index 58986d159..dd474de61 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -4,6 +4,7 @@ use spin_factor_key_value::KeyValueFactor; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::{NetworkedMqttClient, OutboundMqttFactor}; use spin_factor_outbound_networking::OutboundNetworkingFactor; +use spin_factor_outbound_pg::OutboundPgFactor; use spin_factor_outbound_redis::OutboundRedisFactor; use spin_factor_sqlite::SqliteFactor; use spin_factor_variables::VariablesFactor; @@ -21,6 +22,7 @@ pub struct TriggerFactors { pub sqlite: SqliteFactor, pub redis: OutboundRedisFactor, pub mqtt: OutboundMqttFactor, + pub pg: OutboundPgFactor, } impl TriggerFactors { @@ -39,6 +41,7 @@ impl TriggerFactors { sqlite: SqliteFactor::new(default_sqlite_label_resolver), redis: OutboundRedisFactor::new(), mqtt: OutboundMqttFactor::new(NetworkedMqttClient::creator()), + pg: OutboundPgFactor::new(), } } } From 337fdeb2a3c5428d15f8de8e1661a11d407ed51e Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 22 Aug 2024 14:54:26 +0200 Subject: [PATCH 151/195] Integrate mysql into trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 2 ++ crates/runtime-config/Cargo.toml | 1 + crates/runtime-config/src/lib.rs | 7 +++++++ crates/trigger2/Cargo.toml | 1 + crates/trigger2/src/factors.rs | 3 +++ 5 files changed, 14 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 734e5c3fc..f2b404a08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8172,6 +8172,7 @@ dependencies = [ "spin-factor-key-value-spin", "spin-factor-outbound-http", "spin-factor-outbound-mqtt", + "spin-factor-outbound-mysql", "spin-factor-outbound-networking", "spin-factor-outbound-pg", "spin-factor-outbound-redis", @@ -8485,6 +8486,7 @@ dependencies = [ "spin-factor-key-value", "spin-factor-outbound-http", "spin-factor-outbound-mqtt", + "spin-factor-outbound-mysql", "spin-factor-outbound-networking", "spin-factor-outbound-pg", "spin-factor-outbound-redis", diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index a863b1edd..39f5b6f7a 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -19,6 +19,7 @@ spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-outbound-pg = { path = "../factor-outbound-pg" } +spin-factor-outbound-mysql = { path = "../factor-outbound-mysql" } spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-sqlite = { path = "../factor-sqlite" } spin-factor-variables = { path = "../factor-variables" } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 04149d34a..d5b7d88c9 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -5,6 +5,7 @@ use spin_factor_key_value::runtime_config::spin::{self as key_value, MakeKeyValu use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::OutboundMqttFactor; +use spin_factor_outbound_mysql::OutboundMysqlFactor; use spin_factor_outbound_networking::runtime_config::spin::SpinTlsRuntimeConfig; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_pg::OutboundPgFactor; @@ -166,6 +167,12 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(None) + } +} + impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index b11c598bd..a87d63b8c 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -26,6 +26,7 @@ spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-outbound-pg = { path = "../factor-outbound-pg" } +spin-factor-outbound-mysql = { path = "../factor-outbound-mysql" } spin-factor-outbound-redis = { path = "../factor-outbound-redis" } spin-factor-sqlite = { path = "../factor-sqlite" } spin-factor-variables = { path = "../factor-variables" } diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index dd474de61..f7e805f71 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use spin_factor_key_value::KeyValueFactor; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::{NetworkedMqttClient, OutboundMqttFactor}; +use spin_factor_outbound_mysql::OutboundMysqlFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_outbound_pg::OutboundPgFactor; use spin_factor_outbound_redis::OutboundRedisFactor; @@ -23,6 +24,7 @@ pub struct TriggerFactors { pub redis: OutboundRedisFactor, pub mqtt: OutboundMqttFactor, pub pg: OutboundPgFactor, + pub mysql: OutboundMysqlFactor, } impl TriggerFactors { @@ -42,6 +44,7 @@ impl TriggerFactors { redis: OutboundRedisFactor::new(), mqtt: OutboundMqttFactor::new(NetworkedMqttClient::creator()), pg: OutboundPgFactor::new(), + mysql: OutboundMysqlFactor::new(), } } } From 4acac95e0f04bc9aa0b5ac81b6e3a1291bcb5637 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 21 Aug 2024 16:12:01 -0400 Subject: [PATCH 152/195] trigger-http: Move headers tests into headers mod Signed-off-by: Lann Martin --- crates/trigger-http2/src/headers.rs | 173 +++++++++++++++++++++++++++ crates/trigger-http2/src/lib.rs | 176 +--------------------------- 2 files changed, 174 insertions(+), 175 deletions(-) diff --git a/crates/trigger-http2/src/headers.rs b/crates/trigger-http2/src/headers.rs index d9a2843c9..eed6a4075 100644 --- a/crates/trigger-http2/src/headers.rs +++ b/crates/trigger-http2/src/headers.rs @@ -139,6 +139,8 @@ fn prepare_header_key(key: &str) -> String { #[cfg(test)] mod tests { use super::*; + use anyhow::Result; + use spin_http::routes::Router; #[test] fn test_spin_header_keys() { @@ -155,4 +157,175 @@ mod tests { "spin-raw-component-route".to_string() ); } + + #[test] + fn test_default_headers() -> Result<()> { + let scheme = "https"; + let host = "fermyon.dev"; + let trigger_route = "/foo/..."; + let component_path = "/foo"; + let path_info = "/bar"; + let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); + + let req_uri = format!( + "{}://{}{}{}?key1=value1&key2=value2", + scheme, host, component_path, path_info + ); + + let req = http::Request::builder() + .method("POST") + .uri(req_uri) + .body("")?; + + let (router, _) = Router::build("/", [("DUMMY", &trigger_route.into())])?; + let route_match = router.route("/foo/bar")?; + + let default_headers = compute_default_headers(req.uri(), host, &route_match, client_addr)?; + + assert_eq!( + search(&FULL_URL, &default_headers).unwrap(), + "https://fermyon.dev/foo/bar?key1=value1&key2=value2".to_string() + ); + assert_eq!( + search(&PATH_INFO, &default_headers).unwrap(), + "/bar".to_string() + ); + assert_eq!( + search(&MATCHED_ROUTE, &default_headers).unwrap(), + "/foo/...".to_string() + ); + assert_eq!( + search(&BASE_PATH, &default_headers).unwrap(), + "/".to_string() + ); + assert_eq!( + search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo/...".to_string() + ); + assert_eq!( + search(&COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo".to_string() + ); + assert_eq!( + search(&CLIENT_ADDR, &default_headers).unwrap(), + "127.0.0.1:8777".to_string() + ); + + Ok(()) + } + + #[test] + fn test_default_headers_with_named_wildcards() -> Result<()> { + let scheme = "https"; + let host = "fermyon.dev"; + let trigger_route = "/foo/:userid/..."; + let component_path = "/foo"; + let path_info = "/bar"; + let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); + + let req_uri = format!( + "{}://{}{}/42{}?key1=value1&key2=value2", + scheme, host, component_path, path_info + ); + + let req = http::Request::builder() + .method("POST") + .uri(req_uri) + .body("")?; + + let (router, _) = Router::build("/", [("DUMMY", &trigger_route.into())])?; + let route_match = router.route("/foo/42/bar")?; + + let default_headers = compute_default_headers(req.uri(), host, &route_match, client_addr)?; + + assert_eq!( + search(&FULL_URL, &default_headers).unwrap(), + "https://fermyon.dev/foo/42/bar?key1=value1&key2=value2".to_string() + ); + assert_eq!( + search(&PATH_INFO, &default_headers).unwrap(), + "/bar".to_string() + ); + assert_eq!( + search(&MATCHED_ROUTE, &default_headers).unwrap(), + "/foo/:userid/...".to_string() + ); + assert_eq!( + search(&BASE_PATH, &default_headers).unwrap(), + "/".to_string() + ); + assert_eq!( + search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo/:userid/...".to_string() + ); + assert_eq!( + search(&COMPONENT_ROUTE, &default_headers).unwrap(), + "/foo/:userid".to_string() + ); + assert_eq!( + search(&CLIENT_ADDR, &default_headers).unwrap(), + "127.0.0.1:8777".to_string() + ); + + assert_eq!( + search( + &["SPIN_PATH_MATCH_USERID", "X_PATH_MATCH_USERID"], + &default_headers + ) + .unwrap(), + "42".to_string() + ); + + Ok(()) + } + + #[test] + fn forbidden_headers_are_removed() { + let mut req = Request::get("http://test.spin.internal") + .header("Host", "test.spin.internal") + .header("accept", "text/plain") + .body(Default::default()) + .unwrap(); + + strip_forbidden_headers(&mut req); + + assert_eq!(1, req.headers().len()); + assert!(req.headers().get("Host").is_none()); + + let mut req = Request::get("http://test.spin.internal") + .header("Host", "test.spin.internal:1234") + .header("accept", "text/plain") + .body(Default::default()) + .unwrap(); + + strip_forbidden_headers(&mut req); + + assert_eq!(1, req.headers().len()); + assert!(req.headers().get("Host").is_none()); + } + + #[test] + fn non_forbidden_headers_are_not_removed() { + let mut req = Request::get("http://test.example.com") + .header("Host", "test.example.org") + .header("accept", "text/plain") + .body(Default::default()) + .unwrap(); + + strip_forbidden_headers(&mut req); + + assert_eq!(2, req.headers().len()); + assert!(req.headers().get("Host").is_some()); + } + + fn search(keys: &[&str; 2], headers: &[([String; 2], String)]) -> Option { + let mut res: Option = None; + for (k, v) in headers { + if k[0] == keys[0] && k[1] == keys[1] { + res = Some(v.clone()); + } + } + + res + } } diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index 8263915b1..e5dcbaf8a 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -221,142 +221,7 @@ pub fn dns_error(rcode: String, info_code: u16) -> ErrorCode { #[cfg(test)] mod tests { - use anyhow::Result; - use http::Request; - - use super::{headers::*, *}; - - #[test] - fn test_default_headers() -> Result<()> { - let scheme = "https"; - let host = "fermyon.dev"; - let trigger_route = "/foo/..."; - let component_path = "/foo"; - let path_info = "/bar"; - let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); - - let req_uri = format!( - "{}://{}{}{}?key1=value1&key2=value2", - scheme, host, component_path, path_info - ); - - let req = http::Request::builder() - .method("POST") - .uri(req_uri) - .body("")?; - - let (router, _) = Router::build("/", [("DUMMY", &trigger_route.into())])?; - let route_match = router.route("/foo/bar")?; - - let default_headers = compute_default_headers(req.uri(), host, &route_match, client_addr)?; - - assert_eq!( - search(&FULL_URL, &default_headers).unwrap(), - "https://fermyon.dev/foo/bar?key1=value1&key2=value2".to_string() - ); - assert_eq!( - search(&PATH_INFO, &default_headers).unwrap(), - "/bar".to_string() - ); - assert_eq!( - search(&MATCHED_ROUTE, &default_headers).unwrap(), - "/foo/...".to_string() - ); - assert_eq!( - search(&BASE_PATH, &default_headers).unwrap(), - "/".to_string() - ); - assert_eq!( - search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo/...".to_string() - ); - assert_eq!( - search(&COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo".to_string() - ); - assert_eq!( - search(&CLIENT_ADDR, &default_headers).unwrap(), - "127.0.0.1:8777".to_string() - ); - - Ok(()) - } - - #[test] - fn test_default_headers_with_named_wildcards() -> Result<()> { - let scheme = "https"; - let host = "fermyon.dev"; - let trigger_route = "/foo/:userid/..."; - let component_path = "/foo"; - let path_info = "/bar"; - let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); - - let req_uri = format!( - "{}://{}{}/42{}?key1=value1&key2=value2", - scheme, host, component_path, path_info - ); - - let req = http::Request::builder() - .method("POST") - .uri(req_uri) - .body("")?; - - let (router, _) = Router::build("/", [("DUMMY", &trigger_route.into())])?; - let route_match = router.route("/foo/42/bar")?; - - let default_headers = compute_default_headers(req.uri(), host, &route_match, client_addr)?; - - assert_eq!( - search(&FULL_URL, &default_headers).unwrap(), - "https://fermyon.dev/foo/42/bar?key1=value1&key2=value2".to_string() - ); - assert_eq!( - search(&PATH_INFO, &default_headers).unwrap(), - "/bar".to_string() - ); - assert_eq!( - search(&MATCHED_ROUTE, &default_headers).unwrap(), - "/foo/:userid/...".to_string() - ); - assert_eq!( - search(&BASE_PATH, &default_headers).unwrap(), - "/".to_string() - ); - assert_eq!( - search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo/:userid/...".to_string() - ); - assert_eq!( - search(&COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo/:userid".to_string() - ); - assert_eq!( - search(&CLIENT_ADDR, &default_headers).unwrap(), - "127.0.0.1:8777".to_string() - ); - - assert_eq!( - search( - &["SPIN_PATH_MATCH_USERID", "X_PATH_MATCH_USERID"], - &default_headers - ) - .unwrap(), - "42".to_string() - ); - - Ok(()) - } - - fn search(keys: &[&str; 2], headers: &[([String; 2], String)]) -> Option { - let mut res: Option = None; - for (k, v) in headers { - if k[0] == keys[0] && k[1] == keys[1] { - res = Some(v.clone()); - } - } - - res - } + use super::*; #[test] fn parse_listen_addr_prefers_ipv4() { @@ -364,43 +229,4 @@ mod tests { assert_eq!(addr.ip(), Ipv4Addr::LOCALHOST); assert_eq!(addr.port(), 12345); } - - #[test] - fn forbidden_headers_are_removed() { - let mut req = Request::get("http://test.spin.internal") - .header("Host", "test.spin.internal") - .header("accept", "text/plain") - .body(Default::default()) - .unwrap(); - - strip_forbidden_headers(&mut req); - - assert_eq!(1, req.headers().len()); - assert!(req.headers().get("Host").is_none()); - - let mut req = Request::get("http://test.spin.internal") - .header("Host", "test.spin.internal:1234") - .header("accept", "text/plain") - .body(Default::default()) - .unwrap(); - - strip_forbidden_headers(&mut req); - - assert_eq!(1, req.headers().len()); - assert!(req.headers().get("Host").is_none()); - } - - #[test] - fn non_forbidden_headers_are_not_removed() { - let mut req = Request::get("http://test.example.com") - .header("Host", "test.example.org") - .header("accept", "text/plain") - .body(Default::default()) - .unwrap(); - - strip_forbidden_headers(&mut req); - - assert_eq!(2, req.headers().len()); - assert!(req.headers().get("Host").is_some()); - } } From 66516468613c2dc912cadf4787f940e7e8573fe1 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 22 Aug 2024 10:11:30 -0400 Subject: [PATCH 153/195] trigger-http: Move Server init logic into Server::new Signed-off-by: Lann Martin --- crates/trigger-http2/src/lib.rs | 44 +----------------------------- crates/trigger-http2/src/server.rs | 37 +++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 45 deletions(-) diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index e5dcbaf8a..994911099 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -10,7 +10,6 @@ mod wagi; mod wasi; use std::{ - collections::HashMap, error::Error, net::{Ipv4Addr, SocketAddr, ToSocketAddrs}, path::PathBuf, @@ -21,7 +20,6 @@ use anyhow::{bail, Context}; use clap::Args; use serde::Deserialize; use spin_app::App; -use spin_http::{config::HttpTriggerConfig, routes::Router}; use spin_trigger2::Trigger; use wasmtime_wasi_http::bindings::wasi::http::types::ErrorCode; @@ -72,9 +70,6 @@ pub struct HttpTrigger { /// If the port is set to 0, the actual address will be determined by the OS. listen_addr: SocketAddr, tls_config: Option, - router: Router, - // Component ID -> component trigger config - component_trigger_configs: HashMap, } impl Trigger for HttpTrigger { @@ -109,38 +104,9 @@ impl HttpTrigger { ) -> anyhow::Result { Self::validate_app(app)?; - let component_trigger_configs = HashMap::from_iter( - app.trigger_configs::("http")? - .into_iter() - .map(|(_, config)| (config.component.clone(), config)), - ); - - let component_routes = component_trigger_configs - .iter() - .map(|(component_id, config)| (component_id.as_str(), &config.route)); - let (router, duplicate_routes) = Router::build("/", component_routes)?; - if !duplicate_routes.is_empty() { - tracing::error!( - "The following component routes are duplicates and will never be used:" - ); - for dup in &duplicate_routes { - tracing::error!( - " {}: {} (duplicate of {})", - dup.replaced_id, - dup.route(), - dup.effective_id, - ); - } - } - tracing::trace!( - "Constructed router: {:?}", - router.routes().collect::>() - ); Ok(Self { listen_addr, tls_config, - router, - component_trigger_configs, }) } @@ -149,16 +115,8 @@ impl HttpTrigger { let Self { listen_addr, tls_config, - router, - component_trigger_configs, } = self; - let server = Arc::new(HttpServer::new( - listen_addr, - tls_config, - trigger_app, - router, - component_trigger_configs, - )?); + let server = Arc::new(HttpServer::new(listen_addr, tls_config, trigger_app)?); Ok(server) } diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index 3923bc9c4..8ead8feeb 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -58,9 +58,42 @@ impl HttpServer { listen_addr: SocketAddr, tls_config: Option, trigger_app: TriggerApp, - router: Router, - component_trigger_configs: HashMap, ) -> anyhow::Result { + // This needs to be a vec before building the router to handle duplicate routes + let component_trigger_configs = Vec::from_iter( + trigger_app + .app() + .trigger_configs::("http")? + .into_iter() + .map(|(_, config)| (config.component.clone(), config)), + ); + + // Build router + let component_routes = component_trigger_configs + .iter() + .map(|(component_id, config)| (component_id.as_str(), &config.route)); + let (router, duplicate_routes) = Router::build("/", component_routes)?; + if !duplicate_routes.is_empty() { + tracing::error!( + "The following component routes are duplicates and will never be used:" + ); + for dup in &duplicate_routes { + tracing::error!( + " {}: {} (duplicate of {})", + dup.replaced_id, + dup.route(), + dup.effective_id, + ); + } + } + tracing::trace!( + "Constructed router: {:?}", + router.routes().collect::>() + ); + + // Now that router is built we can merge duplicate routes by component + let component_trigger_configs = HashMap::from_iter(component_trigger_configs); + let component_handler_types = component_trigger_configs .keys() .map(|component_id| { From a7c916305bce624a17ea7eeb81a7285231e4622f Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 22 Aug 2024 13:33:46 +0200 Subject: [PATCH 154/195] Integrate llm factor into trigger2 Signed-off-by: Ryan Levick --- Cargo.lock | 39 ++++----- crates/factor-llm/Cargo.toml | 8 +- crates/factor-llm/src/host.rs | 4 +- crates/factor-llm/src/lib.rs | 52 +++++++++--- crates/factor-llm/src/spin.rs | 106 +++++++++++++++++++++++++ crates/factor-llm/tests/factor_test.rs | 8 +- crates/llm-local/Cargo.toml | 1 - crates/llm-local/src/lib.rs | 22 ++--- crates/llm-remote-http/Cargo.toml | 3 - crates/llm-remote-http/src/lib.rs | 9 +-- crates/runtime-config/Cargo.toml | 1 + crates/runtime-config/src/lib.rs | 47 +++++++---- crates/trigger2/Cargo.toml | 1 + crates/trigger2/src/cli.rs | 6 +- crates/trigger2/src/factors.rs | 8 ++ 15 files changed, 243 insertions(+), 72 deletions(-) create mode 100644 crates/factor-llm/src/spin.rs diff --git a/Cargo.lock b/Cargo.lock index f2b404a08..3295d890c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2435,20 +2435,6 @@ dependencies = [ "syn 2.0.58", ] -[[package]] -name = "factor-llm" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "async-trait", - "spin-factors", - "spin-factors-test", - "spin-locked-app", - "spin-world", - "tokio", - "tracing", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -7647,6 +7633,25 @@ dependencies = [ "spin-key-value-sqlite", ] +[[package]] +name = "spin-factor-llm" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "serde 1.0.197", + "spin-factors", + "spin-factors-test", + "spin-llm-local", + "spin-llm-remote-http", + "spin-locked-app", + "spin-world", + "tokio", + "toml 0.8.14", + "tracing", + "url", +] + [[package]] name = "spin-factor-outbound-http" version = "2.8.0-pre0" @@ -7996,7 +8001,6 @@ dependencies = [ "serde 1.0.197", "spin-common", "spin-core", - "spin-llm", "spin-world", "terminal", "tokenizers", @@ -8010,12 +8014,9 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "http 0.2.12", - "llm", "reqwest 0.11.27", "serde 1.0.197", "serde_json", - "spin-core", - "spin-llm", "spin-telemetry", "spin-world", "tracing", @@ -8170,6 +8171,7 @@ dependencies = [ "spin-factor-key-value-azure", "spin-factor-key-value-redis", "spin-factor-key-value-spin", + "spin-factor-llm", "spin-factor-outbound-http", "spin-factor-outbound-mqtt", "spin-factor-outbound-mysql", @@ -8484,6 +8486,7 @@ dependencies = [ "spin-componentize", "spin-core", "spin-factor-key-value", + "spin-factor-llm", "spin-factor-outbound-http", "spin-factor-outbound-mqtt", "spin-factor-outbound-mysql", diff --git a/crates/factor-llm/Cargo.toml b/crates/factor-llm/Cargo.toml index bcdc1e81f..e5a26b018 100644 --- a/crates/factor-llm/Cargo.toml +++ b/crates/factor-llm/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "factor-llm" +name = "spin-factor-llm" version.workspace = true authors.workspace = true edition.workspace = true @@ -11,10 +11,16 @@ rust-version.workspace = true [dependencies] anyhow = "1.0" async-trait = "0.1" +serde = "1.0" spin-factors = { path = "../factors" } +spin-llm-local = { path = "../llm-local" } +spin-llm-remote-http = { path = "../llm-remote-http" } spin-locked-app = { path = "../locked-app" } spin-world = { path = "../world" } tracing = { workspace = true } +tokio = { version = "1", features = ["sync"] } +toml = "0.8" +url = "2" [dev-dependencies] spin-factors-test = { path = "../factors-test" } diff --git a/crates/factor-llm/src/host.rs b/crates/factor-llm/src/host.rs index 748f97b1a..af980ad7e 100644 --- a/crates/factor-llm/src/host.rs +++ b/crates/factor-llm/src/host.rs @@ -16,6 +16,8 @@ impl v2::Host for InstanceState { return Err(access_denied_error(&model)); } self.engine + .lock() + .await .infer( model, prompt, @@ -39,7 +41,7 @@ impl v2::Host for InstanceState { if !self.allowed_models.contains(&m) { return Err(access_denied_error(&m)); } - self.engine.generate_embeddings(m, data).await + self.engine.lock().await.generate_embeddings(m, data).await } fn convert_error(&mut self, error: v2::Error) -> anyhow::Result { diff --git a/crates/factor-llm/src/lib.rs b/crates/factor-llm/src/lib.rs index 3e40f36a2..543e59b61 100644 --- a/crates/factor-llm/src/lib.rs +++ b/crates/factor-llm/src/lib.rs @@ -1,4 +1,5 @@ mod host; +pub mod spin; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -11,26 +12,28 @@ use spin_factors::{ use spin_locked_app::MetadataKey; use spin_world::v1::llm::{self as v1}; use spin_world::v2::llm::{self as v2}; +use tokio::sync::Mutex; pub const ALLOWED_MODELS_KEY: MetadataKey> = MetadataKey::new("ai_models"); +/// The factor for LLMs. pub struct LlmFactor { - create_engine: Box Box + Send + Sync>, + default_engine_creator: Box, } impl LlmFactor { - pub fn new(create_engine: F) -> Self - where - F: Fn() -> Box + Send + Sync + 'static, - { + /// Creates a new LLM factor with the given default engine creator. + /// + /// The default engine creator is used to create the engine if no runtime configuration is provided. + pub fn new(default_engine_creator: F) -> Self { Self { - create_engine: Box::new(create_engine), + default_engine_creator: Box::new(default_engine_creator), } } } impl Factor for LlmFactor { - type RuntimeConfig = (); + type RuntimeConfig = RuntimeConfig; type AppState = AppState; type InstanceBuilder = InstanceState; @@ -45,7 +48,7 @@ impl Factor for LlmFactor { fn configure_app( &self, - ctx: ConfigureAppContext, + mut ctx: ConfigureAppContext, ) -> anyhow::Result { let component_allowed_models = ctx .app() @@ -62,7 +65,12 @@ impl Factor for LlmFactor { )) }) .collect::>()?; + let engine = ctx + .take_runtime_config() + .map(|c| c.engine) + .unwrap_or_else(|| self.default_engine_creator.create()); Ok(AppState { + engine, component_allowed_models, }) } @@ -78,25 +86,35 @@ impl Factor for LlmFactor { .get(ctx.app_component().id()) .cloned() .unwrap_or_default(); + let engine = ctx.app_state().engine.clone(); Ok(InstanceState { - engine: (self.create_engine)(), + engine, allowed_models, }) } } +/// The application state for the LLM factor. pub struct AppState { + engine: Arc>, component_allowed_models: HashMap>>, } +/// The instance state for the LLM factor. pub struct InstanceState { - engine: Box, + engine: Arc>, pub allowed_models: Arc>, } +/// The runtime configuration for the LLM factor. +pub struct RuntimeConfig { + engine: Arc>, +} + impl SelfInstanceBuilder for InstanceState {} +/// The interface for a language model engine. #[async_trait] pub trait LlmEngine: Send + Sync { async fn infer( @@ -112,3 +130,17 @@ pub trait LlmEngine: Send + Sync { data: Vec, ) -> Result; } + +/// A creator for an LLM engine. +pub trait LlmEngineCreator: Send + Sync { + fn create(&self) -> Arc>; +} + +impl LlmEngineCreator for F +where + F: Fn() -> Arc> + Send + Sync, +{ + fn create(&self) -> Arc> { + self() + } +} diff --git a/crates/factor-llm/src/spin.rs b/crates/factor-llm/src/spin.rs new file mode 100644 index 000000000..6ebd7a706 --- /dev/null +++ b/crates/factor-llm/src/spin.rs @@ -0,0 +1,106 @@ +use std::path::PathBuf; +use std::sync::Arc; + +pub use spin_llm_local::LocalLlmEngine; + +use spin_llm_remote_http::RemoteHttpLlmEngine; +use spin_world::async_trait; +use spin_world::v1::llm::{self as v1}; +use spin_world::v2::llm::{self as v2}; +use tokio::sync::Mutex; +use url::Url; + +use crate::{LlmEngine, LlmEngineCreator, RuntimeConfig}; + +#[async_trait] +impl LlmEngine for LocalLlmEngine { + async fn infer( + &mut self, + model: v1::InferencingModel, + prompt: String, + params: v2::InferencingParams, + ) -> Result { + self.infer(model, prompt, params).await + } + + async fn generate_embeddings( + &mut self, + model: v2::EmbeddingModel, + data: Vec, + ) -> Result { + self.generate_embeddings(model, data).await + } +} + +#[async_trait] +impl LlmEngine for RemoteHttpLlmEngine { + async fn infer( + &mut self, + model: v1::InferencingModel, + prompt: String, + params: v2::InferencingParams, + ) -> Result { + self.infer(model, prompt, params).await + } + + async fn generate_embeddings( + &mut self, + model: v2::EmbeddingModel, + data: Vec, + ) -> Result { + self.generate_embeddings(model, data).await + } +} + +pub fn runtime_config_from_toml( + table: &toml::Table, + state_dir: PathBuf, + use_gpu: bool, +) -> anyhow::Result> { + let Some(value) = table.get("llm_compute") else { + return Ok(None); + }; + let config: LlmCompute = value.clone().try_into()?; + + Ok(Some(RuntimeConfig { + engine: config.into_engine(state_dir, use_gpu), + })) +} + +#[derive(Debug, serde::Deserialize)] +#[serde(rename_all = "snake_case", tag = "type")] +pub enum LlmCompute { + Spin, + RemoteHttp(RemoteHttpCompute), +} + +impl LlmCompute { + fn into_engine(self, state_dir: PathBuf, use_gpu: bool) -> Arc> { + match self { + LlmCompute::Spin => default_engine_creator(state_dir, use_gpu).create(), + LlmCompute::RemoteHttp(config) => Arc::new(Mutex::new(RemoteHttpLlmEngine::new( + config.url, + config.auth_token, + ))), + } + } +} + +#[derive(Debug, serde::Deserialize)] +pub struct RemoteHttpCompute { + url: Url, + auth_token: String, +} + +/// The default engine creator for the LLM factor when used in the Spin CLI. +pub fn default_engine_creator( + state_dir: PathBuf, + use_gpu: bool, +) -> impl LlmEngineCreator + 'static { + move || { + Arc::new(Mutex::new(LocalLlmEngine::new( + state_dir.join("ai-models"), + use_gpu, + ))) as _ + } +} diff --git a/crates/factor-llm/tests/factor_test.rs b/crates/factor-llm/tests/factor_test.rs index 4504238b5..a0c4e988a 100644 --- a/crates/factor-llm/tests/factor_test.rs +++ b/crates/factor-llm/tests/factor_test.rs @@ -1,10 +1,12 @@ use std::collections::HashSet; +use std::sync::Arc; -use factor_llm::{LlmEngine, LlmFactor}; +use spin_factor_llm::{LlmEngine, LlmFactor}; use spin_factors::{anyhow, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::v1::llm::{self as v1}; use spin_world::v2::llm::{self as v2, Host}; +use tokio::sync::Mutex; #[derive(RuntimeFactors)] struct TestFactors { @@ -37,9 +39,9 @@ async fn llm_works() -> anyhow::Result<()> { }); let factors = TestFactors { llm: LlmFactor::new(move || { - Box::new(FakeLLm { + Arc::new(Mutex::new(FakeLLm { handle: handle.clone(), - }) as _ + })) as _ }), }; let env = TestEnvironment::new(factors).extend_manifest(toml! { diff --git a/crates/llm-local/Cargo.toml b/crates/llm-local/Cargo.toml index b0d4ea397..5b7331642 100644 --- a/crates/llm-local/Cargo.toml +++ b/crates/llm-local/Cargo.toml @@ -20,7 +20,6 @@ safetensors = "0.3.3" serde = { version = "1.0.150", features = ["derive"] } spin-common = { path = "../common" } spin-core = { path = "../core" } -spin-llm = { path = "../llm" } spin-world = { path = "../world" } terminal = { path = "../terminal" } tokenizers = "0.13.4" diff --git a/crates/llm-local/src/lib.rs b/crates/llm-local/src/lib.rs index f5d00c7a1..cf0b9f992 100644 --- a/crates/llm-local/src/lib.rs +++ b/crates/llm-local/src/lib.rs @@ -10,8 +10,6 @@ use llm::{ }; use rand::SeedableRng; use spin_common::ui::quoted_path; -use spin_core::async_trait; -use spin_llm::{LlmEngine, MODEL_ALL_MINILM_L6_V2}; use spin_world::v2::llm::{self as wasi_llm}; use std::{ collections::hash_map::Entry, @@ -23,6 +21,8 @@ use std::{ use tokenizers::PaddingParams; use tracing::{instrument, Level}; +const MODEL_ALL_MINILM_L6_V2: &str = "all-minilm-l6-v2"; + #[derive(Clone)] pub struct LocalLlmEngine { registry: PathBuf, @@ -31,10 +31,9 @@ pub struct LocalLlmEngine { embeddings_models: HashMap>, } -#[async_trait] -impl LlmEngine for LocalLlmEngine { +impl LocalLlmEngine { #[instrument(name = "spin_llm_local.infer", skip(self, prompt), err(level = Level::INFO))] - async fn infer( + pub async fn infer( &mut self, model: wasi_llm::InferencingModel, prompt: String, @@ -94,7 +93,7 @@ impl LlmEngine for LocalLlmEngine { } #[instrument(name = "spin_llm_local.generate_embeddings", skip(self, data), err(level = Level::INFO))] - async fn generate_embeddings( + pub async fn generate_embeddings( &mut self, model: wasi_llm::EmbeddingModel, data: Vec, @@ -107,18 +106,13 @@ impl LlmEngine for LocalLlmEngine { } impl LocalLlmEngine { - pub async fn new(registry: PathBuf, use_gpu: bool) -> Self { - let mut engine = Self { + pub fn new(registry: PathBuf, use_gpu: bool) -> Self { + Self { registry, use_gpu, inferencing_models: Default::default(), embeddings_models: Default::default(), - }; - - let _ = engine.inferencing_model("llama2-chat".into()).await; - let _ = engine.embeddings_model(MODEL_ALL_MINILM_L6_V2.into()).await; - - engine + } } /// Get embeddings model from cache or load from disk diff --git a/crates/llm-remote-http/Cargo.toml b/crates/llm-remote-http/Cargo.toml index 3a9bb8e12..af05459e5 100644 --- a/crates/llm-remote-http/Cargo.toml +++ b/crates/llm-remote-http/Cargo.toml @@ -7,11 +7,8 @@ edition = { workspace = true } [dependencies] anyhow = "1.0" http = "0.2" -llm = { git = "https://github.com/rustformers/llm", rev = "2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663", default-features = false } serde = { version = "1.0.150", features = ["derive"] } serde_json = "1.0" -spin-core = { path = "../core" } -spin-llm = { path = "../llm" } spin-telemetry = { path = "../telemetry" } spin-world = { path = "../world" } reqwest = { version = "0.11", features = ["gzip", "json"] } diff --git a/crates/llm-remote-http/src/lib.rs b/crates/llm-remote-http/src/lib.rs index 4987b14b0..4a0039539 100644 --- a/crates/llm-remote-http/src/lib.rs +++ b/crates/llm-remote-http/src/lib.rs @@ -5,8 +5,6 @@ use reqwest::{ }; use serde::{Deserialize, Serialize}; use serde_json::json; -use spin_core::async_trait; -use spin_llm::LlmEngine; use spin_world::v2::llm::{self as wasi_llm}; use tracing::{instrument, Level}; @@ -53,10 +51,9 @@ struct EmbeddingResponseBody { usage: EmbeddingUsage, } -#[async_trait] -impl LlmEngine for RemoteHttpLlmEngine { +impl RemoteHttpLlmEngine { #[instrument(name = "spin_llm_remote_http.infer", skip(self, prompt), err(level = Level::INFO), fields(otel.kind = "client"))] - async fn infer( + pub async fn infer( &mut self, model: wasi_llm::InferencingModel, prompt: String, @@ -119,7 +116,7 @@ impl LlmEngine for RemoteHttpLlmEngine { } #[instrument(name = "spin_llm_remote_http.generate_embeddings", skip(self, data), err(level = Level::INFO), fields(otel.kind = "client"))] - async fn generate_embeddings( + pub async fn generate_embeddings( &mut self, model: wasi_llm::EmbeddingModel, data: Vec, diff --git a/crates/runtime-config/Cargo.toml b/crates/runtime-config/Cargo.toml index 39f5b6f7a..267dfdec7 100644 --- a/crates/runtime-config/Cargo.toml +++ b/crates/runtime-config/Cargo.toml @@ -15,6 +15,7 @@ spin-factor-key-value = { path = "../factor-key-value" } spin-factor-key-value-spin = { path = "../factor-key-value-spin" } spin-factor-key-value-redis = { path = "../factor-key-value-redis" } spin-factor-key-value-azure = { path = "../factor-key-value-azure" } +spin-factor-llm = { path = "../factor-llm" } spin-factor-outbound-http = { path = "../factor-outbound-http" } spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index d5b7d88c9..77ca2a8ec 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; use spin_factor_key_value::runtime_config::spin::{self as key_value, MakeKeyValueStore}; use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; +use spin_factor_llm::{spin as llm, LlmFactor}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::OutboundMqttFactor; use spin_factor_outbound_mysql::OutboundMysqlFactor; @@ -39,13 +40,17 @@ where for<'a> >>::Error: Into, { /// Creates a new resolved runtime configuration from a runtime config source TOML file. - pub fn from_file(runtime_config_path: &Path, state_dir: Option<&str>) -> anyhow::Result { + pub fn from_file( + runtime_config_path: &Path, + state_dir: Option<&str>, + use_gpu: bool, + ) -> anyhow::Result { let tls_resolver = SpinTlsRuntimeConfig::new(runtime_config_path); - let key_value_config_resolver = - key_value_config_resolver(PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR))); + let state_dir = PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR)); + let key_value_config_resolver = key_value_config_resolver(state_dir.clone()); - let sqlite_config_resolver = - sqlite_config_resolver(state_dir).context("failed to resolve sqlite runtime config")?; + let sqlite_config_resolver = sqlite_config_resolver(state_dir.clone()) + .context("failed to resolve sqlite runtime config")?; let file = std::fs::read_to_string(runtime_config_path).with_context(|| { format!( @@ -61,9 +66,11 @@ where })?; let runtime_config: T = TomlRuntimeConfigSource::new( &toml, + state_dir, &key_value_config_resolver, &tls_resolver, &sqlite_config_resolver, + use_gpu, ) .try_into() .map_err(Into::into)?; @@ -99,12 +106,11 @@ where impl ResolvedRuntimeConfig { pub fn default(state_dir: Option<&str>) -> Self { + let state_dir = state_dir.unwrap_or(DEFAULT_STATE_DIR); Self { - sqlite_resolver: sqlite_config_resolver(state_dir) + sqlite_resolver: sqlite_config_resolver(PathBuf::from(state_dir)) .expect("failed to resolve sqlite runtime config"), - key_value_resolver: key_value_config_resolver(PathBuf::from( - state_dir.unwrap_or(DEFAULT_STATE_DIR), - )), + key_value_resolver: key_value_config_resolver(PathBuf::from(state_dir)), runtime_config: Default::default(), } } @@ -113,23 +119,29 @@ impl ResolvedRuntimeConfig { /// The TOML based runtime configuration source Spin CLI. pub struct TomlRuntimeConfigSource<'a> { table: TomlKeyTracker<'a>, + state_dir: PathBuf, key_value: &'a key_value::RuntimeConfigResolver, tls: &'a SpinTlsRuntimeConfig, sqlite: &'a sqlite::RuntimeConfigResolver, + use_gpu: bool, } impl<'a> TomlRuntimeConfigSource<'a> { pub fn new( table: &'a toml::Table, + state_dir: PathBuf, key_value: &'a key_value::RuntimeConfigResolver, tls: &'a SpinTlsRuntimeConfig, sqlite: &'a sqlite::RuntimeConfigResolver, + use_gpu: bool, ) -> Self { Self { table: TomlKeyTracker::new(table), + state_dir, key_value, tls, sqlite, + use_gpu, } } } @@ -173,6 +185,16 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource< } } +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { + fn get_runtime_config(&mut self) -> anyhow::Result> { + Ok(llm::runtime_config_from_toml( + self.table.as_ref(), + self.state_dir.clone(), + self.use_gpu, + )?) + } +} + impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) @@ -251,14 +273,11 @@ pub fn key_value_config_resolver( /// The sqlite runtime configuration resolver. /// /// Takes a base path to the state directory. -fn sqlite_config_resolver( - state_dir: Option<&str>, -) -> anyhow::Result { - let default_database_dir = PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR)); +fn sqlite_config_resolver(state_dir: PathBuf) -> anyhow::Result { let local_database_dir = std::env::current_dir().context("failed to get current working directory")?; Ok(sqlite::RuntimeConfigResolver::new( - default_database_dir, + state_dir, local_database_dir, )) } diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index a87d63b8c..820eba767 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -23,6 +23,7 @@ spin-componentize = { path = "../componentize" } spin-core = { path = "../core" } spin-factor-key-value = { path = "../factor-key-value" } spin-factor-outbound-http = { path = "../factor-outbound-http" } +spin-factor-llm = { path = "../factor-llm" } spin-factor-outbound-mqtt = { path = "../factor-outbound-mqtt" } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factor-outbound-pg = { path = "../factor-outbound-pg" } diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index 2aab88adb..9bc787cef 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -10,7 +10,7 @@ use spin_common::ui::quoted_path; use spin_common::url::parse_file_url; use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; -use spin_runtime_config::ResolvedRuntimeConfig; +use spin_runtime_config::{ResolvedRuntimeConfig, DEFAULT_STATE_DIR}; use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; @@ -304,11 +304,13 @@ impl TriggerAppBuilder { }; self.trigger.add_to_linker(core_engine_builder.linker())?; + let use_gpu = true; let runtime_config = match options.runtime_config_file { Some(runtime_config_path) => { ResolvedRuntimeConfig::::from_file( runtime_config_path, options.state_dir, + use_gpu, )? } None => ResolvedRuntimeConfig::default(options.state_dir), @@ -319,10 +321,12 @@ impl TriggerAppBuilder { .await?; let factors = TriggerFactors::new( + options.state_dir.unwrap_or(DEFAULT_STATE_DIR), self.working_dir.clone(), options.allow_transient_write, runtime_config.key_value_resolver, runtime_config.sqlite_resolver, + use_gpu, ); // TODO: move these into Factor methods/constructors diff --git a/crates/trigger2/src/factors.rs b/crates/trigger2/src/factors.rs index f7e805f71..6274f6bcf 100644 --- a/crates/trigger2/src/factors.rs +++ b/crates/trigger2/src/factors.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use spin_factor_key_value::KeyValueFactor; +use spin_factor_llm::LlmFactor; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::{NetworkedMqttClient, OutboundMqttFactor}; use spin_factor_outbound_mysql::OutboundMysqlFactor; @@ -25,14 +26,17 @@ pub struct TriggerFactors { pub mqtt: OutboundMqttFactor, pub pg: OutboundPgFactor, pub mysql: OutboundMysqlFactor, + pub llm: LlmFactor, } impl TriggerFactors { pub fn new( + state_dir: impl Into, working_dir: impl Into, allow_transient_writes: bool, default_key_value_label_resolver: impl spin_factor_key_value::DefaultLabelResolver + 'static, default_sqlite_label_resolver: impl spin_factor_sqlite::DefaultLabelResolver + 'static, + use_gpu: bool, ) -> Self { Self { wasi: wasi_factor(working_dir, allow_transient_writes), @@ -45,6 +49,10 @@ impl TriggerFactors { mqtt: OutboundMqttFactor::new(NetworkedMqttClient::creator()), pg: OutboundPgFactor::new(), mysql: OutboundMysqlFactor::new(), + llm: LlmFactor::new(spin_factor_llm::spin::default_engine_creator( + state_dir.into(), + use_gpu, + )), } } } From c46b41feab02a3e8d3dfdee7b6b4d11ecd060650 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 22 Aug 2024 17:41:07 +0200 Subject: [PATCH 155/195] Fix runtime tests. Spin now has a hard requirement that a Host (pseudo-)header be set. The runtime tests were previously not sending a Host header which caused them to fail. Signed-off-by: Ryan Levick --- tests/runtime-tests/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/runtime-tests/src/lib.rs b/tests/runtime-tests/src/lib.rs index 560fdd851..c4427e508 100644 --- a/tests/runtime-tests/src/lib.rs +++ b/tests/runtime-tests/src/lib.rs @@ -86,7 +86,7 @@ impl RuntimeTest { pub fn run(&mut self) { self.run_test(|env| { let runtime = env.runtime_mut(); - let request = Request::new(Method::Get, "/"); + let request: Request = Request::full(Method::Get, "/", &[("Host", "localhost")], None); let response = runtime.make_http_request(request)?; if response.status() == 200 { return Ok(()); @@ -147,7 +147,7 @@ impl RuntimeTest { pub fn run(&mut self) { self.run_test(|env| { let runtime = env.runtime_mut(); - let response = runtime.make_http_request(Request::new(Method::Get, "/"))?; + let response = runtime.make_http_request(Request::full(Method::Get, "/", &[("Host", "localhost")],None))?; if response.status() == 200 { return Ok(()); } From 1d317654422a88e7604bb13b1d8358e1989148b2 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 22 Aug 2024 17:57:12 +0200 Subject: [PATCH 156/195] Also allow getting authority from the request URI's authority. Signed-off-by: Ryan Levick --- crates/trigger-http2/src/server.rs | 51 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http2/src/server.rs index 3923bc9c4..71acddbc0 100644 --- a/crates/trigger-http2/src/server.rs +++ b/crates/trigger-http2/src/server.rs @@ -1,7 +1,10 @@ use std::{collections::HashMap, future::Future, io::IsTerminal, net::SocketAddr, sync::Arc}; -use anyhow::Context; -use http::{uri::Scheme, Request, Response, StatusCode, Uri}; +use anyhow::{bail, Context}; +use http::{ + uri::{Authority, Scheme}, + Request, Response, StatusCode, Uri, +}; use http_body_util::BodyExt; use hyper::{ body::{Bytes, Incoming}, @@ -359,29 +362,39 @@ impl HttpServer { /// The incoming request's scheme and authority /// /// The incoming request's URI is relative to the server, so we need to set the scheme and authority. -/// The `Host` header is used to set the authority. This function will error if no `Host` header is -/// present or if it is not parsable as an `Authority`. +/// Either the `Host` header or the request's URI's authority is used as the source of truth for the authority. +/// This function will error if the authority cannot be unambiguously determined. fn set_req_uri(req: &mut Request, scheme: Scheme) -> anyhow::Result<()> { let uri = req.uri().clone(); let mut parts = uri.into_parts(); let headers = req.headers(); - let host_header = headers + let header_authority = headers .get(http::header::HOST) - .context("missing 'Host' header")? - .to_str() - .context("'Host' header is not valid UTF-8")?; - let authority = host_header - .parse() - .context("'Host' header contains an invalid authority")?; - // Ensure that if `req.authority` is set, it matches what was in the `Host` header - // https://github.com/hyperium/hyper/issues/1612 - if let Some(a) = parts.authority.as_ref() { - if a != &authority { - return Err(anyhow::anyhow!( - "authority in 'Host' header does not match authority in URI" - )); + .map(|h| -> anyhow::Result { + let host_header = h.to_str().context("'Host' header is not valid UTF-8")?; + host_header + .parse() + .context("'Host' header contains an invalid authority") + }) + .transpose()?; + let uri_authority = parts.authority; + + // Get authority either from request URI or from 'Host' header + let authority = match (header_authority, uri_authority) { + (None, None) => bail!("no 'Host' header present in request"), + (None, Some(a)) => a, + (Some(a), None) => a, + (Some(a1), Some(a2)) => { + // Ensure that if `req.authority` is set, it matches what was in the `Host` header + // https://github.com/hyperium/hyper/issues/1612 + if a1 != a2 { + return Err(anyhow::anyhow!( + "authority in 'Host' header does not match authority in URI" + )); + } + a1 } - } + }; parts.scheme = Some(scheme); parts.authority = Some(authority); *req.uri_mut() = Uri::from_parts(parts).unwrap(); From 85b55a3249eba2ae099ec7dd063551166e123299 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 22 Aug 2024 18:16:53 +0200 Subject: [PATCH 157/195] Put local llm behind feature flags like they used to be. Signed-off-by: Ryan Levick --- Cargo.toml | 7 +-- crates/factor-llm/Cargo.toml | 9 ++- crates/factor-llm/src/spin.rs | 102 +++++++++++++++++++++++--------- crates/trigger-http2/Cargo.toml | 5 ++ crates/trigger2/Cargo.toml | 5 ++ 5 files changed, 94 insertions(+), 34 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f7d32293a..0f270d3b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,10 +113,9 @@ wit-component = "0.19.0" # TODO(factors): default = ["llm"] all-tests = ["extern-dependencies-tests"] extern-dependencies-tests = [] -# TODO(factors): -# llm = ["spin-trigger-http/llm"] -# llm-metal = ["llm", "spin-trigger-http/llm-metal"] -# llm-cublas = ["llm", "spin-trigger-http/llm-cublas"] +llm = ["spin-trigger-http2/llm"] +llm-metal = ["llm", "spin-trigger-http2/llm-metal"] +llm-cublas = ["llm", "spin-trigger-http2/llm-cublas"] [workspace] members = [ diff --git a/crates/factor-llm/Cargo.toml b/crates/factor-llm/Cargo.toml index e5a26b018..b7f0e4107 100644 --- a/crates/factor-llm/Cargo.toml +++ b/crates/factor-llm/Cargo.toml @@ -8,19 +8,24 @@ homepage.workspace = true repository.workspace = true rust-version.workspace = true +[features] +llm = ["spin-llm-local"] +llm-metal = ["llm", "spin-llm-local/metal"] +llm-cublas = ["llm", "spin-llm-local/cublas"] + [dependencies] anyhow = "1.0" async-trait = "0.1" serde = "1.0" spin-factors = { path = "../factors" } -spin-llm-local = { path = "../llm-local" } +spin-llm-local = { path = "../llm-local", optional = true } spin-llm-remote-http = { path = "../llm-remote-http" } spin-locked-app = { path = "../locked-app" } spin-world = { path = "../world" } tracing = { workspace = true } tokio = { version = "1", features = ["sync"] } toml = "0.8" -url = "2" +url = { version = "2", features = ["serde"] } [dev-dependencies] spin-factors-test = { path = "../factors-test" } diff --git a/crates/factor-llm/src/spin.rs b/crates/factor-llm/src/spin.rs index 6ebd7a706..6d59c786c 100644 --- a/crates/factor-llm/src/spin.rs +++ b/crates/factor-llm/src/spin.rs @@ -1,8 +1,6 @@ use std::path::PathBuf; use std::sync::Arc; -pub use spin_llm_local::LocalLlmEngine; - use spin_llm_remote_http::RemoteHttpLlmEngine; use spin_world::async_trait; use spin_world::v1::llm::{self as v1}; @@ -12,26 +10,48 @@ use url::Url; use crate::{LlmEngine, LlmEngineCreator, RuntimeConfig}; -#[async_trait] -impl LlmEngine for LocalLlmEngine { - async fn infer( - &mut self, - model: v1::InferencingModel, - prompt: String, - params: v2::InferencingParams, - ) -> Result { - self.infer(model, prompt, params).await - } +#[cfg(feature = "llm")] +mod local { + use super::*; + pub use spin_llm_local::LocalLlmEngine; - async fn generate_embeddings( - &mut self, - model: v2::EmbeddingModel, - data: Vec, - ) -> Result { - self.generate_embeddings(model, data).await + #[async_trait] + impl LlmEngine for LocalLlmEngine { + async fn infer( + &mut self, + model: v2::InferencingModel, + prompt: String, + params: v2::InferencingParams, + ) -> Result { + self.infer(model, prompt, params).await + } + + async fn generate_embeddings( + &mut self, + model: v2::EmbeddingModel, + data: Vec, + ) -> Result { + self.generate_embeddings(model, data).await + } } } +/// The default engine creator for the LLM factor when used in the Spin CLI. +pub fn default_engine_creator( + state_dir: PathBuf, + use_gpu: bool, +) -> impl LlmEngineCreator + 'static { + #[cfg(feature = "llm")] + let engine = spin_llm_local::LocalLlmEngine::new(state_dir.join("ai-models"), use_gpu); + #[cfg(not(feature = "llm"))] + let engine = { + let _ = (state_dir, use_gpu); + noop::NoopLlmEngine + }; + let engine = Arc::new(Mutex::new(engine)) as Arc>; + move || engine.clone() +} + #[async_trait] impl LlmEngine for RemoteHttpLlmEngine { async fn infer( @@ -77,6 +97,12 @@ pub enum LlmCompute { impl LlmCompute { fn into_engine(self, state_dir: PathBuf, use_gpu: bool) -> Arc> { match self { + #[cfg(not(feature = "llm"))] + LlmCompute::Spin => { + let _ = (state_dir, use_gpu); + Arc::new(Mutex::new(noop::NoopLlmEngine)) + } + #[cfg(feature = "llm")] LlmCompute::Spin => default_engine_creator(state_dir, use_gpu).create(), LlmCompute::RemoteHttp(config) => Arc::new(Mutex::new(RemoteHttpLlmEngine::new( config.url, @@ -92,15 +118,35 @@ pub struct RemoteHttpCompute { auth_token: String, } -/// The default engine creator for the LLM factor when used in the Spin CLI. -pub fn default_engine_creator( - state_dir: PathBuf, - use_gpu: bool, -) -> impl LlmEngineCreator + 'static { - move || { - Arc::new(Mutex::new(LocalLlmEngine::new( - state_dir.join("ai-models"), - use_gpu, - ))) as _ +/// A noop engine used when the local engine feature is disabled. +#[cfg(not(feature = "llm"))] +mod noop { + use super::*; + + #[derive(Clone, Copy)] + pub(super) struct NoopLlmEngine; + + #[async_trait] + impl LlmEngine for NoopLlmEngine { + async fn infer( + &mut self, + _model: v2::InferencingModel, + _prompt: String, + _params: v2::InferencingParams, + ) -> Result { + Err(v2::Error::RuntimeError( + "Local LLM operations are not supported in this version of Spin.".into(), + )) + } + + async fn generate_embeddings( + &mut self, + _model: v2::EmbeddingModel, + _data: Vec, + ) -> Result { + Err(v2::Error::RuntimeError( + "Local LLM operations are not supported in this version of Spin.".into(), + )) + } } } diff --git a/crates/trigger-http2/Cargo.toml b/crates/trigger-http2/Cargo.toml index 0abb81436..d79b7b520 100644 --- a/crates/trigger-http2/Cargo.toml +++ b/crates/trigger-http2/Cargo.toml @@ -7,6 +7,11 @@ edition = { workspace = true } [lib] doctest = false +[features] +llm = ["spin-trigger2/llm"] +llm-metal = ["spin-trigger2/llm-metal"] +llm-cublas = ["spin-trigger2/llm-cublas"] + [dependencies] anyhow = "1.0" async-trait = "0.1" diff --git a/crates/trigger2/Cargo.toml b/crates/trigger2/Cargo.toml index 820eba767..565ee436b 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger2/Cargo.toml @@ -8,6 +8,11 @@ homepage.workspace = true repository.workspace = true rust-version.workspace = true +[features] +llm = ["spin-factor-llm/llm"] +llm-metal = ["spin-factor-llm/llm-metal"] +llm-cublas = ["spin-factor-llm/llm-cublas"] + [dependencies] anyhow = "1" clap = { version = "3.1.18", features = ["derive", "env"] } From 934545cfcf6b7e6e918dac581c02e4febc61ead8 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 22 Aug 2024 18:39:14 +0200 Subject: [PATCH 158/195] Get rid of old non-factors code Signed-off-by: Ryan Levick --- Cargo.lock | 507 +------ build.rs | 22 +- crates/llm/Cargo.toml | 16 - crates/llm/src/host_component.rs | 49 - crates/llm/src/lib.rs | 112 -- crates/oci/Cargo.toml | 3 - crates/oci/src/client.rs | 492 +++---- crates/outbound-http/Cargo.toml | 30 - crates/outbound-http/src/host_component.rs | 41 - crates/outbound-http/src/host_impl.rs | 208 --- crates/outbound-http/src/lib.rs | 11 - crates/outbound-mqtt/Cargo.toml | 23 - crates/outbound-mqtt/src/host_component.rs | 41 - crates/outbound-mqtt/src/lib.rs | 158 --- crates/outbound-mysql/Cargo.toml | 30 - crates/outbound-mysql/src/lib.rs | 442 ------ crates/outbound-pg/Cargo.toml | 25 - crates/outbound-pg/src/lib.rs | 461 ------ crates/outbound-redis/Cargo.toml | 23 - crates/outbound-redis/src/host_component.rs | 42 - crates/outbound-redis/src/lib.rs | 317 ----- crates/testing/Cargo.toml | 19 - crates/testing/src/lib.rs | 244 ---- crates/trigger-http/Cargo.toml | 62 - crates/trigger-http/benches/baseline.rs | 126 -- crates/trigger-http/benches/readme.md | 8 - .../spin-http-benchmark/.cargo/config.toml | 2 - .../benches/spin-http-benchmark/Cargo.toml | 13 - .../benches/spin-http-benchmark/src/lib.rs | 57 - .../benches/wagi-benchmark/.cargo/config.toml | 2 - .../benches/wagi-benchmark/Cargo.toml | 6 - .../benches/wagi-benchmark/src/main.rs | 11 - crates/trigger-http/src/handler.rs | 456 ------ crates/trigger-http/src/instrument.rs | 93 -- crates/trigger-http/src/lib.rs | 1240 ----------------- .../src/testdata/invalid-cert.pem | 20 - .../src/testdata/invalid-private-key.pem | 5 - .../trigger-http/src/testdata/valid-cert.pem | 21 - .../src/testdata/valid-private-key.pem | 5 - crates/trigger-http/src/tls.rs | 141 -- crates/trigger-http/src/wagi.rs | 142 -- crates/trigger-http/tests/local.crt.pem | 17 - crates/trigger-http/tests/local.key.pem | 28 - crates/trigger-redis/Cargo.toml | 30 - crates/trigger-redis/src/lib.rs | 203 --- crates/trigger-redis/src/spin.rs | 65 - crates/trigger-redis/src/tests.rs | 26 - .../trigger-redis/tests/rust/.cargo/.config | 2 - crates/trigger-redis/tests/rust/Cargo.lock | 303 ---- crates/trigger-redis/tests/rust/Cargo.toml | 13 - crates/trigger-redis/tests/rust/src/lib.rs | 25 - crates/trigger/Cargo.toml | 68 - crates/trigger/src/cli.rs | 304 ---- crates/trigger/src/cli/launch_metadata.rs | 86 -- crates/trigger/src/lib.rs | 483 ------- crates/trigger/src/loader.rs | 256 ---- crates/trigger/src/network.rs | 88 -- crates/trigger/src/runtime_config.rs | 840 ----------- .../trigger/src/runtime_config/client_tls.rs | 50 - .../trigger/src/runtime_config/key_value.rs | 195 --- crates/trigger/src/runtime_config/llm.rs | 82 -- crates/trigger/src/runtime_config/sqlite.rs | 240 ---- .../src/runtime_config/variables_provider.rs | 134 -- crates/trigger/src/stdio.rs | 336 ----- crates/variables/Cargo.toml | 29 - crates/variables/src/host_component.rs | 122 -- crates/variables/src/lib.rs | 4 - crates/variables/src/provider.rs | 3 - .../variables/src/provider/azure_key_vault.rs | 143 -- crates/variables/src/provider/env.rs | 141 -- crates/variables/src/provider/vault.rs | 65 - examples/spin-timer/Cargo.lock | 3 - examples/spin-timer/Cargo.toml | 2 +- 73 files changed, 263 insertions(+), 9849 deletions(-) delete mode 100644 crates/llm/Cargo.toml delete mode 100644 crates/llm/src/host_component.rs delete mode 100644 crates/llm/src/lib.rs delete mode 100644 crates/outbound-http/Cargo.toml delete mode 100644 crates/outbound-http/src/host_component.rs delete mode 100644 crates/outbound-http/src/host_impl.rs delete mode 100644 crates/outbound-http/src/lib.rs delete mode 100644 crates/outbound-mqtt/Cargo.toml delete mode 100644 crates/outbound-mqtt/src/host_component.rs delete mode 100644 crates/outbound-mqtt/src/lib.rs delete mode 100644 crates/outbound-mysql/Cargo.toml delete mode 100644 crates/outbound-mysql/src/lib.rs delete mode 100644 crates/outbound-pg/Cargo.toml delete mode 100644 crates/outbound-pg/src/lib.rs delete mode 100644 crates/outbound-redis/Cargo.toml delete mode 100644 crates/outbound-redis/src/host_component.rs delete mode 100644 crates/outbound-redis/src/lib.rs delete mode 100644 crates/testing/Cargo.toml delete mode 100644 crates/testing/src/lib.rs delete mode 100644 crates/trigger-http/Cargo.toml delete mode 100644 crates/trigger-http/benches/baseline.rs delete mode 100644 crates/trigger-http/benches/readme.md delete mode 100644 crates/trigger-http/benches/spin-http-benchmark/.cargo/config.toml delete mode 100644 crates/trigger-http/benches/spin-http-benchmark/Cargo.toml delete mode 100644 crates/trigger-http/benches/spin-http-benchmark/src/lib.rs delete mode 100644 crates/trigger-http/benches/wagi-benchmark/.cargo/config.toml delete mode 100644 crates/trigger-http/benches/wagi-benchmark/Cargo.toml delete mode 100644 crates/trigger-http/benches/wagi-benchmark/src/main.rs delete mode 100644 crates/trigger-http/src/handler.rs delete mode 100644 crates/trigger-http/src/instrument.rs delete mode 100644 crates/trigger-http/src/lib.rs delete mode 100644 crates/trigger-http/src/testdata/invalid-cert.pem delete mode 100644 crates/trigger-http/src/testdata/invalid-private-key.pem delete mode 100644 crates/trigger-http/src/testdata/valid-cert.pem delete mode 100644 crates/trigger-http/src/testdata/valid-private-key.pem delete mode 100644 crates/trigger-http/src/tls.rs delete mode 100644 crates/trigger-http/src/wagi.rs delete mode 100644 crates/trigger-http/tests/local.crt.pem delete mode 100644 crates/trigger-http/tests/local.key.pem delete mode 100644 crates/trigger-redis/Cargo.toml delete mode 100644 crates/trigger-redis/src/lib.rs delete mode 100644 crates/trigger-redis/src/spin.rs delete mode 100644 crates/trigger-redis/src/tests.rs delete mode 100644 crates/trigger-redis/tests/rust/.cargo/.config delete mode 100644 crates/trigger-redis/tests/rust/Cargo.lock delete mode 100644 crates/trigger-redis/tests/rust/Cargo.toml delete mode 100644 crates/trigger-redis/tests/rust/src/lib.rs delete mode 100644 crates/trigger/Cargo.toml delete mode 100644 crates/trigger/src/cli.rs delete mode 100644 crates/trigger/src/cli/launch_metadata.rs delete mode 100644 crates/trigger/src/lib.rs delete mode 100644 crates/trigger/src/loader.rs delete mode 100644 crates/trigger/src/network.rs delete mode 100644 crates/trigger/src/runtime_config.rs delete mode 100644 crates/trigger/src/runtime_config/client_tls.rs delete mode 100644 crates/trigger/src/runtime_config/key_value.rs delete mode 100644 crates/trigger/src/runtime_config/llm.rs delete mode 100644 crates/trigger/src/runtime_config/sqlite.rs delete mode 100644 crates/trigger/src/runtime_config/variables_provider.rs delete mode 100644 crates/trigger/src/stdio.rs delete mode 100644 crates/variables/Cargo.toml delete mode 100644 crates/variables/src/host_component.rs delete mode 100644 crates/variables/src/lib.rs delete mode 100644 crates/variables/src/provider.rs delete mode 100644 crates/variables/src/provider/azure_key_vault.rs delete mode 100644 crates/variables/src/provider/env.rs delete mode 100644 crates/variables/src/provider/vault.rs diff --git a/Cargo.lock b/Cargo.lock index 3295d890c..d6a8e17f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -860,12 +860,6 @@ dependencies = [ "serde 1.0.197", ] -[[package]] -name = "bytesize" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" - [[package]] name = "bzip2" version = "0.4.4" @@ -925,7 +919,7 @@ source = "git+https://github.com/huggingface/candle?rev=b80348d22f8f0dadb6cc4101 dependencies = [ "byteorder", "candle-gemm", - "half 2.4.0", + "half", "memmap2 0.7.1", "num-traits 0.2.18", "num_cpus", @@ -1016,7 +1010,7 @@ dependencies = [ "candle-gemm-common", "candle-gemm-f32", "dyn-stack", - "half 2.4.0", + "half", "lazy_static 1.4.0", "num-complex", "num-traits 0.2.18", @@ -1217,12 +1211,6 @@ dependencies = [ "toml 0.8.14", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.0.90" @@ -1299,17 +1287,6 @@ dependencies = [ "libloading", ] -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "bitflags 1.3.2", - "textwrap 0.11.0", - "unicode-width", -] - [[package]] name = "clap" version = "3.2.25" @@ -1324,7 +1301,7 @@ dependencies = [ "once_cell", "strsim 0.10.0", "termcolor", - "textwrap 0.16.1", + "textwrap", ] [[package]] @@ -1695,44 +1672,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" -dependencies = [ - "atty", - "cast", - "clap 2.34.0", - "criterion-plot", - "csv", - "futures", - "itertools 0.10.5", - "lazy_static 1.4.0", - "num-traits 0.2.18", - "oorandom", - "plotters", - "rayon", - "regex", - "serde 1.0.197", - "serde_cbor", - "serde_derive", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" -dependencies = [ - "cast", - "itertools 0.10.5", -] - [[package]] name = "crossbeam" version = "0.8.4" @@ -1842,27 +1781,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde 1.0.197", -] - -[[package]] -name = "csv-core" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" -dependencies = [ - "memchr", -] - [[package]] name = "ctrlc" version = "3.4.4" @@ -3155,12 +3073,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" - [[package]] name = "half" version = "2.4.0" @@ -4228,12 +4140,7 @@ version = "0.2.0-dev" source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" dependencies = [ "llm-base", - "llm-bloom", - "llm-gpt2", - "llm-gptj", - "llm-gptneox", "llm-llama", - "llm-mpt", "serde 1.0.197", "tracing", ] @@ -4245,7 +4152,7 @@ source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a87 dependencies = [ "bytemuck", "ggml", - "half 2.4.0", + "half", "llm-samplers", "memmap2 0.5.10", "partial_sort", @@ -4258,39 +4165,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "llm-bloom" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "llm-base", -] - -[[package]] -name = "llm-gpt2" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "bytemuck", - "llm-base", -] - -[[package]] -name = "llm-gptj" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "llm-base", -] - -[[package]] -name = "llm-gptneox" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "llm-base", -] - [[package]] name = "llm-llama" version = "0.2.0-dev" @@ -4300,14 +4174,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "llm-mpt" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "llm-base", -] - [[package]] name = "llm-samplers" version = "0.0.6" @@ -5127,12 +4993,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "oorandom" -version = "11.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" - [[package]] name = "opaque-debug" version = "0.3.1" @@ -5313,94 +5173,6 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -[[package]] -name = "outbound-http" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "http 0.2.12", - "reqwest 0.11.27", - "spin-app", - "spin-core", - "spin-expressions", - "spin-locked-app", - "spin-outbound-networking", - "spin-telemetry", - "spin-world", - "terminal", - "tracing", - "url", -] - -[[package]] -name = "outbound-mqtt" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "rumqttc", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tracing", -] - -[[package]] -name = "outbound-mysql" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "flate2", - "mysql_async", - "mysql_common", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "outbound-pg" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "native-tls", - "postgres-native-tls", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tokio-postgres", - "tracing", -] - -[[package]] -name = "outbound-redis" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "redis 0.21.7", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tracing", -] - [[package]] name = "overload" version = "0.1.1" @@ -5722,34 +5494,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "plotters" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" -dependencies = [ - "num-traits 0.2.18", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" - -[[package]] -name = "plotters-svg" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" -dependencies = [ - "plotters-backend", -] - [[package]] name = "polling" version = "2.8.0" @@ -6815,16 +6559,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "sanitize-filename" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" -dependencies = [ - "lazy_static 1.4.0", - "regex", -] - [[package]] name = "sanitize-filename" version = "0.5.0" @@ -6993,16 +6727,6 @@ dependencies = [ "serde 1.0.197", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half 1.8.3", - "serde 1.0.197", -] - [[package]] name = "serde_derive" version = "1.0.197" @@ -7973,18 +7697,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "spin-llm" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "bytesize", - "llm", - "spin-app", - "spin-core", - "spin-world", -] - [[package]] name = "spin-llm-local" version = "2.8.0-pre0" @@ -8114,7 +7826,6 @@ dependencies = [ "spin-loader", "spin-locked-app", "spin-manifest", - "spin-testing", "tempfile", "tokio", "tokio-util 0.7.10", @@ -8290,124 +8001,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "spin-testing" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "http 1.1.0", - "hyper 1.4.1", - "serde 1.0.197", - "serde_json", - "spin-app", - "spin-componentize", - "spin-core", - "spin-http", - "spin-trigger", - "tokio", - "tracing-subscriber", -] - -[[package]] -name = "spin-trigger" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "async-trait", - "clap 3.2.25", - "ctrlc", - "dirs 4.0.0", - "futures", - "http 1.1.0", - "indexmap 1.9.3", - "ipnet", - "outbound-http", - "outbound-mqtt", - "outbound-mysql", - "outbound-pg", - "outbound-redis", - "rustls-pemfile 2.1.2", - "rustls-pki-types", - "sanitize-filename 0.4.0", - "serde 1.0.197", - "serde_json", - "spin-app", - "spin-common", - "spin-componentize", - "spin-core", - "spin-expressions", - "spin-key-value", - "spin-key-value-azure", - "spin-key-value-redis", - "spin-key-value-sqlite", - "spin-llm", - "spin-llm-local", - "spin-llm-remote-http", - "spin-loader", - "spin-manifest", - "spin-outbound-networking", - "spin-serde", - "spin-sqlite", - "spin-sqlite-inproc", - "spin-sqlite-libsql", - "spin-telemetry", - "spin-variables", - "spin-world", - "tempfile", - "terminal", - "tokio", - "toml 0.5.11", - "tracing", - "url", - "wasmtime", - "wasmtime-wasi", - "wasmtime-wasi-http", -] - -[[package]] -name = "spin-trigger-http" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "async-trait", - "clap 3.2.25", - "criterion", - "futures", - "futures-util", - "http 1.1.0", - "http-body-util", - "hyper 1.4.1", - "hyper-util", - "indexmap 1.9.3", - "num_cpus", - "outbound-http", - "percent-encoding", - "rustls 0.22.4", - "rustls-pemfile 2.1.2", - "rustls-pki-types", - "serde 1.0.197", - "serde_json", - "spin-app", - "spin-core", - "spin-http", - "spin-outbound-networking", - "spin-telemetry", - "spin-testing", - "spin-trigger", - "spin-world", - "terminal", - "tls-listener", - "tokio", - "tokio-rustls 0.25.0", - "tracing", - "url", - "wasi-common", - "wasmtime", - "wasmtime-wasi", - "wasmtime-wasi-http", - "webpki-roots 0.26.1", -] - [[package]] name = "spin-trigger-http2" version = "2.8.0-pre0" @@ -8449,27 +8042,6 @@ dependencies = [ "webpki-roots 0.26.1", ] -[[package]] -name = "spin-trigger-redis" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "async-trait", - "futures", - "redis 0.21.7", - "serde 1.0.197", - "spin-app", - "spin-common", - "spin-core", - "spin-expressions", - "spin-telemetry", - "spin-testing", - "spin-trigger", - "spin-world", - "tokio", - "tracing", -] - [[package]] name = "spin-trigger2" version = "2.8.0-pre0" @@ -8478,7 +8050,7 @@ dependencies = [ "clap 3.2.25", "ctrlc", "futures", - "sanitize-filename 0.5.0", + "sanitize-filename", "serde 1.0.197", "serde_json", "spin-app", @@ -8505,29 +8077,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "spin-variables" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "async-trait", - "azure_core", - "azure_identity", - "azure_security_keyvault", - "dotenvy", - "once_cell", - "serde 1.0.197", - "spin-app", - "spin-core", - "spin-expressions", - "spin-world", - "thiserror", - "tokio", - "toml 0.5.11", - "tracing", - "vaultrs", -] - [[package]] name = "spin-world" version = "2.8.0-pre0" @@ -8885,15 +8434,6 @@ dependencies = [ "wasmtime-wasi-http", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "textwrap" version = "0.16.1" @@ -8982,16 +8522,6 @@ dependencies = [ "lazy_static 0.2.11", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde 1.0.197", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -9898,33 +9428,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasi-common" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86fd41e1e26ff6af9451c6a332a5ce5f5283ca51e87d875cdd9a05305598ee3" -dependencies = [ - "anyhow", - "bitflags 2.5.0", - "cap-fs-ext", - "cap-rand", - "cap-std 3.0.0", - "cap-time-ext", - "fs-set-times", - "io-extras", - "io-lifetimes 2.0.3", - "log", - "once_cell", - "rustix 0.38.32", - "system-interface", - "thiserror", - "tokio", - "tracing", - "wasmtime", - "wiggle", - "windows-sys 0.52.0", -] - [[package]] name = "wasite" version = "0.1.0" diff --git a/build.rs b/build.rs index 7ddf012e3..d4cd7acf0 100644 --- a/build.rs +++ b/build.rs @@ -68,17 +68,17 @@ error: the `wasm32-wasi` target is not installed std::fs::create_dir_all("target/test-programs").unwrap(); build_wasm_test_program("core-wasi-test.wasm", "crates/core/tests/core-wasi-test"); - build_wasm_test_program("redis-rust.wasm", "crates/trigger-redis/tests/rust"); - - build_wasm_test_program( - "spin-http-benchmark.wasm", - "crates/trigger-http/benches/spin-http-benchmark", - ); - build_wasm_test_program( - "wagi-benchmark.wasm", - "crates/trigger-http/benches/wagi-benchmark", - ); - build_wasm_test_program("timer_app_example.wasm", "examples/spin-timer/app-example"); + // build_wasm_test_program("redis-rust.wasm", "crates/trigger-redis/tests/rust"); + + // build_wasm_test_program( + // "spin-http-benchmark.wasm", + // "crates/trigger-http/benches/spin-http-benchmark", + // ); + // build_wasm_test_program( + // "wagi-benchmark.wasm", + // "crates/trigger-http/benches/wagi-benchmark", + // ); + // build_wasm_test_program("timer_app_example.wasm", "examples/spin-timer/app-example"); cargo_build(TIMER_TRIGGER_INTEGRATION_TEST); } diff --git a/crates/llm/Cargo.toml b/crates/llm/Cargo.toml deleted file mode 100644 index 08c193e0b..000000000 --- a/crates/llm/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "spin-llm" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[dependencies] -anyhow = "1.0" -bytesize = "1.1" -llm = { git = "https://github.com/rustformers/llm", rev = "2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663", features = [ - "tokenizers-remote", - "models", -], default-features = false } -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-world = { path = "../world" } diff --git a/crates/llm/src/host_component.rs b/crates/llm/src/host_component.rs deleted file mode 100644 index 8574e6bb0..000000000 --- a/crates/llm/src/host_component.rs +++ /dev/null @@ -1,49 +0,0 @@ -use spin_app::DynamicHostComponent; -use spin_core::HostComponent; - -use crate::{LlmDispatch, LlmEngine, AI_MODELS_KEY}; - -pub struct LlmComponent { - create_engine: Box Box + Send + Sync>, -} - -impl LlmComponent { - pub fn new(create_engine: F) -> Self - where - F: Fn() -> Box + Send + Sync + 'static, - { - Self { - create_engine: Box::new(create_engine), - } - } -} - -impl HostComponent for LlmComponent { - type Data = LlmDispatch; - - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - spin_world::v1::llm::add_to_linker(linker, get)?; - spin_world::v2::llm::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - LlmDispatch { - engine: (self.create_engine)(), - allowed_models: Default::default(), - } - } -} - -impl DynamicHostComponent for LlmComponent { - fn update_data( - &self, - data: &mut Self::Data, - component: &spin_app::AppComponent, - ) -> anyhow::Result<()> { - data.allowed_models = component.get_metadata(AI_MODELS_KEY)?.unwrap_or_default(); - Ok(()) - } -} diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs deleted file mode 100644 index 399c2fbca..000000000 --- a/crates/llm/src/lib.rs +++ /dev/null @@ -1,112 +0,0 @@ -pub mod host_component; - -use spin_app::MetadataKey; -use spin_core::async_trait; -use spin_world::v1::llm::{self as v1}; -use spin_world::v2::llm::{self as v2}; -use std::collections::HashSet; - -pub use crate::host_component::LlmComponent; - -pub const MODEL_ALL_MINILM_L6_V2: &str = "all-minilm-l6-v2"; -pub const AI_MODELS_KEY: MetadataKey> = MetadataKey::new("ai_models"); - -#[async_trait] -pub trait LlmEngine: Send + Sync { - async fn infer( - &mut self, - model: v1::InferencingModel, - prompt: String, - params: v2::InferencingParams, - ) -> Result; - - async fn generate_embeddings( - &mut self, - model: v2::EmbeddingModel, - data: Vec, - ) -> Result; -} - -pub struct LlmDispatch { - engine: Box, - allowed_models: HashSet, -} - -#[async_trait] -impl v2::Host for LlmDispatch { - async fn infer( - &mut self, - model: v2::InferencingModel, - prompt: String, - params: Option, - ) -> Result { - if !self.allowed_models.contains(&model) { - return Err(access_denied_error(&model)); - } - self.engine - .infer( - model, - prompt, - params.unwrap_or(v2::InferencingParams { - max_tokens: 100, - repeat_penalty: 1.1, - repeat_penalty_last_n_token_count: 64, - temperature: 0.8, - top_k: 40, - top_p: 0.9, - }), - ) - .await - } - - async fn generate_embeddings( - &mut self, - m: v1::EmbeddingModel, - data: Vec, - ) -> Result { - if !self.allowed_models.contains(&m) { - return Err(access_denied_error(&m)); - } - self.engine.generate_embeddings(m, data).await - } - - fn convert_error(&mut self, error: v2::Error) -> anyhow::Result { - Ok(error) - } -} - -#[async_trait] -impl v1::Host for LlmDispatch { - async fn infer( - &mut self, - model: v1::InferencingModel, - prompt: String, - params: Option, - ) -> Result { - ::infer(self, model, prompt, params.map(Into::into)) - .await - .map(Into::into) - .map_err(Into::into) - } - - async fn generate_embeddings( - &mut self, - model: v1::EmbeddingModel, - data: Vec, - ) -> Result { - ::generate_embeddings(self, model, data) - .await - .map(Into::into) - .map_err(Into::into) - } - - fn convert_error(&mut self, error: v1::Error) -> anyhow::Result { - Ok(error) - } -} - -fn access_denied_error(model: &str) -> v2::Error { - v2::Error::InvalidInput(format!( - "The component does not have access to use '{model}'. To give the component access, add '{model}' to the 'ai_models' key for the component in your spin.toml manifest" - )) -} diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml index 8dcbd55b7..92ac7ae6e 100644 --- a/crates/oci/Cargo.toml +++ b/crates/oci/Cargo.toml @@ -31,6 +31,3 @@ tokio = { version = "1", features = ["fs"] } tokio-util = { version = "0.7.9", features = ["compat"] } tracing = { workspace = true } walkdir = "2.3" - -[dev-dependencies] -spin-testing = { path = "../testing" } diff --git a/crates/oci/src/client.rs b/crates/oci/src/client.rs index 92015b945..218b4cb66 100644 --- a/crates/oci/src/client.rs +++ b/crates/oci/src/client.rs @@ -824,252 +824,252 @@ mod test { } } - #[tokio::test] - async fn can_assemble_layers() { - use spin_locked_app::locked::LockedComponent; - use tokio::io::AsyncWriteExt; - - let working_dir = tempfile::tempdir().unwrap(); - - // Set up component/file directory tree - // - // create component1 and component2 dirs - let _ = tokio::fs::create_dir(working_dir.path().join("component1").as_path()).await; - let _ = tokio::fs::create_dir(working_dir.path().join("component2").as_path()).await; - - // create component "wasm" files - let mut c1 = tokio::fs::File::create(working_dir.path().join("component1.wasm")) - .await - .expect("should create component wasm file"); - c1.write_all(b"c1") - .await - .expect("should write component wasm contents"); - let mut c2 = tokio::fs::File::create(working_dir.path().join("component2.wasm")) - .await - .expect("should create component wasm file"); - c2.write_all(b"c2") - .await - .expect("should write component wasm contents"); - - // component1 files - let mut c1f1 = tokio::fs::File::create(working_dir.path().join("component1").join("bar")) - .await - .expect("should create component file"); - c1f1.write_all(b"bar") - .await - .expect("should write file contents"); - let mut c1f2 = tokio::fs::File::create(working_dir.path().join("component1").join("baz")) - .await - .expect("should create component file"); - c1f2.write_all(b"baz") - .await - .expect("should write file contents"); - - // component2 files - let mut c2f1 = tokio::fs::File::create(working_dir.path().join("component2").join("baz")) - .await - .expect("should create component file"); - c2f1.write_all(b"baz") - .await - .expect("should write file contents"); - - #[derive(Clone)] - struct TestCase { - name: &'static str, - opts: Option, - locked_components: Vec, - expected_layer_count: usize, - expected_error: Option<&'static str>, - } - - let tests: Vec = [ - TestCase { - name: "Two component layers", - opts: None, - locked_components: spin_testing::from_json!([{ - "id": "component1", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), - "digest": "digest", - }}, - { - "id": "component2", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component2.wasm").to_str().unwrap()), - "digest": "digest", - }}]), - expected_layer_count: 2, - expected_error: None, - }, - TestCase { - name: "One component layer and two file layers", - opts: Some(ClientOpts{content_ref_inline_max_size: 0}), - locked_components: spin_testing::from_json!([{ - "id": "component1", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), - "digest": "digest", - }, - "files": [ - { - "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), - "path": working_dir.path().join("component1").join("bar").to_str().unwrap() - }, - { - "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), - "path": working_dir.path().join("component1").join("baz").to_str().unwrap() - } - ] - }]), - expected_layer_count: 3, - expected_error: None, - }, - TestCase { - name: "One component layer and one file with inlined content", - opts: None, - locked_components: spin_testing::from_json!([{ - "id": "component1", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), - "digest": "digest", - }, - "files": [ - { - "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), - "path": working_dir.path().join("component1").join("bar").to_str().unwrap() - } - ] - }]), - expected_layer_count: 1, - expected_error: None, - }, - TestCase { - name: "Component has no source", - opts: None, - locked_components: spin_testing::from_json!([{ - "id": "component1", - "source": { - "content_type": "application/wasm", - "source": "", - "digest": "digest", - } - }]), - expected_layer_count: 0, - expected_error: Some("Invalid URL: \"\""), - }, - TestCase { - name: "Duplicate component sources", - opts: None, - locked_components: spin_testing::from_json!([{ - "id": "component1", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), - "digest": "digest", - }}, - { - "id": "component2", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), - "digest": "digest", - }}]), - expected_layer_count: 1, - expected_error: None, - }, - TestCase { - name: "Duplicate file paths", - opts: Some(ClientOpts{content_ref_inline_max_size: 0}), - locked_components: spin_testing::from_json!([{ - "id": "component1", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), - "digest": "digest", - }, - "files": [ - { - "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), - "path": working_dir.path().join("component1").join("bar").to_str().unwrap() - }, - { - "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), - "path": working_dir.path().join("component1").join("baz").to_str().unwrap() - } - ]}, - { - "id": "component2", - "source": { - "content_type": "application/wasm", - "source": format!("file://{}", working_dir.path().join("component2.wasm").to_str().unwrap()), - "digest": "digest", - }, - "files": [ - { - "source": format!("file://{}", working_dir.path().join("component2").to_str().unwrap()), - "path": working_dir.path().join("component2").join("baz").to_str().unwrap() - } - ] - }]), - expected_layer_count: 4, - expected_error: None, - }, - ] - .to_vec(); - - for tc in tests { - let triggers = Default::default(); - let metadata = Default::default(); - let variables = Default::default(); - let mut locked = LockedApp { - spin_lock_version: Default::default(), - components: tc.locked_components, - triggers, - metadata, - variables, - must_understand: Default::default(), - host_requirements: Default::default(), - }; - - let mut client = Client::new(false, Some(working_dir.path().to_path_buf())) - .await - .expect("should create new client"); - if let Some(o) = tc.opts { - client.opts = o; - } - - match tc.expected_error { - Some(e) => { - assert_eq!( - e, - client - .assemble_layers(&mut locked, AssemblyMode::Simple) - .await - .unwrap_err() - .to_string(), - "{}", - tc.name - ) - } - None => { - assert_eq!( - tc.expected_layer_count, - client - .assemble_layers(&mut locked, AssemblyMode::Simple) - .await - .unwrap() - .len(), - "{}", - tc.name - ) - } - } - } - } + // #[tokio::test] + // async fn can_assemble_layers() { + // use spin_locked_app::locked::LockedComponent; + // use tokio::io::AsyncWriteExt; + + // let working_dir = tempfile::tempdir().unwrap(); + + // // Set up component/file directory tree + // // + // // create component1 and component2 dirs + // let _ = tokio::fs::create_dir(working_dir.path().join("component1").as_path()).await; + // let _ = tokio::fs::create_dir(working_dir.path().join("component2").as_path()).await; + + // // create component "wasm" files + // let mut c1 = tokio::fs::File::create(working_dir.path().join("component1.wasm")) + // .await + // .expect("should create component wasm file"); + // c1.write_all(b"c1") + // .await + // .expect("should write component wasm contents"); + // let mut c2 = tokio::fs::File::create(working_dir.path().join("component2.wasm")) + // .await + // .expect("should create component wasm file"); + // c2.write_all(b"c2") + // .await + // .expect("should write component wasm contents"); + + // // component1 files + // let mut c1f1 = tokio::fs::File::create(working_dir.path().join("component1").join("bar")) + // .await + // .expect("should create component file"); + // c1f1.write_all(b"bar") + // .await + // .expect("should write file contents"); + // let mut c1f2 = tokio::fs::File::create(working_dir.path().join("component1").join("baz")) + // .await + // .expect("should create component file"); + // c1f2.write_all(b"baz") + // .await + // .expect("should write file contents"); + + // // component2 files + // let mut c2f1 = tokio::fs::File::create(working_dir.path().join("component2").join("baz")) + // .await + // .expect("should create component file"); + // c2f1.write_all(b"baz") + // .await + // .expect("should write file contents"); + + // #[derive(Clone)] + // struct TestCase { + // name: &'static str, + // opts: Option, + // locked_components: Vec, + // expected_layer_count: usize, + // expected_error: Option<&'static str>, + // } + + // let tests: Vec = [ + // TestCase { + // name: "Two component layers", + // opts: None, + // locked_components: spin_testing::from_json!([{ + // "id": "component1", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), + // "digest": "digest", + // }}, + // { + // "id": "component2", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component2.wasm").to_str().unwrap()), + // "digest": "digest", + // }}]), + // expected_layer_count: 2, + // expected_error: None, + // }, + // TestCase { + // name: "One component layer and two file layers", + // opts: Some(ClientOpts{content_ref_inline_max_size: 0}), + // locked_components: spin_testing::from_json!([{ + // "id": "component1", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), + // "digest": "digest", + // }, + // "files": [ + // { + // "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), + // "path": working_dir.path().join("component1").join("bar").to_str().unwrap() + // }, + // { + // "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), + // "path": working_dir.path().join("component1").join("baz").to_str().unwrap() + // } + // ] + // }]), + // expected_layer_count: 3, + // expected_error: None, + // }, + // TestCase { + // name: "One component layer and one file with inlined content", + // opts: None, + // locked_components: spin_testing::from_json!([{ + // "id": "component1", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), + // "digest": "digest", + // }, + // "files": [ + // { + // "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), + // "path": working_dir.path().join("component1").join("bar").to_str().unwrap() + // } + // ] + // }]), + // expected_layer_count: 1, + // expected_error: None, + // }, + // TestCase { + // name: "Component has no source", + // opts: None, + // locked_components: spin_testing::from_json!([{ + // "id": "component1", + // "source": { + // "content_type": "application/wasm", + // "source": "", + // "digest": "digest", + // } + // }]), + // expected_layer_count: 0, + // expected_error: Some("Invalid URL: \"\""), + // }, + // TestCase { + // name: "Duplicate component sources", + // opts: None, + // locked_components: spin_testing::from_json!([{ + // "id": "component1", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), + // "digest": "digest", + // }}, + // { + // "id": "component2", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), + // "digest": "digest", + // }}]), + // expected_layer_count: 1, + // expected_error: None, + // }, + // TestCase { + // name: "Duplicate file paths", + // opts: Some(ClientOpts{content_ref_inline_max_size: 0}), + // locked_components: spin_testing::from_json!([{ + // "id": "component1", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component1.wasm").to_str().unwrap()), + // "digest": "digest", + // }, + // "files": [ + // { + // "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), + // "path": working_dir.path().join("component1").join("bar").to_str().unwrap() + // }, + // { + // "source": format!("file://{}", working_dir.path().join("component1").to_str().unwrap()), + // "path": working_dir.path().join("component1").join("baz").to_str().unwrap() + // } + // ]}, + // { + // "id": "component2", + // "source": { + // "content_type": "application/wasm", + // "source": format!("file://{}", working_dir.path().join("component2.wasm").to_str().unwrap()), + // "digest": "digest", + // }, + // "files": [ + // { + // "source": format!("file://{}", working_dir.path().join("component2").to_str().unwrap()), + // "path": working_dir.path().join("component2").join("baz").to_str().unwrap() + // } + // ] + // }]), + // expected_layer_count: 4, + // expected_error: None, + // }, + // ] + // .to_vec(); + + // for tc in tests { + // let triggers = Default::default(); + // let metadata = Default::default(); + // let variables = Default::default(); + // let mut locked = LockedApp { + // spin_lock_version: Default::default(), + // components: tc.locked_components, + // triggers, + // metadata, + // variables, + // must_understand: Default::default(), + // host_requirements: Default::default(), + // }; + + // let mut client = Client::new(false, Some(working_dir.path().to_path_buf())) + // .await + // .expect("should create new client"); + // if let Some(o) = tc.opts { + // client.opts = o; + // } + + // match tc.expected_error { + // Some(e) => { + // assert_eq!( + // e, + // client + // .assemble_layers(&mut locked, AssemblyMode::Simple) + // .await + // .unwrap_err() + // .to_string(), + // "{}", + // tc.name + // ) + // } + // None => { + // assert_eq!( + // tc.expected_layer_count, + // client + // .assemble_layers(&mut locked, AssemblyMode::Simple) + // .await + // .unwrap() + // .len(), + // "{}", + // tc.name + // ) + // } + // } + // } + // } fn annotatable_app() -> LockedApp { let mut meta_builder = spin_locked_app::values::ValuesMapBuilder::new(); diff --git a/crates/outbound-http/Cargo.toml b/crates/outbound-http/Cargo.toml deleted file mode 100644 index 73617a08c..000000000 --- a/crates/outbound-http/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "outbound-http" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[lib] -doctest = false - -[dependencies] -anyhow = "1.0" -http = "0.2" -reqwest = { version = "0.11", features = ["gzip"] } -spin-app = { path = "../app", optional = true } -spin-core = { path = "../core", optional = true } -spin-expressions = { path = "../expressions", optional = true } -spin-locked-app = { path = "../locked-app" } -spin-outbound-networking = { path = "../outbound-networking" } -spin-world = { path = "../world", optional = true } -spin-telemetry = { path = "../telemetry" } -terminal = { path = "../terminal" } -tracing = { workspace = true } -url = "2.2.1" - -[features] -default = ["runtime"] -runtime = ["dep:spin-app", "dep:spin-core", "dep:spin-expressions", "dep:spin-world"] - -[lints] -workspace = true diff --git a/crates/outbound-http/src/host_component.rs b/crates/outbound-http/src/host_component.rs deleted file mode 100644 index 0fd60a05d..000000000 --- a/crates/outbound-http/src/host_component.rs +++ /dev/null @@ -1,41 +0,0 @@ -use anyhow::Result; - -use spin_app::DynamicHostComponent; -use spin_core::{Data, HostComponent, Linker}; -use spin_outbound_networking::{AllowedHostsConfig, ALLOWED_HOSTS_KEY}; -use spin_world::v1::http; - -use crate::host_impl::OutboundHttp; - -pub struct OutboundHttpComponent { - pub resolver: spin_expressions::SharedPreparedResolver, -} - -impl HostComponent for OutboundHttpComponent { - type Data = OutboundHttp; - - fn add_to_linker( - linker: &mut Linker, - get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> Result<()> { - http::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - Default::default() - } -} - -impl DynamicHostComponent for OutboundHttpComponent { - fn update_data( - &self, - data: &mut Self::Data, - component: &spin_app::AppComponent, - ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(ALLOWED_HOSTS_KEY)? - .unwrap_or_default(); - data.allowed_hosts = AllowedHostsConfig::parse(&hosts, self.resolver.get().unwrap())?; - Ok(()) - } -} diff --git a/crates/outbound-http/src/host_impl.rs b/crates/outbound-http/src/host_impl.rs deleted file mode 100644 index cffdc5cc1..000000000 --- a/crates/outbound-http/src/host_impl.rs +++ /dev/null @@ -1,208 +0,0 @@ -use anyhow::Result; -use http::{HeaderMap, Uri}; -use reqwest::Client; -use spin_core::async_trait; -use spin_outbound_networking::{AllowedHostsConfig, OutboundUrl}; -use spin_world::v1::{ - http as outbound_http, - http_types::{self, Headers, HttpError, Method, Request, Response}, -}; -use tracing::{field::Empty, instrument, Level}; - -/// A very simple implementation for outbound HTTP requests. -#[derive(Default, Clone)] -pub struct OutboundHttp { - /// List of hosts guest modules are allowed to make requests to. - pub allowed_hosts: AllowedHostsConfig, - /// During an incoming HTTP request, origin is set to the host of that incoming HTTP request. - /// This is used to direct outbound requests to the same host when allowed. - pub origin: String, - client: Option, -} - -impl OutboundHttp { - /// Check if guest module is allowed to send request to URL, based on the list of - /// allowed hosts defined by the runtime. If the url passed in is a relative path, - /// only allow if allowed_hosts contains `self`. If the list of allowed hosts contains - /// `insecure:allow-all`, then all hosts are allowed. - /// If `None` is passed, the guest module is not allowed to send the request. - fn is_allowed(&mut self, url: &str) -> Result { - if url.starts_with('/') { - return Ok(self.allowed_hosts.allows_relative_url(&["http", "https"])); - } - - Ok(OutboundUrl::parse(url, "https") - .map(|u| self.allowed_hosts.allows(&u)) - .unwrap_or_default()) - } -} - -#[async_trait] -impl outbound_http::Host for OutboundHttp { - #[instrument(name = "spin_outbound_http.send_request", skip_all, err(level = Level::INFO), - fields(otel.kind = "client", url.full = Empty, http.request.method = Empty, - http.response.status_code = Empty, otel.name = Empty, server.address = Empty, server.port = Empty))] - async fn send_request(&mut self, req: Request) -> Result { - let current_span = tracing::Span::current(); - let method = format!("{:?}", req.method) - .strip_prefix("Method::") - .unwrap_or("_OTHER") - .to_uppercase(); - current_span.record("otel.name", method.clone()); - current_span.record("url.full", req.uri.clone()); - current_span.record("http.request.method", method); - if let Ok(uri) = req.uri.parse::() { - if let Some(authority) = uri.authority() { - current_span.record("server.address", authority.host()); - if let Some(port) = authority.port() { - current_span.record("server.port", port.as_u16()); - } - } - } - - tracing::trace!("Attempting to send outbound HTTP request to {}", req.uri); - if !self - .is_allowed(&req.uri) - .map_err(|_| HttpError::RuntimeError)? - { - tracing::info!("Destination not allowed: {}", req.uri); - if let Some((scheme, host_and_port)) = scheme_host_and_port(&req.uri) { - terminal::warn!("A component tried to make a HTTP request to non-allowed host '{host_and_port}'."); - eprintln!("To allow requests, add 'allowed_outbound_hosts = [\"{scheme}://{host_and_port}\"]' to the manifest component section."); - } - return Err(HttpError::DestinationNotAllowed); - } - - let method = method_from(req.method); - - let abs_url = if req.uri.starts_with('/') { - format!("{}{}", self.origin, req.uri) - } else { - req.uri.clone() - }; - - let req_url = reqwest::Url::parse(&abs_url).map_err(|_| HttpError::InvalidUrl)?; - - let mut headers = request_headers(req.headers).map_err(|_| HttpError::RuntimeError)?; - spin_telemetry::inject_trace_context(&mut headers); - let body = req.body.unwrap_or_default().to_vec(); - - if !req.params.is_empty() { - tracing::warn!("HTTP params field is deprecated"); - } - - // Allow reuse of Client's internal connection pool for multiple requests - // in a single component execution - let client = self.client.get_or_insert_with(Default::default); - - let resp = client - .request(method, req_url) - .headers(headers) - .body(body) - .send() - .await - .map_err(log_reqwest_error)?; - tracing::trace!("Returning response from outbound request to {}", req.uri); - current_span.record("http.response.status_code", resp.status().as_u16()); - response_from_reqwest(resp).await - } -} - -impl http_types::Host for OutboundHttp { - fn convert_http_error(&mut self, error: HttpError) -> Result { - Ok(error) - } -} - -fn log_reqwest_error(err: reqwest::Error) -> HttpError { - let error_desc = if err.is_timeout() { - "timeout error" - } else if err.is_connect() { - "connection error" - } else if err.is_body() || err.is_decode() { - "message body error" - } else if err.is_request() { - "request error" - } else { - "error" - }; - tracing::warn!( - "Outbound HTTP {}: URL {}, error detail {:?}", - error_desc, - err.url() - .map(|u| u.to_string()) - .unwrap_or_else(|| "".to_owned()), - err - ); - HttpError::RuntimeError -} - -fn method_from(m: Method) -> http::Method { - match m { - Method::Get => http::Method::GET, - Method::Post => http::Method::POST, - Method::Put => http::Method::PUT, - Method::Delete => http::Method::DELETE, - Method::Patch => http::Method::PATCH, - Method::Head => http::Method::HEAD, - Method::Options => http::Method::OPTIONS, - } -} - -async fn response_from_reqwest(res: reqwest::Response) -> Result { - let status = res.status().as_u16(); - let headers = response_headers(res.headers()).map_err(|_| HttpError::RuntimeError)?; - - let body = Some( - res.bytes() - .await - .map_err(|_| HttpError::RuntimeError)? - .to_vec(), - ); - - Ok(Response { - status, - headers, - body, - }) -} - -fn request_headers(h: Headers) -> anyhow::Result { - let mut res = HeaderMap::new(); - for (k, v) in h { - res.insert( - http::header::HeaderName::try_from(k)?, - http::header::HeaderValue::try_from(v)?, - ); - } - Ok(res) -} - -fn response_headers(h: &HeaderMap) -> anyhow::Result>> { - let mut res: Vec<(String, String)> = vec![]; - - for (k, v) in h { - res.push(( - k.to_string(), - std::str::from_utf8(v.as_bytes())?.to_string(), - )); - } - - Ok(Some(res)) -} - -/// Returns both the scheme and the `$HOST:$PORT` for the url string -/// -/// Returns `None` if the url cannot be parsed or if it does not contain a host -fn scheme_host_and_port(url: &str) -> Option<(String, String)> { - url::Url::parse(url).ok().and_then(|u| { - u.host_str().map(|h| { - let mut host = h.to_owned(); - if let Some(p) = u.port() { - use std::fmt::Write; - write!(&mut host, ":{p}").unwrap(); - } - (u.scheme().to_owned(), host) - }) - }) -} diff --git a/crates/outbound-http/src/lib.rs b/crates/outbound-http/src/lib.rs deleted file mode 100644 index 33a726f34..000000000 --- a/crates/outbound-http/src/lib.rs +++ /dev/null @@ -1,11 +0,0 @@ -#[cfg(feature = "runtime")] -mod host_component; -#[cfg(feature = "runtime")] -mod host_impl; - -#[cfg(feature = "runtime")] -pub use host_component::OutboundHttpComponent; - -use spin_locked_app::MetadataKey; - -pub const ALLOWED_HTTP_HOSTS_KEY: MetadataKey> = MetadataKey::new("allowed_http_hosts"); diff --git a/crates/outbound-mqtt/Cargo.toml b/crates/outbound-mqtt/Cargo.toml deleted file mode 100644 index 385bdfb11..000000000 --- a/crates/outbound-mqtt/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "outbound-mqtt" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[lib] -doctest = false - -[dependencies] -anyhow = "1.0" -rumqttc = { version = "0.24", features = ["url"] } -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-expressions = { path = "../expressions" } -spin-world = { path = "../world" } -spin-outbound-networking = { path = "../outbound-networking" } -table = { path = "../table" } -tokio = { version = "1", features = ["sync"] } -tracing = { workspace = true } - -[lints] -workspace = true diff --git a/crates/outbound-mqtt/src/host_component.rs b/crates/outbound-mqtt/src/host_component.rs deleted file mode 100644 index df242a54b..000000000 --- a/crates/outbound-mqtt/src/host_component.rs +++ /dev/null @@ -1,41 +0,0 @@ -use anyhow::Context; -use spin_app::DynamicHostComponent; -use spin_core::HostComponent; - -use crate::OutboundMqtt; - -pub struct OutboundMqttComponent { - pub resolver: spin_expressions::SharedPreparedResolver, -} - -impl HostComponent for OutboundMqttComponent { - type Data = OutboundMqtt; - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - spin_world::v2::mqtt::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - Default::default() - } -} - -impl DynamicHostComponent for OutboundMqttComponent { - fn update_data( - &self, - data: &mut Self::Data, - component: &spin_app::AppComponent, - ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(spin_outbound_networking::ALLOWED_HOSTS_KEY)? - .unwrap_or_default(); - data.allowed_hosts = spin_outbound_networking::AllowedHostsConfig::parse( - &hosts, - self.resolver.get().unwrap(), - ) - .context("`allowed_outbound_hosts` contained an invalid url")?; - Ok(()) - } -} diff --git a/crates/outbound-mqtt/src/lib.rs b/crates/outbound-mqtt/src/lib.rs deleted file mode 100644 index b17d11055..000000000 --- a/crates/outbound-mqtt/src/lib.rs +++ /dev/null @@ -1,158 +0,0 @@ -mod host_component; - -use std::time::Duration; - -use anyhow::Result; -use rumqttc::{AsyncClient, Event, Incoming, Outgoing, QoS}; -use spin_core::{async_trait, wasmtime::component::Resource}; -use spin_world::v2::mqtt::{self as v2, Connection as MqttConnection, Error, Qos}; - -pub use host_component::OutboundMqttComponent; -use tracing::{instrument, Level}; - -pub struct OutboundMqtt { - allowed_hosts: spin_outbound_networking::AllowedHostsConfig, - connections: table::Table<(AsyncClient, rumqttc::EventLoop)>, -} - -impl Default for OutboundMqtt { - fn default() -> Self { - Self { - allowed_hosts: Default::default(), - connections: table::Table::new(1024), - } - } -} - -const MQTT_CHANNEL_CAP: usize = 1000; - -impl OutboundMqtt { - fn is_address_allowed(&self, address: &str) -> bool { - spin_outbound_networking::check_url(address, "mqtt", &self.allowed_hosts) - } - - async fn establish_connection( - &mut self, - address: String, - username: String, - password: String, - keep_alive_interval: Duration, - ) -> Result, Error> { - let mut conn_opts = rumqttc::MqttOptions::parse_url(address).map_err(|e| { - tracing::error!("MQTT URL parse error: {e:?}"); - Error::InvalidAddress - })?; - conn_opts.set_credentials(username, password); - conn_opts.set_keep_alive(keep_alive_interval); - let (client, event_loop) = AsyncClient::new(conn_opts, MQTT_CHANNEL_CAP); - - self.connections - .push((client, event_loop)) - .map(Resource::new_own) - .map_err(|_| Error::TooManyConnections) - } -} - -impl v2::Host for OutboundMqtt { - fn convert_error(&mut self, error: Error) -> Result { - Ok(error) - } -} - -#[async_trait] -impl v2::HostConnection for OutboundMqtt { - #[instrument(name = "spin_outbound_mqtt.open_connection", skip(self, password), err(level = Level::INFO), fields(otel.kind = "client"))] - async fn open( - &mut self, - address: String, - username: String, - password: String, - keep_alive_interval: u64, - ) -> Result, Error> { - if !self.is_address_allowed(&address) { - return Err(v2::Error::ConnectionFailed(format!( - "address {address} is not permitted" - ))); - } - self.establish_connection( - address, - username, - password, - Duration::from_secs(keep_alive_interval), - ) - .await - } - - /// Publish a message to the MQTT broker. - /// - /// OTEL trace propagation is not directly supported in MQTT V3. You will need to embed the - /// current trace context into the payload yourself. - /// https://w3c.github.io/trace-context-mqtt/#mqtt-v3-recommendation. - #[instrument(name = "spin_outbound_mqtt.publish", skip(self, connection, payload), err(level = Level::INFO), - fields(otel.kind = "producer", otel.name = format!("{} publish", topic), messaging.operation = "publish", - messaging.system = "mqtt"))] - async fn publish( - &mut self, - connection: Resource, - topic: String, - payload: Vec, - qos: Qos, - ) -> Result<(), Error> { - let (client, eventloop) = self.get_conn(connection).await.map_err(other_error)?; - let qos = convert_to_mqtt_qos_value(qos); - - // Message published to EventLoop (not MQTT Broker) - client - .publish_bytes(topic, qos, false, payload.into()) - .await - .map_err(other_error)?; - - // Poll event loop until outgoing publish event is iterated over to send the message to MQTT broker or capture/throw error. - // We may revisit this later to manage long running connections, high throughput use cases and their issues in the connection pool. - loop { - let event = eventloop - .poll() - .await - .map_err(|err| v2::Error::ConnectionFailed(err.to_string()))?; - - match (qos, event) { - (QoS::AtMostOnce, Event::Outgoing(Outgoing::Publish(_))) - | (QoS::AtLeastOnce, Event::Incoming(Incoming::PubAck(_))) - | (QoS::ExactlyOnce, Event::Incoming(Incoming::PubComp(_))) => break, - - (_, _) => continue, - } - } - Ok(()) - } - - fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { - self.connections.remove(connection.rep()); - Ok(()) - } -} - -fn convert_to_mqtt_qos_value(qos: Qos) -> rumqttc::QoS { - match qos { - Qos::AtMostOnce => rumqttc::QoS::AtMostOnce, - Qos::AtLeastOnce => rumqttc::QoS::AtLeastOnce, - Qos::ExactlyOnce => rumqttc::QoS::ExactlyOnce, - } -} - -fn other_error(e: impl std::fmt::Display) -> Error { - Error::Other(e.to_string()) -} - -impl OutboundMqtt { - async fn get_conn( - &mut self, - connection: Resource, - ) -> Result<&mut (AsyncClient, rumqttc::EventLoop), Error> { - self.connections - .get_mut(connection.rep()) - .ok_or(Error::Other( - "could not find connection for resource".into(), - )) - } -} diff --git a/crates/outbound-mysql/Cargo.toml b/crates/outbound-mysql/Cargo.toml deleted file mode 100644 index bfc49cd67..000000000 --- a/crates/outbound-mysql/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "outbound-mysql" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[lib] -doctest = false - -[dependencies] -anyhow = "1.0" -flate2 = "1.0.17" -# Removing default features for mysql_async to remove flate2/zlib feature -mysql_async = { version = "0.33.0", default-features = false, features = [ - "native-tls-tls", -] } -# Removing default features for mysql_common to remove flate2/zlib feature -mysql_common = { version = "0.31.0", default-features = false } -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-expressions = { path = "../expressions" } -spin-outbound-networking = { path = "../outbound-networking" } -spin-world = { path = "../world" } -table = { path = "../table" } -tokio = { version = "1", features = ["rt-multi-thread"] } -tracing = { version = "0.1", features = ["log"] } -url = "2.3.1" - -[lints] -workspace = true diff --git a/crates/outbound-mysql/src/lib.rs b/crates/outbound-mysql/src/lib.rs deleted file mode 100644 index 78f369f47..000000000 --- a/crates/outbound-mysql/src/lib.rs +++ /dev/null @@ -1,442 +0,0 @@ -use anyhow::{Context, Result}; -use mysql_async::{consts::ColumnType, from_value_opt, prelude::*, Opts, OptsBuilder, SslOpts}; -use spin_app::DynamicHostComponent; -use spin_core::wasmtime::component::Resource; -use spin_core::{async_trait, HostComponent}; -use spin_world::v1::mysql as v1; -use spin_world::v2::mysql::{self as v2, Connection}; -use spin_world::v2::rdbms_types as v2_types; -use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue}; -use std::sync::Arc; -use tracing::{instrument, Level}; -use url::Url; - -/// A simple implementation to support outbound mysql connection -pub struct OutboundMysqlComponent { - pub resolver: spin_expressions::SharedPreparedResolver, -} - -#[derive(Default)] -pub struct OutboundMysql { - allowed_hosts: spin_outbound_networking::AllowedHostsConfig, - pub connections: table::Table, -} - -impl OutboundMysql { - async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { - self.connections - .push( - build_conn(address) - .await - .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, - ) - .map_err(|_| v2::Error::ConnectionFailed("too many connections".into())) - .map(Resource::new_own) - } - - async fn get_conn( - &mut self, - connection: Resource, - ) -> Result<&mut mysql_async::Conn, v2::Error> { - self.connections - .get_mut(connection.rep()) - .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) - } - - fn is_address_allowed(&self, address: &str) -> bool { - spin_outbound_networking::check_url(address, "mysql", &self.allowed_hosts) - } -} - -impl HostComponent for OutboundMysqlComponent { - type Data = OutboundMysql; - - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - v2::add_to_linker(linker, get)?; - v1::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - Default::default() - } -} - -impl DynamicHostComponent for OutboundMysqlComponent { - fn update_data( - &self, - data: &mut Self::Data, - component: &spin_app::AppComponent, - ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(spin_outbound_networking::ALLOWED_HOSTS_KEY)? - .unwrap_or_default(); - data.allowed_hosts = spin_outbound_networking::AllowedHostsConfig::parse( - &hosts, - self.resolver.get().unwrap(), - ) - .context("`allowed_outbound_hosts` contained an invalid url")?; - Ok(()) - } -} - -impl v2::Host for OutboundMysql {} - -#[async_trait] -impl v2::HostConnection for OutboundMysql { - #[instrument(name = "spin_outbound_mysql.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql"))] - async fn open(&mut self, address: String) -> Result, v2::Error> { - if !self.is_address_allowed(&address) { - return Err(v2::Error::ConnectionFailed(format!( - "address {address} is not permitted" - ))); - } - self.open_connection(&address).await - } - - #[instrument(name = "spin_outbound_mysql.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] - async fn execute( - &mut self, - connection: Resource, - statement: String, - params: Vec, - ) -> Result<(), v2::Error> { - let db_params = params.into_iter().map(to_sql_parameter).collect::>(); - let parameters = mysql_async::Params::Positional(db_params); - - self.get_conn(connection) - .await? - .exec_batch(&statement, &[parameters]) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - Ok(()) - } - - #[instrument(name = "spin_outbound_mysql.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "mysql", otel.name = statement))] - async fn query( - &mut self, - connection: Resource, - statement: String, - params: Vec, - ) -> Result { - let db_params = params.into_iter().map(to_sql_parameter).collect::>(); - let parameters = mysql_async::Params::Positional(db_params); - - let mut query_result = self - .get_conn(connection) - .await? - .exec_iter(&statement, parameters) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - // We have to get these before collect() destroys them - let columns = convert_columns(query_result.columns()); - - match query_result.collect::().await { - Err(e) => Err(v2::Error::Other(e.to_string())), - Ok(result_set) => { - let rows = result_set - .into_iter() - .map(|row| convert_row(row, &columns)) - .collect::, _>>()?; - - Ok(v2_types::RowSet { columns, rows }) - } - } - } - - fn drop(&mut self, connection: Resource) -> Result<()> { - self.connections.remove(connection.rep()); - Ok(()) - } -} - -impl v2_types::Host for OutboundMysql { - fn convert_error(&mut self, error: v2::Error) -> Result { - Ok(error) - } -} - -/// Delegate a function call to the v2::HostConnection implementation -macro_rules! delegate { - ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ - if !$self.is_address_allowed(&$address) { - return Err(v1::MysqlError::ConnectionFailed(format!( - "address {} is not permitted", $address - ))); - } - let connection = match $self.open_connection(&$address).await { - Ok(c) => c, - Err(e) => return Err(e.into()), - }; - ::$name($self, connection, $($arg),*) - .await - .map_err(Into::into) - }}; -} - -#[async_trait] -impl v1::Host for OutboundMysql { - async fn execute( - &mut self, - address: String, - statement: String, - params: Vec, - ) -> Result<(), v1::MysqlError> { - delegate!(self.execute( - address, - statement, - params.into_iter().map(Into::into).collect() - )) - } - - async fn query( - &mut self, - address: String, - statement: String, - params: Vec, - ) -> Result { - delegate!(self.query( - address, - statement, - params.into_iter().map(Into::into).collect() - )) - .map(Into::into) - } - - fn convert_mysql_error(&mut self, error: v1::MysqlError) -> Result { - Ok(error) - } -} - -fn to_sql_parameter(value: ParameterValue) -> mysql_async::Value { - match value { - ParameterValue::Boolean(v) => mysql_async::Value::from(v), - ParameterValue::Int32(v) => mysql_async::Value::from(v), - ParameterValue::Int64(v) => mysql_async::Value::from(v), - ParameterValue::Int8(v) => mysql_async::Value::from(v), - ParameterValue::Int16(v) => mysql_async::Value::from(v), - ParameterValue::Floating32(v) => mysql_async::Value::from(v), - ParameterValue::Floating64(v) => mysql_async::Value::from(v), - ParameterValue::Uint8(v) => mysql_async::Value::from(v), - ParameterValue::Uint16(v) => mysql_async::Value::from(v), - ParameterValue::Uint32(v) => mysql_async::Value::from(v), - ParameterValue::Uint64(v) => mysql_async::Value::from(v), - ParameterValue::Str(v) => mysql_async::Value::from(v), - ParameterValue::Binary(v) => mysql_async::Value::from(v), - ParameterValue::DbNull => mysql_async::Value::NULL, - } -} - -fn convert_columns(columns: Option>) -> Vec { - match columns { - Some(columns) => columns.iter().map(convert_column).collect(), - None => vec![], - } -} - -fn convert_column(column: &mysql_async::Column) -> Column { - let name = column.name_str().into_owned(); - let data_type = convert_data_type(column); - - Column { name, data_type } -} - -fn convert_data_type(column: &mysql_async::Column) -> DbDataType { - let column_type = column.column_type(); - - if column_type.is_numeric_type() { - convert_numeric_type(column) - } else if column_type.is_character_type() { - convert_character_type(column) - } else { - DbDataType::Other - } -} - -fn convert_character_type(column: &mysql_async::Column) -> DbDataType { - match (column.column_type(), is_binary(column)) { - (ColumnType::MYSQL_TYPE_BLOB, false) => DbDataType::Str, // TEXT type - (ColumnType::MYSQL_TYPE_BLOB, _) => DbDataType::Binary, - (ColumnType::MYSQL_TYPE_LONG_BLOB, _) => DbDataType::Binary, - (ColumnType::MYSQL_TYPE_MEDIUM_BLOB, _) => DbDataType::Binary, - (ColumnType::MYSQL_TYPE_STRING, true) => DbDataType::Binary, // BINARY type - (ColumnType::MYSQL_TYPE_STRING, _) => DbDataType::Str, - (ColumnType::MYSQL_TYPE_VAR_STRING, true) => DbDataType::Binary, // VARBINARY type - (ColumnType::MYSQL_TYPE_VAR_STRING, _) => DbDataType::Str, - (_, _) => DbDataType::Other, - } -} - -fn convert_numeric_type(column: &mysql_async::Column) -> DbDataType { - match (column.column_type(), is_signed(column)) { - (ColumnType::MYSQL_TYPE_DOUBLE, _) => DbDataType::Floating64, - (ColumnType::MYSQL_TYPE_FLOAT, _) => DbDataType::Floating32, - (ColumnType::MYSQL_TYPE_INT24, true) => DbDataType::Int32, - (ColumnType::MYSQL_TYPE_INT24, false) => DbDataType::Uint32, - (ColumnType::MYSQL_TYPE_LONG, true) => DbDataType::Int32, - (ColumnType::MYSQL_TYPE_LONG, false) => DbDataType::Uint32, - (ColumnType::MYSQL_TYPE_LONGLONG, true) => DbDataType::Int64, - (ColumnType::MYSQL_TYPE_LONGLONG, false) => DbDataType::Uint64, - (ColumnType::MYSQL_TYPE_SHORT, true) => DbDataType::Int16, - (ColumnType::MYSQL_TYPE_SHORT, false) => DbDataType::Uint16, - (ColumnType::MYSQL_TYPE_TINY, true) => DbDataType::Int8, - (ColumnType::MYSQL_TYPE_TINY, false) => DbDataType::Uint8, - (_, _) => DbDataType::Other, - } -} - -fn is_signed(column: &mysql_async::Column) -> bool { - !column - .flags() - .contains(mysql_async::consts::ColumnFlags::UNSIGNED_FLAG) -} - -fn is_binary(column: &mysql_async::Column) -> bool { - column - .flags() - .contains(mysql_async::consts::ColumnFlags::BINARY_FLAG) -} - -fn convert_row(mut row: mysql_async::Row, columns: &[Column]) -> Result, v2::Error> { - let mut result = Vec::with_capacity(row.len()); - for index in 0..row.len() { - result.push(convert_entry(&mut row, index, columns)?); - } - Ok(result) -} - -fn convert_entry( - row: &mut mysql_async::Row, - index: usize, - columns: &[Column], -) -> Result { - match (row.take(index), columns.get(index)) { - (None, _) => Ok(DbValue::DbNull), // TODO: is this right or is this an "index out of range" thing - (_, None) => Err(v2::Error::Other(format!( - "Can't get column at index {}", - index - ))), - (Some(mysql_async::Value::NULL), _) => Ok(DbValue::DbNull), - (Some(value), Some(column)) => convert_value(value, column), - } -} - -fn convert_value(value: mysql_async::Value, column: &Column) -> Result { - match column.data_type { - DbDataType::Binary => convert_value_to::>(value).map(DbValue::Binary), - DbDataType::Boolean => convert_value_to::(value).map(DbValue::Boolean), - DbDataType::Floating32 => convert_value_to::(value).map(DbValue::Floating32), - DbDataType::Floating64 => convert_value_to::(value).map(DbValue::Floating64), - DbDataType::Int8 => convert_value_to::(value).map(DbValue::Int8), - DbDataType::Int16 => convert_value_to::(value).map(DbValue::Int16), - DbDataType::Int32 => convert_value_to::(value).map(DbValue::Int32), - DbDataType::Int64 => convert_value_to::(value).map(DbValue::Int64), - DbDataType::Str => convert_value_to::(value).map(DbValue::Str), - DbDataType::Uint8 => convert_value_to::(value).map(DbValue::Uint8), - DbDataType::Uint16 => convert_value_to::(value).map(DbValue::Uint16), - DbDataType::Uint32 => convert_value_to::(value).map(DbValue::Uint32), - DbDataType::Uint64 => convert_value_to::(value).map(DbValue::Uint64), - DbDataType::Other => Err(v2::Error::ValueConversionFailed(format!( - "Cannot convert value {:?} in column {} data type {:?}", - value, column.name, column.data_type - ))), - } -} - -async fn build_conn(address: &str) -> Result { - tracing::debug!("Build new connection: {}", address); - - let opts = build_opts(address)?; - - let connection_pool = mysql_async::Pool::new(opts); - - connection_pool.get_conn().await -} - -fn is_ssl_param(s: &str) -> bool { - ["ssl-mode", "sslmode"].contains(&s.to_lowercase().as_str()) -} - -/// The mysql_async crate blows up if you pass it an SSL parameter and doesn't support SSL opts properly. This function -/// is a workaround to manually set SSL opts if the user requests them. -/// -/// We only support ssl-mode in the query as per -/// https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-connp-props-security.html#cj-conn-prop_sslMode. -/// -/// An issue has been filed in the upstream repository https://github.com/blackbeam/mysql_async/issues/225. -fn build_opts(address: &str) -> Result { - let url = Url::parse(address)?; - - let use_ssl = url - .query_pairs() - .any(|(k, v)| is_ssl_param(&k) && v.to_lowercase() != "disabled"); - - let query_without_ssl: Vec<(_, _)> = url - .query_pairs() - .filter(|(k, _v)| !is_ssl_param(k)) - .collect(); - let mut cleaned_url = url.clone(); - cleaned_url.set_query(None); - cleaned_url - .query_pairs_mut() - .extend_pairs(query_without_ssl); - - Ok(OptsBuilder::from_opts(cleaned_url.as_str()) - .ssl_opts(if use_ssl { - Some(SslOpts::default()) - } else { - None - }) - .into()) -} - -fn convert_value_to(value: mysql_async::Value) -> Result { - from_value_opt::(value).map_err(|e| v2::Error::ValueConversionFailed(format!("{}", e))) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_mysql_address_without_ssl_mode() { - assert!(build_opts("mysql://myuser:password@127.0.0.1/db") - .unwrap() - .ssl_opts() - .is_none()) - } - - #[test] - fn test_mysql_address_with_ssl_mode_disabled() { - assert!( - build_opts("mysql://myuser:password@127.0.0.1/db?ssl-mode=DISABLED") - .unwrap() - .ssl_opts() - .is_none() - ) - } - - #[test] - fn test_mysql_address_with_ssl_mode_verify_ca() { - assert!( - build_opts("mysql://myuser:password@127.0.0.1/db?sslMode=VERIFY_CA") - .unwrap() - .ssl_opts() - .is_some() - ) - } - - #[test] - fn test_mysql_address_with_more_to_query() { - let address = "mysql://myuser:password@127.0.0.1/db?SsLmOdE=VERIFY_CA&pool_max=10"; - assert!(build_opts(address).unwrap().ssl_opts().is_some()); - assert_eq!( - build_opts(address).unwrap().pool_opts().constraints().max(), - 10 - ) - } -} diff --git a/crates/outbound-pg/Cargo.toml b/crates/outbound-pg/Cargo.toml deleted file mode 100644 index d12580978..000000000 --- a/crates/outbound-pg/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "outbound-pg" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[lib] -doctest = false - -[dependencies] -anyhow = "1.0" -native-tls = "0.2.11" -postgres-native-tls = "0.5.0" -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-expressions = { path = "../expressions" } -spin-outbound-networking = { path = "../outbound-networking" } -spin-world = { path = "../world" } -table = { path = "../table" } -tokio = { version = "1", features = ["rt-multi-thread"] } -tokio-postgres = { version = "0.7.7" } -tracing = { workspace = true } - -[lints] -workspace = true diff --git a/crates/outbound-pg/src/lib.rs b/crates/outbound-pg/src/lib.rs deleted file mode 100644 index 23f01c191..000000000 --- a/crates/outbound-pg/src/lib.rs +++ /dev/null @@ -1,461 +0,0 @@ -use anyhow::{anyhow, Context, Result}; -use native_tls::TlsConnector; -use postgres_native_tls::MakeTlsConnector; -use spin_app::DynamicHostComponent; -use spin_core::{async_trait, wasmtime::component::Resource, HostComponent}; -use spin_world::v1::postgres as v1; -use spin_world::v1::rdbms_types as v1_types; -use spin_world::v2::postgres::{self as v2, Connection}; -use spin_world::v2::rdbms_types; -use spin_world::v2::rdbms_types::{Column, DbDataType, DbValue, ParameterValue, RowSet}; -use tokio_postgres::{ - config::SslMode, - types::{ToSql, Type}, - Client, NoTls, Row, Socket, -}; -use tracing::instrument; -use tracing::Level; - -pub struct OutboundPgComponent { - pub resolver: spin_expressions::SharedPreparedResolver, -} - -/// A simple implementation to support outbound pg connection -#[derive(Default)] -pub struct OutboundPg { - allowed_hosts: spin_outbound_networking::AllowedHostsConfig, - pub connections: table::Table, -} - -impl OutboundPg { - async fn open_connection(&mut self, address: &str) -> Result, v2::Error> { - self.connections - .push( - build_client(address) - .await - .map_err(|e| v2::Error::ConnectionFailed(format!("{e:?}")))?, - ) - .map_err(|_| v2::Error::ConnectionFailed("too many connections".into())) - .map(Resource::new_own) - } - - async fn get_client(&mut self, connection: Resource) -> Result<&Client, v2::Error> { - self.connections - .get(connection.rep()) - .ok_or_else(|| v2::Error::ConnectionFailed("no connection found".into())) - } - - fn is_address_allowed(&self, address: &str) -> bool { - let Ok(config) = address.parse::() else { - return false; - }; - for (i, host) in config.get_hosts().iter().enumerate() { - match host { - tokio_postgres::config::Host::Tcp(address) => { - let ports = config.get_ports(); - // The port we use is either: - // * The port at the same index as the host - // * The first port if there is only one port - let port = - ports - .get(i) - .or_else(|| if ports.len() == 1 { ports.get(1) } else { None }); - let port_str = port.map(|p| format!(":{}", p)).unwrap_or_default(); - let url = format!("{address}{port_str}"); - if !spin_outbound_networking::check_url(&url, "postgres", &self.allowed_hosts) { - return false; - } - } - #[cfg(unix)] - tokio_postgres::config::Host::Unix(_) => return false, - } - } - true - } -} - -impl HostComponent for OutboundPgComponent { - type Data = OutboundPg; - - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - v1::add_to_linker(linker, get)?; - v2::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - Default::default() - } -} - -impl DynamicHostComponent for OutboundPgComponent { - fn update_data( - &self, - data: &mut Self::Data, - component: &spin_app::AppComponent, - ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(spin_outbound_networking::ALLOWED_HOSTS_KEY)? - .unwrap_or_default(); - data.allowed_hosts = spin_outbound_networking::AllowedHostsConfig::parse( - &hosts, - self.resolver.get().unwrap(), - ) - .context("`allowed_outbound_hosts` contained an invalid url")?; - Ok(()) - } -} - -#[async_trait] -impl v2::Host for OutboundPg {} - -#[async_trait] -impl v2::HostConnection for OutboundPg { - #[instrument(name = "spin_outbound_pg.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql"))] - async fn open(&mut self, address: String) -> Result, v2::Error> { - if !self.is_address_allowed(&address) { - return Err(v2::Error::ConnectionFailed(format!( - "address {address} is not permitted" - ))); - } - self.open_connection(&address).await - } - - #[instrument(name = "spin_outbound_pg.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql", otel.name = statement))] - async fn execute( - &mut self, - connection: Resource, - statement: String, - params: Vec, - ) -> Result { - let params: Vec<&(dyn ToSql + Sync)> = params - .iter() - .map(to_sql_parameter) - .collect::>>() - .map_err(|e| v2::Error::ValueConversionFailed(format!("{:?}", e)))?; - - let nrow = self - .get_client(connection) - .await? - .execute(&statement, params.as_slice()) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - Ok(nrow) - } - - #[instrument(name = "spin_outbound_pg.query", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "postgresql", otel.name = statement))] - async fn query( - &mut self, - connection: Resource, - statement: String, - params: Vec, - ) -> Result { - let params: Vec<&(dyn ToSql + Sync)> = params - .iter() - .map(to_sql_parameter) - .collect::>>() - .map_err(|e| v2::Error::BadParameter(format!("{:?}", e)))?; - - let results = self - .get_client(connection) - .await? - .query(&statement, params.as_slice()) - .await - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - if results.is_empty() { - return Ok(RowSet { - columns: vec![], - rows: vec![], - }); - } - - let columns = infer_columns(&results[0]); - let rows = results - .iter() - .map(convert_row) - .collect::, _>>() - .map_err(|e| v2::Error::QueryFailed(format!("{:?}", e)))?; - - Ok(RowSet { columns, rows }) - } - - fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { - self.connections.remove(connection.rep()); - Ok(()) - } -} - -impl rdbms_types::Host for OutboundPg { - fn convert_error(&mut self, error: v2::Error) -> Result { - Ok(error) - } -} - -fn to_sql_parameter(value: &ParameterValue) -> anyhow::Result<&(dyn ToSql + Sync)> { - match value { - ParameterValue::Boolean(v) => Ok(v), - ParameterValue::Int32(v) => Ok(v), - ParameterValue::Int64(v) => Ok(v), - ParameterValue::Int8(v) => Ok(v), - ParameterValue::Int16(v) => Ok(v), - ParameterValue::Floating32(v) => Ok(v), - ParameterValue::Floating64(v) => Ok(v), - ParameterValue::Uint8(_) - | ParameterValue::Uint16(_) - | ParameterValue::Uint32(_) - | ParameterValue::Uint64(_) => Err(anyhow!("Postgres does not support unsigned integers")), - ParameterValue::Str(v) => Ok(v), - ParameterValue::Binary(v) => Ok(v), - ParameterValue::DbNull => Ok(&PgNull), - } -} - -fn infer_columns(row: &Row) -> Vec { - let mut result = Vec::with_capacity(row.len()); - for index in 0..row.len() { - result.push(infer_column(row, index)); - } - result -} - -fn infer_column(row: &Row, index: usize) -> Column { - let column = &row.columns()[index]; - let name = column.name().to_owned(); - let data_type = convert_data_type(column.type_()); - Column { name, data_type } -} - -fn convert_data_type(pg_type: &Type) -> DbDataType { - match *pg_type { - Type::BOOL => DbDataType::Boolean, - Type::BYTEA => DbDataType::Binary, - Type::FLOAT4 => DbDataType::Floating32, - Type::FLOAT8 => DbDataType::Floating64, - Type::INT2 => DbDataType::Int16, - Type::INT4 => DbDataType::Int32, - Type::INT8 => DbDataType::Int64, - Type::TEXT | Type::VARCHAR | Type::BPCHAR => DbDataType::Str, - _ => { - tracing::debug!("Couldn't convert Postgres type {} to WIT", pg_type.name(),); - DbDataType::Other - } - } -} - -fn convert_row(row: &Row) -> Result, tokio_postgres::Error> { - let mut result = Vec::with_capacity(row.len()); - for index in 0..row.len() { - result.push(convert_entry(row, index)?); - } - Ok(result) -} - -fn convert_entry(row: &Row, index: usize) -> Result { - let column = &row.columns()[index]; - let value = match column.type_() { - &Type::BOOL => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Boolean(v), - None => DbValue::DbNull, - } - } - &Type::BYTEA => { - let value: Option> = row.try_get(index)?; - match value { - Some(v) => DbValue::Binary(v), - None => DbValue::DbNull, - } - } - &Type::FLOAT4 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Floating32(v), - None => DbValue::DbNull, - } - } - &Type::FLOAT8 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Floating64(v), - None => DbValue::DbNull, - } - } - &Type::INT2 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Int16(v), - None => DbValue::DbNull, - } - } - &Type::INT4 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Int32(v), - None => DbValue::DbNull, - } - } - &Type::INT8 => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Int64(v), - None => DbValue::DbNull, - } - } - &Type::TEXT | &Type::VARCHAR | &Type::BPCHAR => { - let value: Option = row.try_get(index)?; - match value { - Some(v) => DbValue::Str(v), - None => DbValue::DbNull, - } - } - t => { - tracing::debug!( - "Couldn't convert Postgres type {} in column {}", - t.name(), - column.name() - ); - DbValue::Unsupported - } - }; - Ok(value) -} - -async fn build_client(address: &str) -> anyhow::Result { - let config = address.parse::()?; - - tracing::debug!("Build new connection: {}", address); - - if config.get_ssl_mode() == SslMode::Disable { - connect(config).await - } else { - connect_tls(config).await - } -} - -async fn connect(config: tokio_postgres::Config) -> anyhow::Result { - let (client, connection) = config.connect(NoTls).await?; - - spawn(connection); - - Ok(client) -} - -async fn connect_tls(config: tokio_postgres::Config) -> anyhow::Result { - let builder = TlsConnector::builder(); - let connector = MakeTlsConnector::new(builder.build()?); - let (client, connection) = config.connect(connector).await?; - - spawn(connection); - - Ok(client) -} - -fn spawn(connection: tokio_postgres::Connection) -where - T: tokio_postgres::tls::TlsStream + std::marker::Unpin + std::marker::Send + 'static, -{ - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::error!("Postgres connection error: {}", e); - } - }); -} - -/// Although the Postgres crate converts Rust Option::None to Postgres NULL, -/// it enforces the type of the Option as it does so. (For example, trying to -/// pass an Option::::None to a VARCHAR column fails conversion.) As we -/// do not know expected column types, we instead use a "neutral" custom type -/// which allows conversion to any type but always tells the Postgres crate to -/// treat it as a SQL NULL. -struct PgNull; - -impl ToSql for PgNull { - fn to_sql( - &self, - _ty: &Type, - _out: &mut tokio_postgres::types::private::BytesMut, - ) -> Result> - where - Self: Sized, - { - Ok(tokio_postgres::types::IsNull::Yes) - } - - fn accepts(_ty: &Type) -> bool - where - Self: Sized, - { - true - } - - fn to_sql_checked( - &self, - _ty: &Type, - _out: &mut tokio_postgres::types::private::BytesMut, - ) -> Result> { - Ok(tokio_postgres::types::IsNull::Yes) - } -} - -impl std::fmt::Debug for PgNull { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("NULL").finish() - } -} - -/// Delegate a function call to the v2::HostConnection implementation -macro_rules! delegate { - ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ - if !$self.is_address_allowed(&$address) { - return Err(v1::PgError::ConnectionFailed(format!( - "address {} is not permitted", $address - ))); - } - let connection = match $self.open_connection(&$address).await { - Ok(c) => c, - Err(e) => return Err(e.into()), - }; - ::$name($self, connection, $($arg),*) - .await - .map_err(|e| e.into()) - }}; -} - -#[async_trait] -impl v1::Host for OutboundPg { - async fn execute( - &mut self, - address: String, - statement: String, - params: Vec, - ) -> Result { - delegate!(self.execute( - address, - statement, - params.into_iter().map(Into::into).collect() - )) - } - - async fn query( - &mut self, - address: String, - statement: String, - params: Vec, - ) -> Result { - delegate!(self.query( - address, - statement, - params.into_iter().map(Into::into).collect() - )) - .map(Into::into) - } - - fn convert_pg_error(&mut self, error: v1::PgError) -> Result { - Ok(error) - } -} diff --git a/crates/outbound-redis/Cargo.toml b/crates/outbound-redis/Cargo.toml deleted file mode 100644 index 83080c607..000000000 --- a/crates/outbound-redis/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "outbound-redis" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[lib] -doctest = false - -[dependencies] -anyhow = "1.0" -redis = { version = "0.21", features = ["tokio-comp", "tokio-native-tls-comp"] } -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-expressions = { path = "../expressions" } -spin-world = { path = "../world" } -spin-outbound-networking = { path = "../outbound-networking" } -table = { path = "../table" } -tokio = { version = "1", features = ["sync"] } -tracing = { workspace = true } - -[lints] -workspace = true diff --git a/crates/outbound-redis/src/host_component.rs b/crates/outbound-redis/src/host_component.rs deleted file mode 100644 index 464e1712d..000000000 --- a/crates/outbound-redis/src/host_component.rs +++ /dev/null @@ -1,42 +0,0 @@ -use anyhow::Context; -use spin_app::DynamicHostComponent; -use spin_core::HostComponent; - -use crate::OutboundRedis; - -pub struct OutboundRedisComponent { - pub resolver: spin_expressions::SharedPreparedResolver, -} - -impl HostComponent for OutboundRedisComponent { - type Data = OutboundRedis; - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - spin_world::v1::redis::add_to_linker(linker, get)?; - spin_world::v2::redis::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - Default::default() - } -} - -impl DynamicHostComponent for OutboundRedisComponent { - fn update_data( - &self, - data: &mut Self::Data, - component: &spin_app::AppComponent, - ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(spin_outbound_networking::ALLOWED_HOSTS_KEY)? - .unwrap_or_default(); - data.allowed_hosts = spin_outbound_networking::AllowedHostsConfig::parse( - &hosts, - self.resolver.get().unwrap(), - ) - .context("`allowed_outbound_hosts` contained an invalid url")?; - Ok(()) - } -} diff --git a/crates/outbound-redis/src/lib.rs b/crates/outbound-redis/src/lib.rs deleted file mode 100644 index e895efd62..000000000 --- a/crates/outbound-redis/src/lib.rs +++ /dev/null @@ -1,317 +0,0 @@ -mod host_component; - -use anyhow::Result; -use redis::{aio::Connection, AsyncCommands, FromRedisValue, Value}; -use spin_core::{async_trait, wasmtime::component::Resource}; -use spin_world::v1::{redis as v1, redis_types}; -use spin_world::v2::redis::{ - self as v2, Connection as RedisConnection, Error, RedisParameter, RedisResult, -}; - -pub use host_component::OutboundRedisComponent; -use tracing::{instrument, Level}; - -struct RedisResults(Vec); - -impl FromRedisValue for RedisResults { - fn from_redis_value(value: &Value) -> redis::RedisResult { - fn append(values: &mut Vec, value: &Value) { - match value { - Value::Nil | Value::Okay => (), - Value::Int(v) => values.push(RedisResult::Int64(*v)), - Value::Data(bytes) => values.push(RedisResult::Binary(bytes.to_owned())), - Value::Bulk(bulk) => bulk.iter().for_each(|value| append(values, value)), - Value::Status(message) => values.push(RedisResult::Status(message.to_owned())), - } - } - - let mut values = Vec::new(); - append(&mut values, value); - Ok(RedisResults(values)) - } -} - -pub struct OutboundRedis { - allowed_hosts: spin_outbound_networking::AllowedHostsConfig, - connections: table::Table, -} - -impl Default for OutboundRedis { - fn default() -> Self { - Self { - allowed_hosts: Default::default(), - connections: table::Table::new(1024), - } - } -} - -impl OutboundRedis { - fn is_address_allowed(&self, address: &str) -> bool { - spin_outbound_networking::check_url(address, "redis", &self.allowed_hosts) - } - - async fn establish_connection( - &mut self, - address: String, - ) -> Result, Error> { - let conn = redis::Client::open(address.as_str()) - .map_err(|_| Error::InvalidAddress)? - .get_async_connection() - .await - .map_err(other_error)?; - self.connections - .push(conn) - .map(Resource::new_own) - .map_err(|_| Error::TooManyConnections) - } -} - -impl v2::Host for OutboundRedis { - fn convert_error(&mut self, error: Error) -> Result { - Ok(error) - } -} - -#[async_trait] -impl v2::HostConnection for OutboundRedis { - #[instrument(name = "spin_outbound_redis.open_connection", skip(self), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis"))] - async fn open(&mut self, address: String) -> Result, Error> { - if !self.is_address_allowed(&address) { - return Err(Error::InvalidAddress); - } - - self.establish_connection(address).await - } - - #[instrument(name = "spin_outbound_redis.publish", skip(self, connection, payload), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("PUBLISH {}", channel)))] - async fn publish( - &mut self, - connection: Resource, - channel: String, - payload: Vec, - ) -> Result<(), Error> { - let conn = self.get_conn(connection).await.map_err(other_error)?; - conn.publish(&channel, &payload) - .await - .map_err(other_error)?; - Ok(()) - } - - #[instrument(name = "spin_outbound_redis.get", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("GET {}", key)))] - async fn get( - &mut self, - connection: Resource, - key: String, - ) -> Result>, Error> { - let conn = self.get_conn(connection).await.map_err(other_error)?; - let value = conn.get(&key).await.map_err(other_error)?; - Ok(value) - } - - #[instrument(name = "spin_outbound_redis.set", skip(self, connection, value), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SET {}", key)))] - async fn set( - &mut self, - connection: Resource, - key: String, - value: Vec, - ) -> Result<(), Error> { - let conn = self.get_conn(connection).await.map_err(other_error)?; - conn.set(&key, &value).await.map_err(other_error)?; - Ok(()) - } - - #[instrument(name = "spin_outbound_redis.incr", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("INCRBY {} 1", key)))] - async fn incr( - &mut self, - connection: Resource, - key: String, - ) -> Result { - let conn = self.get_conn(connection).await.map_err(other_error)?; - let value = conn.incr(&key, 1).await.map_err(other_error)?; - Ok(value) - } - - #[instrument(name = "spin_outbound_redis.del", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("DEL {}", keys.join(" "))))] - async fn del( - &mut self, - connection: Resource, - keys: Vec, - ) -> Result { - let conn = self.get_conn(connection).await.map_err(other_error)?; - let value = conn.del(&keys).await.map_err(other_error)?; - Ok(value) - } - - #[instrument(name = "spin_outbound_redis.sadd", skip(self, connection, values), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SADD {} {}", key, values.join(" "))))] - async fn sadd( - &mut self, - connection: Resource, - key: String, - values: Vec, - ) -> Result { - let conn = self.get_conn(connection).await.map_err(other_error)?; - let value = conn.sadd(&key, &values).await.map_err(|e| { - if e.kind() == redis::ErrorKind::TypeError { - Error::TypeError - } else { - Error::Other(e.to_string()) - } - })?; - Ok(value) - } - - #[instrument(name = "spin_outbound_redis.smembers", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SMEMBERS {}", key)))] - async fn smembers( - &mut self, - connection: Resource, - key: String, - ) -> Result, Error> { - let conn = self.get_conn(connection).await.map_err(other_error)?; - let value = conn.smembers(&key).await.map_err(other_error)?; - Ok(value) - } - - #[instrument(name = "spin_outbound_redis.srem", skip(self, connection, values), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("SREM {} {}", key, values.join(" "))))] - async fn srem( - &mut self, - connection: Resource, - key: String, - values: Vec, - ) -> Result { - let conn = self.get_conn(connection).await.map_err(other_error)?; - let value = conn.srem(&key, &values).await.map_err(other_error)?; - Ok(value) - } - - #[instrument(name = "spin_outbound_redis.execute", skip(self, connection), err(level = Level::INFO), fields(otel.kind = "client", db.system = "redis", otel.name = format!("{}", command)))] - async fn execute( - &mut self, - connection: Resource, - command: String, - arguments: Vec, - ) -> Result, Error> { - let conn = self.get_conn(connection).await?; - let mut cmd = redis::cmd(&command); - arguments.iter().for_each(|value| match value { - RedisParameter::Int64(v) => { - cmd.arg(v); - } - RedisParameter::Binary(v) => { - cmd.arg(v); - } - }); - - cmd.query_async::<_, RedisResults>(conn) - .await - .map(|values| values.0) - .map_err(other_error) - } - - fn drop(&mut self, connection: Resource) -> anyhow::Result<()> { - self.connections.remove(connection.rep()); - Ok(()) - } -} - -fn other_error(e: impl std::fmt::Display) -> Error { - Error::Other(e.to_string()) -} - -/// Delegate a function call to the v2::HostConnection implementation -macro_rules! delegate { - ($self:ident.$name:ident($address:expr, $($arg:expr),*)) => {{ - if !$self.is_address_allowed(&$address) { - return Err(v1::Error::Error); - } - let connection = match $self.establish_connection($address).await { - Ok(c) => c, - Err(_) => return Err(v1::Error::Error), - }; - ::$name($self, connection, $($arg),*) - .await - .map_err(|_| v1::Error::Error) - }}; -} - -#[async_trait] -impl v1::Host for OutboundRedis { - async fn publish( - &mut self, - address: String, - channel: String, - payload: Vec, - ) -> Result<(), v1::Error> { - delegate!(self.publish(address, channel, payload)) - } - - async fn get(&mut self, address: String, key: String) -> Result, v1::Error> { - delegate!(self.get(address, key)).map(|v| v.unwrap_or_default()) - } - - async fn set(&mut self, address: String, key: String, value: Vec) -> Result<(), v1::Error> { - delegate!(self.set(address, key, value)) - } - - async fn incr(&mut self, address: String, key: String) -> Result { - delegate!(self.incr(address, key)) - } - - async fn del(&mut self, address: String, keys: Vec) -> Result { - delegate!(self.del(address, keys)).map(|v| v as i64) - } - - async fn sadd( - &mut self, - address: String, - key: String, - values: Vec, - ) -> Result { - delegate!(self.sadd(address, key, values)).map(|v| v as i64) - } - - async fn smembers(&mut self, address: String, key: String) -> Result, v1::Error> { - delegate!(self.smembers(address, key)) - } - - async fn srem( - &mut self, - address: String, - key: String, - values: Vec, - ) -> Result { - delegate!(self.srem(address, key, values)).map(|v| v as i64) - } - - async fn execute( - &mut self, - address: String, - command: String, - arguments: Vec, - ) -> Result, v1::Error> { - delegate!(self.execute( - address, - command, - arguments.into_iter().map(Into::into).collect() - )) - .map(|v| v.into_iter().map(Into::into).collect()) - } -} - -impl redis_types::Host for OutboundRedis { - fn convert_error(&mut self, error: redis_types::Error) -> Result { - Ok(error) - } -} - -impl OutboundRedis { - async fn get_conn( - &mut self, - connection: Resource, - ) -> Result<&mut Connection, Error> { - self.connections - .get_mut(connection.rep()) - .ok_or(Error::Other( - "could not find connection for resource".into(), - )) - } -} diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml deleted file mode 100644 index 45204df15..000000000 --- a/crates/testing/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "spin-testing" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[dependencies] -anyhow = "1.0" -http = "1.0.0" -hyper = "1.0.0" -serde = "1.0.188" -serde_json = "1" -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-http = { path = "../http" } -spin-trigger = { path = "../trigger" } -tokio = { version = "1", features = ["macros", "rt"] } -tracing-subscriber = "0.3" -spin-componentize = { workspace = true } diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs deleted file mode 100644 index 48b56bf58..000000000 --- a/crates/testing/src/lib.rs +++ /dev/null @@ -1,244 +0,0 @@ -//! This crates contains common code for use in tests. Many methods will panic -//! in the slightest breeze, so DO NOT USE IN NON-TEST CODE. - -use std::{ - net::SocketAddr, - path::{Path, PathBuf}, - sync::Once, -}; - -use http::Response; -use serde::de::DeserializeOwned; -use serde_json::{json, Value}; -use spin_app::{ - async_trait, - locked::{LockedApp, LockedComponentSource}, - AppComponent, Loader, -}; -use spin_core::{Component, StoreBuilder}; -use spin_http::config::{ - HttpExecutorType, HttpTriggerConfig, HttpTriggerRouteConfig, WagiTriggerConfig, -}; -use spin_trigger::{HostComponentInitData, RuntimeConfig, TriggerExecutor, TriggerExecutorBuilder}; -use tokio::fs; - -pub use tokio; - -// Built by build.rs -const TEST_PROGRAM_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/test-programs"); - -/// Initialize a test writer for `tracing`, making its output compatible with libtest -pub fn init_tracing() { - static ONCE: Once = Once::new(); - ONCE.call_once(|| { - tracing_subscriber::fmt() - // Cranelift is very verbose at INFO, so let's tone that down: - .with_max_level(tracing_subscriber::filter::LevelFilter::WARN) - .with_test_writer() - .init(); - }) -} - -// Convenience wrapper for deserializing from literal JSON -#[macro_export] -macro_rules! from_json { - ($($json:tt)+) => { - serde_json::from_value(serde_json::json!($($json)+)).expect("valid json") - }; -} - -#[derive(Default)] -pub struct HttpTestConfig { - module_path: Option, - http_trigger_config: HttpTriggerConfig, -} - -#[derive(Default)] -pub struct RedisTestConfig { - module_path: Option, - redis_channel: String, -} - -impl HttpTestConfig { - pub fn module_path(&mut self, path: impl Into) -> &mut Self { - init_tracing(); - self.module_path = Some(path.into()); - self - } - - pub fn test_program(&mut self, name: impl AsRef) -> &mut Self { - self.module_path(Path::new(TEST_PROGRAM_PATH).join(name)) - } - - pub fn http_spin_trigger(&mut self, route: impl Into) -> &mut Self { - self.http_trigger_config = HttpTriggerConfig { - component: "test-component".to_string(), - route: route.into(), - executor: None, - }; - self - } - - pub fn http_wagi_trigger( - &mut self, - route: impl Into, - wagi_config: WagiTriggerConfig, - ) -> &mut Self { - self.http_trigger_config = HttpTriggerConfig { - component: "test-component".to_string(), - route: route.into(), - executor: Some(HttpExecutorType::Wagi(wagi_config)), - }; - self - } - - pub fn build_loader(&self) -> impl Loader { - init_tracing(); - TestLoader { - module_path: self.module_path.clone().expect("module path to be set"), - trigger_type: "http".into(), - app_trigger_metadata: json!({"base": "/"}), - trigger_config: serde_json::to_value(&self.http_trigger_config).unwrap(), - } - } - - pub async fn build_trigger(&self) -> Executor - where - Executor::TriggerConfig: DeserializeOwned, - { - TriggerExecutorBuilder::new(self.build_loader()) - .build( - TEST_APP_URI.to_string(), - RuntimeConfig::default(), - HostComponentInitData::default(), - ) - .await - .unwrap() - } -} - -impl RedisTestConfig { - pub fn module_path(&mut self, path: impl Into) -> &mut Self { - init_tracing(); - self.module_path = Some(path.into()); - self - } - - pub fn test_program(&mut self, name: impl AsRef) -> &mut Self { - self.module_path(Path::new(TEST_PROGRAM_PATH).join(name)) - } - - pub fn build_loader(&self) -> impl Loader { - TestLoader { - module_path: self.module_path.clone().expect("module path to be set"), - trigger_type: "redis".into(), - app_trigger_metadata: json!({"address": "test-redis-host"}), - trigger_config: json!({ - "component": "test-component", - "channel": self.redis_channel, - }), - } - } - - pub async fn build_trigger(&mut self, channel: &str) -> Executor - where - Executor::TriggerConfig: DeserializeOwned, - { - self.redis_channel = channel.into(); - - TriggerExecutorBuilder::new(self.build_loader()) - .build( - TEST_APP_URI.to_string(), - RuntimeConfig::default(), - HostComponentInitData::default(), - ) - .await - .unwrap() - } -} - -const TEST_APP_URI: &str = "spin-test:"; - -struct TestLoader { - module_path: PathBuf, - trigger_type: String, - app_trigger_metadata: Value, - trigger_config: Value, -} - -#[async_trait] -impl Loader for TestLoader { - async fn load_app(&self, uri: &str) -> anyhow::Result { - assert_eq!(uri, TEST_APP_URI); - let components = from_json!([{ - "id": "test-component", - "source": { - "content_type": "application/wasm", - "digest": "test-source", - }, - }]); - let triggers = from_json!([ - { - "id": "trigger--test-app", - "trigger_type": self.trigger_type, - "trigger_config": self.trigger_config, - }, - ]); - let mut trigger_meta = self.app_trigger_metadata.clone(); - trigger_meta - .as_object_mut() - .unwrap() - .insert("type".into(), self.trigger_type.clone().into()); - let metadata = from_json!({"name": "test-app", "trigger": trigger_meta}); - let variables = Default::default(); - Ok(LockedApp { - spin_lock_version: Default::default(), - components, - triggers, - metadata, - variables, - must_understand: Default::default(), - host_requirements: Default::default(), - }) - } - - async fn load_component( - &self, - engine: &spin_core::wasmtime::Engine, - source: &LockedComponentSource, - ) -> anyhow::Result { - assert_eq!(source.content.digest.as_deref(), Some("test-source")); - Component::new( - engine, - spin_componentize::componentize_if_necessary(&fs::read(&self.module_path).await?)?, - ) - } - - async fn load_module( - &self, - engine: &spin_core::wasmtime::Engine, - source: &LockedComponentSource, - ) -> anyhow::Result { - assert_eq!(source.content.digest.as_deref(), Some("test-source")); - spin_core::Module::from_file(engine, &self.module_path) - } - - async fn mount_files( - &self, - _store_builder: &mut StoreBuilder, - component: &AppComponent, - ) -> anyhow::Result<()> { - assert_eq!(component.files().len(), 0, "files testing not implemented"); - Ok(()) - } -} - -pub fn test_socket_addr() -> SocketAddr { - "127.0.0.1:55555".parse().unwrap() -} - -pub fn assert_http_response_success(resp: &Response) { - if !resp.status().is_success() { - panic!("non-success response {}: {:?}", resp.status(), resp.body()); - } -} diff --git a/crates/trigger-http/Cargo.toml b/crates/trigger-http/Cargo.toml deleted file mode 100644 index 79ce6c128..000000000 --- a/crates/trigger-http/Cargo.toml +++ /dev/null @@ -1,62 +0,0 @@ -[package] -name = "spin-trigger-http" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[lib] -doctest = false - -[dependencies] -anyhow = "1.0" -async-trait = "0.1" -clap = "3" -futures = "0.3" -futures-util = "0.3.8" -http = "1.0.0" -hyper = { workspace = true } -hyper-util = { version = "0.1.2", features = ["tokio"] } -http-body-util = { workspace = true } -indexmap = "1" -outbound-http = { path = "../outbound-http" } -percent-encoding = "2" -rustls = { version = "0.22.4" } -rustls-pemfile = "2.1.2" -rustls-pki-types = "1.7" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1" -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-http = { path = "../http" } -spin-outbound-networking = { path = "../outbound-networking" } -spin-telemetry = { path = "../telemetry" } -spin-trigger = { path = "../trigger" } -spin-world = { path = "../world" } -terminal = { path = "../terminal" } -tls-listener = { version = "0.10.0", features = ["rustls"] } -tokio = { version = "1.23", features = ["full"] } -tokio-rustls = { version = "0.25.0" } -url = "2.4.1" -tracing = { workspace = true } -wasmtime = { workspace = true } -wasmtime-wasi = { workspace = true } -wasmtime-wasi-http = { workspace = true } -wasi-common-preview1 = { workspace = true } -webpki-roots = { version = "0.26.0" } - -[dev-dependencies] -criterion = { version = "0.3.5", features = ["async_tokio"] } -num_cpus = "1" -spin-testing = { path = "../testing" } - -[[bench]] -name = "baseline" -harness = false - -[features] -llm = ["spin-trigger/llm"] -llm-metal = ["llm", "spin-trigger/llm-metal"] -llm-cublas = ["llm", "spin-trigger/llm-cublas"] - -[lints] -workspace = true diff --git a/crates/trigger-http/benches/baseline.rs b/crates/trigger-http/benches/baseline.rs deleted file mode 100644 index 2eac1aa6b..000000000 --- a/crates/trigger-http/benches/baseline.rs +++ /dev/null @@ -1,126 +0,0 @@ -use std::future::Future; -use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; -use std::sync::Arc; - -use criterion::{criterion_group, criterion_main, Criterion}; - -use http::uri::Scheme; -use http::Request; -use spin_testing::{assert_http_response_success, HttpTestConfig}; -use spin_trigger_http::HttpTrigger; -use tokio::runtime::Runtime; - -criterion_main!(benches); -criterion_group!( - benches, - bench_startup, - bench_spin_concurrency_minimal, - bench_wagi_concurrency_minimal, -); - -async fn spin_trigger() -> Arc { - Arc::new( - HttpTestConfig::default() - .test_program("spin-http-benchmark.wasm") - .http_spin_trigger("/") - .build_trigger() - .await, - ) -} - -async fn wagi_trigger() -> Arc { - Arc::new( - HttpTestConfig::default() - .test_program("wagi-benchmark.wasm") - .http_wagi_trigger("/", Default::default()) - .build_trigger() - .await, - ) -} - -// Benchmark time to start and process one request -fn bench_startup(c: &mut Criterion) { - let async_runtime = Runtime::new().unwrap(); - - let mut group = c.benchmark_group("startup"); - group.bench_function("spin-executor", |b| { - b.to_async(&async_runtime).iter(|| async { - let trigger = spin_trigger().await; - run(&trigger, "/").await; - }); - }); - group.bench_function("spin-wagi-executor", |b| { - b.to_async(&async_runtime).iter(|| async { - let trigger = wagi_trigger().await; - run(&trigger, "/").await; - }); - }); -} - -fn bench_spin_concurrency_minimal(c: &mut Criterion) { - bench_concurrency_minimal(c, "spin-executor", spin_trigger); -} -fn bench_wagi_concurrency_minimal(c: &mut Criterion) { - bench_concurrency_minimal(c, "spin-wagi-executor", wagi_trigger); -} - -fn bench_concurrency_minimal>>( - c: &mut Criterion, - name: &str, - mk: fn() -> F, -) { - let async_runtime = Runtime::new().unwrap(); - let trigger = async_runtime.block_on(mk()); - - for task in ["/?sleep=1", "/?noop", "/?cpu=1"] { - let mut group = c.benchmark_group(format!("{name}{task}")); - for concurrency in concurrency_steps() { - group.bench_function(format!("concurrency-{}", concurrency), |b| { - let done = Arc::new(AtomicBool::new(false)); - let background = (0..concurrency - 1) - .map(|_| { - let trigger = trigger.clone(); - let done = done.clone(); - async_runtime.spawn(async move { - while !done.load(Relaxed) { - run(&trigger, task).await; - } - }) - }) - .collect::>(); - b.to_async(&async_runtime).iter(|| run(&trigger, task)); - done.store(true, Relaxed); - for task in background { - async_runtime.block_on(task).unwrap(); - } - }); - } - } -} - -// Helpers - -fn concurrency_steps() -> [u32; 3] { - let cpus = num_cpus::get() as u32; - if cpus > 1 { - [1, cpus, cpus * 4] - } else { - [1, 2, 4] - } -} - -async fn run(trigger: &HttpTrigger, path: &str) { - let req = Request::get(path.to_string()) - .body(Default::default()) - .unwrap(); - let resp = trigger - .handle( - req, - Scheme::HTTP, - "127.0.0.1:3000".parse().unwrap(), - "127.0.0.1:55555".parse().unwrap(), - ) - .await - .unwrap(); - assert_http_response_success(&resp); -} diff --git a/crates/trigger-http/benches/readme.md b/crates/trigger-http/benches/readme.md deleted file mode 100644 index 18c9ef229..000000000 --- a/crates/trigger-http/benches/readme.md +++ /dev/null @@ -1,8 +0,0 @@ -These benchmarks use [criterion.rs](https://github.com/bheisler/criterion.rs); the recommended way to run them is with the [cargo-criterion](https://github.com/bheisler/cargo-criterion) tool: - -```sh -$ cargo install cargo-criterion -$ cargo criterion --workspace -``` - -HTML reports will be written to `target/criterion/reports` \ No newline at end of file diff --git a/crates/trigger-http/benches/spin-http-benchmark/.cargo/config.toml b/crates/trigger-http/benches/spin-http-benchmark/.cargo/config.toml deleted file mode 100644 index 6b77899cb..000000000 --- a/crates/trigger-http/benches/spin-http-benchmark/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build] -target = "wasm32-wasi" diff --git a/crates/trigger-http/benches/spin-http-benchmark/Cargo.toml b/crates/trigger-http/benches/spin-http-benchmark/Cargo.toml deleted file mode 100644 index dae808036..000000000 --- a/crates/trigger-http/benches/spin-http-benchmark/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "spin-http-benchmark" -version = "0.2.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] - -[dependencies] -wit-bindgen = "0.13.0" -url = "2.4.1" - -[workspace] diff --git a/crates/trigger-http/benches/spin-http-benchmark/src/lib.rs b/crates/trigger-http/benches/spin-http-benchmark/src/lib.rs deleted file mode 100644 index 52e883529..000000000 --- a/crates/trigger-http/benches/spin-http-benchmark/src/lib.rs +++ /dev/null @@ -1,57 +0,0 @@ -wit_bindgen::generate!({ - world: "http-trigger", - path: "../../../../wit/deps/spin@unversioned", - exports: { - "fermyon:spin/inbound-http": SpinHttp, - } -}); - -use exports::fermyon::spin::inbound_http; - -struct SpinHttp; - -impl inbound_http::Guest for SpinHttp { - fn handle_request(req: inbound_http::Request) -> inbound_http::Response { - let params = req.uri.find('?').map(|i| &req.uri[i + 1..]).unwrap_or(""); - for (key, value) in url::form_urlencoded::parse(params.as_bytes()) { - #[allow(clippy::single_match)] - match &*key { - // sleep= param simulates processing time - "sleep" => { - let ms = value.parse().expect("invalid sleep"); - std::thread::sleep(std::time::Duration::from_millis(ms)); - } - // cpu= param simulates compute time - "cpu" => { - let amt = value.parse().expect("invalid cpu"); - for _ in 0..amt { - do_some_work(); - } - } - _ => (), - } - } - inbound_http::Response { - status: 200, - headers: None, - body: None, - } - } -} - -// According to my computer, which is highly accurate, this is the best way to -// simulate precisely 1.5ms of work. That definitely won't change over time. -fn do_some_work() { - const N: usize = 4096; - const AMT: usize = 5_000; - - let mut a = [0u8; N]; - let mut b = [1u8; N]; - - for _ in 0..AMT { - a.copy_from_slice(&b); - std::hint::black_box(&a); - b.copy_from_slice(&a); - std::hint::black_box(&b); - } -} diff --git a/crates/trigger-http/benches/wagi-benchmark/.cargo/config.toml b/crates/trigger-http/benches/wagi-benchmark/.cargo/config.toml deleted file mode 100644 index 6b77899cb..000000000 --- a/crates/trigger-http/benches/wagi-benchmark/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build] -target = "wasm32-wasi" diff --git a/crates/trigger-http/benches/wagi-benchmark/Cargo.toml b/crates/trigger-http/benches/wagi-benchmark/Cargo.toml deleted file mode 100644 index a2b6085cf..000000000 --- a/crates/trigger-http/benches/wagi-benchmark/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "wagi-benchmark" -version = "0.1.0" -edition = "2021" - -[workspace] \ No newline at end of file diff --git a/crates/trigger-http/benches/wagi-benchmark/src/main.rs b/crates/trigger-http/benches/wagi-benchmark/src/main.rs deleted file mode 100644 index a8e17c8ac..000000000 --- a/crates/trigger-http/benches/wagi-benchmark/src/main.rs +++ /dev/null @@ -1,11 +0,0 @@ -fn main() { - for arg in std::env::args() { - // sleep= param simulates processing time - if let Some(ms_str) = arg.strip_prefix("sleep=") { - let ms = ms_str.parse().expect("invalid sleep"); - std::thread::sleep(std::time::Duration::from_millis(ms)); - } - } - - println!("Content-Type: text/plain\n"); -} diff --git a/crates/trigger-http/src/handler.rs b/crates/trigger-http/src/handler.rs deleted file mode 100644 index 4ff4aa398..000000000 --- a/crates/trigger-http/src/handler.rs +++ /dev/null @@ -1,456 +0,0 @@ -use std::{net::SocketAddr, str, str::FromStr}; - -use crate::{Body, ChainedRequestHandler, HttpExecutor, HttpInstance, HttpTrigger, Store}; -use anyhow::{anyhow, Context, Result}; -use futures::TryFutureExt; -use http::{HeaderName, HeaderValue}; -use http_body_util::BodyExt; -use hyper::{Request, Response}; -use outbound_http::OutboundHttpComponent; -use spin_core::async_trait; -use spin_core::wasi_2023_10_18::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_10_18; -use spin_core::wasi_2023_11_10::exports::wasi::http::incoming_handler::Guest as IncomingHandler2023_11_10; -use spin_core::{Component, Engine, Instance}; -use spin_http::body; -use spin_http::routes::RouteMatch; -use spin_trigger::TriggerAppEngine; -use spin_world::v1::http_types; -use std::sync::Arc; -use tokio::{sync::oneshot, task}; -use tracing::{instrument, Instrument, Level}; -use wasmtime_wasi_http::{proxy::Proxy, WasiHttpView}; - -#[derive(Clone)] -pub struct HttpHandlerExecutor; - -#[async_trait] -impl HttpExecutor for HttpHandlerExecutor { - #[instrument(name = "spin_trigger_http.execute_wasm", skip_all, err(level = Level::INFO), fields(otel.name = format!("execute_wasm_component {}", route_match.component_id())))] - async fn execute( - &self, - engine: Arc>, - base: &str, - route_match: &RouteMatch, - req: Request, - client_addr: SocketAddr, - ) -> Result> { - let component_id = route_match.component_id(); - - tracing::trace!( - "Executing request using the Spin executor for component {}", - component_id - ); - - let (instance, mut store) = engine.prepare_instance(component_id).await?; - let HttpInstance::Component(instance, ty) = instance else { - unreachable!() - }; - - set_http_origin_from_request(&mut store, engine.clone(), self, &req); - - // set the client tls options for the current component_id. - // The OutboundWasiHttpHandler in this file is only used - // when making http-request from a http-trigger component. - // The outbound http requests from other triggers such as Redis - // uses OutboundWasiHttpHandler defined in spin_core crate. - store.as_mut().data_mut().as_mut().client_tls_opts = - engine.get_client_tls_opts(component_id); - - let resp = match ty { - HandlerType::Spin => { - Self::execute_spin(store, instance, base, route_match, req, client_addr) - .await - .map_err(contextualise_err)? - } - _ => { - Self::execute_wasi(store, instance, ty, base, route_match, req, client_addr).await? - } - }; - - tracing::info!( - "Request finished, sending response with status code {}", - resp.status() - ); - Ok(resp) - } -} - -impl HttpHandlerExecutor { - pub async fn execute_spin( - mut store: Store, - instance: Instance, - base: &str, - route_match: &RouteMatch, - req: Request, - client_addr: SocketAddr, - ) -> Result> { - let headers = Self::headers(&req, base, route_match, client_addr)?; - let func = instance - .exports(&mut store) - .instance("fermyon:spin/inbound-http") - // Safe since we have already checked that this instance exists - .expect("no fermyon:spin/inbound-http found") - .typed_func::<(http_types::Request,), (http_types::Response,)>("handle-request")?; - - let (parts, body) = req.into_parts(); - let bytes = body.collect().await?.to_bytes().to_vec(); - - let method = if let Some(method) = Self::method(&parts.method) { - method - } else { - return Ok(Response::builder() - .status(http::StatusCode::METHOD_NOT_ALLOWED) - .body(body::empty())?); - }; - - // Preparing to remove the params field. We are leaving it in place for now - // to avoid breaking the ABI, but no longer pass or accept values in it. - // https://github.com/fermyon/spin/issues/663 - let params = vec![]; - - let uri = match parts.uri.path_and_query() { - Some(u) => u.to_string(), - None => parts.uri.to_string(), - }; - - let req = http_types::Request { - method, - uri, - headers, - params, - body: Some(bytes), - }; - - let (resp,) = func.call_async(&mut store, (req,)).await?; - - if resp.status < 100 || resp.status > 600 { - tracing::error!("malformed HTTP status code"); - return Ok(Response::builder() - .status(http::StatusCode::INTERNAL_SERVER_ERROR) - .body(body::empty())?); - }; - - let mut response = http::Response::builder().status(resp.status); - if let Some(headers) = response.headers_mut() { - Self::append_headers(headers, resp.headers)?; - } - - let body = match resp.body { - Some(b) => body::full(b.into()), - None => body::empty(), - }; - - Ok(response.body(body)?) - } - - fn method(m: &http::Method) -> Option { - Some(match *m { - http::Method::GET => http_types::Method::Get, - http::Method::POST => http_types::Method::Post, - http::Method::PUT => http_types::Method::Put, - http::Method::DELETE => http_types::Method::Delete, - http::Method::PATCH => http_types::Method::Patch, - http::Method::HEAD => http_types::Method::Head, - http::Method::OPTIONS => http_types::Method::Options, - _ => return None, - }) - } - - async fn execute_wasi( - mut store: Store, - instance: Instance, - ty: HandlerType, - base: &str, - route_match: &RouteMatch, - mut req: Request, - client_addr: SocketAddr, - ) -> anyhow::Result> { - let headers = Self::headers(&req, base, route_match, client_addr)?; - req.headers_mut().clear(); - req.headers_mut() - .extend(headers.into_iter().filter_map(|(n, v)| { - let Ok(name) = n.parse::() else { - return None; - }; - let Ok(value) = HeaderValue::from_bytes(v.as_bytes()) else { - return None; - }; - Some((name, value)) - })); - let request = store.as_mut().data_mut().new_incoming_request(req)?; - - let (response_tx, response_rx) = oneshot::channel(); - let response = store - .as_mut() - .data_mut() - .new_response_outparam(response_tx)?; - - enum Handler { - Latest(Proxy), - Handler2023_11_10(IncomingHandler2023_11_10), - Handler2023_10_18(IncomingHandler2023_10_18), - } - - let handler = - { - let mut exports = instance.exports(&mut store); - match ty { - HandlerType::Wasi2023_10_18 => { - let mut instance = exports - .instance(WASI_HTTP_EXPORT_2023_10_18) - .ok_or_else(|| { - anyhow!("export of `{WASI_HTTP_EXPORT_2023_10_18}` not an instance") - })?; - Handler::Handler2023_10_18(IncomingHandler2023_10_18::new(&mut instance)?) - } - HandlerType::Wasi2023_11_10 => { - let mut instance = exports - .instance(WASI_HTTP_EXPORT_2023_11_10) - .ok_or_else(|| { - anyhow!("export of `{WASI_HTTP_EXPORT_2023_11_10}` not an instance") - })?; - Handler::Handler2023_11_10(IncomingHandler2023_11_10::new(&mut instance)?) - } - HandlerType::Wasi0_2 => { - drop(exports); - Handler::Latest(Proxy::new(&mut store, &instance)?) - } - HandlerType::Spin => panic!("should have used execute_spin instead"), - } - }; - - let span = tracing::debug_span!("execute_wasi"); - let handle = task::spawn( - async move { - let result = match handler { - Handler::Latest(proxy) => { - proxy - .wasi_http_incoming_handler() - .call_handle(&mut store, request, response) - .instrument(span) - .await - } - Handler::Handler2023_10_18(proxy) => { - proxy - .call_handle(&mut store, request, response) - .instrument(span) - .await - } - Handler::Handler2023_11_10(proxy) => { - proxy - .call_handle(&mut store, request, response) - .instrument(span) - .await - } - }; - - tracing::trace!( - "wasi-http memory consumed: {}", - store.as_ref().data().memory_consumed() - ); - - result - } - .in_current_span(), - ); - - match response_rx.await { - Ok(response) => { - task::spawn( - async move { - handle - .await - .context("guest invocation panicked")? - .context("guest invocation failed")?; - - Ok(()) - } - .map_err(|e: anyhow::Error| { - tracing::warn!("component error after response: {e:?}"); - }), - ); - - Ok(response.context("guest failed to produce a response")?) - } - - Err(_) => { - handle - .await - .context("guest invocation panicked")? - .context("guest invocation failed")?; - - Err(anyhow!( - "guest failed to produce a response prior to returning" - )) - } - } - } - - fn headers( - req: &Request, - base: &str, - route_match: &RouteMatch, - client_addr: SocketAddr, - ) -> Result> { - let mut res = Vec::new(); - for (name, value) in req - .headers() - .iter() - .map(|(name, value)| (name.to_string(), std::str::from_utf8(value.as_bytes()))) - { - let value = value?.to_string(); - res.push((name, value)); - } - - let default_host = http::HeaderValue::from_str("localhost")?; - let host = std::str::from_utf8( - req.headers() - .get("host") - .unwrap_or(&default_host) - .as_bytes(), - )?; - - // Set the environment information (path info, base path, etc) as headers. - // In the future, we might want to have this information in a context - // object as opposed to headers. - for (keys, val) in - crate::compute_default_headers(req.uri(), base, host, route_match, client_addr)? - { - res.push((Self::prepare_header_key(&keys[0]), val)); - } - - Ok(res) - } - - fn prepare_header_key(key: &str) -> String { - key.replace('_', "-").to_ascii_lowercase() - } - - fn append_headers(res: &mut http::HeaderMap, src: Option>) -> Result<()> { - if let Some(src) = src { - for (k, v) in src.iter() { - res.insert( - http::header::HeaderName::from_str(k)?, - http::header::HeaderValue::from_str(v)?, - ); - } - }; - - Ok(()) - } -} - -/// Whether this handler uses the custom Spin http handler interface for wasi-http -#[derive(Copy, Clone)] -pub enum HandlerType { - Spin, - Wasi0_2, - Wasi2023_11_10, - Wasi2023_10_18, -} - -const WASI_HTTP_EXPORT_2023_10_18: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-10-18"; -const WASI_HTTP_EXPORT_2023_11_10: &str = "wasi:http/incoming-handler@0.2.0-rc-2023-11-10"; -const WASI_HTTP_EXPORT_0_2_0: &str = "wasi:http/incoming-handler@0.2.0"; - -impl HandlerType { - /// Determine the handler type from the exports of a component - pub fn from_component(engine: &Engine, component: &Component) -> Result { - let mut handler_ty = None; - - let mut set = |ty: HandlerType| { - if handler_ty.is_none() { - handler_ty = Some(ty); - Ok(()) - } else { - Err(anyhow!( - "component exports multiple different handlers but \ - it's expected to export only one" - )) - } - }; - let ty = component.component_type(); - for (name, _) in ty.exports(engine.as_ref()) { - match name { - WASI_HTTP_EXPORT_2023_10_18 => set(HandlerType::Wasi2023_10_18)?, - WASI_HTTP_EXPORT_2023_11_10 => set(HandlerType::Wasi2023_11_10)?, - WASI_HTTP_EXPORT_0_2_0 => set(HandlerType::Wasi0_2)?, - "fermyon:spin/inbound-http" => set(HandlerType::Spin)?, - _ => {} - } - } - - handler_ty.ok_or_else(|| { - anyhow!( - "Expected component to either export `{WASI_HTTP_EXPORT_2023_10_18}`, \ - `{WASI_HTTP_EXPORT_2023_11_10}`, `{WASI_HTTP_EXPORT_0_2_0}`, \ - or `fermyon:spin/inbound-http` but it exported none of those" - ) - }) - } -} - -fn set_http_origin_from_request( - store: &mut Store, - engine: Arc>, - handler: &HttpHandlerExecutor, - req: &Request, -) { - if let Some(authority) = req.uri().authority() { - if let Some(scheme) = req.uri().scheme_str() { - let origin = format!("{}://{}", scheme, authority); - if let Some(outbound_http_handle) = engine - .engine - .find_host_component_handle::>() - { - let outbound_http_data = store - .host_components_data() - .get_or_insert(outbound_http_handle); - - outbound_http_data.origin.clone_from(&origin); - store.as_mut().data_mut().as_mut().allowed_hosts = - outbound_http_data.allowed_hosts.clone(); - } - - let chained_request_handler = ChainedRequestHandler { - engine: engine.clone(), - executor: handler.clone(), - }; - store.as_mut().data_mut().as_mut().origin = Some(origin); - store.as_mut().data_mut().as_mut().chained_handler = Some(chained_request_handler); - } - } -} - -fn contextualise_err(e: anyhow::Error) -> anyhow::Error { - if e.to_string() - .contains("failed to find function export `canonical_abi_free`") - { - e.context( - "component is not compatible with Spin executor - should this use the Wagi executor?", - ) - } else { - e - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_spin_header_keys() { - assert_eq!( - HttpHandlerExecutor::prepare_header_key("SPIN_FULL_URL"), - "spin-full-url".to_string() - ); - assert_eq!( - HttpHandlerExecutor::prepare_header_key("SPIN_PATH_INFO"), - "spin-path-info".to_string() - ); - assert_eq!( - HttpHandlerExecutor::prepare_header_key("SPIN_RAW_COMPONENT_ROUTE"), - "spin-raw-component-route".to_string() - ); - } -} diff --git a/crates/trigger-http/src/instrument.rs b/crates/trigger-http/src/instrument.rs deleted file mode 100644 index 2e74d97aa..000000000 --- a/crates/trigger-http/src/instrument.rs +++ /dev/null @@ -1,93 +0,0 @@ -use anyhow::Result; -use http::Response; -use tracing::Level; -use wasmtime_wasi_http::body::HyperIncomingBody; - -/// Create a span for an HTTP request. -macro_rules! http_span { - ($request:tt, $addr:tt) => { - tracing::info_span!( - "spin_trigger_http.handle_http_request", - "otel.kind" = "server", - "http.request.method" = %$request.method(), - "network.peer.address" = %$addr.ip(), - "network.peer.port" = %$addr.port(), - "network.protocol.name" = "http", - "url.path" = $request.uri().path(), - "url.query" = $request.uri().query().unwrap_or(""), - "url.scheme" = $request.uri().scheme_str().unwrap_or(""), - "client.address" = $request.headers().get("x-forwarded-for").and_then(|val| val.to_str().ok()), - // Recorded later - "error.type" = Empty, - "http.response.status_code" = Empty, - "http.route" = Empty, - "otel.name" = Empty, - ) - }; -} - -pub(crate) use http_span; - -/// Finish setting attributes on the HTTP span. -pub(crate) fn finalize_http_span( - response: Result>, - method: String, -) -> Result> { - let span = tracing::Span::current(); - match response { - Ok(response) => { - let matched_route = response.extensions().get::(); - // Set otel.name and http.route - if let Some(MatchedRoute { route }) = matched_route { - span.record("http.route", route); - span.record("otel.name", format!("{method} {route}")); - } else { - span.record("otel.name", method); - } - - // Set status code - span.record("http.response.status_code", response.status().as_u16()); - - Ok(response) - } - Err(err) => { - instrument_error(&err); - span.record("http.response.status_code", 500); - span.record("otel.name", method); - Err(err) - } - } -} - -/// Marks the current span as errored. -pub(crate) fn instrument_error(err: &anyhow::Error) { - let span = tracing::Span::current(); - tracing::event!(target:module_path!(), Level::INFO, error = %err); - span.record("error.type", format!("{:?}", err)); -} - -/// MatchedRoute is used as a response extension to track the route that was matched for OTel -/// tracing purposes. -#[derive(Clone)] -pub struct MatchedRoute { - pub route: String, -} - -impl MatchedRoute { - pub fn set_response_extension( - resp: &mut Response, - route: impl Into, - ) { - resp.extensions_mut().insert(MatchedRoute { - route: route.into(), - }); - } - - pub fn with_response_extension( - mut resp: Response, - route: impl Into, - ) -> Response { - Self::set_response_extension(&mut resp, route); - resp - } -} diff --git a/crates/trigger-http/src/lib.rs b/crates/trigger-http/src/lib.rs deleted file mode 100644 index c5c5f066c..000000000 --- a/crates/trigger-http/src/lib.rs +++ /dev/null @@ -1,1240 +0,0 @@ -//! Implementation for the Spin HTTP engine. - -mod handler; -mod instrument; -mod tls; -mod wagi; - -use std::{ - collections::HashMap, - error::Error, - io::IsTerminal, - net::{Ipv4Addr, SocketAddr, ToSocketAddrs}, - path::PathBuf, - str::FromStr, - sync::Arc, -}; - -use anyhow::{Context, Result}; -use async_trait::async_trait; -use clap::Args; -use http::{header::HOST, uri::Authority, uri::Scheme, HeaderValue, StatusCode, Uri}; -use http_body_util::BodyExt; -use hyper::{ - body::{Bytes, Incoming}, - server::conn::http1, - service::service_fn, - Request, Response, -}; -use hyper_util::rt::tokio::TokioIo; -use instrument::{finalize_http_span, http_span}; -use spin_app::{AppComponent, APP_DESCRIPTION_KEY}; -use spin_core::{Engine, OutboundWasiHttpHandler}; -use spin_http::{ - app_info::AppInfo, - body, - config::{HttpExecutorType, HttpTriggerConfig}, - routes::{RouteMatch, Router}, -}; -use spin_outbound_networking::{ - is_service_chaining_host, parse_service_chaining_target, AllowedHostsConfig, OutboundUrl, -}; -use spin_trigger::{ParsedClientTlsOpts, TriggerAppEngine, TriggerExecutor, TriggerInstancePre}; -use tokio::{ - io::{AsyncRead, AsyncWrite}, - net::{TcpListener, TcpStream}, - task, - time::timeout, -}; - -use tracing::{field::Empty, log, Instrument}; -use wasmtime_wasi_http::{ - bindings::wasi::http::{types, types::ErrorCode}, - body::{HyperIncomingBody as Body, HyperOutgoingBody}, - types::HostFutureIncomingResponse, - HttpError, HttpResult, -}; - -use crate::{ - handler::{HandlerType, HttpHandlerExecutor}, - instrument::{instrument_error, MatchedRoute}, - wagi::WagiHttpExecutor, -}; - -pub use tls::TlsConfig; - -pub(crate) type RuntimeData = HttpRuntimeData; -pub(crate) type Store = spin_core::Store; - -/// The Spin HTTP trigger. -pub struct HttpTrigger { - engine: Arc>, - router: Router, - // Base path for component routes. - base: String, - // Component ID -> component trigger config - component_trigger_configs: HashMap, -} - -#[derive(Args)] -pub struct CliArgs { - /// IP address and port to listen on - #[clap(long = "listen", env = "SPIN_HTTP_LISTEN_ADDR", default_value = "127.0.0.1:3000", value_parser = parse_listen_addr)] - pub address: SocketAddr, - - /// The path to the certificate to use for https, if this is not set, normal http will be used. The cert should be in PEM format - #[clap(long, env = "SPIN_TLS_CERT", requires = "tls-key")] - pub tls_cert: Option, - - /// The path to the certificate key to use for https, if this is not set, normal http will be used. The key should be in PKCS#8 format - #[clap(long, env = "SPIN_TLS_KEY", requires = "tls-cert")] - pub tls_key: Option, -} - -impl CliArgs { - fn into_tls_config(self) -> Option { - match (self.tls_cert, self.tls_key) { - (Some(cert_path), Some(key_path)) => Some(TlsConfig { - cert_path, - key_path, - }), - (None, None) => None, - _ => unreachable!(), - } - } -} - -pub enum HttpInstancePre { - Component(spin_core::InstancePre, HandlerType), - Module(spin_core::ModuleInstancePre), -} - -pub enum HttpInstance { - Component(spin_core::Instance, HandlerType), - Module(spin_core::ModuleInstance), -} - -#[async_trait] -impl TriggerExecutor for HttpTrigger { - const TRIGGER_TYPE: &'static str = "http"; - type RuntimeData = RuntimeData; - type TriggerConfig = HttpTriggerConfig; - type RunConfig = CliArgs; - type InstancePre = HttpInstancePre; - - async fn new(engine: TriggerAppEngine) -> Result { - let mut base = engine - .trigger_metadata::()? - .unwrap_or_default() - .base; - - if !base.starts_with('/') { - base = format!("/{base}"); - } - - let component_routes = engine - .trigger_configs() - .map(|(_, config)| (config.component.as_str(), &config.route)); - - let (router, duplicate_routes) = Router::build(&base, component_routes)?; - - if !duplicate_routes.is_empty() { - log::error!("The following component routes are duplicates and will never be used:"); - for dup in &duplicate_routes { - log::error!( - " {}: {} (duplicate of {})", - dup.replaced_id, - dup.route(), - dup.effective_id, - ); - } - } - - log::trace!( - "Constructed router for application {}: {:?}", - engine.app_name, - router.routes().collect::>() - ); - - let component_trigger_configs = engine - .trigger_configs() - .map(|(_, config)| (config.component.clone(), config.clone())) - .collect(); - - Ok(Self { - engine: Arc::new(engine), - router, - base, - component_trigger_configs, - }) - } - - async fn run(self, config: Self::RunConfig) -> Result<()> { - let listen_addr = config.address; - let tls = config.into_tls_config(); - - let listener = TcpListener::bind(listen_addr) - .await - .with_context(|| format!("Unable to listen on {}", listen_addr))?; - - let self_ = Arc::new(self); - if let Some(tls) = tls { - self_.serve_tls(listener, listen_addr, tls).await? - } else { - self_.serve(listener, listen_addr).await? - }; - - Ok(()) - } - - fn supported_host_requirements() -> Vec<&'static str> { - vec![spin_app::locked::SERVICE_CHAINING_KEY] - } -} - -#[async_trait] -impl TriggerInstancePre for HttpInstancePre { - type Instance = HttpInstance; - - async fn instantiate_pre( - engine: &Engine, - component: &AppComponent, - config: &HttpTriggerConfig, - ) -> Result { - if let Some(HttpExecutorType::Wagi(_)) = &config.executor { - let module = component.load_module(engine).await?; - Ok(HttpInstancePre::Module( - engine.module_instantiate_pre(&module)?, - )) - } else { - let comp = component.load_component(engine).await?; - let handler_ty = HandlerType::from_component(engine, &comp)?; - Ok(HttpInstancePre::Component( - engine.instantiate_pre(&comp)?, - handler_ty, - )) - } - } - - async fn instantiate(&self, store: &mut Store) -> Result { - match self { - HttpInstancePre::Component(pre, ty) => Ok(HttpInstance::Component( - pre.instantiate_async(store).await?, - *ty, - )), - HttpInstancePre::Module(pre) => { - pre.instantiate_async(store).await.map(HttpInstance::Module) - } - } - } -} - -impl HttpTrigger { - /// Handles incoming requests using an HTTP executor. - pub async fn handle( - &self, - mut req: Request, - scheme: Scheme, - server_addr: SocketAddr, - client_addr: SocketAddr, - ) -> Result> { - set_req_uri(&mut req, scheme, server_addr)?; - strip_forbidden_headers(&mut req); - - spin_telemetry::extract_trace_context(&req); - - log::info!( - "Processing request for application {} on URI {}", - &self.engine.app_name, - req.uri() - ); - - let path = req.uri().path().to_string(); - - // Handle well-known spin paths - if let Some(well_known) = path.strip_prefix(spin_http::WELL_KNOWN_PREFIX) { - return match well_known { - "health" => Ok(MatchedRoute::with_response_extension( - Response::new(body::full(Bytes::from_static(b"OK"))), - path, - )), - "info" => self.app_info(path), - _ => Self::not_found(NotFoundRouteKind::WellKnown), - }; - } - - // Route to app component - match self.router.route(&path) { - Ok(route_match) => { - spin_telemetry::metrics::monotonic_counter!( - spin.request_count = 1, - trigger_type = "http", - app_id = &self.engine.app_name, - component_id = route_match.component_id() - ); - - let component_id = route_match.component_id(); - - let trigger = self.component_trigger_configs.get(component_id).unwrap(); - - let executor = trigger.executor.as_ref().unwrap_or(&HttpExecutorType::Http); - - let res = match executor { - HttpExecutorType::Http => { - HttpHandlerExecutor - .execute( - self.engine.clone(), - &self.base, - &route_match, - req, - client_addr, - ) - .await - } - HttpExecutorType::Wagi(wagi_config) => { - let executor = WagiHttpExecutor { - wagi_config: wagi_config.clone(), - }; - executor - .execute( - self.engine.clone(), - &self.base, - &route_match, - req, - client_addr, - ) - .await - } - }; - match res { - Ok(res) => Ok(MatchedRoute::with_response_extension( - res, - route_match.raw_route(), - )), - Err(e) => { - log::error!("Error processing request: {:?}", e); - instrument_error(&e); - Self::internal_error(None, route_match.raw_route()) - } - } - } - Err(_) => Self::not_found(NotFoundRouteKind::Normal(path.to_string())), - } - } - - /// Returns spin status information. - fn app_info(&self, route: String) -> Result> { - let info = AppInfo::new(self.engine.app()); - let body = serde_json::to_vec_pretty(&info)?; - Ok(MatchedRoute::with_response_extension( - Response::builder() - .header("content-type", "application/json") - .body(body::full(body.into()))?, - route, - )) - } - - /// Creates an HTTP 500 response. - fn internal_error(body: Option<&str>, route: impl Into) -> Result> { - let body = match body { - Some(body) => body::full(Bytes::copy_from_slice(body.as_bytes())), - None => body::empty(), - }; - - Ok(MatchedRoute::with_response_extension( - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(body)?, - route, - )) - } - - /// Creates an HTTP 404 response. - fn not_found(kind: NotFoundRouteKind) -> Result> { - use std::sync::atomic::{AtomicBool, Ordering}; - static SHOWN_GENERIC_404_WARNING: AtomicBool = AtomicBool::new(false); - if let NotFoundRouteKind::Normal(route) = kind { - if !SHOWN_GENERIC_404_WARNING.fetch_or(true, Ordering::Relaxed) - && std::io::stderr().is_terminal() - { - terminal::warn!("Request to {route} matched no pattern, and received a generic 404 response. To serve a more informative 404 page, add a catch-all (/...) route."); - } - } - Ok(Response::builder() - .status(StatusCode::NOT_FOUND) - .body(body::empty())?) - } - - fn serve_connection( - self: Arc, - stream: S, - server_addr: SocketAddr, - client_addr: SocketAddr, - ) { - task::spawn(async move { - if let Err(e) = http1::Builder::new() - .keep_alive(true) - .serve_connection( - TokioIo::new(stream), - service_fn(move |request| { - self.clone() - .instrumented_service_fn(server_addr, client_addr, request) - }), - ) - .await - { - log::warn!("{e:?}"); - } - }); - } - - async fn instrumented_service_fn( - self: Arc, - server_addr: SocketAddr, - client_addr: SocketAddr, - request: Request, - ) -> Result> { - let span = http_span!(request, client_addr); - let method = request.method().to_string(); - async { - let result = self - .handle( - request.map(|body: Incoming| { - body.map_err(wasmtime_wasi_http::hyper_response_error) - .boxed() - }), - Scheme::HTTP, - server_addr, - client_addr, - ) - .await; - finalize_http_span(result, method) - } - .instrument(span) - .await - } - - async fn serve(self: Arc, listener: TcpListener, listen_addr: SocketAddr) -> Result<()> { - self.print_startup_msgs("http", &listener)?; - loop { - let (stream, client_addr) = listener.accept().await?; - self.clone() - .serve_connection(stream, listen_addr, client_addr); - } - } - - async fn serve_tls( - self: Arc, - listener: TcpListener, - listen_addr: SocketAddr, - tls: TlsConfig, - ) -> Result<()> { - let acceptor = tls.server_config()?; - self.print_startup_msgs("https", &listener)?; - - loop { - let (stream, addr) = listener.accept().await?; - match acceptor.accept(stream).await { - Ok(stream) => self.clone().serve_connection(stream, listen_addr, addr), - Err(err) => tracing::error!(?err, "Failed to start TLS session"), - } - } - } - - fn print_startup_msgs(&self, scheme: &str, listener: &TcpListener) -> Result<()> { - let local_addr = listener.local_addr()?; - let base_url = format!("{scheme}://{local_addr:?}"); - terminal::step!("\nServing", "{}", base_url); - log::info!("Serving {}", base_url); - - println!("Available Routes:"); - for (route, component_id) in self.router.routes() { - println!(" {}: {}{}", component_id, base_url, route); - if let Some(component) = self.engine.app().get_component(component_id) { - if let Some(description) = component.get_metadata(APP_DESCRIPTION_KEY)? { - println!(" {}", description); - } - } - } - Ok(()) - } -} - -fn parse_listen_addr(addr: &str) -> anyhow::Result { - let addrs: Vec = addr.to_socket_addrs()?.collect(); - // Prefer 127.0.0.1 over e.g. [::1] because CHANGE IS HARD - if let Some(addr) = addrs - .iter() - .find(|addr| addr.is_ipv4() && addr.ip() == Ipv4Addr::LOCALHOST) - { - return Ok(*addr); - } - // Otherwise, take the first addr (OS preference) - addrs.into_iter().next().context("couldn't resolve address") -} - -/// The incoming request's scheme and authority -/// -/// The incoming request's URI is relative to the server, so we need to set the scheme and authority -fn set_req_uri(req: &mut Request, scheme: Scheme, addr: SocketAddr) -> Result<()> { - let uri = req.uri().clone(); - let mut parts = uri.into_parts(); - let authority = format!("{}:{}", addr.ip(), addr.port()).parse().unwrap(); - parts.scheme = Some(scheme); - parts.authority = Some(authority); - *req.uri_mut() = Uri::from_parts(parts).unwrap(); - Ok(()) -} - -fn strip_forbidden_headers(req: &mut Request) { - let headers = req.headers_mut(); - if let Some(host_header) = headers.get("Host") { - if let Ok(host) = host_header.to_str() { - if is_service_chaining_host(host) { - headers.remove("Host"); - } - } - } -} - -// We need to make the following pieces of information available to both executors. -// While the values we set are identical, the way they are passed to the -// modules is going to be different, so each executor must must use the info -// in its standardized way (environment variables for the Wagi executor, and custom headers -// for the Spin HTTP executor). -const FULL_URL: [&str; 2] = ["SPIN_FULL_URL", "X_FULL_URL"]; -const PATH_INFO: [&str; 2] = ["SPIN_PATH_INFO", "PATH_INFO"]; -const MATCHED_ROUTE: [&str; 2] = ["SPIN_MATCHED_ROUTE", "X_MATCHED_ROUTE"]; -const COMPONENT_ROUTE: [&str; 2] = ["SPIN_COMPONENT_ROUTE", "X_COMPONENT_ROUTE"]; -const RAW_COMPONENT_ROUTE: [&str; 2] = ["SPIN_RAW_COMPONENT_ROUTE", "X_RAW_COMPONENT_ROUTE"]; -const BASE_PATH: [&str; 2] = ["SPIN_BASE_PATH", "X_BASE_PATH"]; -const CLIENT_ADDR: [&str; 2] = ["SPIN_CLIENT_ADDR", "X_CLIENT_ADDR"]; - -pub(crate) fn compute_default_headers( - uri: &Uri, - base: &str, - host: &str, - route_match: &RouteMatch, - client_addr: SocketAddr, -) -> Result> { - fn owned(strs: &[&'static str; 2]) -> [String; 2] { - [strs[0].to_owned(), strs[1].to_owned()] - } - - let owned_full_url: [String; 2] = owned(&FULL_URL); - let owned_path_info: [String; 2] = owned(&PATH_INFO); - let owned_matched_route: [String; 2] = owned(&MATCHED_ROUTE); - let owned_component_route: [String; 2] = owned(&COMPONENT_ROUTE); - let owned_raw_component_route: [String; 2] = owned(&RAW_COMPONENT_ROUTE); - let owned_base_path: [String; 2] = owned(&BASE_PATH); - let owned_client_addr: [String; 2] = owned(&CLIENT_ADDR); - - let mut res = vec![]; - let abs_path = uri - .path_and_query() - .expect("cannot get path and query") - .as_str(); - - let path_info = route_match.trailing_wildcard(); - - let scheme = uri.scheme_str().unwrap_or("http"); - - let full_url = format!("{}://{}{}", scheme, host, abs_path); - - res.push((owned_path_info, path_info)); - res.push((owned_full_url, full_url)); - res.push((owned_matched_route, route_match.based_route().to_string())); - - res.push((owned_base_path, base.to_string())); - res.push(( - owned_raw_component_route, - route_match.raw_route().to_string(), - )); - res.push((owned_component_route, route_match.raw_route_or_prefix())); - res.push((owned_client_addr, client_addr.to_string())); - - for (wild_name, wild_value) in route_match.named_wildcards() { - let wild_header = format!("SPIN_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer - let wild_wagi_header = format!("X_PATH_MATCH_{}", wild_name.to_ascii_uppercase()); // TODO: safer - res.push(([wild_header, wild_wagi_header], wild_value.clone())); - } - - Ok(res) -} - -/// The HTTP executor trait. -/// All HTTP executors must implement this trait. -#[async_trait] -pub(crate) trait HttpExecutor: Clone + Send + Sync + 'static { - async fn execute( - &self, - engine: Arc>, - base: &str, - route_match: &RouteMatch, - req: Request, - client_addr: SocketAddr, - ) -> Result>; -} - -#[derive(Clone)] -struct ChainedRequestHandler { - engine: Arc>, - executor: HttpHandlerExecutor, -} - -#[derive(Default)] -pub struct HttpRuntimeData { - origin: Option, - chained_handler: Option, - // Optional mapping of authority and TLS options for the current component - client_tls_opts: Option>, - /// The hosts this app is allowed to make outbound requests to - allowed_hosts: AllowedHostsConfig, -} - -impl HttpRuntimeData { - fn chain_request( - data: &mut spin_core::Data, - request: Request, - config: wasmtime_wasi_http::types::OutgoingRequestConfig, - component_id: String, - ) -> HttpResult { - use wasmtime_wasi_http::types::IncomingResponse; - - let this = data.as_ref(); - - let chained_handler = - this.chained_handler - .clone() - .ok_or(HttpError::trap(wasmtime::Error::msg( - "Internal error: internal request chaining not prepared (engine not assigned)", - )))?; - - let engine = chained_handler.engine; - let handler = chained_handler.executor; - - let base = "/"; - let route_match = RouteMatch::synthetic(&component_id, request.uri().path()); - - let client_addr = std::net::SocketAddr::from_str("0.0.0.0:0").unwrap(); - - let between_bytes_timeout = config.between_bytes_timeout; - - let resp_fut = async move { - match handler - .execute(engine.clone(), base, &route_match, request, client_addr) - .await - { - Ok(resp) => Ok(Ok(IncomingResponse { - resp, - between_bytes_timeout, - worker: None, - })), - Err(e) => Err(wasmtime::Error::msg(e)), - } - }; - - let handle = wasmtime_wasi::runtime::spawn(resp_fut); - Ok(HostFutureIncomingResponse::Pending(handle)) - } -} - -fn parse_chaining_target(request: &Request) -> Option { - parse_service_chaining_target(request.uri()) -} - -impl OutboundWasiHttpHandler for HttpRuntimeData { - fn send_request( - data: &mut spin_core::Data, - mut request: Request, - mut config: wasmtime_wasi_http::types::OutgoingRequestConfig, - ) -> HttpResult { - let this = data.as_mut(); - - let is_relative_url = request - .uri() - .authority() - .map(|a| a.host().trim() == "") - .unwrap_or_default(); - if is_relative_url { - // Origin must be set in the incoming http handler - let origin = this.origin.clone().unwrap(); - let path_and_query = request - .uri() - .path_and_query() - .map(|p| p.as_str()) - .unwrap_or("/"); - let uri: Uri = format!("{origin}{path_and_query}") - .parse() - // origin together with the path and query must be a valid URI - .unwrap(); - let host = format!("{}:{}", uri.host().unwrap(), uri.port().unwrap()); - let headers = request.headers_mut(); - headers.insert( - HOST, - HeaderValue::from_str(&host).map_err(|_| ErrorCode::HttpProtocolError)?, - ); - - config.use_tls = uri - .scheme() - .map(|s| s == &Scheme::HTTPS) - .unwrap_or_default(); - // We know that `uri` has an authority because we set it above - *request.uri_mut() = uri; - } - - let uri = request.uri(); - let uri_string = uri.to_string(); - let unallowed_relative = - is_relative_url && !this.allowed_hosts.allows_relative_url(&["http", "https"]); - let unallowed_absolute = !is_relative_url - && !this.allowed_hosts.allows( - &OutboundUrl::parse(uri_string, "https") - .map_err(|_| ErrorCode::HttpRequestUriInvalid)?, - ); - if unallowed_relative || unallowed_absolute { - tracing::error!("Destination not allowed: {}", request.uri()); - let host = if unallowed_absolute { - // Safe to unwrap because absolute urls have a host by definition. - let host = uri.authority().map(|a| a.host()).unwrap(); - let port = uri.authority().map(|a| a.port()).unwrap(); - let port = match port { - Some(port_str) => port_str.to_string(), - None => uri - .scheme() - .and_then(|s| (s == &Scheme::HTTP).then_some(80)) - .unwrap_or(443) - .to_string(), - }; - terminal::warn!( - "A component tried to make a HTTP request to non-allowed host '{host}'." - ); - let scheme = uri.scheme().unwrap_or(&Scheme::HTTPS); - format!("{scheme}://{host}:{port}") - } else { - terminal::warn!("A component tried to make a HTTP request to the same component but it does not have permission."); - "self".into() - }; - eprintln!("To allow requests, add 'allowed_outbound_hosts = [\"{}\"]' to the manifest component section.", host); - return Err(ErrorCode::HttpRequestDenied.into()); - } - - if let Some(component_id) = parse_chaining_target(&request) { - return Self::chain_request(data, request, config, component_id); - } - - let current_span = tracing::Span::current(); - let uri = request.uri(); - if let Some(authority) = uri.authority() { - current_span.record("server.address", authority.host()); - if let Some(port) = authority.port() { - current_span.record("server.port", port.as_u16()); - } - } - - let client_tls_opts = (data.as_ref()).client_tls_opts.clone(); - - // TODO: This is a temporary workaround to make sure that outbound task is instrumented. - // Once Wasmtime gives us the ability to do the spawn ourselves we can just call .instrument - // and won't have to do this workaround. - let response_handle = async move { - let res = send_request_handler(request, config, client_tls_opts).await; - if let Ok(res) = &res { - tracing::Span::current() - .record("http.response.status_code", res.resp.status().as_u16()); - } - Ok(res) - } - .in_current_span(); - Ok(HostFutureIncomingResponse::Pending( - wasmtime_wasi::runtime::spawn(response_handle), - )) - } -} - -#[derive(Debug, PartialEq)] -enum NotFoundRouteKind { - Normal(String), - WellKnown, -} - -/// This is a fork of wasmtime_wasi_http::default_send_request_handler function -/// forked from bytecodealliance/wasmtime commit-sha 29a76b68200fcfa69c8fb18ce6c850754279a05b -/// This fork provides the ability to configure client cert auth for mTLS -pub async fn send_request_handler( - mut request: hyper::Request, - wasmtime_wasi_http::types::OutgoingRequestConfig { - use_tls, - connect_timeout, - first_byte_timeout, - between_bytes_timeout, - }: wasmtime_wasi_http::types::OutgoingRequestConfig, - client_tls_opts: Option>, -) -> Result { - let authority_str = if let Some(authority) = request.uri().authority() { - if authority.port().is_some() { - authority.to_string() - } else { - let port = if use_tls { 443 } else { 80 }; - format!("{}:{port}", authority) - } - } else { - return Err(types::ErrorCode::HttpRequestUriInvalid); - }; - - let authority = &authority_str.parse::().unwrap(); - - let tcp_stream = timeout(connect_timeout, TcpStream::connect(&authority_str)) - .await - .map_err(|_| types::ErrorCode::ConnectionTimeout)? - .map_err(|e| match e.kind() { - std::io::ErrorKind::AddrNotAvailable => { - dns_error("address not available".to_string(), 0) - } - - _ => { - if e.to_string() - .starts_with("failed to lookup address information") - { - dns_error("address not available".to_string(), 0) - } else { - types::ErrorCode::ConnectionRefused - } - } - })?; - - let (mut sender, worker) = if use_tls { - #[cfg(any(target_arch = "riscv64", target_arch = "s390x"))] - { - return Err( - wasmtime_wasi_http::bindings::http::types::ErrorCode::InternalError(Some( - "unsupported architecture for SSL".to_string(), - )), - ); - } - - #[cfg(not(any(target_arch = "riscv64", target_arch = "s390x")))] - { - use rustls::pki_types::ServerName; - let config = - get_client_tls_config_for_authority(authority, client_tls_opts).map_err(|e| { - wasmtime_wasi_http::bindings::http::types::ErrorCode::InternalError(Some( - format!( - "failed to configure client tls config for authority. error: {}", - e - ), - )) - })?; - let connector = tokio_rustls::TlsConnector::from(std::sync::Arc::new(config)); - let mut parts = authority_str.split(':'); - let host = parts.next().unwrap_or(&authority_str); - let domain = ServerName::try_from(host) - .map_err(|e| { - tracing::warn!("dns lookup error: {e:?}"); - dns_error("invalid dns name".to_string(), 0) - })? - .to_owned(); - let stream = connector.connect(domain, tcp_stream).await.map_err(|e| { - tracing::warn!("tls protocol error: {e:?}"); - types::ErrorCode::TlsProtocolError - })?; - let stream = TokioIo::new(stream); - - let (sender, conn) = timeout( - connect_timeout, - hyper::client::conn::http1::handshake(stream), - ) - .await - .map_err(|_| types::ErrorCode::ConnectionTimeout)? - .map_err(hyper_request_error)?; - - let worker = wasmtime_wasi::runtime::spawn(async move { - match conn.await { - Ok(()) => {} - // TODO: shouldn't throw away this error and ideally should - // surface somewhere. - Err(e) => tracing::warn!("dropping error {e}"), - } - }); - - (sender, worker) - } - } else { - let tcp_stream = TokioIo::new(tcp_stream); - let (sender, conn) = timeout( - connect_timeout, - // TODO: we should plumb the builder through the http context, and use it here - hyper::client::conn::http1::handshake(tcp_stream), - ) - .await - .map_err(|_| types::ErrorCode::ConnectionTimeout)? - .map_err(hyper_request_error)?; - - let worker = wasmtime_wasi::runtime::spawn(async move { - match conn.await { - Ok(()) => {} - // TODO: same as above, shouldn't throw this error away. - Err(e) => tracing::warn!("dropping error {e}"), - } - }); - - (sender, worker) - }; - - // at this point, the request contains the scheme and the authority, but - // the http packet should only include those if addressing a proxy, so - // remove them here, since SendRequest::send_request does not do it for us - *request.uri_mut() = http::Uri::builder() - .path_and_query( - request - .uri() - .path_and_query() - .map(|p| p.as_str()) - .unwrap_or("/"), - ) - .build() - .expect("comes from valid request"); - - let resp = timeout(first_byte_timeout, sender.send_request(request)) - .await - .map_err(|_| types::ErrorCode::ConnectionReadTimeout)? - .map_err(hyper_request_error)? - .map(|body| body.map_err(hyper_request_error).boxed()); - - Ok(wasmtime_wasi_http::types::IncomingResponse { - resp, - worker: Some(worker), - between_bytes_timeout, - }) -} - -fn get_client_tls_config_for_authority( - authority: &Authority, - client_tls_opts: Option>, -) -> Result { - // derived from https://github.com/tokio-rs/tls/blob/master/tokio-rustls/examples/client/src/main.rs - let ca_webpki_roots = rustls::RootCertStore { - roots: webpki_roots::TLS_SERVER_ROOTS.into(), - }; - - #[allow(clippy::mutable_key_type)] - let client_tls_opts = match client_tls_opts { - Some(opts) => opts, - _ => { - return Ok(rustls::ClientConfig::builder() - .with_root_certificates(ca_webpki_roots) - .with_no_client_auth()); - } - }; - - let client_tls_opts_for_host = match client_tls_opts.get(authority) { - Some(opts) => opts, - _ => { - return Ok(rustls::ClientConfig::builder() - .with_root_certificates(ca_webpki_roots) - .with_no_client_auth()); - } - }; - - let mut root_cert_store = if client_tls_opts_for_host.ca_webpki_roots { - ca_webpki_roots - } else { - rustls::RootCertStore::empty() - }; - - if let Some(custom_root_ca) = &client_tls_opts_for_host.custom_root_ca { - for cer in custom_root_ca { - match root_cert_store.add(cer.to_owned()) { - Ok(_) => {} - Err(e) => { - return Err(anyhow::anyhow!( - "failed to add custom cert to root_cert_store. error: {}", - e - )); - } - } - } - } - - match ( - &client_tls_opts_for_host.cert_chain, - &client_tls_opts_for_host.private_key, - ) { - (Some(cert_chain), Some(private_key)) => Ok(rustls::ClientConfig::builder() - .with_root_certificates(root_cert_store) - .with_client_auth_cert(cert_chain.to_owned(), private_key.clone_key())?), - _ => Ok(rustls::ClientConfig::builder() - .with_root_certificates(root_cert_store) - .with_no_client_auth()), - } -} - -/// Translate a [`hyper::Error`] to a wasi-http `ErrorCode` in the context of a request. -pub fn hyper_request_error(err: hyper::Error) -> ErrorCode { - // If there's a source, we might be able to extract a wasi-http error from it. - if let Some(cause) = err.source() { - if let Some(err) = cause.downcast_ref::() { - return err.clone(); - } - } - - tracing::warn!("hyper request error: {err:?}"); - - ErrorCode::HttpProtocolError -} - -pub fn dns_error(rcode: String, info_code: u16) -> ErrorCode { - ErrorCode::DnsError(wasmtime_wasi_http::bindings::http::types::DnsErrorPayload { - rcode: Some(rcode), - info_code: Some(info_code), - }) -} - -#[cfg(test)] -mod tests { - use anyhow::Result; - - use super::*; - - #[test] - fn test_default_headers_with_base_path() -> Result<()> { - let scheme = "https"; - let host = "fermyon.dev"; - let base = "/base"; - let trigger_route = "/foo/..."; - let component_path = "/foo"; - let path_info = "/bar"; - let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); - - let req_uri = format!( - "{}://{}{}{}{}?key1=value1&key2=value2", - scheme, host, base, component_path, path_info - ); - - let req = http::Request::builder() - .method("POST") - .uri(req_uri) - .body("")?; - - let (router, _) = Router::build(base, [("DUMMY", &trigger_route.into())])?; - let route_match = router.route("/base/foo/bar")?; - - let default_headers = - crate::compute_default_headers(req.uri(), base, host, &route_match, client_addr)?; - - assert_eq!( - search(&FULL_URL, &default_headers).unwrap(), - "https://fermyon.dev/base/foo/bar?key1=value1&key2=value2".to_string() - ); - assert_eq!( - search(&PATH_INFO, &default_headers).unwrap(), - "/bar".to_string() - ); - assert_eq!( - search(&MATCHED_ROUTE, &default_headers).unwrap(), - "/base/foo/...".to_string() - ); - assert_eq!( - search(&BASE_PATH, &default_headers).unwrap(), - "/base".to_string() - ); - assert_eq!( - search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo/...".to_string() - ); - assert_eq!( - search(&COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo".to_string() - ); - assert_eq!( - search(&CLIENT_ADDR, &default_headers).unwrap(), - "127.0.0.1:8777".to_string() - ); - - Ok(()) - } - - #[test] - fn test_default_headers_without_base_path() -> Result<()> { - let scheme = "https"; - let host = "fermyon.dev"; - let base = "/"; - let trigger_route = "/foo/..."; - let component_path = "/foo"; - let path_info = "/bar"; - let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); - - let req_uri = format!( - "{}://{}{}{}?key1=value1&key2=value2", - scheme, host, component_path, path_info - ); - - let req = http::Request::builder() - .method("POST") - .uri(req_uri) - .body("")?; - - let (router, _) = Router::build(base, [("DUMMY", &trigger_route.into())])?; - let route_match = router.route("/foo/bar")?; - - let default_headers = - crate::compute_default_headers(req.uri(), base, host, &route_match, client_addr)?; - - // TODO: we currently replace the scheme with HTTP. When TLS is supported, this should be fixed. - assert_eq!( - search(&FULL_URL, &default_headers).unwrap(), - "https://fermyon.dev/foo/bar?key1=value1&key2=value2".to_string() - ); - assert_eq!( - search(&PATH_INFO, &default_headers).unwrap(), - "/bar".to_string() - ); - assert_eq!( - search(&MATCHED_ROUTE, &default_headers).unwrap(), - "/foo/...".to_string() - ); - assert_eq!( - search(&BASE_PATH, &default_headers).unwrap(), - "/".to_string() - ); - assert_eq!( - search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo/...".to_string() - ); - assert_eq!( - search(&COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo".to_string() - ); - assert_eq!( - search(&CLIENT_ADDR, &default_headers).unwrap(), - "127.0.0.1:8777".to_string() - ); - - Ok(()) - } - - #[test] - fn test_default_headers_with_named_wildcards() -> Result<()> { - let scheme = "https"; - let host = "fermyon.dev"; - let base = "/"; - let trigger_route = "/foo/:userid/..."; - let component_path = "/foo"; - let path_info = "/bar"; - let client_addr: SocketAddr = "127.0.0.1:8777".parse().unwrap(); - - let req_uri = format!( - "{}://{}{}/42{}?key1=value1&key2=value2", - scheme, host, component_path, path_info - ); - - let req = http::Request::builder() - .method("POST") - .uri(req_uri) - .body("")?; - - let (router, _) = Router::build(base, [("DUMMY", &trigger_route.into())])?; - let route_match = router.route("/foo/42/bar")?; - - let default_headers = - crate::compute_default_headers(req.uri(), base, host, &route_match, client_addr)?; - - // TODO: we currently replace the scheme with HTTP. When TLS is supported, this should be fixed. - assert_eq!( - search(&FULL_URL, &default_headers).unwrap(), - "https://fermyon.dev/foo/42/bar?key1=value1&key2=value2".to_string() - ); - assert_eq!( - search(&PATH_INFO, &default_headers).unwrap(), - "/bar".to_string() - ); - assert_eq!( - search(&MATCHED_ROUTE, &default_headers).unwrap(), - "/foo/:userid/...".to_string() - ); - assert_eq!( - search(&BASE_PATH, &default_headers).unwrap(), - "/".to_string() - ); - assert_eq!( - search(&RAW_COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo/:userid/...".to_string() - ); - assert_eq!( - search(&COMPONENT_ROUTE, &default_headers).unwrap(), - "/foo/:userid".to_string() - ); - assert_eq!( - search(&CLIENT_ADDR, &default_headers).unwrap(), - "127.0.0.1:8777".to_string() - ); - - assert_eq!( - search( - &["SPIN_PATH_MATCH_USERID", "X_PATH_MATCH_USERID"], - &default_headers - ) - .unwrap(), - "42".to_string() - ); - - Ok(()) - } - - fn search(keys: &[&str; 2], headers: &[([String; 2], String)]) -> Option { - let mut res: Option = None; - for (k, v) in headers { - if k[0] == keys[0] && k[1] == keys[1] { - res = Some(v.clone()); - } - } - - res - } - - #[test] - fn parse_listen_addr_prefers_ipv4() { - let addr = parse_listen_addr("localhost:12345").unwrap(); - assert_eq!(addr.ip(), Ipv4Addr::LOCALHOST); - assert_eq!(addr.port(), 12345); - } - - #[test] - fn forbidden_headers_are_removed() { - let mut req = Request::get("http://test.spin.internal") - .header("Host", "test.spin.internal") - .header("accept", "text/plain") - .body(Default::default()) - .unwrap(); - - strip_forbidden_headers(&mut req); - - assert_eq!(1, req.headers().len()); - assert!(req.headers().get("Host").is_none()); - - let mut req = Request::get("http://test.spin.internal") - .header("Host", "test.spin.internal:1234") - .header("accept", "text/plain") - .body(Default::default()) - .unwrap(); - - strip_forbidden_headers(&mut req); - - assert_eq!(1, req.headers().len()); - assert!(req.headers().get("Host").is_none()); - } - - #[test] - fn non_forbidden_headers_are_not_removed() { - let mut req = Request::get("http://test.example.com") - .header("Host", "test.example.org") - .header("accept", "text/plain") - .body(Default::default()) - .unwrap(); - - strip_forbidden_headers(&mut req); - - assert_eq!(2, req.headers().len()); - assert!(req.headers().get("Host").is_some()); - } -} diff --git a/crates/trigger-http/src/testdata/invalid-cert.pem b/crates/trigger-http/src/testdata/invalid-cert.pem deleted file mode 100644 index f1a952b9c..000000000 --- a/crates/trigger-http/src/testdata/invalid-cert.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBkjCCATegAwIBAgIIEOURVvWgx1AwCgYIKoZIzj0EAwIwIzEhMB8GA1UEAwwY -azNzLWNsaWVudC1jYUAxNzE3NzgwNTIwMB4XDTI0MDYwNzE3MTUyMFoXDTI1MDYw -NzE3MTUyMFowMDEXMBUGA1UEChMOc3lzdGVtOm1hc3RlcnMxFTATBgNVBAMTDHN5 -c3RlbTphZG1pbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFGE/CVuauj8kmde -i4AagSJ5GYgGnL0eF55ItiXrKSjMmsIf/N8EyeamxQfWPKVk/1xhH7cS9GcQgNe6 -XrRvmLyjSDBGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAf -BgNVHSMEGDAWgBRpihySeW3DafmU1cw6LMnQCQDD4jAKBggqhkjOPQQDAgNJADBG -AiEA/db1wb4mVrqJVctqbPU9xd0bXzJx7cBDzpWgPP9ISfkCIQDNyuskAkXvUMHH -F73/GJnh8Bt2H38qyzThM8nlR9v1eQ== ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3MtY2xp -ZW50LWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw -WjAjMSEwHwYDVQQDDBhrM3MtY2xpZW50LWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO -PQIBBggqhkjOPQMBBwNCAASozciE0YGl8ak3G0Ll1riwXSScfpK0QRle/cFizdlA -HgDowBssBcla0/2a/eWabxqTPzsZH0cVhL7Tialoj8GNo0IwQDAOBgNVHQ8BAf8E -BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUaYocknltw2n5lNXMOizJ -0AkAw+IwCgYIKoZIzj0EAwIDRwAwRAIgR8YcLA8cH4qAMDRPDsJqLaw4GJFkgjwV -TCrMgyUxSvACIBwyklgm7mgHcC5WM9CqmliAGZJyV0xRPZBK01POrNf0 diff --git a/crates/trigger-http/src/testdata/invalid-private-key.pem b/crates/trigger-http/src/testdata/invalid-private-key.pem deleted file mode 100644 index 39d7e59ee..000000000 --- a/crates/trigger-http/src/testdata/invalid-private-key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIA+FBtmKJbd8wBGOWeuJQfHiCKjjXF8ywEPrvj8S1N3VoAoGCCqGSM49 -AwEHoUQDQgAEUYT8JW5q6PySZ16LgBqBInkZiAacvR4Xnki2JespKMyawh/83wTJ -5qbFB9Y8pWT/XGEftxL0ZxCA17petG+YvA== ------END EC PRIVATE KEY- \ No newline at end of file diff --git a/crates/trigger-http/src/testdata/valid-cert.pem b/crates/trigger-http/src/testdata/valid-cert.pem deleted file mode 100644 index e75166d0e..000000000 --- a/crates/trigger-http/src/testdata/valid-cert.pem +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBkjCCATegAwIBAgIIEOURVvWgx1AwCgYIKoZIzj0EAwIwIzEhMB8GA1UEAwwY -azNzLWNsaWVudC1jYUAxNzE3NzgwNTIwMB4XDTI0MDYwNzE3MTUyMFoXDTI1MDYw -NzE3MTUyMFowMDEXMBUGA1UEChMOc3lzdGVtOm1hc3RlcnMxFTATBgNVBAMTDHN5 -c3RlbTphZG1pbjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFGE/CVuauj8kmde -i4AagSJ5GYgGnL0eF55ItiXrKSjMmsIf/N8EyeamxQfWPKVk/1xhH7cS9GcQgNe6 -XrRvmLyjSDBGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDAjAf -BgNVHSMEGDAWgBRpihySeW3DafmU1cw6LMnQCQDD4jAKBggqhkjOPQQDAgNJADBG -AiEA/db1wb4mVrqJVctqbPU9xd0bXzJx7cBDzpWgPP9ISfkCIQDNyuskAkXvUMHH -F73/GJnh8Bt2H38qyzThM8nlR9v1eQ== ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3MtY2xp -ZW50LWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw -WjAjMSEwHwYDVQQDDBhrM3MtY2xpZW50LWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO -PQIBBggqhkjOPQMBBwNCAASozciE0YGl8ak3G0Ll1riwXSScfpK0QRle/cFizdlA -HgDowBssBcla0/2a/eWabxqTPzsZH0cVhL7Tialoj8GNo0IwQDAOBgNVHQ8BAf8E -BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUaYocknltw2n5lNXMOizJ -0AkAw+IwCgYIKoZIzj0EAwIDRwAwRAIgR8YcLA8cH4qAMDRPDsJqLaw4GJFkgjwV -TCrMgyUxSvACIBwyklgm7mgHcC5WM9CqmliAGZJyV0xRPZBK01POrNf0 ------END CERTIFICATE----- \ No newline at end of file diff --git a/crates/trigger-http/src/testdata/valid-private-key.pem b/crates/trigger-http/src/testdata/valid-private-key.pem deleted file mode 100644 index 2820fbed2..000000000 --- a/crates/trigger-http/src/testdata/valid-private-key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIA+FBtmKJbd8wBGOWeuJQfHiCKjjXF8ywEPrvj8S1N3VoAoGCCqGSM49 -AwEHoUQDQgAEUYT8JW5q6PySZ16LgBqBInkZiAacvR4Xnki2JespKMyawh/83wTJ -5qbFB9Y8pWT/XGEftxL0ZxCA17petG+YvA== ------END EC PRIVATE KEY----- \ No newline at end of file diff --git a/crates/trigger-http/src/tls.rs b/crates/trigger-http/src/tls.rs deleted file mode 100644 index d0486c50e..000000000 --- a/crates/trigger-http/src/tls.rs +++ /dev/null @@ -1,141 +0,0 @@ -use rustls_pemfile::private_key; -use std::{ - fs, io, - path::{Path, PathBuf}, - sync::Arc, -}; -use tokio_rustls::{rustls, TlsAcceptor}; - -/// TLS configuration for the server. -#[derive(Clone)] -pub struct TlsConfig { - /// Path to TLS certificate. - pub cert_path: PathBuf, - /// Path to TLS key. - pub key_path: PathBuf, -} - -impl TlsConfig { - // Creates a TLS acceptor from server config. - pub(super) fn server_config(&self) -> anyhow::Result { - let certs = load_certs(&self.cert_path)?; - let private_key = load_key(&self.key_path)?; - - let cfg = rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, private_key) - .map_err(|e| anyhow::anyhow!("{}", e))?; - - Ok(Arc::new(cfg).into()) - } -} - -// load_certs parse and return the certs from the provided file -pub fn load_certs( - path: impl AsRef, -) -> io::Result>> { - rustls_pemfile::certs(&mut io::BufReader::new(fs::File::open(path).map_err( - |err| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("failed to read cert file {:?}", err), - ) - }, - )?)) - .collect() -} - -// parse and return the first private key from the provided file -pub fn load_key(path: impl AsRef) -> io::Result> { - private_key(&mut io::BufReader::new(fs::File::open(path).map_err( - |err| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("failed to read private key file {:?}", err), - ) - }, - )?)) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid private key")) - .transpose() - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "private key file contains no private keys", - ) - })? -} - -#[cfg(test)] -mod tests { - use super::*; - - fn testdatadir() -> PathBuf { - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push("src"); - path.push("testdata"); - - path - } - - #[test] - fn test_read_non_existing_cert() { - let mut path = testdatadir(); - path.push("non-existing-file.pem"); - - let certs = load_certs(path); - assert!(certs.is_err()); - assert_eq!(certs.err().unwrap().to_string(), "failed to read cert file Os { code: 2, kind: NotFound, message: \"No such file or directory\" }"); - } - - #[test] - fn test_read_invalid_cert() { - let mut path = testdatadir(); - path.push("invalid-cert.pem"); - - let certs = load_certs(path); - assert!(certs.is_err()); - assert_eq!( - certs.err().unwrap().to_string(), - "section end \"-----END CERTIFICATE-----\" missing" - ); - } - - #[test] - fn test_read_valid_cert() { - let mut path = testdatadir(); - path.push("valid-cert.pem"); - - let certs = load_certs(path); - assert!(certs.is_ok()); - assert_eq!(certs.unwrap().len(), 2); - } - - #[test] - fn test_read_non_existing_private_key() { - let mut path = testdatadir(); - path.push("non-existing-file.pem"); - - let keys = load_key(path); - assert!(keys.is_err()); - assert_eq!(keys.err().unwrap().to_string(), "failed to read private key file Os { code: 2, kind: NotFound, message: \"No such file or directory\" }"); - } - - #[test] - fn test_read_invalid_private_key() { - let mut path = testdatadir(); - path.push("invalid-private-key.pem"); - - let keys = load_key(path); - assert!(keys.is_err()); - assert_eq!(keys.err().unwrap().to_string(), "invalid private key"); - } - - #[test] - fn test_read_valid_private_key() { - let mut path = testdatadir(); - path.push("valid-private-key.pem"); - - let keys = load_key(path); - assert!(keys.is_ok()); - } -} diff --git a/crates/trigger-http/src/wagi.rs b/crates/trigger-http/src/wagi.rs deleted file mode 100644 index 0cb020200..000000000 --- a/crates/trigger-http/src/wagi.rs +++ /dev/null @@ -1,142 +0,0 @@ -use std::{io::Cursor, net::SocketAddr, sync::Arc}; - -use crate::HttpInstance; -use anyhow::{anyhow, ensure, Context, Result}; -use async_trait::async_trait; -use http_body_util::BodyExt; -use hyper::{Request, Response}; -use spin_core::WasiVersion; -use spin_http::{config::WagiTriggerConfig, routes::RouteMatch, wagi}; -use spin_trigger::TriggerAppEngine; -use tracing::{instrument, Level}; -use wasi_common_preview1::{pipe::WritePipe, I32Exit}; - -use crate::{Body, HttpExecutor, HttpTrigger}; - -#[derive(Clone)] -pub struct WagiHttpExecutor { - pub wagi_config: WagiTriggerConfig, -} - -#[async_trait] -impl HttpExecutor for WagiHttpExecutor { - #[instrument(name = "spin_trigger_http.execute_wagi", skip_all, err(level = Level::INFO), fields(otel.name = format!("execute_wagi_component {}", route_match.component_id())))] - async fn execute( - &self, - engine: Arc>, - base: &str, - route_match: &RouteMatch, - req: Request, - client_addr: SocketAddr, - ) -> Result> { - let component = route_match.component_id(); - - tracing::trace!( - "Executing request using the Wagi executor for component {}", - component - ); - - let uri_path = req.uri().path(); - - // Build the argv array by starting with the config for `argv` and substituting in - // script name and args where appropriate. - let script_name = uri_path.to_string(); - let args = req.uri().query().unwrap_or_default().replace('&', " "); - let argv = self - .wagi_config - .argv - .clone() - .replace("${SCRIPT_NAME}", &script_name) - .replace("${ARGS}", &args); - - let (parts, body) = req.into_parts(); - - let body = body.collect().await?.to_bytes().to_vec(); - let len = body.len(); - - // TODO - // The default host and TLS fields are currently hard-coded. - let mut headers = - wagi::build_headers(route_match, &parts, len, client_addr, "default_host", false); - - let default_host = http::HeaderValue::from_str("localhost")?; - let host = std::str::from_utf8( - parts - .headers - .get("host") - .unwrap_or(&default_host) - .as_bytes(), - )?; - - // Add the default Spin headers. - // This sets the current environment variables Wagi expects (such as - // `PATH_INFO`, or `X_FULL_URL`). - // Note that this overrides any existing headers previously set by Wagi. - for (keys, val) in - crate::compute_default_headers(&parts.uri, base, host, route_match, client_addr)? - { - headers.insert(keys[1].to_string(), val); - } - - let stdout = WritePipe::new_in_memory(); - - let mut store_builder = engine.store_builder(component, WasiVersion::Preview1)?; - // Set up Wagi environment - store_builder.args(argv.split(' '))?; - store_builder.env(headers)?; - store_builder.stdin_pipe(Cursor::new(body)); - store_builder.stdout(Box::new(stdout.clone()))?; - - let (instance, mut store) = engine - .prepare_instance_with_store(component, store_builder) - .await?; - - let HttpInstance::Module(instance) = instance else { - unreachable!() - }; - - let start = instance - .get_func(&mut store, &self.wagi_config.entrypoint) - .ok_or_else(|| { - anyhow::anyhow!( - "No such function '{}' in {}", - self.wagi_config.entrypoint, - component - ) - })?; - tracing::trace!("Calling Wasm entry point"); - start - .call_async(&mut store, &[], &mut []) - .await - .or_else(ignore_successful_proc_exit_trap) - .with_context(|| { - anyhow!( - "invoking {} for component {component}", - self.wagi_config.entrypoint - ) - })?; - tracing::info!("Module execution complete"); - - // Drop the store so we're left with a unique reference to `stdout`: - drop(store); - - let stdout = stdout.try_into_inner().unwrap().into_inner(); - ensure!( - !stdout.is_empty(), - "The {component:?} component is configured to use the WAGI executor \ - but did not write to stdout. Check the `executor` in spin.toml." - ); - - wagi::compose_response(&stdout) - } -} - -fn ignore_successful_proc_exit_trap(guest_err: anyhow::Error) -> Result<()> { - match guest_err.root_cause().downcast_ref::() { - Some(trap) => match trap.0 { - 0 => Ok(()), - _ => Err(guest_err), - }, - None => Err(guest_err), - } -} diff --git a/crates/trigger-http/tests/local.crt.pem b/crates/trigger-http/tests/local.crt.pem deleted file mode 100644 index efd51f670..000000000 --- a/crates/trigger-http/tests/local.crt.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICujCCAaICCQClexHj2O4K/TANBgkqhkiG9w0BAQsFADAfMQswCQYDVQQGEwJV -UzEQMA4GA1UECgwHRmVybXlvbjAeFw0yMjAyMjUxNzQ3MTFaFw0yMzAyMjUxNzQ3 -MTFaMB8xCzAJBgNVBAYTAlVTMRAwDgYDVQQKDAdGZXJteW9uMIIBIjANBgkqhkiG -9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwMbUZ2eoIaJfgcBJ2fILUViWYApnA9SU+Ruf -nm6DNm9Gy5+YThqxd/0mhbPwYVkfi2/3UddWDl3VPOAYcvYoHDqH0tHm10wo+UzY -DDcNZB9enLRfGCv9Fful4bqNd3Vtx2xNwc8+F0WiljtYeMc+9wp7M5WWbKJqzKPe -VQBADRlfGoG3jCLGaQ2fyVp/73nWdqbbluWJopxHph7v1alb/BxLcDi/tjWKgZut -Vr9ZtBBPDSjRbfjHarn6pibYZAWgzanpfsaSBdbpVNn1MQ/gNXIHmNFwfbsN0V+3 -LN/Z4VNZrkc+C7CjGhJOcBj0xtrSDhoHnOmDS/z+lBUdlNOUrQIDAQABMA0GCSqG -SIb3DQEBCwUAA4IBAQAOnRPnUJoEE8s9+ADUpKkWBXFCiRajtBSBDNDX3phRPwly -q2zG+gXyV+Axx1qvsis9yXQBF9DcD+lx0rEgGzQjYGfmEA45E8Co2Tih2ON7JkCu -bYoT+wMkgfOMci/S2BBOJ+d0LI3K0b1qDfc4KwHe6g3p5ywuEBFOaWKiMemJyywd -zpoD6QmcQ9qlp5/2pf12bNRUIdXe5+vMU3qVIZcWM49u04L2/Swyc6EFXfEtnp/m -6184isfCkc3egMvqEfrKUaf0lgNzCksmRD9sLF8wWaV4lcidzsNDdU47EPFutVMU -3iLgXAhmRuZ+eoBf56QkzVTQWnCYQdlGwZp1Fcoj ------END CERTIFICATE----- diff --git a/crates/trigger-http/tests/local.key.pem b/crates/trigger-http/tests/local.key.pem deleted file mode 100644 index 6b080db69..000000000 --- a/crates/trigger-http/tests/local.key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDAxtRnZ6ghol+B -wEnZ8gtRWJZgCmcD1JT5G5+eboM2b0bLn5hOGrF3/SaFs/BhWR+Lb/dR11YOXdU8 -4Bhy9igcOofS0ebXTCj5TNgMNw1kH16ctF8YK/0V+6Xhuo13dW3HbE3Bzz4XRaKW -O1h4xz73CnszlZZsomrMo95VAEANGV8agbeMIsZpDZ/JWn/vedZ2ptuW5YminEem -Hu/VqVv8HEtwOL+2NYqBm61Wv1m0EE8NKNFt+MdqufqmJthkBaDNqel+xpIF1ulU -2fUxD+A1cgeY0XB9uw3RX7cs39nhU1muRz4LsKMaEk5wGPTG2tIOGgec6YNL/P6U -FR2U05StAgMBAAECggEAfpcSjATJp6yUwwOee3wyamyd8tth4mYKnbrCCqvPhkN0 -XeqjfUaSG5UlYs9SntqDmHEiG6AoZq6/hIY0B+oVVNQqtQoZaHAex/bqOLs+E+11 -l7nqaFkajQD/YUe79iIqwLYiKY8J2wZjSfwWkNlmQ5uiY7FrYlMVhuRk77SGWxKW -UbWfgTTMgEWIK6bU77FShQ7b0px5ZIulRPQeRaH8USdx0yktqUMwUakIrNyZ64u+ -Gx9k4ma2bCmbWxGlCEp0EQsYOlWDBeKu3Elq2g48KmADzbjvKlS7S/0fhcVqi2dE -Fj0BrmzxWjPzJwqxA6Z/8tykqzL5Nr6tOm0e6ZhBEQKBgQDhfy83jLfIWLt3rMcx -dFA4TGFSEUVJE9ESV0Za5zriLeGzM66JGut+Bph9Kk7XmDt+q3ewFJv7oDVibhzG -4nit+TakfSMWUAronsf2wQuUvpE6rNoZlWjhd7AE5f/eBZTYhNm5cp7ujGwnEn47 -vmfSVev+1yQcEUeV10OSWWaCrwKBgQDa2pEwps6htnZqiZsJP86LfxbTA1P+BgsV -nFvVkcCT0Uy7V0pSSdA82Ua/1KfcQ3BAJiBkINSL6Sob1+3lSQTeTHLVbXySacnh -c7UDDoayWJxtYNyjJeBzrjlZCDIkipJqz26pGfIhxePwVgbj30O/EB55y44gkxqn -JIvqIWBlYwKBgQDVqR4DI3lMAw92QKbo7A3KmkyoZybgLD+wgjNulKQNhW3Sz4hz -7qbt3bAFAN59l4ff6PZaR9zYWh/bKPxpUlMIfRdSWiOx05vSeAh+fMHNaZfQIdHx -5cjfwfltWsTLCTzUv2RRPBLtcu5TQ0mKsEpNWQ5ohE95rMHIb5ReCgmAjwKBgCb6 -NlGL49E5Re3DhDEphAekItSCCzt6qA65QkHPK5Un+ZqD+WCedM/hgpA3t42rFRrX -r30lu7UPWciLtHrZflx5ERqh3UXWQXY9vUdGFwc8cN+qGKGV5Vu089G/e+62H02W -lAbZ8B3DuMzdBW0gHliw7jyS3EVA7cZG5ARW3WwxAoGAW+FkrJKsPyyScHBdu/LD -GeDMGRBRBdthXVbtB7xkzi2Tla4TywlHTm32rK3ywtoBxzvhlxbVnbBODMO/83xZ -DKjq2leuKfXUNsuMEre7uhhs7ezEM6QfiKTTosD/D8Z3S8AA4q1NKu3iEBUjtXcS -FSaIdbf6aHPcvbRB9cDv5ho= ------END PRIVATE KEY----- diff --git a/crates/trigger-redis/Cargo.toml b/crates/trigger-redis/Cargo.toml deleted file mode 100644 index 70a90a61d..000000000 --- a/crates/trigger-redis/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "spin-trigger-redis" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[lib] -doctest = false - -[dependencies] -anyhow = "1.0" -async-trait = "0.1" -futures = "0.3" -serde = "1.0.188" -spin-app = { path = "../app" } -spin-common = { path = "../common" } -spin-core = { path = "../core" } -spin-expressions = { path = "../expressions" } -spin-trigger = { path = "../trigger" } -spin-world = { path = "../world" } -redis = { version = "0.21", features = ["tokio-comp"] } -tracing = { workspace = true } -tokio = { version = "1.23", features = ["full"] } -spin-telemetry = { path = "../telemetry" } - -[dev-dependencies] -spin-testing = { path = "../testing" } - -[lints] -workspace = true diff --git a/crates/trigger-redis/src/lib.rs b/crates/trigger-redis/src/lib.rs deleted file mode 100644 index 440fc00b2..000000000 --- a/crates/trigger-redis/src/lib.rs +++ /dev/null @@ -1,203 +0,0 @@ -//! Implementation for the Spin Redis engine. - -mod spin; - -use anyhow::{anyhow, Context, Result}; -use futures::{future::join_all, StreamExt}; -use redis::{Client, ConnectionLike}; -use serde::{de::IgnoredAny, Deserialize, Serialize}; -use spin_common::url::remove_credentials; -use spin_core::{async_trait, InstancePre}; -use spin_trigger::{cli::NoArgs, TriggerAppEngine, TriggerExecutor}; -use std::collections::HashMap; -use std::sync::Arc; -use tracing::{instrument, Level}; - -use crate::spin::SpinRedisExecutor; - -pub(crate) type RuntimeData = (); -pub(crate) type Store = spin_core::Store; - -type ChannelComponents = HashMap>; -/// The Spin Redis trigger. -#[derive(Clone)] -pub struct RedisTrigger { - engine: Arc>, - // Mapping of server url with subscription channel and associated component IDs - server_channels: HashMap, -} - -/// Redis trigger configuration. -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct RedisTriggerConfig { - /// Component ID to invoke - pub component: String, - /// Channel to subscribe to - pub channel: String, - /// optional overide address for trigger - pub address: Option, - /// Trigger executor (currently unused) - #[serde(default, skip_serializing)] - pub executor: IgnoredAny, -} - -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -struct TriggerMetadata { - address: String, -} - -#[async_trait] -impl TriggerExecutor for RedisTrigger { - const TRIGGER_TYPE: &'static str = "redis"; - type RuntimeData = RuntimeData; - type TriggerConfig = RedisTriggerConfig; - type RunConfig = NoArgs; - type InstancePre = InstancePre; - - async fn new(engine: TriggerAppEngine) -> Result { - let default_address: String = engine - .trigger_metadata::()? - .unwrap_or_default() - .address; - let default_address_expr = spin_expressions::Template::new(default_address)?; - let default_address = engine.resolve_template(&default_address_expr)?; - - let mut server_channels: HashMap = HashMap::new(); - - for (_, config) in engine.trigger_configs() { - let address = config.address.clone().unwrap_or(default_address.clone()); - let address_expr = spin_expressions::Template::new(address)?; - let address = engine.resolve_template(&address_expr)?; - let server = server_channels.entry(address).or_default(); - let channel_expr = spin_expressions::Template::new(config.channel.as_str())?; - let channel = engine.resolve_template(&channel_expr)?; - server - .entry(channel) - .or_default() - .push(config.component.clone()); - } - Ok(Self { - engine: Arc::new(engine), - server_channels, - }) - } - - /// Run the Redis trigger indefinitely. - async fn run(self, _config: Self::RunConfig) -> Result<()> { - let tasks: Vec<_> = self - .server_channels - .clone() - .into_iter() - .map(|(server_address, channel_components)| { - let trigger = self.clone(); - tokio::spawn(async move { - trigger - .run_listener(server_address.clone(), channel_components.clone()) - .await - }) - }) - .collect(); - - // wait for the first handle to be returned and drop the rest - let (result, _, rest) = futures::future::select_all(tasks).await; - - drop(rest); - - result? - } -} - -impl RedisTrigger { - // Handle the message. - #[instrument(name = "spin_trigger_redis.handle_message", skip(self, channel_components, msg), - err(level = Level::INFO), fields(otel.name = format!("{} receive", msg.get_channel_name()), - otel.kind = "consumer", messaging.operation = "receive", messaging.system = "redis"))] - async fn handle( - &self, - address: &str, - channel_components: &ChannelComponents, - msg: redis::Msg, - ) -> Result<()> { - let channel = msg.get_channel_name(); - tracing::info!("Received message on channel {address}:{:?}", channel); - - if let Some(component_ids) = channel_components.get(channel) { - let futures = component_ids.iter().map(|id| { - tracing::trace!("Executing Redis component {id:?}"); - SpinRedisExecutor.execute(&self.engine, id, channel, msg.get_payload_bytes()) - }); - let results: Vec<_> = join_all(futures).await.into_iter().collect(); - let errors = results - .into_iter() - .filter_map(|r| r.err()) - .collect::>(); - if !errors.is_empty() { - return Err(anyhow!("{errors:#?}")); - } - } else { - tracing::debug!("No subscription found for {:?}", channel); - } - Ok(()) - } - - async fn run_listener( - &self, - address: String, - channel_components: ChannelComponents, - ) -> Result<()> { - tracing::info!("Connecting to Redis server at {}", address); - let mut client = Client::open(address.to_string())?; - let mut pubsub = client - .get_async_connection() - .await - .with_context(|| anyhow!("Redis trigger failed to connect to {}", address))? - .into_pubsub(); - - let sanitised_addr = remove_credentials(&address)?; - println!("Active Channels on {sanitised_addr}:"); - // Subscribe to channels - for (channel, component) in channel_components.iter() { - tracing::info!("Subscribing component {component:?} to channel {channel:?}"); - pubsub.subscribe(channel).await?; - println!("\t{sanitised_addr}:{channel}: [{}]", component.join(",")); - } - - let mut stream = pubsub.on_message(); - loop { - match stream.next().await { - Some(msg) => { - if let Err(err) = self.handle(&address, &channel_components, msg).await { - tracing::warn!("Error handling message: {err}"); - } - } - None => { - tracing::trace!("Empty message"); - if !client.check_connection() { - tracing::info!("No Redis connection available"); - println!("Disconnected from {address}"); - break; - } - } - }; - } - Ok(()) - } -} - -/// The Redis executor trait. -/// All Redis executors must implement this trait. -#[async_trait] -pub(crate) trait RedisExecutor: Clone + Send + Sync + 'static { - async fn execute( - &self, - engine: &TriggerAppEngine, - component_id: &str, - channel: &str, - payload: &[u8], - ) -> Result<()>; -} - -#[cfg(test)] -mod tests; diff --git a/crates/trigger-redis/src/spin.rs b/crates/trigger-redis/src/spin.rs deleted file mode 100644 index 290265a21..000000000 --- a/crates/trigger-redis/src/spin.rs +++ /dev/null @@ -1,65 +0,0 @@ -use anyhow::{anyhow, Result}; -use async_trait::async_trait; -use spin_core::Instance; -use spin_trigger::TriggerAppEngine; -use spin_world::v1::redis_types::{Error, Payload}; -use tracing::{instrument, Level}; - -use crate::{RedisExecutor, RedisTrigger, Store}; - -#[derive(Clone)] -pub struct SpinRedisExecutor; - -#[async_trait] -impl RedisExecutor for SpinRedisExecutor { - #[instrument(name = "spin_trigger_redis.execute_wasm", skip(self, engine, payload), err(level = Level::INFO), fields(otel.name = format!("execute_wasm_component {}", component_id)))] - async fn execute( - &self, - engine: &TriggerAppEngine, - component_id: &str, - channel: &str, - payload: &[u8], - ) -> Result<()> { - tracing::trace!("Executing request using the Spin executor for component {component_id}"); - - spin_telemetry::metrics::monotonic_counter!( - spin.request_count = 1, - trigger_type = "redis", - app_id = engine.app_name, - component_id = component_id - ); - - let (instance, store) = engine.prepare_instance(component_id).await?; - - match Self::execute_impl(store, instance, channel, payload.to_vec()).await { - Ok(()) => { - tracing::trace!("Request finished OK"); - Ok(()) - } - Err(e) => { - tracing::trace!("Request finished with error from {component_id}: {e}"); - Err(anyhow!("Error from {component_id}: {e}")) - } - } - } -} - -impl SpinRedisExecutor { - pub async fn execute_impl( - mut store: Store, - instance: Instance, - _channel: &str, - payload: Vec, - ) -> Result<()> { - let func = instance - .exports(&mut store) - .instance("fermyon:spin/inbound-redis") - .ok_or_else(|| anyhow!("no fermyon:spin/inbound-redis instance found"))? - .typed_func::<(Payload,), (Result<(), Error>,)>("handle-message")?; - - match func.call_async(store, (payload,)).await? { - (Ok(()) | Err(Error::Success),) => Ok(()), - _ => Err(anyhow!("`handle-message` returned an error")), - } - } -} diff --git a/crates/trigger-redis/src/tests.rs b/crates/trigger-redis/src/tests.rs deleted file mode 100644 index 6e0bb91d8..000000000 --- a/crates/trigger-redis/src/tests.rs +++ /dev/null @@ -1,26 +0,0 @@ -use super::*; -use anyhow::Result; -use redis::{Msg, Value}; -use spin_testing::{tokio, RedisTestConfig}; - -fn create_trigger_event(channel: &str, payload: &str) -> redis::Msg { - Msg::from_value(&redis::Value::Bulk(vec![ - Value::Data("message".into()), - Value::Data(channel.into()), - Value::Data(payload.into()), - ])) - .unwrap() -} - -#[tokio::test] -async fn test_pubsub() -> Result<()> { - let trigger: RedisTrigger = RedisTestConfig::default() - .test_program("redis-rust.wasm") - .build_trigger("messages") - .await; - let test = HashMap::new(); - let msg = create_trigger_event("messages", "hello"); - trigger.handle("", &test, msg).await?; - - Ok(()) -} diff --git a/crates/trigger-redis/tests/rust/.cargo/.config b/crates/trigger-redis/tests/rust/.cargo/.config deleted file mode 100644 index 30c83a790..000000000 --- a/crates/trigger-redis/tests/rust/.cargo/.config +++ /dev/null @@ -1,2 +0,0 @@ -[build] - target = "wasm32-wasi" diff --git a/crates/trigger-redis/tests/rust/Cargo.lock b/crates/trigger-redis/tests/rust/Cargo.lock deleted file mode 100644 index dd62f57f1..000000000 --- a/crates/trigger-redis/tests/rust/Cargo.lock +++ /dev/null @@ -1,303 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "anyhow" -version = "1.0.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" - -[[package]] -name = "bitflags" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "hashbrown" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "id-arena" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" - -[[package]] -name = "indexmap" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" -dependencies = [ - "equivalent", - "hashbrown", - "serde", -] - -[[package]] -name = "itoa" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" - -[[package]] -name = "leb128" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" - -[[package]] -name = "log" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" - -[[package]] -name = "proc-macro2" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rust" -version = "0.1.0" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "ryu" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" - -[[package]] -name = "semver" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" - -[[package]] -name = "serde" -version = "1.0.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "smallvec" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" - -[[package]] -name = "spdx" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b19b32ed6d899ab23174302ff105c1577e45a06b08d4fe0a9dd13ce804bbbf71" -dependencies = [ - "smallvec", -] - -[[package]] -name = "syn" -version = "2.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode-xid" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" - -[[package]] -name = "wasm-encoder" -version = "0.35.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca90ba1b5b0a70d3d49473c5579951f3bddc78d47b59256d2f9d4922b150aca" -dependencies = [ - "leb128", -] - -[[package]] -name = "wasm-metadata" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14abc161bfda5b519aa229758b68f2a52b45a12b993808665c857d1a9a00223c" -dependencies = [ - "anyhow", - "indexmap", - "serde", - "serde_derive", - "serde_json", - "spdx", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.115.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e06c0641a4add879ba71ccb3a1e4278fd546f76f1eafb21d8f7b07733b547cd5" -dependencies = [ - "indexmap", - "semver", -] - -[[package]] -name = "wit-bindgen" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d92ce0ca6b6074059413a9581a637550c3a740581c854f9847ec293c8aed71" -dependencies = [ - "bitflags", - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "565b945ae074886071eccf9cdaf8ccd7b959c2b0d624095bea5fe62003e8b3e0" -dependencies = [ - "anyhow", - "wit-component", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5695ff4e41873ed9ce56d2787e6b5772bdad9e70e2c1d2d160621d1762257f4f" -dependencies = [ - "anyhow", - "heck", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91835ea4231da1fe7971679d505ba14be7826e192b6357f08465866ef482e08" -dependencies = [ - "anyhow", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", - "wit-component", -] - -[[package]] -name = "wit-component" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e87488b57a08e2cbbd076b325acbe7f8666965af174d69d5929cd373bd54547f" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ace9943d89bbf3dbbc71b966da0e7302057b311f36a4ac3d65ddfef17b52cf" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", -] diff --git a/crates/trigger-redis/tests/rust/Cargo.toml b/crates/trigger-redis/tests/rust/Cargo.toml deleted file mode 100644 index ad23592c0..000000000 --- a/crates/trigger-redis/tests/rust/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "rust" -version = "0.1.0" -edition = "2021" -authors = ["Radu Matei "] - -[lib] -crate-type = ["cdylib"] - -[dependencies] -wit-bindgen = "0.13.0" - -[workspace] diff --git a/crates/trigger-redis/tests/rust/src/lib.rs b/crates/trigger-redis/tests/rust/src/lib.rs deleted file mode 100644 index 8371614b9..000000000 --- a/crates/trigger-redis/tests/rust/src/lib.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::str::{from_utf8, Utf8Error}; - -wit_bindgen::generate!({ - world: "redis-trigger", - path: "../../../../wit/deps/spin@unversioned", - exports: { - "fermyon:spin/inbound-redis": SpinRedis, - } -}); -use exports::fermyon::spin::inbound_redis::{self, Error, Payload}; - -struct SpinRedis; - -impl inbound_redis::Guest for SpinRedis { - fn handle_message(message: Payload) -> Result<(), Error> { - println!("Message: {:?}", from_utf8(&message)); - Ok(()) - } -} - -impl From for Error { - fn from(_: Utf8Error) -> Self { - Self::Error - } -} diff --git a/crates/trigger/Cargo.toml b/crates/trigger/Cargo.toml deleted file mode 100644 index 164a265aa..000000000 --- a/crates/trigger/Cargo.toml +++ /dev/null @@ -1,68 +0,0 @@ -[package] -name = "spin-trigger" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[features] -llm = ["spin-llm-local"] -llm-metal = ["llm", "spin-llm-local/metal"] -llm-cublas = ["llm", "spin-llm-local/cublas"] -# Enables loading AOT compiled components, a potentially unsafe operation. See -# `::::enable_loading_aot_compiled_components` -# documentation for more information about the safety risks. -unsafe-aot-compilation = [] - -[dependencies] -anyhow = "1.0" -async-trait = "0.1" -clap = { version = "3.1.15", features = ["derive", "env"] } -ctrlc = { version = "3.2", features = ["termination"] } -dirs = "4" -futures = "0.3" -indexmap = "1" -ipnet = "2.9.0" -http = "1.0.0" -outbound-http = { path = "../outbound-http" } -outbound-redis = { path = "../outbound-redis" } -outbound-mqtt = { path = "../outbound-mqtt" } -outbound-pg = { path = "../outbound-pg" } -outbound-mysql = { path = "../outbound-mysql" } -rustls-pemfile = "2.1.2" -rustls-pki-types = "1.7.0" -spin-common = { path = "../common" } -spin-expressions = { path = "../expressions" } -spin-serde = { path = "../serde" } -spin-key-value = { path = "../key-value" } -spin-key-value-azure = { path = "../key-value-azure" } -spin-key-value-redis = { path = "../key-value-redis" } -spin-key-value-sqlite = { path = "../key-value-sqlite" } -spin-outbound-networking = { path = "../outbound-networking" } -spin-sqlite = { path = "../sqlite" } -spin-sqlite-inproc = { path = "../sqlite-inproc" } -spin-sqlite-libsql = { path = "../sqlite-libsql" } -spin-world = { path = "../world" } -spin-llm = { path = "../llm" } -spin-llm-local = { path = "../llm-local", optional = true } -spin-llm-remote-http = { path = "../llm-remote-http" } -spin-telemetry = { path = "../telemetry" } -sanitize-filename = "0.4" -serde = "1.0.188" -serde_json = "1.0" -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-loader = { path = "../loader" } -spin-manifest = { path = "../manifest" } -spin-variables = { path = "../variables" } -terminal = { path = "../terminal" } -tokio = { version = "1.23", features = ["fs"] } -toml = "0.5.9" -url = "2" -spin-componentize = { workspace = true } -tracing = { workspace = true } -wasmtime = { workspace = true } -wasmtime-wasi = { workspace = true } -wasmtime-wasi-http = { workspace = true } - -[dev-dependencies] -tempfile = "3.8.0" \ No newline at end of file diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs deleted file mode 100644 index ccd63d67c..000000000 --- a/crates/trigger/src/cli.rs +++ /dev/null @@ -1,304 +0,0 @@ -use std::path::PathBuf; - -use anyhow::{Context, Result}; -use clap::{Args, IntoApp, Parser}; -use serde::de::DeserializeOwned; -use spin_app::Loader; -use spin_common::{arg_parser::parse_kv, sloth}; - -use crate::network::Network; -use crate::runtime_config::llm::LLmOptions; -use crate::runtime_config::sqlite::SqlitePersistenceMessageHook; -use crate::runtime_config::SummariseRuntimeConfigHook; -use crate::stdio::StdioLoggingTriggerHooks; -use crate::{ - loader::TriggerLoader, - runtime_config::{key_value::KeyValuePersistenceMessageHook, RuntimeConfig}, - stdio::FollowComponents, -}; -use crate::{TriggerExecutor, TriggerExecutorBuilder}; - -mod launch_metadata; -pub use launch_metadata::LaunchMetadata; - -pub const APP_LOG_DIR: &str = "APP_LOG_DIR"; -pub const DISABLE_WASMTIME_CACHE: &str = "DISABLE_WASMTIME_CACHE"; -pub const FOLLOW_LOG_OPT: &str = "FOLLOW_ID"; -pub const WASMTIME_CACHE_FILE: &str = "WASMTIME_CACHE_FILE"; -pub const RUNTIME_CONFIG_FILE: &str = "RUNTIME_CONFIG_FILE"; - -// Set by `spin up` -pub const SPIN_LOCKED_URL: &str = "SPIN_LOCKED_URL"; -pub const SPIN_LOCAL_APP_DIR: &str = "SPIN_LOCAL_APP_DIR"; -pub const SPIN_WORKING_DIR: &str = "SPIN_WORKING_DIR"; - -/// A command that runs a TriggerExecutor. -#[derive(Parser, Debug)] -#[clap( - usage = "spin [COMMAND] [OPTIONS]", - next_help_heading = help_heading::() -)] -pub struct TriggerExecutorCommand -where - Executor::RunConfig: Args, -{ - /// Log directory for the stdout and stderr of components. Setting to - /// the empty string disables logging to disk. - #[clap( - name = APP_LOG_DIR, - short = 'L', - long = "log-dir", - env = "SPIN_LOG_DIR", - )] - pub log: Option, - - /// Disable Wasmtime cache. - #[clap( - name = DISABLE_WASMTIME_CACHE, - long = "disable-cache", - env = DISABLE_WASMTIME_CACHE, - conflicts_with = WASMTIME_CACHE_FILE, - takes_value = false, - )] - pub disable_cache: bool, - - /// Wasmtime cache configuration file. - #[clap( - name = WASMTIME_CACHE_FILE, - long = "cache", - env = WASMTIME_CACHE_FILE, - conflicts_with = DISABLE_WASMTIME_CACHE, - )] - pub cache: Option, - - /// Disable Wasmtime's pooling instance allocator. - #[clap(long = "disable-pooling")] - pub disable_pooling: bool, - - /// Print output to stdout/stderr only for given component(s) - #[clap( - name = FOLLOW_LOG_OPT, - long = "follow", - multiple_occurrences = true, - )] - pub follow_components: Vec, - - /// Silence all component output to stdout/stderr - #[clap( - long = "quiet", - short = 'q', - aliases = &["sh", "shush"], - conflicts_with = FOLLOW_LOG_OPT, - )] - pub silence_component_logs: bool, - - /// Set the static assets of the components in the temporary directory as writable. - #[clap(long = "allow-transient-write")] - pub allow_transient_write: bool, - - /// Configuration file for config providers and wasmtime config. - #[clap( - name = RUNTIME_CONFIG_FILE, - long = "runtime-config-file", - env = RUNTIME_CONFIG_FILE, - )] - pub runtime_config_file: Option, - - /// Set the application state directory path. This is used in the default - /// locations for logs, key value stores, etc. - /// - /// For local apps, this defaults to `.spin/` relative to the `spin.toml` file. - /// For remote apps, this has no default (unset). - /// Passing an empty value forces the value to be unset. - #[clap(long)] - pub state_dir: Option, - - #[clap(flatten)] - pub run_config: Executor::RunConfig, - - /// Set a key/value pair (key=value) in the application's - /// default store. Any existing value will be overwritten. - /// Can be used multiple times. - #[clap(long = "key-value", parse(try_from_str = parse_kv))] - key_values: Vec<(String, String)>, - - /// Run a SQLite statement such as a migration against the default database. - /// To run from a file, prefix the filename with @ e.g. spin up --sqlite @migration.sql - #[clap(long = "sqlite")] - sqlite_statements: Vec, - - #[clap(long = "help-args-only", hide = true)] - pub help_args_only: bool, - - #[clap(long = "launch-metadata-only", hide = true)] - pub launch_metadata_only: bool, -} - -/// An empty implementation of clap::Args to be used as TriggerExecutor::RunConfig -/// for executors that do not need additional CLI args. -#[derive(Args)] -pub struct NoArgs; - -impl TriggerExecutorCommand -where - Executor::RunConfig: Args, - Executor::TriggerConfig: DeserializeOwned, -{ - /// Create a new TriggerExecutorBuilder from this TriggerExecutorCommand. - pub async fn run(self) -> Result<()> { - if self.help_args_only { - Self::command() - .disable_help_flag(true) - .help_template("{all-args}") - .print_long_help()?; - return Ok(()); - } - - if self.launch_metadata_only { - let lm = LaunchMetadata::infer::(); - let json = serde_json::to_string_pretty(&lm)?; - eprintln!("{json}"); - return Ok(()); - } - - // Required env vars - let working_dir = std::env::var(SPIN_WORKING_DIR).context(SPIN_WORKING_DIR)?; - let locked_url = std::env::var(SPIN_LOCKED_URL).context(SPIN_LOCKED_URL)?; - - let init_data = crate::HostComponentInitData::new( - &*self.key_values, - &*self.sqlite_statements, - LLmOptions { use_gpu: true }, - ); - - let loader = TriggerLoader::new(working_dir, self.allow_transient_write); - let executor = self.build_executor(loader, locked_url, init_data).await?; - - let run_fut = executor.run(self.run_config); - - let (abortable, abort_handle) = futures::future::abortable(run_fut); - ctrlc::set_handler(move || abort_handle.abort())?; - match abortable.await { - Ok(Ok(())) => { - tracing::info!("Trigger executor shut down: exiting"); - Ok(()) - } - Ok(Err(err)) => { - tracing::error!("Trigger executor failed"); - Err(err) - } - Err(_aborted) => { - tracing::info!("User requested shutdown: exiting"); - Ok(()) - } - } - } - - async fn build_executor( - &self, - loader: impl Loader + Send + Sync + 'static, - locked_url: String, - init_data: crate::HostComponentInitData, - ) -> Result { - let runtime_config = self.build_runtime_config()?; - - let _sloth_guard = warn_if_wasm_build_slothful(); - - let mut builder = TriggerExecutorBuilder::new(loader); - self.update_config(builder.config_mut())?; - - builder.hooks(StdioLoggingTriggerHooks::new(self.follow_components())); - builder.hooks(Network::default()); - builder.hooks(SummariseRuntimeConfigHook::new(&self.runtime_config_file)); - builder.hooks(KeyValuePersistenceMessageHook); - builder.hooks(SqlitePersistenceMessageHook); - - builder.build(locked_url, runtime_config, init_data).await - } - - fn build_runtime_config(&self) -> Result { - let local_app_dir = std::env::var_os(SPIN_LOCAL_APP_DIR); - let mut config = RuntimeConfig::new(local_app_dir.map(Into::into)); - if let Some(state_dir) = &self.state_dir { - config.set_state_dir(state_dir); - } - if let Some(log_dir) = &self.log { - config.set_log_dir(log_dir); - } - if let Some(config_file) = &self.runtime_config_file { - config.merge_config_file(config_file)?; - } - Ok(config) - } - - fn follow_components(&self) -> FollowComponents { - if self.silence_component_logs { - FollowComponents::None - } else if self.follow_components.is_empty() { - FollowComponents::All - } else { - let followed = self.follow_components.clone().into_iter().collect(); - FollowComponents::Named(followed) - } - } - - fn update_config(&self, config: &mut spin_core::Config) -> Result<()> { - // Apply --cache / --disable-cache - if !self.disable_cache { - config.enable_cache(&self.cache)?; - } - - if self.disable_pooling { - config.disable_pooling(); - } - - Ok(()) - } -} - -const SLOTH_WARNING_DELAY_MILLIS: u64 = 1250; - -fn warn_if_wasm_build_slothful() -> sloth::SlothGuard { - #[cfg(debug_assertions)] - let message = "\ - This is a debug build - preparing Wasm modules might take a few seconds\n\ - If you're experiencing long startup times please switch to the release build"; - - #[cfg(not(debug_assertions))] - let message = "Preparing Wasm modules is taking a few seconds..."; - - sloth::warn_if_slothful(SLOTH_WARNING_DELAY_MILLIS, format!("{message}\n")) -} - -fn help_heading() -> Option<&'static str> { - if E::TRIGGER_TYPE == help::HelpArgsOnlyTrigger::TRIGGER_TYPE { - Some("TRIGGER OPTIONS") - } else { - let heading = format!("{} TRIGGER OPTIONS", E::TRIGGER_TYPE.to_uppercase()); - let as_str = Box::new(heading).leak(); - Some(as_str) - } -} - -pub mod help { - use super::*; - - /// Null object to support --help-args-only in the absence of - /// a `spin.toml` file. - pub struct HelpArgsOnlyTrigger; - - #[async_trait::async_trait] - impl TriggerExecutor for HelpArgsOnlyTrigger { - const TRIGGER_TYPE: &'static str = "help-args-only"; - type RuntimeData = (); - type TriggerConfig = (); - type RunConfig = NoArgs; - type InstancePre = spin_core::InstancePre; - async fn new(_: crate::TriggerAppEngine) -> Result { - Ok(Self) - } - async fn run(self, _: Self::RunConfig) -> Result<()> { - Ok(()) - } - } -} diff --git a/crates/trigger/src/cli/launch_metadata.rs b/crates/trigger/src/cli/launch_metadata.rs deleted file mode 100644 index 0be69b282..000000000 --- a/crates/trigger/src/cli/launch_metadata.rs +++ /dev/null @@ -1,86 +0,0 @@ -use clap::{Args, CommandFactory}; -use serde::{Deserialize, Serialize}; -use std::ffi::OsString; - -use crate::{cli::TriggerExecutorCommand, TriggerExecutor}; - -/// Contains information about the trigger flags (and potentially -/// in future configuration) that a consumer (such as `spin up`) -/// can query using `--launch-metadata-only`. -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct LaunchMetadata { - all_flags: Vec, -} - -// This assumes no triggers that want to participate in multi-trigger -// use positional arguments. This is a restriction we'll have to make -// anyway: suppose triggers A and B both take one positional arg, and -// the user writes `spin up 123 456` - which value would go to which trigger? -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] -struct LaunchFlag { - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(default)] - short: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(default)] - long: Option, -} - -impl LaunchMetadata { - pub fn infer() -> Self - where - Executor::RunConfig: Args, - { - let all_flags: Vec<_> = TriggerExecutorCommand::::command() - .get_arguments() - .map(LaunchFlag::infer) - .collect(); - - LaunchMetadata { all_flags } - } - - pub fn matches<'a>(&self, groups: &[Vec<&'a OsString>]) -> Vec<&'a OsString> { - let mut matches = vec![]; - - for group in groups { - if group.is_empty() { - continue; - } - if self.is_match(group[0]) { - matches.extend(group); - } - } - - matches - } - - fn is_match(&self, arg: &OsString) -> bool { - self.all_flags.iter().any(|f| f.is_match(arg)) - } - - pub fn is_group_match(&self, group: &[&OsString]) -> bool { - if group.is_empty() { - false - } else { - self.all_flags.iter().any(|f| f.is_match(group[0])) - } - } -} - -impl LaunchFlag { - fn infer(arg: &clap::Arg) -> Self { - Self { - long: arg.get_long().map(|s| format!("--{s}")), - short: arg.get_short().map(|ch| format!("-{ch}")), - } - } - - fn is_match(&self, candidate: &OsString) -> bool { - let Some(s) = candidate.to_str() else { - return false; - }; - let candidate = Some(s.to_owned()); - - candidate == self.long || candidate == self.short - } -} diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs deleted file mode 100644 index 506221c58..000000000 --- a/crates/trigger/src/lib.rs +++ /dev/null @@ -1,483 +0,0 @@ -pub mod cli; -pub mod loader; -pub mod network; -mod runtime_config; -mod stdio; - -use std::{collections::HashMap, marker::PhantomData}; - -use anyhow::{Context, Result}; -pub use async_trait::async_trait; -use http::uri::Authority; -use runtime_config::llm::LLmOptions; -use serde::de::DeserializeOwned; - -use spin_app::{App, AppComponent, AppLoader, AppTrigger, Loader, OwnedApp, APP_NAME_KEY}; -use spin_core::{ - Config, Engine, EngineBuilder, Instance, InstancePre, OutboundWasiHttpHandler, Store, - StoreBuilder, WasiVersion, -}; - -pub use crate::runtime_config::{ParsedClientTlsOpts, RuntimeConfig}; - -#[async_trait] -pub trait TriggerExecutor: Sized + Send + Sync { - const TRIGGER_TYPE: &'static str; - type RuntimeData: OutboundWasiHttpHandler + Default + Send + Sync + 'static; - type TriggerConfig; - type RunConfig; - type InstancePre: TriggerInstancePre; - - /// Create a new trigger executor. - async fn new(engine: TriggerAppEngine) -> Result; - - /// Run the trigger executor. - async fn run(self, config: Self::RunConfig) -> Result<()>; - - /// Make changes to the ExecutionContext using the given Builder. - fn configure_engine(_builder: &mut EngineBuilder) -> Result<()> { - Ok(()) - } - - fn supported_host_requirements() -> Vec<&'static str> { - Vec::new() - } -} - -/// Helper type alias to project the `Instance` of a given `TriggerExecutor`. -pub type ExecutorInstance = <::InstancePre as TriggerInstancePre< - ::RuntimeData, - ::TriggerConfig, ->>::Instance; - -#[async_trait] -pub trait TriggerInstancePre: Sized + Send + Sync -where - T: OutboundWasiHttpHandler + Send + Sync, -{ - type Instance; - - async fn instantiate_pre( - engine: &Engine, - component: &AppComponent, - config: &C, - ) -> Result; - - async fn instantiate(&self, store: &mut Store) -> Result; -} - -#[async_trait] -impl TriggerInstancePre for InstancePre -where - T: OutboundWasiHttpHandler + Send + Sync, -{ - type Instance = Instance; - - async fn instantiate_pre( - engine: &Engine, - component: &AppComponent, - _config: &C, - ) -> Result { - let comp = component.load_component(engine).await?; - Ok(engine - .instantiate_pre(&comp) - .with_context(|| format!("Failed to instantiate component '{}'", component.id()))?) - } - - async fn instantiate(&self, store: &mut Store) -> Result { - self.instantiate_async(store).await - } -} - -pub struct TriggerExecutorBuilder { - loader: AppLoader, - config: Config, - hooks: Vec>, - disable_default_host_components: bool, - _phantom: PhantomData, -} - -impl TriggerExecutorBuilder { - /// Create a new TriggerExecutorBuilder with the given Application. - pub fn new(loader: impl Loader + Send + Sync + 'static) -> Self { - Self { - loader: AppLoader::new(loader), - config: Default::default(), - hooks: Default::default(), - disable_default_host_components: false, - _phantom: PhantomData, - } - } - - /// !!!Warning!!! Using a custom Wasmtime Config is entirely unsupported; - /// many configurations are likely to cause errors or unexpected behavior. - #[doc(hidden)] - pub fn config_mut(&mut self) -> &mut spin_core::Config { - &mut self.config - } - - pub fn hooks(&mut self, hooks: impl TriggerHooks + 'static) -> &mut Self { - self.hooks.push(Box::new(hooks)); - self - } - - pub fn disable_default_host_components(&mut self) -> &mut Self { - self.disable_default_host_components = true; - self - } - - pub async fn build( - mut self, - app_uri: String, - runtime_config: runtime_config::RuntimeConfig, - init_data: HostComponentInitData, - ) -> Result - where - Executor::TriggerConfig: DeserializeOwned, - { - let resolver_cell = std::sync::Arc::new(std::sync::OnceLock::new()); - - let engine = { - let mut builder = Engine::builder(&self.config)?; - - if !self.disable_default_host_components { - // Wasmtime 17: WASI@0.2.0 - builder.link_import(|l, _| { - wasmtime_wasi::add_to_linker_async(l)?; - wasmtime_wasi_http::proxy::add_only_http_to_linker(l) - })?; - - // Wasmtime 15: WASI@0.2.0-rc-2023-11-10 - builder.link_import(|l, _| spin_core::wasi_2023_11_10::add_to_linker(l))?; - - // Wasmtime 14: WASI@0.2.0-rc-2023-10-18 - builder.link_import(|l, _| spin_core::wasi_2023_10_18::add_to_linker(l))?; - - self.loader.add_dynamic_host_component( - &mut builder, - outbound_redis::OutboundRedisComponent { - resolver: resolver_cell.clone(), - }, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - outbound_mqtt::OutboundMqttComponent { - resolver: resolver_cell.clone(), - }, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - outbound_mysql::OutboundMysqlComponent { - resolver: resolver_cell.clone(), - }, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - outbound_pg::OutboundPgComponent { - resolver: resolver_cell.clone(), - }, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - runtime_config::llm::build_component(&runtime_config, init_data.llm.use_gpu) - .await, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - runtime_config::key_value::build_key_value_component( - &runtime_config, - &init_data.kv, - ) - .await?, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - runtime_config::sqlite::build_component(&runtime_config, &init_data.sqlite) - .await?, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - outbound_http::OutboundHttpComponent { - resolver: resolver_cell.clone(), - }, - )?; - self.loader.add_dynamic_host_component( - &mut builder, - spin_variables::VariablesHostComponent::new( - runtime_config.variables_providers()?, - ), - )?; - } - - Executor::configure_engine(&mut builder)?; - builder.build() - }; - - let app = self.loader.load_owned_app(app_uri).await?; - - if let Err(unmet) = app - .borrowed() - .ensure_needs_only(&Executor::supported_host_requirements()) - { - anyhow::bail!("This application requires the following features that are not available in this version of the '{}' trigger: {unmet}", Executor::TRIGGER_TYPE); - } - - let app_name = app.borrowed().require_metadata(APP_NAME_KEY)?; - - let resolver = - spin_variables::make_resolver(app.borrowed(), runtime_config.variables_providers()?)?; - let prepared_resolver = std::sync::Arc::new(resolver.prepare().await?); - resolver_cell - .set(prepared_resolver.clone()) - .map_err(|_| anyhow::anyhow!("resolver cell was already set!"))?; - - self.hooks - .iter_mut() - .try_for_each(|h| h.app_loaded(app.borrowed(), &runtime_config, &prepared_resolver))?; - - // Run trigger executor - Executor::new( - TriggerAppEngine::new( - engine, - app_name, - app, - self.hooks, - &prepared_resolver, - runtime_config.client_tls_opts()?, - ) - .await?, - ) - .await - } -} - -/// Initialization data for host components. -#[derive(Default)] // TODO: the implementation of Default is only for tests - would like to get rid of -pub struct HostComponentInitData { - kv: Vec<(String, String)>, - sqlite: Vec, - llm: LLmOptions, -} - -impl HostComponentInitData { - /// Create an instance of `HostComponentInitData`. `key_value_init_values` - /// will be added to the default key-value store; `sqlite_init_statements` - /// will be run against the default SQLite database. - pub fn new( - key_value_init_values: impl Into>, - sqlite_init_statements: impl Into>, - llm: LLmOptions, - ) -> Self { - Self { - kv: key_value_init_values.into(), - sqlite: sqlite_init_statements.into(), - llm, - } - } -} - -/// Execution context for a TriggerExecutor executing a particular App. -pub struct TriggerAppEngine { - /// Engine to be used with this executor. - pub engine: Engine, - /// Name of the app for e.g. logging. - pub app_name: String, - // An owned wrapper of the App. - app: OwnedApp, - // Trigger hooks - hooks: Vec>, - // Trigger configs for this trigger type, with order matching `app.triggers_with_type(Executor::TRIGGER_TYPE)` - trigger_configs: Vec, - // Map of {Component ID -> InstancePre} for each component. - component_instance_pres: HashMap, - // Resolver for value template expressions - resolver: std::sync::Arc, - // Map of { Component ID -> Map of { Authority -> ParsedClientTlsOpts} } - client_tls_opts: HashMap>, -} - -impl TriggerAppEngine { - /// Returns a new TriggerAppEngine. May return an error if trigger config validation or - /// component pre-instantiation fails. - pub async fn new( - engine: Engine, - app_name: String, - app: OwnedApp, - hooks: Vec>, - resolver: &std::sync::Arc, - client_tls_opts: HashMap>, - ) -> Result - where - ::TriggerConfig: DeserializeOwned, - { - let trigger_configs = app - .borrowed() - .triggers_with_type(Executor::TRIGGER_TYPE) - .map(|trigger| { - Ok(( - trigger.component()?.id().to_owned(), - trigger.typed_config().with_context(|| { - format!("invalid trigger configuration for {:?}", trigger.id()) - })?, - )) - }) - .collect::>>()?; - - let mut component_instance_pres = HashMap::default(); - for component in app.borrowed().components() { - let id = component.id(); - // There is an issue here for triggers that consider the trigger config during - // preinstantiation. We defer this for now because the only case is the HTTP - // `executor` field and that should not differ from trigger to trigger. - let trigger_config = trigger_configs - .iter() - .find(|(c, _)| c == id) - .map(|(_, cfg)| cfg); - if let Some(config) = trigger_config { - component_instance_pres.insert( - id.to_owned(), - Executor::InstancePre::instantiate_pre(&engine, &component, config) - .await - .with_context(|| format!("Failed to instantiate component '{id}'"))?, - ); - } else { - tracing::warn!( - "component '{id}' is not used by any triggers in app '{app_name}'", - id = id, - app_name = app_name - ) - } - } - - Ok(Self { - engine, - app_name, - app, - hooks, - trigger_configs: trigger_configs.into_iter().map(|(_, v)| v).collect(), - component_instance_pres, - resolver: resolver.clone(), - client_tls_opts, - }) - } - - /// Returns a reference to the App. - pub fn app(&self) -> &App { - self.app.borrowed() - } - - pub fn trigger_metadata(&self) -> spin_app::Result> { - self.app().get_trigger_metadata(Executor::TRIGGER_TYPE) - } - - /// Returns AppTriggers and typed TriggerConfigs for this executor type. - pub fn trigger_configs(&self) -> impl Iterator { - self.app() - .triggers_with_type(Executor::TRIGGER_TYPE) - .zip(&self.trigger_configs) - } - - /// Returns a new StoreBuilder for the given component ID. - pub fn store_builder( - &self, - component_id: &str, - wasi_version: WasiVersion, - ) -> Result { - let mut builder = self.engine.store_builder(wasi_version); - let component = self.get_component(component_id)?; - self.hooks - .iter() - .try_for_each(|h| h.component_store_builder(&component, &mut builder))?; - Ok(builder) - } - - /// Returns a new Store and Instance for the given component ID. - pub async fn prepare_instance( - &self, - component_id: &str, - ) -> Result<(ExecutorInstance, Store)> { - let store_builder = self.store_builder(component_id, WasiVersion::Preview2)?; - self.prepare_instance_with_store(component_id, store_builder) - .await - } - - /// Returns a new Store and Instance for the given component ID and StoreBuilder. - pub async fn prepare_instance_with_store( - &self, - component_id: &str, - mut store_builder: StoreBuilder, - ) -> Result<(ExecutorInstance, Store)> { - let component = self.get_component(component_id)?; - - // Build Store - component.apply_store_config(&mut store_builder).await?; - let mut store = store_builder.build()?; - - // Instantiate - let pre = self - .component_instance_pres - .get(component_id) - .expect("component_instance_pres missing valid component_id"); - - let instance = pre.instantiate(&mut store).await.with_context(|| { - format!( - "app {:?} component {:?} instantiation failed", - self.app_name, component_id - ) - })?; - - Ok((instance, store)) - } - - pub fn get_component(&self, component_id: &str) -> Result { - self.app().get_component(component_id).with_context(|| { - format!( - "app {:?} has no component {:?}", - self.app_name, component_id - ) - }) - } - - pub fn get_client_tls_opts( - &self, - component_id: &str, - ) -> Option> { - self.client_tls_opts.get(component_id).cloned() - } - - pub fn resolve_template( - &self, - template: &spin_expressions::Template, - ) -> Result { - self.resolver.resolve_template(template) - } -} - -/// TriggerHooks allows a Spin environment to hook into a TriggerAppEngine's -/// configuration and execution processes. -pub trait TriggerHooks: Send + Sync { - #![allow(unused_variables)] - - /// Called once, immediately after an App is loaded. - fn app_loaded( - &mut self, - app: &App, - runtime_config: &RuntimeConfig, - resolver: &std::sync::Arc, - ) -> Result<()> { - Ok(()) - } - - /// Called while an AppComponent is being prepared for execution. - /// Implementations may update the given StoreBuilder to change the - /// environment of the instance to be executed. - fn component_store_builder( - &self, - component: &AppComponent, - store_builder: &mut StoreBuilder, - ) -> Result<()> { - Ok(()) - } -} - -impl TriggerHooks for () {} diff --git a/crates/trigger/src/loader.rs b/crates/trigger/src/loader.rs deleted file mode 100644 index e46fab104..000000000 --- a/crates/trigger/src/loader.rs +++ /dev/null @@ -1,256 +0,0 @@ -use std::path::{Path, PathBuf}; - -use anyhow::{ensure, Context, Result}; -use async_trait::async_trait; -use spin_app::{ - locked::{LockedApp, LockedComponentSource}, - AppComponent, Loader, -}; -use spin_componentize::bugs::WasiLibc377Bug; -use spin_core::StoreBuilder; -use tokio::fs; - -use spin_common::{ui::quoted_path, url::parse_file_url}; - -/// Compilation status of all components of a Spin application -pub enum AotCompilationStatus { - /// No components are ahead of time (AOT) compiled. - Disabled, - #[cfg(feature = "unsafe-aot-compilation")] - /// All components are componentized and ahead of time (AOT) compiled to cwasm. - Enabled, -} - -/// Loader for the Spin runtime -pub struct TriggerLoader { - /// Working directory where files for mounting exist. - working_dir: PathBuf, - /// Set the static assets of the components in the temporary directory as writable. - allow_transient_write: bool, - /// Declares the compilation status of all components of a Spin application. - aot_compilation_status: AotCompilationStatus, -} - -impl TriggerLoader { - pub fn new(working_dir: impl Into, allow_transient_write: bool) -> Self { - Self { - working_dir: working_dir.into(), - allow_transient_write, - aot_compilation_status: AotCompilationStatus::Disabled, - } - } - - /// Updates the TriggerLoader to load AOT precompiled components - /// - /// **Warning: This feature may bypass important security guarantees of the - /// Wasmtime - // security sandbox if used incorrectly! Read this documentation - // carefully.** - /// - /// Usually, components are compiled just-in-time from portable Wasm - /// sources. This method causes components to instead be loaded - /// ahead-of-time as Wasmtime-precompiled native executable binaries. - /// Precompiled binaries must be produced with a compatible Wasmtime engine - /// using the same Wasmtime version and compiler target settings - typically - /// by a host with the same processor that will be executing them. See the - /// Wasmtime documentation for more information: - /// https://docs.rs/wasmtime/latest/wasmtime/struct.Module.html#method.deserialize - /// - /// # Safety - /// - /// This method is marked as `unsafe` because it enables potentially unsafe - /// behavior if - // used to load malformed or malicious precompiled binaries. Loading sources - // from an - /// incompatible Wasmtime engine will fail but is otherwise safe. This - /// method is safe if it can be guaranteed that `::load_component` will only ever be called with a trusted - /// `LockedComponentSource`. **Precompiled binaries must never be loaded - /// from untrusted sources.** - #[cfg(feature = "unsafe-aot-compilation")] - pub unsafe fn enable_loading_aot_compiled_components(&mut self) { - self.aot_compilation_status = AotCompilationStatus::Enabled; - } -} - -#[async_trait] -impl Loader for TriggerLoader { - async fn load_app(&self, url: &str) -> Result { - let path = parse_file_url(url)?; - let contents = std::fs::read(&path) - .with_context(|| format!("failed to read manifest at {}", quoted_path(&path)))?; - let app = - serde_json::from_slice(&contents).context("failed to parse app lock file JSON")?; - Ok(app) - } - - async fn load_component( - &self, - engine: &spin_core::wasmtime::Engine, - source: &LockedComponentSource, - ) -> Result { - let source = source - .content - .source - .as_ref() - .context("LockedComponentSource missing source field")?; - let path = parse_file_url(source)?; - match self.aot_compilation_status { - #[cfg(feature = "unsafe-aot-compilation")] - AotCompilationStatus::Enabled => { - match engine.detect_precompiled_file(&path)?{ - Some(wasmtime::Precompiled::Component) => { - unsafe { - spin_core::Component::deserialize_file(engine, &path) - .with_context(|| format!("deserializing module {}", quoted_path(&path))) - } - }, - Some(wasmtime::Precompiled::Module) => anyhow::bail!("Spin loader is configured to load only AOT compiled components but an AOT compiled module provided at {}", quoted_path(&path)), - None => { - anyhow::bail!("Spin loader is configured to load only AOT compiled components, but {} is not precompiled", quoted_path(&path)) - } - } - } - AotCompilationStatus::Disabled => { - let bytes = fs::read(&path).await.with_context(|| { - format!( - "failed to read component source from disk at path {}", - quoted_path(&path) - ) - })?; - let component = spin_componentize::componentize_if_necessary(&bytes)?; - spin_core::Component::new(engine, component.as_ref()) - .with_context(|| format!("loading module {}", quoted_path(&path))) - } - } - } - - async fn load_module( - &self, - engine: &spin_core::wasmtime::Engine, - source: &LockedComponentSource, - ) -> Result { - let source = source - .content - .source - .as_ref() - .context("LockedComponentSource missing source field")?; - let path = parse_file_url(source)?; - check_uncomponentizable_module_deprecation(&path); - spin_core::Module::from_file(engine, &path) - .with_context(|| format!("loading module {}", quoted_path(&path))) - } - - async fn mount_files( - &self, - store_builder: &mut StoreBuilder, - component: &AppComponent, - ) -> Result<()> { - for content_dir in component.files() { - let source_uri = content_dir - .content - .source - .as_deref() - .with_context(|| format!("Missing 'source' on files mount {content_dir:?}"))?; - let source_path = self.working_dir.join(parse_file_url(source_uri)?); - ensure!( - source_path.is_dir(), - "TriggerLoader only supports directory mounts; {} is not a directory", - quoted_path(&source_path), - ); - let guest_path = content_dir.path.clone(); - if self.allow_transient_write { - store_builder.read_write_preopened_dir(source_path, guest_path)?; - } else { - store_builder.read_only_preopened_dir(source_path, guest_path)?; - } - } - Ok(()) - } -} - -// Check whether the given module is (likely) susceptible to a wasi-libc bug -// that makes it unsafe to componentize. If so, print a deprecation warning; -// this will turn into a hard error in a future release. -fn check_uncomponentizable_module_deprecation(module_path: &Path) { - let module = match std::fs::read(module_path) { - Ok(module) => module, - Err(err) => { - tracing::warn!("Failed to read {module_path:?}: {err:#}"); - return; - } - }; - match WasiLibc377Bug::detect(&module) { - Ok(WasiLibc377Bug::ProbablySafe) => {} - not_safe @ Ok(WasiLibc377Bug::ProbablyUnsafe | WasiLibc377Bug::Unknown) => { - println!( - "\n!!! DEPRECATION WARNING !!!\n\ - The Wasm module at {path}\n\ - {verbs} have been compiled with wasi-sdk version <19 and is likely to\n\ - contain a critical memory safety bug. Spin has deprecated execution of these\n\ - modules; they will stop working in a future release.\n\ - For more information, see: https://github.com/fermyon/spin/issues/2552\n", - path = module_path.display(), - verbs = if not_safe.unwrap() == WasiLibc377Bug::ProbablyUnsafe { - "appears to" - } else { - "may" - } - ); - } - Err(err) => { - tracing::warn!("Failed to apply wasi-libc bug heuristic on {module_path:?}: {err:#}"); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use spin_app::locked::ContentRef; - use std::io::Write; - use tempfile::NamedTempFile; - - fn precompiled_component(file: &mut NamedTempFile) -> LockedComponentSource { - let wasmtime_engine = wasmtime::Engine::default(); - let component = wasmtime::component::Component::new(&wasmtime_engine, "(component)") - .unwrap() - .serialize() - .unwrap(); - let file_uri = format!("file://{}", file.path().to_str().unwrap()); - file.write_all(&component).unwrap(); - LockedComponentSource { - content: ContentRef { - source: Some(file_uri), - ..Default::default() - }, - content_type: "application/wasm".to_string(), - } - } - - #[cfg(feature = "unsafe-aot-compilation")] - #[tokio::test] - async fn load_component_succeeds_for_precompiled_component() { - let mut file = NamedTempFile::new().unwrap(); - let source = precompiled_component(&mut file); - let mut loader = super::TriggerLoader::new("/unreferenced", false); - unsafe { - loader.enable_loading_aot_compiled_components(); - } - loader - .load_component(&spin_core::wasmtime::Engine::default(), &source) - .await - .unwrap(); - } - - #[tokio::test] - async fn load_component_fails_for_precompiled_component() { - let mut file = NamedTempFile::new().unwrap(); - let source = precompiled_component(&mut file); - let loader = super::TriggerLoader::new("/unreferenced", false); - let result = loader - .load_component(&spin_core::wasmtime::Engine::default(), &source) - .await; - assert!(result.is_err()); - } -} diff --git a/crates/trigger/src/network.rs b/crates/trigger/src/network.rs deleted file mode 100644 index ce7af9988..000000000 --- a/crates/trigger/src/network.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::sync::Arc; - -use crate::TriggerHooks; - -#[derive(Default)] -pub struct Network { - resolver: Arc, -} - -impl TriggerHooks for Network { - fn app_loaded( - &mut self, - _app: &spin_app::App, - _runtime_config: &crate::RuntimeConfig, - resolver: &Arc, - ) -> anyhow::Result<()> { - self.resolver = resolver.clone(); - Ok(()) - } - - fn component_store_builder( - &self, - component: &spin_app::AppComponent, - store_builder: &mut spin_core::StoreBuilder, - ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(spin_outbound_networking::ALLOWED_HOSTS_KEY)? - .unwrap_or_default(); - let allowed_hosts = - spin_outbound_networking::AllowedHostsConfig::parse(&hosts, &self.resolver)?; - match allowed_hosts { - spin_outbound_networking::AllowedHostsConfig::All => store_builder.inherit_network(), - spin_outbound_networking::AllowedHostsConfig::SpecificHosts(configs) => { - for config in configs { - if config.scheme().allows_any() { - match config.host() { - spin_outbound_networking::HostConfig::Any => { - store_builder.inherit_network() - } - spin_outbound_networking::HostConfig::AnySubdomain(_) => continue, - spin_outbound_networking::HostConfig::ToSelf => {} - spin_outbound_networking::HostConfig::List(hosts) => { - for host in hosts { - let Ok(ip_net) = - // Parse the host as an `IpNet` cidr block and if it fails - // then try parsing again with `/32` appended to the end. - host.parse().or_else(|_| format!("{host}/32").parse()) - else { - continue; - }; - add_ip_net(store_builder, ip_net, config.port()); - } - } - spin_outbound_networking::HostConfig::Cidr(ip_net) => { - add_ip_net(store_builder, *ip_net, config.port()) - } - } - } - } - } - } - Ok(()) - } -} - -fn add_ip_net( - store_builder: &mut spin_core::StoreBuilder, - ip_net: ipnet::IpNet, - port: &spin_outbound_networking::PortConfig, -) { - match port { - spin_outbound_networking::PortConfig::Any => { - store_builder.insert_ip_net_port_range(ip_net, 0, None); - } - spin_outbound_networking::PortConfig::List(ports) => { - for port in ports { - match port { - spin_outbound_networking::IndividualPortConfig::Port(p) => { - store_builder.insert_ip_net_port_range(ip_net, *p, p.checked_add(1)); - } - spin_outbound_networking::IndividualPortConfig::Range(r) => { - store_builder.insert_ip_net_port_range(ip_net, r.start, Some(r.end)) - } - } - } - } - } -} diff --git a/crates/trigger/src/runtime_config.rs b/crates/trigger/src/runtime_config.rs deleted file mode 100644 index 4d22c48bc..000000000 --- a/crates/trigger/src/runtime_config.rs +++ /dev/null @@ -1,840 +0,0 @@ -pub mod client_tls; -pub mod key_value; -pub mod llm; -pub mod sqlite; -pub mod variables_provider; - -use std::{ - collections::HashMap, - fs, - path::{Path, PathBuf}, - sync::Arc, -}; - -use anyhow::{Context, Result}; -use http::uri::Authority; -use serde::Deserialize; -use spin_common::ui::quoted_path; -use spin_sqlite::Connection; - -use crate::TriggerHooks; - -use self::{ - client_tls::{load_certs, load_key, ClientTlsOpts}, - key_value::{KeyValueStore, KeyValueStoreOpts}, - llm::LlmComputeOpts, - sqlite::SqliteDatabaseOpts, - variables_provider::{VariablesProvider, VariablesProviderOpts}, -}; - -pub const DEFAULT_STATE_DIR: &str = ".spin"; -const DEFAULT_LOGS_DIR: &str = "logs"; -/// RuntimeConfig allows multiple sources of runtime configuration to be -/// queried uniformly. -#[derive(Debug, Default)] -pub struct RuntimeConfig { - local_app_dir: Option, - files: Vec, - overrides: RuntimeConfigOpts, -} - -impl RuntimeConfig { - // Gives more consistent conditional branches - #![allow(clippy::manual_map)] - - pub fn new(local_app_dir: Option) -> Self { - Self { - local_app_dir, - ..Default::default() - } - } - - /// Load a runtime config file from the given path. Options specified in a - /// later-loaded file take precedence over any earlier-loaded files. - pub fn merge_config_file(&mut self, path: impl Into) -> Result<()> { - let path = path.into(); - let mut opts = RuntimeConfigOpts::parse_file(&path)?; - opts.file_path = Some(path); - self.files.push(opts); - Ok(()) - } - - /// Return a Vec of configured [`VariablesProvider`]s. - pub fn variables_providers(&self) -> Result> { - let default_provider = - VariablesProviderOpts::default_provider_opts(self).build_provider()?; - let mut providers: Vec = vec![default_provider]; - for opts in self.opts_layers() { - for var_provider in &opts.variables_providers { - let provider = var_provider.build_provider()?; - providers.push(provider); - } - } - Ok(providers) - } - - /// Return an iterator of named configured [`KeyValueStore`]s. - pub fn key_value_stores(&self) -> Result> { - let mut stores = HashMap::new(); - // Insert explicitly-configured stores - for opts in self.opts_layers() { - for (name, store) in &opts.key_value_stores { - if !stores.contains_key(name) { - let store = store.build_store(opts)?; - stores.insert(name.to_owned(), store); - } - } - } - // Upsert default store - if !stores.contains_key("default") { - let store = KeyValueStoreOpts::default_store_opts(self) - .build_store(&RuntimeConfigOpts::default())?; - stores.insert("default".into(), store); - } - Ok(stores.into_iter()) - } - - // Return the "default" key value store config. - fn default_key_value_opts(&self) -> KeyValueStoreOpts { - self.opts_layers() - .find_map(|opts| opts.key_value_stores.get("default")) - .cloned() - .unwrap_or_else(|| KeyValueStoreOpts::default_store_opts(self)) - } - - // Return the "default" key value store config. - fn default_sqlite_opts(&self) -> SqliteDatabaseOpts { - self.opts_layers() - .find_map(|opts| opts.sqlite_databases.get("default")) - .cloned() - .unwrap_or_else(|| SqliteDatabaseOpts::default(self)) - } - - /// Return an iterator of named configured [`SqliteDatabase`]s. - pub async fn sqlite_databases( - &self, - ) -> Result)>> { - let mut databases = HashMap::new(); - // Insert explicitly-configured databases - for opts in self.opts_layers() { - for (name, database) in &opts.sqlite_databases { - if !databases.contains_key(name) { - let store = database.build(opts).await?; - databases.insert(name.to_owned(), store); - } - } - } - // Upsert default store - if !databases.contains_key("default") { - let store = SqliteDatabaseOpts::default(self) - .build(&RuntimeConfigOpts::default()) - .await?; - databases.insert("default".into(), store); - } - Ok(databases.into_iter()) - } - - /// Set the state dir, overriding any other runtime config source. - pub fn set_state_dir(&mut self, state_dir: impl Into) { - self.overrides.state_dir = Some(state_dir.into()); - } - - /// Return the state dir if set. - pub fn state_dir(&self) -> Option { - if let Some(path_str) = self.find_opt(|opts| &opts.state_dir) { - if path_str.is_empty() { - None // An empty string forces the state dir to be unset - } else { - Some(path_str.into()) - } - } else if let Some(app_dir) = &self.local_app_dir { - // If we're running a local app, return the default state dir - Some(app_dir.join(DEFAULT_STATE_DIR)) - } else { - None - } - } - - /// Set the log dir, overriding any other runtime config source. - pub fn set_log_dir(&mut self, log_dir: impl Into) { - self.overrides.log_dir = Some(log_dir.into()); - } - - /// Return the log dir if set. - pub fn log_dir(&self) -> Option { - if let Some(path) = self.find_opt(|opts| &opts.log_dir) { - if path.as_os_str().is_empty() { - // If the log dir is explicitly set to "", disable logging - None - } else { - // If there is an explicit log dir set, return it - Some(path.into()) - } - } else if let Some(state_dir) = self.state_dir() { - // If the state dir is set, build the default path - Some(state_dir.join(DEFAULT_LOGS_DIR)) - } else { - None - } - } - - pub fn llm_compute(&self) -> &LlmComputeOpts { - if let Some(compute) = self.find_opt(|opts| &opts.llm_compute) { - compute - } else { - &LlmComputeOpts::Spin - } - } - - // returns the client tls options in form of nested - // HashMap of { Component ID -> HashMap of { Host -> ParsedClientTlsOpts} } - pub fn client_tls_opts( - &self, - ) -> Result>> { - let mut components_map: HashMap> = - HashMap::new(); - - // if available, use the existing client tls opts value for a given component-id and host-authority - // to ensure first-one wins incase of duplicate options - fn use_existing_if_available( - existing_opts: Option<&HashMap>, - host: Authority, - newopts: ParsedClientTlsOpts, - ) -> (Authority, ParsedClientTlsOpts) { - match existing_opts { - None => (host, newopts), - Some(opts) => match opts.get(&host) { - Some(existing_opts_for_component_and_host) => { - (host, existing_opts_for_component_and_host.to_owned()) - } - None => (host, newopts), - }, - } - } - - for opt_layer in self.opts_layers() { - for opts in &opt_layer.client_tls_opts { - let parsed = parse_client_tls_opts(opts).context("parsing client tls options")?; - for component_id in &opts.component_ids { - let existing_opts_for_component = components_map.get(component_id.as_ref()); - #[allow(clippy::mutable_key_type)] - let hostmap = parsed - .hosts - .clone() - .into_iter() - .map(|host| { - use_existing_if_available( - existing_opts_for_component, - host, - parsed.clone(), - ) - }) - .collect::>(); - components_map.insert(component_id.to_string(), hostmap); - } - } - } - - Ok(components_map) - } - - /// Returns an iterator of RuntimeConfigOpts in order of decreasing precedence - fn opts_layers(&self) -> impl Iterator { - std::iter::once(&self.overrides).chain(self.files.iter().rev()) - } - - /// Returns the highest precedence RuntimeConfigOpts Option that is set - fn find_opt(&self, mut f: impl FnMut(&RuntimeConfigOpts) -> &Option) -> Option<&T> { - self.opts_layers().find_map(|opts| f(opts).as_ref()) - } -} - -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RuntimeConfigOpts { - #[serde(default)] - pub state_dir: Option, - - #[serde(default)] - pub log_dir: Option, - - #[serde(default)] - pub llm_compute: Option, - - #[serde(rename = "variables_provider", alias = "config_provider", default)] - pub variables_providers: Vec, - - #[serde(rename = "key_value_store", default)] - pub key_value_stores: HashMap, - - #[serde(rename = "sqlite_database", default)] - pub sqlite_databases: HashMap, - - #[serde(skip)] - pub file_path: Option, - - #[serde(rename = "client_tls", default)] - pub client_tls_opts: Vec, -} - -impl RuntimeConfigOpts { - fn parse_file(path: &Path) -> Result { - let contents = fs::read_to_string(path) - .with_context(|| format!("Failed to read runtime config file {}", quoted_path(path)))?; - let ext = path.extension().unwrap_or_default(); - let is_json = ext != "toml" && (ext == "json" || contents.trim_start().starts_with('{')); - if is_json { - serde_json::from_str(&contents).with_context(|| { - format!( - "Failed to parse runtime config JSON file {}", - quoted_path(path) - ) - }) - } else { - toml::from_str(&contents).with_context(|| { - format!( - "Failed to parse runtime config TOML file {}", - quoted_path(path) - ) - }) - } - } -} - -fn resolve_config_path(path: &Path, config_opts: &RuntimeConfigOpts) -> Result { - if path.is_absolute() { - return Ok(path.to_owned()); - } - let base_path = match &config_opts.file_path { - Some(file_path) => file_path - .parent() - .with_context(|| { - format!( - "failed to get parent of runtime config file path {}", - quoted_path(file_path) - ) - })? - .to_owned(), - None => std::env::current_dir().context("failed to get current directory")?, - }; - Ok(base_path.join(path)) -} - -pub(crate) struct SummariseRuntimeConfigHook { - runtime_config_file: Option, -} - -impl SummariseRuntimeConfigHook { - pub(crate) fn new(runtime_config_file: &Option) -> Self { - Self { - runtime_config_file: runtime_config_file.clone(), - } - } -} - -impl TriggerHooks for SummariseRuntimeConfigHook { - fn app_loaded( - &mut self, - _app: &spin_app::App, - runtime_config: &RuntimeConfig, - _resolver: &Arc, - ) -> anyhow::Result<()> { - if let Some(path) = &self.runtime_config_file { - let mut opts = vec![]; - for opt in runtime_config.opts_layers() { - for (id, opt) in &opt.key_value_stores { - opts.push(Self::summarise_kv(id, opt)); - } - for (id, opt) in &opt.sqlite_databases { - opts.push(Self::summarise_sqlite(id, opt)); - } - if let Some(opt) = &opt.llm_compute { - opts.push(Self::summarise_llm(opt)); - } - } - if !opts.is_empty() { - let opts_text = opts.join(", "); - println!( - "Using {opts_text} runtime config from {}", - quoted_path(path) - ); - } - } - Ok(()) - } -} - -impl SummariseRuntimeConfigHook { - fn summarise_kv(id: &str, opt: &KeyValueStoreOpts) -> String { - let source = match opt { - KeyValueStoreOpts::Spin(_) => "spin", - KeyValueStoreOpts::Redis(_) => "redis", - KeyValueStoreOpts::AzureCosmos(_) => "cosmos", - }; - format!("[key_value_store.{id}: {}]", source) - } - - fn summarise_sqlite(id: &str, opt: &SqliteDatabaseOpts) -> String { - let source = match opt { - SqliteDatabaseOpts::Spin(_) => "spin", - SqliteDatabaseOpts::Libsql(_) => "libsql", - }; - format!("[sqlite_database.{id}: {}]", source) - } - - fn summarise_llm(opt: &LlmComputeOpts) -> String { - let source = match opt { - LlmComputeOpts::Spin => "spin", - LlmComputeOpts::RemoteHttp(_) => "remote-http", - }; - format!("[llm_compute: {}]", source) - } -} - -#[cfg(test)] -mod tests { - use std::io::Write; - - use tempfile::NamedTempFile; - use toml::toml; - - use super::*; - - #[test] - fn defaults_without_local_app_dir() -> Result<()> { - let config = RuntimeConfig::new(None); - - assert_eq!(config.state_dir(), None); - assert_eq!(config.log_dir(), None); - assert_eq!(default_spin_store_path(&config), None); - - Ok(()) - } - - #[test] - fn defaults_with_local_app_dir() -> Result<()> { - let app_dir = tempfile::tempdir()?; - let config = RuntimeConfig::new(Some(app_dir.path().into())); - - let state_dir = config.state_dir().unwrap(); - assert!(state_dir.starts_with(&app_dir)); - - let log_dir = config.log_dir().unwrap(); - assert!(log_dir.starts_with(&state_dir)); - - let default_db_path = default_spin_store_path(&config).unwrap(); - assert!(default_db_path.starts_with(&state_dir)); - - Ok(()) - } - - #[test] - fn state_dir_force_unset() -> Result<()> { - let app_dir = tempfile::tempdir()?; - let mut config = RuntimeConfig::new(Some(app_dir.path().into())); - assert!(config.state_dir().is_some()); - - config.set_state_dir(""); - assert!(config.state_dir().is_none()); - - Ok(()) - } - - #[test] - fn opts_layers_precedence() -> Result<()> { - let mut config = RuntimeConfig::new(None); - - merge_config_toml( - &mut config, - toml! { - state_dir = "file-state-dir" - log_dir = "file-log-dir" - }, - ); - - let state_dir = config.state_dir().unwrap(); - assert_eq!(state_dir.as_os_str(), "file-state-dir"); - - let log_dir = config.log_dir().unwrap(); - assert_eq!(log_dir.as_os_str(), "file-log-dir"); - - config.set_state_dir("override-state-dir"); - config.set_log_dir("override-log-dir"); - - let state_dir = config.state_dir().unwrap(); - assert_eq!(state_dir.as_os_str(), "override-state-dir"); - - let log_dir = config.log_dir().unwrap(); - assert_eq!(log_dir.as_os_str(), "override-log-dir"); - - Ok(()) - } - - #[test] - fn deprecated_config_provider_in_runtime_config_file() -> Result<()> { - let mut config = RuntimeConfig::new(None); - - // One default provider - assert_eq!(config.variables_providers()?.len(), 1); - - merge_config_toml( - &mut config, - toml! { - [[config_provider]] - type = "vault" - url = "http://vault" - token = "secret" - mount = "root" - }, - ); - assert_eq!(config.variables_providers()?.len(), 2); - - Ok(()) - } - - #[test] - fn variables_providers_from_file() -> Result<()> { - let mut config = RuntimeConfig::new(None); - - // One default provider - assert_eq!(config.variables_providers()?.len(), 1); - - merge_config_toml( - &mut config, - toml! { - [[variables_provider]] - type = "vault" - url = "http://vault" - token = "secret" - mount = "root" - }, - ); - assert_eq!(config.variables_providers()?.len(), 2); - - Ok(()) - } - - #[test] - fn key_value_stores_from_file() -> Result<()> { - let mut config = RuntimeConfig::new(None); - - // One default store - assert_eq!(config.key_value_stores().unwrap().into_iter().count(), 1); - - merge_config_toml( - &mut config, - toml! { - [key_value_store.default] - type = "spin" - path = "override.db" - - [key_value_store.other] - type = "spin" - path = "other.db" - }, - ); - assert_eq!(config.key_value_stores().unwrap().into_iter().count(), 2); - - Ok(()) - } - - #[test] - fn default_redis_key_value_store_from_file() -> Result<()> { - let mut config = RuntimeConfig::new(None); - - merge_config_toml( - &mut config, - toml! { - [key_value_store.default] - type = "redis" - url = "redis://127.0.0.1/" - }, - ); - assert_eq!(config.key_value_stores().unwrap().into_iter().count(), 1); - - assert!( - matches!(config.default_key_value_opts(), KeyValueStoreOpts::Redis(_)), - "expected default Redis store", - ); - - Ok(()) - } - - fn to_component_id(inp: &str) -> spin_serde::KebabId { - spin_serde::KebabId::try_from(inp.to_string()).expect("parse component id into kebab id") - } - - #[test] - fn test_parsing_valid_hosts_in_client_tls_opts() { - let input = ClientTlsOpts { - component_ids: vec![to_component_id("component-id-foo")], - hosts: vec!["fermyon.com".to_string(), "fermyon.com:5443".to_string()], - ca_roots_file: None, - cert_chain_file: None, - private_key_file: None, - ca_webpki_roots: None, - }; - - let parsed = parse_client_tls_opts(&input); - assert!(parsed.is_ok()); - assert_eq!(parsed.unwrap().hosts.len(), 2) - } - - #[test] - fn test_parsing_empty_hosts_in_client_tls_opts() { - let input = ClientTlsOpts { - component_ids: vec![to_component_id("component-id-foo")], - hosts: vec!["".to_string(), "fermyon.com:5443".to_string()], - ca_roots_file: None, - cert_chain_file: None, - private_key_file: None, - ca_webpki_roots: None, - }; - - let parsed = parse_client_tls_opts(&input); - assert!(parsed.is_err()); - assert_eq!( - "failed to parse uri ''. error: InvalidUri(Empty)", - parsed.unwrap_err().to_string() - ) - } - - #[test] - fn test_parsing_invalid_hosts_in_client_tls_opts() { - let input = ClientTlsOpts { - component_ids: vec![to_component_id("component-id-foo")], - hosts: vec!["perc%ent:443".to_string(), "fermyon.com:5443".to_string()], - ca_roots_file: None, - cert_chain_file: None, - private_key_file: None, - ca_webpki_roots: None, - }; - - let parsed = parse_client_tls_opts(&input); - assert!(parsed.is_err()); - assert_eq!( - "failed to parse uri 'perc%ent:443'. error: InvalidUri(InvalidAuthority)", - parsed.unwrap_err().to_string() - ) - } - - #[test] - fn test_parsing_multiple_client_tls_opts() { - let custom_root_ca = r#" ------BEGIN CERTIFICATE----- -MIIBeDCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3Mtc2Vy -dmVyLWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw -WjAjMSEwHwYDVQQDDBhrM3Mtc2VydmVyLWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO -PQIBBggqhkjOPQMBBwNCAAQnhGmz/r5E+ZBgkg/kpeSliS4LjMFaeFNM3C0SUksV -cVDbymRZt+D2loVpSIn9PnBHUIiR9kz+cmWJaJDhcY6Ho0IwQDAOBgNVHQ8BAf8E -BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUzXLACkzCDPAXXERIxQim -NdG07zEwCgYIKoZIzj0EAwIDSQAwRgIhALwsHX2R7a7GXfgmn7h8rNRRvlQwyRaG -9hyv0a1cyJr2AiEA8+2vF0CZ/S0MG6rT0Y6xZ+iqi/vhcDnmBhJCxx2rwAI= ------END CERTIFICATE----- -"#; - let mut custom_root_ca_file = NamedTempFile::new().expect("temp file for custom root ca"); - custom_root_ca_file - .write_all(custom_root_ca.as_bytes()) - .expect("write custom root ca file"); - - let runtimeconfig_data = format!( - r#" -[[client_tls]] -hosts = ["localhost:6551"] -component_ids = ["component-no1"] -[[client_tls]] -hosts = ["localhost:6551"] -component_ids = ["component-no2"] -ca_roots_file = "{}" -"#, - custom_root_ca_file.path().to_str().unwrap() - ); - - let mut config = RuntimeConfig::new(None); - merge_config_toml(&mut config, toml::from_str(&runtimeconfig_data).unwrap()); - - let client_tls_opts = config.client_tls_opts(); - assert!(client_tls_opts.is_ok()); - - //assert that component level mapping works as expected - let client_tls_opts_ok = client_tls_opts.as_ref().unwrap(); - - // assert for component-no1 - assert!(client_tls_opts_ok - .get(&"component-no1".to_string()) - .is_some()); - - #[allow(clippy::mutable_key_type)] - let component_no1_client_tls_opts = client_tls_opts_ok - .get(&"component-no1".to_string()) - .expect("get opts for component-no1"); - assert!(component_no1_client_tls_opts - .get(&"localhost:6551".parse::().unwrap()) - .is_some()); - - let component_no1_host_client_tls_opts = component_no1_client_tls_opts - .get(&"localhost:6551".parse::().unwrap()) - .unwrap(); - assert!(component_no1_host_client_tls_opts.custom_root_ca.is_none()); - - // assert for component-no2 - assert!(client_tls_opts_ok - .get(&"component-no2".to_string()) - .is_some()); - - #[allow(clippy::mutable_key_type)] - let component_no2_client_tls_opts = client_tls_opts_ok - .get(&"component-no2".to_string()) - .expect("get opts for component-no2"); - assert!(component_no2_client_tls_opts - .get(&"localhost:6551".parse::().unwrap()) - .is_some()); - - let component_no2_host_client_tls_opts = component_no2_client_tls_opts - .get(&"localhost:6551".parse::().unwrap()) - .unwrap(); - assert!(component_no2_host_client_tls_opts.custom_root_ca.is_some()) - } - - #[test] - fn test_parsing_multiple_overlapping_client_tls_opts() { - let custom_root_ca = r#" ------BEGIN CERTIFICATE----- -MIIBeDCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3Mtc2Vy -dmVyLWNhQDE3MTc3ODA1MjAwHhcNMjQwNjA3MTcxNTIwWhcNMzQwNjA1MTcxNTIw -WjAjMSEwHwYDVQQDDBhrM3Mtc2VydmVyLWNhQDE3MTc3ODA1MjAwWTATBgcqhkjO -PQIBBggqhkjOPQMBBwNCAAQnhGmz/r5E+ZBgkg/kpeSliS4LjMFaeFNM3C0SUksV -cVDbymRZt+D2loVpSIn9PnBHUIiR9kz+cmWJaJDhcY6Ho0IwQDAOBgNVHQ8BAf8E -BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUzXLACkzCDPAXXERIxQim -NdG07zEwCgYIKoZIzj0EAwIDSQAwRgIhALwsHX2R7a7GXfgmn7h8rNRRvlQwyRaG -9hyv0a1cyJr2AiEA8+2vF0CZ/S0MG6rT0Y6xZ+iqi/vhcDnmBhJCxx2rwAI= ------END CERTIFICATE----- -"#; - let mut custom_root_ca_file = NamedTempFile::new().expect("temp file for custom root ca"); - custom_root_ca_file - .write_all(custom_root_ca.as_bytes()) - .expect("write custom root ca file"); - - let runtimeconfig_data = format!( - r#" -[[client_tls]] -hosts = ["localhost:6551"] -component_ids = ["component-no1"] -[[client_tls]] -hosts = ["localhost:6551"] -component_ids = ["component-no1"] -ca_roots_file = "{}" -"#, - custom_root_ca_file.path().to_str().unwrap() - ); - - let mut config = RuntimeConfig::new(None); - merge_config_toml(&mut config, toml::from_str(&runtimeconfig_data).unwrap()); - - let client_tls_opts = config.client_tls_opts(); - assert!(client_tls_opts.is_ok()); - - //assert that component level mapping works as expected - let client_tls_opts_ok = client_tls_opts.as_ref().unwrap(); - - // assert for component-no1 - assert!(client_tls_opts_ok - .get(&"component-no1".to_string()) - .is_some()); - - #[allow(clippy::mutable_key_type)] - let component_no1_client_tls_opts = client_tls_opts_ok - .get(&"component-no1".to_string()) - .expect("get opts for component-no1"); - assert!(component_no1_client_tls_opts - .get(&"localhost:6551".parse::().unwrap()) - .is_some()); - - let component_no1_host_client_tls_opts = component_no1_client_tls_opts - .get(&"localhost:6551".parse::().unwrap()) - .unwrap(); - - // verify that the last client_tls block wins for same component-id and host combination - assert!(component_no1_host_client_tls_opts.custom_root_ca.is_none()); - } - - fn merge_config_toml(config: &mut RuntimeConfig, value: toml::Value) { - let data = toml::to_vec(&value).expect("encode toml"); - let mut file = NamedTempFile::new().expect("temp file"); - file.write_all(&data).expect("write toml"); - config.merge_config_file(file.path()).expect("merge config"); - } - - fn default_spin_store_path(config: &RuntimeConfig) -> Option { - match config.default_key_value_opts() { - KeyValueStoreOpts::Spin(opts) => opts.path, - other => panic!("unexpected default store opts {other:?}"), - } - } -} - -// parsed client tls options -#[derive(Debug, Clone)] -pub struct ParsedClientTlsOpts { - pub components: Vec, - pub hosts: Vec, - pub custom_root_ca: Option>>, - pub cert_chain: Option>>, - pub private_key: Option>>, - pub ca_webpki_roots: bool, -} - -fn parse_client_tls_opts(inp: &ClientTlsOpts) -> Result { - let custom_root_ca = match &inp.ca_roots_file { - Some(path) => Some(load_certs(path).context("loading custom root ca")?), - None => None, - }; - - let cert_chain = match &inp.cert_chain_file { - Some(file) => Some(load_certs(file).context("loading client tls certs")?), - None => None, - }; - - let private_key = match &inp.private_key_file { - Some(file) => { - let privatekey = load_key(file).context("loading private key")?; - Some(Arc::from(privatekey)) - } - None => None, - }; - - let parsed_hosts: Vec = inp - .hosts - .clone() - .into_iter() - .map(|s| { - s.parse::() - .map_err(|e| anyhow::anyhow!("failed to parse uri '{}'. error: {:?}", s, e)) - }) - .collect::, anyhow::Error>>()?; - - let custom_root_ca_provided = custom_root_ca.is_some(); - - // use_ca_webpki_roots is true if - // 1. ca_webpki_roots is explicitly true in runtime config OR - // 2. custom_root_ca is not provided - // - // if custom_root_ca is provided, use_ca_webpki_roots defaults to false - let ca_webpki_roots = inp.ca_webpki_roots.unwrap_or(!custom_root_ca_provided); - - let parsed_component_ids: Vec = inp - .component_ids - .clone() - .into_iter() - .map(|s| s.to_string()) - .collect(); - - Ok(ParsedClientTlsOpts { - hosts: parsed_hosts, - components: parsed_component_ids, - custom_root_ca, - cert_chain, - private_key, - ca_webpki_roots, - }) -} diff --git a/crates/trigger/src/runtime_config/client_tls.rs b/crates/trigger/src/runtime_config/client_tls.rs deleted file mode 100644 index 59390841e..000000000 --- a/crates/trigger/src/runtime_config/client_tls.rs +++ /dev/null @@ -1,50 +0,0 @@ -use anyhow::Context; -use rustls_pemfile::private_key; -use std::io; -use std::{ - fs, - path::{Path, PathBuf}, -}; - -#[derive(Debug, serde::Deserialize)] -#[serde(rename_all = "snake_case", tag = "type")] -pub struct ClientTlsOpts { - pub component_ids: Vec, - pub hosts: Vec, - pub ca_roots_file: Option, - pub cert_chain_file: Option, - pub private_key_file: Option, - pub ca_webpki_roots: Option, -} - -// load_certs parse and return the certs from the provided file -pub fn load_certs( - path: impl AsRef, -) -> io::Result>> { - rustls_pemfile::certs(&mut io::BufReader::new(fs::File::open(path).map_err( - |err| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("failed to read cert file {:?}", err), - ) - }, - )?)) - .collect::>>>() -} - -// load_keys parse and return the first private key from the provided file -pub fn load_key( - path: impl AsRef, -) -> anyhow::Result> { - private_key(&mut io::BufReader::new( - fs::File::open(path).context("loading private key")?, - )) - .map_err(|_| anyhow::anyhow!("invalid input")) - .transpose() - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "private key file contains no private keys", - ) - })? -} diff --git a/crates/trigger/src/runtime_config/key_value.rs b/crates/trigger/src/runtime_config/key_value.rs deleted file mode 100644 index bbfd83572..000000000 --- a/crates/trigger/src/runtime_config/key_value.rs +++ /dev/null @@ -1,195 +0,0 @@ -use std::{collections::HashMap, fs, path::PathBuf, sync::Arc}; - -use crate::{runtime_config::RuntimeConfig, TriggerHooks}; -use anyhow::{bail, Context, Result}; -use serde::Deserialize; -use spin_common::ui::quoted_path; -use spin_key_value::{ - CachingStoreManager, DelegatingStoreManager, KeyValueComponent, StoreManager, - KEY_VALUE_STORES_KEY, -}; -use spin_key_value_azure::{ - KeyValueAzureCosmos, KeyValueAzureCosmosAuthOptions, KeyValueAzureCosmosRuntimeConfigOptions, -}; -use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; - -use super::{resolve_config_path, RuntimeConfigOpts}; - -const DEFAULT_SPIN_STORE_FILENAME: &str = "sqlite_key_value.db"; - -pub type KeyValueStore = Arc; - -/// Builds a [`KeyValueComponent`] from the given [`RuntimeConfig`]. -pub async fn build_key_value_component( - runtime_config: &RuntimeConfig, - init_data: &[(String, String)], -) -> Result { - let stores: HashMap<_, _> = runtime_config - .key_value_stores() - .context("Failed to build key-value component")? - .into_iter() - .collect(); - - // Avoid creating a database as a side-effect if one is not needed. - if !init_data.is_empty() { - if let Some(manager) = stores.get("default") { - let default_store = manager - .get("default") - .await - .context("Failed to access key-value store to set requested entries")?; - for (key, value) in init_data { - default_store - .set(key, value.as_bytes()) - .await - .with_context(|| { - format!("Failed to set requested entry {key} in key-value store") - })?; - } - } else { - bail!("Failed to access key-value store to set requested entries"); - } - } - // This is a temporary addition while factors work is in progress - // The default manager should already be added for all default labels - // and this should never be called. - let default_manager_fn = |_: &str| -> Option> { None }; - let delegating_manager = DelegatingStoreManager::new(stores, Arc::new(default_manager_fn)); - let caching_manager = Arc::new(CachingStoreManager::new(delegating_manager)); - Ok(KeyValueComponent::new(spin_key_value::manager(move |_| { - caching_manager.clone() - }))) -} - -// Holds deserialized options from a `[key_value_store.]` runtime config section. -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "snake_case", tag = "type")] -pub enum KeyValueStoreOpts { - Spin(SpinKeyValueStoreOpts), - Redis(RedisKeyValueStoreOpts), - AzureCosmos(AzureCosmosConfig), -} - -impl KeyValueStoreOpts { - pub fn default_store_opts(runtime_config: &RuntimeConfig) -> Self { - Self::Spin(SpinKeyValueStoreOpts::default_store_opts(runtime_config)) - } - - pub fn build_store(&self, config_opts: &RuntimeConfigOpts) -> Result { - match self { - Self::Spin(opts) => opts.build_store(config_opts), - Self::Redis(opts) => opts.build_store(), - Self::AzureCosmos(opts) => opts.build_store(), - } - } -} - -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SpinKeyValueStoreOpts { - pub path: Option, -} - -impl SpinKeyValueStoreOpts { - fn default_store_opts(runtime_config: &RuntimeConfig) -> Self { - // If the state dir is set, build the default path - let path = runtime_config - .state_dir() - .map(|dir| dir.join(DEFAULT_SPIN_STORE_FILENAME)); - Self { path } - } - - fn build_store(&self, config_opts: &RuntimeConfigOpts) -> Result { - let location = match self.path.as_ref() { - Some(path) => { - let path = resolve_config_path(path, config_opts)?; - // Create the store's parent directory if necessary - fs::create_dir_all(path.parent().unwrap()) - .context("Failed to create key value store")?; - DatabaseLocation::Path(path) - } - None => DatabaseLocation::InMemory, - }; - Ok(Arc::new(KeyValueSqlite::new(location))) - } -} - -#[derive(Clone, Debug, Deserialize)] -pub struct RedisKeyValueStoreOpts { - pub url: String, -} - -impl RedisKeyValueStoreOpts { - fn build_store(&self) -> Result { - let kv_redis = spin_key_value_redis::KeyValueRedis::new(self.url.clone())?; - Ok(Arc::new(kv_redis)) - } -} - -#[derive(Clone, Debug, Deserialize)] -pub struct AzureCosmosConfig { - key: Option, - account: String, - database: String, - container: String, -} - -impl AzureCosmosConfig { - pub fn build_store(&self) -> Result> { - let auth_options = match self.key.clone() { - Some(key) => { - tracing::debug!("Azure key value is using key auth."); - let config_values = KeyValueAzureCosmosRuntimeConfigOptions::new(key); - KeyValueAzureCosmosAuthOptions::RuntimeConfigValues(config_values) - } - None => { - tracing::debug!("Azure key value is using environmental auth."); - KeyValueAzureCosmosAuthOptions::Environmental - } - }; - let kv_azure_cosmos = KeyValueAzureCosmos::new( - self.account.clone(), - self.database.clone(), - self.container.clone(), - auth_options, - )?; - Ok(Arc::new(kv_azure_cosmos)) - } -} - -// Prints startup messages about the default key value store config. -pub struct KeyValuePersistenceMessageHook; - -impl TriggerHooks for KeyValuePersistenceMessageHook { - fn app_loaded( - &mut self, - app: &spin_app::App, - runtime_config: &RuntimeConfig, - _resolver: &Arc, - ) -> Result<()> { - // Only print if the app actually uses KV - if app.components().all(|c| { - c.get_metadata(KEY_VALUE_STORES_KEY) - .unwrap_or_default() - .unwrap_or_default() - .is_empty() - }) { - return Ok(()); - } - match runtime_config.default_key_value_opts() { - KeyValueStoreOpts::Redis(_store_opts) => { - println!("Storing default key-value data to Redis"); - } - KeyValueStoreOpts::Spin(store_opts) => { - if let Some(path) = &store_opts.path { - println!("Storing default key-value data to {}", quoted_path(path)); - } else { - println!("Using in-memory default key-value store; data will not be saved!"); - } - } - KeyValueStoreOpts::AzureCosmos(store_opts) => { - println!("Storing default key-value data to Azure CosmosDB: account: {}, database: {}, container: {}", store_opts.account, store_opts.database, store_opts.container); - } - } - Ok(()) - } -} diff --git a/crates/trigger/src/runtime_config/llm.rs b/crates/trigger/src/runtime_config/llm.rs deleted file mode 100644 index aed48f639..000000000 --- a/crates/trigger/src/runtime_config/llm.rs +++ /dev/null @@ -1,82 +0,0 @@ -use spin_llm_remote_http::RemoteHttpLlmEngine; -use url::Url; - -#[derive(Default)] -pub struct LLmOptions { - pub use_gpu: bool, -} - -pub(crate) async fn build_component( - runtime_config: &crate::RuntimeConfig, - use_gpu: bool, -) -> spin_llm::LlmComponent { - match runtime_config.llm_compute() { - #[cfg(feature = "llm")] - LlmComputeOpts::Spin => { - let path = runtime_config - .state_dir() - .unwrap_or_default() - .join("ai-models"); - let engine = spin_llm_local::LocalLlmEngine::new(path, use_gpu).await; - spin_llm::LlmComponent::new(move || Box::new(engine.clone())) - } - #[cfg(not(feature = "llm"))] - LlmComputeOpts::Spin => { - let _ = use_gpu; - spin_llm::LlmComponent::new(move || Box::new(noop::NoopLlmEngine.clone())) - } - LlmComputeOpts::RemoteHttp(config) => { - tracing::info!("Using remote compute for LLMs"); - let engine = - RemoteHttpLlmEngine::new(config.url.to_owned(), config.auth_token.to_owned()); - spin_llm::LlmComponent::new(move || Box::new(engine.clone())) - } - } -} - -#[derive(Debug, serde::Deserialize)] -#[serde(rename_all = "snake_case", tag = "type")] -pub enum LlmComputeOpts { - Spin, - RemoteHttp(RemoteHttpComputeOpts), -} - -#[derive(Debug, serde::Deserialize)] -pub struct RemoteHttpComputeOpts { - url: Url, - auth_token: String, -} - -#[cfg(not(feature = "llm"))] -mod noop { - use async_trait::async_trait; - use spin_llm::LlmEngine; - use spin_world::v2::llm as wasi_llm; - - #[derive(Clone)] - pub(super) struct NoopLlmEngine; - - #[async_trait] - impl LlmEngine for NoopLlmEngine { - async fn infer( - &mut self, - _model: wasi_llm::InferencingModel, - _prompt: String, - _params: wasi_llm::InferencingParams, - ) -> Result { - Err(wasi_llm::Error::RuntimeError( - "Local LLM operations are not supported in this version of Spin.".into(), - )) - } - - async fn generate_embeddings( - &mut self, - _model: wasi_llm::EmbeddingModel, - _data: Vec, - ) -> Result { - Err(wasi_llm::Error::RuntimeError( - "Local LLM operations are not supported in this version of Spin.".into(), - )) - } - } -} diff --git a/crates/trigger/src/runtime_config/sqlite.rs b/crates/trigger/src/runtime_config/sqlite.rs deleted file mode 100644 index 5163791e9..000000000 --- a/crates/trigger/src/runtime_config/sqlite.rs +++ /dev/null @@ -1,240 +0,0 @@ -use std::{collections::HashMap, path::PathBuf, sync::Arc}; - -use crate::{runtime_config::RuntimeConfig, TriggerHooks}; -use anyhow::Context; -use spin_common::ui::quoted_path; -use spin_sqlite::{Connection, ConnectionsStore, SqliteComponent, DATABASES_KEY}; - -use super::RuntimeConfigOpts; - -const DEFAULT_SQLITE_DB_FILENAME: &str = "sqlite_db.db"; - -pub(crate) async fn build_component( - runtime_config: &RuntimeConfig, - sqlite_statements: &[String], -) -> anyhow::Result { - let databases: HashMap<_, _> = runtime_config - .sqlite_databases() - .await - .context("Failed to build sqlite component")? - .into_iter() - .collect(); - execute_statements(sqlite_statements, &databases).await?; - let connections_store = - Arc::new(SimpleConnectionsStore(databases)) as Arc; - Ok(SqliteComponent::new(move |_| connections_store.clone())) -} - -/// A `ConnectionStore` based on a `HashMap` -struct SimpleConnectionsStore(HashMap>); - -#[async_trait::async_trait] -impl ConnectionsStore for SimpleConnectionsStore { - async fn get_connection( - &self, - database: &str, - ) -> Result>, spin_world::v2::sqlite::Error> { - Ok(self.0.get(database).cloned()) - } - - fn has_connection_for(&self, database: &str) -> bool { - self.0.contains_key(database) - } -} - -async fn execute_statements( - statements: &[String], - databases: &HashMap>, -) -> anyhow::Result<()> { - if statements.is_empty() { - return Ok(()); - } - - for m in statements { - if let Some(config) = m.strip_prefix('@') { - let (file, database) = parse_file_and_label(config)?; - let database = databases.get(database).with_context(|| { - format!( - "based on the '@{config}' a registered database named '{database}' was expected but not found. The registered databases are '{:?}'", databases.keys() - ) - })?; - let sql = std::fs::read_to_string(file).with_context(|| { - format!("could not read file '{file}' containing sql statements") - })?; - database - .execute_batch(&sql) - .await - .with_context(|| format!("failed to execute sql from file '{file}'"))?; - } else { - let Some(default) = databases.get("default") else { - debug_assert!(false, "the 'default' sqlite database should always be available but for some reason was not"); - return Ok(()); - }; - default - .query(m, Vec::new()) - .await - .with_context(|| format!("failed to execute statement: '{m}'"))?; - } - } - Ok(()) -} - -/// Parses a @{file:label} sqlite statement -fn parse_file_and_label(config: &str) -> anyhow::Result<(&str, &str)> { - let config = config.trim(); - let (file, label) = match config.split_once(':') { - Some((_, label)) if label.trim().is_empty() => { - anyhow::bail!("database label is empty in the '@{config}' sqlite statement") - } - Some((file, label)) => (file.trim(), label.trim()), - None => (config, "default"), - }; - Ok((file, label)) -} - -// Holds deserialized options from a `[sqlite_database.]` runtime config section. -#[derive(Clone, Debug, serde::Deserialize)] -#[serde(rename_all = "snake_case", tag = "type")] -pub enum SqliteDatabaseOpts { - Spin(SpinSqliteDatabaseOpts), - Libsql(LibsqlOpts), -} - -impl SqliteDatabaseOpts { - pub fn default(runtime_config: &RuntimeConfig) -> Self { - Self::Spin(SpinSqliteDatabaseOpts::default(runtime_config)) - } - - pub async fn build( - &self, - config_opts: &RuntimeConfigOpts, - ) -> anyhow::Result> { - match self { - Self::Spin(opts) => opts.build(config_opts), - Self::Libsql(opts) => opts.build().await, - } - } -} - -#[derive(Clone, Debug, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SpinSqliteDatabaseOpts { - pub path: Option, -} - -impl SpinSqliteDatabaseOpts { - pub fn default(runtime_config: &RuntimeConfig) -> Self { - let path = runtime_config - .state_dir() - .map(|dir| dir.join(DEFAULT_SQLITE_DB_FILENAME)); - Self { path } - } - - fn build(&self, config_opts: &RuntimeConfigOpts) -> anyhow::Result> { - use spin_sqlite_inproc::{InProcConnection, InProcDatabaseLocation}; - - let location = match self.path.as_ref() { - Some(path) => { - let path = super::resolve_config_path(path, config_opts)?; - // Create the store's parent directory if necessary - std::fs::create_dir_all(path.parent().unwrap()) - .context("Failed to create sqlite database directory")?; - InProcDatabaseLocation::Path(path) - } - None => InProcDatabaseLocation::InMemory, - }; - Ok(Arc::new(InProcConnection::new(location)?)) - } -} - -#[derive(Clone, Debug, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct LibsqlOpts { - url: String, - token: String, -} - -impl LibsqlOpts { - async fn build(&self) -> anyhow::Result> { - let url = check_url(&self.url) - .with_context(|| { - format!( - "unexpected libSQL URL '{}' in runtime config file ", - self.url - ) - })? - .to_owned(); - let client = spin_sqlite_libsql::LibsqlClient::create(url, self.token.clone()) - .await - .context("failed to create SQLite client")?; - Ok(Arc::new(client)) - } -} - -// Checks an incoming url is in the shape we expect -fn check_url(url: &str) -> anyhow::Result<&str> { - if url.starts_with("https://") || url.starts_with("http://") { - Ok(url) - } else { - Err(anyhow::anyhow!( - "URL does not start with 'https://' or 'http://'. Spin currently only supports talking to libSQL databases over HTTP(S)" - )) - } -} - -pub struct SqlitePersistenceMessageHook; - -impl TriggerHooks for SqlitePersistenceMessageHook { - fn app_loaded( - &mut self, - app: &spin_app::App, - runtime_config: &RuntimeConfig, - _resolver: &Arc, - ) -> anyhow::Result<()> { - if app.components().all(|c| { - c.get_metadata(DATABASES_KEY) - .unwrap_or_default() - .unwrap_or_default() - .is_empty() - }) { - return Ok(()); - } - - match runtime_config.default_sqlite_opts() { - SqliteDatabaseOpts::Spin(s) => { - if let Some(path) = &s.path { - println!("Storing default SQLite data to {}", quoted_path(path)); - } else { - println!("Using in-memory default SQLite database; data will not be saved!"); - } - } - SqliteDatabaseOpts::Libsql(l) => { - println!( - "Storing default SQLite data to a libsql database at {}", - l.url - ); - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn can_parse_file_and_label() { - let config = "file:label"; - let result = parse_file_and_label(config).unwrap(); - assert_eq!(result, ("file", "label")); - - let config = "file:"; - let result = parse_file_and_label(config); - assert!(result.is_err()); - - let config = "file"; - let result = parse_file_and_label(config).unwrap(); - assert_eq!(result, ("file", "default")); - } -} diff --git a/crates/trigger/src/runtime_config/variables_provider.rs b/crates/trigger/src/runtime_config/variables_provider.rs deleted file mode 100644 index 3cfc5c1de..000000000 --- a/crates/trigger/src/runtime_config/variables_provider.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::path::PathBuf; - -use anyhow::{anyhow, Result}; -use serde::Deserialize; -use spin_variables::provider::azure_key_vault::{ - AzureKeyVaultAuthOptions, AzureKeyVaultRuntimeConfigOptions, -}; -use spin_variables::provider::{ - azure_key_vault::{AzureAuthorityHost, AzureKeyVaultProvider}, - env::EnvProvider, - vault::VaultProvider, -}; - -use super::RuntimeConfig; - -pub type VariablesProvider = Box; - -// Holds deserialized options from a `[[config_provider]]` runtime config section. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "snake_case", tag = "type")] -pub enum VariablesProviderOpts { - Env(EnvVariablesProviderOpts), - Vault(VaultVariablesProviderOpts), - AzureKeyVault(AzureKeyVaultVariablesProviderOpts), -} - -impl VariablesProviderOpts { - pub fn default_provider_opts(runtime_config: &RuntimeConfig) -> Self { - Self::Env(EnvVariablesProviderOpts::default_provider_opts( - runtime_config, - )) - } - - pub fn build_provider(&self) -> Result { - match self { - Self::Env(opts) => opts.build_provider(), - Self::Vault(opts) => opts.build_provider(), - Self::AzureKeyVault(opts) => opts.build_provider(), - } - } -} - -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EnvVariablesProviderOpts { - /// A prefix to add to variable names when resolving from the environment. - /// Unless empty, joined to the variable name with an underscore. - #[serde(default)] - pub prefix: Option, - /// Optional path to a 'dotenv' file which will be merged into the environment. - #[serde(default)] - pub dotenv_path: Option, -} - -impl EnvVariablesProviderOpts { - pub fn default_provider_opts(runtime_config: &RuntimeConfig) -> Self { - let dotenv_path = runtime_config - .local_app_dir - .as_deref() - .map(|path| path.join(".env")); - Self { - prefix: None, - dotenv_path, - } - } - - pub fn build_provider(&self) -> Result { - Ok(Box::new(EnvProvider::new( - self.prefix.clone(), - self.dotenv_path.clone(), - ))) - } -} - -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct VaultVariablesProviderOpts { - pub url: String, - pub token: String, - pub mount: String, - #[serde(default)] - pub prefix: Option, -} - -impl VaultVariablesProviderOpts { - pub fn build_provider(&self) -> Result { - Ok(Box::new(VaultProvider::new( - &self.url, - &self.token, - &self.mount, - self.prefix.as_deref(), - ))) - } -} - -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct AzureKeyVaultVariablesProviderOpts { - pub vault_url: String, - pub client_id: Option, - pub client_secret: Option, - pub tenant_id: Option, - pub authority_host: Option, -} - -impl AzureKeyVaultVariablesProviderOpts { - pub fn build_provider(&self) -> Result { - let auth_config_runtime_vars = [&self.client_id, &self.tenant_id, &self.client_secret]; - let any_some = auth_config_runtime_vars.iter().any(|&var| var.is_some()); - let any_none = auth_config_runtime_vars.iter().any(|&var| var.is_none()); - - if any_none && any_some { - // some of the service principal auth options were specified, but not enough to authenticate. - return Err(anyhow!("The current runtime config specifies some but not all of the Azure KeyVault 'client_id', 'client_secret', and 'tenant_id' values. Provide the missing values to authenticate to Azure KeyVault with the given service principal, or remove all these values to authenticate using ambient authentication (e.g. env vars, Azure CLI, Managed Identity, Workload Identity).")); - } - - let auth_options = if any_some { - // all the service principal auth options were specified in the runtime config - AzureKeyVaultAuthOptions::RuntimeConfigValues(AzureKeyVaultRuntimeConfigOptions::new( - self.client_id.clone().unwrap(), - self.client_secret.clone().unwrap(), - self.tenant_id.clone().unwrap(), - self.authority_host, - )) - } else { - AzureKeyVaultAuthOptions::Environmental - }; - - Ok(Box::new(AzureKeyVaultProvider::new( - &self.vault_url, - auth_options, - )?)) - } -} diff --git a/crates/trigger/src/stdio.rs b/crates/trigger/src/stdio.rs deleted file mode 100644 index e23a7f8b6..000000000 --- a/crates/trigger/src/stdio.rs +++ /dev/null @@ -1,336 +0,0 @@ -use std::{ - collections::HashSet, - path::{Path, PathBuf}, - task::Poll, -}; - -use anyhow::{Context, Result}; -use spin_common::ui::quoted_path; -use tokio::io::AsyncWrite; - -use crate::{runtime_config::RuntimeConfig, TriggerHooks}; - -/// Which components should have their logs followed on stdout/stderr. -#[derive(Clone, Debug)] -pub enum FollowComponents { - /// No components should have their logs followed. - None, - /// Only the specified components should have their logs followed. - Named(HashSet), - /// All components should have their logs followed. - All, -} - -impl FollowComponents { - /// Whether a given component should have its logs followed on stdout/stderr. - pub fn should_follow(&self, component_id: &str) -> bool { - match self { - Self::None => false, - Self::All => true, - Self::Named(ids) => ids.contains(component_id), - } - } -} - -impl Default for FollowComponents { - fn default() -> Self { - Self::None - } -} - -/// Implements TriggerHooks, writing logs to a log file and (optionally) stderr -pub struct StdioLoggingTriggerHooks { - follow_components: FollowComponents, - log_dir: Option, -} - -impl StdioLoggingTriggerHooks { - pub fn new(follow_components: FollowComponents) -> Self { - Self { - follow_components, - log_dir: None, - } - } - - fn component_stdio_writer( - &self, - component_id: &str, - log_suffix: &str, - log_dir: Option<&Path>, - ) -> Result { - let sanitized_component_id = sanitize_filename::sanitize(component_id); - let log_path = log_dir - .map(|log_dir| log_dir.join(format!("{sanitized_component_id}_{log_suffix}.txt",))); - let log_path = log_path.as_deref(); - - let follow = self.follow_components.should_follow(component_id); - match log_path { - Some(log_path) => ComponentStdioWriter::new_forward(log_path, follow) - .with_context(|| format!("Failed to open log file {}", quoted_path(log_path))), - None => ComponentStdioWriter::new_inherit(), - } - } - - fn validate_follows(&self, app: &spin_app::App) -> anyhow::Result<()> { - match &self.follow_components { - FollowComponents::Named(names) => { - let component_ids: HashSet<_> = - app.components().map(|c| c.id().to_owned()).collect(); - let unknown_names: Vec<_> = names.difference(&component_ids).collect(); - if unknown_names.is_empty() { - Ok(()) - } else { - let unknown_list = bullet_list(&unknown_names); - let actual_list = bullet_list(&component_ids); - let message = anyhow::anyhow!("The following component(s) specified in --follow do not exist in the application:\n{unknown_list}\nThe following components exist:\n{actual_list}"); - Err(message) - } - } - _ => Ok(()), - } - } -} - -impl TriggerHooks for StdioLoggingTriggerHooks { - fn app_loaded( - &mut self, - app: &spin_app::App, - runtime_config: &RuntimeConfig, - _resolver: &std::sync::Arc, - ) -> anyhow::Result<()> { - self.log_dir = runtime_config.log_dir(); - - self.validate_follows(app)?; - - if let Some(dir) = &self.log_dir { - // Ensure log dir exists if set - std::fs::create_dir_all(dir) - .with_context(|| format!("Failed to create log dir {}", quoted_path(dir)))?; - - println!("Logging component stdio to {}", quoted_path(dir.join(""))) - } - - Ok(()) - } - - fn component_store_builder( - &self, - component: &spin_app::AppComponent, - builder: &mut spin_core::StoreBuilder, - ) -> anyhow::Result<()> { - builder.stdout_pipe(self.component_stdio_writer( - component.id(), - "stdout", - self.log_dir.as_deref(), - )?); - builder.stderr_pipe(self.component_stdio_writer( - component.id(), - "stderr", - self.log_dir.as_deref(), - )?); - - Ok(()) - } -} - -/// ComponentStdioWriter forwards output to a log file, (optionally) stderr, and (optionally) to a -/// tracing compatibility layer. -pub struct ComponentStdioWriter { - inner: ComponentStdioWriterInner, -} - -enum ComponentStdioWriterInner { - /// Inherit stdout/stderr from the parent process. - Inherit, - /// Forward stdout/stderr to a file in addition to the inherited stdout/stderr. - Forward { - sync_file: std::fs::File, - async_file: tokio::fs::File, - state: ComponentStdioWriterState, - follow: bool, - }, -} - -#[derive(Debug)] -enum ComponentStdioWriterState { - File, - Follow(std::ops::Range), -} - -impl ComponentStdioWriter { - fn new_forward(log_path: &Path, follow: bool) -> anyhow::Result { - let sync_file = std::fs::File::options() - .create(true) - .append(true) - .open(log_path)?; - - let async_file = sync_file - .try_clone() - .context("could not get async file handle")? - .into(); - - Ok(Self { - inner: ComponentStdioWriterInner::Forward { - sync_file, - async_file, - state: ComponentStdioWriterState::File, - follow, - }, - }) - } - - fn new_inherit() -> anyhow::Result { - Ok(Self { - inner: ComponentStdioWriterInner::Inherit, - }) - } -} - -impl AsyncWrite for ComponentStdioWriter { - fn poll_write( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &[u8], - ) -> Poll> { - let this = self.get_mut(); - - loop { - match &mut this.inner { - ComponentStdioWriterInner::Inherit => { - let written = futures::ready!( - std::pin::Pin::new(&mut tokio::io::stderr()).poll_write(cx, buf) - ); - let written = match written { - Ok(w) => w, - Err(e) => return Poll::Ready(Err(e)), - }; - return Poll::Ready(Ok(written)); - } - ComponentStdioWriterInner::Forward { - async_file, - state, - follow, - .. - } => match &state { - ComponentStdioWriterState::File => { - let written = - futures::ready!(std::pin::Pin::new(async_file).poll_write(cx, buf)); - let written = match written { - Ok(w) => w, - Err(e) => return Poll::Ready(Err(e)), - }; - if *follow { - *state = ComponentStdioWriterState::Follow(0..written); - } else { - return Poll::Ready(Ok(written)); - } - } - ComponentStdioWriterState::Follow(range) => { - let written = futures::ready!(std::pin::Pin::new(&mut tokio::io::stderr()) - .poll_write(cx, &buf[range.clone()])); - let written = match written { - Ok(w) => w, - Err(e) => return Poll::Ready(Err(e)), - }; - if range.start + written >= range.end { - let end = range.end; - *state = ComponentStdioWriterState::File; - return Poll::Ready(Ok(end)); - } else { - *state = ComponentStdioWriterState::Follow( - (range.start + written)..range.end, - ); - }; - } - }, - } - } - } - - fn poll_flush( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> Poll> { - let this = self.get_mut(); - - match &mut this.inner { - ComponentStdioWriterInner::Inherit => { - std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) - } - ComponentStdioWriterInner::Forward { - async_file, state, .. - } => match state { - ComponentStdioWriterState::File => std::pin::Pin::new(async_file).poll_flush(cx), - ComponentStdioWriterState::Follow(_) => { - std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) - } - }, - } - } - - fn poll_shutdown( - self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> Poll> { - let this = self.get_mut(); - - match &mut this.inner { - ComponentStdioWriterInner::Inherit => { - std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) - } - ComponentStdioWriterInner::Forward { - async_file, state, .. - } => match state { - ComponentStdioWriterState::File => std::pin::Pin::new(async_file).poll_shutdown(cx), - ComponentStdioWriterState::Follow(_) => { - std::pin::Pin::new(&mut tokio::io::stderr()).poll_flush(cx) - } - }, - } - } -} - -impl std::io::Write for ComponentStdioWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - spin_telemetry::logs::handle_app_log(buf); - - match &mut self.inner { - ComponentStdioWriterInner::Inherit => { - std::io::stderr().write_all(buf)?; - Ok(buf.len()) - } - ComponentStdioWriterInner::Forward { - sync_file, follow, .. - } => { - let written = sync_file.write(buf)?; - if *follow { - std::io::stderr().write_all(&buf[..written])?; - } - Ok(written) - } - } - } - - fn flush(&mut self) -> std::io::Result<()> { - match &mut self.inner { - ComponentStdioWriterInner::Inherit => std::io::stderr().flush(), - ComponentStdioWriterInner::Forward { - sync_file, follow, .. - } => { - sync_file.flush()?; - if *follow { - std::io::stderr().flush()?; - } - Ok(()) - } - } - } -} - -fn bullet_list(items: impl IntoIterator) -> String { - items - .into_iter() - .map(|item| format!(" - {item}")) - .collect::>() - .join("\n") -} diff --git a/crates/variables/Cargo.toml b/crates/variables/Cargo.toml deleted file mode 100644 index f5de219e4..000000000 --- a/crates/variables/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "spin-variables" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } - -[dependencies] -anyhow = "1.0" -async-trait = "0.1" -dotenvy = "0.15" -once_cell = "1" -spin-app = { path = "../app" } -spin-core = { path = "../core" } -spin-expressions = { path = "../expressions" } -spin-world = { path = "../world" } -thiserror = "1" -tokio = { version = "1", features = ["rt-multi-thread"] } -vaultrs = "0.6.2" -serde = "1.0.188" -tracing = { workspace = true } -azure_security_keyvault = { git = "https://github.com/azure/azure-sdk-for-rust.git", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } -azure_core = { git = "https://github.com/azure/azure-sdk-for-rust.git", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } -azure_identity = { git = "https://github.com/azure/azure-sdk-for-rust.git", rev = "8c4caa251c3903d5eae848b41bb1d02a4d65231c" } - -[dev-dependencies] -toml = "0.5" - -[lints] -workspace = true diff --git a/crates/variables/src/host_component.rs b/crates/variables/src/host_component.rs deleted file mode 100644 index 821191add..000000000 --- a/crates/variables/src/host_component.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use anyhow::Result; -use once_cell::sync::OnceCell; -use spin_app::{AppComponent, DynamicHostComponent}; -use spin_core::{async_trait, HostComponent}; -use spin_world::v1::config::Error as V1ConfigError; -use spin_world::v2::variables; - -use spin_expressions::{Error, Key, Provider, ProviderResolver}; - -pub struct VariablesHostComponent { - providers: Mutex>>, - resolver: Arc>, -} - -impl VariablesHostComponent { - pub fn new(providers: Vec>) -> Self { - Self { - providers: Mutex::new(providers), - resolver: Default::default(), - } - } -} - -impl HostComponent for VariablesHostComponent { - type Data = ComponentVariables; - - fn add_to_linker( - linker: &mut spin_core::Linker, - get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, - ) -> anyhow::Result<()> { - spin_world::v1::config::add_to_linker(linker, get)?; - variables::add_to_linker(linker, get) - } - - fn build_data(&self) -> Self::Data { - ComponentVariables { - resolver: self.resolver.clone(), - component_id: None, - } - } -} - -impl DynamicHostComponent for VariablesHostComponent { - fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()> { - self.resolver.get_or_try_init(|| { - make_resolver(component.app, self.providers.lock().unwrap().drain(..)) - })?; - data.component_id = Some(component.id().to_string()); - Ok(()) - } -} - -pub fn make_resolver( - app: &spin_app::App, - providers: impl IntoIterator>, -) -> anyhow::Result { - let mut resolver = - ProviderResolver::new(app.variables().map(|(key, var)| (key.clone(), var.clone())))?; - for component in app.components() { - resolver.add_component_variables( - component.id(), - component.config().map(|(k, v)| (k.into(), v.into())), - )?; - } - for provider in providers { - resolver.add_provider(provider); - } - Ok(resolver) -} - -/// A component variables interface implementation. -pub struct ComponentVariables { - resolver: Arc>, - component_id: Option, -} - -#[async_trait] -impl variables::Host for ComponentVariables { - async fn get(&mut self, key: String) -> Result { - // Set by DynamicHostComponent::update_data - let component_id = self.component_id.as_deref().unwrap(); - let key = Key::new(&key).map_err(as_wit)?; - self.resolver - .get() - .unwrap() - .resolve(component_id, key) - .await - .map_err(as_wit) - } - - fn convert_error(&mut self, error: variables::Error) -> Result { - Ok(error) - } -} - -#[async_trait] -impl spin_world::v1::config::Host for ComponentVariables { - async fn get_config(&mut self, key: String) -> Result { - ::get(self, key) - .await - .map_err(|err| match err { - variables::Error::InvalidName(msg) => V1ConfigError::InvalidKey(msg), - variables::Error::Undefined(msg) => V1ConfigError::Provider(msg), - other => V1ConfigError::Other(format!("{other}")), - }) - } - - fn convert_error(&mut self, error: V1ConfigError) -> Result { - Ok(error) - } -} - -fn as_wit(err: Error) -> variables::Error { - match err { - Error::InvalidName(msg) => variables::Error::InvalidName(msg), - Error::Undefined(msg) => variables::Error::Undefined(msg), - Error::Provider(err) => variables::Error::Provider(err.to_string()), - other => variables::Error::Other(format!("{other}")), - } -} diff --git a/crates/variables/src/lib.rs b/crates/variables/src/lib.rs deleted file mode 100644 index 620e6171c..000000000 --- a/crates/variables/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod host_component; -pub mod provider; - -pub use host_component::{make_resolver, VariablesHostComponent}; diff --git a/crates/variables/src/provider.rs b/crates/variables/src/provider.rs deleted file mode 100644 index e311b3d01..000000000 --- a/crates/variables/src/provider.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod azure_key_vault; -pub mod env; -pub mod vault; diff --git a/crates/variables/src/provider/azure_key_vault.rs b/crates/variables/src/provider/azure_key_vault.rs deleted file mode 100644 index c9009a52f..000000000 --- a/crates/variables/src/provider/azure_key_vault.rs +++ /dev/null @@ -1,143 +0,0 @@ -use std::sync::Arc; - -use anyhow::{Context, Result}; -use async_trait::async_trait; -use azure_core::auth::TokenCredential; -use azure_core::Url; -use azure_security_keyvault::SecretClient; -use serde::Deserialize; -use spin_expressions::{Key, Provider}; -use tracing::{instrument, Level}; - -/// Azure KeyVault runtime config literal options for authentication -#[derive(Clone, Debug)] -pub struct AzureKeyVaultRuntimeConfigOptions { - client_id: String, - client_secret: String, - tenant_id: String, - authority_host: AzureAuthorityHost, -} - -impl AzureKeyVaultRuntimeConfigOptions { - pub fn new( - client_id: String, - client_secret: String, - tenant_id: String, - authority_host: Option, - ) -> Self { - Self { - client_id, - client_secret, - tenant_id, - authority_host: authority_host.unwrap_or_default(), - } - } -} - -/// Azure Cosmos Key / Value enumeration for the possible authentication options -#[derive(Clone, Debug)] -pub enum AzureKeyVaultAuthOptions { - /// Runtime Config values indicates the service principal credentials have been supplied - RuntimeConfigValues(AzureKeyVaultRuntimeConfigOptions), - /// Environmental indicates that the environment variables of the process should be used to - /// create the TokenCredential for the Cosmos client. This will use the Azure Rust SDK's - /// DefaultCredentialChain to derive the TokenCredential based on what environment variables - /// have been set. - /// - /// Service Principal with client secret: - /// - `AZURE_TENANT_ID`: ID of the service principal's Azure tenant. - /// - `AZURE_CLIENT_ID`: the service principal's client ID. - /// - `AZURE_CLIENT_SECRET`: one of the service principal's secrets. - /// - /// Service Principal with certificate: - /// - `AZURE_TENANT_ID`: ID of the service principal's Azure tenant. - /// - `AZURE_CLIENT_ID`: the service principal's client ID. - /// - `AZURE_CLIENT_CERTIFICATE_PATH`: path to a PEM or PKCS12 certificate file including the private key. - /// - `AZURE_CLIENT_CERTIFICATE_PASSWORD`: (optional) password for the certificate file. - /// - /// Workload Identity (Kubernetes, injected by the Workload Identity mutating webhook): - /// - `AZURE_TENANT_ID`: ID of the service principal's Azure tenant. - /// - `AZURE_CLIENT_ID`: the service principal's client ID. - /// - `AZURE_FEDERATED_TOKEN_FILE`: TokenFilePath is the path of a file containing a Kubernetes service account token. - /// - /// Managed Identity (User Assigned or System Assigned identities): - /// - `AZURE_CLIENT_ID`: (optional) if using a user assigned identity, this will be the client ID of the identity. - /// - /// Azure CLI: - /// - `AZURE_TENANT_ID`: (optional) use a specific tenant via the Azure CLI. - /// - /// Common across each: - /// - `AZURE_AUTHORITY_HOST`: (optional) the host for the identity provider. For example, for Azure public cloud the host defaults to "https://login.microsoftonline.com". - /// See also: https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/identity/README.md - Environmental, -} - -#[derive(Debug)] -pub struct AzureKeyVaultProvider { - secret_client: SecretClient, -} - -impl AzureKeyVaultProvider { - pub fn new( - vault_url: impl Into, - auth_options: AzureKeyVaultAuthOptions, - ) -> Result { - let http_client = azure_core::new_http_client(); - let token_credential = match auth_options.clone() { - AzureKeyVaultAuthOptions::RuntimeConfigValues(config) => { - let credential = azure_identity::ClientSecretCredential::new( - http_client, - config.authority_host.into(), - config.tenant_id.to_string(), - config.client_id.to_string(), - config.client_secret.to_string(), - ); - Arc::new(credential) as Arc - } - AzureKeyVaultAuthOptions::Environmental => azure_identity::create_default_credential()?, - }; - - Ok(Self { - secret_client: SecretClient::new(&vault_url.into(), token_credential)?, - }) - } -} - -#[async_trait] -impl Provider for AzureKeyVaultProvider { - #[instrument(name = "spin_variables.get_from_azure_key_vault", skip(self), err(level = Level::INFO), fields(otel.kind = "client"))] - async fn get(&self, key: &Key) -> Result> { - let secret = self - .secret_client - .get(key.as_str()) - .await - .context("Failed to read variable from Azure Key Vault")?; - Ok(Some(secret.value)) - } -} - -#[derive(Debug, Copy, Clone, Deserialize)] -pub enum AzureAuthorityHost { - AzurePublicCloud, - AzureChina, - AzureGermany, - AzureGovernment, -} - -impl Default for AzureAuthorityHost { - fn default() -> Self { - Self::AzurePublicCloud - } -} - -impl From for Url { - fn from(value: AzureAuthorityHost) -> Self { - let url = match value { - AzureAuthorityHost::AzureChina => "https://login.chinacloudapi.cn/", - AzureAuthorityHost::AzureGovernment => "https://login.microsoftonline.us/", - AzureAuthorityHost::AzureGermany => "https://login.microsoftonline.de/", - AzureAuthorityHost::AzurePublicCloud => "https://login.microsoftonline.com/", - }; - Url::parse(url).unwrap() - } -} diff --git a/crates/variables/src/provider/env.rs b/crates/variables/src/provider/env.rs deleted file mode 100644 index 90ba581f3..000000000 --- a/crates/variables/src/provider/env.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::{collections::HashMap, path::PathBuf, sync::Mutex}; - -use anyhow::{Context, Result}; -use async_trait::async_trait; - -use spin_expressions::{Key, Provider}; -use tracing::{instrument, Level}; - -const DEFAULT_ENV_PREFIX: &str = "SPIN_VARIABLE"; -const LEGACY_ENV_PREFIX: &str = "SPIN_CONFIG"; - -/// A config Provider that uses environment variables. -#[derive(Debug)] -pub struct EnvProvider { - prefix: Option, - dotenv_path: Option, - dotenv_cache: Mutex>>, -} - -impl EnvProvider { - /// Creates a new EnvProvider. - pub fn new(prefix: Option>, dotenv_path: Option) -> Self { - Self { - prefix: prefix.map(Into::into), - dotenv_path, - dotenv_cache: Default::default(), - } - } - - fn query_env(&self, env_key: &str) -> Result> { - match std::env::var(env_key) { - Err(std::env::VarError::NotPresent) => self.get_dotenv(env_key), - other => other - .map(Some) - .with_context(|| format!("failed to resolve env var {env_key}")), - } - } - - fn get_sync(&self, key: &Key) -> Result> { - let prefix = self - .prefix - .clone() - .unwrap_or(DEFAULT_ENV_PREFIX.to_string()); - let use_fallback = self.prefix.is_none(); - - let upper_key = key.as_ref().to_ascii_uppercase(); - let env_key = format!("{prefix}_{upper_key}"); - - match self.query_env(&env_key)? { - None if use_fallback => { - let old_key = format!("{LEGACY_ENV_PREFIX}_{upper_key}"); - let result = self.query_env(&old_key); - if let Ok(Some(_)) = &result { - eprintln!("Warning: variable '{key}': {env_key} was not set, so used {old_key}. The {LEGACY_ENV_PREFIX} prefix is deprecated; please switch to the {DEFAULT_ENV_PREFIX} prefix.", key = key.as_ref()); - } - result - } - other => Ok(other), - } - } - - fn get_dotenv(&self, key: &str) -> Result> { - if self.dotenv_path.is_none() { - return Ok(None); - } - let mut maybe_cache = self - .dotenv_cache - .lock() - .expect("dotenv_cache lock poisoned"); - let cache = match maybe_cache.as_mut() { - Some(cache) => cache, - None => maybe_cache.insert(self.load_dotenv()?), - }; - Ok(cache.get(key).cloned()) - } - - fn load_dotenv(&self) -> Result> { - let path = self.dotenv_path.as_deref().unwrap(); - Ok(dotenvy::from_path_iter(path) - .into_iter() - .flatten() - .collect::, _>>()?) - } -} - -#[async_trait] -impl Provider for EnvProvider { - #[instrument(name = "spin_variables.get_from_env", skip(self), err(level = Level::INFO))] - async fn get(&self, key: &Key) -> Result> { - tokio::task::block_in_place(|| self.get_sync(key)) - } -} - -#[cfg(test)] -mod test { - use std::env::temp_dir; - - use super::*; - - #[test] - fn provider_get() { - std::env::set_var("TESTING_SPIN_ENV_KEY1", "val"); - let key1 = Key::new("env_key1").unwrap(); - let mut envs = HashMap::new(); - envs.insert( - "TESTING_SPIN_ENV_KEY1".to_string(), - "dotenv_val".to_string(), - ); - assert_eq!( - EnvProvider::new(Some("TESTING_SPIN"), None) - .get_sync(&key1) - .unwrap(), - Some("val".to_string()) - ); - } - - #[test] - fn provider_get_dotenv() { - let dotenv_path = temp_dir().join("spin-env-provider-test"); - std::fs::write(&dotenv_path, b"TESTING_SPIN_ENV_KEY2=dotenv_val").unwrap(); - - let key = Key::new("env_key2").unwrap(); - assert_eq!( - EnvProvider::new(Some("TESTING_SPIN"), Some(dotenv_path)) - .get_sync(&key) - .unwrap(), - Some("dotenv_val".to_string()) - ); - } - - #[test] - fn provider_get_missing() { - let key = Key::new("please_do_not_ever_set_this_during_tests").unwrap(); - assert_eq!( - EnvProvider::new(Some("TESTING_SPIN"), Default::default()) - .get_sync(&key) - .unwrap(), - None - ); - } -} diff --git a/crates/variables/src/provider/vault.rs b/crates/variables/src/provider/vault.rs deleted file mode 100644 index d48505c38..000000000 --- a/crates/variables/src/provider/vault.rs +++ /dev/null @@ -1,65 +0,0 @@ -use anyhow::{Context, Result}; -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use tracing::{instrument, Level}; -use vaultrs::{ - client::{VaultClient, VaultClientSettingsBuilder}, - error::ClientError, - kv2, -}; - -use spin_expressions::{Key, Provider}; - -/// A config Provider that uses HashiCorp Vault. -#[derive(Debug)] -pub struct VaultProvider { - url: String, - token: String, - mount: String, - prefix: Option, -} - -impl VaultProvider { - pub fn new( - url: impl Into, - token: impl Into, - mount: impl Into, - prefix: Option>, - ) -> Self { - Self { - url: url.into(), - token: token.into(), - mount: mount.into(), - prefix: prefix.map(Into::into), - } - } -} - -#[derive(Deserialize, Serialize)] -struct Secret { - value: String, -} - -#[async_trait] -impl Provider for VaultProvider { - #[instrument(name = "spin_variables.get_from_vault", skip(self), err(level = Level::INFO), fields(otel.kind = "client"))] - async fn get(&self, key: &Key) -> Result> { - let client = VaultClient::new( - VaultClientSettingsBuilder::default() - .address(&self.url) - .token(&self.token) - .build()?, - )?; - let path = match &self.prefix { - Some(prefix) => format!("{}/{}", prefix, key.as_str()), - None => key.as_str().to_string(), - }; - match kv2::read::(&client, &self.mount, &path).await { - Ok(secret) => Ok(Some(secret.value)), - // Vault doesn't have this entry so pass along the chain - Err(ClientError::APIError { code: 404, .. }) => Ok(None), - // Other Vault error so bail rather than looking elsewhere - Err(e) => Err(e).context("Failed to check Vault for config"), - } - } -} diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index 425b5c4b7..9a04ac97f 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -5730,12 +5730,9 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "http 0.2.11", - "llm", "reqwest 0.11.24", "serde 1.0.203", "serde_json", - "spin-core", - "spin-llm", "spin-telemetry", "spin-world", "tracing", diff --git a/examples/spin-timer/Cargo.toml b/examples/spin-timer/Cargo.toml index 43472f0ec..ec535dcf0 100644 --- a/examples/spin-timer/Cargo.toml +++ b/examples/spin-timer/Cargo.toml @@ -11,7 +11,7 @@ futures = "0.3.25" serde = "1.0.188" spin-app = { path = "../../crates/app" } spin-core = { path = "../../crates/core" } -spin-trigger = { path = "../../crates/trigger" } +# spin-trigger = { path = "../../crates/trigger" } tokio = { version = "1.11", features = ["full"] } tokio-scoped = "0.2.0" wasmtime = "22.0.0" From 6f54439b1588b852e4b7845c969d9ca8928796c8 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 22 Aug 2024 15:24:09 -0400 Subject: [PATCH 159/195] factors: Port redis trigger Signed-off-by: Lann Martin --- Cargo.lock | 88 +++++++++-- crates/expressions/src/lib.rs | 2 +- crates/factor-variables/src/lib.rs | 12 +- crates/factors-executor/src/lib.rs | 4 + crates/trigger-http2/src/lib.rs | 4 +- crates/trigger-redis/Cargo.toml | 24 +++ crates/trigger-redis/src/lib.rs | 226 +++++++++++++++++++++++++++++ crates/trigger2/src/cli.rs | 4 +- crates/trigger2/src/lib.rs | 11 +- 9 files changed, 347 insertions(+), 28 deletions(-) create mode 100644 crates/trigger-redis/Cargo.toml create mode 100644 crates/trigger-redis/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index d6a8e17f3..6b4b3bfcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,6 +181,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayvec" version = "0.5.2" @@ -479,9 +485,9 @@ checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", @@ -1737,7 +1743,7 @@ dependencies = [ "bitflags 1.3.2", "crossterm_winapi", "libc", - "mio", + "mio 0.8.11", "parking_lot", "signal-hook", "signal-hook-mio", @@ -4493,6 +4499,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + [[package]] name = "mirai-annotations" version = "1.12.0" @@ -4541,7 +4559,7 @@ dependencies = [ "keyed_priority_queue", "lazy_static 1.4.0", "lru 0.12.3", - "mio", + "mio 0.8.11", "mysql_common", "native-tls", "once_cell", @@ -4712,7 +4730,7 @@ dependencies = [ "inotify", "kqueue", "libc", - "mio", + "mio 0.8.11", "walkdir", "windows-sys 0.45.0", ] @@ -4743,11 +4761,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits 0.2.18", ] @@ -6035,6 +6052,29 @@ dependencies = [ "url", ] +[[package]] +name = "redis" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e902a69d09078829137b4a5d9d082e0490393537badd7c91a3d69d14639e115f" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2 0.5.6", + "tokio", + "tokio-util 0.7.10", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -6960,7 +7000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio", + "mio 0.8.11", "signal-hook", ] @@ -8042,6 +8082,23 @@ dependencies = [ "webpki-roots 0.26.1", ] +[[package]] +name = "spin-trigger-redis" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "redis 0.26.1", + "serde 1.0.197", + "spin-factor-variables", + "spin-telemetry", + "spin-trigger2", + "spin-world", + "tokio", + "tracing", +] + [[package]] name = "spin-trigger2" version = "2.8.0-pre0" @@ -8588,21 +8645,20 @@ dependencies = [ [[package]] name = "tokio" -version = "1.37.0" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", "libc", - "mio", - "num_cpus", + "mio 1.0.2", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.5.6", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -8617,9 +8673,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", diff --git a/crates/expressions/src/lib.rs b/crates/expressions/src/lib.rs index 811c0e864..350914b55 100644 --- a/crates/expressions/src/lib.rs +++ b/crates/expressions/src/lib.rs @@ -267,7 +267,7 @@ impl<'a> AsRef for Key<'a> { } } -type Result = std::result::Result; +pub type Result = std::result::Result; /// A variable resolution error. #[derive(Debug, thiserror::Error)] diff --git a/crates/factor-variables/src/lib.rs b/crates/factor-variables/src/lib.rs index 9ea892a2b..3c9b6603a 100644 --- a/crates/factor-variables/src/lib.rs +++ b/crates/factor-variables/src/lib.rs @@ -5,7 +5,7 @@ pub mod spin_cli; use std::sync::Arc; use runtime_config::RuntimeConfig; -use spin_expressions::ProviderResolver as ExpressionResolver; +use spin_expressions::{ProviderResolver as ExpressionResolver, Template}; use spin_factors::{ anyhow, ConfigureAppContext, Factor, InitContext, InstanceBuilders, PrepareContext, RuntimeFactors, SelfInstanceBuilder, @@ -70,6 +70,16 @@ pub struct AppState { expression_resolver: Arc, } +impl AppState { + pub async fn resolve_expression( + &self, + expr: impl Into>, + ) -> spin_expressions::Result { + let template = Template::new(expr)?; + self.expression_resolver.resolve_template(&template).await + } +} + pub struct InstanceState { component_id: String, expression_resolver: Arc, diff --git a/crates/factors-executor/src/lib.rs b/crates/factors-executor/src/lib.rs index 2d1c00330..f204dd1a7 100644 --- a/crates/factors-executor/src/lib.rs +++ b/crates/factors-executor/src/lib.rs @@ -115,6 +115,10 @@ impl FactorsExecutorApp { &self.executor.core_engine } + pub fn configured_app(&self) -> &ConfiguredApp { + &self.configured_app + } + pub fn app(&self) -> &App { self.configured_app.app() } diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http2/src/lib.rs index 994911099..705c522f2 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http2/src/lib.rs @@ -60,8 +60,6 @@ impl CliArgs { } } -pub(crate) type InstanceState = (); - /// The Spin HTTP trigger. pub struct HttpTrigger { /// The address the server should listen on. @@ -76,7 +74,7 @@ impl Trigger for HttpTrigger { const TYPE: &'static str = "http"; type CliArgs = CliArgs; - type InstanceState = InstanceState; + type InstanceState = (); fn new(cli_args: Self::CliArgs, app: &spin_app::App) -> anyhow::Result { Self::new(app, cli_args.address, cli_args.into_tls_config()) diff --git a/crates/trigger-redis/Cargo.toml b/crates/trigger-redis/Cargo.toml new file mode 100644 index 000000000..c2cba3a57 --- /dev/null +++ b/crates/trigger-redis/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "spin-trigger-redis" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[lib] +doctest = false + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +futures = "0.3" +serde = "1.0.188" +spin-factor-variables = { path = "../factor-variables" } +spin-telemetry = { path = "../telemetry" } +spin-trigger2 = { path = "../trigger2" } +spin-world = { path = "../world" } +redis = { version = "0.26.1", features = ["tokio-comp"] } +tracing = { workspace = true } +tokio = { version = "1.39.3", features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/crates/trigger-redis/src/lib.rs b/crates/trigger-redis/src/lib.rs new file mode 100644 index 000000000..d0d3fbbda --- /dev/null +++ b/crates/trigger-redis/src/lib.rs @@ -0,0 +1,226 @@ +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Context; +use futures::{StreamExt, TryFutureExt}; +use redis::{Client, Msg}; +use serde::Deserialize; +use spin_factor_variables::VariablesFactor; +use spin_trigger2::{cli::NoCliArgs, App, Trigger, TriggerApp}; +use spin_world::exports::fermyon::spin::inbound_redis; +use tracing::{instrument, Level}; + +pub struct RedisTrigger; + +/// Redis trigger metadata. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct TriggerMetadata { + address: String, +} + +/// Redis trigger configuration. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct TriggerConfig { + /// Component ID to invoke + component: String, + /// Channel to subscribe to + channel: String, + /// Optionally override address for trigger + address: Option, +} + +impl Trigger for RedisTrigger { + const TYPE: &'static str = "redis"; + + type CliArgs = NoCliArgs; + + type InstanceState = (); + + fn new(_cli_args: Self::CliArgs, _app: &App) -> anyhow::Result { + Ok(Self) + } + + async fn run(self, trigger_app: spin_trigger2::TriggerApp) -> anyhow::Result<()> { + let app_variables = trigger_app + .configured_app() + .app_state::() + .context("RedisTrigger depends on VariablesFactor")?; + + let app = trigger_app.app(); + let metadata = app + .get_trigger_metadata::(Self::TYPE)? + .unwrap_or_default(); + let default_address_expr = &metadata.address; + let default_address = app_variables + .resolve_expression(default_address_expr.clone()) + .await + .with_context(|| { + format!("failed to resolve redis trigger default address {default_address_expr:?}") + })?; + + // Maps -> -> + let mut server_channel_components: HashMap = HashMap::new(); + + // Resolve trigger configs before starting any subscribers + for (_, config) in app + .trigger_configs::(Self::TYPE)? + .into_iter() + .collect::>() + { + let component_id = config.component; + + let address_expr = config.address.as_ref().unwrap_or(&default_address); + let address = app_variables + .resolve_expression(address_expr.clone()) + .await + .with_context(|| { + format!( + "failed to resolve redis trigger address {address_expr:?} for component {component_id}" + ) + })?; + + let channel_expr = &config.channel; + let channel = app_variables + .resolve_expression(channel_expr.clone()) + .await + .with_context(|| { + format!( + "failed to resolve redis trigger channel {channel_expr:?} for component {component_id}" + ) + })?; + + server_channel_components + .entry(address) + .or_default() + .entry(channel) + .or_default() + .push(component_id); + } + + // Start subscriber(s) + let trigger_app = Arc::new(trigger_app); + let mut subscriber_tasks = Vec::new(); + for (address, channel_components) in server_channel_components { + let subscriber = Subscriber::new(address, trigger_app.clone(), channel_components)?; + let task = tokio::spawn(subscriber.run_listener()); + subscriber_tasks.push(task); + } + + // Wait for any task to complete + let (res, _, _) = futures::future::select_all(subscriber_tasks).await; + res? + } +} + +/// Maps -> +type ChannelComponents = HashMap>; + +/// Subscribes to channels from a single Redis server. +struct Subscriber { + client: Client, + trigger_app: Arc>, + channel_components: ChannelComponents, +} + +impl Subscriber { + fn new( + address: String, + trigger_app: Arc>, + channel_components: ChannelComponents, + ) -> anyhow::Result { + let client = Client::open(address)?; + Ok(Self { + client, + trigger_app, + channel_components, + }) + } + + async fn run_listener(self) -> anyhow::Result<()> { + let server_addr = &self.client.get_connection_info().addr; + + tracing::info!("Connecting to Redis server at {server_addr}"); + let mut pubsub = self + .client + .get_async_pubsub() + .await + .with_context(|| format!("Redis trigger failed to connect to {server_addr}"))?; + + println!("Active Channels on {server_addr}:"); + + // Subscribe to channels + for (channel, components) in &self.channel_components { + tracing::info!("Subscribing to {channel:?} on {server_addr}"); + pubsub.subscribe(channel).await.with_context(|| { + format!("Redis trigger failed to subscribe to channel {channel:?} on {server_addr}") + })?; + println!("\t{server_addr}/{channel}: [{}]", components.join(",")); + } + + let mut message_stream = pubsub.on_message(); + while let Some(msg) = message_stream.next().await { + if let Err(err) = self.handle_message(msg).await { + tracing::error!("Error handling message from {server_addr}: {err}"); + } + } + Err(anyhow::anyhow!("disconnected from {server_addr}")) + } + + #[instrument(name = "spin_trigger_redis.handle_message", skip_all, err(level = Level::INFO), fields( + otel.name = format!("{} receive", msg.get_channel_name()), + otel.kind = "consumer", + messaging.operation = "receive", + messaging.system = "redis" + ))] + async fn handle_message(&self, msg: Msg) -> anyhow::Result<()> { + let server_addr = &self.client.get_connection_info().addr; + let channel = msg.get_channel_name(); + tracing::trace!(%server_addr, %channel, "Received message"); + + let Some(component_ids) = self.channel_components.get(channel) else { + anyhow::bail!("message from unexpected channel {channel:?}"); + }; + + let dispatch_futures = component_ids.iter().map(|component_id| { + tracing::trace!("Executing Redis component {component_id}"); + self.dispatch_handler(&msg, component_id) + .inspect_err(move |err| { + tracing::info!("Component {component_id} handler failed: {err}"); + }) + }); + futures::future::join_all(dispatch_futures).await; + + Ok(()) + } + + async fn dispatch_handler(&self, msg: &Msg, component_id: &str) -> anyhow::Result<()> { + spin_telemetry::metrics::monotonic_counter!( + spin.request_count = 1, + trigger_type = "redis", + app_id = self.trigger_app.app().id(), + component_id = component_id + ); + + let (instance, mut store) = self + .trigger_app + .prepare(component_id)? + .instantiate(()) + .await?; + + let guest = { + let exports = &mut instance.exports(&mut store); + let mut inbound_redis_export = exports + .instance("fermyon:spin/inbound-redis") + .context("no fermyon:spin/inbound-redis instance found")?; + inbound_redis::Guest::new(&mut inbound_redis_export)? + }; + + let payload = msg.get_payload_bytes().to_vec(); + + guest + .call_handle_message(&mut store, &payload) + .await? + .context("Redis handler returned an error") + } +} diff --git a/crates/trigger2/src/cli.rs b/crates/trigger2/src/cli.rs index 9bc787cef..4b6dc422b 100644 --- a/crates/trigger2/src/cli.rs +++ b/crates/trigger2/src/cli.rs @@ -130,7 +130,7 @@ pub struct FactorsTriggerCommand { /// An empty implementation of clap::Args to be used as TriggerExecutor::RunConfig /// for executors that do not need additional CLI args. #[derive(Args)] -pub struct NoArgs; +pub struct NoCliArgs; impl FactorsTriggerCommand { /// Create a new TriggerExecutorBuilder from this TriggerExecutorCommand. @@ -402,7 +402,7 @@ pub mod help { impl Trigger for HelpArgsOnlyTrigger { const TYPE: &'static str = "help-args-only"; - type CliArgs = NoArgs; + type CliArgs = NoCliArgs; type InstanceState = (); fn new(_cli_args: Self::CliArgs, _app: &App) -> anyhow::Result { diff --git a/crates/trigger2/src/lib.rs b/crates/trigger2/src/lib.rs index 095d30e3b..0fde4a021 100644 --- a/crates/trigger2/src/lib.rs +++ b/crates/trigger2/src/lib.rs @@ -1,14 +1,15 @@ +pub mod cli; +mod factors; +mod stdio; + use std::future::Future; use clap::Args; use factors::{TriggerFactors, TriggerFactorsInstanceState}; -use spin_app::App; use spin_core::Linker; use spin_factors_executor::{FactorsExecutorApp, FactorsInstanceBuilder}; -pub mod cli; -mod factors; -mod stdio; +pub use spin_app::App; /// Type alias for a [`FactorsConfiguredApp`] specialized to a [`Trigger`]. pub type TriggerApp = FactorsExecutorApp::InstanceState>; @@ -54,7 +55,7 @@ pub trait Trigger: Sized + Send { /// Run this trigger. fn run( self, - configured_app: TriggerApp, + trigger_app: TriggerApp, ) -> impl Future> + Send; /// Returns a list of host requirements supported by this trigger specifically. From 381bd802c91bc636286828488df5b8979390e7e4 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 22 Aug 2024 15:29:37 -0400 Subject: [PATCH 160/195] Reintroduce redis trigger to spin up Signed-off-by: Lann Martin --- Cargo.lock | 1 + Cargo.toml | 2 +- build.rs | 1 - src/bin/spin.rs | 6 +++--- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b4b3bfcd..b5da52fe9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7237,6 +7237,7 @@ dependencies = [ "spin-telemetry", "spin-templates", "spin-trigger-http2", + "spin-trigger-redis", "spin-trigger2", "subprocess", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 0f270d3b3..664d92b06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ spin-telemetry = { path = "crates/telemetry", features = [ spin-templates = { path = "crates/templates" } spin-trigger2 = { path = "crates/trigger2" } spin-trigger-http2 = { path = "crates/trigger-http2" } -# TODO: spin-trigger-redis = { path = "crates/trigger-redis" } +spin-trigger-redis = { path = "crates/trigger-redis" } tempfile = "3.8.0" tokio = { version = "1.23", features = ["full"] } diff --git a/build.rs b/build.rs index d4cd7acf0..3246e3ec7 100644 --- a/build.rs +++ b/build.rs @@ -69,7 +69,6 @@ error: the `wasm32-wasi` target is not installed build_wasm_test_program("core-wasi-test.wasm", "crates/core/tests/core-wasi-test"); // build_wasm_test_program("redis-rust.wasm", "crates/trigger-redis/tests/rust"); - // build_wasm_test_program( // "spin-http-benchmark.wasm", // "crates/trigger-http/benches/spin-http-benchmark", diff --git a/src/bin/spin.rs b/src/bin/spin.rs index 42e6539dc..3cce1b3ca 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -18,7 +18,7 @@ use spin_cli::{build_info::*, subprocess::ExitStatusError}; use spin_trigger2::cli::help::HelpArgsOnlyTrigger; use spin_trigger2::cli::FactorsTriggerCommand; use spin_trigger_http2::HttpTrigger; -// TODO(factors): use spin_trigger_redis::RedisTrigger; +use spin_trigger_redis::RedisTrigger; #[tokio::main] async fn main() { @@ -137,7 +137,7 @@ enum SpinApp { #[derive(Subcommand)] enum TriggerCommands { Http(FactorsTriggerCommand), - // TODO(factors): Redis(TriggerExecutorCommand), + Redis(FactorsTriggerCommand), #[clap(name = spin_cli::HELP_ARGS_ONLY_TRIGGER_TYPE, hide = true)] HelpArgsOnly(FactorsTriggerCommand), } @@ -155,7 +155,7 @@ impl SpinApp { Self::Registry(cmd) => cmd.run().await, Self::Build(cmd) => cmd.run().await, Self::Trigger(TriggerCommands::Http(cmd)) => cmd.run().await, - // TODO(factors): Self::Trigger(TriggerCommands::Redis(cmd)) => cmd.run().await, + Self::Trigger(TriggerCommands::Redis(cmd)) => cmd.run().await, Self::Trigger(TriggerCommands::HelpArgsOnly(cmd)) => cmd.run().await, Self::Plugins(cmd) => cmd.run().await, Self::External(cmd) => execute_external_subcommand(cmd, app).await, From 22d136d1ea2ac438cfa0eebcafa733f153ffb861 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 23 Aug 2024 10:55:02 +0200 Subject: [PATCH 161/195] It's just trigger and http-trigger - no 2 for you Signed-off-by: Ryan Levick --- Cargo.lock | 84 +++++++++---------- Cargo.toml | 10 +-- .../Cargo.toml | 10 +-- .../src/headers.rs | 0 .../src/instrument.rs | 0 .../src/lib.rs | 6 +- .../src/outbound_http.rs | 0 .../src/server.rs | 0 .../src/spin.rs | 0 .../src/tls.rs | 0 .../src/wagi.rs | 0 .../src/wasi.rs | 0 .../testdata/invalid-cert.pem | 0 .../testdata/invalid-private-key.pem | 0 .../testdata/valid-cert.pem | 0 .../testdata/valid-private-key.pem | 0 crates/trigger-redis/Cargo.toml | 2 +- crates/trigger-redis/src/lib.rs | 4 +- crates/{trigger2 => trigger}/Cargo.toml | 2 +- crates/{trigger2 => trigger}/src/cli.rs | 0 .../src/cli/launch_metadata.rs | 0 crates/{trigger2 => trigger}/src/factors.rs | 0 crates/{trigger2 => trigger}/src/lib.rs | 0 crates/{trigger2 => trigger}/src/stdio.rs | 0 run-factors-tests.sh | 2 +- src/bin/spin.rs | 6 +- src/commands/up.rs | 2 +- tests/testing-framework/Cargo.toml | 4 +- .../src/runtimes/in_process_spin.rs | 4 +- 29 files changed, 68 insertions(+), 68 deletions(-) rename crates/{trigger-http2 => trigger-http}/Cargo.toml (88%) rename crates/{trigger-http2 => trigger-http}/src/headers.rs (100%) rename crates/{trigger-http2 => trigger-http}/src/instrument.rs (100%) rename crates/{trigger-http2 => trigger-http}/src/lib.rs (96%) rename crates/{trigger-http2 => trigger-http}/src/outbound_http.rs (100%) rename crates/{trigger-http2 => trigger-http}/src/server.rs (100%) rename crates/{trigger-http2 => trigger-http}/src/spin.rs (100%) rename crates/{trigger-http2 => trigger-http}/src/tls.rs (100%) rename crates/{trigger-http2 => trigger-http}/src/wagi.rs (100%) rename crates/{trigger-http2 => trigger-http}/src/wasi.rs (100%) rename crates/{trigger-http2 => trigger-http}/testdata/invalid-cert.pem (100%) rename crates/{trigger-http2 => trigger-http}/testdata/invalid-private-key.pem (100%) rename crates/{trigger-http2 => trigger-http}/testdata/valid-cert.pem (100%) rename crates/{trigger-http2 => trigger-http}/testdata/valid-private-key.pem (100%) rename crates/{trigger2 => trigger}/Cargo.toml (98%) rename crates/{trigger2 => trigger}/src/cli.rs (100%) rename crates/{trigger2 => trigger}/src/cli/launch_metadata.rs (100%) rename crates/{trigger2 => trigger}/src/factors.rs (100%) rename crates/{trigger2 => trigger}/src/lib.rs (100%) rename crates/{trigger2 => trigger}/src/stdio.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index b5da52fe9..52fa76a10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7236,9 +7236,9 @@ dependencies = [ "spin-plugins", "spin-telemetry", "spin-templates", - "spin-trigger-http2", + "spin-trigger", + "spin-trigger-http", "spin-trigger-redis", - "spin-trigger2", "subprocess", "tempfile", "terminal", @@ -8043,7 +8043,42 @@ dependencies = [ ] [[package]] -name = "spin-trigger-http2" +name = "spin-trigger" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "clap 3.2.25", + "ctrlc", + "futures", + "sanitize-filename", + "serde 1.0.197", + "serde_json", + "spin-app", + "spin-common", + "spin-componentize", + "spin-core", + "spin-factor-key-value", + "spin-factor-llm", + "spin-factor-outbound-http", + "spin-factor-outbound-mqtt", + "spin-factor-outbound-mysql", + "spin-factor-outbound-networking", + "spin-factor-outbound-pg", + "spin-factor-outbound-redis", + "spin-factor-sqlite", + "spin-factor-variables", + "spin-factor-wasi", + "spin-factors", + "spin-factors-executor", + "spin-runtime-config", + "spin-telemetry", + "terminal", + "tokio", + "tracing", +] + +[[package]] +name = "spin-trigger-http" version = "2.8.0-pre0" dependencies = [ "anyhow", @@ -8069,7 +8104,7 @@ dependencies = [ "spin-http", "spin-outbound-networking", "spin-telemetry", - "spin-trigger2", + "spin-trigger", "spin-world", "terminal", "tls-listener", @@ -8094,47 +8129,12 @@ dependencies = [ "serde 1.0.197", "spin-factor-variables", "spin-telemetry", - "spin-trigger2", + "spin-trigger", "spin-world", "tokio", "tracing", ] -[[package]] -name = "spin-trigger2" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "clap 3.2.25", - "ctrlc", - "futures", - "sanitize-filename", - "serde 1.0.197", - "serde_json", - "spin-app", - "spin-common", - "spin-componentize", - "spin-core", - "spin-factor-key-value", - "spin-factor-llm", - "spin-factor-outbound-http", - "spin-factor-outbound-mqtt", - "spin-factor-outbound-mysql", - "spin-factor-outbound-networking", - "spin-factor-outbound-pg", - "spin-factor-outbound-redis", - "spin-factor-sqlite", - "spin-factor-variables", - "spin-factor-wasi", - "spin-factors", - "spin-factors-executor", - "spin-runtime-config", - "spin-telemetry", - "terminal", - "tokio", - "tracing", -] - [[package]] name = "spin-world" version = "2.8.0-pre0" @@ -8483,8 +8483,8 @@ dependencies = [ "spin-factors-executor", "spin-http", "spin-loader", - "spin-trigger-http2", - "spin-trigger2", + "spin-trigger", + "spin-trigger-http", "temp-dir", "test-environment", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 664d92b06..089e4f1af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,8 +62,8 @@ spin-telemetry = { path = "crates/telemetry", features = [ "tracing-log-compat", ] } spin-templates = { path = "crates/templates" } -spin-trigger2 = { path = "crates/trigger2" } -spin-trigger-http2 = { path = "crates/trigger-http2" } +spin-trigger = { path = "crates/trigger" } +spin-trigger-http = { path = "crates/trigger-http" } spin-trigger-redis = { path = "crates/trigger-redis" } tempfile = "3.8.0" @@ -113,9 +113,9 @@ wit-component = "0.19.0" # TODO(factors): default = ["llm"] all-tests = ["extern-dependencies-tests"] extern-dependencies-tests = [] -llm = ["spin-trigger-http2/llm"] -llm-metal = ["llm", "spin-trigger-http2/llm-metal"] -llm-cublas = ["llm", "spin-trigger-http2/llm-cublas"] +llm = ["spin-trigger-http/llm"] +llm-metal = ["llm", "spin-trigger-http/llm-metal"] +llm-cublas = ["llm", "spin-trigger-http/llm-cublas"] [workspace] members = [ diff --git a/crates/trigger-http2/Cargo.toml b/crates/trigger-http/Cargo.toml similarity index 88% rename from crates/trigger-http2/Cargo.toml rename to crates/trigger-http/Cargo.toml index d79b7b520..65030acc6 100644 --- a/crates/trigger-http2/Cargo.toml +++ b/crates/trigger-http/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "spin-trigger-http2" +name = "spin-trigger-http" version = { workspace = true } authors = { workspace = true } edition = { workspace = true } @@ -8,9 +8,9 @@ edition = { workspace = true } doctest = false [features] -llm = ["spin-trigger2/llm"] -llm-metal = ["spin-trigger2/llm-metal"] -llm-cublas = ["spin-trigger2/llm-cublas"] +llm = ["spin-trigger/llm"] +llm-metal = ["spin-trigger/llm-metal"] +llm-cublas = ["spin-trigger/llm-cublas"] [dependencies] anyhow = "1.0" @@ -36,7 +36,7 @@ spin-factor-wasi = { path = "../factor-wasi" } spin-http = { path = "../http" } spin-outbound-networking = { path = "../outbound-networking" } spin-telemetry = { path = "../telemetry" } -spin-trigger2 = { path = "../trigger2" } +spin-trigger = { path = "../trigger" } spin-world = { path = "../world" } terminal = { path = "../terminal" } tls-listener = { version = "0.10.0", features = ["rustls"] } diff --git a/crates/trigger-http2/src/headers.rs b/crates/trigger-http/src/headers.rs similarity index 100% rename from crates/trigger-http2/src/headers.rs rename to crates/trigger-http/src/headers.rs diff --git a/crates/trigger-http2/src/instrument.rs b/crates/trigger-http/src/instrument.rs similarity index 100% rename from crates/trigger-http2/src/instrument.rs rename to crates/trigger-http/src/instrument.rs diff --git a/crates/trigger-http2/src/lib.rs b/crates/trigger-http/src/lib.rs similarity index 96% rename from crates/trigger-http2/src/lib.rs rename to crates/trigger-http/src/lib.rs index 705c522f2..636bff940 100644 --- a/crates/trigger-http2/src/lib.rs +++ b/crates/trigger-http/src/lib.rs @@ -20,7 +20,7 @@ use anyhow::{bail, Context}; use clap::Args; use serde::Deserialize; use spin_app::App; -use spin_trigger2::Trigger; +use spin_trigger::Trigger; use wasmtime_wasi_http::bindings::wasi::http::types::ErrorCode; pub use server::HttpServer; @@ -29,8 +29,8 @@ pub use tls::TlsConfig; pub(crate) use wasmtime_wasi_http::body::HyperIncomingBody as Body; -pub(crate) type TriggerApp = spin_trigger2::TriggerApp; -pub(crate) type TriggerInstanceBuilder<'a> = spin_trigger2::TriggerInstanceBuilder<'a, HttpTrigger>; +pub(crate) type TriggerApp = spin_trigger::TriggerApp; +pub(crate) type TriggerInstanceBuilder<'a> = spin_trigger::TriggerInstanceBuilder<'a, HttpTrigger>; #[derive(Args)] pub struct CliArgs { diff --git a/crates/trigger-http2/src/outbound_http.rs b/crates/trigger-http/src/outbound_http.rs similarity index 100% rename from crates/trigger-http2/src/outbound_http.rs rename to crates/trigger-http/src/outbound_http.rs diff --git a/crates/trigger-http2/src/server.rs b/crates/trigger-http/src/server.rs similarity index 100% rename from crates/trigger-http2/src/server.rs rename to crates/trigger-http/src/server.rs diff --git a/crates/trigger-http2/src/spin.rs b/crates/trigger-http/src/spin.rs similarity index 100% rename from crates/trigger-http2/src/spin.rs rename to crates/trigger-http/src/spin.rs diff --git a/crates/trigger-http2/src/tls.rs b/crates/trigger-http/src/tls.rs similarity index 100% rename from crates/trigger-http2/src/tls.rs rename to crates/trigger-http/src/tls.rs diff --git a/crates/trigger-http2/src/wagi.rs b/crates/trigger-http/src/wagi.rs similarity index 100% rename from crates/trigger-http2/src/wagi.rs rename to crates/trigger-http/src/wagi.rs diff --git a/crates/trigger-http2/src/wasi.rs b/crates/trigger-http/src/wasi.rs similarity index 100% rename from crates/trigger-http2/src/wasi.rs rename to crates/trigger-http/src/wasi.rs diff --git a/crates/trigger-http2/testdata/invalid-cert.pem b/crates/trigger-http/testdata/invalid-cert.pem similarity index 100% rename from crates/trigger-http2/testdata/invalid-cert.pem rename to crates/trigger-http/testdata/invalid-cert.pem diff --git a/crates/trigger-http2/testdata/invalid-private-key.pem b/crates/trigger-http/testdata/invalid-private-key.pem similarity index 100% rename from crates/trigger-http2/testdata/invalid-private-key.pem rename to crates/trigger-http/testdata/invalid-private-key.pem diff --git a/crates/trigger-http2/testdata/valid-cert.pem b/crates/trigger-http/testdata/valid-cert.pem similarity index 100% rename from crates/trigger-http2/testdata/valid-cert.pem rename to crates/trigger-http/testdata/valid-cert.pem diff --git a/crates/trigger-http2/testdata/valid-private-key.pem b/crates/trigger-http/testdata/valid-private-key.pem similarity index 100% rename from crates/trigger-http2/testdata/valid-private-key.pem rename to crates/trigger-http/testdata/valid-private-key.pem diff --git a/crates/trigger-redis/Cargo.toml b/crates/trigger-redis/Cargo.toml index c2cba3a57..0ace95427 100644 --- a/crates/trigger-redis/Cargo.toml +++ b/crates/trigger-redis/Cargo.toml @@ -14,7 +14,7 @@ futures = "0.3" serde = "1.0.188" spin-factor-variables = { path = "../factor-variables" } spin-telemetry = { path = "../telemetry" } -spin-trigger2 = { path = "../trigger2" } +spin-trigger = { path = "../trigger" } spin-world = { path = "../world" } redis = { version = "0.26.1", features = ["tokio-comp"] } tracing = { workspace = true } diff --git a/crates/trigger-redis/src/lib.rs b/crates/trigger-redis/src/lib.rs index d0d3fbbda..5932b8939 100644 --- a/crates/trigger-redis/src/lib.rs +++ b/crates/trigger-redis/src/lib.rs @@ -5,7 +5,7 @@ use futures::{StreamExt, TryFutureExt}; use redis::{Client, Msg}; use serde::Deserialize; use spin_factor_variables::VariablesFactor; -use spin_trigger2::{cli::NoCliArgs, App, Trigger, TriggerApp}; +use spin_trigger::{cli::NoCliArgs, App, Trigger, TriggerApp}; use spin_world::exports::fermyon::spin::inbound_redis; use tracing::{instrument, Level}; @@ -41,7 +41,7 @@ impl Trigger for RedisTrigger { Ok(Self) } - async fn run(self, trigger_app: spin_trigger2::TriggerApp) -> anyhow::Result<()> { + async fn run(self, trigger_app: spin_trigger::TriggerApp) -> anyhow::Result<()> { let app_variables = trigger_app .configured_app() .app_state::() diff --git a/crates/trigger2/Cargo.toml b/crates/trigger/Cargo.toml similarity index 98% rename from crates/trigger2/Cargo.toml rename to crates/trigger/Cargo.toml index 565ee436b..733a33530 100644 --- a/crates/trigger2/Cargo.toml +++ b/crates/trigger/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "spin-trigger2" +name = "spin-trigger" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/crates/trigger2/src/cli.rs b/crates/trigger/src/cli.rs similarity index 100% rename from crates/trigger2/src/cli.rs rename to crates/trigger/src/cli.rs diff --git a/crates/trigger2/src/cli/launch_metadata.rs b/crates/trigger/src/cli/launch_metadata.rs similarity index 100% rename from crates/trigger2/src/cli/launch_metadata.rs rename to crates/trigger/src/cli/launch_metadata.rs diff --git a/crates/trigger2/src/factors.rs b/crates/trigger/src/factors.rs similarity index 100% rename from crates/trigger2/src/factors.rs rename to crates/trigger/src/factors.rs diff --git a/crates/trigger2/src/lib.rs b/crates/trigger/src/lib.rs similarity index 100% rename from crates/trigger2/src/lib.rs rename to crates/trigger/src/lib.rs diff --git a/crates/trigger2/src/stdio.rs b/crates/trigger/src/stdio.rs similarity index 100% rename from crates/trigger2/src/stdio.rs rename to crates/trigger/src/stdio.rs diff --git a/run-factors-tests.sh b/run-factors-tests.sh index 3b9ca4e42..26fe7b4b7 100755 --- a/run-factors-tests.sh +++ b/run-factors-tests.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # TODO(factors): Remove after enabling CI for factors branch -cargo test -p '*factor*' -p spin-trigger2 -p spin-trigger-http2 +cargo test -p '*factor*' -p spin-trigger -p spin-trigger-http diff --git a/src/bin/spin.rs b/src/bin/spin.rs index 3cce1b3ca..0834f1380 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -15,9 +15,9 @@ use spin_cli::commands::{ watch::WatchCommand, }; use spin_cli::{build_info::*, subprocess::ExitStatusError}; -use spin_trigger2::cli::help::HelpArgsOnlyTrigger; -use spin_trigger2::cli::FactorsTriggerCommand; -use spin_trigger_http2::HttpTrigger; +use spin_trigger::cli::help::HelpArgsOnlyTrigger; +use spin_trigger::cli::FactorsTriggerCommand; +use spin_trigger_http::HttpTrigger; use spin_trigger_redis::RedisTrigger; #[tokio::main] diff --git a/src/commands/up.rs b/src/commands/up.rs index 1b8afd106..87f5d61cf 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -15,7 +15,7 @@ use spin_app::locked::LockedApp; use spin_common::ui::quoted_path; use spin_loader::FilesMountStrategy; use spin_oci::OciLoader; -use spin_trigger2::cli::{LaunchMetadata, SPIN_LOCAL_APP_DIR, SPIN_LOCKED_URL, SPIN_WORKING_DIR}; +use spin_trigger::cli::{LaunchMetadata, SPIN_LOCAL_APP_DIR, SPIN_LOCKED_URL, SPIN_WORKING_DIR}; use tempfile::TempDir; use crate::opts::*; diff --git a/tests/testing-framework/Cargo.toml b/tests/testing-framework/Cargo.toml index 2118f93b3..8448faaef 100644 --- a/tests/testing-framework/Cargo.toml +++ b/tests/testing-framework/Cargo.toml @@ -18,8 +18,8 @@ spin-app = { path = "../../crates/app" } spin-factors-executor = { path = "../../crates/factors-executor" } spin-http = { path = "../../crates/http" } spin-loader = { path = "../../crates/loader" } -spin-trigger2 = { path = "../../crates/trigger2" } -spin-trigger-http2 = { path = "../../crates/trigger-http2" } +spin-trigger = { path = "../../crates/trigger" } +spin-trigger-http = { path = "../../crates/trigger-http" } toml = "0.8.6" tokio = "1.23" wasmtime-wasi-http = { workspace = true } diff --git a/tests/testing-framework/src/runtimes/in_process_spin.rs b/tests/testing-framework/src/runtimes/in_process_spin.rs index 7580842a2..35a8ce9af 100644 --- a/tests/testing-framework/src/runtimes/in_process_spin.rs +++ b/tests/testing-framework/src/runtimes/in_process_spin.rs @@ -3,8 +3,8 @@ use std::{path::PathBuf, sync::Arc}; use anyhow::Context as _; -use spin_trigger2::cli::{TriggerAppBuilder, TriggerAppOptions}; -use spin_trigger_http2::{HttpServer, HttpTrigger}; +use spin_trigger::cli::{TriggerAppBuilder, TriggerAppOptions}; +use spin_trigger_http::{HttpServer, HttpTrigger}; use test_environment::{ http::{Request, Response}, services::ServicesConfig, From 7d15bdda6ed6d15ac73970ebcfc92531876ff6ab Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 23 Aug 2024 11:45:52 +0200 Subject: [PATCH 162/195] Fix checking of cidr ranges Signed-off-by: Ryan Levick --- crates/outbound-networking/src/lib.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/outbound-networking/src/lib.rs b/crates/outbound-networking/src/lib.rs index aec793419..a2be72471 100644 --- a/crates/outbound-networking/src/lib.rs +++ b/crates/outbound-networking/src/lib.rs @@ -175,7 +175,7 @@ impl HostConfig { HostConfig::List(l) => l.iter().any(|h| h.as_str() == host), HostConfig::ToSelf => false, HostConfig::Cidr(c) => { - let Ok(ip) = host.parse::() else { + let Ok(ip) = host.parse::() else { return false; }; c.contains(&ip) @@ -776,4 +776,11 @@ mod test { .allows(&OutboundUrl::parse("mysql://user%3Apass%23word@xyz.com", "mysql").unwrap())); assert!(allowed.allows(&OutboundUrl::parse("user%3Apass%23word@xyz.com", "mysql").unwrap())); } + + #[test] + fn test_cidr() { + let allowed = + AllowedHostsConfig::parse(&["*://127.0.0.1/24:63551"], &dummy_resolver()).unwrap(); + assert!(allowed.allows(&OutboundUrl::parse("tcp://127.0.0.1:63551", "tcp").unwrap())); + } } From 49ec0f05a5d7e3637fe9cbc0e6655783a0b11099 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 23 Aug 2024 10:54:12 -0400 Subject: [PATCH 163/195] factors: Port spin-timer to factors Signed-off-by: Lann Martin --- examples/spin-timer/Cargo.lock | 4122 +++++++------------------------ examples/spin-timer/Cargo.toml | 4 +- examples/spin-timer/src/lib.rs | 75 +- examples/spin-timer/src/main.rs | 5 +- 4 files changed, 962 insertions(+), 3244 deletions(-) diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index 9a04ac97f..c46d33839 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -18,15 +18,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] -name = "aes" -version = "0.8.4" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "ahash" @@ -42,27 +37,18 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "ambient-authority" @@ -85,64 +71,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - -[[package]] -name = "anstream" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" - -[[package]] -name = "anstyle-parse" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - [[package]] name = "anyhow" version = "1.0.79" @@ -155,22 +83,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - -[[package]] -name = "async-broadcast" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c48ccdbf6ca6b121e0f586cbc0e73ae440e56c67c30fa0873b4e110d9c26d2b" -dependencies = [ - "event-listener 2.5.3", - "futures-core", -] - [[package]] name = "async-channel" version = "1.9.0" @@ -189,16 +101,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener-strategy 0.5.2", + "event-listener-strategy", "futures-core", "pin-project-lite", ] [[package]] name = "async-compression" -version = "0.4.6" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "flate2", "futures-core", @@ -207,65 +119,20 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-executor" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8828ec6e544c02b0d6691d21ed9f9218d0384a82542855073c2a3f58304aaf0" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.3.0", - "slab", -] - -[[package]] -name = "async-fs" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279cf904654eeebfa37ac9bb1598880884924aab82e290aa65c9e77a0e142e06" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "blocking", - "futures-lite 1.13.0", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling 2.8.0", - "rustix 0.37.27", - "slab", - "socket2 0.4.10", - "waker-fn", -] - [[package]] name = "async-io" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +checksum = "0d6baa8f0178795da0e71bc42c9e5d13261aac7ee549853162e66a241ba17964" dependencies = [ - "async-lock 3.3.0", + "async-lock", "cfg-if", "concurrent-queue", "futures-io", "futures-lite 2.3.0", "parking", - "polling 3.7.0", - "rustix 0.38.31", + "polling", + "rustix", "slab", "tracing", "windows-sys 0.52.0", @@ -273,88 +140,51 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" -dependencies = [ - "event-listener 2.5.3", -] - -[[package]] -name = "async-lock" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", + "event-listener 5.3.1", + "event-listener-strategy", "pin-project-lite", ] [[package]] name = "async-process" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6438ba0a08d81529c69b36700fa2f95837bfe3e776ab39cde9c14d9149da88" -dependencies = [ - "async-io 1.13.0", - "async-lock 2.8.0", - "async-signal", - "blocking", - "cfg-if", - "event-listener 3.1.0", - "futures-lite 1.13.0", - "rustix 0.38.31", - "windows-sys 0.48.0", -] - -[[package]] -name = "async-process" -version = "2.2.2" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a53fc6301894e04a92cb2584fedde80cb25ba8e02d9dc39d4a87d036e22f397d" +checksum = "f7eda79bbd84e29c2b308d1dc099d7de8dcc7035e48f4bf5dc4a531a44ff5e2a" dependencies = [ "async-channel 2.3.1", - "async-io 2.3.2", - "async-lock 3.3.0", + "async-io", + "async-lock", "async-signal", "async-task", "blocking", "cfg-if", - "event-listener 5.3.0", + "event-listener 5.3.1", "futures-lite 2.3.0", - "rustix 0.38.31", + "rustix", "tracing", "windows-sys 0.52.0", ] -[[package]] -name = "async-recursion" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "async-signal" -version = "0.2.5" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" +checksum = "dfb3634b73397aa844481f814fad23bbf07fdb0eabec10f2eb95e58944b1ec32" dependencies = [ - "async-io 2.3.2", - "async-lock 2.8.0", + "async-io", + "async-lock", "atomic-waker", "cfg-if", "futures-core", "futures-io", - "rustix 0.38.31", + "rustix", "signal-hook-registry", "slab", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -376,7 +206,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -393,7 +223,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -419,6 +249,33 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-lc-rs" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" +dependencies = [ + "bindgen 0.69.4", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "axum" version = "0.6.20" @@ -430,9 +287,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.30", "itoa", "matchit", "memchr", @@ -440,8 +297,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustversion", - "serde 1.0.203", - "sync_wrapper", + "serde", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", @@ -456,7 +313,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "mime", "rustversion", @@ -467,10 +324,10 @@ dependencies = [ [[package]] name = "azure_core" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", - "base64 0.22.0", + "base64 0.22.1", "bytes", "dyn-clone", "futures", @@ -481,9 +338,9 @@ dependencies = [ "paste", "pin-project", "rand 0.8.5", - "reqwest 0.12.4", + "reqwest 0.12.5", "rustc_version", - "serde 1.0.203", + "serde", "serde_json", "sha2", "time", @@ -495,13 +352,13 @@ dependencies = [ [[package]] name = "azure_data_cosmos" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "azure_core", "bytes", "futures", - "serde 1.0.203", + "serde", "serde_json", "thiserror", "time", @@ -513,16 +370,16 @@ dependencies = [ [[package]] name = "azure_identity" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ - "async-lock 3.3.0", - "async-process 2.2.2", + "async-lock", + "async-process", "async-trait", "azure_core", "futures", "oauth2", "pin-project", - "serde 1.0.203", + "serde", "time", "tracing", "tz-rs", @@ -533,12 +390,12 @@ dependencies = [ [[package]] name = "azure_security_keyvault" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "azure_core", "futures", - "serde 1.0.203", + "serde", "serde_json", "time", ] @@ -553,17 +410,11 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.2", "object 0.32.2", "rustc-demangle", ] -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base64" version = "0.13.1" @@ -578,21 +429,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" - -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "beef" -version = "0.5.2" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bindgen" @@ -603,15 +442,36 @@ dependencies = [ "bitflags 2.4.2", "cexpr", "clang-sys", - "itertools 0.12.1", - "lazy_static 1.4.0", + "itertools", + "lazy_static", "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.75", + "which", +] + +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.4.2", + "cexpr", + "clang-sys", + "itertools", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -626,15 +486,6 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -644,15 +495,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "blocking" version = "1.6.1" @@ -672,7 +514,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" dependencies = [ - "num-traits 0.2.19", + "num-traits", ] [[package]] @@ -681,12 +523,6 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" -[[package]] -name = "bytemuck" -version = "1.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" - [[package]] name = "byteorder" version = "1.5.0" @@ -699,56 +535,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" dependencies = [ - "serde 1.0.203", -] - -[[package]] -name = "bytesize" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" - -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cached-path" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097968e38f1319207f057d0f4d76452e4f4f847a5de61c5215379f297fa034f3" -dependencies = [ - "flate2", - "fs2", - "glob", - "indicatif", - "log", - "rand 0.8.5", - "reqwest 0.11.24", - "serde 1.0.203", - "serde_json", - "sha2", - "tar", - "tempfile", - "thiserror", - "zip", + "serde", ] [[package]] @@ -759,7 +546,7 @@ checksum = "769f8cd02eb04d57f14e2e371ebb533f96817f9b2525d73a5c72b61ca7973747" dependencies = [ "cap-primitives", "cap-std", - "io-lifetimes 2.0.3", + "io-lifetimes", "windows-sys 0.52.0", ] @@ -771,7 +558,7 @@ checksum = "59ff6d3fb274292a9af283417e383afe6ded1fe66f6472d2c781216d3d80c218" dependencies = [ "cap-primitives", "cap-std", - "rustix 0.38.31", + "rustix", "smallvec", ] @@ -784,19 +571,19 @@ dependencies = [ "ambient-authority", "fs-set-times", "io-extras", - "io-lifetimes 2.0.3", + "io-lifetimes", "ipnet", "maybe-owned", - "rustix 0.38.31", + "rustix", "windows-sys 0.52.0", "winx", ] [[package]] name = "cap-rand" -version = "3.0.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4327f08daac33a99bb03c54ae18c8f32c3ba31c728a33ddf683c6c6a5043de68" +checksum = "dbcb16a619d8b8211ed61f42bd290d2a1ac71277a69cf8417ec0996fa92f5211" dependencies = [ "ambient-authority", "rand 0.8.5", @@ -810,8 +597,8 @@ checksum = "266626ce180cf9709f317d0bf9754e3a5006359d87f4bf792f06c9c5f1b63c0f" dependencies = [ "cap-primitives", "io-extras", - "io-lifetimes 2.0.3", - "rustix 0.38.31", + "io-lifetimes", + "rustix", ] [[package]] @@ -824,17 +611,16 @@ dependencies = [ "cap-primitives", "iana-time-zone", "once_cell", - "rustix 0.38.31", + "rustix", "winx", ] [[package]] -name = "cbc" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +name = "cargo-target-dep" +version = "0.1.0" +source = "git+https://github.com/fermyon/cargo-target-dep?rev=482f269eceb7b1a7e8fc618bf8c082dd24979cf1#482f269eceb7b1a7e8fc618bf8c082dd24979cf1" dependencies = [ - "cipher", + "glob", ] [[package]] @@ -854,7 +640,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -878,27 +664,17 @@ dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", - "num-traits 0.2.19", - "serde 1.0.203", + "num-traits", + "serde", "wasm-bindgen", "windows-targets 0.52.4", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -913,44 +689,22 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "atty", "bitflags 1.3.2", - "clap_derive 3.2.25", - "clap_lex 0.2.4", + "clap_derive", + "clap_lex", "indexmap 1.9.3", "once_cell", - "strsim 0.10.0", + "strsim", "termcolor", "textwrap", ] -[[package]] -name = "clap" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" -dependencies = [ - "clap_builder", - "clap_derive 4.5.4", -] - -[[package]] -name = "clap_builder" -version = "4.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" -dependencies = [ - "anstream", - "anstyle", - "clap_lex 0.7.0", - "strsim 0.11.1", -] - [[package]] name = "clap_derive" version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -958,37 +712,19 @@ dependencies = [ ] [[package]] -name = "clap_derive" -version = "4.5.4" +name = "clap_lex" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.48", + "os_str_bytes", ] -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "clap_lex" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" - [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" dependencies = [ "cc", ] @@ -999,24 +735,18 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" -[[package]] -name = "colorchoice" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" - [[package]] name = "combine" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "futures-core", "memchr", "pin-project-lite", "tokio", - "tokio-util 0.7.10", + "tokio-util", ] [[package]] @@ -1028,53 +758,12 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "config" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1b9d958c2b1368a663f05538fc1b5975adce1e19f435acceae987aceeeb369" -dependencies = [ - "lazy_static 1.4.0", - "nom 5.1.3", - "rust-ini", - "serde 1.0.203", - "serde-hjson", - "serde_json", - "toml 0.5.11", - "yaml-rust", -] - -[[package]] -name = "console" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" -dependencies = [ - "encode_unicode", - "lazy_static 1.4.0", - "libc", - "unicode-width", - "windows-sys 0.52.0", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const_fn" version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d" -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "core-foundation" version = "0.9.4" @@ -1087,9 +776,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpp_demangle" @@ -1170,7 +859,7 @@ version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8cfdc315e5d18997093e040a8d234bea1ac1e118a716d3e30f40d449e78207b" dependencies = [ - "serde 1.0.203", + "serde", "serde_derive", ] @@ -1212,7 +901,7 @@ dependencies = [ "cranelift-codegen", "cranelift-entity", "cranelift-frontend", - "itertools 0.12.1", + "itertools", "log", "smallvec", "wasmparser 0.209.1", @@ -1243,9 +932,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -1284,24 +973,6 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -1318,7 +989,7 @@ version = "3.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" dependencies = [ - "nix 0.28.0", + "nix", "windows-sys 0.52.0", ] @@ -1328,18 +999,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" -dependencies = [ - "darling_core 0.20.9", - "darling_macro 0.20.9", + "darling_core", + "darling_macro", ] [[package]] @@ -1352,46 +1013,21 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim", "syn 1.0.109", ] -[[package]] -name = "darling_core" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.11.1", - "syn 2.0.48", -] - [[package]] name = "darling_macro" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core 0.14.4", + "darling_core", "quote", "syn 1.0.109", ] -[[package]] -name = "darling_macro" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" -dependencies = [ - "darling_core 0.20.9", - "quote", - "syn 2.0.48", -] - [[package]] name = "debugid" version = "0.8.0" @@ -1401,17 +1037,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "der" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - [[package]] name = "deranged" version = "0.3.11" @@ -1419,18 +1044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", - "serde 1.0.203", -] - -[[package]] -name = "derivative" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "serde", ] [[package]] @@ -1439,16 +1053,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" dependencies = [ - "derive_builder_macro 0.11.2", -] - -[[package]] -name = "derive_builder" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" -dependencies = [ - "derive_builder_macro 0.12.0", + "derive_builder_macro", ] [[package]] @@ -1457,19 +1062,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_core" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" -dependencies = [ - "darling 0.14.4", + "darling", "proc-macro2", "quote", "syn 1.0.109", @@ -1481,33 +1074,10 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" dependencies = [ - "derive_builder_core 0.11.2", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_macro" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" -dependencies = [ - "derive_builder_core 0.12.0", + "derive_builder_core", "syn 1.0.109", ] -[[package]] -name = "dialoguer" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" -dependencies = [ - "console", - "shell-words", - "tempfile", - "thiserror", - "zeroize", -] - [[package]] name = "digest" version = "0.10.7" @@ -1515,20 +1085,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "const-oid", "crypto-common", "subtle", ] -[[package]] -name = "directories" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" -dependencies = [ - "dirs-sys 0.3.7", -] - [[package]] name = "directories-next" version = "2.0.0" @@ -1545,16 +1105,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "dirs-sys 0.3.7", -] - -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys 0.4.1", + "dirs-sys", ] [[package]] @@ -1568,18 +1119,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1591,17 +1130,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "docker_credential" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31951f49556e34d90ed28342e1df7e1cb7a229c4cab0aecc627b5d91edd41d07" -dependencies = [ - "base64 0.21.7", - "serde 1.0.203", - "serde_json", -] - [[package]] name = "dotenvy" version = "0.15.7" @@ -1610,9 +1138,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "dyn-clone" @@ -1620,58 +1148,18 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der", - "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", -] - [[package]] name = "either" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest", - "ff", - "generic-array", - "group", - "pem-rfc7468", - "pkcs8", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "embedded-io" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "encoding_rs" version = "0.8.33" @@ -1681,27 +1169,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "enumflags2" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d232db7f5956f3f14313dc2f87985c58bd2c695ce124c8cdd984e08e15ac133d" -dependencies = [ - "enumflags2_derive", - "serde 1.0.203", -] - -[[package]] -name = "enumflags2_derive" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -1718,12 +1185,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "esaxx-rs" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" - [[package]] name = "event-listener" version = "2.5.3" @@ -1732,54 +1193,22 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener" -version = "5.3.0" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", "pin-project-lite", ] -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.3", - "pin-project-lite", -] - [[package]] name = "event-listener-strategy" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.3.1", "pin-project-lite", ] @@ -1812,9 +1241,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fd-lock" @@ -1823,52 +1252,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947" dependencies = [ "cfg-if", - "rustix 0.38.31", - "windows-sys 0.52.0", -] - -[[package]] -name = "ff" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "filetime" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", + "rustix", "windows-sys 0.52.0", ] -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -1918,20 +1313,16 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "033b337d725b97690d86893f9de22b67b80dcc4e9ad815f348254c38119db8fb" dependencies = [ - "io-lifetimes 2.0.3", - "rustix 0.38.31", + "io-lifetimes", + "rustix", "windows-sys 0.52.0", ] [[package]] -name = "fs2" -version = "0.4.3" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" @@ -2002,7 +1393,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.0.1", + "fastrand 2.1.0", "futures-core", "futures-io", "parking", @@ -2017,7 +1408,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -2068,7 +1459,7 @@ dependencies = [ "bitflags 2.4.2", "debugid", "fxhash", - "serde 1.0.203", + "serde", "serde_json", ] @@ -2080,7 +1471,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -2107,24 +1497,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "ggml" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "ggml-sys", - "memmap2", - "thiserror", -] - -[[package]] -name = "ggml-sys" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "cc", -] - [[package]] name = "gimli" version = "0.28.1" @@ -2143,19 +1515,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "h2" -version = "0.3.26" +name = "h2" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ @@ -2164,43 +1525,33 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.11", + "http 0.2.12", "indexmap 2.2.6", "slab", "tokio", - "tokio-util 0.7.10", + "tokio-util", "tracing", ] [[package]] name = "h2" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http 1.1.0", "indexmap 2.2.6", "slab", "tokio", - "tokio-util 0.7.10", + "tokio-util", "tracing", ] -[[package]] -name = "half" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" -dependencies = [ - "cfg-if", - "crunchy", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -2224,7 +1575,7 @@ checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ "ahash", "allocator-api2", - "serde 1.0.203", + "serde", ] [[package]] @@ -2242,12 +1593,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - [[package]] name = "hermit-abi" version = "0.1.19" @@ -2264,34 +1609,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] -name = "hex" -version = "0.4.3" +name = "hermit-abi" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" [[package]] -name = "hkdf" -version = "0.12.4" +name = "hmac" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "hmac", + "digest", ] [[package]] -name = "hmac" -version = "0.12.1" +name = "home" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "digest", + "windows-sys 0.52.0", ] [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -2309,15 +1654,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-auth" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643c9bbf6a4ea8a656d6b4cd53d34f79e3f841ad5203c1a55fb7d761923bc255" -dependencies = [ - "memchr", -] - [[package]] name = "http-body" version = "0.4.6" @@ -2325,15 +1661,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.11", + "http 0.2.12", "pin-project-lite", ] [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -2341,14 +1677,14 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -2365,7 +1701,7 @@ dependencies = [ "infer", "pin-project-lite", "rand 0.7.3", - "serde 1.0.203", + "serde", "serde_json", "serde_qs", "serde_urlencoded", @@ -2374,9 +1710,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -2386,22 +1722,22 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", "h2 0.3.26", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2", "tokio", "tower-service", "tracing", @@ -2410,16 +1746,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", + "h2 0.4.6", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2436,9 +1772,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http 0.2.11", - "hyper 0.14.28", - "rustls 0.21.10", + "http 0.2.12", + "hyper 0.14.30", + "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", ] @@ -2450,15 +1786,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "399c78f9338483cb7e630c8474b07268983c6bd5acee012e4211f9f7bb21b070" dependencies = [ "futures-util", - "http 0.2.11", - "hyper 0.14.28", + "http 0.2.12", + "hyper 0.14.30", "log", "rustls 0.22.4", "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.25.0", - "webpki-roots 0.26.1", + "webpki-roots 0.26.3", ] [[package]] @@ -2467,7 +1803,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.30", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -2480,7 +1816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.28", + "hyper 0.14.30", "native-tls", "tokio", "tokio-native-tls", @@ -2494,7 +1830,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "native-tls", "tokio", @@ -2504,18 +1840,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", - "hyper 1.3.1", + "http-body 1.0.1", + "hyper 1.4.1", "pin-project-lite", - "socket2 0.5.5", + "socket2", "tokio", "tower", "tower-service", @@ -2567,20 +1903,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "im-rc" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1955a75fa080c677d3972822ec4bad316169ab1cfc6c257a942c2265dbe5fe" -dependencies = [ - "bitmaps", - "rand_core 0.6.4", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -2589,7 +1911,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", - "serde 1.0.203", ] [[package]] @@ -2600,19 +1921,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", - "serde 1.0.203", -] - -[[package]] -name = "indicatif" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" -dependencies = [ - "console", - "lazy_static 1.4.0", - "number_prefix", - "regex", + "serde", ] [[package]] @@ -2621,46 +1930,25 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] [[package]] name = "io-extras" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c301e73fb90e8a29e600a9f402d095765f74310d582916a952f618836a1bd1ed" +checksum = "c9f046b9af244f13b3bd939f55d16830ac3a201e8a9ba9661bfcb03e2be72b9b" dependencies = [ - "io-lifetimes 2.0.3", + "io-lifetimes", "windows-sys 0.52.0", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.9", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "io-lifetimes" version = "2.0.3" @@ -2673,48 +1961,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" -[[package]] -name = "is_terminal_polyfill" -version = "1.70.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" - -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.1" @@ -2761,28 +2007,13 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jwt" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" -dependencies = [ - "base64 0.13.1", - "crypto-common", - "digest", - "hmac", - "serde 1.0.203", - "serde_json", - "sha2", -] - [[package]] name = "keyed_priority_queue" version = "0.4.2" @@ -2792,31 +2023,11 @@ dependencies = [ "indexmap 2.2.6", ] -[[package]] -name = "keyring" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "363387f0019d714aa60cc30ab4fe501a747f4c08fc58f069dd14be971bd495a0" -dependencies = [ - "byteorder", - "lazy_static 1.4.0", - "linux-keyutils", - "secret-service", - "security-framework", - "windows-sys 0.52.0", -] - -[[package]] -name = "lazy_static" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" - [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -2830,19 +2041,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" -[[package]] -name = "lexical-core" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" -dependencies = [ - "arrayvec", - "bitflags 1.3.2", - "cfg-if", - "ryu", - "static_assertions", -] - [[package]] name = "libc" version = "0.2.153" @@ -2851,12 +2049,12 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.8.1" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-targets 0.52.4", ] [[package]] @@ -2877,9 +2075,9 @@ dependencies = [ [[package]] name = "libsql" -version = "0.3.2" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3879a4ed80a245fd4dd8c8fa139245653e86184ed3ab97a6d6ea592045d25793" +checksum = "1bd17bcc143f2a5be449680dc63b91327d953bcabebe34a69c549fca8934ec9d" dependencies = [ "async-stream", "async-trait", @@ -2888,30 +2086,30 @@ dependencies = [ "bytes", "fallible-iterator 0.3.0", "futures", - "http 0.2.11", - "hyper 0.14.28", + "http 0.2.12", + "hyper 0.14.30", "hyper-rustls 0.25.0", "libsql-hrana", "libsql-sqlite3-parser", - "serde 1.0.203", + "serde", "serde_json", "thiserror", "tokio", - "tokio-util 0.7.10", + "tokio-util", "tower", "tracing", ] [[package]] name = "libsql-hrana" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f256c5c98e84808e067133253471d6f5961c670f0127150694210fb8e6116a" +checksum = "220a925fe6d49dbfa7523b20f5a5391f579b5d9dcf9dd1225606d00929fcab3a" dependencies = [ "base64 0.21.7", "bytes", "prost", - "serde 1.0.203", + "serde", ] [[package]] @@ -2945,380 +2143,137 @@ dependencies = [ ] [[package]] -name = "linked-hash-map" -version = "0.5.6" +name = "linux-raw-sys" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] -name = "linux-keyutils" -version = "0.2.4" +name = "lock_api" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761e49ec5fd8a5a463f9b84e877c373d888935b71c6be78f3767fe2ae6bed18e" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ - "bitflags 2.4.2", - "libc", + "autocfg", + "scopeguard", ] [[package]] -name = "linux-raw-sys" -version = "0.3.8" +name = "log" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] -name = "linux-raw-sys" -version = "0.4.13" +name = "lru" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" - -[[package]] -name = "llm" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" +checksum = "71e7d46de488603ffdd5f30afbc64fbba2378214a2c3a2fb83abf3d33126df17" dependencies = [ - "llm-base", - "llm-bloom", - "llm-gpt2", - "llm-gptj", - "llm-gptneox", - "llm-llama", - "llm-mpt", - "serde 1.0.203", - "tracing", + "hashbrown 0.13.2", ] [[package]] -name = "llm-base" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" +name = "lru" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" dependencies = [ - "bytemuck", - "ggml", - "half", - "llm-samplers", - "memmap2", - "partial_sort", - "rand 0.8.5", - "regex", - "serde 1.0.203", - "serde_bytes", - "thiserror", - "tokenizers", - "tracing", + "hashbrown 0.14.3", ] [[package]] -name = "llm-bloom" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ - "llm-base", + "libc", ] [[package]] -name = "llm-gpt2" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "bytemuck", - "llm-base", + "regex-automata 0.1.10", ] [[package]] -name = "llm-gptj" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "llm-base", -] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] -name = "llm-gptneox" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "llm-base", -] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" [[package]] -name = "llm-llama" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "llm-base", - "tracing", + "cfg-if", + "digest", ] [[package]] -name = "llm-mpt" -version = "0.2.0-dev" -source = "git+https://github.com/rustformers/llm?rev=2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663#2f6ffd4435799ceaa1d1bcb5a8790e5b3e0c5663" -dependencies = [ - "llm-base", -] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] -name = "llm-samplers" -version = "0.0.6" +name = "memfd" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7553f60d113c9cdc6a5402456a31cd9a273bef79f6f16d8a4f7b4bedf5f754b2" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "anyhow", - "num-traits 0.2.19", - "rand 0.8.5", - "thiserror", + "rustix", ] [[package]] -name = "lock_api" -version = "0.4.11" +name = "memoffset" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", - "scopeguard", ] [[package]] -name = "log" -version = "0.4.20" +name = "mime" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] -name = "logos" -version = "0.13.0" +name = "minimal-lexical" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" -dependencies = [ - "logos-derive 0.13.0", -] +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "logos" -version = "0.14.0" +name = "miniz_oxide" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161971eb88a0da7ae0c333e1063467c5b5727e7fb6b710b8db4814eade3a42e8" -dependencies = [ - "logos-derive 0.14.0", -] - -[[package]] -name = "logos-codegen" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" -dependencies = [ - "beef", - "fnv", - "proc-macro2", - "quote", - "regex-syntax 0.6.29", - "syn 2.0.48", -] - -[[package]] -name = "logos-codegen" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e31badd9de5131fdf4921f6473d457e3dd85b11b7f091ceb50e4df7c3eeb12a" -dependencies = [ - "beef", - "fnv", - "lazy_static 1.4.0", - "proc-macro2", - "quote", - "regex-syntax 0.8.2", - "syn 2.0.48", -] - -[[package]] -name = "logos-derive" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" -dependencies = [ - "logos-codegen 0.13.0", -] - -[[package]] -name = "logos-derive" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c2a69b3eb68d5bd595107c9ee58d7e07fe2bb5e360cc85b0f084dedac80de0a" -dependencies = [ - "logos-codegen 0.14.0", -] - -[[package]] -name = "lru" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e7d46de488603ffdd5f30afbc64fbba2378214a2c3a2fb83abf3d33126df17" -dependencies = [ - "hashbrown 0.13.2", -] - -[[package]] -name = "lru" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" -dependencies = [ - "hashbrown 0.14.3", -] - -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - -[[package]] -name = "macro_rules_attribute" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "maybe-owned" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] -name = "memchr" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" - -[[package]] -name = "memfd" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" -dependencies = [ - "rustix 0.38.31", -] - -[[package]] -name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - -[[package]] -name = "miette" -version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" -dependencies = [ - "cfg-if", - "miette-derive", - "thiserror", - "unicode-width", -] - -[[package]] -name = "miette-derive" -version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ - "mime", - "unicase", + "adler", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -3334,31 +2289,10 @@ dependencies = [ ] [[package]] -name = "monostate" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878c2a1f1c70e5724fa28f101ca787b6a7e8ad5c5e4ae4ca3b0fa4a419fa9075" -dependencies = [ - "monostate-impl", - "serde 1.0.203", -] - -[[package]] -name = "monostate-impl" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f686d68a09079e63b1d2c64aa305095887ce50565f00a922ebfaeeee0d9ba6ce" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "multimap" -version = "0.8.3" +name = "mirai-annotations" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" [[package]] name = "mysql_async" @@ -3373,8 +2307,8 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lazy_static 1.4.0", - "lru 0.12.3", + "lazy_static", + "lru 0.12.4", "mio", "mysql_common", "native-tls", @@ -3383,13 +2317,13 @@ dependencies = [ "percent-encoding", "pin-project", "rand 0.8.5", - "serde 1.0.203", + "serde", "serde_json", - "socket2 0.5.5", + "socket2", "thiserror", "tokio", "tokio-native-tls", - "tokio-util 0.7.10", + "tokio-util", "twox-hash", "url", ] @@ -3401,7 +2335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" dependencies = [ "base64 0.21.7", - "bindgen", + "bindgen 0.70.1", "bitflags 2.4.2", "btoi", "byteorder", @@ -3410,13 +2344,13 @@ dependencies = [ "cmake", "crc32fast", "flate2", - "lazy_static 1.4.0", + "lazy_static", "num-bigint", - "num-traits 0.2.19", + "num-traits", "rand 0.8.5", "regex", "saturating", - "serde 1.0.203", + "serde", "serde_json", "sha1 0.10.6", "sha2", @@ -3429,11 +2363,10 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static 1.4.0", "libc", "log", "openssl", @@ -3445,18 +2378,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "nix" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", - "memoffset 0.7.1", -] - [[package]] name = "nix" version = "0.28.0" @@ -3469,17 +2390,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nom" -version = "5.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08959a387a676302eebf4ddbcbc611da04285579f76f88ee0506c63b1a61dd4b" -dependencies = [ - "lexical-core", - "memchr", - "version_check", -] - [[package]] name = "nom" version = "7.1.3" @@ -3490,15 +2400,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "normpath" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5831952a9476f2fed74b77d74182fa5ddc4d21c72ec45a333b250e3ed0272804" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3509,37 +2410,14 @@ dependencies = [ "winapi", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits 0.2.19", -] - [[package]] name = "num-bigint" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" -dependencies = [ - "num-integer", - "num-traits 0.2.19", -] - -[[package]] -name = "num-complex" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "num-traits 0.2.19", + "num-integer", + "num-traits", ] [[package]] @@ -3554,38 +2432,7 @@ version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "num-traits 0.2.19", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits 0.2.19", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits 0.2.19", -] - -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -dependencies = [ - "num-traits 0.2.19", + "num-traits", ] [[package]] @@ -3609,19 +2456,13 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "oauth2" version = "4.4.2" @@ -3631,9 +2472,9 @@ dependencies = [ "base64 0.13.1", "chrono", "getrandom 0.2.12", - "http 0.2.11", + "http 0.2.12", "rand 0.8.5", - "serde 1.0.203", + "serde", "serde_json", "serde_path_to_error", "sha2", @@ -3662,92 +2503,17 @@ dependencies = [ "memchr", ] -[[package]] -name = "oci-distribution" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95a2c51531af0cb93761f66094044ca6ea879320bccd35ab747ff3fcab3f422" -dependencies = [ - "bytes", - "chrono", - "futures-util", - "http 1.1.0", - "http-auth", - "jwt", - "lazy_static 1.4.0", - "olpc-cjson", - "regex", - "reqwest 0.12.4", - "serde 1.0.203", - "serde_json", - "sha2", - "thiserror", - "tokio", - "tracing", - "unicase", -] - -[[package]] -name = "oci-wasm" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91502e5352f927156f2b6a28d2558cc59558b1f441b681df3f706ced6937e07" -dependencies = [ - "anyhow", - "chrono", - "oci-distribution", - "serde 1.0.203", - "serde_json", - "sha2", - "tokio", - "wit-component 0.209.1", - "wit-parser 0.209.1", -] - -[[package]] -name = "olpc-cjson" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d637c9c15b639ccff597da8f4fa968300651ad2f1e968aefc3b4927a6fb2027a" -dependencies = [ - "serde 1.0.203", - "serde_json", - "unicode-normalization", -] - [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "onig" -version = "6.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" -dependencies = [ - "bitflags 1.3.2", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "openssl" -version = "0.10.64" +version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags 2.4.2", "cfg-if", @@ -3766,7 +2532,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -3777,9 +2543,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.102" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", @@ -3810,9 +2576,9 @@ checksum = "7690dc77bf776713848c4faa6501157469017eaf332baccd4eb1cea928743d94" dependencies = [ "async-trait", "bytes", - "http 0.2.11", + "http 0.2.12", "opentelemetry", - "reqwest 0.11.24", + "reqwest 0.11.27", ] [[package]] @@ -3823,14 +2589,14 @@ checksum = "1a016b8d9495c639af2145ac22387dcb88e44118e45320d9238fbf4e7889abcb" dependencies = [ "async-trait", "futures-core", - "http 0.2.11", + "http 0.2.12", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", "opentelemetry-semantic-conventions", "opentelemetry_sdk", "prost", - "reqwest 0.11.24", + "reqwest 0.11.27", "thiserror", "tokio", "tonic", @@ -3868,7 +2634,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.2.0", + "ordered-float", "percent-encoding", "rand 0.8.5", "serde_json", @@ -3877,38 +2643,13 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits 0.2.19", -] - [[package]] name = "ordered-float" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" -dependencies = [ - "num-traits 0.2.19", -] - -[[package]] -name = "ordered-stream" -version = "0.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ - "futures-core", - "pin-project-lite", + "num-traits", ] [[package]] @@ -3917,112 +2658,12 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -[[package]] -name = "outbound-http" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "http 0.2.11", - "reqwest 0.11.24", - "spin-app", - "spin-core", - "spin-expressions", - "spin-locked-app", - "spin-outbound-networking", - "spin-telemetry", - "spin-world", - "terminal", - "tracing", - "url", -] - -[[package]] -name = "outbound-mqtt" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "rumqttc", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tracing", -] - -[[package]] -name = "outbound-mysql" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "flate2", - "mysql_async", - "mysql_common", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "outbound-pg" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "native-tls", - "postgres-native-tls", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tokio-postgres", - "tracing", -] - -[[package]] -name = "outbound-redis" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "redis", - "spin-app", - "spin-core", - "spin-expressions", - "spin-outbound-networking", - "spin-world", - "table", - "tokio", - "tracing", -] - [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2", -] - [[package]] name = "parking" version = "2.2.0" @@ -4052,119 +2693,20 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "partial_sort" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7924d1d0ad836f665c9065e26d016c673ece3993f30d340068b16f282afc1156" - -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "paste" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" -[[package]] -name = "path-absolutize" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" -dependencies = [ - "path-dedot", -] - -[[package]] -name = "path-dedot" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" -dependencies = [ - "once_cell", -] - -[[package]] -name = "pathdiff" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" - -[[package]] -name = "pbjson" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" -dependencies = [ - "base64 0.21.7", - "serde 1.0.203", -] - -[[package]] -name = "pbjson-build" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" -dependencies = [ - "heck 0.4.1", - "itertools 0.11.0", - "prost", - "prost-types", -] - -[[package]] -name = "pbjson-types" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" -dependencies = [ - "bytes", - "chrono", - "pbjson", - "pbjson-build", - "prost", - "prost-build", - "serde 1.0.203", -] - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - [[package]] name = "pem" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" -dependencies = [ - "base64 0.21.7", - "serde 1.0.203", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64ct", + "base64 0.22.1", + "serde", ] [[package]] @@ -4173,16 +2715,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset", - "indexmap 2.2.6", -] - [[package]] name = "phf" version = "0.11.2" @@ -4239,7 +2771,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -4256,25 +2788,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" dependencies = [ "atomic-waker", - "fastrand 2.0.1", + "fastrand 2.1.0", "futures-io", ] -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - [[package]] name = "pkg-config" version = "0.3.29" @@ -4283,31 +2805,15 @@ checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "polling" -version = "3.7.0" +version = "3.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645493cf344456ef24219d02a768cf1fb92ddf8c92161679ae3d91b91a637be3" +checksum = "a3ed00ed3fbf728b5816498ecd316d1716eecaced9c0c8d2c5a6740ca214985b" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi 0.3.9", + "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.31", + "rustix", "tracing", "windows-sys 0.52.0", ] @@ -4320,7 +2826,7 @@ checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" dependencies = [ "cobs", "embedded-io", - "serde 1.0.203", + "serde", ] [[package]] @@ -4338,11 +2844,11 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" +checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "byteorder", "bytes", "fallible-iterator 0.2.0", @@ -4356,9 +2862,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" +checksum = "02048d9e032fb3cc3413bbf7b83a15d84a5d419778e2628751896d856498eee9" dependencies = [ "bytes", "fallible-iterator 0.2.0", @@ -4373,42 +2879,26 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "prettyplease" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "proc-macro-error" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ @@ -4432,9 +2922,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -4449,27 +2939,6 @@ dependencies = [ "prost-derive", ] -[[package]] -name = "prost-build" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" -dependencies = [ - "bytes", - "heck 0.5.0", - "itertools 0.12.1", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.48", - "tempfile", -] - [[package]] name = "prost-derive" version = "0.12.6" @@ -4477,59 +2946,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools", "proc-macro2", "quote", - "syn 2.0.48", -] - -[[package]] -name = "prost-reflect" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5eec97d5d34bdd17ad2db2219aabf46b054c6c41bd5529767c9ce55be5898f" -dependencies = [ - "logos 0.14.0", - "miette", - "once_cell", - "prost", - "prost-types", -] - -[[package]] -name = "prost-types" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" -dependencies = [ - "prost", -] - -[[package]] -name = "protox" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29b3c5596eb23a849deba860b53ffd468199d9ad5fe4402a7d55379e16aa2d2" -dependencies = [ - "bytes", - "miette", - "prost", - "prost-reflect", - "prost-types", - "protox-parse", - "thiserror", -] - -[[package]] -name = "protox-parse" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "033b939d76d358f7c32120c86c71f515bae45e64f2bde455200356557276276c" -dependencies = [ - "logos 0.13.0", - "miette", - "prost-types", - "thiserror", + "syn 2.0.75", ] [[package]] @@ -4541,22 +2961,6 @@ dependencies = [ "cc", ] -[[package]] -name = "ptree" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0de80796b316aec75344095a6d2ef68ec9b8f573b9e7adc821149ba3598e270" -dependencies = [ - "ansi_term", - "atty", - "config", - "directories", - "petgraph", - "serde 1.0.203", - "serde-value", - "tint", -] - [[package]] name = "quote" version = "1.0.35" @@ -4637,15 +3041,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rayon" version = "1.8.1" @@ -4656,17 +3051,6 @@ dependencies = [ "rayon-core", ] -[[package]] -name = "rayon-cond" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd1259362c9065e5ea39a789ef40b1e3fd934c94beb7b5ab3ac6629d3b5e7cb7" -dependencies = [ - "either", - "itertools 0.8.2", - "rayon", -] - [[package]] name = "rayon-core" version = "1.12.1" @@ -4695,7 +3079,7 @@ dependencies = [ "sha1 0.6.1", "tokio", "tokio-native-tls", - "tokio-util 0.7.10", + "tokio-util", "url", ] @@ -4734,14 +3118,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ - "aho-corasick 1.1.2", + "aho-corasick", "memchr", "regex-automata 0.4.7", - "regex-syntax 0.8.2", + "regex-syntax 0.8.4", ] [[package]] @@ -4759,9 +3143,9 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ - "aho-corasick 1.1.2", + "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.4", ] [[package]] @@ -4772,21 +3156,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - -[[package]] -name = "regex-syntax" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.11.24" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "async-compression", "base64 0.21.7", @@ -4795,9 +3173,9 @@ dependencies = [ "futures-core", "futures-util", "h2 0.3.26", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.30", "hyper-rustls 0.24.2", "hyper-tls 0.5.0", "ipnet", @@ -4808,17 +3186,17 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.10", + "rustls 0.21.12", "rustls-pemfile 1.0.4", - "serde 1.0.203", + "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", - "tokio-util 0.7.10", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -4830,20 +3208,18 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "bytes", - "encoding_rs", "futures-core", "futures-util", - "h2 0.4.4", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -4854,16 +3230,14 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.1.2", - "serde 1.0.203", + "rustls-pemfile 2.1.3", + "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", - "system-configuration", + "sync_wrapper 1.0.1", "tokio", "tokio-native-tls", - "tokio-socks", - "tokio-util 0.7.10", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -4873,16 +3247,6 @@ dependencies = [ "winreg 0.52.0", ] -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac", - "subtle", -] - [[package]] name = "ring" version = "0.17.8" @@ -4909,8 +3273,8 @@ dependencies = [ "futures-util", "log", "rustls-native-certs", - "rustls-pemfile 2.1.2", - "rustls-webpki 0.102.2", + "rustls-pemfile 2.1.3", + "rustls-webpki 0.102.6", "thiserror", "tokio", "tokio-rustls 0.25.0", @@ -4931,12 +3295,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "rust-ini" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -4967,10 +3325,10 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "http 0.2.11", - "reqwest 0.11.24", + "http 0.2.12", + "reqwest 0.11.27", "rustify_derive", - "serde 1.0.203", + "serde", "serde_json", "serde_urlencoded", "thiserror", @@ -4980,9 +3338,9 @@ dependencies = [ [[package]] name = "rustify_derive" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58135536c18c04f4634bedad182a3f41baf33ef811cc38a3ec7b7061c57134c8" +checksum = "7345f32672da54338227b727bd578c897859ddfaad8952e0b0d787fb4e58f07d" dependencies = [ "proc-macro2", "quote", @@ -4992,20 +3350,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "rustix" -version = "0.37.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes 1.0.11", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - [[package]] name = "rustix" version = "0.38.31" @@ -5016,16 +3360,16 @@ dependencies = [ "errno", "itoa", "libc", - "linux-raw-sys 0.4.13", + "linux-raw-sys", "once_cell", "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -5042,19 +3386,34 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.2", + "rustls-webpki 0.102.6", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.2", + "rustls-pemfile 2.1.3", "rustls-pki-types", "schannel", "security-framework", @@ -5071,19 +3430,19 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -5097,10 +3456,11 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.2" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5108,9 +3468,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" @@ -5118,22 +3478,13 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "sanitize-filename" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" +checksum = "2ed72fbaf78e6f2d41744923916966c4fbe3d7c74e3037a8ee482f1115572603" dependencies = [ - "lazy_static 1.4.0", + "lazy_static", "regex", ] @@ -5168,54 +3519,11 @@ dependencies = [ "untrusted", ] -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", - "subtle", - "zeroize", -] - -[[package]] -name = "secrecy" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" -dependencies = [ - "serde 1.0.203", - "zeroize", -] - -[[package]] -name = "secret-service" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5204d39df37f06d1944935232fd2dfe05008def7ca599bf28c0800366c8a8f9" -dependencies = [ - "aes", - "cbc", - "futures-util", - "generic-array", - "hkdf", - "num", - "once_cell", - "rand 0.8.5", - "serde 1.0.203", - "sha2", - "zbus", -] - [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -5226,9 +3534,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -5239,15 +3547,6 @@ name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" -dependencies = [ - "serde 1.0.203", -] - -[[package]] -name = "serde" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" @@ -5258,37 +3557,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-hjson" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" -dependencies = [ - "lazy_static 1.4.0", - "num-traits 0.1.43", - "regex", - "serde 0.8.23", -] - -[[package]] -name = "serde-value" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" -dependencies = [ - "ordered-float 2.10.1", - "serde 1.0.203", -] - -[[package]] -name = "serde_bytes" -version = "0.11.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" -dependencies = [ - "serde 1.0.203", -] - [[package]] name = "serde_derive" version = "1.0.203" @@ -5297,7 +3565,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -5308,7 +3576,7 @@ checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", - "serde 1.0.203", + "serde", ] [[package]] @@ -5318,7 +3586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", - "serde 1.0.203", + "serde", ] [[package]] @@ -5328,28 +3596,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" dependencies = [ "percent-encoding", - "serde 1.0.203", + "serde", "thiserror", ] -[[package]] -name = "serde_repr" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "serde_spanned" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ - "serde 1.0.203", + "serde", ] [[package]] @@ -5361,50 +3618,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.203", -] - -[[package]] -name = "serde_with" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" -dependencies = [ - "base64 0.22.0", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.2.6", - "serde 1.0.203", - "serde_derive", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" -dependencies = [ - "darling 0.20.9", - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.2.6", - "itoa", - "ryu", - "serde 1.0.203", - "unsafe-libyaml", + "serde", ] [[package]] @@ -5429,9 +3643,9 @@ dependencies = [ [[package]] name = "sha1_smol" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" [[package]] name = "sha2" @@ -5444,50 +3658,22 @@ dependencies = [ "digest", ] -[[package]] -name = "sha256" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18278f6a914fa3070aa316493f7d2ddfb9ac86ebc06fa3b83bffda487e9065b0" -dependencies = [ - "async-trait", - "bytes", - "hex", - "sha2", - "tokio", -] - [[package]] name = "sharded-slab" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ - "lazy_static 1.4.0", + "lazy_static", ] -[[package]] -name = "shell-words" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" - [[package]] name = "shellexpand" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" dependencies = [ - "dirs 4.0.0", -] - -[[package]] -name = "shellexpand" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" -dependencies = [ - "dirs 5.0.1", + "dirs", ] [[package]] @@ -5505,32 +3691,12 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - [[package]] name = "siphasher" version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "slab" version = "0.4.9" @@ -5552,17 +3718,7 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" dependencies = [ - "serde 1.0.203", -] - -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", + "serde", ] [[package]] @@ -5577,9 +3733,9 @@ dependencies = [ [[package]] name = "spdx" -version = "0.10.3" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bde1398b09b9f93fc2fc9b9da86e362693e999d3a54a8ac47a99a5a73f638b" +checksum = "47317bbaf63785b53861e1ae2d11b80d6b624211d42cb20efcd210ee6f8a14bc" dependencies = [ "smallvec", ] @@ -5599,7 +3755,7 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "async-trait", - "serde 1.0.203", + "serde", "serde_json", "spin-locked-app", "spin-serde", @@ -5611,7 +3767,7 @@ name = "spin-common" version = "2.8.0-pre0" dependencies = [ "anyhow", - "dirs 4.0.0", + "dirs", "sha2", "tempfile", "tokio", @@ -5625,9 +3781,9 @@ dependencies = [ "anyhow", "tracing", "wasm-encoder 0.200.0", - "wasm-metadata 0.200.0", + "wasm-metadata", "wasmparser 0.200.0", - "wit-component 0.200.0", + "wit-component", "wit-parser 0.200.0", ] @@ -5649,19 +3805,102 @@ dependencies = [ "async-trait", "dotenvy", "once_cell", - "serde 1.0.203", + "serde", "spin-locked-app", "thiserror", ] [[package]] -name = "spin-key-value" +name = "spin-factor-key-value" version = "2.8.0-pre0" dependencies = [ "anyhow", - "lru 0.9.0", - "spin-app", + "serde", + "spin-factors", + "spin-key-value", + "spin-world", + "toml", +] + +[[package]] +name = "spin-factor-key-value-azure" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "serde", + "spin-factor-key-value", + "spin-key-value-azure", +] + +[[package]] +name = "spin-factor-key-value-redis" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "serde", + "spin-factor-key-value", + "spin-key-value-redis", +] + +[[package]] +name = "spin-factor-key-value-spin" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "serde", + "spin-factor-key-value", + "spin-key-value-sqlite", +] + +[[package]] +name = "spin-factor-llm" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "serde", + "spin-factors", + "spin-llm-remote-http", + "spin-locked-app", + "spin-world", + "tokio", + "toml", + "tracing", + "url", +] + +[[package]] +name = "spin-factor-outbound-http" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "http 1.1.0", + "http-body-util", + "hyper 1.4.1", + "reqwest 0.11.27", + "rustls 0.23.12", + "spin-factor-outbound-networking", + "spin-factors", + "spin-telemetry", + "spin-world", + "terminal", + "tokio", + "tokio-rustls 0.26.0", + "tracing", + "wasmtime", + "wasmtime-wasi", + "wasmtime-wasi-http", +] + +[[package]] +name = "spin-factor-outbound-mqtt" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "rumqttc", "spin-core", + "spin-factor-outbound-networking", + "spin-factors", "spin-world", "table", "tokio", @@ -5669,139 +3908,243 @@ dependencies = [ ] [[package]] -name = "spin-key-value-azure" +name = "spin-factor-outbound-mysql" version = "2.8.0-pre0" dependencies = [ "anyhow", - "azure_data_cosmos", - "azure_identity", - "futures", - "serde 1.0.203", + "flate2", + "mysql_async", + "mysql_common", + "spin-app", "spin-core", - "spin-key-value", + "spin-expressions", + "spin-factor-outbound-networking", + "spin-factors", + "spin-outbound-networking", + "spin-world", + "table", "tokio", "tracing", "url", ] [[package]] -name = "spin-key-value-redis" +name = "spin-factor-outbound-networking" version = "2.8.0-pre0" dependencies = [ "anyhow", - "redis", + "futures-util", + "http 1.1.0", + "ipnet", + "rustls 0.23.12", + "rustls-pemfile 2.1.3", + "rustls-pki-types", + "serde", + "spin-factor-variables", + "spin-factor-wasi", + "spin-factors", + "spin-outbound-networking", + "spin-serde", + "tracing", + "webpki-roots 0.26.3", +] + +[[package]] +name = "spin-factor-outbound-pg" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "native-tls", + "postgres-native-tls", "spin-core", - "spin-key-value", + "spin-factor-outbound-networking", + "spin-factors", "spin-world", + "table", "tokio", + "tokio-postgres", "tracing", - "url", ] [[package]] -name = "spin-key-value-sqlite" +name = "spin-factor-outbound-redis" version = "2.8.0-pre0" dependencies = [ "anyhow", - "once_cell", - "rusqlite", + "redis", "spin-core", - "spin-key-value", + "spin-factor-outbound-networking", + "spin-factors", + "spin-world", + "table", + "tracing", +] + +[[package]] +name = "spin-factor-sqlite" +version = "2.8.0-pre0" +dependencies = [ + "async-trait", + "serde", + "spin-factors", + "spin-locked-app", + "spin-sqlite", + "spin-sqlite-inproc", + "spin-sqlite-libsql", + "spin-world", + "table", + "tokio", + "toml", +] + +[[package]] +name = "spin-factor-variables" +version = "2.8.0-pre0" +dependencies = [ + "azure_core", + "azure_identity", + "azure_security_keyvault", + "dotenvy", + "serde", + "spin-expressions", + "spin-factors", "spin-world", "tokio", + "toml", "tracing", + "vaultrs", +] + +[[package]] +name = "spin-factor-wasi" +version = "2.8.0-pre0" +dependencies = [ + "async-trait", + "bytes", + "cap-primitives", + "spin-common", + "spin-factors", + "tokio", + "wasmtime", + "wasmtime-wasi", ] [[package]] -name = "spin-llm" +name = "spin-factors" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "cargo-target-dep", + "serde", + "spin-app", + "spin-factors-derive", + "thiserror", + "toml", + "tracing", + "wasmtime", +] + +[[package]] +name = "spin-factors-derive" +version = "2.8.0-pre0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.75", +] + +[[package]] +name = "spin-factors-executor" version = "2.8.0-pre0" dependencies = [ "anyhow", - "bytesize", - "llm", "spin-app", "spin-core", - "spin-world", + "spin-factors", ] [[package]] -name = "spin-llm-remote-http" +name = "spin-key-value" version = "2.8.0-pre0" dependencies = [ "anyhow", - "http 0.2.11", - "reqwest 0.11.24", - "serde 1.0.203", - "serde_json", - "spin-telemetry", + "lru 0.9.0", + "spin-app", + "spin-core", "spin-world", + "table", + "tokio", "tracing", ] [[package]] -name = "spin-loader" +name = "spin-key-value-azure" version = "2.8.0-pre0" dependencies = [ "anyhow", - "async-trait", - "bytes", - "dirs 4.0.0", - "dunce", + "azure_data_cosmos", + "azure_identity", "futures", - "glob", - "indexmap 1.9.3", - "itertools 0.10.5", - "lazy_static 1.4.0", - "mime_guess", - "path-absolutize", - "regex", - "reqwest 0.11.24", - "semver", - "serde 1.0.203", - "serde_json", - "sha2", - "shellexpand 3.1.0", - "spin-common", - "spin-locked-app", - "spin-manifest", - "spin-outbound-networking", - "tempfile", - "terminal", - "thiserror", + "serde", + "spin-core", + "spin-key-value", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "spin-key-value-redis" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "redis", + "spin-core", + "spin-key-value", + "spin-world", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "spin-key-value-sqlite" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "once_cell", + "rusqlite", + "spin-core", + "spin-key-value", + "spin-world", "tokio", - "tokio-util 0.6.10", - "toml 0.8.14", "tracing", - "walkdir", - "wasm-pkg-loader", ] [[package]] -name = "spin-locked-app" +name = "spin-llm-remote-http" version = "2.8.0-pre0" dependencies = [ "anyhow", - "async-trait", - "serde 1.0.203", + "http 0.2.12", + "reqwest 0.11.27", + "serde", "serde_json", - "spin-serde", - "thiserror", + "spin-telemetry", + "spin-world", + "tracing", ] [[package]] -name = "spin-manifest" +name = "spin-locked-app" version = "2.8.0-pre0" dependencies = [ "anyhow", - "indexmap 1.9.3", - "semver", - "serde 1.0.203", + "async-trait", + "serde", + "serde_json", "spin-serde", - "terminal", "thiserror", - "toml 0.8.14", - "url", - "wasm-pkg-common", ] [[package]] @@ -5818,12 +4161,35 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "spin-runtime-config" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "spin-factor-key-value", + "spin-factor-key-value-azure", + "spin-factor-key-value-redis", + "spin-factor-key-value-spin", + "spin-factor-llm", + "spin-factor-outbound-http", + "spin-factor-outbound-mqtt", + "spin-factor-outbound-mysql", + "spin-factor-outbound-networking", + "spin-factor-outbound-pg", + "spin-factor-outbound-redis", + "spin-factor-sqlite", + "spin-factor-variables", + "spin-factor-wasi", + "spin-factors", + "toml", +] + [[package]] name = "spin-serde" version = "2.8.0-pre0" dependencies = [ "base64 0.21.7", - "serde 1.0.203", + "serde", ] [[package]] @@ -5875,7 +4241,7 @@ name = "spin-telemetry" version = "2.8.0-pre0" dependencies = [ "anyhow", - "http 0.2.11", + "http 0.2.12", "http 1.1.0", "opentelemetry", "opentelemetry-otlp", @@ -5894,75 +4260,34 @@ name = "spin-trigger" version = "2.8.0-pre0" dependencies = [ "anyhow", - "async-trait", - "clap 3.2.25", + "clap", "ctrlc", - "dirs 4.0.0", "futures", - "http 1.1.0", - "indexmap 1.9.3", - "ipnet", - "outbound-http", - "outbound-mqtt", - "outbound-mysql", - "outbound-pg", - "outbound-redis", - "rustls-pemfile 2.1.2", - "rustls-pki-types", "sanitize-filename", - "serde 1.0.203", + "serde", "serde_json", "spin-app", "spin-common", "spin-componentize", "spin-core", - "spin-expressions", - "spin-key-value", - "spin-key-value-azure", - "spin-key-value-redis", - "spin-key-value-sqlite", - "spin-llm", - "spin-llm-remote-http", - "spin-loader", - "spin-manifest", - "spin-outbound-networking", - "spin-serde", - "spin-sqlite", - "spin-sqlite-inproc", - "spin-sqlite-libsql", + "spin-factor-key-value", + "spin-factor-llm", + "spin-factor-outbound-http", + "spin-factor-outbound-mqtt", + "spin-factor-outbound-mysql", + "spin-factor-outbound-networking", + "spin-factor-outbound-pg", + "spin-factor-outbound-redis", + "spin-factor-sqlite", + "spin-factor-variables", + "spin-factor-wasi", + "spin-factors", + "spin-factors-executor", + "spin-runtime-config", "spin-telemetry", - "spin-variables", - "spin-world", "terminal", "tokio", - "toml 0.5.11", - "tracing", - "url", - "wasmtime", - "wasmtime-wasi", - "wasmtime-wasi-http", -] - -[[package]] -name = "spin-variables" -version = "2.8.0-pre0" -dependencies = [ - "anyhow", - "async-trait", - "azure_core", - "azure_identity", - "azure_security_keyvault", - "dotenvy", - "once_cell", - "serde 1.0.203", - "spin-app", - "spin-core", - "spin-expressions", - "spin-world", - "thiserror", - "tokio", "tracing", - "vaultrs", ] [[package]] @@ -5973,28 +4298,6 @@ dependencies = [ "wasmtime", ] -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64 0.13.1", - "nom 7.1.3", - "serde 1.0.203", - "unicode-segmentation", -] - [[package]] name = "sptr" version = "0.3.2" @@ -6024,13 +4327,13 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] @@ -6039,12 +4342,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "subprocess" version = "0.2.9" @@ -6057,9 +4354,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -6074,9 +4371,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -6089,6 +4386,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "synstructure" version = "0.12.6" @@ -6132,8 +4435,8 @@ dependencies = [ "cap-fs-ext", "cap-std", "fd-lock", - "io-lifetimes 2.0.3", - "rustix 0.38.31", + "io-lifetimes", + "rustix", "windows-sys 0.52.0", "winx", ] @@ -6142,17 +4445,6 @@ dependencies = [ name = "table" version = "2.8.0-pre0" -[[package]] -name = "tar" -version = "0.4.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "target-lexicon" version = "0.12.13" @@ -6161,13 +4453,14 @@ checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" +checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" dependencies = [ "cfg-if", - "fastrand 2.0.1", - "rustix 0.38.31", + "fastrand 2.1.0", + "once_cell", + "rustix", "windows-sys 0.52.0", ] @@ -6212,7 +4505,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -6238,7 +4531,7 @@ dependencies = [ "num-conv", "num_threads", "powerfmt", - "serde 1.0.203", + "serde", "time-core", "time-macros", ] @@ -6259,20 +4552,11 @@ dependencies = [ "time-core", ] -[[package]] -name = "tint" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af24570664a3074673dbbf69a65bdae0ae0b72f2949b1adfbacb736ee4d6896" -dependencies = [ - "lazy_static 0.2.11", -] - [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -6283,40 +4567,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tokenizers" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aea68938177975ab09da68552b720eac941779ff386baceaf77e0f5f9cea645f" -dependencies = [ - "aho-corasick 0.7.20", - "cached-path", - "derive_builder 0.12.0", - "dirs 4.0.0", - "esaxx-rs", - "getrandom 0.2.12", - "itertools 0.9.0", - "lazy_static 1.4.0", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand 0.8.5", - "rayon", - "rayon-cond", - "regex", - "regex-syntax 0.7.5", - "reqwest 0.11.24", - "serde 1.0.203", - "serde_json", - "spm_precompiled", - "thiserror", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - [[package]] name = "tokio" version = "1.36.0" @@ -6331,7 +4581,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2", "tokio-macros", "windows-sys 0.48.0", ] @@ -6354,7 +4604,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -6369,9 +4619,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" +checksum = "03adcf0147e203b6032c0b2d30be1415ba03bc348901f3ff1cc0df6a733e60c3" dependencies = [ "async-trait", "byteorder", @@ -6387,9 +4637,9 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.8.5", - "socket2 0.5.5", + "socket2", "tokio", - "tokio-util 0.7.10", + "tokio-util", "whoami", ] @@ -6399,7 +4649,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.10", + "rustls 0.21.12", "tokio", ] @@ -6415,25 +4665,24 @@ dependencies = [ ] [[package]] -name = "tokio-scoped" -version = "0.2.0" +name = "tokio-rustls" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4beb8ba13bc53ac53ce1d52b42f02e5d8060f0f42138862869beb769722b256" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ + "rustls 0.23.12", + "rustls-pki-types", "tokio", - "tokio-stream", ] [[package]] -name = "tokio-socks" -version = "0.5.1" +name = "tokio-scoped" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0" +checksum = "e4beb8ba13bc53ac53ce1d52b42f02e5d8060f0f42138862869beb769722b256" dependencies = [ - "either", - "futures-util", - "thiserror", "tokio", + "tokio-stream", ] [[package]] @@ -6449,39 +4698,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde 1.0.203", ] [[package]] @@ -6490,11 +4715,10 @@ version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ - "indexmap 2.2.6", - "serde 1.0.203", + "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit", ] [[package]] @@ -6503,18 +4727,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ - "serde 1.0.203", -] - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", + "serde", ] [[package]] @@ -6524,10 +4737,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ "indexmap 2.2.6", - "serde 1.0.203", + "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.13", + "winnow", ] [[package]] @@ -6542,9 +4755,9 @@ dependencies = [ "base64 0.21.7", "bytes", "h2 0.3.26", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.30", "hyper-timeout", "percent-encoding", "pin-project", @@ -6571,7 +4784,7 @@ dependencies = [ "rand 0.8.5", "slab", "tokio", - "tokio-util 0.7.10", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -6579,15 +4792,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -6621,7 +4834,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -6657,7 +4870,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" dependencies = [ - "serde 1.0.203", + "serde", "tracing-core", ] @@ -6671,7 +4884,7 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", - "serde 1.0.203", + "serde", "serde_json", "sharded-slab", "smallvec", @@ -6686,11 +4899,9 @@ name = "trigger-timer" version = "0.1.0" dependencies = [ "anyhow", - "clap 3.2.25", + "clap", "futures", - "serde 1.0.203", - "spin-app", - "spin-core", + "serde", "spin-trigger", "tokio", "tokio-scoped", @@ -6729,17 +4940,6 @@ dependencies = [ "const_fn", ] -[[package]] -name = "uds_windows" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89daebc3e6fd160ac4aa9fc8b3bf71e1f74fbf92367ae71fb83a037e8bf164b9" -dependencies = [ - "memoffset 0.9.0", - "tempfile", - "winapi", -] - [[package]] name = "uncased" version = "0.9.10" @@ -6749,15 +4949,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.15" @@ -6772,27 +4963,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.11.0" +name = "unicode-properties" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" [[package]] name = "unicode-width" @@ -6806,18 +4988,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -6826,14 +4996,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", "percent-encoding", - "serde 1.0.203", + "serde", ] [[package]] @@ -6842,12 +5012,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - [[package]] name = "uuid" version = "1.7.0" @@ -6871,191 +5035,43 @@ checksum = "267f958930e08323a44c12e6c5461f3eaaa16d88785e9ec8550215b8aafc3d0b" dependencies = [ "async-trait", "bytes", - "derive_builder 0.11.2", - "http 0.2.11", - "reqwest 0.11.24", + "derive_builder", + "http 0.2.12", + "reqwest 0.11.27", "rustify", "rustify_derive", - "serde 1.0.203", - "serde_json", - "thiserror", - "tracing", - "url", -] - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "waker-fn" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" - -[[package]] -name = "walkdir" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "warg-api" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a22d3c9026f2f6a628cf386963844cdb7baea3b3419ba090c9096da114f977d" -dependencies = [ - "indexmap 2.2.6", - "itertools 0.12.1", - "serde 1.0.203", - "serde_with", - "thiserror", - "warg-crypto", - "warg-protocol", -] - -[[package]] -name = "warg-client" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8b5a2b17e737e1847dbf4642e4ebe49f5df32a574520251ff080ef0a120423" -dependencies = [ - "anyhow", - "async-recursion", - "async-trait", - "bytes", - "clap 4.5.4", - "dialoguer", - "dirs 5.0.1", - "futures-util", - "indexmap 2.2.6", - "itertools 0.12.1", - "keyring", - "libc", - "normpath", - "once_cell", - "pathdiff", - "ptree", - "reqwest 0.12.4", - "secrecy", - "semver", - "serde 1.0.203", + "serde", "serde_json", - "sha256", - "tempfile", "thiserror", - "tokio", - "tokio-util 0.7.10", "tracing", "url", - "walkdir", - "warg-api", - "warg-crypto", - "warg-protocol", - "warg-transparency", - "wasm-compose", - "wasm-encoder 0.41.2", - "wasmparser 0.121.2", - "wasmprinter 0.2.80", - "windows-sys 0.52.0", ] [[package]] -name = "warg-crypto" -version = "0.7.0" +name = "vcpkg" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "834bf58863aa4bc3821732afb0c77e08a5cbf05f63ee93116acae694eab04460" -dependencies = [ - "anyhow", - "base64 0.21.7", - "digest", - "hex", - "leb128", - "once_cell", - "p256", - "rand_core 0.6.4", - "secrecy", - "serde 1.0.203", - "sha2", - "signature", - "thiserror", -] +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] -name = "warg-protobuf" -version = "0.7.0" +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8a2dee6b14f5b0b0c461711a81cdef45d45ea94f8460cb6205cada7fec732a" -dependencies = [ - "anyhow", - "pbjson", - "pbjson-build", - "pbjson-types", - "prost", - "prost-build", - "prost-types", - "protox", - "regex", - "serde 1.0.203", - "warg-crypto", -] +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "warg-protocol" -version = "0.7.0" +name = "waker-fn" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4053a3276d3fee83645411b1b5f462f72402e70fbf645164274a3a0a2fd72538" -dependencies = [ - "anyhow", - "base64 0.21.7", - "hex", - "indexmap 2.2.6", - "pbjson-types", - "prost", - "prost-types", - "semver", - "serde 1.0.203", - "serde_with", - "thiserror", - "warg-crypto", - "warg-protobuf", - "warg-transparency", - "wasmparser 0.121.2", -] +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" [[package]] -name = "warg-transparency" -version = "0.7.0" +name = "want" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513ef81a5bb1ac5d7bd04f90d3c192dad8f590f4c02b3ef68d3ae4fbbb53c1d7" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "anyhow", - "indexmap 2.2.6", - "prost", - "thiserror", - "warg-crypto", - "warg-protobuf", + "try-lock", ] [[package]] @@ -7078,34 +5094,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -7115,9 +5132,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7125,53 +5142,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" - -[[package]] -name = "wasm-compose" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd324927af875ebedb1b820c00e3c585992d33c2c787c5021fe6d8982527359b" -dependencies = [ - "anyhow", - "heck 0.4.1", - "im-rc", - "indexmap 2.2.6", - "log", - "petgraph", - "serde 1.0.203", - "serde_derive", - "serde_yaml", - "smallvec", - "wasm-encoder 0.41.2", - "wasmparser 0.121.2", - "wat", -] - -[[package]] -name = "wasm-encoder" -version = "0.41.2" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "972f97a5d8318f908dded23594188a90bcd09365986b1163e66d70170e5287ae" -dependencies = [ - "leb128", - "wasmparser 0.121.2", -] +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-encoder" @@ -7208,7 +5194,7 @@ checksum = "c31b8cc0c21f46d55b0aaa419cacce1eadcf28eaebd0e1488d6a6313ee71a586" dependencies = [ "anyhow", "indexmap 2.2.6", - "serde 1.0.203", + "serde", "serde_derive", "serde_json", "spdx", @@ -7216,71 +5202,6 @@ dependencies = [ "wasmparser 0.200.0", ] -[[package]] -name = "wasm-metadata" -version = "0.209.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d32029ce424f6d3c2b39b4419fb45a0e2d84fb0751e0c0a32b7ce8bd5d97f46" -dependencies = [ - "anyhow", - "indexmap 2.2.6", - "serde 1.0.203", - "serde_derive", - "serde_json", - "spdx", - "wasm-encoder 0.209.1", - "wasmparser 0.209.1", -] - -[[package]] -name = "wasm-pkg-common" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7a687d110f68a65227a644c7040c7720220e8cb0bb8c803e2b5dcb7fd72468" -dependencies = [ - "anyhow", - "dirs 5.0.1", - "http 1.1.0", - "reqwest 0.12.4", - "semver", - "serde 1.0.203", - "serde_json", - "thiserror", - "toml 0.8.14", - "tracing", -] - -[[package]] -name = "wasm-pkg-loader" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11338b173351bc505bc752c00068a7d1da5106a9d351753f0d01267dcc4747b2" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.22.0", - "bytes", - "dirs 5.0.1", - "docker_credential", - "futures-util", - "oci-distribution", - "oci-wasm", - "secrecy", - "serde 1.0.203", - "serde_json", - "sha2", - "thiserror", - "tokio", - "tokio-util 0.7.10", - "toml 0.8.14", - "tracing", - "tracing-subscriber", - "url", - "warg-client", - "warg-protocol", - "wasm-pkg-common", -] - [[package]] name = "wasm-streams" version = "0.4.0" @@ -7294,17 +5215,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.121.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" -dependencies = [ - "bitflags 2.4.2", - "indexmap 2.2.6", - "semver", -] - [[package]] name = "wasmparser" version = "0.200.0" @@ -7327,17 +5237,7 @@ dependencies = [ "hashbrown 0.14.3", "indexmap 2.2.6", "semver", - "serde 1.0.203", -] - -[[package]] -name = "wasmprinter" -version = "0.2.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e73986a6b7fdfedb7c5bf9e7eb71135486507c8fbc4c0c42cffcb6532988b7" -dependencies = [ - "anyhow", - "wasmparser 0.121.2", + "serde", ] [[package]] @@ -7373,16 +5273,16 @@ dependencies = [ "log", "mach2", "memfd", - "memoffset 0.9.0", + "memoffset", "object 0.36.0", "once_cell", "paste", "postcard", "psm", "rayon", - "rustix 0.38.31", + "rustix", "semver", - "serde 1.0.203", + "serde", "serde_derive", "serde_json", "smallvec", @@ -7426,11 +5326,11 @@ dependencies = [ "directories-next", "log", "postcard", - "rustix 0.38.31", - "serde 1.0.203", + "rustix", + "serde", "serde_derive", "sha2", - "toml 0.8.14", + "toml", "windows-sys 0.52.0", "zstd 0.13.1", ] @@ -7444,7 +5344,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser 0.209.1", @@ -7495,12 +5395,12 @@ dependencies = [ "object 0.36.0", "postcard", "rustc-demangle", - "serde 1.0.203", + "serde", "serde_derive", "target-lexicon", "wasm-encoder 0.209.1", "wasmparser 0.209.1", - "wasmprinter 0.209.1", + "wasmprinter", "wasmtime-component-util", "wasmtime-types", ] @@ -7514,7 +5414,7 @@ dependencies = [ "anyhow", "cc", "cfg-if", - "rustix 0.38.31", + "rustix", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", "windows-sys 0.52.0", @@ -7528,7 +5428,7 @@ checksum = "9bc54198c6720f098210a85efb3ba8c078d1de4d373cdb6778850a66ae088d11" dependencies = [ "object 0.36.0", "once_cell", - "rustix 0.38.31", + "rustix", "wasmtime-versioned-export-macros", ] @@ -7557,7 +5457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "412463e9000e14cf6856be48628d2213c20c153e29ffc22b036980c892ea6964" dependencies = [ "cranelift-entity", - "serde 1.0.203", + "serde", "serde_derive", "smallvec", "wasmparser 0.209.1", @@ -7571,7 +5471,7 @@ checksum = "de5a9bc4f44ceeb168e9e8e3be4e0b4beb9095b468479663a9e24c667e36826f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] @@ -7592,9 +5492,9 @@ dependencies = [ "fs-set-times", "futures", "io-extras", - "io-lifetimes 2.0.3", + "io-lifetimes", "once_cell", - "rustix 0.38.31", + "rustix", "system-interface", "thiserror", "tokio", @@ -7616,16 +5516,16 @@ dependencies = [ "bytes", "futures", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "rustls 0.22.4", "tokio", "tokio-rustls 0.25.0", "tracing", "wasmtime", "wasmtime-wasi", - "webpki-roots 0.26.1", + "webpki-roots 0.26.3", ] [[package]] @@ -7652,7 +5552,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc077306b38288262e5ba01d4b21532a6987416cdc0aedf04bb06c22a68fdc" dependencies = [ "anyhow", - "heck 0.4.1", + "heck", "indexmap 2.2.6", "wit-parser 0.209.1", ] @@ -7690,9 +5590,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -7716,13 +5616,25 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "whoami" version = "1.5.1" @@ -7756,11 +5668,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "557567f2793508760cd855f7659b7a0b9dc4dbc451f53f1415d6943a15311ade" dependencies = [ "anyhow", - "heck 0.4.1", + "heck", "proc-macro2", "quote", - "shellexpand 2.1.2", - "syn 2.0.48", + "shellexpand", + "syn 2.0.75", "witx", ] @@ -7772,7 +5684,7 @@ checksum = "cc26129a8aea20b62c961d1b9ab4a3c3b56b10042ed85d004f8678af0f21ba6e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", "wiggle-generate", ] @@ -7965,15 +5877,6 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.6.13" @@ -8023,34 +5926,15 @@ dependencies = [ "bitflags 2.4.2", "indexmap 2.2.6", "log", - "serde 1.0.203", + "serde", "serde_derive", "serde_json", "wasm-encoder 0.200.0", - "wasm-metadata 0.200.0", + "wasm-metadata", "wasmparser 0.200.0", "wit-parser 0.200.0", ] -[[package]] -name = "wit-component" -version = "0.209.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a2bb5b039f9cb03425e1d5a6e54b441ca4ca1b1d4fa6a0924db67a55168f99" -dependencies = [ - "anyhow", - "bitflags 2.4.2", - "indexmap 2.2.6", - "log", - "serde 1.0.203", - "serde_derive", - "serde_json", - "wasm-encoder 0.209.1", - "wasm-metadata 0.209.1", - "wasmparser 0.209.1", - "wit-parser 0.209.1", -] - [[package]] name = "wit-parser" version = "0.200.0" @@ -8062,7 +5946,7 @@ dependencies = [ "indexmap 2.2.6", "log", "semver", - "serde 1.0.203", + "serde", "serde_derive", "serde_json", "unicode-xid", @@ -8080,7 +5964,7 @@ dependencies = [ "indexmap 2.2.6", "log", "semver", - "serde 1.0.203", + "serde", "serde_derive", "serde_json", "unicode-xid", @@ -8099,108 +5983,13 @@ dependencies = [ "wast 35.0.2", ] -[[package]] -name = "xattr" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" -dependencies = [ - "libc", - "linux-raw-sys 0.4.13", - "rustix 0.38.31", -] - -[[package]] -name = "xdg-home" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca91dcf8f93db085f3a0a29358cd0b9d670915468f4290e8b85d118a34211ab8" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - -[[package]] -name = "zbus" -version = "3.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "675d170b632a6ad49804c8cf2105d7c31eddd3312555cffd4b740e08e97c25e6" -dependencies = [ - "async-broadcast", - "async-executor", - "async-fs", - "async-io 1.13.0", - "async-lock 2.8.0", - "async-process 1.8.1", - "async-recursion", - "async-task", - "async-trait", - "blocking", - "byteorder", - "derivative", - "enumflags2", - "event-listener 2.5.3", - "futures-core", - "futures-sink", - "futures-util", - "hex", - "nix 0.26.4", - "once_cell", - "ordered-stream", - "rand 0.8.5", - "serde 1.0.203", - "serde_repr", - "sha1 0.10.6", - "static_assertions", - "tracing", - "uds_windows", - "winapi", - "xdg-home", - "zbus_macros", - "zbus_names", - "zvariant", -] - -[[package]] -name = "zbus_macros" -version = "3.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7131497b0f887e8061b430c530240063d33bf9455fa34438f388a245da69e0a5" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "regex", - "syn 1.0.109", - "zvariant_utils", -] - -[[package]] -name = "zbus_names" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "437d738d3750bed6ca9b8d423ccc7a8eb284f6b1d6d4e225a0e4e6258d864c8d" -dependencies = [ - "serde 1.0.203", - "static_assertions", - "zvariant", -] - [[package]] name = "zerocopy" version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -8212,42 +6001,27 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.75", ] [[package]] name = "zeroize" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" - -[[package]] -name = "zip" -version = "0.6.6" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ - "aes", - "byteorder", - "bzip2", - "constant_time_eq", - "crc32fast", - "crossbeam-utils", - "flate2", - "hmac", - "pbkdf2", - "sha1 0.10.6", - "time", - "zstd 0.11.2+zstd.1.5.2", + "zeroize_derive", ] [[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" +name = "zeroize_derive" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", + "proc-macro2", + "quote", + "syn 2.0.75", ] [[package]] @@ -8268,16 +6042,6 @@ dependencies = [ "zstd-safe 7.1.0", ] -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - [[package]] name = "zstd-safe" version = "6.0.6" @@ -8306,41 +6070,3 @@ dependencies = [ "cc", "pkg-config", ] - -[[package]] -name = "zvariant" -version = "3.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eef2be88ba09b358d3b58aca6e41cd853631d44787f319a1383ca83424fb2db" -dependencies = [ - "byteorder", - "enumflags2", - "libc", - "serde 1.0.203", - "static_assertions", - "zvariant_derive", -] - -[[package]] -name = "zvariant_derive" -version = "3.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c24dc0bed72f5f90d1f8bb5b07228cbf63b3c6e9f82d82559d4bae666e7ed9" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 1.0.109", - "zvariant_utils", -] - -[[package]] -name = "zvariant_utils" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7234f0d811589db492d16893e3f21e8e2fd282e6d01b0cddee310322062cc200" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] diff --git a/examples/spin-timer/Cargo.toml b/examples/spin-timer/Cargo.toml index ec535dcf0..5d2d37d4b 100644 --- a/examples/spin-timer/Cargo.toml +++ b/examples/spin-timer/Cargo.toml @@ -9,9 +9,7 @@ anyhow = "1.0.68" clap = { version = "3.1.15", features = ["derive", "env"] } futures = "0.3.25" serde = "1.0.188" -spin-app = { path = "../../crates/app" } -spin-core = { path = "../../crates/core" } -# spin-trigger = { path = "../../crates/trigger" } +spin-trigger = { path = "../../crates/trigger" } tokio = { version = "1.11", features = ["full"] } tokio-scoped = "0.2.0" wasmtime = "22.0.0" diff --git a/examples/spin-timer/src/lib.rs b/examples/spin-timer/src/lib.rs index a9e9be54e..58bb7e8e9 100644 --- a/examples/spin-timer/src/lib.rs +++ b/examples/spin-timer/src/lib.rs @@ -2,9 +2,7 @@ use std::collections::HashMap; use clap::Args; use serde::{Deserialize, Serialize}; -use spin_app::MetadataKey; -use spin_core::{async_trait, InstancePre}; -use spin_trigger::{TriggerAppEngine, TriggerExecutor}; +use spin_trigger::{App, Trigger, TriggerApp}; wasmtime::component::bindgen!({ path: ".", @@ -12,9 +10,6 @@ wasmtime::component::bindgen!({ async: true }); -pub(crate) type RuntimeData = (); -pub(crate) type _Store = spin_core::Store; - #[derive(Args)] pub struct CliArgs { /// If true, run each component once and exit @@ -24,7 +19,7 @@ pub struct CliArgs { // The trigger structure with all values processed and ready pub struct TimerTrigger { - engine: TriggerAppEngine, + test: bool, speedup: u64, component_timings: HashMap, } @@ -50,45 +45,36 @@ pub struct TimerTriggerConfig { interval_secs: u64, } -const TRIGGER_METADATA_KEY: MetadataKey = MetadataKey::new("triggers"); - -#[async_trait] -impl TriggerExecutor for TimerTrigger { - const TRIGGER_TYPE: &'static str = "timer"; - - type RuntimeData = RuntimeData; +impl Trigger for TimerTrigger { + const TYPE: &'static str = "timer"; - type TriggerConfig = TimerTriggerConfig; + type CliArgs = CliArgs; - type RunConfig = CliArgs; + type InstanceState = (); - type InstancePre = InstancePre; + fn new(cli_args: Self::CliArgs, app: &App) -> anyhow::Result { + let metadata = app + .get_trigger_metadata::(Self::TYPE)? + .unwrap_or_default(); + let speedup = metadata.speedup.unwrap_or(1); - async fn new(engine: spin_trigger::TriggerAppEngine) -> anyhow::Result { - let speedup = engine - .app() - .require_metadata(TRIGGER_METADATA_KEY)? - .timer - .unwrap_or_default() - .speedup - .unwrap_or(1); - - let component_timings = engine - .trigger_configs() + let component_timings = app + .trigger_configs::(Self::TYPE)? + .into_iter() .map(|(_, config)| (config.component.clone(), config.interval_secs)) .collect(); Ok(Self { - engine, + test: cli_args.test, speedup, component_timings, }) } - async fn run(self, config: Self::RunConfig) -> anyhow::Result<()> { - if config.test { + async fn run(self, trigger_app: TriggerApp) -> anyhow::Result<()> { + if self.test { for component in self.component_timings.keys() { - self.handle_timer_event(component).await?; + self.handle_timer_event(&trigger_app, component).await?; } } else { // This trigger spawns threads, which Ctrl+C does not kill. So @@ -102,12 +88,16 @@ impl TriggerExecutor for TimerTrigger { let speedup = self.speedup; tokio_scoped::scope(|scope| { // For each component, run its own timer loop - for (c, d) in &self.component_timings { + for (component_id, interval_secs) in &self.component_timings { scope.spawn(async { - let duration = tokio::time::Duration::from_millis(*d * 1000 / speedup); + let duration = + tokio::time::Duration::from_millis(*interval_secs * 1000 / speedup); loop { tokio::time::sleep(duration).await; - self.handle_timer_event(c).await.unwrap(); + + self.handle_timer_event(&trigger_app, component_id) + .await + .unwrap(); } }); } @@ -118,11 +108,14 @@ impl TriggerExecutor for TimerTrigger { } impl TimerTrigger { - async fn handle_timer_event(&self, component_id: &str) -> anyhow::Result<()> { - // Load the guest... - let (instance, mut store) = self.engine.prepare_instance(component_id).await?; - let instance = SpinTimer::new(&mut store, &instance)?; - // ...and call the entry point - instance.call_handle_timer_request(&mut store).await + async fn handle_timer_event( + &self, + trigger_app: &TriggerApp, + component_id: &str, + ) -> anyhow::Result<()> { + let instance_builder = trigger_app.prepare(component_id)?; + let (instance, mut store) = instance_builder.instantiate(()).await?; + let timer = SpinTimer::new(&mut store, &instance)?; + timer.call_handle_timer_request(&mut store).await } } diff --git a/examples/spin-timer/src/main.rs b/examples/spin-timer/src/main.rs index 49ba3dfab..d9c7203b3 100644 --- a/examples/spin-timer/src/main.rs +++ b/examples/spin-timer/src/main.rs @@ -1,9 +1,10 @@ use anyhow::Error; use clap::Parser; -use spin_trigger::cli::TriggerExecutorCommand; +use spin_trigger::cli::FactorsTriggerCommand; + use trigger_timer::TimerTrigger; -type Command = TriggerExecutorCommand; +type Command = FactorsTriggerCommand; #[tokio::main] async fn main() -> Result<(), Error> { From 0759b927ad938dfbd534ffa31b0dcd57981f2b35 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 23 Aug 2024 10:59:40 -0400 Subject: [PATCH 164/195] factors: Fix clippy lints Signed-off-by: Lann Martin --- crates/factor-key-value-azure/src/lib.rs | 3 ++- crates/factor-key-value-redis/src/lib.rs | 3 ++- crates/factor-outbound-redis/src/lib.rs | 3 ++- crates/factor-sqlite/src/host.rs | 3 ++- crates/factor-sqlite/tests/factor_test.rs | 3 ++- crates/factor-variables/src/spin_cli/azure_key_vault.rs | 1 + crates/factor-variables/tests/factor_test.rs | 2 +- crates/runtime-config/src/lib.rs | 6 +----- 8 files changed, 13 insertions(+), 11 deletions(-) diff --git a/crates/factor-key-value-azure/src/lib.rs b/crates/factor-key-value-azure/src/lib.rs index bf5e8067a..cad2da1ee 100644 --- a/crates/factor-key-value-azure/src/lib.rs +++ b/crates/factor-key-value-azure/src/lib.rs @@ -5,6 +5,7 @@ use spin_key_value_azure::{ }; /// A key-value store that uses Azure Cosmos as the backend. +#[derive(Default)] pub struct AzureKeyValueStore { _priv: (), } @@ -12,7 +13,7 @@ pub struct AzureKeyValueStore { impl AzureKeyValueStore { /// Creates a new `AzureKeyValueStore`. pub fn new() -> Self { - Self { _priv: () } + Self::default() } } diff --git a/crates/factor-key-value-redis/src/lib.rs b/crates/factor-key-value-redis/src/lib.rs index f8da8a980..67d71ac3e 100644 --- a/crates/factor-key-value-redis/src/lib.rs +++ b/crates/factor-key-value-redis/src/lib.rs @@ -3,6 +3,7 @@ use spin_factor_key_value::runtime_config::spin::MakeKeyValueStore; use spin_key_value_redis::KeyValueRedis; /// A key-value store that uses Redis as the backend. +#[derive(Default)] pub struct RedisKeyValueStore { _priv: (), } @@ -10,7 +11,7 @@ pub struct RedisKeyValueStore { impl RedisKeyValueStore { /// Creates a new `RedisKeyValueStore`. pub fn new() -> Self { - Self { _priv: () } + Self::default() } } diff --git a/crates/factor-outbound-redis/src/lib.rs b/crates/factor-outbound-redis/src/lib.rs index c0ff5924d..1c9f13752 100644 --- a/crates/factor-outbound-redis/src/lib.rs +++ b/crates/factor-outbound-redis/src/lib.rs @@ -8,13 +8,14 @@ use spin_factors::{ }; /// The [`Factor`] for `fermyon:spin/outbound-redis`. +#[derive(Default)] pub struct OutboundRedisFactor { _priv: (), } impl OutboundRedisFactor { pub fn new() -> Self { - Self { _priv: () } + Self::default() } } diff --git a/crates/factor-sqlite/src/host.rs b/crates/factor-sqlite/src/host.rs index 8911c37c4..92d4b146d 100644 --- a/crates/factor-sqlite/src/host.rs +++ b/crates/factor-sqlite/src/host.rs @@ -44,9 +44,10 @@ impl InstanceState { fn get_connection( &self, connection: Resource, - ) -> Result<&Box, v2::Error> { + ) -> Result<&dyn Connection, v2::Error> { self.connections .get(connection.rep()) + .map(|conn| conn.as_ref()) .ok_or(v2::Error::InvalidConnection) } } diff --git a/crates/factor-sqlite/tests/factor_test.rs b/crates/factor-sqlite/tests/factor_test.rs index 5b9972c9e..e2e2ef213 100644 --- a/crates/factor-sqlite/tests/factor_test.rs +++ b/crates/factor-sqlite/tests/factor_test.rs @@ -104,7 +104,8 @@ impl FactorRuntimeConfigSource for TomlRuntimeSource<'_> { impl RuntimeConfigSourceFinalizer for TomlRuntimeSource<'_> { fn finalize(&mut self) -> anyhow::Result<()> { - Ok(self.table.validate_all_keys_used().unwrap()) + self.table.validate_all_keys_used()?; + Ok(()) } } diff --git a/crates/factor-variables/src/spin_cli/azure_key_vault.rs b/crates/factor-variables/src/spin_cli/azure_key_vault.rs index a832fa536..54fe15bbc 100644 --- a/crates/factor-variables/src/spin_cli/azure_key_vault.rs +++ b/crates/factor-variables/src/spin_cli/azure_key_vault.rs @@ -90,6 +90,7 @@ pub enum AzureKeyVaultAuthOptions { /// /// Common across each: /// - `AZURE_AUTHORITY_HOST`: (optional) the host for the identity provider. For example, for Azure public cloud the host defaults to "https://login.microsoftonline.com". + /// /// See also: https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/identity/README.md Environmental, } diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index 1ce403000..49358ee62 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -31,7 +31,7 @@ async fn static_provider_works() -> anyhow::Result<()> { }))?; let mut state = env.build_instance_state().await?; - let val = state.variables.get("baz".try_into().unwrap()).await?; + let val = state.variables.get("baz".into()).await?; assert_eq!(val, ""); Ok(()) } diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 77ca2a8ec..cc5e07811 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -187,11 +187,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource< impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { - Ok(llm::runtime_config_from_toml( - self.table.as_ref(), - self.state_dir.clone(), - self.use_gpu, - )?) + llm::runtime_config_from_toml(self.table.as_ref(), self.state_dir.clone(), self.use_gpu) } } From eddff506c890afd78a622549929d234f46f74c59 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 23 Aug 2024 14:36:46 -0400 Subject: [PATCH 165/195] factors: Fix wagi support Signed-off-by: Lann Martin --- crates/componentize/src/bugs.rs | 87 +++++++++---------- crates/componentize/src/lib.rs | 68 ++++++++------- crates/componentize/src/module_info.rs | 111 +++++++++++++++++++++++++ crates/http/src/config.rs | 3 +- crates/trigger-http/src/server.rs | 29 +++++-- crates/trigger-http/src/wagi.rs | 35 +++----- crates/trigger-http/src/wasi.rs | 3 +- crates/trigger/src/cli.rs | 2 +- examples/spin-wagi-http/spin.toml | 2 +- examples/wagi-http-rust/spin.toml | 2 +- 10 files changed, 229 insertions(+), 113 deletions(-) create mode 100644 crates/componentize/src/module_info.rs diff --git a/crates/componentize/src/bugs.rs b/crates/componentize/src/bugs.rs index 1e7ed9e63..053dc4aa3 100644 --- a/crates/componentize/src/bugs.rs +++ b/crates/componentize/src/bugs.rs @@ -1,6 +1,6 @@ -use anyhow::bail; -use wasm_metadata::Producers; -use wasmparser::{Encoding, ExternalKind, Parser, Payload}; +use crate::module_info::ModuleInfo; + +pub const EARLIEST_PROBABLY_SAFE_CLANG_VERSION: &str = "15.0.7"; /// Represents the detected likelihood of the allocation bug fixed in /// https://github.com/WebAssembly/wasi-libc/pull/377 being present in a Wasm @@ -8,53 +8,35 @@ use wasmparser::{Encoding, ExternalKind, Parser, Payload}; #[derive(Debug, PartialEq)] pub enum WasiLibc377Bug { ProbablySafe, - ProbablyUnsafe, + ProbablyUnsafe { clang_version: String }, Unknown, } impl WasiLibc377Bug { - pub fn detect(module: &[u8]) -> anyhow::Result { - for payload in Parser::new(0).parse_all(module) { - match payload? { - Payload::Version { encoding, .. } if encoding != Encoding::Module => { - bail!("detection only applicable to modules"); - } - Payload::ExportSection(reader) => { - for export in reader { - let export = export?; - if export.kind == ExternalKind::Func && export.name == "cabi_realloc" { - // `cabi_realloc` is a good signal that this module - // uses wit-bindgen, making it probably-safe. - tracing::debug!("Found cabi_realloc export"); - return Ok(Self::ProbablySafe); - } - } - } - Payload::CustomSection(c) if c.name() == "producers" => { - let producers = Producers::from_bytes(c.data(), c.data_offset())?; - if let Some(clang_version) = - producers.get("processed-by").and_then(|f| f.get("clang")) - { - tracing::debug!(clang_version, "Parsed producers.processed-by.clang"); - - // Clang/LLVM version is a good proxy for wasi-sdk - // version; the allocation bug was fixed in wasi-sdk-18 - // and LLVM was updated to 15.0.7 in wasi-sdk-19. - if let Some((major, minor, patch)) = parse_clang_version(clang_version) { - return if (major, minor, patch) >= (15, 0, 7) { - Ok(Self::ProbablySafe) - } else { - Ok(Self::ProbablyUnsafe) - }; - } else { - tracing::warn!( - clang_version, - "Unexpected producers.processed-by.clang version" - ); - } - } - } - _ => (), + pub fn detect(module_info: &ModuleInfo) -> anyhow::Result { + if module_info.probably_uses_wit_bindgen() { + // Modules built with wit-bindgen are probably safe. + return Ok(Self::ProbablySafe); + } + if let Some(clang_version) = &module_info.clang_version { + // Clang/LLVM version is a good proxy for wasi-sdk + // version; the allocation bug was fixed in wasi-sdk-18 + // and LLVM was updated to 15.0.7 in wasi-sdk-19. + if let Some((major, minor, patch)) = parse_clang_version(clang_version) { + let earliest_safe = + parse_clang_version(EARLIEST_PROBABLY_SAFE_CLANG_VERSION).unwrap(); + return if (major, minor, patch) >= earliest_safe { + Ok(Self::ProbablySafe) + } else { + Ok(Self::ProbablyUnsafe { + clang_version: clang_version.clone(), + }) + }; + } else { + tracing::warn!( + clang_version, + "Unexpected producers.processed-by.clang version" + ); } } Ok(Self::Unknown) @@ -98,11 +80,15 @@ mod tests { ), ( r#"(module (@producers (processed-by "clang" "15.0.6")))"#, - ProbablyUnsafe, + ProbablyUnsafe { + clang_version: "15.0.6".into(), + }, ), ( - r#"(module (@producers (processed-by "clang" "14.0.0")))"#, - ProbablyUnsafe, + r#"(module (@producers (processed-by "clang" "14.0.0 extra-stuff")))"#, + ProbablyUnsafe { + clang_version: "14.0.0 extra-stuff".into(), + }, ), ( r#"(module (@producers (processed-by "clang" "a.b.c")))"#, @@ -111,7 +97,8 @@ mod tests { ] { eprintln!("WAT: {wasm}"); let module = wat::parse_str(wasm).unwrap(); - let detected = WasiLibc377Bug::detect(&module).unwrap(); + let module_info = ModuleInfo::from_module(&module).unwrap(); + let detected = WasiLibc377Bug::detect(&module_info).unwrap(); assert_eq!(detected, expected); } } diff --git a/crates/componentize/src/lib.rs b/crates/componentize/src/lib.rs index f4be1bcb1..f59caefb7 100644 --- a/crates/componentize/src/lib.rs +++ b/crates/componentize/src/lib.rs @@ -3,6 +3,7 @@ use { anyhow::{anyhow, Context, Result}, convert::{IntoEntityType, IntoExportKind}, + module_info::ModuleInfo, std::{borrow::Cow, collections::HashSet}, wasm_encoder::{CustomSection, ExportSection, ImportSection, Module, RawSection}, wasmparser::{Encoding, Parser, Payload}, @@ -14,6 +15,7 @@ pub mod bugs; #[cfg(test)] mod abi_conformance; mod convert; +mod module_info; const SPIN_ADAPTER: &[u8] = include_bytes!(concat!( env!("OUT_DIR"), @@ -51,8 +53,9 @@ pub fn componentize_if_necessary(module_or_component: &[u8]) -> Result } pub fn componentize(module: &[u8]) -> Result> { - match WitBindgenVersion::from_module(module)? { - WitBindgenVersion::V0_2 => componentize_old_bindgen(module), + let module_info = ModuleInfo::from_module(module)?; + match WitBindgenVersion::detect(&module_info)? { + WitBindgenVersion::V0_2OrNone => componentize_old_module(module, &module_info), WitBindgenVersion::GreaterThanV0_4 => componentize_new_bindgen(module), WitBindgenVersion::Other(other) => Err(anyhow::anyhow!( "cannot adapt modules created with wit-bindgen version {other}" @@ -65,40 +68,36 @@ pub fn componentize(module: &[u8]) -> Result> { #[derive(Debug)] enum WitBindgenVersion { GreaterThanV0_4, - V0_2, + V0_2OrNone, Other(String), } impl WitBindgenVersion { - fn from_module(module: &[u8]) -> Result { - let (_, bindgen) = metadata::decode(module)?; - if let Some(producers) = bindgen.producers { - if let Some(processors) = producers.get("processed-by") { - let bindgen_version = processors.iter().find_map(|(key, value)| { - key.starts_with("wit-bindgen").then_some(value.as_str()) - }); - if let Some(v) = bindgen_version { - let mut parts = v.split('.'); - let Some(major) = parts.next().and_then(|p| p.parse::().ok()) else { - return Ok(Self::Other(v.to_owned())); - }; - let Some(minor) = parts.next().and_then(|p| p.parse::().ok()) else { - return Ok(Self::Other(v.to_owned())); - }; - if (major == 0 && minor < 5) || major >= 1 { - return Ok(Self::Other(v.to_owned())); - } - // Either there should be no patch version or nothing after patch - if parts.next().is_none() || parts.next().is_none() { - return Ok(Self::GreaterThanV0_4); - } else { - return Ok(Self::Other(v.to_owned())); - } + fn detect(module_info: &ModuleInfo) -> Result { + if let Some(processors) = module_info.bindgen_processors() { + let bindgen_version = processors + .iter() + .find_map(|(key, value)| key.starts_with("wit-bindgen").then_some(value.as_str())); + if let Some(v) = bindgen_version { + let mut parts = v.split('.'); + let Some(major) = parts.next().and_then(|p| p.parse::().ok()) else { + return Ok(Self::Other(v.to_owned())); + }; + let Some(minor) = parts.next().and_then(|p| p.parse::().ok()) else { + return Ok(Self::Other(v.to_owned())); + }; + if (major == 0 && minor < 5) || major >= 1 { + return Ok(Self::Other(v.to_owned())); + } + // Either there should be no patch version or nothing after patch + if parts.next().is_none() || parts.next().is_none() { + return Ok(Self::GreaterThanV0_4); + } else { + return Ok(Self::Other(v.to_owned())); } } } - - Ok(Self::V0_2) + Ok(Self::V0_2OrNone) } } @@ -111,6 +110,17 @@ pub fn componentize_new_bindgen(module: &[u8]) -> Result> { .encode() } +/// Modules *not* produced with wit-bindgen >= 0.5 could be old wit-bindgen or no wit-bindgen +pub fn componentize_old_module(module: &[u8], module_info: &ModuleInfo) -> Result> { + // If the module has a _start export and doesn't obviously use wit-bindgen + // it is likely an old p1 command module. + if module_info.has_start_export && !module_info.probably_uses_wit_bindgen() { + componentize_command(module) + } else { + componentize_old_bindgen(module) + } +} + /// Modules produced with wit-bindgen 0.2 need more extensive adaption pub fn componentize_old_bindgen(module: &[u8]) -> Result> { let (module, exports) = retarget_imports_and_get_exports(ADAPTER_NAME, module)?; diff --git a/crates/componentize/src/module_info.rs b/crates/componentize/src/module_info.rs new file mode 100644 index 000000000..e02ece13c --- /dev/null +++ b/crates/componentize/src/module_info.rs @@ -0,0 +1,111 @@ +use wasm_metadata::Producers; +use wasmparser::{Encoding, ExternalKind, Parser, Payload}; +use wit_component::metadata::Bindgen; + +// wit-bindgen has used both of these historically. +const CANONICAL_ABI_REALLOC_EXPORTS: &[&str] = &["cabi_realloc", "canonical_abi_realloc"]; + +/// Stores various bits of info parsed from a Wasm module that are relevant to +/// componentization. +#[derive(Default)] +pub struct ModuleInfo { + pub bindgen: Option, + pub clang_version: Option, + pub realloc_export: Option, + pub has_start_export: bool, +} + +impl ModuleInfo { + /// Parses info from the given binary module bytes. + pub fn from_module(module: &[u8]) -> anyhow::Result { + let mut info = Self::default(); + for payload in Parser::new(0).parse_all(module) { + match payload? { + Payload::Version { encoding, .. } => { + anyhow::ensure!( + encoding == Encoding::Module, + "ModuleInfo::from_module is only applicable to Modules; got a {encoding:?}" + ); + } + Payload::ExportSection(reader) => { + for export in reader { + let export = export?; + if export.kind == ExternalKind::Func { + if CANONICAL_ABI_REALLOC_EXPORTS.contains(&export.name) { + tracing::debug!( + "Found canonical ABI realloc export {:?}", + export.name + ); + info.realloc_export = Some(export.name.to_string()); + } else if export.name == "_start" { + tracing::debug!("Found _start export"); + info.has_start_export = true; + } + } + } + } + Payload::CustomSection(c) => { + let section_name = c.name(); + if section_name == "producers" { + let producers = Producers::from_bytes(c.data(), c.data_offset())?; + if let Some(clang_version) = + producers.get("processed-by").and_then(|f| f.get("clang")) + { + tracing::debug!(clang_version, "Parsed producers.processed-by.clang"); + info.clang_version = Some(clang_version.to_string()); + } + } else if section_name.starts_with("component-type") { + match decode_bindgen_custom_section(section_name, c.data()) { + Ok(bindgen) => { + tracing::debug!("Parsed bindgen section {section_name:?}"); + info.bindgen = Some(bindgen); + } + Err(err) => tracing::warn!( + "Error parsing bindgen section {section_name:?}: {err}" + ), + } + } + } + _ => (), + } + } + Ok(info) + } + + /// Returns true if the given module was heuristically probably compiled + /// with wit-bindgen. + pub fn probably_uses_wit_bindgen(&self) -> bool { + if self.bindgen.is_some() { + // Presence of bindgen metadata is a strong signal + true + } else if self.realloc_export.is_some() { + // A canonical ABI realloc export is a decent signal + true + } else { + false + } + } + + /// Returns the wit-bindgen metadata producers processed-by field, if + /// present. + pub fn bindgen_processors(&self) -> Option { + self.bindgen + .as_ref()? + .producers + .as_ref()? + .get("processed-by") + } +} + +/// This is a silly workaround for the limited public interface available in +/// [`wit_component::metadata`]. +// TODO: Make Bindgen::decode_custom_section public? +fn decode_bindgen_custom_section(name: &str, data: &[u8]) -> anyhow::Result { + let mut module = wasm_encoder::Module::new(); + module.section(&wasm_encoder::CustomSection { + name: name.into(), + data: data.into(), + }); + let (_, bindgen) = wit_component::metadata::decode(module.as_slice())?; + Ok(bindgen) +} diff --git a/crates/http/src/config.rs b/crates/http/src/config.rs index 2c675f1ca..5ccee8e57 100644 --- a/crates/http/src/config.rs +++ b/crates/http/src/config.rs @@ -63,7 +63,8 @@ pub enum HttpExecutorType { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] pub struct WagiTriggerConfig { - /// The name of the entrypoint. + /// The name of the entrypoint. (DEPRECATED) + #[serde(skip_serializing)] pub entrypoint: String, /// A string representation of the argv array. diff --git a/crates/trigger-http/src/server.rs b/crates/trigger-http/src/server.rs index 9fab3d6f1..b9ee135b7 100644 --- a/crates/trigger-http/src/server.rs +++ b/crates/trigger-http/src/server.rs @@ -98,10 +98,21 @@ impl HttpServer { let component_trigger_configs = HashMap::from_iter(component_trigger_configs); let component_handler_types = component_trigger_configs - .keys() - .map(|component_id| { - let component = trigger_app.get_component(component_id)?; - let handler_type = HandlerType::from_component(trigger_app.engine(), component)?; + .iter() + .map(|(component_id, trigger_config)| { + let handler_type = match &trigger_config.executor { + None | Some(HttpExecutorType::Http) => { + let component = trigger_app.get_component(component_id)?; + HandlerType::from_component(trigger_app.engine(), component)? + } + Some(HttpExecutorType::Wagi(wagi_config)) => { + anyhow::ensure!( + wagi_config.entrypoint == "_start", + "Wagi component '{component_id}' cannot use deprecated 'entrypoint' field" + ); + HandlerType::Wagi + } + }; Ok((component_id.clone(), handler_type)) }) .collect::>()?; @@ -253,6 +264,7 @@ impl HttpServer { .execute(instance_builder, &route_match, req, client_addr) .await } + HandlerType::Wagi => unreachable!(), }, HttpExecutorType::Wagi(wagi_config) => { let executor = WagiHttpExecutor { @@ -446,9 +458,10 @@ pub(crate) trait HttpExecutor: Clone + Send + Sync + 'static { } /// Whether this handler uses the custom Spin http handler interface for wasi-http -#[derive(Copy, Clone)] +#[derive(Clone, Copy)] pub enum HandlerType { Spin, + Wagi, Wasi0_2, Wasi2023_11_10, Wasi2023_10_18, @@ -490,8 +503,10 @@ impl HandlerType { handler_ty.ok_or_else(|| { anyhow::anyhow!( - "Expected component to either export `{WASI_HTTP_EXPORT_2023_10_18}`, \ - `{WASI_HTTP_EXPORT_2023_11_10}`, `{WASI_HTTP_EXPORT_0_2_0}`, \ + "Expected component to export one of \ + `{WASI_HTTP_EXPORT_2023_10_18}`, \ + `{WASI_HTTP_EXPORT_2023_11_10}`, \ + `{WASI_HTTP_EXPORT_0_2_0}`, \ or `fermyon:spin/inbound-http` but it exported none of those" ) }) diff --git a/crates/trigger-http/src/wagi.rs b/crates/trigger-http/src/wagi.rs index c0a28485f..e447e27e0 100644 --- a/crates/trigger-http/src/wagi.rs +++ b/crates/trigger-http/src/wagi.rs @@ -1,6 +1,6 @@ use std::{io::Cursor, net::SocketAddr}; -use anyhow::{anyhow, ensure, Context, Result}; +use anyhow::{ensure, Result}; use http_body_util::BodyExt; use hyper::{Request, Response}; use spin_http::{config::WagiTriggerConfig, routes::RouteMatch, wagi}; @@ -83,27 +83,18 @@ impl HttpExecutor for WagiHttpExecutor { let (instance, mut store) = instance_builder.instantiate(()).await?; - let start = instance - .get_func(&mut store, &self.wagi_config.entrypoint) - .ok_or_else(|| { - anyhow::anyhow!( - "No such function '{}' in {}", - self.wagi_config.entrypoint, - component - ) - })?; + let command = wasmtime_wasi::bindings::Command::new(&mut store, &instance)?; + tracing::trace!("Calling Wasm entry point"); - start - .call_async(&mut store, &[], &mut []) + if let Err(()) = command + .wasi_cli_run() + .call_run(&mut store) .await - .or_else(ignore_successful_proc_exit_trap) - .with_context(|| { - anyhow!( - "invoking {} for component {component}", - self.wagi_config.entrypoint - ) - })?; - tracing::info!("Module execution complete"); + .or_else(ignore_successful_proc_exit_trap)? + { + tracing::error!("Wagi main function returned unsuccessful result"); + } + tracing::info!("Wagi execution complete"); // Drop the store so we're left with a unique reference to `stdout`: drop(store); @@ -119,13 +110,13 @@ impl HttpExecutor for WagiHttpExecutor { } } -fn ignore_successful_proc_exit_trap(guest_err: anyhow::Error) -> Result<()> { +fn ignore_successful_proc_exit_trap(guest_err: anyhow::Error) -> Result> { match guest_err .root_cause() .downcast_ref::() { Some(trap) => match trap.0 { - 0 => Ok(()), + 0 => Ok(Ok(())), _ => Err(guest_err), }, None => Err(guest_err), diff --git a/crates/trigger-http/src/wasi.rs b/crates/trigger-http/src/wasi.rs index 05d6b33b7..d61f3bdb2 100644 --- a/crates/trigger-http/src/wasi.rs +++ b/crates/trigger-http/src/wasi.rs @@ -93,7 +93,8 @@ impl HttpExecutor for WasiHttpExecutor { drop(exports); Handler::Latest(Proxy::new(&mut store, &instance)?) } - HandlerType::Spin => panic!("should have used execute_spin instead"), + HandlerType::Spin => unreachable!("should have used SpinHttpExecutor"), + HandlerType::Wagi => unreachable!("should have used WagiExecutor instead"), } }; diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index 4b6dc422b..b5800cc9c 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -358,7 +358,7 @@ impl TriggerAppBuilder { ) })?; let component = spin_componentize::componentize_if_necessary(&bytes)?; - spin_core::Component::new(engine, component.as_ref()) + spin_core::Component::new(engine, component) .with_context(|| format!("loading module {}", quoted_path(&path))) } } diff --git a/examples/spin-wagi-http/spin.toml b/examples/spin-wagi-http/spin.toml index e6280308f..37ecaee46 100644 --- a/examples/spin-wagi-http/spin.toml +++ b/examples/spin-wagi-http/spin.toml @@ -9,7 +9,7 @@ version = "1.0.0" [[trigger.http]] route = "/hello" component = "hello" -executor = { type = "wagi" } # _start (the default entrypoint) is automatically mapped to main() +executor = { type = "wagi" } [[trigger.http]] route = "/goodbye" diff --git a/examples/wagi-http-rust/spin.toml b/examples/wagi-http-rust/spin.toml index c1cc3f065..c5d1a7821 100644 --- a/examples/wagi-http-rust/spin.toml +++ b/examples/wagi-http-rust/spin.toml @@ -9,7 +9,7 @@ version = "1.0.0" [[trigger.http]] route = "/env" component = "env" -executor = { type = "wagi" } # _start (the default entrypoint) is automatically mapped to main() +executor = { type = "wagi" } [component.env] source = "target/wasm32-wasi/release/wagihelloworld.wasm" From c5ce6f9f01bb9d0cb96b71da8026fb5a8a8de9f2 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 23 Aug 2024 15:50:34 -0400 Subject: [PATCH 166/195] factors: Fix wasi-libc#377 bug detection Signed-off-by: Lann Martin --- crates/componentize/src/bugs.rs | 72 +++++++++++++++++++-------------- crates/componentize/src/lib.rs | 1 + crates/trigger/src/cli.rs | 5 ++- 3 files changed, 45 insertions(+), 33 deletions(-) diff --git a/crates/componentize/src/bugs.rs b/crates/componentize/src/bugs.rs index 053dc4aa3..099b4adb6 100644 --- a/crates/componentize/src/bugs.rs +++ b/crates/componentize/src/bugs.rs @@ -2,21 +2,19 @@ use crate::module_info::ModuleInfo; pub const EARLIEST_PROBABLY_SAFE_CLANG_VERSION: &str = "15.0.7"; -/// Represents the detected likelihood of the allocation bug fixed in -/// https://github.com/WebAssembly/wasi-libc/pull/377 being present in a Wasm -/// module. +/// This error represents the likely presence of the allocation bug fixed in +/// https://github.com/WebAssembly/wasi-libc/pull/377 in a Wasm module. #[derive(Debug, PartialEq)] -pub enum WasiLibc377Bug { - ProbablySafe, - ProbablyUnsafe { clang_version: String }, - Unknown, +pub struct WasiLibc377Bug { + clang_version: Option, } impl WasiLibc377Bug { - pub fn detect(module_info: &ModuleInfo) -> anyhow::Result { + /// Detects the likely presence of this bug. + pub fn check(module_info: &ModuleInfo) -> Result<(), Self> { if module_info.probably_uses_wit_bindgen() { // Modules built with wit-bindgen are probably safe. - return Ok(Self::ProbablySafe); + return Ok(()); } if let Some(clang_version) = &module_info.clang_version { // Clang/LLVM version is a good proxy for wasi-sdk @@ -25,12 +23,12 @@ impl WasiLibc377Bug { if let Some((major, minor, patch)) = parse_clang_version(clang_version) { let earliest_safe = parse_clang_version(EARLIEST_PROBABLY_SAFE_CLANG_VERSION).unwrap(); - return if (major, minor, patch) >= earliest_safe { - Ok(Self::ProbablySafe) + if (major, minor, patch) >= earliest_safe { + return Ok(()); } else { - Ok(Self::ProbablyUnsafe { - clang_version: clang_version.clone(), - }) + return Err(Self { + clang_version: Some(clang_version.clone()), + }); }; } else { tracing::warn!( @@ -39,10 +37,27 @@ impl WasiLibc377Bug { ); } } - Ok(Self::Unknown) + // If we can't assert that the module uses wit-bindgen OR was compiled + // with a new-enough wasi-sdk, conservatively assume it may be buggy. + Err(Self { + clang_version: None, + }) } } +impl std::fmt::Display for WasiLibc377Bug { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "This Wasm module may have been compiled with wasi-sdk version <19 which \ + contains a critical memory safety bug. For more information, see: \ + https://github.com/fermyon/spin/issues/2552" + ) + } +} + +impl std::error::Error for WasiLibc377Bug {} + fn parse_clang_version(ver: &str) -> Option<(u16, u16, u16)> { // Strip optional trailing detail after space let ver = ver.split(' ').next().unwrap(); @@ -59,47 +74,42 @@ mod tests { #[test] fn wasi_libc_377_detect() { - use WasiLibc377Bug::*; - for (wasm, expected) in [ - (r#"(module)"#, Unknown), + for (wasm, safe) in [ + (r#"(module)"#, false), ( r#"(module (func (export "cabi_realloc") (unreachable)))"#, - ProbablySafe, + true, ), ( r#"(module (func (export "some_other_function") (unreachable)))"#, - Unknown, + false, ), ( r#"(module (@producers (processed-by "clang" "16.0.0 extra-stuff")))"#, - ProbablySafe, + true, ), ( r#"(module (@producers (processed-by "clang" "15.0.7")))"#, - ProbablySafe, + true, ), ( r#"(module (@producers (processed-by "clang" "15.0.6")))"#, - ProbablyUnsafe { - clang_version: "15.0.6".into(), - }, + false, ), ( r#"(module (@producers (processed-by "clang" "14.0.0 extra-stuff")))"#, - ProbablyUnsafe { - clang_version: "14.0.0 extra-stuff".into(), - }, + false, ), ( r#"(module (@producers (processed-by "clang" "a.b.c")))"#, - Unknown, + false, ), ] { eprintln!("WAT: {wasm}"); let module = wat::parse_str(wasm).unwrap(); let module_info = ModuleInfo::from_module(&module).unwrap(); - let detected = WasiLibc377Bug::detect(&module_info).unwrap(); - assert_eq!(detected, expected); + let detected = WasiLibc377Bug::check(&module_info); + assert!(detected.is_ok() == safe, "{wasm} -> {detected:?}"); } } } diff --git a/crates/componentize/src/lib.rs b/crates/componentize/src/lib.rs index f59caefb7..e78269c25 100644 --- a/crates/componentize/src/lib.rs +++ b/crates/componentize/src/lib.rs @@ -115,6 +115,7 @@ pub fn componentize_old_module(module: &[u8], module_info: &ModuleInfo) -> Resul // If the module has a _start export and doesn't obviously use wit-bindgen // it is likely an old p1 command module. if module_info.has_start_export && !module_info.probably_uses_wit_bindgen() { + bugs::WasiLibc377Bug::check(module_info)?; componentize_command(module) } else { componentize_old_bindgen(module) diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index b5800cc9c..5678c6875 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -357,9 +357,10 @@ impl TriggerAppBuilder { quoted_path(&path) ) })?; - let component = spin_componentize::componentize_if_necessary(&bytes)?; + let component = spin_componentize::componentize_if_necessary(&bytes) + .with_context(|| format!("preparing wasm {}", quoted_path(&path)))?; spin_core::Component::new(engine, component) - .with_context(|| format!("loading module {}", quoted_path(&path))) + .with_context(|| format!("compiling wasm {}", quoted_path(&path))) } } From 8050ec961cee870c670c918dcf778cfae1f8a8cb Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 23 Aug 2024 15:30:10 +0200 Subject: [PATCH 167/195] Fix path handling in sqlite and kv Signed-off-by: Ryan Levick --- crates/factor-key-value-spin/src/lib.rs | 59 +++---- crates/factor-key-value/src/runtime_config.rs | 2 + .../src/runtime_config/spin.rs | 16 +- crates/factor-key-value/tests/factor_test.rs | 146 ++++++++++-------- .../factor-sqlite/src/runtime_config/spin.rs | 17 +- crates/factor-sqlite/tests/factor_test.rs | 2 +- crates/factors/tests/smoke.rs | 23 +-- crates/key-value-sqlite/src/lib.rs | 1 + crates/runtime-config/src/lib.rs | 43 +++--- 9 files changed, 169 insertions(+), 140 deletions(-) diff --git a/crates/factor-key-value-spin/src/lib.rs b/crates/factor-key-value-spin/src/lib.rs index 38e9ef442..062398a01 100644 --- a/crates/factor-key-value-spin/src/lib.rs +++ b/crates/factor-key-value-spin/src/lib.rs @@ -11,12 +11,16 @@ use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; /// A key-value store that uses SQLite as the backend. pub struct SpinKeyValueStore { /// The base path or directory for the SQLite database file. - base_path: PathBuf, + base_path: Option, } impl SpinKeyValueStore { /// Create a new SpinKeyValueStore with the given base path. - pub fn new(base_path: PathBuf) -> Self { + /// + /// If the database directory is None, the database will always be in-memory. + /// If it's `Some`, the database will be stored at the combined `base_path` and + /// the `path` specified in the runtime configuration. + pub fn new(base_path: Option) -> Self { Self { base_path } } } @@ -31,25 +35,14 @@ pub struct SpinKeyValueRuntimeConfig { impl SpinKeyValueRuntimeConfig { /// The default filename for the SQLite database. const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; - - /// Create a new runtime configuration with the given directory. - /// - /// If the database directory is None, the database is in-memory. - /// If the database directory is Some, the database is stored in a file in the given directory. - pub fn default(default_database_dir: Option) -> Self { - let path = default_database_dir.map(|dir| dir.join(Self::DEFAULT_SPIN_STORE_FILENAME)); - Self { path } - } } -/// Resolve a relative path against a base dir. -/// -/// If the path is absolute, it is returned as is. Otherwise, it is resolved against the base dir. -fn resolve_relative_path(path: &Path, base_dir: &Path) -> PathBuf { - if path.is_absolute() { - return path.to_owned(); +impl Default for SpinKeyValueRuntimeConfig { + fn default() -> Self { + Self { + path: Some(PathBuf::from(Self::DEFAULT_SPIN_STORE_FILENAME)), + } } - base_dir.join(path) } impl MakeKeyValueStore for SpinKeyValueStore { @@ -63,16 +56,30 @@ impl MakeKeyValueStore for SpinKeyValueStore { &self, runtime_config: Self::RuntimeConfig, ) -> anyhow::Result { - let location = match runtime_config.path { - Some(path) => { - let path = resolve_relative_path(&path, &self.base_path); + // The base path and the subpath must both be set otherwise, we default to in-memory. + let location = + if let (Some(base_path), Some(path)) = (&self.base_path, &runtime_config.path) { + let path = resolve_relative_path(path, base_path); // Create the store's parent directory if necessary - fs::create_dir_all(path.parent().unwrap()) - .context("Failed to create key value store")?; + let parent = path.parent().unwrap(); + if !parent.exists() { + fs::create_dir_all(parent) + .context("Failed to create key value store's parent directory")?; + } DatabaseLocation::Path(path) - } - None => DatabaseLocation::InMemory, - }; + } else { + DatabaseLocation::InMemory + }; Ok(KeyValueSqlite::new(location)) } } + +/// Resolve a relative path against a base dir. +/// +/// If the path is absolute, it is returned as is. Otherwise, it is resolved against the base dir. +fn resolve_relative_path(path: &Path, base_dir: &Path) -> PathBuf { + if path.is_absolute() { + return path.to_owned(); + } + base_dir.join(path) +} diff --git a/crates/factor-key-value/src/runtime_config.rs b/crates/factor-key-value/src/runtime_config.rs index 97e2a3ece..0c83243a3 100644 --- a/crates/factor-key-value/src/runtime_config.rs +++ b/crates/factor-key-value/src/runtime_config.rs @@ -13,6 +13,8 @@ pub struct RuntimeConfig { impl RuntimeConfig { /// Adds a store manager for the store with the given label to the runtime configuration. + /// + /// If a store manager already exists for the given label, it will be replaced. pub fn add_store_manager(&mut self, label: String, store_manager: Arc) { self.store_managers.insert(label, store_manager); } diff --git a/crates/factor-key-value/src/runtime_config/spin.rs b/crates/factor-key-value/src/runtime_config/spin.rs index 3f3e487f5..860208b7e 100644 --- a/crates/factor-key-value/src/runtime_config/spin.rs +++ b/crates/factor-key-value/src/runtime_config/spin.rs @@ -62,8 +62,20 @@ impl RuntimeConfigResolver { /// /// Users must ensure that the store type for `config` has been registered with /// the resolver using [`Self::register_store_type`]. - pub fn add_default_store(&mut self, label: &'static str, config: StoreConfig) { - self.defaults.insert(label, config); + pub fn add_default_store( + &mut self, + label: &'static str, + config: T::RuntimeConfig, + ) -> anyhow::Result<()> + where + T: MakeKeyValueStore, + T::RuntimeConfig: Serialize, + { + self.defaults.insert( + label, + StoreConfig::new(T::RUNTIME_CONFIG_TYPE.to_owned(), config)?, + ); + Ok(()) } /// Registers a store type to the resolver. diff --git a/crates/factor-key-value/tests/factor_test.rs b/crates/factor-key-value/tests/factor_test.rs index 9412b9e51..f796162d6 100644 --- a/crates/factor-key-value/tests/factor_test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -1,12 +1,13 @@ -use anyhow::Context; +use anyhow::Context as _; use spin_factor_key_value::{ - runtime_config::spin::{MakeKeyValueStore, RuntimeConfigResolver, StoreConfig}, + runtime_config::spin::{MakeKeyValueStore, RuntimeConfigResolver}, KeyValueFactor, RuntimeConfig, }; use spin_factor_key_value_redis::RedisKeyValueStore; -use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; +use spin_factor_key_value_spin::SpinKeyValueStore; use spin_factors::{FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; +use spin_world::v2::key_value::HostStore; use std::{collections::HashSet, sync::Arc}; #[derive(RuntimeFactors)] @@ -14,25 +15,11 @@ struct TestFactors { key_value: KeyValueFactor, } -fn default_key_value_resolver() -> anyhow::Result<(RuntimeConfigResolver, tempdir::TempDir)> { - let mut test_resolver = RuntimeConfigResolver::new(); - test_resolver.register_store_type(SpinKeyValueStore::new( - std::env::current_dir().context("failed to get current directory")?, - ))?; - let tmp_dir = tempdir::TempDir::new("example")?; - let path = tmp_dir.path().to_path_buf(); - let default_config = SpinKeyValueRuntimeConfig::default(Some(path)); - let store_config = StoreConfig::new( - SpinKeyValueStore::RUNTIME_CONFIG_TYPE.to_string(), - default_config, - )?; - test_resolver.add_default_store("default", store_config); - Ok((test_resolver, tmp_dir)) -} - #[tokio::test] async fn default_key_value_works() -> anyhow::Result<()> { - let (test_resolver, dir) = default_key_value_resolver()?; + let mut test_resolver = RuntimeConfigResolver::new(); + test_resolver.register_store_type(SpinKeyValueStore::new(None))?; + test_resolver.add_default_store::("default", Default::default())?; let factors = TestFactors { key_value: KeyValueFactor::new(test_resolver), }; @@ -47,8 +34,6 @@ async fn default_key_value_works() -> anyhow::Result<()> { state.key_value.allowed_stores(), &["default".into()].into_iter().collect::>() ); - // Ensure the database directory is created - assert!(dir.path().exists()); Ok(()) } @@ -56,7 +41,7 @@ async fn run_test_with_config_and_stores_for_label( runtime_config: Option, store_types: Vec, labels: Vec<&str>, -) -> anyhow::Result<()> { +) -> anyhow::Result { let mut test_resolver = RuntimeConfigResolver::new(); for store_type in store_types { test_resolver.register_store_type(store_type)?; @@ -79,7 +64,7 @@ async fn run_test_with_config_and_stores_for_label( state.key_value.allowed_stores().iter().collect::>() ); - Ok(()) + Ok(state) } #[tokio::test] @@ -94,7 +79,8 @@ async fn overridden_default_key_value_works() -> anyhow::Result<()> { vec![RedisKeyValueStore::new()], vec!["default"], ) - .await + .await?; + Ok(()) } #[tokio::test] @@ -105,52 +91,69 @@ async fn custom_spin_key_value_works() -> anyhow::Result<()> { }; run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new( - std::env::current_dir().context("failed to get current directory")?, - )], + vec![SpinKeyValueStore::new(None)], vec!["custom"], ) - .await + .await?; + Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn custom_spin_key_value_works_with_absolute_path() -> anyhow::Result<()> { let tmp_dir = tempdir::TempDir::new("example")?; - let path = tmp_dir.path().join("custom.db"); - let path_str = path.to_str().unwrap(); + let db_path = tmp_dir.path().join("foo/custom.db"); + // Check that the db does not exist yet - it will exist by the end of the test + assert!(!db_path.exists()); + + let path_str = db_path.to_str().unwrap(); let runtime_config = toml::toml! { [key_value_store.custom] type = "spin" path = path_str }; - run_test_with_config_and_stores_for_label( + let mut state = run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new( + vec![SpinKeyValueStore::new(Some( std::env::current_dir().context("failed to get current directory")?, - )], + ))], vec!["custom"], ) .await?; - assert!(tmp_dir.path().exists()); + + // Actually et a key since store creation is lazy + let store = state.key_value.open("custom".to_owned()).await??; + let _ = state.key_value.get(store, "foo".to_owned()).await??; + + // Check that the parent has been created + assert!(db_path.exists()); Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn custom_spin_key_value_works_with_relative_path() -> anyhow::Result<()> { let tmp_dir = tempdir::TempDir::new("example")?; - let path = tmp_dir.path().to_owned(); + let db_path = tmp_dir.path().join("custom.db"); + // Check that the db does not exist yet - it will exist by the end of the test + assert!(!db_path.exists()); + let runtime_config = toml::toml! { [key_value_store.custom] type = "spin" path = "custom.db" }; - run_test_with_config_and_stores_for_label( + let mut state = run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new(path)], + vec![SpinKeyValueStore::new(Some(tmp_dir.path().to_owned()))], vec!["custom"], ) .await?; - assert!(tmp_dir.path().exists()); + + // Actually et a key since store creation is lazy + let store = state.key_value.open("custom".to_owned()).await??; + let _ = state.key_value.get(store, "foo".to_owned()).await??; + + // Check that the correct store in the config was chosen by verifying the existence of the DB + assert!(db_path.exists()); Ok(()) } @@ -166,64 +169,75 @@ async fn custom_redis_key_value_works() -> anyhow::Result<()> { vec![RedisKeyValueStore::new()], vec!["custom"], ) - .await + .await?; + Ok(()) } #[tokio::test] async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { + let tmp_dir = tempdir::TempDir::new("example")?; let runtime_config = toml::toml! { [key_value_store.custom] type = "spin" path = "/$$&/bad/path/foo.db" }; - assert!(run_test_with_config_and_stores_for_label( + let result = run_test_with_config_and_stores_for_label( Some(runtime_config), - vec![SpinKeyValueStore::new( - std::env::current_dir().context("failed to get current directory")? - )], - vec!["custom"] + vec![SpinKeyValueStore::new(Some(tmp_dir.path().to_owned()))], + vec!["custom"], ) - .await - .is_err()); + .await; + // TODO(rylev): This only fails on my machine due to a read-only file system error. + // We should consider adding a check for the error message. + assert!(result.is_err()); Ok(()) } -#[tokio::test] -async fn multiple_custom_key_value_uses_first_store() -> anyhow::Result<()> { +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +// TODO(rylev): consider removing this test as it is really only a consequence of +// toml deserialization and not a feature of the key-value store. +async fn multiple_custom_key_value_uses_second_store() -> anyhow::Result<()> { let tmp_dir = tempdir::TempDir::new("example")?; + let db_path = tmp_dir.path().join("custom.db"); + // Check that the db does not exist yet - it will exist by the end of the test + assert!(!db_path.exists()); + let mut test_resolver = RuntimeConfigResolver::new(); test_resolver.register_store_type(RedisKeyValueStore::new())?; - test_resolver.register_store_type(SpinKeyValueStore::new(tmp_dir.path().to_owned()))?; + test_resolver.register_store_type(SpinKeyValueStore::new(Some(tmp_dir.path().to_owned())))?; let test_resolver = Arc::new(test_resolver); let factors = TestFactors { key_value: KeyValueFactor::new(test_resolver.clone()), }; + let runtime_config = toml::toml! { + [key_value_store.custom] + type = "redis" + url = "redis://localhost:6379" + + [key_value_store.custom] + type = "spin" + path = "custom.db" + + }; let env = TestEnvironment::new(factors) .extend_manifest(toml! { [component.test-component] source = "does-not-exist.wasm" key_value_stores = ["custom"] }) - .runtime_config(TomlConfig::new( - test_resolver, - Some(toml::toml! { - [key_value_store.custom] - type = "spin" - path = "custom.db" + .runtime_config(TomlConfig::new(test_resolver, Some(runtime_config)))?; + let mut state = env.build_instance_state().await?; - [key_value_store.custom] - type = "redis" - url = "redis://localhost:6379" - }), - ))?; - let state = env.build_instance_state().await?; + // Actually et a key since store creation is lazy + let store = state.key_value.open("custom".to_owned()).await??; + let _ = state.key_value.get(store, "foo".to_owned()).await??; assert_eq!( state.key_value.allowed_stores(), &["custom".into()].into_iter().collect::>() ); - // Check that the first store in the config was chosen by verifying the existence of the DB directory - assert!(tmp_dir.path().exists()); + // Check that the correct store in the config was chosen by verifying the existence of the DB + assert!(db_path.exists()); Ok(()) } diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index a98387df1..6274fd97d 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -20,7 +20,7 @@ use crate::{Connection, ConnectionCreator, DefaultLabelResolver}; /// This type implements how Spin CLI's SQLite implementation is configured /// through the runtime config toml as well as the behavior of the "default" label. pub struct RuntimeConfigResolver { - default_database_dir: PathBuf, + default_database_dir: Option, local_database_dir: PathBuf, } @@ -29,11 +29,12 @@ impl RuntimeConfigResolver { /// /// This takes as arguments: /// * the directory to use as the default location for SQLite databases. - /// Usually this will be the path to the `.spin` state directory. + /// Usually this will be the path to the `.spin` state directory. If + /// `None`, the default database will be in-memory. /// * the path to the directory from which relative paths to /// local SQLite databases are resolved. (this should most likely be the /// path to the runtime-config file or the current working dir). - pub fn new(default_database_dir: PathBuf, local_database_dir: PathBuf) -> Self { + pub fn new(default_database_dir: Option, local_database_dir: PathBuf) -> Self { Self { default_database_dir, local_database_dir, @@ -102,9 +103,15 @@ impl DefaultLabelResolver for RuntimeConfigResolver { return None; } - let path = self.default_database_dir.join(DEFAULT_SQLITE_DB_FILENAME); + let path = self + .default_database_dir + .as_deref() + .map(|p| p.join(DEFAULT_SQLITE_DB_FILENAME)); let factory = move || { - let location = spin_sqlite_inproc::InProcDatabaseLocation::Path(path.clone()); + let location = match &path { + Some(path) => spin_sqlite_inproc::InProcDatabaseLocation::Path(path.clone()), + None => spin_sqlite_inproc::InProcDatabaseLocation::InMemory, + }; let connection = spin_sqlite_inproc::InProcConnection::new(location)?; Ok(Box::new(connection) as _) }; diff --git a/crates/factor-sqlite/tests/factor_test.rs b/crates/factor-sqlite/tests/factor_test.rs index e2e2ef213..b668bf343 100644 --- a/crates/factor-sqlite/tests/factor_test.rs +++ b/crates/factor-sqlite/tests/factor_test.rs @@ -66,7 +66,7 @@ async fn no_error_when_database_is_configured() -> anyhow::Result<()> { [sqlite_database.foo] type = "spin" }; - let sqlite_config = RuntimeConfigResolver::new("/".into(), "/".into()); + let sqlite_config = RuntimeConfigResolver::new(None, "/".into()); let env = TestEnvironment::new(factors) .extend_manifest(toml! { [component.test-component] diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 3b31c701d..95e77d7b6 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -1,14 +1,11 @@ -use std::{path::PathBuf, sync::Arc}; +use std::sync::Arc; use anyhow::Context; use http_body_util::BodyExt; use spin_app::App; -use spin_factor_key_value::{ - runtime_config::spin::{MakeKeyValueStore, RuntimeConfigResolver, StoreConfig}, - KeyValueFactor, -}; +use spin_factor_key_value::{runtime_config::spin::RuntimeConfigResolver, KeyValueFactor}; use spin_factor_key_value_redis::RedisKeyValueStore; -use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; +use spin_factor_key_value_spin::SpinKeyValueStore; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; @@ -42,18 +39,10 @@ impl AsInstanceState for Data { #[tokio::test(flavor = "multi_thread")] async fn smoke_test_works() -> anyhow::Result<()> { let mut key_value_resolver = RuntimeConfigResolver::default(); - let default_config = - SpinKeyValueRuntimeConfig::default(Some(PathBuf::from("tests/smoke-app/.spin"))); - key_value_resolver.add_default_store( - "default", - StoreConfig { - type_: SpinKeyValueStore::RUNTIME_CONFIG_TYPE.to_string(), - config: toml::value::Table::try_from(default_config)?, - }, - ); - key_value_resolver.register_store_type(SpinKeyValueStore::new( + key_value_resolver.add_default_store::("default", Default::default())?; + key_value_resolver.register_store_type(SpinKeyValueStore::new(Some( std::env::current_dir().context("failed to get current directory")?, - ))?; + )))?; key_value_resolver.register_store_type(RedisKeyValueStore::new())?; let key_value_resolver = Arc::new(key_value_resolver); diff --git a/crates/key-value-sqlite/src/lib.rs b/crates/key-value-sqlite/src/lib.rs index 337667b69..152da5ba3 100644 --- a/crates/key-value-sqlite/src/lib.rs +++ b/crates/key-value-sqlite/src/lib.rs @@ -10,6 +10,7 @@ use std::{ use tokio::task; use tracing::{instrument, Level}; +#[derive(Clone, Debug)] pub enum DatabaseLocation { InMemory, Path(PathBuf), diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index cc5e07811..801b60417 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -1,8 +1,9 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; -use spin_factor_key_value::runtime_config::spin::{self as key_value, MakeKeyValueStore}; +use spin_factor_key_value::runtime_config::spin::{self as key_value}; use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; +use spin_factor_key_value_spin::SpinKeyValueStore; use spin_factor_llm::{spin as llm, LlmFactor}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::OutboundMqttFactor; @@ -46,10 +47,9 @@ where use_gpu: bool, ) -> anyhow::Result { let tls_resolver = SpinTlsRuntimeConfig::new(runtime_config_path); - let state_dir = PathBuf::from(state_dir.unwrap_or(DEFAULT_STATE_DIR)); - let key_value_config_resolver = key_value_config_resolver(state_dir.clone()); + let key_value_config_resolver = key_value_config_resolver(state_dir.map(Into::into)); - let sqlite_config_resolver = sqlite_config_resolver(state_dir.clone()) + let sqlite_config_resolver = sqlite_config_resolver(state_dir.map(Into::into)) .context("failed to resolve sqlite runtime config")?; let file = std::fs::read_to_string(runtime_config_path).with_context(|| { @@ -66,7 +66,7 @@ where })?; let runtime_config: T = TomlRuntimeConfigSource::new( &toml, - state_dir, + state_dir.unwrap_or(DEFAULT_STATE_DIR).into(), &key_value_config_resolver, &tls_resolver, &sqlite_config_resolver, @@ -106,11 +106,11 @@ where impl ResolvedRuntimeConfig { pub fn default(state_dir: Option<&str>) -> Self { - let state_dir = state_dir.unwrap_or(DEFAULT_STATE_DIR); + let state_dir = state_dir.map(PathBuf::from); Self { - sqlite_resolver: sqlite_config_resolver(PathBuf::from(state_dir)) + sqlite_resolver: sqlite_config_resolver(state_dir.clone()) .expect("failed to resolve sqlite runtime config"), - key_value_resolver: key_value_config_resolver(PathBuf::from(state_dir)), + key_value_resolver: key_value_config_resolver(state_dir), runtime_config: Default::default(), } } @@ -227,14 +227,13 @@ impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_> { } } -const DEFAULT_KEY_VALUE_STORE_FILENAME: &str = "sqlite_key_value.db"; const DEFAULT_KEY_VALUE_STORE_LABEL: &str = "default"; /// The key-value runtime configuration resolver. /// /// Takes a base path for the local store. pub fn key_value_config_resolver( - local_store_base_path: PathBuf, + local_store_base_path: Option, ) -> key_value::RuntimeConfigResolver { let mut key_value = key_value::RuntimeConfigResolver::new(); @@ -242,7 +241,7 @@ pub fn key_value_config_resolver( // Unwraps are safe because the store types are known to not overlap. key_value .register_store_type(spin_factor_key_value_spin::SpinKeyValueStore::new( - local_store_base_path, + local_store_base_path.clone(), )) .unwrap(); key_value @@ -253,27 +252,25 @@ pub fn key_value_config_resolver( .unwrap(); // Add handling of "default" store. - key_value.add_default_store( - DEFAULT_KEY_VALUE_STORE_LABEL, - key_value::StoreConfig { - type_: spin_factor_key_value_spin::SpinKeyValueStore::RUNTIME_CONFIG_TYPE.to_owned(), - config: toml::toml! { - path = DEFAULT_KEY_VALUE_STORE_FILENAME - }, - }, - ); + // Unwraps are safe because the store is known to be serializable as toml. + key_value + .add_default_store::(DEFAULT_KEY_VALUE_STORE_LABEL, Default::default()) + .unwrap(); key_value } /// The sqlite runtime configuration resolver. /// -/// Takes a base path to the state directory. -fn sqlite_config_resolver(state_dir: PathBuf) -> anyhow::Result { +/// Takes a path to the directory where the default database should be stored. +/// If the path is `None`, the default database will be in-memory. +fn sqlite_config_resolver( + default_database_dir: Option, +) -> anyhow::Result { let local_database_dir = std::env::current_dir().context("failed to get current working directory")?; Ok(sqlite::RuntimeConfigResolver::new( - state_dir, + default_database_dir, local_database_dir, )) } From 7c4b268158e596121072ed0b037acd31c5d935d4 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 26 Aug 2024 14:46:21 +0200 Subject: [PATCH 168/195] Properly handle state_dir Signed-off-by: Ryan Levick --- crates/factor-llm/src/spin.rs | 19 ++++++++----- crates/runtime-config/src/lib.rs | 46 +++++++++++++++++++++++++------- crates/trigger/src/cli.rs | 7 ++--- crates/trigger/src/factors.rs | 17 ++++++------ 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/crates/factor-llm/src/spin.rs b/crates/factor-llm/src/spin.rs index 6d59c786c..b9b91c699 100644 --- a/crates/factor-llm/src/spin.rs +++ b/crates/factor-llm/src/spin.rs @@ -38,18 +38,25 @@ mod local { /// The default engine creator for the LLM factor when used in the Spin CLI. pub fn default_engine_creator( - state_dir: PathBuf, + state_dir: Option, use_gpu: bool, -) -> impl LlmEngineCreator + 'static { +) -> anyhow::Result { #[cfg(feature = "llm")] - let engine = spin_llm_local::LocalLlmEngine::new(state_dir.join("ai-models"), use_gpu); + let engine = { + use anyhow::Context as _; + let models_dir_parent = match state_dir { + Some(ref dir) => dir.clone(), + None => std::env::current_dir().context("failed to get current working directory")?, + }; + spin_llm_local::LocalLlmEngine::new(models_dir_parent.join("ai-models"), use_gpu) + }; #[cfg(not(feature = "llm"))] let engine = { let _ = (state_dir, use_gpu); noop::NoopLlmEngine }; let engine = Arc::new(Mutex::new(engine)) as Arc>; - move || engine.clone() + Ok(move || engine.clone()) } #[async_trait] @@ -74,7 +81,7 @@ impl LlmEngine for RemoteHttpLlmEngine { pub fn runtime_config_from_toml( table: &toml::Table, - state_dir: PathBuf, + state_dir: Option, use_gpu: bool, ) -> anyhow::Result> { let Some(value) = table.get("llm_compute") else { @@ -95,7 +102,7 @@ pub enum LlmCompute { } impl LlmCompute { - fn into_engine(self, state_dir: PathBuf, use_gpu: bool) -> Arc> { + fn into_engine(self, state_dir: Option, use_gpu: bool) -> Arc> { match self { #[cfg(not(feature = "llm"))] LlmCompute::Spin => { diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 801b60417..a7217c8e9 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -16,6 +16,7 @@ use spin_factor_sqlite::runtime_config::spin as sqlite; use spin_factor_sqlite::SqliteFactor; use spin_factor_variables::{spin_cli as variables, VariablesFactor}; use spin_factor_wasi::WasiFactor; +use spin_factors::runtime_config::toml::GetTomlValue as _; use spin_factors::{ runtime_config::toml::TomlKeyTracker, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, }; @@ -33,6 +34,10 @@ pub struct ResolvedRuntimeConfig { pub key_value_resolver: key_value::RuntimeConfigResolver, /// The resolver used to resolve sqlite databases from runtime configuration. pub sqlite_resolver: sqlite::RuntimeConfigResolver, + /// The fully resolved state directory. + /// + /// `None` is used for an "unset" state directory which each factor will treat differently. + pub state_dir: Option, } impl ResolvedRuntimeConfig @@ -41,6 +46,9 @@ where for<'a> >>::Error: Into, { /// Creates a new resolved runtime configuration from a runtime config source TOML file. + /// + /// `state_dir` is the explicitly provided state directory, if any. Some("") will be treated as + /// as `None`. pub fn from_file( runtime_config_path: &Path, state_dir: Option<&str>, @@ -64,21 +72,22 @@ where runtime_config_path.display() ) })?; - let runtime_config: T = TomlRuntimeConfigSource::new( + let source = TomlRuntimeConfigSource::new( &toml, - state_dir.unwrap_or(DEFAULT_STATE_DIR).into(), + state_dir, &key_value_config_resolver, &tls_resolver, &sqlite_config_resolver, use_gpu, - ) - .try_into() - .map_err(Into::into)?; + ); + let state_dir = source.state_dir(); + let runtime_config: T = source.try_into().map_err(Into::into)?; Ok(Self { runtime_config, key_value_resolver: key_value_config_resolver, sqlite_resolver: sqlite_config_resolver, + state_dir, }) } @@ -102,16 +111,23 @@ where } Ok(()) } + + /// The fully resolved state directory. + pub fn state_dir(&self) -> Option { + self.state_dir.clone() + } } impl ResolvedRuntimeConfig { + /// Creates a new resolved runtime configuration with default values. pub fn default(state_dir: Option<&str>) -> Self { let state_dir = state_dir.map(PathBuf::from); Self { sqlite_resolver: sqlite_config_resolver(state_dir.clone()) .expect("failed to resolve sqlite runtime config"), - key_value_resolver: key_value_config_resolver(state_dir), + key_value_resolver: key_value_config_resolver(state_dir.clone()), runtime_config: Default::default(), + state_dir, } } } @@ -119,7 +135,8 @@ impl ResolvedRuntimeConfig { /// The TOML based runtime configuration source Spin CLI. pub struct TomlRuntimeConfigSource<'a> { table: TomlKeyTracker<'a>, - state_dir: PathBuf, + /// Explicitly provided state directory. + state_dir: Option<&'a str>, key_value: &'a key_value::RuntimeConfigResolver, tls: &'a SpinTlsRuntimeConfig, sqlite: &'a sqlite::RuntimeConfigResolver, @@ -129,7 +146,7 @@ pub struct TomlRuntimeConfigSource<'a> { impl<'a> TomlRuntimeConfigSource<'a> { pub fn new( table: &'a toml::Table, - state_dir: PathBuf, + state_dir: Option<&'a str>, key_value: &'a key_value::RuntimeConfigResolver, tls: &'a SpinTlsRuntimeConfig, sqlite: &'a sqlite::RuntimeConfigResolver, @@ -144,6 +161,17 @@ impl<'a> TomlRuntimeConfigSource<'a> { use_gpu, } } + + /// Get the configured state_directory. + pub fn state_dir(&self) -> Option { + let from_toml = || self.table.get("state_dir").and_then(|v| v.as_str()); + // Prefer explicitly provided state directory, then take from toml. + self.state_dir + .or_else(from_toml) + // Treat "" as None. + .filter(|s| !s.is_empty()) + .map(PathBuf::from) + } } impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { @@ -187,7 +215,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource< impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { fn get_runtime_config(&mut self) -> anyhow::Result> { - llm::runtime_config_from_toml(self.table.as_ref(), self.state_dir.clone(), self.use_gpu) + llm::runtime_config_from_toml(self.table.as_ref(), self.state_dir(), self.use_gpu) } } diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index 5678c6875..6aa99ab4a 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -10,7 +10,7 @@ use spin_common::ui::quoted_path; use spin_common::url::parse_file_url; use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; -use spin_runtime_config::{ResolvedRuntimeConfig, DEFAULT_STATE_DIR}; +use spin_runtime_config::ResolvedRuntimeConfig; use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; @@ -321,13 +321,14 @@ impl TriggerAppBuilder { .await?; let factors = TriggerFactors::new( - options.state_dir.unwrap_or(DEFAULT_STATE_DIR), + runtime_config.state_dir(), self.working_dir.clone(), options.allow_transient_write, runtime_config.key_value_resolver, runtime_config.sqlite_resolver, use_gpu, - ); + ) + .context("failed to create factors")?; // TODO: move these into Factor methods/constructors // let init_data = crate::HostComponentInitData::new( diff --git a/crates/trigger/src/factors.rs b/crates/trigger/src/factors.rs index 6274f6bcf..1aa92f13e 100644 --- a/crates/trigger/src/factors.rs +++ b/crates/trigger/src/factors.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use anyhow::Context as _; use spin_factor_key_value::KeyValueFactor; use spin_factor_llm::LlmFactor; use spin_factor_outbound_http::OutboundHttpFactor; @@ -31,14 +32,14 @@ pub struct TriggerFactors { impl TriggerFactors { pub fn new( - state_dir: impl Into, + state_dir: Option, working_dir: impl Into, allow_transient_writes: bool, default_key_value_label_resolver: impl spin_factor_key_value::DefaultLabelResolver + 'static, default_sqlite_label_resolver: impl spin_factor_sqlite::DefaultLabelResolver + 'static, use_gpu: bool, - ) -> Self { - Self { + ) -> anyhow::Result { + Ok(Self { wasi: wasi_factor(working_dir, allow_transient_writes), variables: VariablesFactor::default(), key_value: KeyValueFactor::new(default_key_value_label_resolver), @@ -49,11 +50,11 @@ impl TriggerFactors { mqtt: OutboundMqttFactor::new(NetworkedMqttClient::creator()), pg: OutboundPgFactor::new(), mysql: OutboundMysqlFactor::new(), - llm: LlmFactor::new(spin_factor_llm::spin::default_engine_creator( - state_dir.into(), - use_gpu, - )), - } + llm: LlmFactor::new( + spin_factor_llm::spin::default_engine_creator(state_dir, use_gpu) + .context("failed to configure LLM factor")?, + ), + }) } } From dab87ce88af05abacdc1c167b59547626f3b9934 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 26 Aug 2024 16:28:46 +0200 Subject: [PATCH 169/195] Make sure resolved state_dir is used everywhere Signed-off-by: Ryan Levick --- crates/factor-key-value-spin/src/lib.rs | 72 ++++--- .../src/runtime_config/spin.rs | 13 +- crates/factor-key-value/tests/factor_test.rs | 5 +- crates/factors/src/runtime_config/toml.rs | 1 + crates/factors/tests/smoke.rs | 5 +- crates/runtime-config/src/lib.rs | 188 +++++++++++------- crates/trigger/src/cli.rs | 19 +- crates/trigger/src/factors.rs | 4 +- 8 files changed, 187 insertions(+), 120 deletions(-) diff --git a/crates/factor-key-value-spin/src/lib.rs b/crates/factor-key-value-spin/src/lib.rs index 062398a01..636b6c491 100644 --- a/crates/factor-key-value-spin/src/lib.rs +++ b/crates/factor-key-value-spin/src/lib.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::Context; +use anyhow::{bail, Context}; use serde::{Deserialize, Serialize}; use spin_factor_key_value::runtime_config::spin::MakeKeyValueStore; use spin_key_value_sqlite::{DatabaseLocation, KeyValueSqlite}; @@ -25,26 +25,6 @@ impl SpinKeyValueStore { } } -/// Runtime configuration for the SQLite key-value store. -#[derive(Deserialize, Serialize)] -pub struct SpinKeyValueRuntimeConfig { - /// The path to the SQLite database file. - path: Option, -} - -impl SpinKeyValueRuntimeConfig { - /// The default filename for the SQLite database. - const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; -} - -impl Default for SpinKeyValueRuntimeConfig { - fn default() -> Self { - Self { - path: Some(PathBuf::from(Self::DEFAULT_SPIN_STORE_FILENAME)), - } - } -} - impl MakeKeyValueStore for SpinKeyValueStore { const RUNTIME_CONFIG_TYPE: &'static str = "spin"; @@ -56,24 +36,50 @@ impl MakeKeyValueStore for SpinKeyValueStore { &self, runtime_config: Self::RuntimeConfig, ) -> anyhow::Result { - // The base path and the subpath must both be set otherwise, we default to in-memory. - let location = - if let (Some(base_path), Some(path)) = (&self.base_path, &runtime_config.path) { + let location = match (&self.base_path, &runtime_config.path) { + // If both the base path and the path are specified, resolve the path against the base path + (Some(base_path), Some(path)) => { let path = resolve_relative_path(path, base_path); - // Create the store's parent directory if necessary - let parent = path.parent().unwrap(); - if !parent.exists() { - fs::create_dir_all(parent) - .context("Failed to create key value store's parent directory")?; - } DatabaseLocation::Path(path) - } else { - DatabaseLocation::InMemory - }; + } + // If the base path is `None` but path is an absolute path, use the absolute path + (None, Some(path)) if path.is_absolute() => DatabaseLocation::Path(path.clone()), + // If the base path is `None` but path is a relative path, error out + (None, Some(path)) => { + bail!( + "key-value store path '{}' is relative, but no base path is set", + path.display() + ) + } + // Otherwise, use an in-memory database + (None | Some(_), None) => DatabaseLocation::InMemory, + }; + if let DatabaseLocation::Path(path) = &location { + // Create the store's parent directory if necessary + if let Some(parent) = path.parent().filter(|p| !p.exists()) { + fs::create_dir_all(parent) + .context("Failed to create key value store's parent directory")?; + } + } Ok(KeyValueSqlite::new(location)) } } +/// The serialized runtime configuration for the SQLite key-value store. +#[derive(Deserialize, Serialize)] +pub struct SpinKeyValueRuntimeConfig { + /// The path to the SQLite database file. + path: Option, +} + +impl SpinKeyValueRuntimeConfig { + /// Create a new SpinKeyValueRuntimeConfig with the given parent directory + /// where the key-value store will live. + pub fn new(path: Option) -> Self { + Self { path } + } +} + /// Resolve a relative path against a base dir. /// /// If the path is absolute, it is returned as is. Otherwise, it is resolved against the base dir. diff --git a/crates/factor-key-value/src/runtime_config/spin.rs b/crates/factor-key-value/src/runtime_config/spin.rs index 860208b7e..e604d3a5f 100644 --- a/crates/factor-key-value/src/runtime_config/spin.rs +++ b/crates/factor-key-value/src/runtime_config/spin.rs @@ -28,11 +28,12 @@ type StoreFromToml = /// Creates a `StoreFromToml` function from a `MakeKeyValueStore` implementation. fn store_from_toml_fn(provider_type: T) -> StoreFromToml { Arc::new(move |table| { - let runtime_config: T::RuntimeConfig = - table.try_into().context("could not parse runtime config")?; + let runtime_config: T::RuntimeConfig = table + .try_into() + .context("could not parse key-value runtime config")?; let provider = provider_type .make_store(runtime_config) - .context("could not make store")?; + .context("could not make key-value store from runtime config")?; Ok(Arc::new(provider)) }) } @@ -108,7 +109,9 @@ impl RuntimeConfigResolver { let mut runtime_config = RuntimeConfig::default(); for (label, config) in table { - let store_manager = self.store_manager_from_config(config)?; + let store_manager = self.store_manager_from_config(config).with_context(|| { + format!("could not configure key-value store with label '{label}'") + })?; runtime_config.add_store_manager(label.clone(), store_manager); } Ok(Some(runtime_config)) @@ -133,6 +136,8 @@ impl RuntimeConfigResolver { impl DefaultLabelResolver for RuntimeConfigResolver { fn default(&self, label: &str) -> Option> { let config = self.defaults.get(label)?; + // TODO(rylev): The unwrap here is not ideal. We should return a Result instead. + // Piping that through `DefaultLabelResolver` is a bit awkward, though. Some(self.store_manager_from_config(config.clone()).unwrap()) } } diff --git a/crates/factor-key-value/tests/factor_test.rs b/crates/factor-key-value/tests/factor_test.rs index f796162d6..1b0facbf6 100644 --- a/crates/factor-key-value/tests/factor_test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -4,7 +4,7 @@ use spin_factor_key_value::{ KeyValueFactor, RuntimeConfig, }; use spin_factor_key_value_redis::RedisKeyValueStore; -use spin_factor_key_value_spin::SpinKeyValueStore; +use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factors::{FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, RuntimeFactors}; use spin_factors_test::{toml, TestEnvironment}; use spin_world::v2::key_value::HostStore; @@ -19,7 +19,8 @@ struct TestFactors { async fn default_key_value_works() -> anyhow::Result<()> { let mut test_resolver = RuntimeConfigResolver::new(); test_resolver.register_store_type(SpinKeyValueStore::new(None))?; - test_resolver.add_default_store::("default", Default::default())?; + test_resolver + .add_default_store::("default", SpinKeyValueRuntimeConfig::new(None))?; let factors = TestFactors { key_value: KeyValueFactor::new(test_resolver), }; diff --git a/crates/factors/src/runtime_config/toml.rs b/crates/factors/src/runtime_config/toml.rs index 42231a148..23e3ef1e2 100644 --- a/crates/factors/src/runtime_config/toml.rs +++ b/crates/factors/src/runtime_config/toml.rs @@ -13,6 +13,7 @@ impl GetTomlValue for toml::Table { } } +#[derive(Debug, Clone)] /// A helper for tracking which keys have been used in a TOML table. pub struct TomlKeyTracker<'a> { unused_keys: RefCell>, diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs index 95e77d7b6..628b29136 100644 --- a/crates/factors/tests/smoke.rs +++ b/crates/factors/tests/smoke.rs @@ -5,7 +5,7 @@ use http_body_util::BodyExt; use spin_app::App; use spin_factor_key_value::{runtime_config::spin::RuntimeConfigResolver, KeyValueFactor}; use spin_factor_key_value_redis::RedisKeyValueStore; -use spin_factor_key_value_spin::SpinKeyValueStore; +use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_networking::OutboundNetworkingFactor; use spin_factor_variables::VariablesFactor; @@ -39,7 +39,8 @@ impl AsInstanceState for Data { #[tokio::test(flavor = "multi_thread")] async fn smoke_test_works() -> anyhow::Result<()> { let mut key_value_resolver = RuntimeConfigResolver::default(); - key_value_resolver.add_default_store::("default", Default::default())?; + key_value_resolver + .add_default_store::("default", SpinKeyValueRuntimeConfig::new(None))?; key_value_resolver.register_store_type(SpinKeyValueStore::new(Some( std::env::current_dir().context("failed to get current directory")?, )))?; diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index a7217c8e9..8fe53a17c 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context as _; use spin_factor_key_value::runtime_config::spin::{self as key_value}; use spin_factor_key_value::{DefaultLabelResolver as _, KeyValueFactor}; -use spin_factor_key_value_spin::SpinKeyValueStore; +use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; use spin_factor_llm::{spin as llm, LlmFactor}; use spin_factor_outbound_http::OutboundHttpFactor; use spin_factor_outbound_mqtt::OutboundMqttFactor; @@ -42,24 +42,31 @@ pub struct ResolvedRuntimeConfig { impl ResolvedRuntimeConfig where - T: for<'a> TryFrom>, - for<'a> >>::Error: Into, + T: for<'a, 'b> TryFrom>, + for<'a, 'b> >>::Error: Into, { + /// Creates a new resolved runtime configuration from an optional runtime config source TOML file. + pub fn from_optional_file( + runtime_config_path: Option<&Path>, + provided_state_dir: Option<&Path>, + use_gpu: bool, + ) -> anyhow::Result { + match runtime_config_path { + Some(runtime_config_path) => { + Self::from_file(runtime_config_path, provided_state_dir, use_gpu) + } + None => Self::new(Default::default(), None, provided_state_dir, use_gpu), + } + } + /// Creates a new resolved runtime configuration from a runtime config source TOML file. /// - /// `state_dir` is the explicitly provided state directory, if any. Some("") will be treated as - /// as `None`. + /// `provided_state_dir` is the explicitly provided state directory, if any. pub fn from_file( runtime_config_path: &Path, - state_dir: Option<&str>, + provided_state_dir: Option<&Path>, use_gpu: bool, ) -> anyhow::Result { - let tls_resolver = SpinTlsRuntimeConfig::new(runtime_config_path); - let key_value_config_resolver = key_value_config_resolver(state_dir.map(Into::into)); - - let sqlite_config_resolver = sqlite_config_resolver(state_dir.map(Into::into)) - .context("failed to resolve sqlite runtime config")?; - let file = std::fs::read_to_string(runtime_config_path).with_context(|| { format!( "failed to read runtime config file '{}'", @@ -72,22 +79,37 @@ where runtime_config_path.display() ) })?; + + Self::new(toml, Some(runtime_config_path), provided_state_dir, use_gpu) + } + + /// Creates a new resolved runtime configuration from a TOML table. + pub fn new( + toml: toml::Table, + runtime_config_path: Option<&Path>, + provided_state_dir: Option<&Path>, + use_gpu: bool, + ) -> anyhow::Result { + let toml_resolver = TomlResolver::new(&toml, provided_state_dir); + let tls_resolver = runtime_config_path.map(SpinTlsRuntimeConfig::new); + let key_value_config_resolver = key_value_config_resolver(toml_resolver.state_dir()?); + let sqlite_config_resolver = sqlite_config_resolver(toml_resolver.state_dir()?) + .context("failed to resolve sqlite runtime config")?; + let source = TomlRuntimeConfigSource::new( - &toml, - state_dir, + toml_resolver.clone(), &key_value_config_resolver, - &tls_resolver, + tls_resolver.as_ref(), &sqlite_config_resolver, use_gpu, ); - let state_dir = source.state_dir(); let runtime_config: T = source.try_into().map_err(Into::into)?; Ok(Self { runtime_config, key_value_resolver: key_value_config_resolver, sqlite_resolver: sqlite_config_resolver, - state_dir, + state_dir: toml_resolver.state_dir()?, }) } @@ -118,140 +140,164 @@ where } } -impl ResolvedRuntimeConfig { - /// Creates a new resolved runtime configuration with default values. - pub fn default(state_dir: Option<&str>) -> Self { - let state_dir = state_dir.map(PathBuf::from); +#[derive(Clone, Debug)] +/// Resolves runtime configuration from a TOML file. +pub struct TomlResolver<'a> { + table: TomlKeyTracker<'a>, + /// Explicitly provided state directory. + state_dir: Option<&'a Path>, +} + +impl<'a> TomlResolver<'a> { + /// Create a new TOML resolver. + /// + /// The `state_dir` is the explicitly provided state directory, if any. + pub fn new(table: &'a toml::Table, state_dir: Option<&'a Path>) -> Self { Self { - sqlite_resolver: sqlite_config_resolver(state_dir.clone()) - .expect("failed to resolve sqlite runtime config"), - key_value_resolver: key_value_config_resolver(state_dir.clone()), - runtime_config: Default::default(), + table: TomlKeyTracker::new(table), state_dir, } } + + /// Get the configured state_directory. + /// + /// Errors if the path cannot be converted to an absolute path. + pub fn state_dir(&self) -> std::io::Result> { + let from_toml = || { + self.table + .get("state_dir") + .and_then(|v| v.as_str()) + .filter(|v| !v.is_empty()) + .map(Path::new) + }; + // Prefer explicitly provided state directory, then take from toml. + self.state_dir + .or_else(from_toml) + .map(PathBuf::from) + .map(std::fs::canonicalize) + .transpose() + } + + /// Validate that all keys in the TOML file have been used. + pub fn validate_all_keys_used(&self) -> spin_factors::Result<()> { + self.table.validate_all_keys_used() + } +} + +impl AsRef for TomlResolver<'_> { + fn as_ref(&self) -> &toml::Table { + self.table.as_ref() + } } /// The TOML based runtime configuration source Spin CLI. -pub struct TomlRuntimeConfigSource<'a> { - table: TomlKeyTracker<'a>, - /// Explicitly provided state directory. - state_dir: Option<&'a str>, +pub struct TomlRuntimeConfigSource<'a, 'b> { + toml: TomlResolver<'b>, key_value: &'a key_value::RuntimeConfigResolver, - tls: &'a SpinTlsRuntimeConfig, + tls: Option<&'a SpinTlsRuntimeConfig>, sqlite: &'a sqlite::RuntimeConfigResolver, use_gpu: bool, } -impl<'a> TomlRuntimeConfigSource<'a> { +impl<'a, 'b> TomlRuntimeConfigSource<'a, 'b> { pub fn new( - table: &'a toml::Table, - state_dir: Option<&'a str>, + toml_resolver: TomlResolver<'b>, key_value: &'a key_value::RuntimeConfigResolver, - tls: &'a SpinTlsRuntimeConfig, + tls: Option<&'a SpinTlsRuntimeConfig>, sqlite: &'a sqlite::RuntimeConfigResolver, use_gpu: bool, ) -> Self { Self { - table: TomlKeyTracker::new(table), - state_dir, + toml: toml_resolver, key_value, tls, sqlite, use_gpu, } } - - /// Get the configured state_directory. - pub fn state_dir(&self) -> Option { - let from_toml = || self.table.get("state_dir").and_then(|v| v.as_str()); - // Prefer explicitly provided state directory, then take from toml. - self.state_dir - .or_else(from_toml) - // Treat "" as None. - .filter(|s| !s.is_empty()) - .map(PathBuf::from) - } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config( &mut self, ) -> anyhow::Result> { - self.key_value.resolve_from_toml(Some(self.table.as_ref())) + self.key_value.resolve_from_toml(Some(self.toml.as_ref())) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config( &mut self, ) -> anyhow::Result::RuntimeConfig>> { - self.tls.config_from_table(self.table.as_ref()) + let Some(tls) = self.tls else { + return Ok(None); + }; + tls.config_from_table(self.toml.as_ref()) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config( &mut self, ) -> anyhow::Result::RuntimeConfig>> { Ok(Some(variables::runtime_config_from_toml( - self.table.as_ref(), + self.toml.as_ref(), )?)) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { - llm::runtime_config_from_toml(self.table.as_ref(), self.state_dir(), self.use_gpu) + llm::runtime_config_from_toml(self.toml.as_ref(), self.toml.state_dir()?, self.use_gpu) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { Ok(None) } } -impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_> { +impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { - self.sqlite.resolve_from_toml(self.table.as_ref()) + self.sqlite.resolve_from_toml(self.toml.as_ref()) } } -impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_> { +impl RuntimeConfigSourceFinalizer for TomlRuntimeConfigSource<'_, '_> { fn finalize(&mut self) -> anyhow::Result<()> { - Ok(self.table.validate_all_keys_used()?) + Ok(self.toml.validate_all_keys_used()?) } } @@ -259,7 +305,8 @@ const DEFAULT_KEY_VALUE_STORE_LABEL: &str = "default"; /// The key-value runtime configuration resolver. /// -/// Takes a base path for the local store. +/// Takes a base path that all local key-value stores which are configured with +/// relative paths will be relative to. pub fn key_value_config_resolver( local_store_base_path: Option, ) -> key_value::RuntimeConfigResolver { @@ -282,11 +329,18 @@ pub fn key_value_config_resolver( // Add handling of "default" store. // Unwraps are safe because the store is known to be serializable as toml. key_value - .add_default_store::(DEFAULT_KEY_VALUE_STORE_LABEL, Default::default()) + .add_default_store::( + DEFAULT_KEY_VALUE_STORE_LABEL, + SpinKeyValueRuntimeConfig::new( + local_store_base_path.map(|p| p.join(DEFAULT_SPIN_STORE_FILENAME)), + ), + ) .unwrap(); key_value } +/// The default filename for the SQLite database. +const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; /// The sqlite runtime configuration resolver. /// diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index 6aa99ab4a..bee7f81e1 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -304,17 +304,16 @@ impl TriggerAppBuilder { }; self.trigger.add_to_linker(core_engine_builder.linker())?; + // Hardcode `use_gpu` to true for now let use_gpu = true; - let runtime_config = match options.runtime_config_file { - Some(runtime_config_path) => { - ResolvedRuntimeConfig::::from_file( - runtime_config_path, - options.state_dir, - use_gpu, - )? - } - None => ResolvedRuntimeConfig::default(options.state_dir), - }; + // Make sure `--state-dir=""` unsets the state dir + let state_dir = options.state_dir.filter(|s| !s.is_empty()).map(Path::new); + let runtime_config = + ResolvedRuntimeConfig::::from_optional_file( + options.runtime_config_file, + state_dir, + use_gpu, + )?; runtime_config .set_initial_key_values(&options.initial_key_values) diff --git a/crates/trigger/src/factors.rs b/crates/trigger/src/factors.rs index 1aa92f13e..3a1b0993a 100644 --- a/crates/trigger/src/factors.rs +++ b/crates/trigger/src/factors.rs @@ -81,10 +81,10 @@ fn outbound_networking_factor() -> OutboundNetworkingFactor { factor } -impl TryFrom> for TriggerFactorsRuntimeConfig { +impl TryFrom> for TriggerFactorsRuntimeConfig { type Error = anyhow::Error; - fn try_from(value: TomlRuntimeConfigSource<'_>) -> Result { + fn try_from(value: TomlRuntimeConfigSource<'_, '_>) -> Result { Self::from_source(value) } } From 79caa0b1da0ea07546234d9134af3a5908618dd6 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 26 Aug 2024 17:02:31 +0200 Subject: [PATCH 170/195] Default state dir to local app dir location Signed-off-by: Ryan Levick --- crates/trigger/src/cli.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index bee7f81e1..c5cc2cc0c 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -155,6 +155,7 @@ impl FactorsTriggerCommand { // Required env vars let working_dir = std::env::var(SPIN_WORKING_DIR).context(SPIN_WORKING_DIR)?; let locked_url = std::env::var(SPIN_LOCKED_URL).context(SPIN_LOCKED_URL)?; + let local_app_dir = std::env::var(SPIN_LOCAL_APP_DIR).ok(); let follow_components = self.follow_components(); @@ -192,6 +193,7 @@ impl FactorsTriggerCommand { TriggerAppOptions { runtime_config_file: self.runtime_config_file.as_deref(), state_dir: self.state_dir.as_deref(), + local_app_dir: local_app_dir.as_deref(), initial_key_values: self.key_values, allow_transient_write: self.allow_transient_write, follow_components, @@ -268,6 +270,8 @@ pub struct TriggerAppOptions<'a> { runtime_config_file: Option<&'a Path>, /// Path to the state directory. state_dir: Option<&'a str>, + /// Path to the local app directory. + local_app_dir: Option<&'a str>, /// Initial key/value pairs to set in the app's default store. initial_key_values: Vec<(String, String)>, /// Whether to allow transient writes to mounted files @@ -306,12 +310,17 @@ impl TriggerAppBuilder { // Hardcode `use_gpu` to true for now let use_gpu = true; - // Make sure `--state-dir=""` unsets the state dir - let state_dir = options.state_dir.filter(|s| !s.is_empty()).map(Path::new); + let state_dir = match options.state_dir { + // Make sure `--state-dir=""` unsets the state dir + Some(s) if s.is_empty() => None, + Some(s) => Some(PathBuf::from(s)), + // Default to `.spin/` in the local app dir + None => options.local_app_dir.map(|d| Path::new(d).join(".spin")), + }; let runtime_config = ResolvedRuntimeConfig::::from_optional_file( options.runtime_config_file, - state_dir, + state_dir.as_deref(), use_gpu, )?; From 30c3009cf01b132b227f31357fd2f8d413c48d92 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 26 Aug 2024 18:45:58 +0200 Subject: [PATCH 171/195] Make Clippy happy Signed-off-by: Ryan Levick --- crates/componentize/src/module_info.rs | 11 +++-------- crates/factor-sqlite/src/runtime_config/spin.rs | 2 +- crates/runtime-config/src/lib.rs | 3 ++- crates/trigger/src/cli.rs | 2 +- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/crates/componentize/src/module_info.rs b/crates/componentize/src/module_info.rs index e02ece13c..c951b2443 100644 --- a/crates/componentize/src/module_info.rs +++ b/crates/componentize/src/module_info.rs @@ -75,15 +75,10 @@ impl ModuleInfo { /// Returns true if the given module was heuristically probably compiled /// with wit-bindgen. pub fn probably_uses_wit_bindgen(&self) -> bool { - if self.bindgen.is_some() { - // Presence of bindgen metadata is a strong signal - true - } else if self.realloc_export.is_some() { + // Presence of bindgen metadata is a strong signal + self.bindgen.is_some() || // A canonical ABI realloc export is a decent signal - true - } else { - false - } + self.realloc_export.is_some() } /// Returns the wit-bindgen metadata producers processed-by field, if diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index 6274fd97d..b12a87616 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -30,7 +30,7 @@ impl RuntimeConfigResolver { /// This takes as arguments: /// * the directory to use as the default location for SQLite databases. /// Usually this will be the path to the `.spin` state directory. If - /// `None`, the default database will be in-memory. + /// `None`, the default database will be in-memory. /// * the path to the directory from which relative paths to /// local SQLite databases are resolved. (this should most likely be the /// path to the runtime-config file or the current working dir). diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 8fe53a17c..2448605a0 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -339,8 +339,9 @@ pub fn key_value_config_resolver( key_value } + /// The default filename for the SQLite database. -const DEFAULT_SPIN_STORE_FILENAME: &'static str = "sqlite_key_value.db"; +const DEFAULT_SPIN_STORE_FILENAME: &str = "sqlite_key_value.db"; /// The sqlite runtime configuration resolver. /// diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index c5cc2cc0c..cd492a31d 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -312,7 +312,7 @@ impl TriggerAppBuilder { let use_gpu = true; let state_dir = match options.state_dir { // Make sure `--state-dir=""` unsets the state dir - Some(s) if s.is_empty() => None, + Some(s) if s.trim().is_empty() => None, Some(s) => Some(PathBuf::from(s)), // Default to `.spin/` in the local app dir None => options.local_app_dir.map(|d| Path::new(d).join(".spin")), From 9576081eb3ec33cc9a723705ab9fd4cadccc3ac4 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 26 Aug 2024 18:53:41 +0200 Subject: [PATCH 172/195] Remove smoke test Signed-off-by: Ryan Levick --- Cargo.lock | 23 +-- crates/factors/Cargo.toml | 20 --- crates/factors/build.rs | 13 -- crates/factors/tests/smoke-app/.gitignore | 3 - crates/factors/tests/smoke-app/Cargo.toml | 15 -- crates/factors/tests/smoke-app/spin.toml | 21 --- crates/factors/tests/smoke-app/src/lib.rs | 19 -- crates/factors/tests/smoke.rs | 202 ---------------------- 8 files changed, 4 insertions(+), 312 deletions(-) delete mode 100644 crates/factors/tests/smoke-app/.gitignore delete mode 100644 crates/factors/tests/smoke-app/Cargo.toml delete mode 100644 crates/factors/tests/smoke-app/spin.toml delete mode 100644 crates/factors/tests/smoke-app/src/lib.rs delete mode 100644 crates/factors/tests/smoke.rs diff --git a/Cargo.lock b/Cargo.lock index 52fa76a10..92c6a8ed6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,7 +598,7 @@ dependencies = [ [[package]] name = "azure_core" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "base64 0.22.0", @@ -626,7 +626,7 @@ dependencies = [ [[package]] name = "azure_data_cosmos" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "azure_core", @@ -644,7 +644,7 @@ dependencies = [ [[package]] name = "azure_identity" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-lock 3.3.0", "async-process 2.2.2", @@ -664,7 +664,7 @@ dependencies = [ [[package]] name = "azure_security_keyvault" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "azure_core", @@ -7601,28 +7601,13 @@ name = "spin-factors" version = "2.8.0-pre0" dependencies = [ "anyhow", - "cargo-target-dep", - "http 1.1.0", - "http-body-util", "serde 1.0.197", - "serde_json", "spin-app", - "spin-componentize", - "spin-factor-key-value", - "spin-factor-key-value-redis", - "spin-factor-key-value-spin", - "spin-factor-outbound-http", - "spin-factor-outbound-networking", - "spin-factor-variables", - "spin-factor-wasi", "spin-factors-derive", - "spin-loader", "thiserror", - "tokio", "toml 0.8.14", "tracing", "wasmtime", - "wasmtime-wasi-http", ] [[package]] diff --git a/crates/factors/Cargo.toml b/crates/factors/Cargo.toml index 71b97694e..075e9a211 100644 --- a/crates/factors/Cargo.toml +++ b/crates/factors/Cargo.toml @@ -15,25 +15,5 @@ toml = "0.8" tracing = { workspace = true } wasmtime = { workspace = true } -[dev-dependencies] -http = "1.1.0" -http-body-util = "0.1.2" -serde_json = "1.0" -spin-componentize = { path = "../componentize" } -spin-factors-derive = { path = "../factors-derive", features = ["expander"] } -spin-factor-key-value = { path = "../factor-key-value" } -spin-factor-key-value-redis = { path = "../factor-key-value-redis" } -spin-factor-key-value-spin = { path = "../factor-key-value-spin" } -spin-factor-outbound-http = { path = "../factor-outbound-http" } -spin-factor-outbound-networking = { path = "../factor-outbound-networking" } -spin-factor-variables = { path = "../factor-variables" } -spin-factor-wasi = { path = "../factor-wasi" } -spin-loader = { path = "../loader" } -tokio = { version = "1", features = ["macros", "rt", "sync"] } -wasmtime-wasi-http = { workspace = true } - -[build-dependencies] -cargo-target-dep = { git = "https://github.com/fermyon/cargo-target-dep", rev = "482f269eceb7b1a7e8fc618bf8c082dd24979cf1" } - [lints] workspace = true diff --git a/crates/factors/build.rs b/crates/factors/build.rs index 923b145fe..c96556b06 100644 --- a/crates/factors/build.rs +++ b/crates/factors/build.rs @@ -1,19 +1,6 @@ -use std::path::Path; - -use cargo_target_dep::build_target_dep; - fn main() { println!("cargo:rerun-if-changed=build.rs"); // Enable spin-factors-derive to emit expanded macro output. let out_dir = std::env::var("OUT_DIR").unwrap(); println!("cargo:rustc-env=SPIN_FACTORS_DERIVE_EXPAND_DIR={out_dir}"); - - let root = "tests/smoke-app"; - build_target_dep(root, Path::new("tests/smoke-app/smoke_app.wasm")) - .release() - .target("wasm32-wasi") - .build(); - println!("cargo:rerun-if-changed={root}/Cargo.toml"); - println!("cargo:rerun-if-changed={root}/Cargo.lock"); - println!("cargo:rerun-if-changed={root}/src"); } diff --git a/crates/factors/tests/smoke-app/.gitignore b/crates/factors/tests/smoke-app/.gitignore deleted file mode 100644 index 671b13b2a..000000000 --- a/crates/factors/tests/smoke-app/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -target/ -.spin/ -*.wasm diff --git a/crates/factors/tests/smoke-app/Cargo.toml b/crates/factors/tests/smoke-app/Cargo.toml deleted file mode 100644 index d4fc36af1..000000000 --- a/crates/factors/tests/smoke-app/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "smoke-app" -authors = ["Lann Martin "] -description = "" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] - -[dependencies] -anyhow = "1" -spin-sdk = "3.0.1" - -[workspace] diff --git a/crates/factors/tests/smoke-app/spin.toml b/crates/factors/tests/smoke-app/spin.toml deleted file mode 100644 index a49ca70f8..000000000 --- a/crates/factors/tests/smoke-app/spin.toml +++ /dev/null @@ -1,21 +0,0 @@ -spin_manifest_version = 2 - -[application] -name = "smoke-app" -version = "0.1.0" -authors = ["Lann Martin "] -description = "" - -[variables] -host = { required = true } -other = { default = "other value" } - -[[trigger.http]] -route = "/..." -component = "smoke-app" - -[component.smoke-app] -source = "smoke_app.wasm" -allowed_outbound_hosts = ["https://{{ host }}"] -key_value_stores = ["default"] -variables = { "other" = "<{{ other }}>"} \ No newline at end of file diff --git a/crates/factors/tests/smoke-app/src/lib.rs b/crates/factors/tests/smoke-app/src/lib.rs deleted file mode 100644 index 214373d29..000000000 --- a/crates/factors/tests/smoke-app/src/lib.rs +++ /dev/null @@ -1,19 +0,0 @@ -use spin_sdk::http::{IntoResponse, Request, Response}; -use spin_sdk::http_component; - -/// A simple Spin HTTP component. -#[http_component] -fn handle_smoke_app(_req: Request) -> anyhow::Result { - let var_val = spin_sdk::variables::get("other")?; - let kv_val = { - let store = spin_sdk::key_value::Store::open_default()?; - store.set("k", b"v")?; - store.get("k")? - }; - let body = format!("Test response\nVariable: {var_val}\nKV: {kv_val:?}"); - Ok(Response::builder() - .status(200) - .header("content-type", "text/plain") - .body(body) - .build()) -} diff --git a/crates/factors/tests/smoke.rs b/crates/factors/tests/smoke.rs deleted file mode 100644 index 628b29136..000000000 --- a/crates/factors/tests/smoke.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::sync::Arc; - -use anyhow::Context; -use http_body_util::BodyExt; -use spin_app::App; -use spin_factor_key_value::{runtime_config::spin::RuntimeConfigResolver, KeyValueFactor}; -use spin_factor_key_value_redis::RedisKeyValueStore; -use spin_factor_key_value_spin::{SpinKeyValueRuntimeConfig, SpinKeyValueStore}; -use spin_factor_outbound_http::OutboundHttpFactor; -use spin_factor_outbound_networking::OutboundNetworkingFactor; -use spin_factor_variables::VariablesFactor; -use spin_factor_wasi::{DummyFilesMounter, WasiFactor}; -use spin_factors::{ - AsInstanceState, Factor, FactorRuntimeConfigSource, RuntimeConfigSourceFinalizer, - RuntimeFactors, -}; -use wasmtime_wasi_http::WasiHttpView; - -#[derive(RuntimeFactors)] -struct Factors { - wasi: WasiFactor, - variables: VariablesFactor, - outbound_networking: OutboundNetworkingFactor, - outbound_http: OutboundHttpFactor, - key_value: KeyValueFactor, -} - -struct Data { - factors_instance_state: FactorsInstanceState, - _other_data: usize, -} - -impl AsInstanceState for Data { - fn as_instance_state(&mut self) -> &mut FactorsInstanceState { - &mut self.factors_instance_state - } -} - -#[tokio::test(flavor = "multi_thread")] -async fn smoke_test_works() -> anyhow::Result<()> { - let mut key_value_resolver = RuntimeConfigResolver::default(); - key_value_resolver - .add_default_store::("default", SpinKeyValueRuntimeConfig::new(None))?; - key_value_resolver.register_store_type(SpinKeyValueStore::new(Some( - std::env::current_dir().context("failed to get current directory")?, - )))?; - key_value_resolver.register_store_type(RedisKeyValueStore::new())?; - let key_value_resolver = Arc::new(key_value_resolver); - - let mut factors = Factors { - wasi: WasiFactor::new(DummyFilesMounter), - variables: VariablesFactor::default(), - outbound_networking: OutboundNetworkingFactor::new(), - outbound_http: OutboundHttpFactor::new(), - key_value: KeyValueFactor::new(key_value_resolver.clone()), - }; - - let locked = spin_loader::from_file( - "tests/smoke-app/spin.toml", - spin_loader::FilesMountStrategy::Direct, - None, - ) - .await?; - let app = App::new("test-app", locked); - - let engine = wasmtime::Engine::new(wasmtime::Config::new().async_support(true))?; - let mut linker = wasmtime::component::Linker::new(&engine); - - factors.init(&mut linker)?; - - let source = TestSource { key_value_resolver }; - let configured_app = factors.configure_app(app, source.try_into()?)?; - let builders = factors.prepare(&configured_app, "smoke-app")?; - let state = factors.build_instance_state(builders)?; - - assert_eq!( - state - .variables - .expression_resolver() - .resolve("smoke-app", "other".try_into().unwrap()) - .await - .unwrap(), - "" - ); - - let data = Data { - factors_instance_state: state, - _other_data: 1, - }; - - let mut store = wasmtime::Store::new(&engine, data); - - let component = configured_app.app().components().next().unwrap(); - let wasm_path = component - .source() - .content - .source - .as_deref() - .unwrap() - .strip_prefix("file://") - .unwrap(); - let wasm_bytes = std::fs::read(wasm_path) - .with_context(|| format!("wasm binary not found at '{wasm_path}'. Did you remember to run `spin build` in the `smoke-app` directory?"))?; - let component_bytes = spin_componentize::componentize_if_necessary(&wasm_bytes)?; - let component = wasmtime::component::Component::new(&engine, component_bytes)?; - let instance = linker.instantiate_async(&mut store, &component).await?; - - // Invoke handler - let req = http::Request::get("/").body(Default::default()).unwrap(); - let mut wasi_http = - OutboundHttpFactor::get_wasi_http_impl(store.data_mut().as_instance_state()).unwrap(); - let request = wasi_http.new_incoming_request(req)?; - let (response_tx, response_rx) = tokio::sync::oneshot::channel(); - let response = wasi_http.new_response_outparam(response_tx)?; - drop(wasi_http); - - let guest = wasmtime_wasi_http::proxy::Proxy::new(&mut store, &instance)?; - let call_task = tokio::spawn(async move { - guest - .wasi_http_incoming_handler() - .call_handle(&mut store, request, response) - .await - }); - let resp_task = tokio::spawn(async { - let resp = response_rx.await.unwrap().unwrap(); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - eprintln!("Response: {body:?}"); - }); - let (call_res, resp_res) = tokio::join!(call_task, resp_task); - let _ = call_res?; - resp_res?; - Ok(()) -} - -struct TestSource { - key_value_resolver: Arc, -} - -impl TryFrom for FactorsRuntimeConfig { - type Error = anyhow::Error; - - fn try_from(value: TestSource) -> Result { - Self::from_source(value) - } -} - -impl FactorRuntimeConfigSource for TestSource { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result::RuntimeConfig>> { - let config = toml::toml! { - [other] - type = "redis" - url = "redis://localhost:6379" - }; - self.key_value_resolver.resolve_from_toml(Some(&config)) - } -} - -impl FactorRuntimeConfigSource for TestSource { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result::RuntimeConfig>> { - spin_factor_variables::spin_cli::runtime_config_from_toml(&toml::toml! { - [[variable_provider]] - type = "static" - [variable_provider.values] - foo = "bar" - }) - .map(Some) - } -} - -impl FactorRuntimeConfigSource for TestSource { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result::RuntimeConfig>> { - Ok(None) - } -} - -impl FactorRuntimeConfigSource for TestSource { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result::RuntimeConfig>> { - Ok(None) - } -} - -impl FactorRuntimeConfigSource for TestSource { - fn get_runtime_config( - &mut self, - ) -> anyhow::Result::RuntimeConfig>> { - Ok(None) - } -} - -impl RuntimeConfigSourceFinalizer for TestSource { - fn finalize(&mut self) -> anyhow::Result<()> { - Ok(()) - } -} From 4430e5f4ee76b60016f38d9a99a9393d670b5fc4 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Mon, 26 Aug 2024 18:56:00 +0200 Subject: [PATCH 173/195] Remove factors specific CI Signed-off-by: Ryan Levick --- .github/workflows/factors.yml | 20 -------------------- run-factors-tests.sh | 4 ---- 2 files changed, 24 deletions(-) delete mode 100644 .github/workflows/factors.yml delete mode 100755 run-factors-tests.sh diff --git a/.github/workflows/factors.yml b/.github/workflows/factors.yml deleted file mode 100644 index 68ecf8fb6..000000000 --- a/.github/workflows/factors.yml +++ /dev/null @@ -1,20 +0,0 @@ -# TODO(factors): remove after factors branch passes regular CI -name: Factors -on: - push: - branches: ["factors"] - pull_request: - branches: ["factors"] -jobs: - factors-tests: - runs-on: ubuntu-22.04-4core-spin - steps: - - uses: actions/checkout@v3 - - name: setup dependencies - uses: ./.github/actions/spin-ci-dependencies - with: - rust: true - rust-wasm: true - rust-cache: true - - name: Run factors tests - run: ./run-factors-tests.sh diff --git a/run-factors-tests.sh b/run-factors-tests.sh deleted file mode 100755 index 26fe7b4b7..000000000 --- a/run-factors-tests.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -# TODO(factors): Remove after enabling CI for factors branch - -cargo test -p '*factor*' -p spin-trigger -p spin-trigger-http From a33b386330c3312a7ccbde932789667aebea2d19 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 13:36:38 -0400 Subject: [PATCH 174/195] Replace tempdir with tempfile Signed-off-by: Lann Martin --- Cargo.lock | 80 ++++---------------- crates/factor-key-value/Cargo.toml | 2 +- crates/factor-key-value/tests/factor_test.rs | 8 +- 3 files changed, 19 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92c6a8ed6..75c0efb31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2564,12 +2564,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "futures" version = "0.3.30" @@ -5842,19 +5836,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -5899,21 +5880,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -6000,15 +5966,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "reborrow" version = "0.5.5" @@ -6167,15 +6124,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "reqwest" version = "0.11.27" @@ -7363,7 +7311,7 @@ dependencies = [ "spin-factors-test", "spin-key-value", "spin-world", - "tempdir", + "tempfile", "tokio", "toml 0.8.14", ] @@ -8369,26 +8317,17 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f227968ec00f0e5322f9b8173c7a0cbcff6181a0a5b28e9892491c286277231" -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] - [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand 2.0.2", + "once_cell", "rustix 0.38.32", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -10413,6 +10352,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.42.2" diff --git a/crates/factor-key-value/Cargo.toml b/crates/factor-key-value/Cargo.toml index cab19b014..64df40352 100644 --- a/crates/factor-key-value/Cargo.toml +++ b/crates/factor-key-value/Cargo.toml @@ -18,7 +18,7 @@ spin-factors-test = { path = "../factors-test" } tokio = { version = "1", features = ["macros", "rt"] } spin-factor-key-value-spin = { path = "../factor-key-value-spin" } spin-factor-key-value-redis = { path = "../factor-key-value-redis" } -tempdir = "0.3" +tempfile = "3.12.0" [lints] diff --git a/crates/factor-key-value/tests/factor_test.rs b/crates/factor-key-value/tests/factor_test.rs index 1b0facbf6..890e95ca9 100644 --- a/crates/factor-key-value/tests/factor_test.rs +++ b/crates/factor-key-value/tests/factor_test.rs @@ -101,7 +101,7 @@ async fn custom_spin_key_value_works() -> anyhow::Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn custom_spin_key_value_works_with_absolute_path() -> anyhow::Result<()> { - let tmp_dir = tempdir::TempDir::new("example")?; + let tmp_dir = tempfile::TempDir::with_prefix("example")?; let db_path = tmp_dir.path().join("foo/custom.db"); // Check that the db does not exist yet - it will exist by the end of the test assert!(!db_path.exists()); @@ -132,7 +132,7 @@ async fn custom_spin_key_value_works_with_absolute_path() -> anyhow::Result<()> #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn custom_spin_key_value_works_with_relative_path() -> anyhow::Result<()> { - let tmp_dir = tempdir::TempDir::new("example")?; + let tmp_dir = tempfile::TempDir::with_prefix("example")?; let db_path = tmp_dir.path().join("custom.db"); // Check that the db does not exist yet - it will exist by the end of the test assert!(!db_path.exists()); @@ -176,7 +176,7 @@ async fn custom_redis_key_value_works() -> anyhow::Result<()> { #[tokio::test] async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { - let tmp_dir = tempdir::TempDir::new("example")?; + let tmp_dir = tempfile::TempDir::with_prefix("example")?; let runtime_config = toml::toml! { [key_value_store.custom] type = "spin" @@ -198,7 +198,7 @@ async fn misconfigured_spin_key_value_fails() -> anyhow::Result<()> { // TODO(rylev): consider removing this test as it is really only a consequence of // toml deserialization and not a feature of the key-value store. async fn multiple_custom_key_value_uses_second_store() -> anyhow::Result<()> { - let tmp_dir = tempdir::TempDir::new("example")?; + let tmp_dir = tempfile::TempDir::with_prefix("example")?; let db_path = tmp_dir.path().join("custom.db"); // Check that the db does not exist yet - it will exist by the end of the test assert!(!db_path.exists()); From ef5be9350ce83737d931383cfc3728d15b1bde75 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 13:40:05 -0400 Subject: [PATCH 175/195] Fix formatting Signed-off-by: Lann Martin --- crates/trigger/src/lib.rs | 5 +---- tests/integration.rs | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 0fde4a021..0cc119f16 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -53,10 +53,7 @@ pub trait Trigger: Sized + Send { } /// Run this trigger. - fn run( - self, - trigger_app: TriggerApp, - ) -> impl Future> + Send; + fn run(self, trigger_app: TriggerApp) -> impl Future> + Send; /// Returns a list of host requirements supported by this trigger specifically. /// diff --git a/tests/integration.rs b/tests/integration.rs index 2c4b8ad04..9ada8bb9d 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -422,11 +422,7 @@ Caused by: ) }; ensure_success("/hello", 200, "I'm a teapot")?; - ensure_success( - "/hello/wildcards/should/be/handled", - 200, - "I'm a teapot", - )?; + ensure_success("/hello/wildcards/should/be/handled", 200, "I'm a teapot")?; ensure_success("/thisshouldfail", 404, "")?; ensure_success("/hello/test-placement", 200, "text for test")?; Ok(()) From 3e6708dbf6a0e6958584c23b39eacce2086a7706 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 13:47:39 -0400 Subject: [PATCH 176/195] Fix-a-lint Signed-off-by: Lann Martin --- crates/trigger/src/cli.rs | 2 +- examples/spin-timer/Cargo.lock | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index cd492a31d..8dce80e1c 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -312,7 +312,7 @@ impl TriggerAppBuilder { let use_gpu = true; let state_dir = match options.state_dir { // Make sure `--state-dir=""` unsets the state dir - Some(s) if s.trim().is_empty() => None, + Some("") => None, Some(s) => Some(PathBuf::from(s)), // Default to `.spin/` in the local app dir None => options.local_app_dir.map(|d| Path::new(d).join(".spin")), diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index c46d33839..79231a983 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -615,14 +615,6 @@ dependencies = [ "winx", ] -[[package]] -name = "cargo-target-dep" -version = "0.1.0" -source = "git+https://github.com/fermyon/cargo-target-dep?rev=482f269eceb7b1a7e8fc618bf8c082dd24979cf1#482f269eceb7b1a7e8fc618bf8c082dd24979cf1" -dependencies = [ - "glob", -] - [[package]] name = "cc" version = "1.0.99" @@ -4034,7 +4026,6 @@ name = "spin-factors" version = "2.8.0-pre0" dependencies = [ "anyhow", - "cargo-target-dep", "serde", "spin-app", "spin-factors-derive", From 18bb9442c04575d19f29b3b3a33ec29f9be81e3c Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 15:06:39 -0400 Subject: [PATCH 177/195] factors: Remove dependency on aws-lc-rs Signed-off-by: Lann Martin --- Cargo.lock | 53 +------- crates/factor-outbound-http/Cargo.toml | 4 +- crates/factor-outbound-networking/Cargo.toml | 2 +- examples/spin-timer/Cargo.lock | 124 +------------------ 4 files changed, 6 insertions(+), 177 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75c0efb31..ce4d4be2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,33 +523,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" -[[package]] -name = "aws-lc-rs" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" -dependencies = [ - "aws-lc-sys", - "mirai-annotations", - "paste", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" -dependencies = [ - "bindgen", - "cc", - "cmake", - "dunce", - "fs_extra", - "libc", - "paste", -] - [[package]] name = "axum" version = "0.6.20" @@ -737,15 +710,12 @@ dependencies = [ "itertools 0.12.1", "lazy_static 1.4.0", "lazycell", - "log", - "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", "syn 2.0.58", - "which", ] [[package]] @@ -4505,12 +4475,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "monostate" version = "0.1.11" @@ -6447,9 +6411,9 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebbbdb961df0ad3f2652da8f3fdc4b36122f568f968f45ad3316f26c025c677b" dependencies = [ - "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.102.2", "subtle", @@ -6510,7 +6474,6 @@ version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -10829,20 +10792,6 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.58", -] [[package]] name = "zip" diff --git a/crates/factor-outbound-http/Cargo.toml b/crates/factor-outbound-http/Cargo.toml index e62c6c52f..9d22b5908 100644 --- a/crates/factor-outbound-http/Cargo.toml +++ b/crates/factor-outbound-http/Cargo.toml @@ -10,14 +10,14 @@ http = "1.1.0" http-body-util = "0.1" hyper = "1.4.1" reqwest = { version = "0.11", features = ["gzip"] } -rustls = "0.23" +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } spin-factor-outbound-networking = { path = "../factor-outbound-networking" } spin-factors = { path = "../factors" } spin-telemetry = { path = "../telemetry" } spin-world = { path = "../world" } terminal = { path = "../terminal" } tokio = { version = "1", features = ["macros", "rt"] } -tokio-rustls = "0.26" +tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } tracing = { workspace = true } wasmtime = { workspace = true } wasmtime-wasi = { workspace = true } diff --git a/crates/factor-outbound-networking/Cargo.toml b/crates/factor-outbound-networking/Cargo.toml index e232d06ec..03fd55e4c 100644 --- a/crates/factor-outbound-networking/Cargo.toml +++ b/crates/factor-outbound-networking/Cargo.toml @@ -9,7 +9,7 @@ anyhow = "1" futures-util = "0.3" http = "1.1.0" ipnet = "2.9.0" -rustls = "0.23" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } rustls-pemfile = { version = "2.1.2", optional = true } rustls-pki-types = "1.7.0" serde = { version = "1", features = ["derive"] } diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index 79231a983..900be37f8 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -249,33 +249,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "aws-lc-rs" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" -dependencies = [ - "aws-lc-sys", - "mirai-annotations", - "paste", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" -dependencies = [ - "bindgen 0.69.4", - "cc", - "cmake", - "dunce", - "fs_extra", - "libc", - "paste", -] - [[package]] name = "axum" version = "0.6.20" @@ -433,29 +406,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.4.2", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.75", - "which", -] - [[package]] name = "bindgen" version = "0.70.1" @@ -1128,12 +1078,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.17" @@ -1310,12 +1254,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.30" @@ -1615,15 +1553,6 @@ dependencies = [ "digest", ] -[[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "http" version = "0.2.12" @@ -2021,12 +1950,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128" version = "0.2.5" @@ -2280,12 +2203,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "mysql_async" version = "0.33.0" @@ -2327,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" dependencies = [ "base64 0.21.7", - "bindgen 0.70.1", + "bindgen", "bitflags 2.4.2", "btoi", "byteorder", @@ -2878,16 +2795,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" -dependencies = [ - "proc-macro2", - "syn 2.0.75", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3389,9 +3296,9 @@ version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ - "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.102.6", "subtle", @@ -3452,7 +3359,6 @@ version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5614,18 +5520,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "whoami" version = "1.5.1" @@ -6000,20 +5894,6 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.75", -] [[package]] name = "zstd" From 7af0c0c08b3bbb60f7f41438d518181ef0da9534 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 15:46:52 -0400 Subject: [PATCH 178/195] factors: Fix startup with missing state_dir Signed-off-by: Lann Martin --- crates/runtime-config/src/lib.rs | 3 +-- src/bin/spin.rs | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 2448605a0..660f0f90d 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -173,8 +173,7 @@ impl<'a> TomlResolver<'a> { // Prefer explicitly provided state directory, then take from toml. self.state_dir .or_else(from_toml) - .map(PathBuf::from) - .map(std::fs::canonicalize) + .map(std::path::absolute) .transpose() } diff --git a/src/bin/spin.rs b/src/bin/spin.rs index 0834f1380..ce5c83671 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -73,7 +73,10 @@ async fn _main() -> anyhow::Result<()> { } } - SpinApp::from_arg_matches(&matches)?.run(cmd).await + SpinApp::from_arg_matches(&matches)? + .run(cmd) + .await + .inspect_err(|err| tracing::debug!(?err)) } fn print_error_chain(err: anyhow::Error) { From 859ff47c8181c6253dd8b09bb2b6244986d8fe6a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 15:53:31 -0400 Subject: [PATCH 179/195] Fix bad_build_test with ambient RUST_LOG set Signed-off-by: Lann Martin --- tests/integration.rs | 7 ++++++- tests/testing-framework/src/runtimes/spin_cli.rs | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration.rs b/tests/integration.rs index 9ada8bb9d..d262325d2 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -341,8 +341,13 @@ mod integration_tests { app_type: SpinAppType::None, }, ServicesConfig::none(), - |_| Ok(()), + |env| { + // Since this test asserts exact stderr output, disable logging + env.set_env_var("RUST_LOG", "off"); + Ok(()) + }, )?; + let expected = r#"Error: Couldn't find trigger executor for local app "spin.toml" Caused by: diff --git a/tests/testing-framework/src/runtimes/spin_cli.rs b/tests/testing-framework/src/runtimes/spin_cli.rs index 8bb46ee3b..44332433e 100644 --- a/tests/testing-framework/src/runtimes/spin_cli.rs +++ b/tests/testing-framework/src/runtimes/spin_cli.rs @@ -59,6 +59,7 @@ impl SpinCli { let port = get_random_port()?; let mut spin_cmd = Command::new(spin_config.binary_path); let child = spin_cmd + .envs(env.env_vars()) .arg("up") .current_dir(env.path()) .args(["--listen", &format!("127.0.0.1:{port}")]) @@ -117,6 +118,7 @@ impl SpinCli { env: &mut TestEnvironment, ) -> anyhow::Result { let mut child = Command::new(spin_config.binary_path) + .envs(env.env_vars()) .arg("up") .current_dir(env.path()) .args(spin_config.spin_up_args) @@ -149,6 +151,7 @@ impl SpinCli { env: &mut TestEnvironment, ) -> anyhow::Result { let mut child = Command::new(spin_config.binary_path) + .envs(env.env_vars()) .arg("up") .current_dir(env.path()) .args(spin_config.spin_up_args) From 013fed7b863bfa8ab3531ef16492b6a1ba6c0e72 Mon Sep 17 00:00:00 2001 From: Caleb Schoepp Date: Mon, 26 Aug 2024 12:26:29 -0600 Subject: [PATCH 180/195] Only open the default store in set_initial_key_values when we have to Signed-off-by: Caleb Schoepp --- crates/runtime-config/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 660f0f90d..bd8d7745f 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -118,6 +118,12 @@ where &self, initial_key_values: impl IntoIterator, ) -> anyhow::Result<()> { + // We don't want to unnecessarily interact with the default store + let mut iter = initial_key_values.into_iter().peekable(); + if iter.peek().is_none() { + return Ok(()); + } + let store = self .key_value_resolver .default(DEFAULT_KEY_VALUE_STORE_LABEL) @@ -125,7 +131,7 @@ where .get(DEFAULT_KEY_VALUE_STORE_LABEL) .await .expect("trigger was misconfigured and lacks a default store"); - for (key, value) in initial_key_values { + for (key, value) in iter { store .set(key, value.as_bytes()) .await From 5cb45a2ccdd0226eb55d831a633d87b58009f634 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 16:13:38 -0400 Subject: [PATCH 181/195] Bump rust-version to 1.79 Signed-off-by: Lann Martin --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 089e4f1af..591c7c01f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ edition = "2021" license = "Apache-2.0 WITH LLVM-exception" homepage = "https://developer.fermyon.com/spin" repository = "https://github.com/fermyon/spin" -rust-version = "1.76" +rust-version = "1.79" [dependencies] anyhow = { workspace = true } From c58a6b876da8ec78e80aaa78009254540e28d4b0 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 16:15:22 -0400 Subject: [PATCH 182/195] ci: Bump rust-version to 1.79 Signed-off-by: Lann Martin --- .github/actions/spin-ci-dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/spin-ci-dependencies/action.yml b/.github/actions/spin-ci-dependencies/action.yml index 4cdb3db4b..78d3a6541 100644 --- a/.github/actions/spin-ci-dependencies/action.yml +++ b/.github/actions/spin-ci-dependencies/action.yml @@ -8,7 +8,7 @@ inputs: type: bool rust-version: description: 'Rust version to setup' - default: '1.76' + default: '1.79' required: false type: string From 4eab3ba1a7386d7ee02f8634c96c1ad18d298870 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 26 Aug 2024 16:48:55 -0400 Subject: [PATCH 183/195] factors: Fix toml key tracking Signed-off-by: Lann Martin --- .../src/runtime_config/spin.rs | 3 ++- crates/factor-llm/src/spin.rs | 3 ++- .../src/runtime_config/spin.rs | 4 ++-- .../factor-sqlite/src/runtime_config/spin.rs | 4 ++-- crates/factor-variables/src/spin_cli/mod.rs | 4 ++-- crates/runtime-config/src/lib.rs | 18 +++++------------- 6 files changed, 15 insertions(+), 21 deletions(-) diff --git a/crates/factor-key-value/src/runtime_config/spin.rs b/crates/factor-key-value/src/runtime_config/spin.rs index e604d3a5f..64c4e1d57 100644 --- a/crates/factor-key-value/src/runtime_config/spin.rs +++ b/crates/factor-key-value/src/runtime_config/spin.rs @@ -4,6 +4,7 @@ use crate::{DefaultLabelResolver, RuntimeConfig}; use anyhow::Context as _; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; +use spin_factors::runtime_config::toml::GetTomlValue; use spin_key_value::StoreManager; use std::{collections::HashMap, sync::Arc}; @@ -100,7 +101,7 @@ impl RuntimeConfigResolver { /// Resolves a toml table into a runtime config. pub fn resolve_from_toml( &self, - table: Option<&toml::Table>, + table: Option<&impl GetTomlValue>, ) -> anyhow::Result> { let Some(table) = table.and_then(|t| t.get("key_value_store")) else { return Ok(None); diff --git a/crates/factor-llm/src/spin.rs b/crates/factor-llm/src/spin.rs index b9b91c699..5f04478aa 100644 --- a/crates/factor-llm/src/spin.rs +++ b/crates/factor-llm/src/spin.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use std::sync::Arc; +use spin_factors::runtime_config::toml::GetTomlValue; use spin_llm_remote_http::RemoteHttpLlmEngine; use spin_world::async_trait; use spin_world::v1::llm::{self as v1}; @@ -80,7 +81,7 @@ impl LlmEngine for RemoteHttpLlmEngine { } pub fn runtime_config_from_toml( - table: &toml::Table, + table: &impl GetTomlValue, state_dir: Option, use_gpu: bool, ) -> anyhow::Result> { diff --git a/crates/factor-outbound-networking/src/runtime_config/spin.rs b/crates/factor-outbound-networking/src/runtime_config/spin.rs index c8e2a3fc5..863ebed7f 100644 --- a/crates/factor-outbound-networking/src/runtime_config/spin.rs +++ b/crates/factor-outbound-networking/src/runtime_config/spin.rs @@ -37,9 +37,9 @@ impl SpinTlsRuntimeConfig { /// client_cert_file = "path/to/client.crt" /// client_private_key_file = "path/to/client.key" /// ``` - pub fn config_from_table( + pub fn config_from_table( &self, - table: &T, + table: &impl GetTomlValue, ) -> anyhow::Result> { let Some(tls_configs) = self.tls_configs_from_table(table)? else { return Ok(None); diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index b12a87616..b51fd4d50 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -49,9 +49,9 @@ impl RuntimeConfigResolver { /// type = "$database-type" /// ... extra type specific configuration ... /// ``` - pub fn resolve_from_toml( + pub fn resolve_from_toml( &self, - table: &T, + table: &impl GetTomlValue, ) -> anyhow::Result> { let Some(table) = table.get("sqlite_database") else { return Ok(None); diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index f86c52d4a..f5d809778 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -12,12 +12,12 @@ pub use vault::*; use serde::Deserialize; use spin_expressions::Provider; -use spin_factors::anyhow; +use spin_factors::{anyhow, runtime_config::toml::GetTomlValue}; use crate::runtime_config::RuntimeConfig; /// Resolves a runtime configuration for the variables factor from a TOML table. -pub fn runtime_config_from_toml(table: &toml::Table) -> anyhow::Result { +pub fn runtime_config_from_toml(table: &impl GetTomlValue) -> anyhow::Result { // Always include the environment variable provider. let mut providers = vec![Box::::default() as _]; let Some(array) = table.get("variable_provider") else { diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index bd8d7745f..85351b9cc 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -189,12 +189,6 @@ impl<'a> TomlResolver<'a> { } } -impl AsRef for TomlResolver<'_> { - fn as_ref(&self) -> &toml::Table { - self.table.as_ref() - } -} - /// The TOML based runtime configuration source Spin CLI. pub struct TomlRuntimeConfigSource<'a, 'b> { toml: TomlResolver<'b>, @@ -226,7 +220,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, ' fn get_runtime_config( &mut self, ) -> anyhow::Result> { - self.key_value.resolve_from_toml(Some(self.toml.as_ref())) + self.key_value.resolve_from_toml(Some(&self.toml.table)) } } @@ -238,7 +232,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSo let Some(tls) = self.tls else { return Ok(None); }; - tls.config_from_table(self.toml.as_ref()) + tls.config_from_table(&self.toml.table) } } @@ -246,9 +240,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, fn get_runtime_config( &mut self, ) -> anyhow::Result::RuntimeConfig>> { - Ok(Some(variables::runtime_config_from_toml( - self.toml.as_ref(), - )?)) + Ok(Some(variables::runtime_config_from_toml(&self.toml.table)?)) } } @@ -266,7 +258,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource< impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { - llm::runtime_config_from_toml(self.toml.as_ref(), self.toml.state_dir()?, self.use_gpu) + llm::runtime_config_from_toml(&self.toml.table, self.toml.state_dir()?, self.use_gpu) } } @@ -296,7 +288,7 @@ impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<' impl FactorRuntimeConfigSource for TomlRuntimeConfigSource<'_, '_> { fn get_runtime_config(&mut self) -> anyhow::Result> { - self.sqlite.resolve_from_toml(self.toml.as_ref()) + self.sqlite.resolve_from_toml(&self.toml.table) } } From 340653b34cf28c6b7355b834dbf7dfd1333cf174 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 11:37:47 +0200 Subject: [PATCH 184/195] Make sure default SQLite database parent dir is created. Signed-off-by: Ryan Levick --- .../factor-sqlite/src/runtime_config/spin.rs | 22 ++++++------------ crates/sqlite-inproc/src/lib.rs | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/crates/factor-sqlite/src/runtime_config/spin.rs b/crates/factor-sqlite/src/runtime_config/spin.rs index b51fd4d50..ef84ca160 100644 --- a/crates/factor-sqlite/src/runtime_config/spin.rs +++ b/crates/factor-sqlite/src/runtime_config/spin.rs @@ -10,6 +10,7 @@ use spin_factors::{ anyhow::{self, Context as _}, runtime_config::toml::GetTomlValue, }; +use spin_sqlite_inproc::InProcDatabaseLocation; use spin_world::v2::sqlite as v2; use tokio::sync::OnceCell; @@ -108,10 +109,7 @@ impl DefaultLabelResolver for RuntimeConfigResolver { .as_deref() .map(|p| p.join(DEFAULT_SQLITE_DB_FILENAME)); let factory = move || { - let location = match &path { - Some(path) => spin_sqlite_inproc::InProcDatabaseLocation::Path(path.clone()), - None => spin_sqlite_inproc::InProcDatabaseLocation::InMemory, - }; + let location = InProcDatabaseLocation::from_path(path.clone())?; let connection = spin_sqlite_inproc::InProcConnection::new(location)?; Ok(Box::new(connection) as _) }; @@ -202,17 +200,11 @@ impl LocalDatabase { /// /// `base_dir` is the base directory path from which `path` is resolved if it is a relative path. fn connection_creator(self, base_dir: &Path) -> anyhow::Result { - let location = match self.path { - Some(path) => { - let path = resolve_relative_path(&path, base_dir); - // Create the store's parent directory if necessary - // unwrapping the parent is fine, because `resolve_relative_path`` will always return a path with a parent - std::fs::create_dir_all(path.parent().unwrap()) - .context("Failed to create sqlite database directory")?; - spin_sqlite_inproc::InProcDatabaseLocation::Path(path) - } - None => spin_sqlite_inproc::InProcDatabaseLocation::InMemory, - }; + let path = self + .path + .as_ref() + .map(|p| resolve_relative_path(p, base_dir)); + let location = InProcDatabaseLocation::from_path(path)?; let factory = move || { let connection = spin_sqlite_inproc::InProcConnection::new(location.clone())?; Ok(Box::new(connection) as _) diff --git a/crates/sqlite-inproc/src/lib.rs b/crates/sqlite-inproc/src/lib.rs index 77877325d..63d24d8fb 100644 --- a/crates/sqlite-inproc/src/lib.rs +++ b/crates/sqlite-inproc/src/lib.rs @@ -16,6 +16,29 @@ pub enum InProcDatabaseLocation { Path(PathBuf), } +impl InProcDatabaseLocation { + /// Convert an optional path to a database location. + /// + /// Ensures that the parent directory of the database exists. + pub fn from_path(path: Option) -> anyhow::Result { + match path { + Some(path) => { + // Create the store's parent directory if necessary + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create sqlite database directory '{}'", + parent.display() + ) + })?; + } + Ok(Self::Path(path)) + } + None => Ok(Self::InMemory), + } + } +} + /// A connection to a sqlite database pub struct InProcConnection { location: InProcDatabaseLocation, From d1cd7fe8604a8402d5fede3f4cace20f97098e3b Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 11:38:17 +0200 Subject: [PATCH 185/195] Run canary conformance tests. Signed-off-by: Ryan Levick --- tests/conformance-tests/src/main.rs | 2 +- tests/runtime.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conformance-tests/src/main.rs b/tests/conformance-tests/src/main.rs index b6e5804c3..c297b2d07 100644 --- a/tests/conformance-tests/src/main.rs +++ b/tests/conformance-tests/src/main.rs @@ -3,7 +3,7 @@ fn main() { .nth(1) .expect("expected first argument to be path to spin binary") .into(); - conformance_tests::run_tests("v0.1.0", move |test| { + conformance_tests::run_tests("canary", move |test| { conformance::run_test(test, &spin_binary) }) .unwrap(); diff --git a/tests/runtime.rs b/tests/runtime.rs index b18bc3d51..ba3b6b41c 100644 --- a/tests/runtime.rs +++ b/tests/runtime.rs @@ -28,7 +28,7 @@ mod runtime_tests { #[test] fn conformance_tests() { - conformance_tests::run_tests("v0.1.0", move |test| { + conformance_tests::run_tests("canary", move |test| { conformance::run_test(test, &spin_binary()) }) .unwrap(); From a79898c6e1e751309775721dc5be2c5a637b7f43 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 11:38:35 +0200 Subject: [PATCH 186/195] Update to using Rust 1.79 in all CI workflows. Signed-off-by: Ryan Levick --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47645d25c..1ca7deeca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ concurrency: env: CARGO_TERM_COLOR: always - RUST_VERSION: 1.76 + RUST_VERSION: 1.79 jobs: dependency-review: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 104bb2ef7..cae50a614 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ on: concurrency: ${{ github.workflow }}-${{ github.ref }} env: - RUST_VERSION: 1.76 + RUST_VERSION: 1.79 jobs: build-and-sign: From 933b3267582baa7b8df7cbe4d40f0b4023ba850e Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 12:08:59 +0200 Subject: [PATCH 187/195] Turn llm feature on by default. Signed-off-by: Ryan Levick --- Cargo.toml | 2 +- crates/factor-llm/src/spin.rs | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 591c7c01f..c32cfc09c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ vergen = { version = "^8.2.1", default-features = false, features = [ wit-component = "0.19.0" [features] -# TODO(factors): default = ["llm"] +default = ["llm"] all-tests = ["extern-dependencies-tests"] extern-dependencies-tests = [] llm = ["spin-trigger-http/llm"] diff --git a/crates/factor-llm/src/spin.rs b/crates/factor-llm/src/spin.rs index 5f04478aa..87e80bbf1 100644 --- a/crates/factor-llm/src/spin.rs +++ b/crates/factor-llm/src/spin.rs @@ -91,7 +91,7 @@ pub fn runtime_config_from_toml( let config: LlmCompute = value.clone().try_into()?; Ok(Some(RuntimeConfig { - engine: config.into_engine(state_dir, use_gpu), + engine: config.into_engine(state_dir, use_gpu)?, })) } @@ -103,20 +103,25 @@ pub enum LlmCompute { } impl LlmCompute { - fn into_engine(self, state_dir: Option, use_gpu: bool) -> Arc> { - match self { + fn into_engine( + self, + state_dir: Option, + use_gpu: bool, + ) -> anyhow::Result>> { + let engine = match self { #[cfg(not(feature = "llm"))] LlmCompute::Spin => { let _ = (state_dir, use_gpu); Arc::new(Mutex::new(noop::NoopLlmEngine)) } #[cfg(feature = "llm")] - LlmCompute::Spin => default_engine_creator(state_dir, use_gpu).create(), + LlmCompute::Spin => default_engine_creator(state_dir, use_gpu)?.create(), LlmCompute::RemoteHttp(config) => Arc::new(Mutex::new(RemoteHttpLlmEngine::new( config.url, config.auth_token, ))), - } + }; + Ok(engine) } } From d5ea75a225a159bc44cf6ac007b68dd0d79abf9e Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 12:50:53 +0200 Subject: [PATCH 188/195] Fix logic for when and where to log by default. Signed-off-by: Ryan Levick --- crates/runtime-config/src/lib.rs | 67 ++++++++++++++++++++++++++++---- crates/trigger/src/cli.rs | 19 +++++---- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 85351b9cc..679382934 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -38,6 +38,10 @@ pub struct ResolvedRuntimeConfig { /// /// `None` is used for an "unset" state directory which each factor will treat differently. pub state_dir: Option, + /// The fully resolved log directory. + /// + /// `None` is used for an "unset" log directory. + pub log_dir: Option, } impl ResolvedRuntimeConfig @@ -49,13 +53,23 @@ where pub fn from_optional_file( runtime_config_path: Option<&Path>, provided_state_dir: Option<&Path>, + provided_log_dir: LogDir, use_gpu: bool, ) -> anyhow::Result { match runtime_config_path { - Some(runtime_config_path) => { - Self::from_file(runtime_config_path, provided_state_dir, use_gpu) - } - None => Self::new(Default::default(), None, provided_state_dir, use_gpu), + Some(runtime_config_path) => Self::from_file( + runtime_config_path, + provided_state_dir, + provided_log_dir, + use_gpu, + ), + None => Self::new( + Default::default(), + None, + provided_state_dir, + provided_log_dir, + use_gpu, + ), } } @@ -65,6 +79,7 @@ where pub fn from_file( runtime_config_path: &Path, provided_state_dir: Option<&Path>, + provided_log_dir: LogDir, use_gpu: bool, ) -> anyhow::Result { let file = std::fs::read_to_string(runtime_config_path).with_context(|| { @@ -80,7 +95,13 @@ where ) })?; - Self::new(toml, Some(runtime_config_path), provided_state_dir, use_gpu) + Self::new( + toml, + Some(runtime_config_path), + provided_state_dir, + provided_log_dir, + use_gpu, + ) } /// Creates a new resolved runtime configuration from a TOML table. @@ -88,9 +109,10 @@ where toml: toml::Table, runtime_config_path: Option<&Path>, provided_state_dir: Option<&Path>, + provided_log_dir: LogDir, use_gpu: bool, ) -> anyhow::Result { - let toml_resolver = TomlResolver::new(&toml, provided_state_dir); + let toml_resolver = TomlResolver::new(&toml, provided_state_dir, provided_log_dir); let tls_resolver = runtime_config_path.map(SpinTlsRuntimeConfig::new); let key_value_config_resolver = key_value_config_resolver(toml_resolver.state_dir()?); let sqlite_config_resolver = sqlite_config_resolver(toml_resolver.state_dir()?) @@ -110,6 +132,7 @@ where key_value_resolver: key_value_config_resolver, sqlite_resolver: sqlite_config_resolver, state_dir: toml_resolver.state_dir()?, + log_dir: toml_resolver.log_dir()?, }) } @@ -144,6 +167,11 @@ where pub fn state_dir(&self) -> Option { self.state_dir.clone() } + + /// The fully resolved state directory. + pub fn log_dir(&self) -> Option { + self.log_dir.clone() + } } #[derive(Clone, Debug)] @@ -152,16 +180,19 @@ pub struct TomlResolver<'a> { table: TomlKeyTracker<'a>, /// Explicitly provided state directory. state_dir: Option<&'a Path>, + /// Explicitly provided log directory. + log_dir: LogDir, } impl<'a> TomlResolver<'a> { /// Create a new TOML resolver. /// /// The `state_dir` is the explicitly provided state directory, if any. - pub fn new(table: &'a toml::Table, state_dir: Option<&'a Path>) -> Self { + pub fn new(table: &'a toml::Table, state_dir: Option<&'a Path>, log_dir: LogDir) -> Self { Self { table: TomlKeyTracker::new(table), state_dir, + log_dir, } } @@ -183,6 +214,17 @@ impl<'a> TomlResolver<'a> { .transpose() } + /// Get the configured log directory. + /// + /// Errors if the path cannot be converted to an absolute path. + pub fn log_dir(&self) -> std::io::Result> { + match &self.log_dir { + LogDir::Provided(p) => Ok(Some(std::path::absolute(p)?)), + LogDir::Default => Ok(self.state_dir()?.map(|p| p.join("logs"))), + LogDir::None => Ok(None), + } + } + /// Validate that all keys in the TOML file have been used. pub fn validate_all_keys_used(&self) -> spin_factors::Result<()> { self.table.validate_all_keys_used() @@ -354,3 +396,14 @@ fn sqlite_config_resolver( local_database_dir, )) } + +/// The log directory for the trigger. +#[derive(Clone, Debug)] +pub enum LogDir { + /// Use the explicitly provided log directory. + Provided(PathBuf), + /// Use the default log directory. + Default, + /// Do not log. + None, +} diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index 8dce80e1c..5351d2676 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -10,7 +10,7 @@ use spin_common::ui::quoted_path; use spin_common::url::parse_file_url; use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; -use spin_runtime_config::ResolvedRuntimeConfig; +use spin_runtime_config::{LogDir, ResolvedRuntimeConfig}; use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; @@ -317,10 +317,17 @@ impl TriggerAppBuilder { // Default to `.spin/` in the local app dir None => options.local_app_dir.map(|d| Path::new(d).join(".spin")), }; + let log_dir = match &options.log_dir { + // Make sure `--log-dir=""` unsets the log dir + Some(p) if p.as_os_str().is_empty() => LogDir::None, + Some(p) => LogDir::Provided(p.clone()), + None => LogDir::Default, + }; let runtime_config = ResolvedRuntimeConfig::::from_optional_file( options.runtime_config_file, state_dir.as_deref(), + log_dir, use_gpu, )?; @@ -328,6 +335,7 @@ impl TriggerAppBuilder { .set_initial_key_values(&options.initial_key_values) .await?; + let log_dir = runtime_config.log_dir(); let factors = TriggerFactors::new( runtime_config.state_dir(), self.working_dir.clone(), @@ -338,12 +346,7 @@ impl TriggerAppBuilder { ) .context("failed to create factors")?; - // TODO: move these into Factor methods/constructors - // let init_data = crate::HostComponentInitData::new( - // &*self.key_values, - // &*self.sqlite_statements, - // LLmOptions { use_gpu: true }, - // ); + // TODO(factors): handle: self.sqlite_statements // TODO: port the rest of the component loader logic struct SimpleComponentLoader; @@ -377,7 +380,7 @@ impl TriggerAppBuilder { executor.add_hooks(StdioLoggingExecutorHooks::new( options.follow_components, - options.log_dir, + log_dir, )); // TODO: // builder.hooks(SummariseRuntimeConfigHook::new(&self.runtime_config_file)); From 783e7b2f5adb175aa8fa8c742f0e8d62c59a872c Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 27 Aug 2024 08:30:11 -0400 Subject: [PATCH 189/195] Fix LlmCompute::into_engine type inference Signed-off-by: Lann Martin --- crates/factor-llm/src/spin.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/factor-llm/src/spin.rs b/crates/factor-llm/src/spin.rs index 87e80bbf1..ab3a167fa 100644 --- a/crates/factor-llm/src/spin.rs +++ b/crates/factor-llm/src/spin.rs @@ -108,7 +108,7 @@ impl LlmCompute { state_dir: Option, use_gpu: bool, ) -> anyhow::Result>> { - let engine = match self { + let engine: Arc> = match self { #[cfg(not(feature = "llm"))] LlmCompute::Spin => { let _ = (state_dir, use_gpu); From c4d239a0f977b94f7df0da0f3572e7e2c7d70094 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 27 Aug 2024 09:30:39 -0400 Subject: [PATCH 190/195] Relax buggy wasi-sdk heuristic Unfortunately there are some toolchains that don't emit enough information for this heuristic to detect that they are positively safe, so we'll need to default to allow. Signed-off-by: Lann Martin --- crates/componentize/src/bugs.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/crates/componentize/src/bugs.rs b/crates/componentize/src/bugs.rs index 099b4adb6..d6299606a 100644 --- a/crates/componentize/src/bugs.rs +++ b/crates/componentize/src/bugs.rs @@ -23,9 +23,7 @@ impl WasiLibc377Bug { if let Some((major, minor, patch)) = parse_clang_version(clang_version) { let earliest_safe = parse_clang_version(EARLIEST_PROBABLY_SAFE_CLANG_VERSION).unwrap(); - if (major, minor, patch) >= earliest_safe { - return Ok(()); - } else { + if (major, minor, patch) < earliest_safe { return Err(Self { clang_version: Some(clang_version.clone()), }); @@ -37,11 +35,7 @@ impl WasiLibc377Bug { ); } } - // If we can't assert that the module uses wit-bindgen OR was compiled - // with a new-enough wasi-sdk, conservatively assume it may be buggy. - Err(Self { - clang_version: None, - }) + Ok(()) } } @@ -49,8 +43,8 @@ impl std::fmt::Display for WasiLibc377Bug { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, - "This Wasm module may have been compiled with wasi-sdk version <19 which \ - contains a critical memory safety bug. For more information, see: \ + "This Wasm module appears to have been compiled with wasi-sdk version <19 \ + which contains a critical memory safety bug. For more information, see: \ https://github.com/fermyon/spin/issues/2552" ) } From fe17796eb71bc9d88114b681d6621123427526bc Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 15:30:12 +0200 Subject: [PATCH 191/195] Fix subtle runtime config issues. Signed-off-by: Ryan Levick --- crates/factor-variables/src/spin_cli/mod.rs | 5 +- crates/runtime-config/src/lib.rs | 116 ++++++++++++++------ crates/trigger/src/cli.rs | 30 +++-- 3 files changed, 104 insertions(+), 47 deletions(-) diff --git a/crates/factor-variables/src/spin_cli/mod.rs b/crates/factor-variables/src/spin_cli/mod.rs index f5d809778..6a31a5b09 100644 --- a/crates/factor-variables/src/spin_cli/mod.rs +++ b/crates/factor-variables/src/spin_cli/mod.rs @@ -20,7 +20,10 @@ use crate::runtime_config::RuntimeConfig; pub fn runtime_config_from_toml(table: &impl GetTomlValue) -> anyhow::Result { // Always include the environment variable provider. let mut providers = vec![Box::::default() as _]; - let Some(array) = table.get("variable_provider") else { + let value = table + .get("variables_provider") + .or_else(|| table.get("config_provider")); + let Some(array) = value else { return Ok(RuntimeConfig { providers }); }; diff --git a/crates/runtime-config/src/lib.rs b/crates/runtime-config/src/lib.rs index 679382934..362812c09 100644 --- a/crates/runtime-config/src/lib.rs +++ b/crates/runtime-config/src/lib.rs @@ -52,13 +52,15 @@ where /// Creates a new resolved runtime configuration from an optional runtime config source TOML file. pub fn from_optional_file( runtime_config_path: Option<&Path>, - provided_state_dir: Option<&Path>, - provided_log_dir: LogDir, + local_app_dir: Option, + provided_state_dir: UserProvidedPath, + provided_log_dir: UserProvidedPath, use_gpu: bool, ) -> anyhow::Result { match runtime_config_path { Some(runtime_config_path) => Self::from_file( runtime_config_path, + local_app_dir, provided_state_dir, provided_log_dir, use_gpu, @@ -66,6 +68,7 @@ where None => Self::new( Default::default(), None, + local_app_dir, provided_state_dir, provided_log_dir, use_gpu, @@ -78,8 +81,9 @@ where /// `provided_state_dir` is the explicitly provided state directory, if any. pub fn from_file( runtime_config_path: &Path, - provided_state_dir: Option<&Path>, - provided_log_dir: LogDir, + local_app_dir: Option, + provided_state_dir: UserProvidedPath, + provided_log_dir: UserProvidedPath, use_gpu: bool, ) -> anyhow::Result { let file = std::fs::read_to_string(runtime_config_path).with_context(|| { @@ -98,6 +102,7 @@ where Self::new( toml, Some(runtime_config_path), + local_app_dir, provided_state_dir, provided_log_dir, use_gpu, @@ -108,11 +113,13 @@ where pub fn new( toml: toml::Table, runtime_config_path: Option<&Path>, - provided_state_dir: Option<&Path>, - provided_log_dir: LogDir, + local_app_dir: Option, + provided_state_dir: UserProvidedPath, + provided_log_dir: UserProvidedPath, use_gpu: bool, ) -> anyhow::Result { - let toml_resolver = TomlResolver::new(&toml, provided_state_dir, provided_log_dir); + let toml_resolver = + TomlResolver::new(&toml, local_app_dir, provided_state_dir, provided_log_dir); let tls_resolver = runtime_config_path.map(SpinTlsRuntimeConfig::new); let key_value_config_resolver = key_value_config_resolver(toml_resolver.state_dir()?); let sqlite_config_resolver = sqlite_config_resolver(toml_resolver.state_dir()?) @@ -178,19 +185,25 @@ where /// Resolves runtime configuration from a TOML file. pub struct TomlResolver<'a> { table: TomlKeyTracker<'a>, + /// The local app directory. + local_app_dir: Option, /// Explicitly provided state directory. - state_dir: Option<&'a Path>, + state_dir: UserProvidedPath, /// Explicitly provided log directory. - log_dir: LogDir, + log_dir: UserProvidedPath, } impl<'a> TomlResolver<'a> { /// Create a new TOML resolver. - /// - /// The `state_dir` is the explicitly provided state directory, if any. - pub fn new(table: &'a toml::Table, state_dir: Option<&'a Path>, log_dir: LogDir) -> Self { + pub fn new( + table: &'a toml::Table, + local_app_dir: Option, + state_dir: UserProvidedPath, + log_dir: UserProvidedPath, + ) -> Self { Self { table: TomlKeyTracker::new(table), + local_app_dir, state_dir, log_dir, } @@ -200,28 +213,63 @@ impl<'a> TomlResolver<'a> { /// /// Errors if the path cannot be converted to an absolute path. pub fn state_dir(&self) -> std::io::Result> { - let from_toml = || { - self.table - .get("state_dir") - .and_then(|v| v.as_str()) - .filter(|v| !v.is_empty()) - .map(Path::new) - }; - // Prefer explicitly provided state directory, then take from toml. - self.state_dir - .or_else(from_toml) - .map(std::path::absolute) - .transpose() + let mut state_dir = self.state_dir.clone(); + // If the state_dir is not explicitly provided, check the toml. + if matches!(state_dir, UserProvidedPath::Default) { + let from_toml = + self.table + .get("state_dir") + .and_then(|v| v.as_str()) + .map(|toml_value| { + if toml_value.is_empty() { + // If the toml value is empty, treat it as unset. + UserProvidedPath::Unset + } else { + // Otherwise, treat the toml value as a provided path. + UserProvidedPath::Provided(PathBuf::from(toml_value)) + } + }); + // If toml value is not provided, use the original value after all. + state_dir = from_toml.unwrap_or(state_dir); + } + + match (state_dir, &self.local_app_dir) { + (UserProvidedPath::Provided(p), _) => Ok(Some(std::path::absolute(p)?)), + (UserProvidedPath::Default, Some(local_app_dir)) => { + Ok(Some(local_app_dir.join(".spin"))) + } + (UserProvidedPath::Default | UserProvidedPath::Unset, _) => Ok(None), + } } /// Get the configured log directory. /// /// Errors if the path cannot be converted to an absolute path. pub fn log_dir(&self) -> std::io::Result> { - match &self.log_dir { - LogDir::Provided(p) => Ok(Some(std::path::absolute(p)?)), - LogDir::Default => Ok(self.state_dir()?.map(|p| p.join("logs"))), - LogDir::None => Ok(None), + let mut log_dir = self.log_dir.clone(); + // If the log_dir is not explicitly provided, check the toml. + if matches!(log_dir, UserProvidedPath::Default) { + let from_toml = self + .table + .get("log_dir") + .and_then(|v| v.as_str()) + .map(|toml_value| { + if toml_value.is_empty() { + // If the toml value is empty, treat it as unset. + UserProvidedPath::Unset + } else { + // Otherwise, treat the toml value as a provided path. + UserProvidedPath::Provided(PathBuf::from(toml_value)) + } + }); + // If toml value is not provided, use the original value after all. + log_dir = from_toml.unwrap_or(log_dir); + } + + match log_dir { + UserProvidedPath::Provided(p) => Ok(Some(std::path::absolute(p)?)), + UserProvidedPath::Default => Ok(self.state_dir()?.map(|p| p.join("logs"))), + UserProvidedPath::Unset => Ok(None), } } @@ -397,13 +445,13 @@ fn sqlite_config_resolver( )) } -/// The log directory for the trigger. +/// A user provided option which be either be provided, default, or explicitly none. #[derive(Clone, Debug)] -pub enum LogDir { - /// Use the explicitly provided log directory. +pub enum UserProvidedPath { + /// Use the explicitly provided directory. Provided(PathBuf), - /// Use the default log directory. + /// Use the default. Default, - /// Do not log. - None, + /// Explicitly unset. + Unset, } diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index 5351d2676..5465a1901 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -10,7 +10,7 @@ use spin_common::ui::quoted_path; use spin_common::url::parse_file_url; use spin_common::{arg_parser::parse_kv, sloth}; use spin_factors_executor::{ComponentLoader, FactorsExecutor}; -use spin_runtime_config::{LogDir, ResolvedRuntimeConfig}; +use spin_runtime_config::{ResolvedRuntimeConfig, UserProvidedPath}; use crate::factors::{TriggerFactors, TriggerFactorsRuntimeConfig}; use crate::stdio::{FollowComponents, StdioLoggingExecutorHooks}; @@ -308,25 +308,31 @@ impl TriggerAppBuilder { }; self.trigger.add_to_linker(core_engine_builder.linker())?; - // Hardcode `use_gpu` to true for now - let use_gpu = true; + let runtime_config_path = options.runtime_config_file; + let local_app_dir = options + .local_app_dir + .map(std::path::absolute) + .transpose() + .context("failed to resolve local app directory path to an absolute path")?; let state_dir = match options.state_dir { // Make sure `--state-dir=""` unsets the state dir - Some("") => None, - Some(s) => Some(PathBuf::from(s)), - // Default to `.spin/` in the local app dir - None => options.local_app_dir.map(|d| Path::new(d).join(".spin")), + Some("") => UserProvidedPath::Unset, + Some(s) => UserProvidedPath::Provided(PathBuf::from(s)), + None => UserProvidedPath::Default, }; let log_dir = match &options.log_dir { // Make sure `--log-dir=""` unsets the log dir - Some(p) if p.as_os_str().is_empty() => LogDir::None, - Some(p) => LogDir::Provided(p.clone()), - None => LogDir::Default, + Some(p) if p.as_os_str().is_empty() => UserProvidedPath::Unset, + Some(p) => UserProvidedPath::Provided(p.clone()), + None => UserProvidedPath::Default, }; + // Hardcode `use_gpu` to true for now + let use_gpu = true; let runtime_config = ResolvedRuntimeConfig::::from_optional_file( - options.runtime_config_file, - state_dir.as_deref(), + runtime_config_path, + local_app_dir, + state_dir, log_dir, use_gpu, )?; From f1ce58796de3148b78d0153fc48494f530a43657 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 27 Aug 2024 10:18:49 -0400 Subject: [PATCH 192/195] Fix WasiLibc377Bug tests Signed-off-by: Lann Martin --- crates/componentize/src/bugs.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/crates/componentize/src/bugs.rs b/crates/componentize/src/bugs.rs index d6299606a..0b9dbbfae 100644 --- a/crates/componentize/src/bugs.rs +++ b/crates/componentize/src/bugs.rs @@ -69,15 +69,11 @@ mod tests { #[test] fn wasi_libc_377_detect() { for (wasm, safe) in [ - (r#"(module)"#, false), + (r#"(module)"#, true), ( r#"(module (func (export "cabi_realloc") (unreachable)))"#, true, ), - ( - r#"(module (func (export "some_other_function") (unreachable)))"#, - false, - ), ( r#"(module (@producers (processed-by "clang" "16.0.0 extra-stuff")))"#, true, @@ -94,10 +90,6 @@ mod tests { r#"(module (@producers (processed-by "clang" "14.0.0 extra-stuff")))"#, false, ), - ( - r#"(module (@producers (processed-by "clang" "a.b.c")))"#, - false, - ), ] { eprintln!("WAT: {wasm}"); let module = wat::parse_str(wasm).unwrap(); From c7e310d18e89741b7ca5871ae738613cf8139d06 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 16:23:21 +0200 Subject: [PATCH 193/195] Fix variables test Signed-off-by: Ryan Levick --- crates/factor-variables/tests/factor_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/factor-variables/tests/factor_test.rs b/crates/factor-variables/tests/factor_test.rs index 49358ee62..33df9f1b2 100644 --- a/crates/factor-variables/tests/factor_test.rs +++ b/crates/factor-variables/tests/factor_test.rs @@ -25,7 +25,7 @@ async fn static_provider_works() -> anyhow::Result<()> { variables = { baz = "<{{ foo }}>" } }) .runtime_config(TomlConfig::new(toml! { - [[variable_provider]] + [[variables_provider]] type = "static" values = { foo = "bar" } }))?; From 822a45454b970b1a81e324650a6ae9d4fa49f691 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 16:51:54 +0200 Subject: [PATCH 194/195] Remove use of absolute for local_app_dir Signed-off-by: Ryan Levick --- crates/trigger/src/cli.rs | 6 +----- crates/trigger/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index 5465a1901..74c5be271 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -309,11 +309,7 @@ impl TriggerAppBuilder { self.trigger.add_to_linker(core_engine_builder.linker())?; let runtime_config_path = options.runtime_config_file; - let local_app_dir = options - .local_app_dir - .map(std::path::absolute) - .transpose() - .context("failed to resolve local app directory path to an absolute path")?; + let local_app_dir = options.local_app_dir.map(PathBuf::from); let state_dir = match options.state_dir { // Make sure `--state-dir=""` unsets the state dir Some("") => UserProvidedPath::Unset, diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 0cc119f16..b5f1fd337 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -11,7 +11,7 @@ use spin_factors_executor::{FactorsExecutorApp, FactorsInstanceBuilder}; pub use spin_app::App; -/// Type alias for a [`FactorsConfiguredApp`] specialized to a [`Trigger`]. +/// Type alias for a [`FactorsExecutorApp`] specialized to a [`Trigger`]. pub type TriggerApp = FactorsExecutorApp::InstanceState>; pub type TriggerInstanceBuilder<'a, T> = From 03ba8630990036c1669316395d0112f0adebfd32 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 27 Aug 2024 17:59:06 +0200 Subject: [PATCH 195/195] Ignore Swift and Grain template integration tests Signed-off-by: Ryan Levick --- tests/integration.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration.rs b/tests/integration.rs index d262325d2..c09e241d3 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -628,6 +628,7 @@ Caused by: #[test] #[cfg(target_arch = "x86_64")] #[cfg(feature = "extern-dependencies-tests")] + #[ignore = "https://github.com/fermyon/spin/issues/2774"] fn http_grain_template_smoke_test() -> anyhow::Result<()> { http_smoke_test_template( "http-grain", @@ -656,6 +657,7 @@ Caused by: #[test] #[cfg(feature = "extern-dependencies-tests")] + #[ignore = "https://github.com/fermyon/spin/issues/2774"] fn http_swift_template_smoke_test() -> anyhow::Result<()> { http_smoke_test_template( "http-swift",