diff --git a/.gitmodules b/.gitmodules index 9a6eb9bbe..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "webassembly-benchmarks"] - path = benches/webassembly-benchmarks - url = https://github.com/jedisct1/webassembly-benchmarks diff --git a/Cargo.lock b/Cargo.lock index ca2bd0e34..9dc931cc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,11 +830,7 @@ dependencies = [ name = "containerd-shim-benchmarks" version = "0.4.0" dependencies = [ - "containerd-shim-wasm", - "containerd-shim-wasmedge", - "containerd-shim-wasmtime", "criterion", - "wasmtime", ] [[package]] diff --git a/benches/containerd-shim-benchmarks/Cargo.toml b/benches/containerd-shim-benchmarks/Cargo.toml index 18ea2f980..fa7cc8c33 100644 --- a/benches/containerd-shim-benchmarks/Cargo.toml +++ b/benches/containerd-shim-benchmarks/Cargo.toml @@ -3,23 +3,9 @@ name = "containerd-shim-benchmarks" version.workspace = true edition.workspace = true -[dependencies] -containerd-shim-wasm = { path = "../../crates/containerd-shim-wasm", features = ["testing"] } -containerd-shim-wasmedge = { path = "../../crates/containerd-shim-wasmedge" } -containerd-shim-wasmtime = { path = "../../crates/containerd-shim-wasmtime" } -wasmtime = { workspace = true } - [dev-dependencies] criterion = { version = "0.5", features = ["html_reports"] } -[[bench]] -name = "wasmtime-benchmarks" -harness = false - -[[bench]] -name = "wasmedge-benchmarks" -harness = false - [[bench]] name = "wasi-demo-app-benchmarks" harness = false diff --git a/benches/containerd-shim-benchmarks/benches/wasmedge-benchmarks.rs b/benches/containerd-shim-benchmarks/benches/wasmedge-benchmarks.rs deleted file mode 100644 index 9c39acef7..000000000 --- a/benches/containerd-shim-benchmarks/benches/wasmedge-benchmarks.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::borrow::Cow; -use std::time::Duration; - -use containerd_shim_wasm::container::Instance; -use containerd_shim_wasm::sandbox::Error; -use containerd_shim_wasm::testing::WasiTest; -use containerd_shim_wasmedge::instance::WasmEdgeEngine; -use criterion::measurement::WallTime; -use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; - -/* - For benchmarking try to choose cases which run fast enough -- the idea is to - get a rough idea of the performance rather than run a hours-long benchmark - suite. - - Because of this, only select the benchmarks which run in under 5 seconds on - a desktop computer using WasmEdge. Note that this selection is pretty - arbitrary and we can add or remove benchmarks easily, also from other - sources. This is the selection algorithm: - - $ for file in *; do if timeout 5s wasmedge "${file}" > /dev/null; then echo "$file"; fi; done - aead_chacha20poly1305, - aead_xchacha20poly1305, - auth, - auth2, - auth3, - auth6, - box_seed, - generichash2, - generichash3, - hash, - hash3, - kdf, - keygen, - onetimeauth, - onetimeauth2, - scalarmult2, - secretbox, - secretbox2, - secretbox_easy, - secretstream_xchacha20poly1305, - shorthash, - siphashx24, - stream3, - stream4 - - Criterion is set to run each benchmark ten times, ten being the minimum - number of iterations that Criterion accepts. If we need more statistically - meaningful results we can increase the number of iterations (with the cost - of a longer benchmarking time). Running the whole suite on a desktop - computer takes now a bit over 10 minutes. -*/ - -type WasmedgeTestInstance = Instance; - -fn run_wasmedge_test_with_spec(wasmbytes: &[u8]) -> Result { - let (exit_code, _, _) = WasiTest::::builder()? - .with_wasm(wasmbytes)? - .build()? - .start()? - .wait(Duration::from_secs(10))?; - Ok(exit_code) -} - -fn run_wasmedge_benchmark(group: &mut BenchmarkGroup, bytes: &[u8]) { - group.bench_function("Wasmedge", |b| { - b.iter(|| { - let res = run_wasmedge_test_with_spec(bytes); - match res { - Err(e) => { - panic!("Error running Wasmedge benchmark: {}", e); - } - Ok(status) => { - assert_eq!(status, 0); - } - } - }) - }); -} - -macro_rules! bench_wasm { - ($name:ident) => { - fn $name(c: &mut Criterion) { - let wasmbytes = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../benches/webassembly-benchmarks/2022-12/wasm/", stringify!($name), ".wasm")); - let bytes = Cow::from(wasmbytes); - let mut group = c.benchmark_group(stringify!($name)); - run_wasmedge_benchmark(&mut group, &bytes); - group.finish(); - } - }; - ($name:ident, $($rest:tt),+) => { - bench_wasm!($name); - bench_wasm!($($rest),+); - }; -} - -bench_wasm! { - aead_chacha20poly1305, - aead_xchacha20poly1305, - auth, - auth2, - auth3, - auth6, - box_seed, - generichash2, - generichash3, - hash, - hash3, - kdf, - keygen, - onetimeauth, - onetimeauth2, - scalarmult2, - secretbox, - secretbox2, - secretbox_easy, - secretstream_xchacha20poly1305, - shorthash, - siphashx24, - stream3, - stream4 -} - -criterion_group! { - name = benches; - config = Criterion::default().sample_size(10); - targets = aead_chacha20poly1305, - aead_xchacha20poly1305, - auth, - auth2, - auth3, - auth6, - box_seed, - generichash2, - generichash3, - hash, - hash3, - kdf, - keygen, - onetimeauth, - onetimeauth2, - scalarmult2, - secretbox, - secretbox2, - secretbox_easy, - secretstream_xchacha20poly1305, - shorthash, - siphashx24, - stream3, - stream4 -} - -criterion_main!(benches); diff --git a/benches/containerd-shim-benchmarks/benches/wasmtime-benchmarks.rs b/benches/containerd-shim-benchmarks/benches/wasmtime-benchmarks.rs deleted file mode 100644 index a0a8a1b68..000000000 --- a/benches/containerd-shim-benchmarks/benches/wasmtime-benchmarks.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::borrow::Cow; -use std::time::Duration; - -use containerd_shim_wasm::container::Instance; -use containerd_shim_wasm::sandbox::Error; -use containerd_shim_wasm::testing::WasiTest; -use containerd_shim_wasmtime::instance::{WasiConfig, WasmtimeEngine}; -use criterion::measurement::WallTime; -use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; - -/* - For benchmarking try to choose cases which run fast enough -- the idea is to - get a rough idea of the performance rather than run a hours-long benchmark - suite. - - Because of this, only select the benchmarks which run in under 5 seconds on - a desktop computer using WasmEdge. Note that this selection is pretty - arbitrary and we can add or remove benchmarks easily, also from other - sources. This is the selection algorithm: - - $ for file in *; do if timeout 5s wasmedge "${file}" > /dev/null; then echo "$file"; fi; done - aead_chacha20poly13052.wasm - aead_chacha20poly1305.wasm - aead_xchacha20poly1305.wasm - auth2.wasm - auth3.wasm - auth6.wasm - auth.wasm - box_seed.wasm - generichash2.wasm - generichash3.wasm - hash3.wasm - hash.wasm - kdf.wasm - keygen.wasm - onetimeauth2.wasm - onetimeauth.wasm - scalarmult2.wasm - scalarmult5.wasm - scalarmult6.wasm - secretbox2.wasm - secretbox_easy.wasm - secretbox.wasm - secretstream_xchacha20poly1305.wasm - shorthash.wasm - siphashx24.wasm - stream3.wasm - stream4.wasm - - Criterion is set to run each benchmark ten times, ten being the minimum - number of iterations that Criterion accepts. If we need more statistically - meaningful results we can increase the number of iterations (with the cost - of a longer benchmarking time). Running the whole suite on a desktop - computer takes now a bit over 10 minutes. -*/ -#[derive(Clone)] -struct WasiTestConfig {} - -impl WasiConfig for WasiTestConfig { - fn new_config() -> wasmtime::Config { - let mut config = wasmtime::Config::new(); - // Disable Wasmtime parallel compilation for the tests - // see https://github.com/containerd/runwasi/pull/405#issuecomment-1928468714 for details - config.parallel_compilation(false); - config.wasm_component_model(true); // enable component linking - config - } -} - -type WasmtimeTestInstance = Instance>; - -fn run_wasmtime_test_with_spec(wasmbytes: &[u8]) -> Result { - let (exit_code, _, _) = WasiTest::::builder()? - .with_wasm(wasmbytes)? - .build()? - .start()? - .wait(Duration::from_secs(10))?; - Ok(exit_code) -} - -fn run_wasmtime_benchmark(group: &mut BenchmarkGroup, bytes: &[u8]) { - group.bench_function("Wasmtime", |b| { - b.iter(|| { - let res = run_wasmtime_test_with_spec(bytes); - match res { - Err(e) => { - panic!("Error running Wasmtime benchmark: {}", e); - } - Ok(status) => { - assert_eq!(status, 0); - } - } - }) - }); -} - -macro_rules! bench_wasm { - ($name:ident) => { - fn $name(c: &mut Criterion) { - let wasmbytes = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../benches/webassembly-benchmarks/2022-12/wasm/", stringify!($name), ".wasm")); - let bytes = Cow::from(wasmbytes); - let mut group = c.benchmark_group(stringify!($name)); - run_wasmtime_benchmark(&mut group, &bytes); - group.finish(); - } - }; - ($name:ident, $($rest:tt),+) => { - bench_wasm!($name); - bench_wasm!($($rest),+); - }; -} - -bench_wasm! { - aead_chacha20poly13052, - aead_chacha20poly1305, - aead_xchacha20poly1305, - auth2, - auth3, - auth6, - auth, - box_seed, - generichash2, - generichash3, - hash3, - hash, - kdf, - keygen, - onetimeauth2, - onetimeauth, - scalarmult2, - scalarmult5, - scalarmult6, - secretbox2, - secretbox_easy, - secretbox, - secretstream_xchacha20poly1305, - shorthash, - siphashx24, - stream3, - stream4 -} - -criterion_group! { - name = benches; - config = Criterion::default().sample_size(10); - targets = aead_chacha20poly13052, - aead_chacha20poly1305, - aead_xchacha20poly1305, - auth2, - auth3, - auth6, - auth, - box_seed, - generichash2, - generichash3, - hash3, - hash, - kdf, - keygen, - onetimeauth2, - onetimeauth, - scalarmult2, - scalarmult5, - scalarmult6, - secretbox2, - secretbox_easy, - secretbox, - secretstream_xchacha20poly1305, - shorthash, - siphashx24, - stream3, - stream4 -} - -criterion_main!(benches); diff --git a/benches/webassembly-benchmarks b/benches/webassembly-benchmarks deleted file mode 160000 index 7e86d68e9..000000000 --- a/benches/webassembly-benchmarks +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7e86d68e99e60130899fbe3b3ab6e9dce9187a7c