From 4d4618d1eeea284ef8e175f881a09b9a11cc525f Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Tue, 22 Oct 2024 12:53:22 +0100 Subject: [PATCH 1/7] feat: Add features for dev and prod wasmer configurations --- .github/workflows/test.yml | 27 +++- Cargo.lock | 128 +++++++++++++++--- crates/host/Cargo.toml | 14 +- crates/host/src/module/wasmer_sys.rs | 11 +- flake.nix | 23 +++- ...-wasmer_sys.sh => bench-wasmer_sys_dev.sh} | 2 +- scripts/bench-wasmer_sys_prod.sh | 5 + scripts/bench.sh | 3 +- scripts/fuzz-wasmer_sys.sh | 2 +- scripts/fuzz-wasmer_sys_prod.sh | 5 + scripts/fuzz.sh | 3 +- ...t-wasmer_sys.sh => test-wasmer_sys_dev.sh} | 6 +- scripts/test-wasmer_sys_prod.sh | 23 ++++ scripts/test.sh | 3 +- test/Cargo.lock | 83 +++++++++++- test/Cargo.toml | 14 +- test/src/wasms.rs | 27 ++-- 17 files changed, 321 insertions(+), 58 deletions(-) rename scripts/{bench-wasmer_sys.sh => bench-wasmer_sys_dev.sh} (73%) create mode 100755 scripts/bench-wasmer_sys_prod.sh create mode 100755 scripts/fuzz-wasmer_sys_prod.sh rename scripts/{test-wasmer_sys.sh => test-wasmer_sys_dev.sh} (81%) create mode 100755 scripts/test-wasmer_sys_prod.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a227266c..6f5c619b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: matrix: script: ["test", "bench"] os: ["ubuntu-latest", "macos-latest"] - wasmer-feature: ["wasmer_sys", "wasmer_wamr"] + wasmer-feature: ["wasmer_sys_dev", "wasmer_sys_prod", "wasmer_wamr"] exclude: # TODO bench suite on macos-latest is killed by system due to running out of swap space # All benches run fine individually @@ -33,9 +33,9 @@ jobs: size: 15G - uses: actions/checkout@v4 - name: Install nix - uses: cachix/install-nix-action@v26 + uses: cachix/install-nix-action@v30 - name: Setup cachix - uses: cachix/cachix-action@v14 + uses: cachix/cachix-action@v15 if: ${{ ! contains(matrix.platform.runs-on, 'self-hosted') }} with: name: holochain-ci @@ -47,7 +47,8 @@ jobs: fail-fast: false matrix: wasmer-feature: - - "wasmer_sys" + - "wasmer_sys_dev" + - "wasmer_sys_prod" # TODO Building with wasmer_wamr feature flag on windows is not currently working. # See https://github.com/holochain/holochain-wasmer/issues/117 # - "wasmer_wamr" @@ -57,7 +58,21 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-unknown-unknown + - name: Install LLVM + env: + LLVM_DIR: .llvm + shell: bash + run: | + LLVM_DIR=$(pwd)/${{ env.LLVM_DIR }} + mkdir -p ${LLVM_DIR} + curl --proto '=https' --tlsv1.2 -sSf "https://github.com/wasmerio/llvm-custom-builds/releases/download/15.x/llvm-windows-amd64.tar.xz" -L -o - | tar xJv -C ${LLVM_DIR} - name: test root - run: cargo test --release --no-default-features --features error_as_host,${{ matrix.wasmer-feature }} -- --nocapture + shell: pwsh + run: | + $env:LLVM_SYS_150_PREFIX="$(pwd)/.llvm" + cargo test --release --no-default-features --features error_as_host,${{ matrix.wasmer-feature }} -- --nocapture - name: test - run: cargo test --release --manifest-path test/Cargo.toml --no-default-features --features ${{ matrix.wasmer-feature }} -- --nocapture + shell: pwsh + run: | + $env:LLVM_SYS_150_PREFIX="$(pwd)/.llvm" + cargo test --release --manifest-path test/Cargo.toml --no-default-features --features ${{ matrix.wasmer-feature }} -- --nocapture diff --git a/Cargo.lock b/Cargo.lock index 8c31c5cd..d98c7e67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide 0.7.4", - "object", + "object 0.36.3", "rustc-demangle", ] @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", @@ -1050,6 +1050,40 @@ dependencies = [ "serde", ] +[[package]] +name = "inkwell" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbac11e485159a525867fb7e6aa61981453e6a72f625fde6a4ab3047b0c6dec9" +dependencies = [ + "either", + "inkwell_internals", + "libc", + "llvm-sys", + "once_cell", + "parking_lot", +] + +[[package]] +name = "inkwell_internals" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d00c17e264ce02be5bc23d7bff959188ec7137beddd06b8b6b05a7c680ea85" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1140,6 +1174,19 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" +[[package]] +name = "llvm-sys" +version = "150.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa55828745895d37233756307ded95a235b058aeb89cd12717ec7c3912089ee9" +dependencies = [ + "cc", + "lazy_static", + "libc", + "regex", + "semver", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -1253,6 +1300,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.28.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" +dependencies = [ + "memchr", +] + [[package]] name = "object" version = "0.36.3" @@ -1469,9 +1525,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -1481,9 +1537,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -1492,9 +1548,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "region" @@ -1596,6 +1652,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.34" @@ -1611,9 +1676,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "log", "once_cell", @@ -1626,15 +1691,15 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -2135,9 +2200,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unsafe-libyaml" @@ -2283,6 +2348,7 @@ dependencies = [ "wasm-bindgen", "wasmer-compiler", "wasmer-compiler-cranelift", + "wasmer-compiler-llvm", "wasmer-derive", "wasmer-types", "wasmer-vm", @@ -2338,6 +2404,30 @@ dependencies = [ "wasmer-types", ] +[[package]] +name = "wasmer-compiler-llvm" +version = "4.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86da768569a3b509285e486a93a3058b4837668c9c7fb54d7d208469fb06f028" +dependencies = [ + "byteorder", + "cc", + "inkwell", + "itertools 0.10.5", + "lazy_static", + "libc", + "object 0.28.4", + "rayon", + "regex", + "rustc_version", + "semver", + "smallvec", + "target-lexicon", + "wasmer-compiler", + "wasmer-types", + "wasmer-vm", +] + [[package]] name = "wasmer-config" version = "0.4.0" @@ -2495,9 +2585,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index dbd29d6a..994ac7d1 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -7,8 +7,8 @@ authors = ["thedavidmeister", "thedavidmeister@gmail.com"] edition = "2021" [dependencies] -wasmer = { version = "=4.3.6", optional = true, default-feature = false } -wasmer-middlewares = { version = "=4.3.6", optional = true, default-feature = false } +wasmer = { version = "=4.3.6", optional = true, default-features = false } +wasmer-middlewares = { version = "=4.3.6", optional = true, default-features = false } # Temporarily include a fork of wasmer from the git branch 'wamr', until it is officially released in wasmer v5 hc-wasmer = { version="=4.3.6-hc.1", optional = true, default-features = false } @@ -30,14 +30,22 @@ crate-type = ["cdylib", "rlib"] path = "src/lib.rs" [features] -default = ["error_as_host", "wasmer_sys"] +default = ["error_as_host", "wasmer_sys_dev"] debug_memory = [] error_as_host = ["holochain_wasmer_common/error_as_host"] wasmer_sys = [ "dep:wasmer", "dep:wasmer-middlewares", + "wasmer/sys", +] +wasmer_sys_dev = [ + "wasmer_sys", "wasmer/default", ] +wasmer_sys_prod = [ + "wasmer_sys", + "wasmer/llvm", +] wasmer_wamr = [ "dep:hc-wasmer", "hc-wasmer/wamr" diff --git a/crates/host/src/module/wasmer_sys.rs b/crates/host/src/module/wasmer_sys.rs index 2778e918..2eadd709 100644 --- a/crates/host/src/module/wasmer_sys.rs +++ b/crates/host/src/module/wasmer_sys.rs @@ -5,7 +5,6 @@ use wasmer::sys::BaseTunables; use wasmer::sys::CompilerConfig; use wasmer::wasmparser; use wasmer::CompileError; -use wasmer::Cranelift; use wasmer::DeserializeError; use wasmer::Engine; use wasmer::Module; @@ -29,9 +28,15 @@ pub fn make_engine() -> Engine { // @todo 100 giga-ops is totally arbitrary cutoff so we probably // want to make the limit configurable somehow. let metering = Arc::new(Metering::new(WASM_METERING_LIMIT, cost_function)); + // the only place where the wasm compiler engine is set - let mut compiler = Cranelift::default(); - compiler.canonicalize_nans(true).push_middleware(metering); + #[cfg(feature = "wasmer_sys_dev")] + let mut compiler = wasmer::Cranelift::default(); + #[cfg(feature = "wasmer_sys_prod")] + let mut compiler = wasmer::LLVM::default(); + + compiler.canonicalize_nans(true); + compiler.push_middleware(metering); // Workaround for invalid memory access on iOS. // https://github.com/holochain/holochain/issues/3096 diff --git a/flake.nix b/flake.nix index efd75ec6..98974d5c 100644 --- a/flake.nix +++ b/flake.nix @@ -21,13 +21,26 @@ inputs.holochain-flake.devShells.${system}.rustDev ]; - # These packages and env vars are required to build wasmer on the 'wamr' branch (i.e. the hc-wasmer dependency) - packages = [ - pkgs.cmake - pkgs.clang - pkgs.llvmPackages.libclang.lib + packages = with pkgs; [ + # These packages and env vars are required to build Wasmer with the 'wamr' feature (i.e. the hc-wasmer dependency) + cmake + clang + llvmPackages.libclang.lib + # These packages are required to build Wasmer with the production config + llvm_15 + libffi + libxml2 + zlib + ncurses ]; + # Used by `wamr` LIBCLANG_PATH="${pkgs.llvmPackages.libclang.lib}/lib"; + # Used by wasmer production config + shellHook = '' + # This binary lives in a different derivation to `llvm_15` and isn't re-exported through that derivation + export LLVM_SYS_150_PREFIX=$(which llvm-config | xargs dirname | xargs dirname) + export LD_LIBRARY_PATH="${pkgs.stdenv.cc.cc.lib}/lib:${pkgs.libffi}/lib:${pkgs.zlib}/lib:${pkgs.ncurses}/lib" + ''; }; }; }; diff --git a/scripts/bench-wasmer_sys.sh b/scripts/bench-wasmer_sys_dev.sh similarity index 73% rename from scripts/bench-wasmer_sys.sh rename to scripts/bench-wasmer_sys_dev.sh index 63909ae5..ab6ded2c 100755 --- a/scripts/bench-wasmer_sys.sh +++ b/scripts/bench-wasmer_sys_dev.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash ( cd test && \ - cargo bench --no-default-features --features wasmer_sys + cargo bench --no-default-features --features wasmer_sys_dev ) # it's possible to flamegraph the benchmarks like this: diff --git a/scripts/bench-wasmer_sys_prod.sh b/scripts/bench-wasmer_sys_prod.sh new file mode 100755 index 00000000..fd73eced --- /dev/null +++ b/scripts/bench-wasmer_sys_prod.sh @@ -0,0 +1,5 @@ +#! /usr/bin/env bash +( + cd test && \ + cargo bench --no-default-features --features wasmer_sys_prod +) diff --git a/scripts/bench.sh b/scripts/bench.sh index ec8a8bb4..3dad9a66 100644 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash -./bench-wasmer_sys.sh && \ +./bench-wasmer_sys_dev.sh +./bench-wasmer_sys_prod.sh ./bench-wasmer_wamr.sh \ No newline at end of file diff --git a/scripts/fuzz-wasmer_sys.sh b/scripts/fuzz-wasmer_sys.sh index 80472e81..f6832e9f 100755 --- a/scripts/fuzz-wasmer_sys.sh +++ b/scripts/fuzz-wasmer_sys.sh @@ -2,4 +2,4 @@ cargo test cargo test --manifest-path test/Cargo.toml -cargo test-fuzz "$FUZZ_TARGET" --no-default-features --features wasmer_sys \ No newline at end of file +cargo test-fuzz "$FUZZ_TARGET" --no-default-features --features wasmer_sys_dev diff --git a/scripts/fuzz-wasmer_sys_prod.sh b/scripts/fuzz-wasmer_sys_prod.sh new file mode 100755 index 00000000..5a9d3a4b --- /dev/null +++ b/scripts/fuzz-wasmer_sys_prod.sh @@ -0,0 +1,5 @@ +#! /usr/bin/env bash +cargo test +cargo test --manifest-path test/Cargo.toml + +cargo test-fuzz "$FUZZ_TARGET" --no-default-features --features wasmer_sys_prod diff --git a/scripts/fuzz.sh b/scripts/fuzz.sh index c2b47543..814a1053 100644 --- a/scripts/fuzz.sh +++ b/scripts/fuzz.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash -./fuzz-wasmer_sys.sh && \ +./fuzz-wasmer_sys_dev.sh +./fuzz-wasmer_sys_prod.sh ./fuzz-wasmer_wamr.sh \ No newline at end of file diff --git a/scripts/test-wasmer_sys.sh b/scripts/test-wasmer_sys_dev.sh similarity index 81% rename from scripts/test-wasmer_sys.sh rename to scripts/test-wasmer_sys_dev.sh index e748cad9..80dd2bcd 100755 --- a/scripts/test-wasmer_sys.sh +++ b/scripts/test-wasmer_sys_dev.sh @@ -14,10 +14,10 @@ cargo clippy ( cd crates/guest && cargo clippy --target wasm32-unknown-unknown ) # tests the root workspace that doesn't include any wasm code -cargo test --no-default-features --features error_as_host,wasmer_sys ${1-} -- --nocapture +cargo test --no-default-features --features error_as_host,wasmer_sys_dev ${1-} -- --nocapture # test that everything builds cargo build --release --manifest-path test/test_wasm/Cargo.toml --target wasm32-unknown-unknown -# build wasm and run the "full" tests for wasmer_sys -cargo test --release --manifest-path test/Cargo.toml --no-default-features --features wasmer_sys ${1-} -- --nocapture +# build wasm and run the "full" tests for wasmer_sys_dev +cargo test --release --manifest-path test/Cargo.toml --no-default-features --features wasmer_sys_dev ${1-} -- --nocapture diff --git a/scripts/test-wasmer_sys_prod.sh b/scripts/test-wasmer_sys_prod.sh new file mode 100755 index 00000000..885eb3a7 --- /dev/null +++ b/scripts/test-wasmer_sys_prod.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euxo pipefail + +export RUST_BACKTRACE=full +export WASMER_BACKTRACE=1 + +# static tests +cargo fmt +( cd test && cargo fmt --check ) +( cd crates/guest && cargo fmt --check ) + +cargo clippy +( cd test && cargo clippy ) +( cd crates/guest && cargo clippy --target wasm32-unknown-unknown ) + +# tests the root workspace that doesn't include any wasm code +cargo test --no-default-features --features error_as_host,wasmer_sys_prod ${1-} -- --nocapture + +# test that everything builds +cargo build --release --manifest-path test/test_wasm/Cargo.toml --target wasm32-unknown-unknown + +# build wasm and run the "full" tests for wasmer_sys_prod +cargo test --release --manifest-path test/Cargo.toml --no-default-features --features wasmer_sys_prod ${1-} -- --nocapture diff --git a/scripts/test.sh b/scripts/test.sh index 30908fcb..f27eaeba 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash -./test-wasmer_sys.sh && \ +./test-wasmer_sys_dev.sh +./test-wasmer_sys_prod.sh ./test-wasmer_wamr.sh \ No newline at end of file diff --git a/test/Cargo.lock b/test/Cargo.lock index cbb07f9e..61361628 100644 --- a/test/Cargo.lock +++ b/test/Cargo.lock @@ -98,7 +98,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.35.0", "rustc-demangle", ] @@ -1224,6 +1224,31 @@ dependencies = [ "serde", ] +[[package]] +name = "inkwell" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbac11e485159a525867fb7e6aa61981453e6a72f625fde6a4ab3047b0c6dec9" +dependencies = [ + "either", + "inkwell_internals", + "libc", + "llvm-sys", + "once_cell", + "parking_lot", +] + +[[package]] +name = "inkwell_internals" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d00c17e264ce02be5bc23d7bff959188ec7137beddd06b8b6b05a7c680ea85" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "is-terminal" version = "0.4.12" @@ -1314,6 +1339,19 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" +[[package]] +name = "llvm-sys" +version = "150.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa55828745895d37233756307ded95a235b058aeb89cd12717ec7c3912089ee9" +dependencies = [ + "cc", + "lazy_static", + "libc", + "regex", + "semver", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -1418,6 +1456,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.28.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" +dependencies = [ + "memchr", +] + [[package]] name = "object" version = "0.35.0" @@ -1801,6 +1848,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.34" @@ -2580,6 +2636,7 @@ dependencies = [ "wasm-bindgen", "wasmer-compiler", "wasmer-compiler-cranelift", + "wasmer-compiler-llvm", "wasmer-derive", "wasmer-types", "wasmer-vm", @@ -2635,6 +2692,30 @@ dependencies = [ "wasmer-types", ] +[[package]] +name = "wasmer-compiler-llvm" +version = "4.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86da768569a3b509285e486a93a3058b4837668c9c7fb54d7d208469fb06f028" +dependencies = [ + "byteorder", + "cc", + "inkwell", + "itertools 0.10.5", + "lazy_static", + "libc", + "object 0.28.4", + "rayon", + "regex", + "rustc_version", + "semver", + "smallvec", + "target-lexicon", + "wasmer-compiler", + "wasmer-types", + "wasmer-vm", +] + [[package]] name = "wasmer-config" version = "0.2.0" diff --git a/test/Cargo.toml b/test/Cargo.toml index 5d34c246..dd9d380d 100644 --- a/test/Cargo.toml +++ b/test/Cargo.toml @@ -42,12 +42,20 @@ debug = true [features] debug_memory = ["holochain_wasmer_host/debug_memory"] -default = ["wasmer_sys"] +default = ["wasmer_sys_dev"] wasmer_sys = [ "dep:wasmer", - "dep:wasmer-middlewares", + "dep:wasmer-middlewares", + "wasmer/sys", +] +wasmer_sys_dev = [ + "wasmer_sys", "wasmer/default", - "holochain_wasmer_host/wasmer_sys" + "holochain_wasmer_host/wasmer_sys_dev" +] +wasmer_sys_prod = [ + "wasmer_sys", + "holochain_wasmer_host/wasmer_sys_prod" ] wasmer_wamr = [ "dep:hc-wasmer", diff --git a/test/src/wasms.rs b/test/src/wasms.rs index 9d90a217..9f9146fc 100644 --- a/test/src/wasms.rs +++ b/test/src/wasms.rs @@ -10,8 +10,6 @@ use wasmer::wasmparser::Operator; use wasmer::AsStoreMut; #[cfg(feature = "wasmer_sys")] use wasmer::CompilerConfig; -#[cfg(feature = "wasmer_sys")] -use wasmer::Cranelift; use wasmer::Engine; use wasmer::FunctionEnv; use wasmer::Imports; @@ -92,15 +90,24 @@ impl TestWasm { let cranelift_fn = || { let cost_function = |_operator: &Operator| -> u64 { 1 }; let metering = Arc::new(Metering::new(10_000_000_000, cost_function)); - let mut cranelift = Cranelift::default(); - cranelift.canonicalize_nans(true).push_middleware(metering); - Engine::from(cranelift) + #[cfg(feature = "wasmer_sys_dev")] + let mut compiler = wasmer::Cranelift::default(); + #[cfg(feature = "wasmer_sys_prod")] + let mut compiler = wasmer::LLVM::default(); + + compiler.canonicalize_nans(true); + compiler.push_middleware(metering); + Engine::from(compiler) }; - let cranelift_fn_unmetered = || { - let mut cranelift = Cranelift::default(); - cranelift.canonicalize_nans(true); - Engine::from(cranelift) + let compiler_fn_unmetered = || { + #[cfg(feature = "wasmer_sys_dev")] + let mut compiler = wasmer::Cranelift::default(); + #[cfg(feature = "wasmer_sys_prod")] + let mut compiler = wasmer::LLVM::default(); + + compiler.canonicalize_nans(true); + Engine::from(compiler) }; // This will error if the cache is already initialized @@ -111,7 +118,7 @@ impl TestWasm { if metered { cranelift_fn } else { - cranelift_fn_unmetered + compiler_fn_unmetered }, None, ), From 8c6751e1543a1b1fcf3057a4ebd9593c42c52556 Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Fri, 25 Oct 2024 15:02:38 +0100 Subject: [PATCH 2/7] Fix fuzz scripts, although fuzzing doesn't just work --- scripts/{fuzz-wasmer_sys.sh => fuzz-wasmer_sys_dev.sh} | 0 scripts/fuzz.sh | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename scripts/{fuzz-wasmer_sys.sh => fuzz-wasmer_sys_dev.sh} (100%) mode change 100644 => 100755 scripts/fuzz.sh diff --git a/scripts/fuzz-wasmer_sys.sh b/scripts/fuzz-wasmer_sys_dev.sh similarity index 100% rename from scripts/fuzz-wasmer_sys.sh rename to scripts/fuzz-wasmer_sys_dev.sh diff --git a/scripts/fuzz.sh b/scripts/fuzz.sh old mode 100644 new mode 100755 index 814a1053..5b37c563 --- a/scripts/fuzz.sh +++ b/scripts/fuzz.sh @@ -1,5 +1,5 @@ #! /usr/bin/env bash -./fuzz-wasmer_sys_dev.sh -./fuzz-wasmer_sys_prod.sh -./fuzz-wasmer_wamr.sh \ No newline at end of file +./scripts/fuzz-wasmer_sys_dev.sh +./scripts/fuzz-wasmer_sys_prod.sh +./scripts/fuzz-wasmer_wamr.sh \ No newline at end of file From 723295d14b59b4bd637a2bb2a9fd60e739923292 Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Wed, 30 Oct 2024 18:23:53 +0000 Subject: [PATCH 3/7] Format check, rather than actually formatting --- scripts/test-wasmer_sys_dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-wasmer_sys_dev.sh b/scripts/test-wasmer_sys_dev.sh index 80dd2bcd..619d337d 100755 --- a/scripts/test-wasmer_sys_dev.sh +++ b/scripts/test-wasmer_sys_dev.sh @@ -5,7 +5,7 @@ export RUST_BACKTRACE=full export WASMER_BACKTRACE=1 # static tests -cargo fmt +cargo fmt --check ( cd test && cargo fmt --check ) ( cd crates/guest && cargo fmt --check ) From abff424e53be109740d1e257b7f30852d1ac2799 Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Wed, 30 Oct 2024 18:37:30 +0000 Subject: [PATCH 4/7] Fix lint checking and then lint issues --- crates/host/src/plru.rs | 4 ---- scripts/test-wasmer_sys_dev.sh | 6 +++--- scripts/test-wasmer_sys_prod.sh | 9 --------- test/wasm_empty/src/wasm.rs | 3 ++- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/crates/host/src/plru.rs b/crates/host/src/plru.rs index 3742b033..19c588c4 100644 --- a/crates/host/src/plru.rs +++ b/crates/host/src/plru.rs @@ -77,8 +77,6 @@ //! ✔ Idiomatic Rust //! //! ✔ Production-quality - -#![cfg_attr(feature = "no_std", no_std)] #![warn(missing_docs)] extern crate core; @@ -105,7 +103,6 @@ pub type DynamicCache = Cache>; /// Create a heap allocated cache of a fixed (but dynamic) number of cache lines. /// /// Ideally, 64 should divide `len` to optimize space efficiency. -#[cfg(not(feature = "no_std"))] pub fn create(lines: usize) -> DynamicCache { // Unfortunately, `AtomicU64` doesn't implement `Clone`, so we cannot use the `vec![]` macro. // We need to manually construct our vector. @@ -312,7 +309,6 @@ impl> fmt::Debug for Cache { #[cfg(test)] mod tests { #[test] - #[cfg(not(feature = "no_std"))] fn create() { assert_eq!(super::create(10).len(), 64); assert_eq!(super::create(64).len(), 64); diff --git a/scripts/test-wasmer_sys_dev.sh b/scripts/test-wasmer_sys_dev.sh index 619d337d..fcaf35da 100755 --- a/scripts/test-wasmer_sys_dev.sh +++ b/scripts/test-wasmer_sys_dev.sh @@ -9,9 +9,9 @@ cargo fmt --check ( cd test && cargo fmt --check ) ( cd crates/guest && cargo fmt --check ) -cargo clippy -( cd test && cargo clippy ) -( cd crates/guest && cargo clippy --target wasm32-unknown-unknown ) +cargo clippy -- --deny warnings +( cd test && cargo clippy -- --deny warnings ) +( cd crates/guest && cargo clippy --target wasm32-unknown-unknown -- --deny warnings ) # tests the root workspace that doesn't include any wasm code cargo test --no-default-features --features error_as_host,wasmer_sys_dev ${1-} -- --nocapture diff --git a/scripts/test-wasmer_sys_prod.sh b/scripts/test-wasmer_sys_prod.sh index 885eb3a7..b36be088 100755 --- a/scripts/test-wasmer_sys_prod.sh +++ b/scripts/test-wasmer_sys_prod.sh @@ -4,15 +4,6 @@ set -euxo pipefail export RUST_BACKTRACE=full export WASMER_BACKTRACE=1 -# static tests -cargo fmt -( cd test && cargo fmt --check ) -( cd crates/guest && cargo fmt --check ) - -cargo clippy -( cd test && cargo clippy ) -( cd crates/guest && cargo clippy --target wasm32-unknown-unknown ) - # tests the root workspace that doesn't include any wasm code cargo test --no-default-features --features error_as_host,wasmer_sys_prod ${1-} -- --nocapture diff --git a/test/wasm_empty/src/wasm.rs b/test/wasm_empty/src/wasm.rs index b60bbdb9..22eba27a 100644 --- a/test/wasm_empty/src/wasm.rs +++ b/test/wasm_empty/src/wasm.rs @@ -1,2 +1,3 @@ +#![allow(unused_imports)] use holochain_wasmer_guest::allocation::__hc__allocate_1; -use holochain_wasmer_guest::allocation::__hc__deallocate_1; \ No newline at end of file +use holochain_wasmer_guest::allocation::__hc__deallocate_1; From 0d499c8455ac157559bbb722254e7afe3974eb5f Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Wed, 30 Oct 2024 18:43:56 +0000 Subject: [PATCH 5/7] Fix another lint issue --- test/test_wasm/src/wasm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_wasm/src/wasm.rs b/test/test_wasm/src/wasm.rs index 4128dbfc..5ae52acc 100644 --- a/test/test_wasm/src/wasm.rs +++ b/test/test_wasm/src/wasm.rs @@ -67,7 +67,7 @@ pub extern "C" fn process_native(guest_ptr: usize, len: usize) -> DoubleUSize { Err(err_ptr) => return err_ptr, }; let processed: SomeStruct = try_ptr!( - host_call(__hc__test_process_struct_2, &input), + host_call(__hc__test_process_struct_2, input), "could not deserialize SomeStruct in process_native" ); return_ptr(processed) From f9d4542e73e61b1243c5014d76643bdc8e0e003c Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Wed, 30 Oct 2024 19:27:02 +0000 Subject: [PATCH 6/7] Update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc8dca17..acfe364b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +### Changed +- **BREAKING CHANGE** The `wasmer_sys` has been renamed to `wasmer_sys_dev` + +### Added +- A new feature flag, `wasmer_sys_prod` which enables the Wasmer LLVM compiler. The default, with the `wasmer_sys_dev` feature + is the Cranelift compiler. The Cranelift compiler is fast, and recommended for development, but the LLVM compiler is supposed + to be faster and more optimized for production. In testing so far, the compile step is slower with LLVM but the runtime is + faster. More testing is needed yet to confirm the difference. + ## [0.0.93] - 2024-04-24 ### Changed From 1f6cb88b4dcc5dbf17e64c48859e002be79e2011 Mon Sep 17 00:00:00 2001 From: ThetaSinner Date: Thu, 31 Oct 2024 10:50:27 +0000 Subject: [PATCH 7/7] Update CHANGELOG.md Co-authored-by: Callum Dunster --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acfe364b..eaf41d9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] ### Changed -- **BREAKING CHANGE** The `wasmer_sys` has been renamed to `wasmer_sys_dev` +- **BREAKING CHANGE** The `wasmer_sys` feature has been renamed to `wasmer_sys_dev` ### Added - A new feature flag, `wasmer_sys_prod` which enables the Wasmer LLVM compiler. The default, with the `wasmer_sys_dev` feature