From 24354d2e11c69e38e4e021aa4acf1525b376b2b1 Mon Sep 17 00:00:00 2001 From: Tyler Hou Date: Wed, 22 Feb 2023 15:13:42 -0800 Subject: [PATCH] Get hydroflow to compile to WASM (#329) Tests do not work yet; I have a followup PR. 1. Turn off diagnostics for non-proc-macro crates. Diagnostics don't work when building for WASM. The proc-macro crates that are compiled at compile time still enable diagonstics because they target the (presumably non-WASM) host architecture where diagonstics work. 2. Use version 2 for the Cargo feature resolver. The version 1 resolver does not treat proc-macro edges differently from regular dependency edges. That is, suppose we depend on package P transitively via both a proc-macro edge and a regular edge, and the proc-macro edge enables feature X but the regular edge does not. With the version 1 resolver, Cargo will compile P only once with X enabled (assuming same host and target arch). With the v2 resolver, Cargo will compile P (at least) twice; once with X enabled for the proc-macro crate, once with P disabled. In our project, P = {hydroflow_datalog_core, hydroflow_lang}, X = diagnostics. The version 1 resolver works for non-WASM builds (because diagnostics compiles fine), but for WASM builds the v1 resolver attempts to build hydroflow_datalog_core for WASM with diagnostics enabled (because of feature resolution). Note that enabling the v2 feature resolver should not change compile times: on non-WASM, diagnostics is always enabled so we only compile P once. On WASM builds, we would have compiled twice anyway (once for the host architecture for the proc-macro package, another for WASM). 3. Turn on only a subset of Tokio features when compiling for WASM. In particular, the WASM environment does not natively support threads and fails to compile with threading features enabled. 4. Manually enable the "js" feature for hydroflow/getrandom. See the comment in hydroflow/Cargo.toml. --- Cargo.lock | 3 +++ Cargo.toml | 2 ++ hydroflow/Cargo.toml | 23 +++++++++++++++---- hydroflow/src/scheduled/net/mod.rs | 2 ++ hydroflow/src/scheduled/net/network_vertex.rs | 2 ++ hydroflow/src/util/mod.rs | 5 ++++ hydroflow/src/util/tcp.rs | 2 ++ hydroflow/src/util/udp.rs | 2 ++ hydroflow/tests/surface_async.rs | 2 ++ hydroflow_datalog/Cargo.toml | 5 +++- hydroflow_datalog_core/Cargo.toml | 4 ++-- hydroflow_internalmacro.d | 5 ++++ hydroflow_internalmacro/Cargo.toml | 2 +- hydroflow_lang/Cargo.toml | 2 +- hydroflow_macro/Cargo.toml | 5 +++- 15 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 hydroflow_internalmacro.d diff --git a/Cargo.lock b/Cargo.lock index b0c61cfa5d2f..c394a9adf235 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1219,8 +1219,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.10.2+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1423,6 +1425,7 @@ dependencies = [ "core_affinity", "criterion", "futures", + "getrandom 0.2.6", "hdrhistogram", "hydroflow_datalog", "hydroflow_lang", diff --git a/Cargo.toml b/Cargo.toml index 576e994e7683..dc03e5693c77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,8 @@ members = [ "variadics", ] +resolver = "2" + [profile.release] strip = true # Strip symbols from the binary opt-level = "z" # Optimize for size diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index 152dbf2a8db9..5c1f063fc4dc 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -63,18 +63,29 @@ serde = { version = "1", features = [ "derive" ] } serde_json = "1" slotmap = "1.0" static_assertions = "1.1.0" -tokio = { version = "1.16", features = [ "full" ] } tokio-stream = { version = "0.1.10", features = [ "io-util" ] } -tokio-util = { version = "0.7.4", features = [ "net", "codec" ] } variadics = { path = "../variadics" } rustc-hash = "1.1.0" tempfile = { version = "3.3.0", optional = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { version = "1.16", features = [ "full" ] } +tokio-util = { version = "0.7.4", features = [ "net", "codec" ] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +tokio = { version = "1.16", features = [ "rt" , "sync", "macros", "io-util", "time" ] } +tokio-util = { version = "0.7.4", features = [ "codec" ] } +# We depend on getrandom transitively through rand. To compile getrandom to +# WASM, we need to enable its "js" feature. However, rand does not expose a +# passthrough to enable "js" on getrandom. As a workaround, we enable the +# getrandom js feature here; when the feature resolver computes the set of +# features for getrandom (unification), it will include "js". +getrandom = { version = "0.2.6", features = [ "js" ] } + [dev-dependencies] -chrono = { version = "0.4.20", features = [ "serde" ], default-features = false } +chrono = { version = "0.4.20", features = [ "serde", "clock" ], default-features = false } clap = { version = "4.1.8", features = [ "derive" ] } colored = "2.0" -criterion = { version = "0.4", features = [ "async_tokio", "html_reports" ] } futures = { version = "0.3" } itertools = "0.10.3" rand = "0.8.4" @@ -84,3 +95,7 @@ trybuild = "1.0.71" hdrhistogram = "7" zipf = "7.0.0" core_affinity = "0.5.10" + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +# Rayon (rust data-parallelism library) does not compile on WASM. +criterion = { version = "0.4", features = [ "async_tokio", "html_reports" ] } diff --git a/hydroflow/src/scheduled/net/mod.rs b/hydroflow/src/scheduled/net/mod.rs index 0ecf48ee682f..b6f18af149ad 100644 --- a/hydroflow/src/scheduled/net/mod.rs +++ b/hydroflow/src/scheduled/net/mod.rs @@ -1,3 +1,5 @@ +#![cfg(not(target_arch = "wasm32"))] + //! This module contiains networking code. //! //! ## How Tokio interacts with Hydroflow (Mingwei 2021-12-07) diff --git a/hydroflow/src/scheduled/net/network_vertex.rs b/hydroflow/src/scheduled/net/network_vertex.rs index d16cd33f8ecc..e5668a081869 100644 --- a/hydroflow/src/scheduled/net/network_vertex.rs +++ b/hydroflow/src/scheduled/net/network_vertex.rs @@ -1,3 +1,5 @@ +#![cfg(not(target_arch = "wasm32"))] + use std::collections::HashMap; use futures::{SinkExt, StreamExt}; diff --git a/hydroflow/src/util/mod.rs b/hydroflow/src/util/mod.rs index 2017107a9b86..8b07cb32cf86 100644 --- a/hydroflow/src/util/mod.rs +++ b/hydroflow/src/util/mod.rs @@ -1,8 +1,11 @@ //! Helper utilities for the Hydroflow surface syntax. mod udp; +#[cfg(not(target_arch = "wasm32"))] pub use udp::*; + mod tcp; +#[cfg(not(target_arch = "wasm32"))] pub use tcp::*; #[cfg(unix)] @@ -106,11 +109,13 @@ pub fn ipv4_resolve(addr: &str) -> Result { } } +#[cfg(not(target_arch = "wasm32"))] pub async fn bind_udp_bytes(addr: SocketAddr) -> (UdpSink, UdpStream, SocketAddr) { let socket = tokio::net::UdpSocket::bind(addr).await.unwrap(); udp_bytes(socket) } +#[cfg(not(target_arch = "wasm32"))] pub async fn bind_udp_lines(addr: SocketAddr) -> (UdpLinesSink, UdpLinesStream, SocketAddr) { let socket = tokio::net::UdpSocket::bind(addr).await.unwrap(); udp_lines(socket) diff --git a/hydroflow/src/util/tcp.rs b/hydroflow/src/util/tcp.rs index a1610455adb2..f466b3c4bac4 100644 --- a/hydroflow/src/util/tcp.rs +++ b/hydroflow/src/util/tcp.rs @@ -1,3 +1,5 @@ +#![cfg(not(target_arch = "wasm32"))] + use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; use tokio_util::codec::{ diff --git a/hydroflow/src/util/udp.rs b/hydroflow/src/util/udp.rs index c6891651854b..1dc0e4ea117b 100644 --- a/hydroflow/src/util/udp.rs +++ b/hydroflow/src/util/udp.rs @@ -1,3 +1,5 @@ +#![cfg(not(target_arch = "wasm32"))] + use std::net::SocketAddr; use bytes::Bytes; diff --git a/hydroflow/tests/surface_async.rs b/hydroflow/tests/surface_async.rs index 562c06161827..6662b8993234 100644 --- a/hydroflow/tests/surface_async.rs +++ b/hydroflow/tests/surface_async.rs @@ -1,3 +1,5 @@ +#![cfg(not(target_arch = "wasm32"))] + //! Surface syntax tests of asynchrony and networking. use std::collections::{BTreeSet, HashSet}; diff --git a/hydroflow_datalog/Cargo.toml b/hydroflow_datalog/Cargo.toml index 7695693a0a4c..16e1601d168c 100644 --- a/hydroflow_datalog/Cargo.toml +++ b/hydroflow_datalog/Cargo.toml @@ -12,4 +12,7 @@ quote = "1.0.0" syn = { version = "1.0.0", features = [ "parsing", "extra-traits" ] } proc-macro2 = "1.0.27" proc-macro-crate = "1.1.0" -hydroflow_datalog_core = { path = "../hydroflow_datalog_core" } +# Note: If we ever compile this proc macro crate to WASM (e.g., if we are +# building on a WASM host), we may need to turn diagnostics off for WASM if +# proc_macro2 does not support WASM at that time. +hydroflow_datalog_core = { path = "../hydroflow_datalog_core", features = [ "diagnostics" ] } diff --git a/hydroflow_datalog_core/Cargo.toml b/hydroflow_datalog_core/Cargo.toml index 109102d96516..a6dde58859ae 100644 --- a/hydroflow_datalog_core/Cargo.toml +++ b/hydroflow_datalog_core/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" path = "src/lib.rs" [features] -default = [ "diagnostics" ] +default = [] diagnostics = [ "hydroflow_lang/diagnostics" ] [dependencies] @@ -17,7 +17,7 @@ syn = { version = "1.0.0", features = [ "parsing", "extra-traits" ] } proc-macro2 = "1.0.27" proc-macro-crate = "1.1.0" rust-sitter = "0.2.1" -hydroflow_lang = { path = "../hydroflow_lang", default-features = false } +hydroflow_lang = { path = "../hydroflow_lang" } [build-dependencies] rust-sitter-tool = "0.2.1" diff --git a/hydroflow_internalmacro.d b/hydroflow_internalmacro.d new file mode 100644 index 000000000000..35d6c1f41ccd --- /dev/null +++ b/hydroflow_internalmacro.d @@ -0,0 +1,5 @@ +hydroflow_internalmacro.d: hydroflow_internalmacro/src/lib.rs + +hydroflow_internalmacro/src/lib.rs: + +# env-dep:CARGO_MANIFEST_DIR=/Users/tylerhou/code/hydroflow/hydroflow_internalmacro diff --git a/hydroflow_internalmacro/Cargo.toml b/hydroflow_internalmacro/Cargo.toml index 9316ab8b9df4..e74ca092cdbd 100644 --- a/hydroflow_internalmacro/Cargo.toml +++ b/hydroflow_internalmacro/Cargo.toml @@ -10,4 +10,4 @@ proc-macro = true proc-macro2 = "1.0.0" proc-macro-crate = "1.1.0" quote = "1.0.0" -syn = { version = "1.0.0", features = [ "parsing", "extra-traits" ] } +syn = { version = "1.0.0", features = [ "parsing", "extra-traits", "full" ] } diff --git a/hydroflow_lang/Cargo.toml b/hydroflow_lang/Cargo.toml index 8c96aef40d44..7dab161a5d19 100644 --- a/hydroflow_lang/Cargo.toml +++ b/hydroflow_lang/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [features] -default = [ "diagnostics" ] +default = [] diagnostics = [] [dependencies] diff --git a/hydroflow_macro/Cargo.toml b/hydroflow_macro/Cargo.toml index 1485be195823..5c83c3a0b2a5 100644 --- a/hydroflow_macro/Cargo.toml +++ b/hydroflow_macro/Cargo.toml @@ -7,7 +7,10 @@ edition = "2021" proc-macro = true [dependencies] -hydroflow_lang = { path = "../hydroflow_lang" } +# Note: If we ever compile this proc macro crate to WASM (e.g., if we are +# building on a WASM host), we may need to turn diagnostics off for WASM if +# proc_macro2 still does not support WASM. +hydroflow_lang = { path = "../hydroflow_lang", features = [ "diagnostics" ] } proc-macro2 = "1.0.0" proc-macro-crate = "1.1.0" quote = "1.0.0"