Compile rustc+LLVM to wasm (#6)

* Compile rustc for wasm15 with llvm * Some fixes * Fix llvm-project submodule * Another fix * Fix linking libc++ on wasi * Disable linker optimizations for wasm This is really slow when producing a 156MB rustc.wasm output. * Better linking of libc++ * Faster LLVM compiles * Remove some unnecessary configs * Some preparation for once LLVM can compile for wasm32-wasip1 without threads * Rustfmt * Simplify LLVM build code a bit --------- Co-authored-by: bjorn3 <[email protected]>
bjorn3 · Sep 25, 2024 · e19cccd · e19cccd
1 parent 214c2df
commit e19cccd
Show file tree

Hide file tree

Showing 14 changed files with 283 additions and 16 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -32,8 +32,8 @@
 	shallow = true
 [submodule "src/llvm-project"]
 	path = src/llvm-project
-	url = https://github.com/rust-lang/llvm-project.git
-	branch = rustc/19.1-2024-07-30
+	url = https://github.com/YoWASP/llvm-project
+	branch = main+wasm
 	shallow = true
 [submodule "src/doc/embedded-book"]
 	path = src/doc/embedded-book

diff --git a/Cargo.lock b/Cargo.lock
@@ -407,9 +407,12 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.0.105"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cfg-if"

diff --git a/comment.txt b/comment.txt
@@ -16,3 +16,25 @@ $ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.* dist/lib/rustlib/x86_64-u
 $ ./rust_out
 Hello World!
 ```
+
+use LLVM
+Install example:
+WASI_SDK_PATH=`pwd`/wasi-sdk-24.0-x86_64-linux WASI_SYSROOT=`pwd`/wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot ./x.py install
+
+If you just want to run it, https://github.com/oligamiq/rust_wasm/tree/main/rustc_llvm
+```
+$ mkdir tmp
+$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps
+$ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.*.o dist/lib/rustlib/x86_64-unknown-linux-gnu/lib/lib*.rlib -o rust_out
+$ ./rust_out
+Hello World!
+```
+
+to Wasi
+```
+$ mkdir tmp
+$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps
+$ wasi-sdk-24.0-x86_64-linux/bin/wasm-ld --shared-memory --max-memory=1073741824 --import-memory --export __main_void -z stack-size=1048576 --stack-first --allow-undefined --no-demangle --import-memory --export-memory --shared-memory dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained/crt1-command.o tmp/rust_out.*.o dist/lib/rustlib/wasm32-wasip1-threads/lib/lib*.rlib -L dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained -lc -o rust_out.wasm
+$ wasmtime run -Sthreads=y rust_out.wasm
+Hello World!
+```
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 ar_archive_writer = "0.4.2"
 arrayvec = { version = "0.7", default-features = false }
 bitflags = "2.4.1"
-cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013
+cc = "1.1.15"
 either = "1.5.0"
 itertools = "0.12"
 jobserver = "0.1.28"

diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -657,7 +657,7 @@ impl<'a> Linker for GccLinker<'a> {
         if self.sess.opts.optimize == config::OptLevel::Default
             || self.sess.opts.optimize == config::OptLevel::Aggressive
         {
-            self.link_arg("-O1");
+            //self.link_arg("-O1");
         }
     }
 

diff --git a/compiler/rustc_fs_util/src/lib.rs b/compiler/rustc_fs_util/src/lib.rs
@@ -1,4 +1,4 @@
-#[cfg(any(unix, windows))]
+#[cfg(any(unix, windows, target_os = "wasi"))]
 use std::ffi::CString;
 use std::path::{absolute, Path, PathBuf};
 use std::{fs, io};
@@ -84,7 +84,7 @@ pub fn path_to_c_string(p: &Path) -> CString {
     let p: &OsStr = p.as_ref();
     CString::new(p.as_bytes()).unwrap()
 }
-#[cfg(windows)]
+#[cfg(any(windows, target_os = "wasi"))]
 pub fn path_to_c_string(p: &Path) -> CString {
     CString::new(p.to_str().unwrap()).unwrap()
 }

diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
@@ -103,10 +103,8 @@ fn run_in_thread_with_globals<F: FnOnce(CurrentGcx) -> R + Send, R: Send>(
         // name contains null bytes.
         let r = builder
             .spawn_scoped(s, move || {*/
-                rustc_span::create_session_globals_then(edition, Some(sm_inputs), || {
-                    f(CurrentGcx::new())
-                })
-            /*})
+    rustc_span::create_session_globals_then(edition, Some(sm_inputs), || f(CurrentGcx::new()))
+    /*})
             .unwrap()
             .join();
 

diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml
@@ -10,5 +10,5 @@ libc = "0.2.73"
 
 [build-dependencies]
 # tidy-alphabetical-start
-cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013
+cc = "1.1.15"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
@@ -102,6 +102,10 @@ fn output(cmd: &mut Command) -> String {
 }
 
 fn main() {
+    if env::var("TARGET").expect("TARGET was not set").contains("wasi") {
+        std::env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set");
+    }
+
     for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) {
         println!("cargo:rustc-check-cfg=cfg(llvm_component,values(\"{component}\"))");
     }
@@ -201,6 +205,15 @@ fn main() {
         cfg.define("NDEBUG", None);
     }
 
+    if target.contains("wasi") {
+        // ref src/bootstrap/src/core/build_steps/llvm.rs
+
+        let wasi_sysroot = env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set");
+        cfg.compiler(format!("{wasi_sysroot}/../../bin/{target}-clang++"));
+        cfg.flag("-pthread");
+        cfg.flag("-D_WASI_EMULATED_MMAN");
+    }
+
     rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper"));
     cfg.file("llvm-wrapper/PassWrapper.cpp")
         .file("llvm-wrapper/RustWrapper.cpp")
@@ -363,6 +376,7 @@ fn main() {
         || target.contains("freebsd")
         || target.contains("windows-gnullvm")
         || target.contains("aix")
+        || target.contains("wasi")
     {
         "c++"
     } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() {

diff --git a/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs b/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs
@@ -17,7 +17,13 @@ pub(crate) fn target() -> Target {
 
     options.add_pre_link_args(
         LinkerFlavor::WasmLld(Cc::No),
-        &["--import-memory", "--export-memory", "--shared-memory", "-Wl,--max-memory=1073741824"],
+        &[
+            "--import-memory",
+            "--export-memory",
+            "--shared-memory",
+            "--max-memory=1073741824",
+            "-lwasi-emulated-mman",
+        ],
     );
     options.add_pre_link_args(
         LinkerFlavor::WasmLld(Cc::Yes),
@@ -27,6 +33,7 @@ pub(crate) fn target() -> Target {
             "-Wl,--export-memory,",
             "-Wl,--shared-memory",
             "-Wl,--max-memory=1073741824",
+            "-lwasi-emulated-mman",
         ],
     );
 

diff --git a/config.llvm.toml b/config.llvm.toml
@@ -0,0 +1,45 @@
+# Includes one of the default files in src/bootstrap/defaults
+profile = "compiler"
+change-id = 9999999
+
+[rust]
+codegen-backends = ["llvm"]
+deny-warnings = false
+llvm-bitcode-linker = false
+debug = false
+debuginfo-level = 0
+
+[llvm]
+cflags = "-march=native"
+cxxflags = "-march=native"
+static-libstdcpp = true
+ninja = false
+download-ci-llvm = false
+link-shared = false
+targets = "WebAssembly;X86"
+experimental-targets = ""
+
+[build]
+docs = false
+extended = false
+tools = []
+host = ["wasm32-wasip1-threads"]
+target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1-threads"]
+cargo-native-static = true
+
+[install]
+prefix = "dist"
+sysconfdir = "etc"
+
+[target.'wasm32-wasip1']
+wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot"
+linker = "wasi-sdk-24.0-x86_64-linux/bin/clang"
+codegen-backends = ["llvm"]
+
+[target.'wasm32-wasip1-threads']
+wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot"
+linker = "wasi-sdk-24.0-x86_64-linux/bin/clang"
+codegen-backends = ["llvm"]
+
+[target.'x86_64-unknown-linux-gnu']
+cc = "gcc"
diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
@@ -1264,7 +1264,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect
             &builder.cxx(target).unwrap(),
             target,
             CLang::Cxx,
-            "libstdc++.a",
+            if target.contains("wasi") { "libc++.a" } else { "libstdc++.a" },
         );
         cargo.env("LLVM_STATIC_STDCPP", file);
     }