From 8bb9d30a02552cb04fe9d7f7b57ee378fbe6f246 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 15 Apr 2024 09:27:34 -0700 Subject: [PATCH] Update how WASI toolchains are used in CI and bootstrap This commit updates how the WASI targets are configured with their toolchain. Long ago a `config.toml` option of `wasi-root` was added to enable building with the WASI files produced by wasi-libc. Additionally for CI testing and release building the Rust toolchain has been using a hard-coded commit of wasi-libc which is bundled with the release of the `wasm32-wasip1` target, for example. Nowadays though the wasi-sdk project, the C/C++ toolchain for WASI, is the go-to solution for compiling/linking WASI code and contains the more-or-less official releases of wasi-libc. This commit migrates CI to using wasi-sdk releases and additionally updates `bootstrap` to recognize when this is configured. This means that with `$WASI_SDK_PATH` configured there's no further configuration necessary to get a working build. Notably this also works better for the new targets of WASI as well, such as `wasm32-wasip2` and `wasm32-wasip1-threads` where the wasi-sdk release now has libraries for all targets bundled within it. --- src/bootstrap/src/core/build_steps/compile.rs | 25 +++++--------- src/bootstrap/src/lib.rs | 21 ++++++++++-- src/bootstrap/src/utils/cc_detect.rs | 12 ++++++- .../host-x86_64/dist-various-2/Dockerfile | 11 ++---- .../build-wasi-threads-toolchain.sh | 24 ------------- .../dist-various-2/build-wasi-toolchain.sh | 23 ------------- .../host-x86_64/test-various/Dockerfile | 11 +++--- .../src/platform-support/wasm32-wasip1.md | 34 +++++++++++-------- .../src/platform-support/wasm32-wasip2.md | 33 +++++++++++++++--- 9 files changed, 93 insertions(+), 101 deletions(-) delete mode 100755 src/ci/docker/host-x86_64/dist-various-2/build-wasi-threads-toolchain.sh delete mode 100755 src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 9420e40d6c2..50149d370d5 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -394,16 +394,13 @@ fn copy_self_contained_objects( target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); } } else if target.contains("-wasi") { - let srcdir = builder - .wasi_root(target) - .unwrap_or_else(|| { - panic!( - "Target {:?} does not have a \"wasi-root\" key in Config.toml", - target.triple - ) - }) - .join("lib") - .join(target.to_string().replace("-preview1", "").replace("p2", "").replace("p1", "")); + let srcdir = builder.wasi_libdir(target).unwrap_or_else(|| { + panic!( + "Target {:?} does not have a \"wasi-root\" key in Config.toml \ + or `$WASI_SDK_PATH` set", + target.triple + ) + }); for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { copy_and_stamp( builder, @@ -514,12 +511,8 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car } if target.contains("-wasi") { - if let Some(p) = builder.wasi_root(target) { - let root = format!( - "native={}/lib/{}", - p.to_str().unwrap(), - target.to_string().replace("-preview1", "") - ); + if let Some(dir) = builder.wasi_libdir(target) { + let root = format!("native={}", dir.to_str().unwrap()); cargo.rustflag("-L").rustflag(&root); } } diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 1a8322c0dfd..027bcac045c 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -1351,9 +1351,24 @@ impl Build { self.musl_root(target).map(|root| root.join("lib")) } - /// Returns the sysroot for the wasi target, if defined - fn wasi_root(&self, target: TargetSelection) -> Option<&Path> { - self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p) + /// Returns the `lib` directory for the WASI target specified, if + /// configured. + /// + /// This first consults `wasi-root` as configured in per-target + /// configuration, and failing that it assumes that `$WASI_SDK_PATH` is + /// set in the environment, and failing that `None` is returned. + fn wasi_libdir(&self, target: TargetSelection) -> Option { + let configured = + self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p); + if let Some(path) = configured { + return Some(path.join("lib").join(target.to_string())); + } + let mut env_root = PathBuf::from(std::env::var_os("WASI_SDK_PATH")?); + env_root.push("share"); + env_root.push("wasi-sysroot"); + env_root.push("lib"); + env_root.push(target.to_string()); + Some(env_root) } /// Returns `true` if this is a no-std `target`, if defined diff --git a/src/bootstrap/src/utils/cc_detect.rs b/src/bootstrap/src/utils/cc_detect.rs index 3ba4e0cb686..7e59b7f6f57 100644 --- a/src/bootstrap/src/utils/cc_detect.rs +++ b/src/bootstrap/src/utils/cc_detect.rs @@ -47,7 +47,7 @@ fn cc2ar(cc: &Path, target: TargetSelection) -> Option { Some(PathBuf::from("ar")) } else if target.contains("vxworks") { Some(PathBuf::from("wr-ar")) - } else if target.contains("android") { + } else if target.contains("android") || target.contains("-wasi") { Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar"))) } else { let parent = cc.parent().unwrap(); @@ -223,6 +223,16 @@ fn default_compiler( } } + t if t.contains("-wasi") => { + let root = PathBuf::from(std::env::var_os("WASI_SDK_PATH")?); + let compiler = match compiler { + Language::C => format!("{t}-clang"), + Language::CPlusPlus => format!("{t}-clang++"), + }; + let compiler = root.join("bin").join(compiler); + Some(compiler) + } + _ => None, } } diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index 9b15bb3530b..bb6254942cb 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -85,11 +85,9 @@ RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc sun COPY host-x86_64/dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/ RUN /tmp/build-x86_64-fortanix-unknown-sgx-toolchain.sh -COPY host-x86_64/dist-various-2/build-wasi-toolchain.sh /tmp/ -RUN /tmp/build-wasi-toolchain.sh - -COPY host-x86_64/dist-various-2/build-wasi-threads-toolchain.sh /tmp/ -RUN /tmp/build-wasi-threads-toolchain.sh +RUN curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0-linux.tar.gz | \ + tar -xz +ENV WASI_SDK_PATH=/tmp/wasi-sdk-22.0 COPY scripts/freebsd-toolchain.sh /tmp/ RUN /tmp/freebsd-toolchain.sh i686 @@ -136,9 +134,6 @@ ENV TARGETS=$TARGETS,x86_64-unknown-uefi RUN ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/include/asm ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --enable-llvm-bitcode-linker --disable-docs \ - --set target.wasm32-wasi.wasi-root=/wasm32-wasip1 \ - --set target.wasm32-wasip1.wasi-root=/wasm32-wasip1 \ - --set target.wasm32-wasip1-threads.wasi-root=/wasm32-wasip1-threads \ --musl-root-armv7=/musl-armv7 ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-wasi-threads-toolchain.sh b/src/ci/docker/host-x86_64/dist-various-2/build-wasi-threads-toolchain.sh deleted file mode 100755 index 8f802eeaa8c..00000000000 --- a/src/ci/docker/host-x86_64/dist-various-2/build-wasi-threads-toolchain.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - -set -ex - -# Originally from https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz -curl https://ci-mirrors.rust-lang.org/rustc/2023-05-17-clang%2Bllvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz | \ - tar xJf - -bin="$PWD/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04/bin" - -git clone https://github.com/WebAssembly/wasi-libc - -cd wasi-libc -git reset --hard ec4566beae84e54952637f0bf61bee4b4cacc087 -make -j$(nproc) \ - CC="$bin/clang" \ - NM="$bin/llvm-nm" \ - AR="$bin/llvm-ar" \ - THREAD_MODEL=posix \ - INSTALL_DIR=/wasm32-wasip1-threads \ - install - -cd .. -rm -rf wasi-libc -rm -rf clang+llvm* diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh deleted file mode 100755 index a2447494078..00000000000 --- a/src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -set -ex - -# Originally from https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz -curl https://ci-mirrors.rust-lang.org/rustc/2023-05-17-clang%2Bllvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz | \ - tar xJf - -bin="$PWD/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04/bin" - -git clone https://github.com/WebAssembly/wasi-libc - -cd wasi-libc -git reset --hard ec4566beae84e54952637f0bf61bee4b4cacc087 -make -j$(nproc) \ - CC="$bin/clang" \ - NM="$bin/llvm-nm" \ - AR="$bin/llvm-ar" \ - INSTALL_DIR=/wasm32-wasip1 \ - install - -cd .. -rm -rf wasi-libc -rm -rf clang+llvm* diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile index 4de9afdb171..8875e226fd8 100644 --- a/src/ci/docker/host-x86_64/test-various/Dockerfile +++ b/src/ci/docker/host-x86_64/test-various/Dockerfile @@ -39,14 +39,14 @@ WORKDIR / COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -COPY host-x86_64/dist-various-2/build-wasi-toolchain.sh /tmp/ -RUN /tmp/build-wasi-toolchain.sh +RUN curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0-linux.tar.gz | \ + tar -xz +ENV WASI_SDK_PATH=/wasi-sdk-22.0 ENV RUST_CONFIGURE_ARGS \ --musl-root-x86_64=/usr/local/x86_64-linux-musl \ --set build.nodejs=/node-v18.12.0-linux-x64/bin/node \ - --set rust.lld \ - --set target.wasm32-wasip1.wasi-root=/wasm32-wasip1 + --set rust.lld # Some run-make tests have assertions about code size, and enabling debug # assertions in libstd causes the binary to be much bigger than it would @@ -68,9 +68,6 @@ ENV WASM_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $WASM_T tests/codegen \ tests/assembly \ library/core -ENV CC_wasm32_wasip1=clang-11 \ - CFLAGS_wasm32_wasip1="--sysroot /wasm32-wasip1" \ - AR_wasm32_wasip1=llvm-ar-11 ENV NVPTX_TARGETS=nvptx64-nvidia-cuda ENV NVPTX_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $NVPTX_TARGETS \ diff --git a/src/doc/rustc/src/platform-support/wasm32-wasip1.md b/src/doc/rustc/src/platform-support/wasm32-wasip1.md index a1ca81d1fec..77efa9c3282 100644 --- a/src/doc/rustc/src/platform-support/wasm32-wasip1.md +++ b/src/doc/rustc/src/platform-support/wasm32-wasip1.md @@ -73,19 +73,21 @@ be used instead. ## Building the target -To build this target a compiled version of [`wasi-libc`] is required to be -present at build time. This can be installed through -[`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk) as well. This is the -configured with: +To build this target first acquire a copy of +[`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk/). At this time version 22 +is the minimum needed. -```toml -[target.wasm32-wasip1] -wasi-root = ".../wasi-libc/sysroot" +Next configure the `WASI_SDK_PATH` environment variable to point to where this +is installed. For example: + +```text +export WASI_SDK_PATH=/path/to/wasi-sdk-22.0 ``` -Additionally users will need to enable LLD when building Rust from source as -LLVM's `wasm-ld` driver for LLD is required when linking WebAssembly code -together. +Next be sure to enable LLD when building Rust from source as LLVM's `wasm-ld` +driver for LLD is required when linking WebAssembly code together. Rust's build +system will automatically pick up any necessary binaries and programs from +`WASI_SDK_PATH`. ## Building Rust programs @@ -112,8 +114,10 @@ This target can be cross-compiled from any hosts. ## Testing -Currently the WASI target is not tested in rust-lang/rust CI. This means that -tests in the repository are not guaranteed to pass. This is theoretically -possibly by installing a standalone WebAssembly runtime and using it as a -"runner" for all tests, but there are various failures that will need to be -waded through to adjust tests to work on the WASI target. +This target is tested in rust-lang/rust CI on all merges. A subset of tests are +run in the `test-various` builder such as the UI tests and libcore tests. This +can be tested locally, for example, with: + +```text +./x.py test --target wasm32-wasip1 tests/ui +``` diff --git a/src/doc/rustc/src/platform-support/wasm32-wasip2.md b/src/doc/rustc/src/platform-support/wasm32-wasip2.md index 4466d2c0840..a385600cc22 100644 --- a/src/doc/rustc/src/platform-support/wasm32-wasip2.md +++ b/src/doc/rustc/src/platform-support/wasm32-wasip2.md @@ -22,9 +22,34 @@ This target is cross-compiled. The target supports `std` fully. ## Platform requirements -The WebAssembly runtime should support the wasi preview 2 API set. +The WebAssembly runtime should support the wasi preview 2 API set. Runtimes also +are required to support components since this target outputs a component as +opposed to a core wasm module. As of the time of this writing Wasmtime 17 and +above is able to run this target natively with no extra flags. -This target is not a stable target. This means that there are only a few engines -which implement wasi preview 2, for example: +## Building the target -* Wasmtime - `-W component-model` +To build this target first acquire a copy of +[`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk/). At this time version 22 +is the minimum needed. + +Next configure the `WASI_SDK_PATH` environment variable to point to where this +is installed. For example: + +```text +export WASI_SDK_PATH=/path/to/wasi-sdk-22.0 +``` + +Next be sure to enable LLD when building Rust from source as LLVM's `wasm-ld` +driver for LLD is required when linking WebAssembly code together. Rust's build +system will automatically pick up any necessary binaries and programs from +`WASI_SDK_PATH`. + +## Testing + +This target is not tested in CI at this time. Locally it can be tested with a +`wasmtime` binary in `PATH` like so: + +```text +./x.py test --target wasm32-wasip2 tests/ui +```