Auto merge of #123978 - alexcrichton:update-wasi-toolchain, r=Mark-Simulacrum

Update how WASI toolchains are used in CI and bootstrap

This commit updates how the WASI targets are configured with their toolchain. Long ago a `config.toml` option of `wasi-root` was added to enable building with the WASI files produced by wasi-libc. Additionally for CI testing and release building the Rust toolchain has been using a hard-coded commit of wasi-libc which is bundled with the release of the `wasm32-wasip1` target, for example.

Nowadays though the wasi-sdk project, the C/C++ toolchain for WASI, is the go-to solution for compiling/linking WASI code and contains the more-or-less official releases of wasi-libc. This commit migrates CI to using wasi-sdk releases and additionally updates `bootstrap` to recognize when this is configured. This means that with `$WASI_SDK_PATH` configured there's no further configuration necessary to get a working build. Notably this also works better for the new targets of WASI as well, such as `wasm32-wasip2` and `wasm32-wasip1-threads` where the wasi-sdk release now has libraries for all targets bundled within it.

src/bootstrap/src/core/build_steps/compile.rs

@@ -394,16 +394,13 @@ fn copy_self_contained_objects(
             target_deps.push((libunwind_path, DependencyType::TargetSelfContained));
         }
     } else if target.contains("-wasi") {
-        let srcdir = builder
+        let srcdir = builder.wasi_libdir(target).unwrap_or_else(|| {
-            .wasi_root(target)
-            .unwrap_or_else(|| {
             panic!(
-                    "Target {:?} does not have a \"wasi-root\" key in Config.toml",
+                "Target {:?} does not have a \"wasi-root\" key in Config.toml \
+                    or `$WASI_SDK_PATH` set",
                 target.triple
             )
-            })
+        });
-            .join("lib")
-            .join(target.to_string().replace("-preview1", "").replace("p2", "").replace("p1", ""));
         for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] {
             copy_and_stamp(
                 builder,
@@ -514,12 +511,8 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car
         }
 
         if target.contains("-wasi") {
-            if let Some(p) = builder.wasi_root(target) {
+            if let Some(dir) = builder.wasi_libdir(target) {
-                let root = format!(
+                let root = format!("native={}", dir.to_str().unwrap());
-                    "native={}/lib/{}",
-                    p.to_str().unwrap(),
-                    target.to_string().replace("-preview1", "")
-                );
                 cargo.rustflag("-L").rustflag(&root);
             }
         }

src/bootstrap/src/lib.rs

@@ -1373,9 +1373,24 @@ impl Build {
         self.musl_root(target).map(|root| root.join("lib"))
     }
 
-    /// Returns the sysroot for the wasi target, if defined
+    /// Returns the `lib` directory for the WASI target specified, if
-    fn wasi_root(&self, target: TargetSelection) -> Option<&Path> {
+    /// configured.
-        self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p)
+    ///
+    /// This first consults `wasi-root` as configured in per-target
+    /// configuration, and failing that it assumes that `$WASI_SDK_PATH` is
+    /// set in the environment, and failing that `None` is returned.
+    fn wasi_libdir(&self, target: TargetSelection) -> Option<PathBuf> {
+        let configured =
+            self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p);
+        if let Some(path) = configured {
+            return Some(path.join("lib").join(target.to_string()));
+        }
+        let mut env_root = PathBuf::from(std::env::var_os("WASI_SDK_PATH")?);
+        env_root.push("share");
+        env_root.push("wasi-sysroot");
+        env_root.push("lib");
+        env_root.push(target.to_string());
+        Some(env_root)
     }
 
     /// Returns `true` if this is a no-std `target`, if defined

src/bootstrap/src/utils/cc_detect.rs

@@ -47,7 +47,7 @@ fn cc2ar(cc: &Path, target: TargetSelection) -> Option<PathBuf> {
         Some(PathBuf::from("ar"))
     } else if target.contains("vxworks") {
         Some(PathBuf::from("wr-ar"))
-    } else if target.contains("android") {
+    } else if target.contains("android") || target.contains("-wasi") {
         Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar")))
     } else {
         let parent = cc.parent().unwrap();
@@ -223,6 +223,16 @@ fn default_compiler(
             }
         }
 
+        t if t.contains("-wasi") => {
+            let root = PathBuf::from(std::env::var_os("WASI_SDK_PATH")?);
+            let compiler = match compiler {
+                Language::C => format!("{t}-clang"),
+                Language::CPlusPlus => format!("{t}-clang++"),
+            };
+            let compiler = root.join("bin").join(compiler);
+            Some(compiler)
+        }
+
         _ => None,
     }
 }

src/ci/docker/host-x86_64/dist-various-2/Dockerfile

@@ -85,11 +85,9 @@ RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc sun
 COPY host-x86_64/dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/
 RUN /tmp/build-x86_64-fortanix-unknown-sgx-toolchain.sh
 
-COPY host-x86_64/dist-various-2/build-wasi-toolchain.sh /tmp/
+RUN curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0-linux.tar.gz | \
-RUN /tmp/build-wasi-toolchain.sh
+  tar -xz
-
+ENV WASI_SDK_PATH=/tmp/wasi-sdk-22.0
-COPY host-x86_64/dist-various-2/build-wasi-threads-toolchain.sh /tmp/
-RUN /tmp/build-wasi-threads-toolchain.sh
 
 COPY scripts/freebsd-toolchain.sh /tmp/
 RUN /tmp/freebsd-toolchain.sh i686
@@ -136,9 +134,6 @@ ENV TARGETS=$TARGETS,x86_64-unknown-uefi
 RUN ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/include/asm
 
 ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --enable-llvm-bitcode-linker --disable-docs \
-  --set target.wasm32-wasi.wasi-root=/wasm32-wasip1 \
-  --set target.wasm32-wasip1.wasi-root=/wasm32-wasip1 \
-  --set target.wasm32-wasip1-threads.wasi-root=/wasm32-wasip1-threads \
   --musl-root-armv7=/musl-armv7
 
 ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS

src/ci/docker/host-x86_64/dist-various-2/build-wasi-threads-toolchain.sh

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-# Originally from https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz
-curl https://ci-mirrors.rust-lang.org/rustc/2023-05-17-clang%2Bllvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz | \
-  tar xJf -
-bin="$PWD/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04/bin"
-
-git clone https://github.com/WebAssembly/wasi-libc
-
-cd wasi-libc
-git reset --hard ec4566beae84e54952637f0bf61bee4b4cacc087
-make -j$(nproc) \
-    CC="$bin/clang" \
-    NM="$bin/llvm-nm" \
-    AR="$bin/llvm-ar" \
-    THREAD_MODEL=posix \
-    INSTALL_DIR=/wasm32-wasip1-threads \
-    install
-
-cd ..
-rm -rf wasi-libc
-rm -rf clang+llvm*

src/ci/docker/host-x86_64/dist-various-2/build-wasi-toolchain.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-# Originally from https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz
-curl https://ci-mirrors.rust-lang.org/rustc/2023-05-17-clang%2Bllvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04.tar.xz | \
-  tar xJf -
-bin="$PWD/clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04/bin"
-
-git clone https://github.com/WebAssembly/wasi-libc
-
-cd wasi-libc
-git reset --hard ec4566beae84e54952637f0bf61bee4b4cacc087
-make -j$(nproc) \
-    CC="$bin/clang" \
-    NM="$bin/llvm-nm" \
-    AR="$bin/llvm-ar" \
-    INSTALL_DIR=/wasm32-wasip1 \
-    install
-
-cd ..
-rm -rf wasi-libc
-rm -rf clang+llvm*

src/ci/docker/host-x86_64/test-various/Dockerfile

@@ -39,14 +39,14 @@ WORKDIR /
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-COPY host-x86_64/dist-various-2/build-wasi-toolchain.sh /tmp/
+RUN curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0-linux.tar.gz | \
-RUN /tmp/build-wasi-toolchain.sh
+  tar -xz
+ENV WASI_SDK_PATH=/wasi-sdk-22.0
 
 ENV RUST_CONFIGURE_ARGS \
   --musl-root-x86_64=/usr/local/x86_64-linux-musl \
   --set build.nodejs=/node-v18.12.0-linux-x64/bin/node \
-  --set rust.lld \
+  --set rust.lld
-  --set target.wasm32-wasip1.wasi-root=/wasm32-wasip1
 
 # Some run-make tests have assertions about code size, and enabling debug
 # assertions in libstd causes the binary to be much bigger than it would
@@ -68,9 +68,6 @@ ENV WASM_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $WASM_T
   tests/codegen \
   tests/assembly \
   library/core
-ENV CC_wasm32_wasip1=clang-11 \
-    CFLAGS_wasm32_wasip1="--sysroot /wasm32-wasip1" \
-    AR_wasm32_wasip1=llvm-ar-11
 
 ENV NVPTX_TARGETS=nvptx64-nvidia-cuda
 ENV NVPTX_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $NVPTX_TARGETS \

src/doc/rustc/src/platform-support/wasm32-wasip1.md

@@ -73,19 +73,21 @@ be used instead.
 
 ## Building the target
 
-To build this target a compiled version of [`wasi-libc`] is required to be
+To build this target first acquire a copy of
-present at build time. This can be installed through
+[`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk/). At this time version 22
-[`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk) as well. This is the
+is the minimum needed.
-configured with:
 
-```toml
+Next configure the `WASI_SDK_PATH` environment variable to point to where this
-[target.wasm32-wasip1]
+is installed. For example:
-wasi-root = ".../wasi-libc/sysroot"
+
+```text
+export WASI_SDK_PATH=/path/to/wasi-sdk-22.0
 ```
 
-Additionally users will need to enable LLD when building Rust from source as
+Next be sure to enable LLD when building Rust from source as LLVM's `wasm-ld`
-LLVM's `wasm-ld` driver for LLD is required when linking WebAssembly code
+driver for LLD is required when linking WebAssembly code together. Rust's build
-together.
+system will automatically pick up any necessary binaries and programs from
+`WASI_SDK_PATH`.
 
 ## Building Rust programs
 
@@ -112,8 +114,10 @@ This target can be cross-compiled from any hosts.
 
 ## Testing
 
-Currently the WASI target is not tested in rust-lang/rust CI. This means that
+This target is tested in rust-lang/rust CI on all merges. A subset of tests are
-tests in the repository are not guaranteed to pass. This is theoretically
+run in the `test-various` builder such as the UI tests and libcore tests. This
-possibly by installing a standalone WebAssembly runtime and using it as a
+can be tested locally, for example, with:
-"runner" for all tests, but there are various failures that will need to be
+
-waded through to adjust tests to work on the WASI target.
+```text
+./x.py test --target wasm32-wasip1 tests/ui
+```

src/doc/rustc/src/platform-support/wasm32-wasip2.md

@@ -22,9 +22,34 @@ This target is cross-compiled. The target supports `std` fully.
 
 ## Platform requirements
 
-The WebAssembly runtime should support the wasi preview 2 API set.
+The WebAssembly runtime should support the wasi preview 2 API set. Runtimes also
+are required to support components since this target outputs a component as
+opposed to a core wasm module. As of the time of this writing Wasmtime 17 and
+above is able to run this target natively with no extra flags.
 
-This target is not a stable target. This means that there are only a few engines
+## Building the target
-which implement wasi preview 2, for example:
 
-* Wasmtime - `-W component-model`
+To build this target first acquire a copy of
+[`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk/). At this time version 22
+is the minimum needed.
+
+Next configure the `WASI_SDK_PATH` environment variable to point to where this
+is installed. For example:
+
+```text
+export WASI_SDK_PATH=/path/to/wasi-sdk-22.0
+```
+
+Next be sure to enable LLD when building Rust from source as LLVM's `wasm-ld`
+driver for LLD is required when linking WebAssembly code together. Rust's build
+system will automatically pick up any necessary binaries and programs from
+`WASI_SDK_PATH`.
+
+## Testing
+
+This target is not tested in CI at this time. Locally it can be tested with a
+`wasmtime` binary in `PATH` like so:
+
+```text
+./x.py test --target wasm32-wasip2 tests/ui
+```