Auto merge of #130040 - onur-ozkan:llvm-tools-with-ci-rustc, r=Kobzol

unify `llvm-bitcode-linker`, `wasm-component-ld` and llvm-tools logics

To use the precompiled `ci-rustc` in CI, we need to install `llvm-bitcode-linker` and LLVM tools into ci-rustc's sysroot. Without them some CI pipelines may fail, as shown [here](https://github.com/rust-lang/rust/pull/122709#issuecomment-2334365988).

Blocker for https://github.com/rust-lang/rust/pull/122709

src/bootstrap/src/core/build_steps/compile.rs

@@ -29,7 +29,7 @@
 use crate::utils::helpers::{
     self, exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, symlink_dir, t, up_to_date,
 };
-use crate::{CLang, Compiler, DependencyType, GitRepo, Mode, LLVM_TOOLS};
+use crate::{CLang, Compiler, DependencyType, GitRepo, Mode};
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Std {
@@ -1908,52 +1908,26 @@ fn run(self, builder: &Builder<'_>) -> Compiler {
         // delegates to the `rust-lld` binary for linking and then runs
         // logic to create the final binary. This is used by the
         // `wasm32-wasip2` target of Rust.
-        if builder.tool_enabled("wasm-component-ld") {
+        dist::maybe_install_wasm_component_ld(
-            let wasm_component_ld_exe =
+            builder,
-                builder.ensure(crate::core::build_steps::tool::WasmComponentLd {
+            build_compiler,
-                    compiler: build_compiler,
+            target_compiler.host,
-                    target: target_compiler.host,
+            &libdir_bin,
-                });
+            false,
-            builder.copy_link(
+        );
-                &wasm_component_ld_exe,
-                &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()),
-            );
-        }
 
-        if builder.config.llvm_enabled(target_compiler.host) {
+        dist::maybe_install_llvm_tools(builder, target_compiler.host, &libdir_bin, false);
-            let llvm::LlvmResult { llvm_config, .. } =
-                builder.ensure(llvm::Llvm { target: target_compiler.host });
-            if !builder.config.dry_run() && builder.config.llvm_tools_enabled {
-                let llvm_bin_dir =
-                    command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout();
-                let llvm_bin_dir = Path::new(llvm_bin_dir.trim());
 
-                // Since we've already built the LLVM tools, install them to the sysroot.
+        let self_contained_bin_dir = libdir_bin.join("self-contained");
-                // This is the equivalent of installing the `llvm-tools-preview` component via
+        t!(fs::create_dir_all(&self_contained_bin_dir));
-                // rustup, and lets developers use a locally built toolchain to
-                // build projects that expect llvm tools to be present in the sysroot
-                // (e.g. the `bootimage` crate).
-                for tool in LLVM_TOOLS {
-                    let tool_exe = exe(tool, target_compiler.host);
-                    let src_path = llvm_bin_dir.join(&tool_exe);
-                    // When using `download-ci-llvm`, some of the tools
-                    // may not exist, so skip trying to copy them.
-                    if src_path.exists() {
-                        builder.copy_link(&src_path, &libdir_bin.join(&tool_exe));
-                    }
-                }
-            }
-        }
 
-        if builder.config.llvm_bitcode_linker_enabled {
+        dist::maybe_install_llvm_bitcode_linker(
-            let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker {
+            builder,
-                compiler: build_compiler,
+            build_compiler,
-                target: target_compiler.host,
+            target_compiler.host,
-                extra_features: vec![],
+            &self_contained_bin_dir,
-            });
+            false,
-            let tool_exe = exe("llvm-bitcode-linker", target_compiler.host);
+        );
-            builder.copy_link(&src_path, &libdir_bin.join(tool_exe));
-        }
 
         // Ensure that `libLLVM.so` ends up in the newly build compiler directory,
         // so that it can be found when the newly built `rustc` is run.

src/bootstrap/src/core/build_steps/dist.rs

@@ -473,11 +473,29 @@ fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
                     );
                 }
             }
-            if builder.tool_enabled("wasm-component-ld") {
+
-                let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
+            let builder_compiler =
-                let ld = exe("wasm-component-ld", compiler.host);
+                builder.compiler_for(builder.top_stage, builder.config.build, compiler.host);
-                builder.copy_link(&src_dir.join(&ld), &dst_dir.join(&ld));
+
-            }
+            maybe_install_wasm_component_ld(
+                builder,
+                builder_compiler,
+                compiler.host,
+                &dst_dir,
+                true,
+            );
+
+            let self_contained_bin_dir = dst_dir.join("self-contained");
+            t!(fs::create_dir_all(&self_contained_bin_dir));
+            maybe_install_llvm_bitcode_linker(
+                builder,
+                builder_compiler,
+                compiler.host,
+                &self_contained_bin_dir,
+                true,
+            );
+
+            maybe_install_llvm_tools(builder, compiler.host, &dst_dir, true);
 
             // Man pages
             t!(fs::create_dir_all(image.join("share/man/man1")));
@@ -2086,6 +2104,85 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection
     }
 }
 
+/// Maybe add LLVM tools to the rustc sysroot.
+pub fn maybe_install_llvm_tools(
+    builder: &Builder<'_>,
+    target: TargetSelection,
+    dst_dir: &Path,
+    dereference_symlinks: bool,
+) {
+    if builder.config.llvm_enabled(target) {
+        let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target });
+        if !builder.config.dry_run() && builder.config.llvm_tools_enabled {
+            let llvm_bin_dir =
+                command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout();
+            let llvm_bin_dir = Path::new(llvm_bin_dir.trim());
+
+            // Since we've already built the LLVM tools, install them to the sysroot.
+            // This is the equivalent of installing the `llvm-tools-preview` component via
+            // rustup, and lets developers use a locally built toolchain to
+            // build projects that expect llvm tools to be present in the sysroot
+            // (e.g. the `bootimage` crate).
+            for tool in LLVM_TOOLS {
+                let tool_exe = exe(tool, target);
+                let src_path = llvm_bin_dir.join(&tool_exe);
+                // When using `download-ci-llvm`, some of the tools
+                // may not exist, so skip trying to copy them.
+                if src_path.exists() {
+                    builder.copy_link_internal(
+                        &src_path,
+                        &dst_dir.join(&tool_exe),
+                        dereference_symlinks,
+                    );
+                }
+            }
+        }
+    }
+}
+
+/// Maybe add `llvm-bitcode-linker` to the rustc sysroot.
+pub fn maybe_install_llvm_bitcode_linker(
+    builder: &Builder<'_>,
+    builder_compiler: Compiler,
+    target: TargetSelection,
+    dst_dir: &Path,
+    dereference_symlinks: bool,
+) {
+    if builder.config.llvm_bitcode_linker_enabled {
+        let llvm_bitcode_linker_exe = builder.ensure(tool::LlvmBitcodeLinker {
+            compiler: builder_compiler,
+            target,
+            extra_features: vec![],
+        });
+
+        builder.copy_link_internal(
+            &llvm_bitcode_linker_exe,
+            &dst_dir.join(llvm_bitcode_linker_exe.file_name().unwrap()),
+            dereference_symlinks,
+        );
+    }
+}
+
+/// Maybe add `wasm-component-ld` to the rustc sysroot.
+pub fn maybe_install_wasm_component_ld(
+    builder: &Builder<'_>,
+    builder_compiler: Compiler,
+    target: TargetSelection,
+    dst_dir: &Path,
+    dereference_symlinks: bool,
+) {
+    if builder.tool_enabled("wasm-component-ld") {
+        let wasm_component_ld_exe =
+            builder.ensure(tool::WasmComponentLd { compiler: builder_compiler, target });
+
+        builder.copy_link_internal(
+            &wasm_component_ld_exe,
+            &dst_dir.join(wasm_component_ld_exe.file_name().unwrap()),
+            dereference_symlinks,
+        );
+    }
+}
+
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub struct LlvmTools {
     pub target: TargetSelection,