Auto merge of #120348 - bjorn3:per_target_backend_selection, r=albertlarsan68

Support configuring the set of codegen backends to build per host triple

This allows building the compiler itself with one backend while using another backend at runtime. For example this allows compiling rustc to wasm using LLVM, while using Cranelift at runtime to produce actual code. Cranelift can't compile to wasm, but is perfectly capable of running on wasm. LLVM can compile to wasm, but can't run on wasm. [^1]

[^1]: The prototype of this still requires a couple of other patches.

config.example.toml

@@ -829,6 +829,11 @@
 # target triples containing `-none`, `nvptx`, `switch`, or `-uefi`.
 #no-std = <platform-specific> (bool)
 
+# This is an array of the codegen backends that will be compiled a rustc
+# compiled for this target, overriding the global rust.codegen-backends option.
+# See that option for more info.
+#codegen-backends = rust.codegen-backends (array)
+
 # =============================================================================
 # Distribution options
 #

src/bootstrap/src/core/build_steps/compile.rs

@@ -1039,7 +1039,7 @@ fn run(self, builder: &Builder<'_>) -> u32 {
 pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) {
     cargo
         .arg("--features")
-        .arg(builder.rustc_features(builder.kind))
+        .arg(builder.rustc_features(builder.kind, target))
         .arg("--manifest-path")
         .arg(builder.src.join("compiler/rustc/Cargo.toml"));
 
@@ -1096,7 +1096,7 @@ pub fn rustc_cargo_env(
         cargo.env("CFG_OMIT_GIT_HASH", "1");
     }
 
-    if let Some(backend) = builder.config.default_codegen_backend() {
+    if let Some(backend) = builder.config.default_codegen_backend(target) {
         cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend);
     }
 
@@ -1137,7 +1137,7 @@ pub fn rustc_cargo_env(
     // build. If we are in a check build we still go ahead here presuming we've
     // detected that LLVM is already built and good to go which helps prevent
     // busting caches (e.g. like #71152).
-    if builder.config.llvm_enabled() {
+    if builder.config.llvm_enabled(target) {
         let building_is_expensive =
             crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err();
         // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler
@@ -1281,7 +1281,7 @@ fn needs_codegen_config(run: &RunConfig<'_>) -> bool {
 fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool {
     if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) {
         let mut needs_codegen_backend_config = true;
-        for &backend in &run.builder.config.rust_codegen_backends {
+        for &backend in run.builder.config.codegen_backends(run.target) {
             if path
                 .path
                 .to_str()
@@ -1318,7 +1318,7 @@ fn make_run(run: RunConfig<'_>) {
             return;
         }
 
-        for &backend in &run.builder.config.rust_codegen_backends {
+        for &backend in run.builder.config.codegen_backends(run.target) {
             if backend == "llvm" {
                 continue; // Already built as part of rustc
             }
@@ -1425,7 +1425,7 @@ fn copy_codegen_backends_to_sysroot(
         return;
     }
 
-    for backend in builder.config.rust_codegen_backends.iter() {
+    for backend in builder.config.codegen_backends(target) {
         if backend == "llvm" {
             continue; // Already built as part of rustc
         }
@@ -1732,7 +1732,7 @@ fn run(self, builder: &Builder<'_>) -> Compiler {
         // to not fail while linking the artifacts.
         build_compiler.stage = actual_stage;
 
-        for &backend in builder.config.rust_codegen_backends.iter() {
+        for &backend in builder.config.codegen_backends(target_compiler.host) {
             if backend == "llvm" {
                 continue; // Already built as part of rustc
             }
@@ -1817,7 +1817,7 @@ fn run(self, builder: &Builder<'_>) -> Compiler {
             }
         }
 
-        if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) {
+        if builder.config.llvm_enabled(target_compiler.host) {
             let llvm::LlvmResult { llvm_config, .. } =
                 builder.ensure(llvm::Llvm { target: target_compiler.host });
             if !builder.config.dry_run() && builder.config.llvm_tools_enabled {

src/bootstrap/src/core/build_steps/dist.rs

@@ -1278,7 +1278,7 @@ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        for &backend in &run.builder.config.rust_codegen_backends {
+        for &backend in run.builder.config.codegen_backends(run.target) {
             if backend == "llvm" {
                 continue; // Already built as part of rustc
             }
@@ -1302,7 +1302,7 @@ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
             return None;
         }
 
-        if !builder.config.rust_codegen_backends.contains(&self.backend) {
+        if !builder.config.codegen_backends(self.compiler.host).contains(&self.backend) {
             return None;
         }
 

src/bootstrap/src/core/build_steps/test.rs

@@ -1798,7 +1798,7 @@ fn run(self, builder: &Builder<'_>) {
 
         let mut llvm_components_passed = false;
         let mut copts_passed = false;
-        if builder.config.llvm_enabled() {
+        if builder.config.llvm_enabled(compiler.host) {
             let llvm::LlvmResult { llvm_config, .. } =
                 builder.ensure(llvm::Llvm { target: builder.config.build });
             if !builder.config.dry_run() {
@@ -3121,7 +3121,8 @@ fn make_run(run: RunConfig<'_>) {
             return;
         }
 
-        if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) {
+        if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("cranelift"))
+        {
             builder.info("cranelift not in rust.codegen-backends. skipping");
             return;
         }
@@ -3245,7 +3246,7 @@ fn make_run(run: RunConfig<'_>) {
             return;
         }
 
-        if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) {
+        if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("gcc")) {
             builder.info("gcc not in rust.codegen-backends. skipping");
             return;
         }

src/bootstrap/src/core/builder.rs

@@ -1229,7 +1229,7 @@ pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
     /// Note that this returns `None` if LLVM is disabled, or if we're in a
     /// check build or dry-run, where there's no need to build all of LLVM.
     fn llvm_config(&self, target: TargetSelection) -> Option<PathBuf> {
-        if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() {
+        if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run() {
             let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target });
             if llvm_config.is_file() {
                 return Some(llvm_config);
@@ -1991,7 +1991,8 @@ fn cargo(
             };
 
             if let Some(limit) = limit {
-                if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm"
+                if stage == 0
+                    || self.config.default_codegen_backend(target).unwrap_or_default() == "llvm"
                 {
                     rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}"));
                 }

src/bootstrap/src/core/config/config.rs

@@ -577,6 +577,7 @@ pub struct Target {
     pub wasi_root: Option<PathBuf>,
     pub qemu_rootfs: Option<PathBuf>,
     pub no_std: bool,
+    pub codegen_backends: Option<Vec<Interned<String>>>,
 }
 
 impl Target {
@@ -1135,6 +1136,7 @@ struct TomlTarget {
         wasi_root: Option<String> = "wasi-root",
         qemu_rootfs: Option<String> = "qemu-rootfs",
         no_std: Option<bool> = "no-std",
+        codegen_backends: Option<Vec<String>> = "codegen-backends",
     }
 }
 
@@ -1840,6 +1842,24 @@ fn get_table(option: &str) -> Result<TomlConfig, toml::de::Error> {
                 target.profiler = cfg.profiler;
                 target.rpath = cfg.rpath;
 
+                if let Some(ref backends) = cfg.codegen_backends {
+                    let available_backends = vec!["llvm", "cranelift", "gcc"];
+
+                    target.codegen_backends = Some(backends.iter().map(|s| {
+                        if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) {
+                            if available_backends.contains(&backend) {
+                                panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'.");
+                            } else {
+                                println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \
+                                    Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \
+                                    In this case, it would be referred to as '{backend}'.");
+                            }
+                        }
+
+                        INTERNER.intern_str(s)
+                    }).collect());
+                }
+
                 config.target_config.insert(TargetSelection::from_user(&triple), target);
             }
         }
@@ -2222,8 +2242,8 @@ pub fn rpath_enabled(&self, target: TargetSelection) -> bool {
         self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath)
     }
 
-    pub fn llvm_enabled(&self) -> bool {
-        self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm"))
+    pub fn llvm_enabled(&self, target: TargetSelection) -> bool {
+        self.codegen_backends(target).contains(&INTERNER.intern_str("llvm"))
     }
 
     pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind {
@@ -2242,8 +2262,15 @@ pub fn submodules(&self, rust_info: &GitInfo) -> bool {
         self.submodules.unwrap_or(rust_info.is_managed_git_subrepository())
     }
 
-    pub fn default_codegen_backend(&self) -> Option<Interned<String>> {
-        self.rust_codegen_backends.get(0).cloned()
+    pub fn codegen_backends(&self, target: TargetSelection) -> &[Interned<String>] {
+        self.target_config
+            .get(&target)
+            .and_then(|cfg| cfg.codegen_backends.as_deref())
+            .unwrap_or(&self.rust_codegen_backends)
+    }
+
+    pub fn default_codegen_backend(&self, target: TargetSelection) -> Option<Interned<String>> {
+        self.codegen_backends(target).get(0).cloned()
     }
 
     pub fn git_config(&self) -> GitConfig<'_> {

src/bootstrap/src/core/sanity.rs

@@ -16,7 +16,6 @@
 use std::process::Command;
 
 use crate::core::config::Target;
-use crate::utils::cache::INTERNER;
 use crate::utils::helpers::output;
 use crate::Build;
 
@@ -88,19 +87,19 @@ pub fn check(build: &mut Build) {
     }
 
     // We need cmake, but only if we're actually building LLVM or sanitizers.
-    let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm"))
-        && build
-            .hosts
-            .iter()
-            .map(|host| {
-                build
+    let building_llvm = build
+        .hosts
+        .iter()
+        .map(|host| {
+            build.config.llvm_enabled(*host)
+                && build
                     .config
                     .target_config
                     .get(host)
                     .map(|config| config.llvm_config.is_none())
                     .unwrap_or(true)
-            })
-            .any(|build_llvm_ourselves| build_llvm_ourselves);
+        })
+        .any(|build_llvm_ourselves| build_llvm_ourselves);
 
     let need_cmake = building_llvm || build.config.any_sanitizers_to_build();
     if need_cmake && cmd_finder.maybe_have("cmake").is_none() {
@@ -190,13 +189,16 @@ pub fn check(build: &mut Build) {
         if !build.config.dry_run() {
             cmd_finder.must_have(build.cxx(*host).unwrap());
         }
-    }
 
-    if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) {
-        // Externally configured LLVM requires FileCheck to exist
-        let filecheck = build.llvm_filecheck(build.build);
-        if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests {
-            panic!("FileCheck executable {filecheck:?} does not exist");
+        if build.config.llvm_enabled(*host) {
+            // Externally configured LLVM requires FileCheck to exist
+            let filecheck = build.llvm_filecheck(build.build);
+            if !filecheck.starts_with(&build.out)
+                && !filecheck.exists()
+                && build.config.codegen_tests
+            {
+                panic!("FileCheck executable {filecheck:?} does not exist");
+            }
         }
     }
 

src/bootstrap/src/lib.rs

@@ -731,12 +731,12 @@ fn std_features(&self, target: TargetSelection) -> String {
     }
 
     /// Gets the space-separated set of activated features for the compiler.
-    fn rustc_features(&self, kind: Kind) -> String {
+    fn rustc_features(&self, kind: Kind, target: TargetSelection) -> String {
         let mut features = vec![];
         if self.config.jemalloc {
             features.push("jemalloc");
         }
-        if self.config.llvm_enabled() || kind == Kind::Check {
+        if self.config.llvm_enabled(target) || kind == Kind::Check {
             features.push("llvm");
         }
         // keep in sync with `bootstrap/compile.rs:rustc_cargo_env`
@@ -1561,7 +1561,8 @@ fn in_tree_crates(&self, root: &str, target: Option<TargetSelection>) -> Vec<&Cr
                         || target
                             .map(|t| self.config.profiler_enabled(t))
                             .unwrap_or_else(|| self.config.any_profiler_enabled()))
-                    && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled())
+                    && (dep != "rustc_codegen_llvm"
+                        || self.config.hosts.iter().any(|host| self.config.llvm_enabled(*host)))
                 {
                     list.push(*dep);
                 }

src/bootstrap/src/utils/change_tracker.rs

@@ -114,4 +114,9 @@ pub fn find_recent_config_change_ids(current_id: usize) -> Vec<ChangeInfo> {
         severity: ChangeSeverity::Warning,
         summary: "A new `optimized-compiler-builtins` option has been introduced. Whether to build llvm's `compiler-rt` from source is no longer implicitly controlled by git state. See the PR for more details.",
     },
+    ChangeInfo {
+        change_id: 120348,
+        severity: ChangeSeverity::Info,
+        summary: "New option `target.<triple>.codegen-backends` added to config.toml.",
+    },
 ];