From 5b3bc29008643203b4de3ffb4c5b5141039c88e6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 15 Feb 2023 19:32:58 +0000 Subject: [PATCH] Allow testing a cranelift backend built into rustc itself This avoids building cranelift twice in rust's CI and is a lot easier than trying to make building of codegen backends work from within a cargo invocation done by rust's build system. --- build_system/abi_cafe.rs | 15 +++++++---- build_system/build_sysroot.rs | 50 +++++++++++++++++++++++------------ build_system/mod.rs | 29 +++++++++++++++----- build_system/tests.rs | 5 ++-- build_system/usage.txt | 4 +++ scripts/cargo-clif.rs | 26 ++++++++++-------- scripts/rustc-clif.rs | 10 ++++--- scripts/rustdoc-clif.rs | 10 ++++--- 8 files changed, 101 insertions(+), 48 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index 8ffd4852083..9634430d116 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -1,10 +1,8 @@ -use std::path::Path; - use super::build_sysroot; use super::path::Dirs; use super::prepare::GitRepo; use super::utils::{spawn_and_wait, CargoProject, Compiler}; -use super::SysrootKind; +use super::{CodegenBackend, SysrootKind}; static ABI_CAFE_REPO: GitRepo = GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe"); @@ -15,7 +13,7 @@ pub(crate) fn run( channel: &str, sysroot_kind: SysrootKind, dirs: &Dirs, - cg_clif_dylib: &Path, + cg_clif_dylib: &CodegenBackend, rustup_toolchain_name: Option<&str>, bootstrap_host_compiler: &Compiler, ) { @@ -41,7 +39,14 @@ pub(crate) fn run( cmd.arg("--pairs"); cmd.args(pairs); cmd.arg("--add-rustc-codegen-backend"); - cmd.arg(format!("cgclif:{}", cg_clif_dylib.display())); + match cg_clif_dylib { + CodegenBackend::Local(path) => { + cmd.arg(format!("cgclif:{}", path.display())); + } + CodegenBackend::Builtin(name) => { + cmd.arg(format!("cgclif:{name}")); + } + } cmd.current_dir(ABI_CAFE.source_dir(dirs)); spawn_and_wait(cmd); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index d2e712941bf..dab9c77d1a4 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -5,7 +5,7 @@ use super::path::{Dirs, RelPath}; use super::rustc_info::{get_file_name, get_rustc_version}; use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler}; -use super::SysrootKind; +use super::{CodegenBackend, SysrootKind}; static DIST_DIR: RelPath = RelPath::DIST; static BIN_DIR: RelPath = RelPath::DIST.join("bin"); @@ -15,7 +15,7 @@ pub(crate) fn build_sysroot( dirs: &Dirs, channel: &str, sysroot_kind: SysrootKind, - cg_clif_dylib_src: &Path, + cg_clif_dylib_src: &CodegenBackend, bootstrap_host_compiler: &Compiler, rustup_toolchain_name: Option<&str>, target_triple: String, @@ -28,17 +28,23 @@ pub(crate) fn build_sysroot( let is_native = bootstrap_host_compiler.triple == target_triple; - // Copy the backend - let cg_clif_dylib_path = if cfg!(windows) { - // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the - // binaries. - BIN_DIR - } else { - LIB_DIR - } - .to_path(dirs) - .join(cg_clif_dylib_src.file_name().unwrap()); - try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path); + let cg_clif_dylib_path = match cg_clif_dylib_src { + CodegenBackend::Local(src_path) => { + // Copy the backend + let cg_clif_dylib_path = if cfg!(windows) { + // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the + // binaries. + BIN_DIR + } else { + LIB_DIR + } + .to_path(dirs) + .join(src_path.file_name().unwrap()); + try_hard_link(src_path, &cg_clif_dylib_path); + CodegenBackend::Local(cg_clif_dylib_path) + } + CodegenBackend::Builtin(name) => CodegenBackend::Builtin(name.clone()), + }; // Build and copy rustc and cargo wrappers let wrapper_base_name = get_file_name(&bootstrap_host_compiler.rustc, "____", "bin"); @@ -65,6 +71,9 @@ pub(crate) fn build_sysroot( .env("RUSTC", &bootstrap_host_compiler.rustc) .env("RUSTDOC", &bootstrap_host_compiler.rustdoc); } + if let CodegenBackend::Builtin(name) = cg_clif_dylib_src { + build_cargo_wrapper_cmd.env("BUILTIN_BACKEND", name); + } spawn_and_wait(build_cargo_wrapper_cmd); try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name)); } @@ -159,7 +168,7 @@ fn build_sysroot_for_triple( dirs: &Dirs, channel: &str, compiler: Compiler, - cg_clif_dylib_path: &Path, + cg_clif_dylib_path: &CodegenBackend, sysroot_kind: SysrootKind, ) -> SysrootTarget { match sysroot_kind { @@ -167,7 +176,7 @@ fn build_sysroot_for_triple( .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }), SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler), SysrootKind::Clif => { - build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path) + build_clif_sysroot_for_triple(dirs, channel, compiler, cg_clif_dylib_path) } } } @@ -211,7 +220,7 @@ fn build_clif_sysroot_for_triple( dirs: &Dirs, channel: &str, mut compiler: Compiler, - cg_clif_dylib_path: &Path, + cg_clif_dylib_path: &CodegenBackend, ) -> SysrootTarget { match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) { Err(e) => { @@ -249,7 +258,14 @@ fn build_clif_sysroot_for_triple( // Build sysroot let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string(); - rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap())); + match cg_clif_dylib_path { + CodegenBackend::Local(path) => { + rustflags.push_str(&format!(" -Zcodegen-backend={}", path.to_str().unwrap())); + } + CodegenBackend::Builtin(name) => { + rustflags.push_str(&format!(" -Zcodegen-backend={name}")); + } + }; // Necessary for MinGW to find rsbegin.o and rsend.o rustflags .push_str(&format!(" --sysroot {}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap())); diff --git a/build_system/mod.rs b/build_system/mod.rs index d1d6f34dcff..2ca5316408e 100644 --- a/build_system/mod.rs +++ b/build_system/mod.rs @@ -43,6 +43,12 @@ pub(crate) enum SysrootKind { Llvm, } +#[derive(Clone, Debug)] +pub(crate) enum CodegenBackend { + Local(PathBuf), + Builtin(String), +} + pub(crate) fn main() { if env::var("RUST_BACKTRACE").is_err() { env::set_var("RUST_BACKTRACE", "1"); @@ -79,6 +85,7 @@ pub(crate) fn main() { let mut sysroot_kind = SysrootKind::Clif; let mut use_unstable_features = true; let mut frozen = false; + let mut use_backend = None; while let Some(arg) = args.next().as_deref() { match arg { "--out-dir" => { @@ -98,6 +105,12 @@ pub(crate) fn main() { } "--no-unstable-features" => use_unstable_features = false, "--frozen" => frozen = true, + "--use-backend" => { + use_backend = Some(match args.next() { + Some(name) => name, + None => arg_error!("--use-backend requires argument"), + }); + } flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag), arg => arg_error!("Unexpected argument {}", arg), } @@ -164,12 +177,16 @@ pub(crate) fn main() { env::set_var("RUSTC", "rustc_should_be_set_explicitly"); env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly"); - let cg_clif_dylib = build_backend::build_backend( - &dirs, - channel, - &bootstrap_host_compiler, - use_unstable_features, - ); + let cg_clif_dylib = if let Some(name) = use_backend { + CodegenBackend::Builtin(name) + } else { + CodegenBackend::Local(build_backend::build_backend( + &dirs, + channel, + &bootstrap_host_compiler, + use_unstable_features, + )) + }; match command { Command::Prepare => { // Handled above diff --git a/build_system/tests.rs b/build_system/tests.rs index 40bcf1e0c1e..13bf9c70c3e 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -3,11 +3,10 @@ use super::path::{Dirs, RelPath}; use super::prepare::GitRepo; use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler}; -use super::SysrootKind; +use super::{CodegenBackend, SysrootKind}; use std::env; use std::ffi::OsStr; use std::fs; -use std::path::Path; use std::process::Command; static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example"); @@ -215,7 +214,7 @@ pub(crate) fn run_tests( dirs: &Dirs, channel: &str, sysroot_kind: SysrootKind, - cg_clif_dylib: &Path, + cg_clif_dylib: &CodegenBackend, bootstrap_host_compiler: &Compiler, rustup_toolchain_name: Option<&str>, target_triple: String, diff --git a/build_system/usage.txt b/build_system/usage.txt index 1aee083f8df..3bc18a93331 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -29,6 +29,10 @@ OPTIONS: --frozen Require Cargo.lock and cache are up to date + --use-backend NAME + Use the existing Cranelift (or other) backend of the rustc with which we built. + Warning: This is meant for use in rust's CI only! + REQUIREMENTS: * Rustup: By default rustup is used to install the right nightly version. If you don't want to use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and diff --git a/scripts/cargo-clif.rs b/scripts/cargo-clif.rs index 0d5d9f7db01..99b97be24e6 100644 --- a/scripts/cargo-clif.rs +++ b/scripts/cargo-clif.rs @@ -12,17 +12,21 @@ fn main() { let mut rustflags = String::new(); rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend="); - rustflags.push_str( - sysroot - .join(if cfg!(windows) { "bin" } else { "lib" }) - .join( - env::consts::DLL_PREFIX.to_string() - + "rustc_codegen_cranelift" - + env::consts::DLL_SUFFIX, - ) - .to_str() - .unwrap(), - ); + if let Some(name) = option_env!("BUILTIN_BACKEND") { + rustflags.push_str(name); + } else { + rustflags.push_str( + sysroot + .join(if cfg!(windows) { "bin" } else { "lib" }) + .join( + env::consts::DLL_PREFIX.to_string() + + "rustc_codegen_cranelift" + + env::consts::DLL_SUFFIX, + ) + .to_str() + .unwrap(), + ); + } rustflags.push_str(" --sysroot "); rustflags.push_str(sysroot.to_str().unwrap()); env::set_var("RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + &rustflags); diff --git a/scripts/rustc-clif.rs b/scripts/rustc-clif.rs index df94b80b34f..33d51bdddea 100644 --- a/scripts/rustc-clif.rs +++ b/scripts/rustc-clif.rs @@ -19,9 +19,13 @@ fn main() { let mut args = vec![]; args.push(OsString::from("-Cpanic=abort")); args.push(OsString::from("-Zpanic-abort-tests")); - let mut codegen_backend_arg = OsString::from("-Zcodegen-backend="); - codegen_backend_arg.push(cg_clif_dylib_path); - args.push(codegen_backend_arg); + if let Some(name) = option_env!("BUILTIN_BACKEND") { + args.push(OsString::from(format!("-Zcodegen-backend={name}"))) + } else { + let mut codegen_backend_arg = OsString::from("-Zcodegen-backend="); + codegen_backend_arg.push(cg_clif_dylib_path); + args.push(codegen_backend_arg); + } if !passed_args.iter().any(|arg| { arg == "--sysroot" || arg.to_str().map(|s| s.starts_with("--sysroot=")) == Some(true) }) { diff --git a/scripts/rustdoc-clif.rs b/scripts/rustdoc-clif.rs index 36a00dc676e..10582cc7bb3 100644 --- a/scripts/rustdoc-clif.rs +++ b/scripts/rustdoc-clif.rs @@ -19,9 +19,13 @@ fn main() { let mut args = vec![]; args.push(OsString::from("-Cpanic=abort")); args.push(OsString::from("-Zpanic-abort-tests")); - let mut codegen_backend_arg = OsString::from("-Zcodegen-backend="); - codegen_backend_arg.push(cg_clif_dylib_path); - args.push(codegen_backend_arg); + if let Some(name) = option_env!("BUILTIN_BACKEND") { + args.push(OsString::from(format!("-Zcodegen-backend={name}"))) + } else { + let mut codegen_backend_arg = OsString::from("-Zcodegen-backend="); + codegen_backend_arg.push(cg_clif_dylib_path); + args.push(codegen_backend_arg); + } if !passed_args.iter().any(|arg| { arg == "--sysroot" || arg.to_str().map(|s| s.starts_with("--sysroot=")) == Some(true) }) {