From a58327dc014dabac8d69b9602011fee75d201815 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 2 Feb 2021 19:45:17 +0100 Subject: [PATCH] Distribute cg_clif as a rustup component --- src/bootstrap/src/core/build_steps/dist.rs | 109 +++++++++++++++--- src/bootstrap/src/core/build_steps/install.rs | 14 +++ src/bootstrap/src/core/build_steps/test.rs | 16 +-- src/bootstrap/src/core/builder.rs | 1 + src/bootstrap/src/utils/helpers.rs | 13 +++ src/bootstrap/src/utils/tarball.rs | 7 ++ src/ci/run.sh | 3 +- src/tools/build-manifest/src/main.rs | 16 ++- src/tools/build-manifest/src/versions.rs | 4 + 9 files changed, 152 insertions(+), 31 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 47f41ab288d..acab3cffe07 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -27,7 +27,7 @@ use crate::core::config::TargetSelection; use crate::utils::cache::{Interned, INTERNER}; use crate::utils::channel; -use crate::utils::helpers::{exe, is_dylib, output, t, timeit}; +use crate::utils::helpers::{exe, is_dylib, output, t, target_supports_cranelift_backend, timeit}; use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball}; use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; @@ -443,19 +443,6 @@ fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { } } - // Copy over the codegen backends - let backends_src = builder.sysroot_codegen_backends(compiler); - let backends_rel = backends_src - .strip_prefix(&src) - .unwrap() - .strip_prefix(builder.sysroot_libdir_relative(compiler)) - .unwrap(); - // Don't use custom libdir here because ^lib/ will be resolved again with installer - let backends_dst = image.join("lib").join(&backends_rel); - - t!(fs::create_dir_all(&backends_dst)); - builder.cp_r(&backends_src, &backends_dst); - // Copy libLLVM.so to the lib dir as well, if needed. While not // technically needed by rustc itself it's needed by lots of other // components like the llvm tools and LLD. LLD is included below and @@ -1282,6 +1269,91 @@ fn run(self, builder: &Builder<'_>) -> Option { } } +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct CodegenBackend { + pub compiler: Compiler, + pub backend: Interned, +} + +impl Step for CodegenBackend { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc_codegen_cranelift") + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &run.builder.config.rust_codegen_backends { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + backend, + }); + } + } + + fn run(self, builder: &Builder<'_>) -> Option { + // This prevents rustc_codegen_cranelift from being built for "dist" + // or "install" on the stable/beta channels. It is not yet stable and + // should not be included. + if !builder.build.unstable_features() { + return None; + } + + if self.backend == "cranelift" { + if !target_supports_cranelift_backend(self.compiler.host) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return None; + } + + if self.compiler.host.contains("windows") { + builder.info( + "dist currently disabled for windows by rustc_codegen_cranelift. skipping", + ); + return None; + } + } + + let compiler = self.compiler; + let backend = self.backend; + + let mut tarball = + Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple); + if backend == "cranelift" { + tarball.set_overlay(OverlayKind::RustcCodegenCranelift); + } else { + panic!("Unknown backend rustc_codegen_{}", backend); + } + tarball.is_preview(true); + tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend)); + + let src = builder.sysroot(compiler); + let backends_src = builder.sysroot_codegen_backends(compiler); + let backends_rel = backends_src + .strip_prefix(&src) + .unwrap() + .strip_prefix(builder.sysroot_libdir_relative(compiler)) + .unwrap(); + // Don't use custom libdir here because ^lib/ will be resolved again with installer + let backends_dst = PathBuf::from("lib").join(&backends_rel); + + let backend_name = format!("rustc_codegen_{}", backend); + for backend in fs::read_dir(&backends_src).unwrap() { + let file_name = backend.unwrap().file_name(); + if file_name.to_str().unwrap().contains(&backend_name) { + tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); + } + } + + Some(tarball.generate()) + } +} + #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] pub struct Rustfmt { pub compiler: Compiler, @@ -1452,6 +1524,10 @@ macro_rules! add_component { add_component!("clippy" => Clippy { compiler, target }); add_component!("miri" => Miri { compiler, target }); add_component!("analysis" => Analysis { compiler, target }); + add_component!("rustc-codegen-cranelift" => CodegenBackend { + compiler: builder.compiler(stage, target), + backend: INTERNER.intern_str("cranelift"), + }); let etc = builder.src.join("src/etc/installer"); @@ -1548,6 +1624,7 @@ fn filter(contents: &str, marker: &str) -> String { prepare(tool); } } + prepare("rustc-codegen-cranelift"); // create an 'uninstall' package builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); pkgbuild("uninstall"); @@ -1587,6 +1664,10 @@ fn filter(contents: &str, marker: &str) -> String { "rust-demangler-preview".to_string() } else if name == "miri" { "miri-preview".to_string() + } else if name == "rustc-codegen-cranelift" { + // FIXME add installer support for cg_clif once it is ready to be distributed on + // windows. + unreachable!("cg_clif shouldn't be built for windows"); } else { name.to_string() }; diff --git a/src/bootstrap/src/core/build_steps/install.rs b/src/bootstrap/src/core/build_steps/install.rs index 391995b7c3b..75d263d7794 100644 --- a/src/bootstrap/src/core/build_steps/install.rs +++ b/src/bootstrap/src/core/build_steps/install.rs @@ -13,6 +13,7 @@ use crate::core::config::{Config, TargetSelection}; use crate::utils::helpers::t; use crate::utils::tarball::GeneratedTarball; +use crate::INTERNER; use crate::{Compiler, Kind}; #[cfg(target_os = "illumos")] @@ -281,6 +282,19 @@ fn run($sel, $builder: &Builder<'_>) { }); install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); }; + RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::CodegenBackend { + compiler: self.compiler, + backend: INTERNER.intern_str("cranelift"), + }) { + install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 27eac212b14..9d2eeaee03e 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -29,7 +29,8 @@ use crate::utils::cache::{Interned, INTERNER}; use crate::utils::exec::BootstrapCommand; use crate::utils::helpers::{ - self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date, + self, add_link_lib_path, dylib_path, dylib_path_var, output, t, + target_supports_cranelift_backend, up_to_date, }; use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests}; use crate::{envify, CLang, DocTests, GitRepo, Mode}; @@ -2998,18 +2999,7 @@ fn make_run(run: RunConfig<'_>) { return; } - let triple = run.target.triple; - let target_supported = if triple.contains("linux") { - triple.contains("x86_64") - || triple.contains("aarch64") - || triple.contains("s390x") - || triple.contains("riscv64gc") - } else if triple.contains("darwin") || triple.contains("windows") { - triple.contains("x86_64") - } else { - false - }; - if !target_supported { + if !target_supports_cranelift_backend(run.target) { builder.info("target not supported by rustc_codegen_cranelift. skipping"); return; } diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 36653dcfb20..90e09d12a9d 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -815,6 +815,7 @@ macro_rules! describe { dist::JsonDocs, dist::Mingw, dist::Rustc, + dist::CodegenBackend, dist::Std, dist::RustcDev, dist::Analysis, diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index b58a1c25842..82a5607d903 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -183,6 +183,19 @@ pub fn use_host_linker(target: TargetSelection) -> bool { || target.contains("switch")) } +pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool { + if target.contains("linux") { + target.contains("x86_64") + || target.contains("aarch64") + || target.contains("s390x") + || target.contains("riscv64gc") + } else if target.contains("darwin") || target.contains("windows") { + target.contains("x86_64") + } else { + false + } +} + pub fn is_valid_test_suite_arg<'a, P: AsRef>( path: &'a Path, suite_path: P, diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs index b437456f8a1..a8393f88f8a 100644 --- a/src/bootstrap/src/utils/tarball.rs +++ b/src/bootstrap/src/utils/tarball.rs @@ -19,6 +19,7 @@ pub(crate) enum OverlayKind { RustDemangler, RLS, RustAnalyzer, + RustcCodegenCranelift, } impl OverlayKind { @@ -58,6 +59,11 @@ fn legal_and_readme(&self) -> &[&str] { "src/tools/rust-analyzer/LICENSE-APACHE", "src/tools/rust-analyzer/LICENSE-MIT", ], + OverlayKind::RustcCodegenCranelift => &[ + "compiler/rustc_codegen_cranelift/Readme.md", + "compiler/rustc_codegen_cranelift/LICENSE-APACHE", + "compiler/rustc_codegen_cranelift/LICENSE-MIT", + ], } } @@ -80,6 +86,7 @@ fn version(&self, builder: &Builder<'_>) -> String { OverlayKind::RustAnalyzer => builder .rust_analyzer_info .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), + OverlayKind::RustcCodegenCranelift => builder.rust_version(), } } } diff --git a/src/ci/run.sh b/src/ci/run.sh index abeff7b837d..17ce7a807b6 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -104,6 +104,8 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" fi + + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" else # We almost always want debug assertions enabled, but sometimes this takes too # long for too little benefit, so we just turn them off. @@ -124,7 +126,6 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" - # Test the Cranelift backend in on CI, but don't ship it. RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" # We enable this for non-dist builders, since those aren't trying to produce diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index b768722acf8..2c795ebb214 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -192,7 +192,8 @@ static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; -static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs]; +static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = + &[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift]; macro_rules! t { ($e:expr) => { @@ -336,7 +337,15 @@ fn add_profiles_to(&mut self, manifest: &mut Manifest) { // NOTE: this profile is effectively deprecated; do not add new components to it. let mut complete = default; - complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]); + complete.extend([ + Rls, + RustAnalyzer, + RustSrc, + LlvmTools, + RustAnalysis, + Miri, + RustcCodegenCranelift, + ]); profile("complete", &complete); // The compiler libraries are not stable for end users, and they're also huge, so we only @@ -423,7 +432,8 @@ fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option | PkgType::Rustfmt | PkgType::LlvmTools | PkgType::RustAnalysis - | PkgType::JsonDocs => { + | PkgType::JsonDocs + | PkgType::RustcCodegenCranelift => { extensions.push(host_component(pkg)); } PkgType::RustcDev | PkgType::RustcDocs => { diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 7a4c15d01ea..57f94bdd90a 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -57,6 +57,7 @@ pub(crate) fn all() -> &'static [PkgType] { LlvmTools = "llvm-tools"; preview = true, Miri = "miri"; preview = true, JsonDocs = "rust-docs-json"; preview = true, + RustcCodegenCranelift = "rustc-codegen-cranelift"; preview = true, } impl PkgType { @@ -80,6 +81,7 @@ fn should_use_rust_version(&self) -> bool { PkgType::Rustfmt => false, PkgType::LlvmTools => false, PkgType::Miri => false, + PkgType::RustcCodegenCranelift => false, PkgType::Rust => true, PkgType::RustStd => true, @@ -106,6 +108,8 @@ pub(crate) fn targets(&self) -> &[&str] { ReproducibleArtifacts => HOSTS, RustcDocs => HOSTS, Cargo => HOSTS, + // FIXME should this use the exact list of targets for which we build cg_clif? + RustcCodegenCranelift => HOSTS, RustMingw => MINGW, RustStd => TARGETS, HtmlDocs => HOSTS,