diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3680136d89f..0c20e4dc535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -393,7 +393,7 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-i686-msvc diff --git a/Cargo.lock b/Cargo.lock index 295a5b43a6f..51fce77150c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -858,14 +858,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.3", + "darling_macro 0.20.3", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", ] [[package]] @@ -882,13 +906,24 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ - "darling_core", + "darling_core 0.20.3", "quote", "syn 2.0.29", ] @@ -919,6 +954,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -938,7 +1004,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d" dependencies = [ - "darling", + "darling 0.20.3", "proc-macro2", "quote", "syn 2.0.29", @@ -2584,6 +2650,8 @@ dependencies = [ "anyhow", "build_helper", "camino", + "clap", + "derive_builder", "env_logger 0.10.0", "fs_extra", "glob", diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index f6954275ad4..6f1b2a6a638 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.lto=thin ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ - ./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \ + ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ build-manifest bootstrap diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 89b82d59d31..1712d65ece9 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -624,7 +624,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index 3f7dba81c3a..f1c3dd6aa9b 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" glob = "0.3" tempfile = "3.5" +derive_builder = "0.12" +clap = { version = "4", features = ["derive"] } diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs new file mode 100644 index 00000000000..e730e610604 --- /dev/null +++ b/src/tools/opt-dist/src/environment.rs @@ -0,0 +1,100 @@ +use camino::Utf8PathBuf; +use derive_builder::Builder; + +#[derive(Builder)] +pub struct Environment { + host_triple: String, + python_binary: String, + /// The rustc checkout, where the compiler source is located. + checkout_dir: Utf8PathBuf, + /// The main directory where the build occurs. + build_dir: Utf8PathBuf, + /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) + /// will be stored. + artifact_dir: Utf8PathBuf, + /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. + host_llvm_dir: Utf8PathBuf, + /// List of test paths that should be skipped when testing the optimized artifacts. + skipped_tests: Vec<&'static str>, + use_bolt: bool, + shared_llvm: bool, +} + +impl Environment { + pub fn host_triple(&self) -> &str { + &self.host_triple + } + + pub fn python_binary(&self) -> &str { + &self.python_binary + } + + pub fn checkout_path(&self) -> Utf8PathBuf { + self.checkout_dir.clone() + } + + pub fn build_root(&self) -> Utf8PathBuf { + self.build_dir.clone() + } + + pub fn build_artifacts(&self) -> Utf8PathBuf { + self.build_root().join("build").join(&self.host_triple) + } + + pub fn artifact_dir(&self) -> Utf8PathBuf { + self.artifact_dir.clone() + } + + pub fn cargo_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("cargo{}", executable_extension())) + } + + pub fn rustc_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + pub fn rustc_stage_2(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage2") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + /// Path to the built rustc-perf benchmark suite. + pub fn rustc_perf_dir(&self) -> Utf8PathBuf { + self.artifact_dir.join("rustc-perf") + } + + pub fn host_llvm_dir(&self) -> Utf8PathBuf { + self.host_llvm_dir.clone() + } + + pub fn use_bolt(&self) -> bool { + self.use_bolt + } + + pub fn supports_shared_llvm(&self) -> bool { + self.shared_llvm + } + + pub fn skipped_tests(&self) -> &[&'static str] { + &self.skipped_tests + } +} + +/// What is the extension of binary executables on this platform? +#[cfg(target_family = "unix")] +pub fn executable_extension() -> &'static str { + "" +} + +#[cfg(target_family = "windows")] +pub fn executable_extension() -> &'static str { + ".exe" +} diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs deleted file mode 100644 index 61f05137298..00000000000 --- a/src/tools/opt-dist/src/environment/linux.rs +++ /dev/null @@ -1,66 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::copy_directory; -use camino::{Utf8Path, Utf8PathBuf}; - -pub(super) struct LinuxEnvironment { - target_triple: String, -} - -impl LinuxEnvironment { - pub fn new(target_triple: String) -> Self { - Self { target_triple } - } -} - -impl Environment for LinuxEnvironment { - fn host_triple(&self) -> &str { - &self.target_triple - } - - fn python_binary(&self) -> &'static str { - "python3" - } - - fn checkout_path(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/checkout") - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/rustroot") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path().join("obj") - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // /tmp/rustc-perf comes from the x64 dist Dockerfile - copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?; - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - Ok(()) - } - - fn supports_bolt(&self) -> bool { - true - } - - fn supports_shared_llvm(&self) -> bool { - true - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs deleted file mode 100644 index 271b41316ab..00000000000 --- a/src/tools/opt-dist/src/environment/mod.rs +++ /dev/null @@ -1,83 +0,0 @@ -use camino::Utf8PathBuf; - -#[cfg(target_family = "unix")] -mod linux; -#[cfg(target_family = "windows")] -mod windows; - -pub trait Environment { - fn host_triple(&self) -> &str; - - fn python_binary(&self) -> &'static str; - - /// The rustc checkout, where the compiler source is located. - fn checkout_path(&self) -> Utf8PathBuf; - - /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. - fn host_llvm_dir(&self) -> Utf8PathBuf; - - /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) - /// will be stored. - fn opt_artifacts(&self) -> Utf8PathBuf; - - /// The main directory where the build occurs. - fn build_root(&self) -> Utf8PathBuf; - - fn build_artifacts(&self) -> Utf8PathBuf { - self.build_root().join("build").join(self.host_triple()) - } - - fn cargo_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("cargo{}", executable_extension())) - } - - fn rustc_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("rustc{}", executable_extension())) - } - - fn rustc_stage_2(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage2") - .join("bin") - .join(format!("rustc{}", executable_extension())) - } - - /// Path to the built rustc-perf benchmark suite. - fn rustc_perf_dir(&self) -> Utf8PathBuf { - self.opt_artifacts().join("rustc-perf") - } - - /// Download and/or compile rustc-perf. - fn prepare_rustc_perf(&self) -> anyhow::Result<()>; - - fn supports_bolt(&self) -> bool; - - fn supports_shared_llvm(&self) -> bool; - - /// List of test paths that should be skipped when testing the optimized artifacts. - fn skipped_tests(&self) -> &'static [&'static str]; -} - -pub fn create_environment(target_triple: String) -> Box { - #[cfg(target_family = "unix")] - return Box::new(linux::LinuxEnvironment::new(target_triple)); - #[cfg(target_family = "windows")] - return Box::new(windows::WindowsEnvironment::new(target_triple)); -} - -/// What is the extension of binary executables on this platform? -#[cfg(target_family = "unix")] -pub fn executable_extension() -> &'static str { - "" -} - -#[cfg(target_family = "windows")] -pub fn executable_extension() -> &'static str { - ".exe" -} diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs deleted file mode 100644 index f705461ac81..00000000000 --- a/src/tools/opt-dist/src/environment/windows.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::move_directory; -use crate::utils::retry_action; -use camino::Utf8PathBuf; -use std::io::Cursor; -use std::time::Duration; -use zip::ZipArchive; - -pub(super) struct WindowsEnvironment { - checkout_dir: Utf8PathBuf, - target_triple: String, -} - -impl WindowsEnvironment { - pub fn new(target_triple: String) -> Self { - Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple } - } -} - -impl Environment for WindowsEnvironment { - fn host_triple(&self) -> &str { - &self.target_triple - } - - fn python_binary(&self) -> &'static str { - "python" - } - - fn checkout_path(&self) -> Utf8PathBuf { - self.checkout_dir.clone() - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - self.checkout_path().join("citools").join("clang-rust") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - self.checkout_path().join("opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path() - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // FIXME: add some mechanism for synchronization of this commit SHA with - // Linux (which builds rustc-perf in a Dockerfile) - // rustc-perf version from 2023-05-30 - const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; - - let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(60 * 2)) - .connect_timeout(Duration::from_secs(60 * 2)) - .build()?; - let response = retry_action( - || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), - "Download rustc-perf archive", - 5, - )?; - - let mut archive = ZipArchive::new(Cursor::new(response))?; - archive.extract(self.rustc_perf_dir())?; - move_directory( - &self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), - &self.rustc_perf_dir(), - )?; - - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - - Ok(()) - } - - fn supports_bolt(&self) -> bool { - false - } - - fn supports_shared_llvm(&self) -> bool { - false - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails as of June 2023. - "tests\\codegen\\vec-shrink-panik.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs index 4765dceb5dd..04e0184528a 100644 --- a/src/tools/opt-dist/src/exec.rs +++ b/src/tools/opt-dist/src/exec.rs @@ -96,7 +96,7 @@ pub struct Bootstrap { } impl Bootstrap { - pub fn build(env: &dyn Environment) -> Self { + pub fn build(env: &Environment) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&[ env.python_binary(), @@ -114,7 +114,7 @@ pub fn build(env: &dyn Environment) -> Self { Self { cmd, metrics_path } } - pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self { + pub fn dist(env: &Environment, dist_args: &[String]) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::>()) .env("RUST_BACKTRACE", "full"); diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 484ca5b3b86..6a3bdd7cc75 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -1,17 +1,22 @@ use crate::bolt::{bolt_optimize, with_bolt_instrumented}; use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; +use clap::Parser; use log::LevelFilter; +use std::io::Cursor; +use std::time::Duration; use utils::io; +use zip::ZipArchive; -use crate::environment::{create_environment, Environment}; -use crate::exec::Bootstrap; +use crate::environment::{Environment, EnvironmentBuilder}; +use crate::exec::{cmd, Bootstrap}; use crate::tests::run_tests; use crate::timer::Timer; use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles}; -use crate::utils::io::reset_directory; +use crate::utils::io::{copy_directory, move_directory, reset_directory}; use crate::utils::{ clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space, - with_log_group, + retry_action, with_log_group, }; mod bolt; @@ -23,24 +28,104 @@ mod training; mod utils; +#[derive(clap::Parser, Debug)] +struct Args { + #[clap(subcommand)] + env: EnvironmentCmd, +} + +#[derive(clap::Parser, Clone, Debug)] +struct SharedArgs { + // Arguments passed to `x` to perform the final (dist) build. + build_args: Vec, +} + +#[derive(clap::Parser, Clone, Debug)] +enum EnvironmentCmd { + LinuxCi { + #[clap(flatten)] + shared: SharedArgs, + }, + WindowsCi { + #[clap(flatten)] + shared: SharedArgs, + }, +} + fn is_try_build() -> bool { std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0" } +fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> { + let (env, args) = match args.env { + EnvironmentCmd::LinuxCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir = Utf8PathBuf::from("/checkout"); + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python3".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(Utf8PathBuf::from("/rustroot")) + .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) + .build_dir(checkout_dir.join("obj")) + .shared_llvm(true) + .use_bolt(true) + .skipped_tests(vec![ + // Fails because of linker errors, as of June 2023. + "tests/ui/process/nofile-limit.rs", + ]) + .build()?; + // /tmp/rustc-perf comes from the x64 dist Dockerfile + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?) + })?; + + (env, shared.build_args) + } + EnvironmentCmd::WindowsCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?; + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(checkout_dir.join("citools").join("clang-rust")) + .artifact_dir(checkout_dir.join("opt-artifacts")) + .build_dir(checkout_dir) + .shared_llvm(false) + .use_bolt(false) + .skipped_tests(vec![ + // Fails as of June 2023. + "tests\\codegen\\vec-shrink-panik.rs", + ]) + .build()?; + + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) + })?; + + (env, shared.build_args) + } + }; + Ok((env, args)) +} + fn execute_pipeline( - env: &dyn Environment, + env: &Environment, timer: &mut Timer, dist_args: Vec, ) -> anyhow::Result<()> { - reset_directory(&env.opt_artifacts())?; - - with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?; + reset_directory(&env.artifact_dir())?; // Stage 1: Build PGO instrumented rustc // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the // same time can cause issues, because the host and in-tree LLVM versions can diverge. let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| { - let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo"); + let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo"); stage.section("Build PGO instrumented rustc and LLVM", |section| { let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root); @@ -74,7 +159,7 @@ fn execute_pipeline( // Remove the previous, uninstrumented build of LLVM. clear_llvm_files(env)?; - let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo"); + let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo"); stage.section("Build PGO instrumented LLVM", |section| { Bootstrap::build(env) @@ -95,7 +180,7 @@ fn execute_pipeline( Ok(profile) })?; - let llvm_bolt_profile = if env.supports_bolt() { + let llvm_bolt_profile = if env.use_bolt() { // Stage 3: Build BOLT instrumented LLVM // We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles. // Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build. @@ -171,10 +256,9 @@ fn main() -> anyhow::Result<()> { .parse_default_env() .init(); - let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + let args = Args::parse(); - let mut build_args: Vec = std::env::args().skip(1).collect(); - println!("Running optimized build pipeline with args `{}`", build_args.join(" ")); + println!("Running optimized build pipeline with args `{:?}`", args); with_log_group("Environment values", || { println!("Environment values\n{}", format_env_variables()); @@ -186,6 +270,8 @@ fn main() -> anyhow::Result<()> { } }); + let (env, mut build_args) = create_environment(args).context("Cannot create environment")?; + // Skip components that are not needed for try builds to speed them up if is_try_build() { log::info!("Skipping building of unimportant components for a try build"); @@ -204,14 +290,56 @@ fn main() -> anyhow::Result<()> { } let mut timer = Timer::new(); - let env = create_environment(target_triple); - let result = execute_pipeline(env.as_ref(), &mut timer, build_args); + let result = execute_pipeline(&env, &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); print_free_disk_space()?; result.context("Optimized build pipeline has failed")?; - print_binary_sizes(env.as_ref())?; + print_binary_sizes(&env)?; Ok(()) } + +// Copy rustc-perf from the given path into the environment and build it. +fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { + copy_directory(dir, &env.rustc_perf_dir())?; + build_rustc_perf(env) +} + +// Download and build rustc-perf into the given environment. +fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { + // FIXME: add some mechanism for synchronization of this commit SHA with + // Linux (which builds rustc-perf in a Dockerfile) + // rustc-perf version from 2023-05-30 + const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; + + let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(60 * 2)) + .connect_timeout(Duration::from_secs(60 * 2)) + .build()?; + let response = retry_action( + || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), + "Download rustc-perf archive", + 5, + )?; + + let mut archive = ZipArchive::new(Cursor::new(response))?; + archive.extract(env.rustc_perf_dir())?; + move_directory( + &env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), + &env.rustc_perf_dir(), + )?; + + build_rustc_perf(env) +} + +fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> { + cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"]) + .workdir(&env.rustc_perf_dir()) + .env("RUSTC", &env.rustc_stage_0().into_string()) + .env("RUSTC_BOOTSTRAP", "1") + .run()?; + Ok(()) +} diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index e6b8df9c862..54f7185f88a 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -5,7 +5,7 @@ use camino::{Utf8Path, Utf8PathBuf}; /// Run tests on optimized dist artifacts. -pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { +pub fn run_tests(env: &Environment) -> anyhow::Result<()> { // After `dist` is executed, we extract its archived components into a sysroot directory, // and then use that extracted rustc as a stage0 compiler. // Then we run a subset of tests using that compiler, to have a basic smoke test which checks diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 950dd1127a9..dd8aa532408 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -30,7 +30,7 @@ const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES; fn init_compiler_benchmarks( - env: &dyn Environment, + env: &Environment, profiles: &[&str], scenarios: &[&str], crates: &[&str], @@ -75,7 +75,7 @@ enum LlvmProfdata { } fn merge_llvm_profiles( - env: &dyn Environment, + env: &Environment, merged_path: &Utf8Path, profile_dir: &Utf8Path, profdata: LlvmProfdata, @@ -116,7 +116,7 @@ fn log_profile_stats( pub struct LlvmPGOProfile(pub Utf8PathBuf); pub fn gather_llvm_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented LLVM"); @@ -127,7 +127,7 @@ pub fn gather_llvm_profiles( .context("Cannot gather LLVM PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata"); + let merged_profile = env.artifact_dir().join("llvm-pgo.profdata"); log::info!("Merging LLVM PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?; @@ -143,7 +143,7 @@ pub fn gather_llvm_profiles( pub struct RustcPGOProfile(pub Utf8PathBuf); pub fn gather_rustc_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented rustc"); @@ -163,7 +163,7 @@ pub fn gather_rustc_profiles( .context("Cannot gather rustc PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata"); + let merged_profile = env.artifact_dir().join("rustc-pgo.profdata"); log::info!("Merging Rustc PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?; @@ -178,7 +178,7 @@ pub fn gather_rustc_profiles( pub struct LlvmBoltProfile(pub Utf8PathBuf); -pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result { +pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result { log::info!("Running benchmarks with BOLT instrumented LLVM"); with_log_group("Running benchmarks", || { @@ -187,7 +187,7 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result anyhow::Result<()> { Ok(()) } -pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { +pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> { use std::fmt::Write; let root = env.build_artifacts().join("stage2"); @@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { Ok(()) } -pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> { +pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> { // Bootstrap currently doesn't support rebuilding LLVM when PGO options // change (or any other llvm-related options); so just clear out the relevant // directories ourselves.