mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor Environment

Browse Source
This commit is contained in:
Jakub Beránek 2023-09-12 18:32:32 +02:00
parent 95500f4941
commit f17047bc90
No known key found for this signature in database
GPG Key ID: 909CD0D26483516B
14 changed files with 335 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@ -393,7 +393,7 @@ jobs:
- name: dist-x86_64-msvc
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-2019-8core-32gb
- name: dist-i686-msvc

76
Cargo.lock
View File

@ -858,14 +858,38 @@ dependencies = [
"winapi",
]
[[package]]
name = "darling"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core 0.14.4",
"darling_macro 0.14.4",
]
[[package]]
name = "darling"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
dependencies = [
"darling_core",
"darling_macro",
"darling_core 0.20.3",
"darling_macro 0.20.3",
]
[[package]]
name = "darling_core"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 1.0.109",
]
[[package]]
@ -882,13 +906,24 @@ dependencies = [
"syn 2.0.29",
]
[[package]]
name = "darling_macro"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core 0.14.4",
"quote",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [
"darling_core",
"darling_core 0.20.3",
"quote",
"syn 2.0.29",
]
@ -919,6 +954,37 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "derive_builder"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
dependencies = [
"derive_builder_macro",
]
[[package]]
name = "derive_builder_core"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
dependencies = [
"darling 0.14.4",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "derive_builder_macro"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
dependencies = [
"derive_builder_core",
"syn 1.0.109",
]
[[package]]
name = "derive_more"
version = "0.99.17"
@ -938,7 +1004,7 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
dependencies = [
"darling",
"darling 0.20.3",
"proc-macro2",
"quote",
"syn 2.0.29",
@ -2584,6 +2650,8 @@ dependencies = [
"anyhow",
"build_helper",
"camino",
"clap",
"derive_builder",
"env_logger 0.10.0",
"fs_extra",
"glob",

2
src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
View File

@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
--set rust.lto=thin
ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
--host $HOSTS --target $HOSTS \
--include-default-paths \
build-manifest bootstrap

2
src/ci/github-actions/ci.yml
View File

@ -624,7 +624,7 @@ jobs:
--target=x86_64-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-8c

2
src/tools/opt-dist/Cargo.toml
View File

@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
glob = "0.3"
tempfile = "3.5"
derive_builder = "0.12"
clap = { version = "4", features = ["derive"] }

100
src/tools/opt-dist/src/environment.rs Normal file
View File

@ -0,0 +1,100 @@
use camino::Utf8PathBuf;
use derive_builder::Builder;
#[derive(Builder)]
pub struct Environment {
host_triple: String,
python_binary: String,
/// The rustc checkout, where the compiler source is located.
checkout_dir: Utf8PathBuf,
/// The main directory where the build occurs.
build_dir: Utf8PathBuf,
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
/// will be stored.
artifact_dir: Utf8PathBuf,
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
host_llvm_dir: Utf8PathBuf,
/// List of test paths that should be skipped when testing the optimized artifacts.
skipped_tests: Vec<&'static str>,
use_bolt: bool,
shared_llvm: bool,
}
impl Environment {
pub fn host_triple(&self) -> &str {
&self.host_triple
}
pub fn python_binary(&self) -> &str {
&self.python_binary
}
pub fn checkout_path(&self) -> Utf8PathBuf {
self.checkout_dir.clone()
}
pub fn build_root(&self) -> Utf8PathBuf {
self.build_dir.clone()
}
pub fn build_artifacts(&self) -> Utf8PathBuf {
self.build_root().join("build").join(&self.host_triple)
}
pub fn artifact_dir(&self) -> Utf8PathBuf {
self.artifact_dir.clone()
}
pub fn cargo_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("cargo{}", executable_extension()))
}
pub fn rustc_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}
pub fn rustc_stage_2(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage2")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}
/// Path to the built rustc-perf benchmark suite.
pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
self.artifact_dir.join("rustc-perf")
}
pub fn host_llvm_dir(&self) -> Utf8PathBuf {
self.host_llvm_dir.clone()
}
pub fn use_bolt(&self) -> bool {
self.use_bolt
}
pub fn supports_shared_llvm(&self) -> bool {
self.shared_llvm
}
pub fn skipped_tests(&self) -> &[&'static str] {
&self.skipped_tests
}
}
/// What is the extension of binary executables on this platform?
#[cfg(target_family = "unix")]
pub fn executable_extension() -> &'static str {
""
}
#[cfg(target_family = "windows")]
pub fn executable_extension() -> &'static str {
".exe"
}

66
src/tools/opt-dist/src/environment/linux.rs
View File

@ -1,66 +0,0 @@
use crate::environment::Environment;
use crate::exec::cmd;
use crate::utils::io::copy_directory;
use camino::{Utf8Path, Utf8PathBuf};
pub(super) struct LinuxEnvironment {
target_triple: String,
}
impl LinuxEnvironment {
pub fn new(target_triple: String) -> Self {
Self { target_triple }
}
}
impl Environment for LinuxEnvironment {
fn host_triple(&self) -> &str {
&self.target_triple
}
fn python_binary(&self) -> &'static str {
"python3"
}
fn checkout_path(&self) -> Utf8PathBuf {
Utf8PathBuf::from("/checkout")
}
fn host_llvm_dir(&self) -> Utf8PathBuf {
Utf8PathBuf::from("/rustroot")
}
fn opt_artifacts(&self) -> Utf8PathBuf {
Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")
}
fn build_root(&self) -> Utf8PathBuf {
self.checkout_path().join("obj")
}
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
// /tmp/rustc-perf comes from the x64 dist Dockerfile
copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?;
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
.workdir(&self.rustc_perf_dir())
.env("RUSTC", &self.rustc_stage_0().into_string())
.env("RUSTC_BOOTSTRAP", "1")
.run()?;
Ok(())
}
fn supports_bolt(&self) -> bool {
true
}
fn supports_shared_llvm(&self) -> bool {
true
}
fn skipped_tests(&self) -> &'static [&'static str] {
&[
// Fails because of linker errors, as of June 2023.
"tests/ui/process/nofile-limit.rs",
]
}
}

83
src/tools/opt-dist/src/environment/mod.rs
View File

@ -1,83 +0,0 @@
use camino::Utf8PathBuf;
#[cfg(target_family = "unix")]
mod linux;
#[cfg(target_family = "windows")]
mod windows;
pub trait Environment {
fn host_triple(&self) -> &str;
fn python_binary(&self) -> &'static str;
/// The rustc checkout, where the compiler source is located.
fn checkout_path(&self) -> Utf8PathBuf;
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
fn host_llvm_dir(&self) -> Utf8PathBuf;
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
/// will be stored.
fn opt_artifacts(&self) -> Utf8PathBuf;
/// The main directory where the build occurs.
fn build_root(&self) -> Utf8PathBuf;
fn build_artifacts(&self) -> Utf8PathBuf {
self.build_root().join("build").join(self.host_triple())
}
fn cargo_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("cargo{}", executable_extension()))
}
fn rustc_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}
fn rustc_stage_2(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage2")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}
/// Path to the built rustc-perf benchmark suite.
fn rustc_perf_dir(&self) -> Utf8PathBuf {
self.opt_artifacts().join("rustc-perf")
}
/// Download and/or compile rustc-perf.
fn prepare_rustc_perf(&self) -> anyhow::Result<()>;
fn supports_bolt(&self) -> bool;
fn supports_shared_llvm(&self) -> bool;
/// List of test paths that should be skipped when testing the optimized artifacts.
fn skipped_tests(&self) -> &'static [&'static str];
}
pub fn create_environment(target_triple: String) -> Box<dyn Environment> {
#[cfg(target_family = "unix")]
return Box::new(linux::LinuxEnvironment::new(target_triple));
#[cfg(target_family = "windows")]
return Box::new(windows::WindowsEnvironment::new(target_triple));
}
/// What is the extension of binary executables on this platform?
#[cfg(target_family = "unix")]
pub fn executable_extension() -> &'static str {
""
}
#[cfg(target_family = "windows")]
pub fn executable_extension() -> &'static str {
".exe"
}

93
src/tools/opt-dist/src/environment/windows.rs
View File

@ -1,93 +0,0 @@
use crate::environment::Environment;
use crate::exec::cmd;
use crate::utils::io::move_directory;
use crate::utils::retry_action;
use camino::Utf8PathBuf;
use std::io::Cursor;
use std::time::Duration;
use zip::ZipArchive;
pub(super) struct WindowsEnvironment {
checkout_dir: Utf8PathBuf,
target_triple: String,
}
impl WindowsEnvironment {
pub fn new(target_triple: String) -> Self {
Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple }
}
}
impl Environment for WindowsEnvironment {
fn host_triple(&self) -> &str {
&self.target_triple
}
fn python_binary(&self) -> &'static str {
"python"
}
fn checkout_path(&self) -> Utf8PathBuf {
self.checkout_dir.clone()
}
fn host_llvm_dir(&self) -> Utf8PathBuf {
self.checkout_path().join("citools").join("clang-rust")
}
fn opt_artifacts(&self) -> Utf8PathBuf {
self.checkout_path().join("opt-artifacts")
}
fn build_root(&self) -> Utf8PathBuf {
self.checkout_path()
}
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
// FIXME: add some mechanism for synchronization of this commit SHA with
// Linux (which builds rustc-perf in a Dockerfile)
// rustc-perf version from 2023-05-30
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(60 * 2))
.connect_timeout(Duration::from_secs(60 * 2))
.build()?;
let response = retry_action(
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
"Download rustc-perf archive",
5,
)?;
let mut archive = ZipArchive::new(Cursor::new(response))?;
archive.extract(self.rustc_perf_dir())?;
move_directory(
&self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
&self.rustc_perf_dir(),
)?;
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
.workdir(&self.rustc_perf_dir())
.env("RUSTC", &self.rustc_stage_0().into_string())
.env("RUSTC_BOOTSTRAP", "1")
.run()?;
Ok(())
}
fn supports_bolt(&self) -> bool {
false
}
fn supports_shared_llvm(&self) -> bool {
false
}
fn skipped_tests(&self) -> &'static [&'static str] {
&[
// Fails as of June 2023.
"tests\\codegen\\vec-shrink-panik.rs",
]
}
}

4
src/tools/opt-dist/src/exec.rs
View File

@ -96,7 +96,7 @@ pub struct Bootstrap {
}
impl Bootstrap {
pub fn build(env: &dyn Environment) -> Self {
pub fn build(env: &Environment) -> Self {
let metrics_path = env.build_root().join("build").join("metrics.json");
let cmd = cmd(&[
env.python_binary(),
@ -114,7 +114,7 @@ pub fn build(env: &dyn Environment) -> Self {
Self { cmd, metrics_path }
}
pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self {
pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
let metrics_path = env.build_root().join("build").join("metrics.json");
let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>())
.env("RUST_BACKTRACE", "full");

162
src/tools/opt-dist/src/main.rs
View File

@ -1,17 +1,22 @@
use crate::bolt::{bolt_optimize, with_bolt_instrumented};
use anyhow::Context;
use camino::{Utf8Path, Utf8PathBuf};
use clap::Parser;
use log::LevelFilter;
use std::io::Cursor;
use std::time::Duration;
use utils::io;
use zip::ZipArchive;
use crate::environment::{create_environment, Environment};
use crate::exec::Bootstrap;
use crate::environment::{Environment, EnvironmentBuilder};
use crate::exec::{cmd, Bootstrap};
use crate::tests::run_tests;
use crate::timer::Timer;
use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
use crate::utils::io::reset_directory;
use crate::utils::io::{copy_directory, move_directory, reset_directory};
use crate::utils::{
clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
with_log_group,
retry_action, with_log_group,
};
mod bolt;
@ -23,24 +28,104 @@
mod training;
mod utils;
#[derive(clap::Parser, Debug)]
struct Args {
#[clap(subcommand)]
env: EnvironmentCmd,
}
#[derive(clap::Parser, Clone, Debug)]
struct SharedArgs {
// Arguments passed to `x` to perform the final (dist) build.
build_args: Vec<String>,
}
#[derive(clap::Parser, Clone, Debug)]
enum EnvironmentCmd {
LinuxCi {
#[clap(flatten)]
shared: SharedArgs,
},
WindowsCi {
#[clap(flatten)]
shared: SharedArgs,
},
}
fn is_try_build() -> bool {
std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0"
}
fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)> {
let (env, args) = match args.env {
EnvironmentCmd::LinuxCi { shared } => {
let target_triple =
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
let checkout_dir = Utf8PathBuf::from("/checkout");
let env = EnvironmentBuilder::default()
.host_triple(target_triple)
.python_binary("python3".to_string())
.checkout_dir(checkout_dir.clone())
.host_llvm_dir(Utf8PathBuf::from("/rustroot"))
.artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
.build_dir(checkout_dir.join("obj"))
.shared_llvm(true)
.use_bolt(true)
.skipped_tests(vec![
// Fails because of linker errors, as of June 2023.
"tests/ui/process/nofile-limit.rs",
])
.build()?;
// /tmp/rustc-perf comes from the x64 dist Dockerfile
with_log_group("Building rustc-perf", || {
Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?)
})?;
(env, shared.build_args)
}
EnvironmentCmd::WindowsCi { shared } => {
let target_triple =
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?;
let env = EnvironmentBuilder::default()
.host_triple(target_triple)
.python_binary("python".to_string())
.checkout_dir(checkout_dir.clone())
.host_llvm_dir(checkout_dir.join("citools").join("clang-rust"))
.artifact_dir(checkout_dir.join("opt-artifacts"))
.build_dir(checkout_dir)
.shared_llvm(false)
.use_bolt(false)
.skipped_tests(vec![
// Fails as of June 2023.
"tests\\codegen\\vec-shrink-panik.rs",
])
.build()?;
with_log_group("Building rustc-perf", || {
Ok::<(), anyhow::Error>(download_rustc_perf(&env)?)
})?;
(env, shared.build_args)
}
};
Ok((env, args))
}
fn execute_pipeline(
env: &dyn Environment,
env: &Environment,
timer: &mut Timer,
dist_args: Vec<String>,
) -> anyhow::Result<()> {
reset_directory(&env.opt_artifacts())?;
with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?;
reset_directory(&env.artifact_dir())?;
// Stage 1: Build PGO instrumented rustc
// We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
// same time can cause issues, because the host and in-tree LLVM versions can diverge.
let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| {
let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo");
let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
stage.section("Build PGO instrumented rustc and LLVM", |section| {
let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root);
@ -74,7 +159,7 @@ fn execute_pipeline(
// Remove the previous, uninstrumented build of LLVM.
clear_llvm_files(env)?;
let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo");
let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo");
stage.section("Build PGO instrumented LLVM", |section| {
Bootstrap::build(env)
@ -95,7 +180,7 @@ fn execute_pipeline(
Ok(profile)
})?;
let llvm_bolt_profile = if env.supports_bolt() {
let llvm_bolt_profile = if env.use_bolt() {
// Stage 3: Build BOLT instrumented LLVM
// We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles.
// Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build.
@ -171,10 +256,9 @@ fn main() -> anyhow::Result<()> {
.parse_default_env()
.init();
let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
let args = Args::parse();
let mut build_args: Vec<String> = std::env::args().skip(1).collect();
println!("Running optimized build pipeline with args `{}`", build_args.join(" "));
println!("Running optimized build pipeline with args `{:?}`", args);
with_log_group("Environment values", || {
println!("Environment values\n{}", format_env_variables());
@ -186,6 +270,8 @@ fn main() -> anyhow::Result<()> {
}
});
let (env, mut build_args) = create_environment(args).context("Cannot create environment")?;
// Skip components that are not needed for try builds to speed them up
if is_try_build() {
log::info!("Skipping building of unimportant components for a try build");
@ -204,14 +290,56 @@ fn main() -> anyhow::Result<()> {
}
let mut timer = Timer::new();
let env = create_environment(target_triple);
let result = execute_pipeline(env.as_ref(), &mut timer, build_args);
let result = execute_pipeline(&env, &mut timer, build_args);
log::info!("Timer results\n{}", timer.format_stats());
print_free_disk_space()?;
result.context("Optimized build pipeline has failed")?;
print_binary_sizes(env.as_ref())?;
print_binary_sizes(&env)?;
Ok(())
}
// Copy rustc-perf from the given path into the environment and build it.
fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
copy_directory(dir, &env.rustc_perf_dir())?;
build_rustc_perf(env)
}
// Download and build rustc-perf into the given environment.
fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
// FIXME: add some mechanism for synchronization of this commit SHA with
// Linux (which builds rustc-perf in a Dockerfile)
// rustc-perf version from 2023-05-30
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(60 * 2))
.connect_timeout(Duration::from_secs(60 * 2))
.build()?;
let response = retry_action(
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
"Download rustc-perf archive",
5,
)?;
let mut archive = ZipArchive::new(Cursor::new(response))?;
archive.extract(env.rustc_perf_dir())?;
move_directory(
&env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
&env.rustc_perf_dir(),
)?;
build_rustc_perf(env)
}
fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> {
cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"])
.workdir(&env.rustc_perf_dir())
.env("RUSTC", &env.rustc_stage_0().into_string())
.env("RUSTC_BOOTSTRAP", "1")
.run()?;
Ok(())
}

2
src/tools/opt-dist/src/tests.rs
View File

@ -5,7 +5,7 @@
use camino::{Utf8Path, Utf8PathBuf};
/// Run tests on optimized dist artifacts.
pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
// After `dist` is executed, we extract its archived components into a sysroot directory,
// and then use that extracted rustc as a stage0 compiler.
// Then we run a subset of tests using that compiler, to have a basic smoke test which checks

16
src/tools/opt-dist/src/training.rs
View File

@ -30,7 +30,7 @@
const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES;
fn init_compiler_benchmarks(
env: &dyn Environment,
env: &Environment,
profiles: &[&str],
scenarios: &[&str],
crates: &[&str],
@ -75,7 +75,7 @@ enum LlvmProfdata {
}
fn merge_llvm_profiles(
env: &dyn Environment,
env: &Environment,
merged_path: &Utf8Path,
profile_dir: &Utf8Path,
profdata: LlvmProfdata,
@ -116,7 +116,7 @@ fn log_profile_stats(
pub struct LlvmPGOProfile(pub Utf8PathBuf);
pub fn gather_llvm_profiles(
env: &dyn Environment,
env: &Environment,
profile_root: &Utf8Path,
) -> anyhow::Result<LlvmPGOProfile> {
log::info!("Running benchmarks with PGO instrumented LLVM");
@ -127,7 +127,7 @@ pub fn gather_llvm_profiles(
.context("Cannot gather LLVM PGO profiles")
})?;
let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
let merged_profile = env.artifact_dir().join("llvm-pgo.profdata");
log::info!("Merging LLVM PGO profiles to {merged_profile}");
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?;
@ -143,7 +143,7 @@ pub fn gather_llvm_profiles(
pub struct RustcPGOProfile(pub Utf8PathBuf);
pub fn gather_rustc_profiles(
env: &dyn Environment,
env: &Environment,
profile_root: &Utf8Path,
) -> anyhow::Result<RustcPGOProfile> {
log::info!("Running benchmarks with PGO instrumented rustc");
@ -163,7 +163,7 @@ pub fn gather_rustc_profiles(
.context("Cannot gather rustc PGO profiles")
})?;
let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
let merged_profile = env.artifact_dir().join("rustc-pgo.profdata");
log::info!("Merging Rustc PGO profiles to {merged_profile}");
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?;
@ -178,7 +178,7 @@ pub fn gather_rustc_profiles(
pub struct LlvmBoltProfile(pub Utf8PathBuf);
pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result<LlvmBoltProfile> {
log::info!("Running benchmarks with BOLT instrumented LLVM");
with_log_group("Running benchmarks", || {
@ -187,7 +187,7 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBo
.context("Cannot gather LLVM BOLT profiles")
})?;
let merged_profile = env.opt_artifacts().join("llvm-bolt.profdata");
let merged_profile = env.artifact_dir().join("llvm-bolt.profdata");
let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
log::info!("Merging LLVM BOLT profiles to {merged_profile}");

4
src/tools/opt-dist/src/utils/mod.rs
View File

@ -26,7 +26,7 @@ pub fn print_free_disk_space() -> anyhow::Result<()> {
Ok(())
}
pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> {
use std::fmt::Write;
let root = env.build_artifacts().join("stage2");
@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
Ok(())
}
pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> {
// Bootstrap currently doesn't support rebuilding LLVM when PGO options
// change (or any other llvm-related options); so just clear out the relevant
// directories ourselves.