From f2feed109559009603fa2a8a7dd9c7380a731377 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 5 Dec 2023 18:08:12 +0300 Subject: [PATCH] Dump command invocations from bootstrap shims When making changes to the bootstrap that shouldn't change its behavior, this feature will help developers perform comparisons to check whether the bootstrap behavior has changed or not. This can also be used for different purposes. For example, allowing CI to dump the shims and upload them so that developers can download them and compare with their local dump to see if CI affects the bootstrap unexpectedly. Or, make CI perform comparisons on specific bootstrap tests to check for behavior changes between the master and PR branches. Signed-off-by: onur-ozkan --- src/bootstrap/src/bin/rustc.rs | 6 ++-- src/bootstrap/src/bin/rustdoc.rs | 2 ++ src/bootstrap/src/core/build_steps/clean.rs | 1 + src/bootstrap/src/core/builder.rs | 14 ++++++++-- src/bootstrap/src/core/config/config.rs | 2 ++ src/bootstrap/src/core/config/flags.rs | 3 ++ src/bootstrap/src/lib.rs | 19 +++++++++++++ src/bootstrap/src/utils/bin_helpers.rs | 31 ++++++++++++++++++--- 8 files changed, 69 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index af66cb3ffd7..18136635c42 100644 --- a/src/bootstrap/src/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -32,9 +32,7 @@ fn main() { let args = env::args_os().skip(1).collect::>(); let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str()); - // We don't use the stage in this shim, but let's parse it to make sure that we're invoked - // by bootstrap, or that we provide a helpful error message if not. - bin_helpers::parse_rustc_stage(); + let stage = bin_helpers::parse_rustc_stage(); let verbose = bin_helpers::parse_rustc_verbose(); // Detect whether or not we're a build script depending on whether --target @@ -214,6 +212,8 @@ fn main() { } } + bin_helpers::maybe_dump(format!("stage{stage}-rustc"), &cmd); + let start = Instant::now(); let (child, status) = { let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); diff --git a/src/bootstrap/src/bin/rustdoc.rs b/src/bootstrap/src/bin/rustdoc.rs index 90bd822699c..b4d1415189c 100644 --- a/src/bootstrap/src/bin/rustdoc.rs +++ b/src/bootstrap/src/bin/rustdoc.rs @@ -62,6 +62,8 @@ fn main() { cmd.arg("-Zunstable-options"); cmd.arg("--check-cfg=cfg(bootstrap)"); + bin_helpers::maybe_dump(format!("stage{stage}-rustdoc"), &cmd); + if verbose > 1 { eprintln!( "rustdoc command: {:?}={:?} {:?}", diff --git a/src/bootstrap/src/core/build_steps/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs index a3fb330ddfb..6372db96afb 100644 --- a/src/bootstrap/src/core/build_steps/clean.rs +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -146,6 +146,7 @@ fn clean_default(build: &Build) { rm_rf(&build.out.join("tmp")); rm_rf(&build.out.join("dist")); rm_rf(&build.out.join("bootstrap").join(".last-warned-change-id")); + rm_rf(&build.out.join("bootstrap-shims-dump")); rm_rf(&build.out.join("rustfmt.stamp")); for host in &build.hosts { diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 2356565190b..6e3adac83b7 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -21,9 +21,9 @@ use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection}; use crate::utils::cache::{Cache, Interned, INTERNER}; use crate::utils::helpers::{self, add_dylib_path, add_link_lib_path, exe, linker_args}; use crate::utils::helpers::{libdir, linker_flags, output, t, LldThreads}; -use crate::Crate; use crate::EXTRA_CHECK_CFGS; -use crate::{Build, CLang, DocTests, GitRepo, Mode}; +use crate::prepare_behaviour_dump_dir; +use crate::{Crate, Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; @@ -1788,6 +1788,16 @@ impl<'a> Builder<'a> { // Enable usage of unstable features cargo.env("RUSTC_BOOTSTRAP", "1"); + + if self.config.dump_bootstrap_shims { + prepare_behaviour_dump_dir(&self.build); + + cargo + .env("DUMP_BOOTSTRAP_SHIMS", self.build.out.join("bootstrap-shims-dump")) + .env("BUILD_OUT", &self.build.out) + .env("CARGO_HOME", t!(home::cargo_home())); + }; + self.add_rust_test_threads(&mut cargo); // Almost all of the crates that we compile as part of the bootstrap may diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 5a9e78f19d6..c3db5641ea4 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -193,6 +193,7 @@ pub struct Config { pub cmd: Subcommand, pub incremental: bool, pub dry_run: DryRun, + pub dump_bootstrap_shims: bool, /// Arguments appearing after `--` to be forwarded to tools, /// e.g. `--fix-broken` or test arguments. pub free_args: Vec, @@ -1210,6 +1211,7 @@ impl Config { config.cmd = flags.cmd; config.incremental = flags.incremental; config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; + config.dump_bootstrap_shims = flags.dump_bootstrap_shims; config.keep_stage = flags.keep_stage; config.keep_stage_std = flags.keep_stage_std; config.color = flags.color; diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs index 27d0425df54..0b13726081b 100644 --- a/src/bootstrap/src/core/config/flags.rs +++ b/src/bootstrap/src/core/config/flags.rs @@ -85,6 +85,9 @@ pub struct Flags { #[arg(global(true), long)] /// dry run; don't build anything pub dry_run: bool, + /// Indicates whether to dump the work done from bootstrap shims + #[arg(global(true), long)] + pub dump_bootstrap_shims: bool, #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index fd0b61eae8d..18f6dc53376 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -1871,3 +1871,22 @@ pub fn generate_smart_stamp_hash(dir: &Path, additional_input: &str) -> String { hex::encode(hasher.finalize().as_slice()) } + +/// Ensures that the behavior dump directory is properly initialized. +pub fn prepare_behaviour_dump_dir(build: &Build) { + static INITIALIZED: OnceLock = OnceLock::new(); + + let dump_path = build.out.join("bootstrap-shims-dump"); + + let initialized = INITIALIZED.get().unwrap_or_else(|| &false); + if !initialized { + // clear old dumps + if dump_path.exists() { + t!(fs::remove_dir_all(&dump_path)); + } + + t!(fs::create_dir_all(&dump_path)); + + t!(INITIALIZED.set(true)); + } +} diff --git a/src/bootstrap/src/utils/bin_helpers.rs b/src/bootstrap/src/utils/bin_helpers.rs index c90fd2805e2..9c4e039ea69 100644 --- a/src/bootstrap/src/utils/bin_helpers.rs +++ b/src/bootstrap/src/utils/bin_helpers.rs @@ -2,14 +2,18 @@ //! dependency on the bootstrap library. This reduces the binary size and //! improves compilation time by reducing the linking time. +use std::env; +use std::fs::OpenOptions; +use std::io::Write; +use std::process::Command; +use std::str::FromStr; + /// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. /// If it was not defined, returns 0 by default. /// /// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. pub(crate) fn parse_rustc_verbose() -> usize { - use std::str::FromStr; - - match std::env::var("RUSTC_VERBOSE") { + match env::var("RUSTC_VERBOSE") { Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), Err(_) => 0, } @@ -19,10 +23,29 @@ pub(crate) fn parse_rustc_verbose() -> usize { /// /// If "RUSTC_STAGE" was not set, the program will be terminated with 101. pub(crate) fn parse_rustc_stage() -> String { - std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { + env::var("RUSTC_STAGE").unwrap_or_else(|_| { // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. eprintln!("rustc shim: FATAL: RUSTC_STAGE was not set"); eprintln!("rustc shim: NOTE: use `x.py build -vvv` to see all environment variables set by bootstrap"); std::process::exit(101); }) } + +/// Writes the command invocation to a file if `DUMP_BOOTSTRAP_SHIMS` is set during bootstrap. +/// +/// Before writing it, replaces user-specific values to create generic dumps for cross-environment +/// comparisons. +pub(crate) fn maybe_dump(dump_name: String, cmd: &Command) { + if let Ok(dump_dir) = env::var("DUMP_BOOTSTRAP_SHIMS") { + let dump_file = format!("{dump_dir}/{dump_name}"); + + let mut file = + OpenOptions::new().create(true).write(true).append(true).open(&dump_file).unwrap(); + + let cmd_dump = format!("{:?}\n", cmd); + let cmd_dump = cmd_dump.replace(&env::var("BUILD_OUT").unwrap(), "${BUILD_OUT}"); + let cmd_dump = cmd_dump.replace(&env::var("CARGO_HOME").unwrap(), "${CARGO_HOME}"); + + file.write_all(cmd_dump.as_bytes()).expect("Unable to write file"); + } +}