From acef1c2c576c778e7a3c8e249298eb10450ae7e8 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 7 Oct 2023 23:33:27 +0300 Subject: [PATCH 1/6] reorganize bootstrap bins and helper module utilizations Signed-off-by: onur-ozkan --- src/bootstrap/bin/_helper.rs | 25 ------------------- src/bootstrap/{ => src}/bin/main.rs | 0 src/bootstrap/{ => src}/bin/rustc.rs | 15 +++++++---- src/bootstrap/{ => src}/bin/rustdoc.rs | 14 +++++++---- .../{ => src}/bin/sccache-plus-cl.rs | 0 5 files changed, 19 insertions(+), 35 deletions(-) delete mode 100644 src/bootstrap/bin/_helper.rs rename src/bootstrap/{ => src}/bin/main.rs (100%) rename src/bootstrap/{ => src}/bin/rustc.rs (98%) rename src/bootstrap/{ => src}/bin/rustdoc.rs (90%) rename src/bootstrap/{ => src}/bin/sccache-plus-cl.rs (100%) diff --git a/src/bootstrap/bin/_helper.rs b/src/bootstrap/bin/_helper.rs deleted file mode 100644 index 46c574c5bf4..00000000000 --- a/src/bootstrap/bin/_helper.rs +++ /dev/null @@ -1,25 +0,0 @@ -/// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. -/// If it was not defined, returns 0 by default. -/// -/// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. -fn parse_rustc_verbose() -> usize { - use std::str::FromStr; - - match std::env::var("RUSTC_VERBOSE") { - Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), - Err(_) => 0, - } -} - -/// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. -/// -/// If "RUSTC_STAGE" was not set, the program will be terminated with 101. -#[allow(unused)] -fn parse_rustc_stage() -> String { - std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { - // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. - eprintln!("rustc shim: fatal: RUSTC_STAGE was not set"); - eprintln!("rustc shim: note: use `x.py build -vvv` to see all environment variables set by bootstrap"); - exit(101); - }) -} diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/src/bin/main.rs similarity index 100% rename from src/bootstrap/bin/main.rs rename to src/bootstrap/src/bin/main.rs diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs similarity index 98% rename from src/bootstrap/bin/rustc.rs rename to src/bootstrap/src/bin/rustc.rs index 6cc5162120a..b050713265b 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -15,19 +15,24 @@ //! switching compilers for the bootstrap and for build scripts will probably //! never get replaced. -include!("../dylib_util.rs"); -include!("./_helper.rs"); - use std::env; use std::path::PathBuf; -use std::process::{exit, Child, Command}; +use std::process::{Child, Command}; use std::time::Instant; +use dylib_util::{dylib_path, dylib_path_var}; + +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib_util.rs"] +mod dylib_util; + fn main() { let args = env::args_os().skip(1).collect::>(); let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str()); - let verbose = parse_rustc_verbose(); + let verbose = bin_helpers::parse_rustc_verbose(); // Detect whether or not we're a build script depending on whether --target // is passed (a bit janky...) diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/src/bin/rustdoc.rs similarity index 90% rename from src/bootstrap/bin/rustdoc.rs rename to src/bootstrap/src/bin/rustdoc.rs index 6561c1c1933..85d685475b4 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/src/bin/rustdoc.rs @@ -5,17 +5,21 @@ use std::env; use std::ffi::OsString; use std::path::PathBuf; -use std::process::{exit, Command}; +use std::process::Command; -include!("../dylib_util.rs"); +use dylib_util::{dylib_path, dylib_path_var}; -include!("./_helper.rs"); +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib_util.rs"] +mod dylib_util; fn main() { let args = env::args_os().skip(1).collect::>(); - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); + let stage = bin_helpers::parse_rustc_stage(); + let verbose = bin_helpers::parse_rustc_verbose(); let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); diff --git a/src/bootstrap/bin/sccache-plus-cl.rs b/src/bootstrap/src/bin/sccache-plus-cl.rs similarity index 100% rename from src/bootstrap/bin/sccache-plus-cl.rs rename to src/bootstrap/src/bin/sccache-plus-cl.rs From c68ab9347ee7d9bf8c93dfb444b9ec5ec0ccc43f Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 7 Oct 2023 23:35:02 +0300 Subject: [PATCH 2/6] improve bootstrap tests structure Signed-off-by: onur-ozkan --- .../{builder/tests.rs => src/tests/builder.rs} | 10 +++++----- src/bootstrap/{config/tests.rs => src/tests/config.rs} | 10 +++++----- src/bootstrap/{setup/tests.rs => src/tests/setup.rs} | 0 3 files changed, 10 insertions(+), 10 deletions(-) rename src/bootstrap/{builder/tests.rs => src/tests/builder.rs} (99%) rename src/bootstrap/{config/tests.rs => src/tests/config.rs} (97%) rename src/bootstrap/{setup/tests.rs => src/tests/setup.rs} (100%) diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/src/tests/builder.rs similarity index 99% rename from src/bootstrap/builder/tests.rs rename to src/bootstrap/src/tests/builder.rs index 0294102286e..96139f7b099 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/src/tests/builder.rs @@ -1,6 +1,6 @@ use super::*; -use crate::config::{Config, DryRun, TargetSelection}; -use crate::doc::DocumentationFormat; +use crate::core::config::{Config, DryRun, TargetSelection}; +use crate::core::build_steps::doc::DocumentationFormat; use std::thread; fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { @@ -158,7 +158,7 @@ fn alias_and_path_for_library() { #[test] fn test_beta_rev_parsing() { - use crate::extract_beta_rev; + use crate::utils::helpers::extract_beta_rev; // single digit revision assert_eq!(extract_beta_rev("1.99.9-beta.7 (xxxxxx)"), Some("7".to_string())); @@ -174,7 +174,7 @@ fn test_beta_rev_parsing() { mod defaults { use super::{configure, first, run_build}; - use crate::builder::*; + use crate::core::builder::*; use crate::Config; use pretty_assertions::assert_eq; @@ -285,7 +285,7 @@ mod defaults { mod dist { use super::{first, run_build, Config}; - use crate::builder::*; + use crate::core::builder::*; use pretty_assertions::assert_eq; fn configure(host: &[&str], target: &[&str]) -> Config { diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/src/tests/config.rs similarity index 97% rename from src/bootstrap/config/tests.rs rename to src/bootstrap/src/tests/config.rs index ae8363b6de9..59bd52a94dc 100644 --- a/src/bootstrap/config/tests.rs +++ b/src/bootstrap/src/tests/config.rs @@ -1,6 +1,6 @@ -use crate::config::TomlConfig; - +use crate::core::config::TomlConfig; use super::{Config, Flags}; + use clap::CommandFactory; use serde::Deserialize; use std::{ @@ -18,7 +18,7 @@ fn parse(config: &str) -> Config { #[test] fn download_ci_llvm() { - if crate::llvm::is_ci_llvm_modified(&parse("")) { + if crate::core::build_steps::llvm::is_ci_llvm_modified(&parse("")) { eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); return; } @@ -137,7 +137,7 @@ build-config = {} assert_eq!(config.change_id, Some(1), "setting top-level value"); assert_eq!( config.rust_lto, - crate::config::RustcLto::Fat, + crate::core::config::RustcLto::Fat, "setting string value without quotes" ); assert_eq!(config.gdb, Some("bar".into()), "setting string value with quotes"); @@ -175,7 +175,7 @@ fn profile_user_dist() { "profile = \"user\"".to_owned() } else { assert!(file.ends_with("config.dist.toml")); - std::fs::read_to_string(dbg!(file)).unwrap() + std::fs::read_to_string(file).unwrap() }; toml::from_str(&contents) .and_then(|table: toml::Value| TomlConfig::deserialize(table)) diff --git a/src/bootstrap/setup/tests.rs b/src/bootstrap/src/tests/setup.rs similarity index 100% rename from src/bootstrap/setup/tests.rs rename to src/bootstrap/src/tests/setup.rs From 2bce0207d4b16393937b268ce7676c6b83813867 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 7 Oct 2023 23:37:06 +0300 Subject: [PATCH 3/6] move bootstrap core implementation to bootstrap/src/core module Signed-off-by: onur-ozkan --- .../{ => src/core/build_steps}/check.rs | 22 +- .../{ => src/core/build_steps}/clean.rs | 6 +- .../{ => src/core/build_steps}/compile.rs | 28 +- .../{ => src/core/build_steps}/dist.rs | 30 +- .../{ => src/core/build_steps}/doc.rs | 14 +- .../{ => src/core/build_steps}/format.rs | 4 +- .../{ => src/core/build_steps}/install.rs | 12 +- .../{ => src/core/build_steps}/llvm.rs | 27 +- src/bootstrap/src/core/build_steps/mod.rs | 15 + .../{ => src/core/build_steps}/run.rs | 16 +- .../{ => src/core/build_steps}/setup.rs | 5 +- .../{ => src/core/build_steps}/suggest.rs | 11 +- .../core/build_steps}/synthetic_targets.rs | 6 +- .../{ => src/core/build_steps}/test.rs | 106 ++-- .../{ => src/core/build_steps}/tool.rs | 12 +- .../{ => src/core/build_steps}/toolstate.rs | 4 +- src/bootstrap/{ => src/core}/builder.rs | 38 +- src/bootstrap/{ => src/core/config}/config.rs | 29 +- src/bootstrap/src/core/config/flags.rs | 565 ++++++++++++++++++ src/bootstrap/src/core/config/mod.rs | 4 + src/bootstrap/{ => src/core}/download.rs | 11 +- src/bootstrap/{ => src/core}/metadata.rs | 4 +- src/bootstrap/src/core/mod.rs | 6 + src/bootstrap/{ => src/core}/sanity.rs | 6 +- 24 files changed, 792 insertions(+), 189 deletions(-) rename src/bootstrap/{ => src/core/build_steps}/check.rs (96%) rename src/bootstrap/{ => src/core/build_steps}/clean.rs (98%) rename src/bootstrap/{ => src/core/build_steps}/compile.rs (98%) rename src/bootstrap/{ => src/core/build_steps}/dist.rs (98%) rename src/bootstrap/{ => src/core/build_steps}/doc.rs (98%) rename src/bootstrap/{ => src/core/build_steps}/format.rs (99%) rename src/bootstrap/{ => src/core/build_steps}/install.rs (98%) rename src/bootstrap/{ => src/core/build_steps}/llvm.rs (98%) create mode 100644 src/bootstrap/src/core/build_steps/mod.rs rename src/bootstrap/{ => src/core/build_steps}/run.rs (96%) rename src/bootstrap/{ => src/core/build_steps}/setup.rs (99%) rename src/bootstrap/{ => src/core/build_steps}/suggest.rs (93%) rename src/bootstrap/{ => src/core/build_steps}/synthetic_targets.rs (95%) rename src/bootstrap/{ => src/core/build_steps}/test.rs (97%) rename src/bootstrap/{ => src/core/build_steps}/tool.rs (99%) rename src/bootstrap/{ => src/core/build_steps}/toolstate.rs (99%) rename src/bootstrap/{ => src/core}/builder.rs (99%) rename src/bootstrap/{ => src/core/config}/config.rs (98%) create mode 100644 src/bootstrap/src/core/config/flags.rs create mode 100644 src/bootstrap/src/core/config/mod.rs rename src/bootstrap/{ => src/core}/download.rs (99%) rename src/bootstrap/{ => src/core}/metadata.rs (98%) create mode 100644 src/bootstrap/src/core/mod.rs rename src/bootstrap/{ => src/core}/sanity.rs (98%) diff --git a/src/bootstrap/check.rs b/src/bootstrap/src/core/build_steps/check.rs similarity index 96% rename from src/bootstrap/check.rs rename to src/bootstrap/src/core/build_steps/check.rs index b417abc00f5..121925b56a0 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -1,10 +1,12 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. -use crate::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo}; -use crate::config::TargetSelection; -use crate::tool::{prepare_tool_cargo, SourceType}; +use crate::core::build_steps::compile::{ + add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, +}; +use crate::core::build_steps::tool::{prepare_tool_cargo, SourceType}; +use crate::core::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::Interned; use crate::INTERNER; use crate::{Compiler, Mode, Subcommand}; use std::path::{Path, PathBuf}; @@ -16,7 +18,7 @@ pub struct Std { /// /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. /// - /// [`compile::Rustc`]: crate::compile::Rustc + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc crates: Interned>, } @@ -193,7 +195,7 @@ pub struct Rustc { /// /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. /// - /// [`compile::Rustc`]: crate::compile::Rustc + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc crates: Interned>, } @@ -237,8 +239,8 @@ impl Step for Rustc { // the sysroot for the compiler to find. Otherwise, we're going to // fail when building crates that need to generate code (e.g., build // scripts and their dependencies). - builder.ensure(crate::compile::Std::new(compiler, compiler.host)); - builder.ensure(crate::compile::Std::new(compiler, target)); + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host)); + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target)); } else { builder.ensure(Std::new(target)); } @@ -387,7 +389,7 @@ impl Step for RustAnalyzer { &["rust-analyzer/in-rust-tree".to_owned()], ); - cargo.allow_features(crate::tool::RustAnalyzer::ALLOW_FEATURES); + cargo.allow_features(crate::core::build_steps::tool::RustAnalyzer::ALLOW_FEATURES); // For ./x.py clippy, don't check those targets because // linting tests and benchmarks can produce very noisy results diff --git a/src/bootstrap/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs similarity index 98% rename from src/bootstrap/clean.rs rename to src/bootstrap/src/core/build_steps/clean.rs index 7389816b44c..679770ce0ec 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -9,9 +9,9 @@ use std::fs; use std::io::{self, ErrorKind}; use std::path::Path; -use crate::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::util::t; +use crate::core::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; +use crate::utils::cache::Interned; +use crate::utils::helpers::t; use crate::{Build, Compiler, Mode, Subcommand}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/src/bootstrap/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs similarity index 98% rename from src/bootstrap/compile.rs rename to src/bootstrap/src/core/build_steps/compile.rs index 623fa5fa111..441931e415c 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -19,16 +19,17 @@ use std::str; use serde_derive::Deserialize; -use crate::builder::crate_description; -use crate::builder::Cargo; -use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; -use crate::cache::{Interned, INTERNER}; -use crate::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; -use crate::dist; -use crate::llvm; -use crate::tool::SourceType; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date}; +use crate::core::build_steps::dist; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::SourceType; +use crate::core::builder::crate_description; +use crate::core::builder::Cargo; +use crate::core::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; +use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{ + exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date, +}; use crate::LLVM_TOOLS; use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; use filetime::FileTime; @@ -510,7 +511,7 @@ impl Step for StdLink { let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder.ensure(crate::compile::Sysroot { + let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot { compiler: self.compiler, force_recompile: self.force_recompile, }); @@ -1016,7 +1017,8 @@ pub fn rustc_cargo_env( // detected that LLVM is already built and good to go which helps prevent // busting caches (e.g. like #71152). if builder.config.llvm_enabled() { - let building_is_expensive = crate::llvm::prebuilt_llvm_config(builder, target).is_err(); + let building_is_expensive = + crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err(); // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; let should_skip_build = building_is_expensive && can_skip_build; @@ -1684,7 +1686,7 @@ impl Step for Assemble { builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); let self_contained_lld_dir = libdir_bin.join("gcc-ld"); t!(fs::create_dir(&self_contained_lld_dir)); - let lld_wrapper_exe = builder.ensure(crate::tool::LldWrapper { + let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { compiler: build_compiler, target: target_compiler.host, }); diff --git a/src/bootstrap/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs similarity index 98% rename from src/bootstrap/dist.rs rename to src/bootstrap/src/core/build_steps/dist.rs index 7bb59e5a611..47f41ab288d 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -19,16 +19,16 @@ use std::process::Command; use object::read::archive::ArchiveFile; use object::BinaryFormat; -use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::channel; -use crate::compile; -use crate::config::TargetSelection; -use crate::doc::DocumentationFormat; -use crate::llvm; -use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; -use crate::tool::{self, Tool}; -use crate::util::{exe, is_dylib, output, t, timeit}; +use crate::core::build_steps::compile; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, Tool}; +use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::channel; +use crate::utils::helpers::{exe, is_dylib, output, t, timeit}; +use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball}; use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { @@ -104,7 +104,7 @@ impl Step for JsonDocs { /// Builds the `rust-docs-json` installer component. fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; - builder.ensure(crate::doc::Std::new( + builder.ensure(crate::core::build_steps::doc::Std::new( builder.top_stage, host, builder, @@ -488,7 +488,7 @@ impl Step for Rustc { let man_src = builder.src.join("src/doc/man"); let man_dst = image.join("share/man/man1"); - // don't use our `bootstrap::util::{copy, cp_r}`, because those try + // don't use our `bootstrap::{copy, cp_r}`, because those try // to hardlink, and we don't want to edit the source templates for file_entry in builder.read_dir(&man_src) { let page_src = file_entry.path(); @@ -2060,7 +2060,7 @@ impl Step for LlvmTools { } } - builder.ensure(crate::llvm::Llvm { target }); + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); tarball.set_overlay(OverlayKind::LLVM); @@ -2119,10 +2119,10 @@ impl Step for RustDev { let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); tarball.set_overlay(OverlayKind::LLVM); - builder.ensure(crate::llvm::Llvm { target }); + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); // We want to package `lld` to use it with `download-ci-llvm`. - builder.ensure(crate::llvm::Lld { target }); + builder.ensure(crate::core::build_steps::llvm::Lld { target }); let src_bindir = builder.llvm_out(target).join("bin"); // If updating this list, you likely want to change diff --git a/src/bootstrap/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs similarity index 98% rename from src/bootstrap/doc.rs rename to src/bootstrap/src/core/build_steps/doc.rs index 505f06ed12d..628a4ece8e9 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -10,13 +10,13 @@ use std::fs; use std::path::{Path, PathBuf}; -use crate::builder::crate_description; -use crate::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::compile; -use crate::config::{Config, TargetSelection}; -use crate::tool::{self, prepare_tool_cargo, SourceType, Tool}; -use crate::util::{dir_is_empty, symlink_dir, t, up_to_date}; +use crate::core::build_steps::compile; +use crate::core::build_steps::tool::{self, prepare_tool_cargo, SourceType, Tool}; +use crate::core::builder::crate_description; +use crate::core::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{dir_is_empty, symlink_dir, t, up_to_date}; use crate::Mode; macro_rules! submodule_helper { diff --git a/src/bootstrap/format.rs b/src/bootstrap/src/core/build_steps/format.rs similarity index 99% rename from src/bootstrap/format.rs rename to src/bootstrap/src/core/build_steps/format.rs index 11f2762f766..0e260e69c85 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -1,7 +1,7 @@ //! Runs rustfmt on the repository. -use crate::builder::Builder; -use crate::util::{output, program_out_of_date, t}; +use crate::core::builder::Builder; +use crate::utils::helpers::{output, program_out_of_date, t}; use build_helper::ci::CiEnv; use build_helper::git::get_git_modified_files; use ignore::WalkBuilder; diff --git a/src/bootstrap/install.rs b/src/bootstrap/src/core/build_steps/install.rs similarity index 98% rename from src/bootstrap/install.rs rename to src/bootstrap/src/core/build_steps/install.rs index 885b3a78236..391995b7c3b 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/src/core/build_steps/install.rs @@ -8,15 +8,13 @@ use std::fs; use std::path::{Component, Path, PathBuf}; use std::process::Command; -use crate::util::t; - -use crate::dist; -use crate::tarball::GeneratedTarball; +use crate::core::build_steps::dist; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::helpers::t; +use crate::utils::tarball::GeneratedTarball; use crate::{Compiler, Kind}; -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::{Config, TargetSelection}; - #[cfg(target_os = "illumos")] const SHELL: &str = "bash"; #[cfg(not(target_os = "illumos"))] diff --git a/src/bootstrap/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs similarity index 98% rename from src/bootstrap/llvm.rs rename to src/bootstrap/src/core/build_steps/llvm.rs index 4556831589b..24351118a5a 100644 --- a/src/bootstrap/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -16,11 +16,10 @@ use std::io; use std::path::{Path, PathBuf}; use std::process::Command; -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::channel; -use crate::config::{Config, TargetSelection}; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{self, exe, output, t, up_to_date}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::channel; +use crate::utils::helpers::{self, exe, get_clang_cl_resource_dir, output, t, up_to_date}; use crate::{CLang, GitRepo, Kind}; use build_helper::ci::CiEnv; @@ -281,7 +280,7 @@ impl Step for Llvm { let _guard = builder.msg_unstaged(Kind::Build, "LLVM", target); t!(stamp.remove()); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); t!(fs::create_dir_all(&out_dir)); // https://llvm.org/docs/CMake.html @@ -410,7 +409,7 @@ impl Step for Llvm { let mut enabled_llvm_projects = Vec::new(); - if util::forcing_clang_based_tests() { + if helpers::forcing_clang_based_tests() { enabled_llvm_projects.push("clang"); enabled_llvm_projects.push("compiler-rt"); } @@ -528,8 +527,12 @@ impl Step for Llvm { // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its // debuginfo. - crate::compile::strip_debug(builder, target, &out_dir.join("lib").join(&lib_name)); - crate::compile::strip_debug( + crate::core::build_steps::compile::strip_debug( + builder, + target, + &out_dir.join("lib").join(&lib_name), + ); + crate::core::build_steps::compile::strip_debug( builder, target, &out_dir.join("build").join("lib").join(&lib_name), @@ -846,7 +849,7 @@ impl Step for Lld { } let _guard = builder.msg_unstaged(Kind::Build, "LLD", target); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); t!(fs::create_dir_all(&out_dir)); let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); @@ -877,7 +880,7 @@ impl Step for Lld { // `LD_LIBRARY_PATH` overrides) // if builder.config.rpath_enabled(target) - && util::use_host_linker(target) + && helpers::use_host_linker(target) && builder.config.llvm_link_shared() && target.contains("linux") { @@ -970,7 +973,7 @@ impl Step for Sanitizers { let _guard = builder.msg_unstaged(Kind::Build, "sanitizers", self.target); t!(stamp.remove()); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let mut cfg = cmake::Config::new(&compiler_rt_dir); cfg.profile("Release"); diff --git a/src/bootstrap/src/core/build_steps/mod.rs b/src/bootstrap/src/core/build_steps/mod.rs new file mode 100644 index 00000000000..50d83789be8 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/mod.rs @@ -0,0 +1,15 @@ +pub(crate) mod check; +pub(crate) mod clean; +pub(crate) mod compile; +pub(crate) mod dist; +pub(crate) mod doc; +pub(crate) mod format; +pub(crate) mod install; +pub(crate) mod llvm; +pub(crate) mod run; +pub(crate) mod setup; +pub(crate) mod suggest; +pub(crate) mod synthetic_targets; +pub(crate) mod test; +pub(crate) mod tool; +pub(crate) mod toolstate; diff --git a/src/bootstrap/run.rs b/src/bootstrap/src/core/build_steps/run.rs similarity index 96% rename from src/bootstrap/run.rs rename to src/bootstrap/src/core/build_steps/run.rs index f253f5225a1..29272bebf99 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -1,13 +1,15 @@ use std::path::PathBuf; use std::process::Command; -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::TargetSelection; -use crate::dist::distdir; -use crate::flags::get_completion; -use crate::test; -use crate::tool::{self, SourceType, Tool}; -use crate::util::output; +use clap_complete::shells; + +use crate::core::build_steps::dist::distdir; +use crate::core::build_steps::test; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::TargetSelection; +use crate::utils::helpers::output; use crate::Mode; #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] diff --git a/src/bootstrap/setup.rs b/src/bootstrap/src/core/build_steps/setup.rs similarity index 99% rename from src/bootstrap/setup.rs rename to src/bootstrap/src/core/build_steps/setup.rs index 14ec33147fb..ebe093674fc 100644 --- a/src/bootstrap/setup.rs +++ b/src/bootstrap/src/core/build_steps/setup.rs @@ -1,4 +1,4 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::Config; use crate::{t, CONFIG_CHANGE_HISTORY}; use sha2::Digest; @@ -12,6 +12,7 @@ use std::str::FromStr; use std::{fmt, fs, io}; #[cfg(test)] +#[path = "../../tests/setup.rs"] mod tests; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] @@ -35,7 +36,7 @@ static SETTINGS_HASHES: &[&str] = &[ "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", "b526bd58d0262dd4dda2bff5bc5515b705fb668a46235ace3e057f807963a11a", ]; -static RUST_ANALYZER_SETTINGS: &str = include_str!("../etc/rust_analyzer_settings.json"); +static RUST_ANALYZER_SETTINGS: &str = include_str!("../../../../etc/rust_analyzer_settings.json"); impl Profile { fn include_path(&self, src_path: &Path) -> PathBuf { diff --git a/src/bootstrap/suggest.rs b/src/bootstrap/src/core/build_steps/suggest.rs similarity index 93% rename from src/bootstrap/suggest.rs rename to src/bootstrap/src/core/build_steps/suggest.rs index f225104bda9..82fb10cebe0 100644 --- a/src/bootstrap/suggest.rs +++ b/src/bootstrap/src/core/build_steps/suggest.rs @@ -1,12 +1,11 @@ #![cfg_attr(feature = "build-metrics", allow(unused))] +use clap::Parser; +use std::path::PathBuf; use std::str::FromStr; -use std::path::PathBuf; - -use clap::Parser; - -use crate::{builder::Builder, tool::Tool}; +use crate::core::build_steps::tool::Tool; +use crate::core::builder::Builder; /// Suggests a list of possible `x.py` commands to run based on modified files in branch. pub fn suggest(builder: &Builder<'_>, run: bool) { @@ -62,7 +61,7 @@ pub fn suggest(builder: &Builder<'_>, run: bool) { for sug in suggestions { let mut build: crate::Build = builder.build.clone(); build.config.paths = sug.2; - build.config.cmd = crate::flags::Flags::parse_from(["x.py", sug.0]).cmd; + build.config.cmd = crate::core::config::flags::Flags::parse_from(["x.py", sug.0]).cmd; if let Some(stage) = sug.1 { build.config.stage = stage; } diff --git a/src/bootstrap/synthetic_targets.rs b/src/bootstrap/src/core/build_steps/synthetic_targets.rs similarity index 95% rename from src/bootstrap/synthetic_targets.rs rename to src/bootstrap/src/core/build_steps/synthetic_targets.rs index 7eeac9025c9..d2c65b740da 100644 --- a/src/bootstrap/synthetic_targets.rs +++ b/src/bootstrap/src/core/build_steps/synthetic_targets.rs @@ -7,8 +7,8 @@ //! one of the target specs already defined in this module, or create new ones by adding a new step //! that calls create_synthetic_target. -use crate::builder::{Builder, ShouldRun, Step}; -use crate::config::TargetSelection; +use crate::core::builder::{Builder, ShouldRun, Step}; +use crate::core::config::TargetSelection; use crate::Compiler; use std::process::{Command, Stdio}; @@ -76,7 +76,7 @@ fn create_synthetic_target( std::fs::write(&path, &serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); let target = TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - crate::cc_detect::find_target(builder, target); + crate::utils::cc_detect::find_target(builder, target); target } diff --git a/src/bootstrap/test.rs b/src/bootstrap/src/core/build_steps/test.rs similarity index 97% rename from src/bootstrap/test.rs rename to src/bootstrap/src/core/build_steps/test.rs index fb8ec0355c2..831a86940fb 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -13,21 +13,24 @@ use std::process::{Command, Stdio}; use clap_complete::shells; -use crate::builder::crate_description; -use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::cache::INTERNER; -use crate::compile; -use crate::config::TargetSelection; -use crate::dist; -use crate::doc::DocumentationFormat; -use crate::flags::Subcommand; -use crate::llvm; -use crate::render_tests::add_flags_and_try_run_tests; -use crate::synthetic_targets::MirOptPanicAbortSyntheticTarget; -use crate::tool::{self, SourceType, Tool}; -use crate::toolstate::ToolState; -use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date}; +use crate::core::build_steps::compile; +use crate::core::build_steps::dist; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::synthetic_targets::MirOptPanicAbortSyntheticTarget; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::crate_description; +use crate::core::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::flags::Subcommand; +use crate::core::config::TargetSelection; +use crate::utils; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{ + self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date, +}; +use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests}; use crate::{envify, CLang, DocTests, GitRepo, Mode}; const ADB_TEST_DIR: &str = "/data/local/tmp/work"; @@ -167,7 +170,7 @@ You can skip linkcheck with --skip src/tools/linkchecker" // Run the linkchecker. let _guard = builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run_delaying_failure(linkchecker.arg(builder.out.join(host.triple).join("doc"))); } @@ -219,7 +222,11 @@ impl Step for HtmlCheck { } // Ensure that a few different kinds of documentation are available. builder.default_doc(&[]); - builder.ensure(crate::doc::Rustc::new(builder.top_stage, self.target, builder)); + builder.ensure(crate::core::build_steps::doc::Rustc::new( + builder.top_stage, + self.target, + builder, + )); builder.run_delaying_failure( builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)), @@ -260,7 +267,7 @@ impl Step for Cargotest { let out_dir = builder.out.join("ct"); t!(fs::create_dir_all(&out_dir)); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let mut cmd = builder.tool_cmd(Tool::CargoTest); builder.run_delaying_failure( cmd.arg(&cargo) @@ -328,7 +335,7 @@ impl Step for Cargo { builder, ); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); add_flags_and_try_run_tests(builder, &mut cargo); } } @@ -642,7 +649,7 @@ impl Step for Miri { // does not understand the flags added by `add_flags_and_try_run_test`. let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); { - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run(&mut cargo); } @@ -658,7 +665,7 @@ impl Step for Miri { let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); { - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run(&mut cargo); } } @@ -698,7 +705,7 @@ impl Step for Miri { let mut cargo = Command::from(cargo); { - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run(&mut cargo); } } @@ -859,7 +866,7 @@ impl Step for RustdocTheme { if builder.is_fuse_ld_lld(self.compiler.host) { cmd.env( "RUSTDOC_LLD_NO_THREADS", - util::lld_flag_no_threads(self.compiler.host.contains("windows")), + helpers::lld_flag_no_threads(self.compiler.host.contains("windows")), ); } builder.run_delaying_failure(&mut cmd); @@ -900,7 +907,8 @@ impl Step for RustdocJSStd { .arg("--test-folder") .arg(builder.src.join("tests/rustdoc-js-std")); for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) + { if !p.ends_with(".js") { eprintln!("A non-js file was given: `{}`", path.display()); panic!("Cannot run rustdoc-js-std tests"); @@ -908,7 +916,7 @@ impl Step for RustdocJSStd { command.arg("--test-file").arg(path); } } - builder.ensure(crate::doc::Std::new( + builder.ensure(crate::core::build_steps::doc::Std::new( builder.top_stage, self.target, builder, @@ -1035,7 +1043,7 @@ impl Step for RustdocGUI { .env("RUSTC", builder.rustc(self.compiler)); for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { if !p.ends_with(".goml") { eprintln!("A non-goml file was given: `{}`", path.display()); panic!("Cannot run rustdoc-gui tests"); @@ -1058,7 +1066,7 @@ impl Step for RustdocGUI { cmd.arg("--npm").arg(npm); } - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let _guard = builder.msg_sysroot_tool( Kind::Test, self.compiler.stage, @@ -1066,7 +1074,7 @@ impl Step for RustdocGUI { self.compiler.host, self.target, ); - crate::render_tests::try_run_tests(builder, &mut cmd, true); + try_run_tests(builder, &mut cmd, true); } } @@ -1126,7 +1134,7 @@ help: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to ); crate::exit!(1); } - crate::format::format(&builder, !builder.config.cmd.bless(), &[]); + crate::core::build_steps::format::format(&builder, !builder.config.cmd.bless(), &[]); } builder.info("tidy check"); @@ -1138,10 +1146,10 @@ help: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"] .map(|filename| builder.src.join("src/etc/completions").join(filename)); if builder.config.cmd.bless() { - builder.ensure(crate::run::GenerateCompletions); - } else if crate::flags::get_completion(shells::Bash, &bash).is_some() - || crate::flags::get_completion(shells::Fish, &fish).is_some() - || crate::flags::get_completion(shells::PowerShell, &powershell).is_some() + builder.ensure(crate::core::build_steps::run::GenerateCompletions); + } else if get_completion(shells::Bash, &bash).is_some() + || get_completion(shells::Fish, &fish).is_some() + || get_completion(shells::PowerShell, &powershell).is_some() || crate::flags::get_completion(shells::Zsh, &zsh).is_some() { eprintln!( @@ -1403,10 +1411,10 @@ impl Step for MirOpt { // have been detected by bootstrap if the target we're testing wasn't in the // --target flags. if !builder.cc.borrow().contains_key(&target_32bit) { - crate::cc_detect::find_target(builder, target_32bit); + utils::cc_detect::find_target(builder, target_32bit); } if !builder.cc.borrow().contains_key(&target_64bit) { - crate::cc_detect::find_target(builder, target_64bit); + utils::cc_detect::find_target(builder, target_64bit); } vec![target_32bit, target_64bit] @@ -1679,7 +1687,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the } } - if util::forcing_clang_based_tests() { + if helpers::forcing_clang_based_tests() { let clang_exe = builder.llvm_out(target).join("bin").join("clang"); cmd.arg("--run-clang-based-tests-with").arg(clang_exe); } @@ -1698,7 +1706,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the // Get test-args by striping suite path let mut test_args: Vec<&str> = paths .iter() - .filter_map(|p| util::is_valid_test_suite_arg(p, suite_path, builder)) + .filter_map(|p| helpers::is_valid_test_suite_arg(p, suite_path, builder)) .collect(); test_args.append(&mut builder.config.test_args()); @@ -1887,7 +1895,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the compiler.host, target, ); - crate::render_tests::try_run_tests(builder, &mut cmd, false); + try_run_tests(builder, &mut cmd, false); if let Some(compare_mode) = compare_mode { cmd.arg("--compare-mode").arg(compare_mode); @@ -1909,8 +1917,8 @@ note: if you're sure you want to do this, please open an issue as to why. In the "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", suite, mode, compare_mode, &compiler.host, target )); - let _time = util::timeit(&builder); - crate::render_tests::try_run_tests(builder, &mut cmd, false); + let _time = helpers::timeit(&builder); + try_run_tests(builder, &mut cmd, false); } } } @@ -1981,7 +1989,7 @@ impl BookTest { compiler.host, compiler.host, ); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let toolstate = if builder.run_delaying_failure(&mut rustbook_cmd) { ToolState::TestPass } else { @@ -2003,7 +2011,7 @@ impl BookTest { // Do a breadth-first traversal of the `src/doc` directory and just run // tests for all files that end in `*.md` let mut stack = vec![builder.src.join(self.path)]; - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let mut files = Vec::new(); while let Some(p) = stack.pop() { if p.is_dir() { @@ -2114,7 +2122,7 @@ impl Step for ErrorIndex { let guard = builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); builder.run_quiet(&mut tool); drop(guard); // The tests themselves need to link to std, so make sure it is @@ -2236,7 +2244,7 @@ fn run_cargo_test<'a>( ) -> bool { let mut cargo = prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); let _group = description.into().and_then(|what| { builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) }); @@ -2631,7 +2639,7 @@ impl Step for RemoteCopyLibs { for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) { let f = t!(f); let name = f.file_name().into_string().unwrap(); - if util::is_dylib(&name) { + if helpers::is_dylib(&name) { builder.run(Command::new(&tool).arg("push").arg(f.path())); } } @@ -2676,7 +2684,9 @@ impl Step for Distcheck { .current_dir(&dir), ); builder.run( - Command::new(util::make(&builder.config.build.triple)).arg("check").current_dir(&dir), + Command::new(helpers::make(&builder.config.build.triple)) + .arg("check") + .current_dir(&dir), ); // Now make sure that rust-src has all of libstd's dependencies @@ -2833,7 +2843,7 @@ impl Step for LintDocs { /// Tests that the lint examples in the rustc book generate the correct /// lints and have the expected format. fn run(self, builder: &Builder<'_>) { - builder.ensure(crate::doc::RustcBook { + builder.ensure(crate::core::build_steps::doc::RustcBook { compiler: self.compiler, target: self.target, validate: true, @@ -3052,7 +3062,7 @@ impl Step for CodegenCranelift { &compiler.host, target )); - let _time = util::timeit(&builder); + let _time = helpers::timeit(&builder); // FIXME handle vendoring for source tarballs before removing the --skip-test below let download_dir = builder.out.join("cg_clif_download"); diff --git a/src/bootstrap/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs similarity index 99% rename from src/bootstrap/tool.rs rename to src/bootstrap/src/core/build_steps/tool.rs index f094dd9d7c9..5702fa62d7c 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -3,12 +3,12 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; -use crate::channel::GitInfo; -use crate::compile; -use crate::config::TargetSelection; -use crate::toolstate::ToolState; -use crate::util::{add_dylib_path, exe, t}; +use crate::core::build_steps::compile; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::channel::GitInfo; +use crate::utils::helpers::{add_dylib_path, exe, t}; use crate::Compiler; use crate::Mode; use crate::{gha, Kind}; diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/src/core/build_steps/toolstate.rs similarity index 99% rename from src/bootstrap/toolstate.rs rename to src/bootstrap/src/core/build_steps/toolstate.rs index 308023537d5..f892577ccbf 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/src/core/build_steps/toolstate.rs @@ -1,5 +1,5 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::util::t; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::utils::helpers::t; use serde_derive::{Deserialize, Serialize}; use std::collections::HashMap; use std::env; diff --git a/src/bootstrap/builder.rs b/src/bootstrap/src/core/builder.rs similarity index 99% rename from src/bootstrap/builder.rs rename to src/bootstrap/src/core/builder.rs index c714b09ec3c..039a87e760d 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -12,20 +12,15 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{Duration, Instant}; -use crate::cache::{Cache, Interned, INTERNER}; -use crate::config::{DryRun, SplitDebuginfo, TargetSelection}; -use crate::doc; -use crate::flags::{Color, Subcommand}; -use crate::install; -use crate::llvm; -use crate::run; -use crate::setup; -use crate::test; -use crate::tool::{self, SourceType}; -use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, SourceType}; +use crate::core::build_steps::{check, clean, compile, dist, doc, install, run, setup, test}; +use crate::core::config::flags::{Color, Subcommand}; +use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection}; +use crate::utils::cache::{Cache, Interned, INTERNER}; +use crate::utils::helpers::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; +use crate::Crate; use crate::EXTRA_CHECK_CFGS; -use crate::{check, compile, Crate}; -use crate::{clean, dist}; use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; @@ -36,6 +31,10 @@ pub use crate::Compiler; use clap::ValueEnum; use once_cell::sync::{Lazy, OnceCell}; +#[cfg(test)] +#[path = "../tests/builder.rs"] +mod tests; + pub struct Builder<'a> { pub build: &'a Build, pub top_stage: u32, @@ -723,7 +722,7 @@ impl<'a> Builder<'a> { check::Bootstrap ), Kind::Test => describe!( - crate::toolstate::ToolStateCheck, + crate::core::build_steps::toolstate::ToolStateCheck, test::ExpandYamlAnchors, test::Tidy, test::Ui, @@ -1297,8 +1296,8 @@ impl<'a> Builder<'a> { // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config // needs to not accidentally link to libLLVM in stage0/lib. - cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var()); - if let Some(e) = env::var_os(util::dylib_path_var()) { + cargo.env("REAL_LIBRARY_PATH_VAR", &helpers::dylib_path_var()); + if let Some(e) = env::var_os(helpers::dylib_path_var()) { cargo.env("REAL_LIBRARY_PATH", e); } @@ -1311,7 +1310,7 @@ impl<'a> Builder<'a> { // rustc_llvm. But if LLVM is stale, that'll be a tiny amount // of work comparatively, and we'd likely need to rebuild it anyway, // so that's okay. - if crate::llvm::prebuilt_llvm_config(self, target).is_err() { + if crate::core::build_steps::llvm::prebuilt_llvm_config(self, target).is_err() { cargo.env("RUST_CHECK", "1"); } } @@ -1643,7 +1642,7 @@ impl<'a> Builder<'a> { // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it // fun to pass a flag to a tool to pass a flag to pass a flag to a tool // to change a flag in a binary? - if self.config.rpath_enabled(target) && util::use_host_linker(target) { + if self.config.rpath_enabled(target) && helpers::use_host_linker(target) { let libdir = self.sysroot_libdir_relative(compiler).to_str().unwrap(); let rpath = if target.contains("apple") { // Note that we need to take one extra step on macOS to also pass @@ -2197,9 +2196,6 @@ impl<'a> Builder<'a> { } } -#[cfg(test)] -mod tests; - /// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler later. /// /// `-Z crate-attr` flags will be applied recursively on the target code using the `rustc_parse::parser::Parser`. diff --git a/src/bootstrap/config.rs b/src/bootstrap/src/core/config/config.rs similarity index 98% rename from src/bootstrap/config.rs rename to src/bootstrap/src/core/config/config.rs index 759d874a089..7ca1bbad968 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -4,6 +4,7 @@ //! how the build runs. #[cfg(test)] +#[path = "../../tests/config.rs"] mod tests; use std::cell::{Cell, RefCell}; @@ -17,19 +18,20 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; -use crate::cache::{Interned, INTERNER}; -use crate::cc_detect::{ndk_compiler, Language}; -use crate::channel::{self, GitInfo}; -use crate::compile::CODEGEN_BACKEND_PREFIX; -pub use crate::flags::Subcommand; -use crate::flags::{Color, Flags, Warnings}; -use crate::util::{exe, output, t}; +use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::config::flags::{Color, Flags, Warnings}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::cc_detect::{ndk_compiler, Language}; +use crate::utils::channel::{self, GitInfo}; +use crate::utils::helpers::{exe, output, t}; use build_helper::exit; use once_cell::sync::OnceCell; use semver::Version; use serde::{Deserialize, Deserializer}; use serde_derive::Deserialize; +pub use crate::core::config::flags::Subcommand; + macro_rules! check_ci_llvm { ($name:expr) => { assert!( @@ -547,7 +549,7 @@ impl Target { /// `Config` structure. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] -struct TomlConfig { +pub(crate) struct TomlConfig { changelog_seen: Option, // FIXME: Deprecated field. Remove it at 2024. change_id: Option, build: Option, @@ -1269,7 +1271,7 @@ impl Config { // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. if !config.out.is_absolute() { // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = crate::util::absolute(&config.out); + config.out = crate::utils::helpers::absolute(&config.out); } config.initial_rustc = if let Some(rustc) = build.rustc { @@ -1527,11 +1529,12 @@ impl Config { config.llvm_from_ci = match llvm.download_ci_llvm { Some(StringOrBool::String(s)) => { assert_eq!(s, "if-available", "unknown option `{s}` for download-ci-llvm"); - crate::llvm::is_ci_llvm_available(&config, asserts) + crate::core::build_steps::llvm::is_ci_llvm_available(&config, asserts) } Some(StringOrBool::Bool(b)) => b, None => { - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, asserts) + config.channel == "dev" + && crate::core::build_steps::llvm::is_ci_llvm_available(&config, asserts) } }; @@ -1573,8 +1576,8 @@ impl Config { config.llvm_link_shared.set(Some(true)); } } else { - config.llvm_from_ci = - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, false); + config.llvm_from_ci = config.channel == "dev" + && crate::core::build_steps::llvm::is_ci_llvm_available(&config, false); } if let Some(t) = toml.target { diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs new file mode 100644 index 00000000000..92aea76382a --- /dev/null +++ b/src/bootstrap/src/core/config/flags.rs @@ -0,0 +1,565 @@ +//! Command-line interface of the rustbuild build system. +//! +//! This module implements the command-line parsing of the build system which +//! has various flags to configure how it's run. + +use std::path::{Path, PathBuf}; + +use clap::{CommandFactory, Parser, ValueEnum}; + +use crate::core::build_steps::setup::Profile; +use crate::core::builder::{Builder, Kind}; +use crate::core::config::{target_selection_list, Config, TargetSelectionList}; +use crate::{Build, DocTests}; + +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Color { + Always, + Never, + #[default] + Auto, +} + +/// Whether to deny warnings, emit them as warnings, or use the default behavior +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Warnings { + Deny, + Warn, + #[default] + Default, +} + +/// Deserialized version of all flags for this compile. +#[derive(Debug, Parser)] +#[clap( + override_usage = "x.py [options] [...]", + disable_help_subcommand(true), + about = "", + next_line_help(false) +)] +pub struct Flags { + #[command(subcommand)] + pub cmd: Subcommand, + + #[arg(global(true), short, long, action = clap::ArgAction::Count)] + /// use verbose output (-vv for very verbose) + pub verbose: u8, // each extra -v after the first is passed to Cargo + #[arg(global(true), short, long)] + /// use incremental compilation + pub incremental: bool, + #[arg(global(true), long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] + /// TOML configuration file for build + pub config: Option, + #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// Build directory, overrides `build.build-dir` in `config.toml` + pub build_dir: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] + /// build target of the stage0 compiler + pub build: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] + /// host targets to build + pub host: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] + /// target targets to build + pub target: Option, + + #[arg(global(true), long, value_name = "PATH")] + /// build paths to exclude + pub exclude: Vec, // keeping for client backward compatibility + #[arg(global(true), long, value_name = "PATH")] + /// build paths to skip + pub skip: Vec, + #[arg(global(true), long)] + /// include default paths in addition to the provided ones + pub include_default_paths: bool, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long)] + pub rustc_error_format: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] + /// command to run on failure + pub on_fail: Option, + #[arg(global(true), long)] + /// dry run; don't build anything + pub dry_run: bool, + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the + /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) + pub stage: Option, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage: Vec, + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) of the standard library to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage_std: Vec, + #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// path to the root of the rust checkout + pub src: Option, + + #[arg( + global(true), + short, + long, + value_hint = clap::ValueHint::Other, + default_value_t = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), + value_name = "JOBS" + )] + /// number of jobs to run in parallel + pub jobs: usize, + // This overrides the deny-warnings configuration option, + // which passes -Dwarnings to the compiler invocations. + #[arg(global(true), long)] + #[clap(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] + /// if value is deny, will deny warnings + /// if value is warn, will emit warnings + /// otherwise, use the default configured behaviour + pub warnings: Warnings, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] + /// rustc error format + pub error_format: Option, + #[arg(global(true), long)] + /// use message-format=json + pub json_output: bool, + + #[arg(global(true), long, value_name = "STYLE")] + #[clap(value_enum, default_value_t = Color::Auto)] + /// whether to use color in cargo and rustc output + pub color: Color, + + /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml + #[arg(global(true), long, value_name = "VALUE")] + pub llvm_skip_rebuild: Option, + /// generate PGO profile with rustc build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_generate: Option, + /// use PGO profile for rustc build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_use: Option, + /// use PGO profile for LLVM build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub llvm_profile_use: Option, + // LLVM doesn't support a custom location for generating profile + // information. + // + // llvm_out/build/profiles/ is the location this writes to. + /// generate PGO profile with llvm built for rustc + #[arg(global(true), long)] + pub llvm_profile_generate: bool, + /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. + #[arg(global(true), long)] + pub reproducible_artifact: Vec, + #[arg(global(true))] + /// paths for the subcommand + pub paths: Vec, + /// override options in config.toml + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] + pub set: Vec, + /// arguments passed to subcommands + #[arg(global(true), last(true), value_name = "ARGS")] + pub free_args: Vec, +} + +impl Flags { + pub fn parse(args: &[String]) -> Self { + let first = String::from("x.py"); + let it = std::iter::once(&first).chain(args.iter()); + // We need to check for ` -h -v`, in which case we list the paths + #[derive(Parser)] + #[clap(disable_help_flag(true))] + struct HelpVerboseOnly { + #[arg(short, long)] + help: bool, + #[arg(global(true), short, long, action = clap::ArgAction::Count)] + pub verbose: u8, + #[arg(value_enum)] + cmd: Kind, + } + if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = + HelpVerboseOnly::try_parse_from(it.clone()) + { + println!("note: updating submodules before printing available paths"); + let config = Config::parse(&[String::from("build")]); + let build = Build::new(config); + let paths = Builder::get_help(&build, subcommand); + if let Some(s) = paths { + println!("{s}"); + } else { + panic!("No paths available for subcommand `{}`", subcommand.as_str()); + } + crate::exit!(0); + } + + Flags::parse_from(it) + } +} + +#[derive(Debug, Clone, Default, clap::Subcommand)] +pub enum Subcommand { + #[clap(aliases = ["b"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example, for a quick build of a usable + compiler: + ./x.py build --stage 1 library/std + This will build a compiler and standard library from the local source code. + Once this is done, build/$ARCH/stage1 contains a usable compiler. + If no arguments are passed then the default artifacts for that stage are + compiled. For example: + ./x.py build --stage 0 + ./x.py build ")] + /// Compile either the compiler or libraries + #[default] + Build, + #[clap(aliases = ["c"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example: + ./x.py check library/std + If no arguments are passed then many artifacts are checked.")] + /// Compile either the compiler or libraries, using cargo check + Check { + #[arg(long)] + /// Check all targets + all_targets: bool, + }, + /// Run Clippy (uses rustup/cargo-installed clippy binary) + #[clap(long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run clippy against. For example: + ./x.py clippy library/core + ./x.py clippy library/core library/proc_macro")] + Clippy { + #[arg(long)] + fix: bool, + /// clippy lints to allow + #[arg(global(true), short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] + allow: Vec, + /// clippy lints to deny + #[arg(global(true), short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] + deny: Vec, + /// clippy lints to warn on + #[arg(global(true), short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] + warn: Vec, + /// clippy lints to forbid + #[arg(global(true), short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] + forbid: Vec, + }, + /// Run cargo fix + #[clap(long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run `cargo fix` against. For example: + ./x.py fix library/core + ./x.py fix library/core library/proc_macro")] + Fix, + #[clap( + name = "fmt", + long_about = "\n + Arguments: + This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and + fails if it is not. For example: + ./x.py fmt + ./x.py fmt --check" + )] + /// Run rustfmt + Format { + /// check formatting instead of applying + #[arg(long)] + check: bool, + }, + #[clap(aliases = ["d"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories of documentation + to build. For example: + ./x.py doc src/doc/book + ./x.py doc src/doc/nomicon + ./x.py doc src/doc/book library/std + ./x.py doc library/std --json + ./x.py doc library/std --open + If no arguments are passed then everything is documented: + ./x.py doc + ./x.py doc --stage 1")] + /// Build documentation + Doc { + #[arg(long)] + /// open the docs in a browser + open: bool, + #[arg(long)] + /// render the documentation in JSON format in addition to the usual HTML format + json: bool, + }, + #[clap(aliases = ["t"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to test directories that + should be compiled and run. For example: + ./x.py test tests/ui + ./x.py test library/std --test-args hash_map + ./x.py test library/std --stage 0 --no-doc + ./x.py test tests/ui --bless + ./x.py test tests/ui --compare-mode next-solver + Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; + just like `build library/std --stage N` it tests the compiler produced by the previous + stage. + Execute tool tests with a tool name argument: + ./x.py test tidy + If no arguments are passed then the complete artifacts for that stage are + compiled and tested. + ./x.py test + ./x.py test --stage 1")] + /// Build and run some test suites + Test { + #[arg(long)] + /// run all tests regardless of failure + no_fail_fast: bool, + #[arg(long, value_name = "SUBSTRING")] + /// skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times + skip: Vec, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + /// extra arguments to be passed for the test tool being used + /// (e.g. libtest, compiletest or rustdoc) + test_args: Vec, + /// extra options to pass the compiler when running tests + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + rustc_args: Vec, + #[arg(long)] + /// do not run doc tests + no_doc: bool, + #[arg(long)] + /// only run doc tests + doc: bool, + #[arg(long)] + /// whether to automatically update stderr/stdout files + bless: bool, + #[arg(long)] + /// comma-separated list of other files types to check (accepts py, py:lint, + /// py:fmt, shell) + extra_checks: Option, + #[arg(long)] + /// rerun tests even if the inputs are unchanged + force_rerun: bool, + #[arg(long)] + /// only run tests that result has been changed + only_modified: bool, + #[arg(long, value_name = "COMPARE MODE")] + /// mode describing what file the actual ui output will be compared to + compare_mode: Option, + #[arg(long, value_name = "check | build | run")] + /// force {check,build,run}-pass tests to this mode. + pass: Option, + #[arg(long, value_name = "auto | always | never")] + /// whether to execute run-* tests + run: Option, + #[arg(long)] + /// enable this to generate a Rustfix coverage file, which is saved in + /// `//rustfix_missing_coverage.txt` + rustfix_coverage: bool, + }, + /// Build and run some benchmarks + Bench { + #[arg(long, allow_hyphen_values(true))] + test_args: Vec, + }, + /// Clean out build directories + Clean { + #[arg(long)] + /// Clean the entire build directory (not used by default) + all: bool, + #[arg(long, value_name = "N")] + /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. + stage: Option, + }, + /// Build distribution artifacts + Dist, + /// Install distribution artifacts + Install, + #[clap(aliases = ["r"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to tools to build and run. For + example: + ./x.py run src/tools/expand-yaml-anchors + At least a tool needs to be called.")] + /// Run tools contained in this repository + Run { + /// arguments for the tool + #[arg(long, allow_hyphen_values(true))] + args: Vec, + }, + /// Set up the environment for development + #[clap(long_about = format!( + "\n +x.py setup creates a `config.toml` which changes the defaults for x.py itself, +as well as setting up a git pre-push hook, VS Code config and toolchain link. +Arguments: + This subcommand accepts a 'profile' to use for builds. For example: + ./x.py setup library + The profile is optional and you will be prompted interactively if it is not given. + The following profiles are available: +{} + To only set up the git hook, VS Code config or toolchain link, you may use + ./x.py setup hook + ./x.py setup vscode + ./x.py setup link", Profile::all_for_help(" ").trim_end()))] + Setup { + /// Either the profile for `config.toml` or another setup action. + /// May be omitted to set up interactively + #[arg(value_name = "|hook|vscode|link")] + profile: Option, + }, + /// Suggest a subset of tests to run, based on modified files + #[clap(long_about = "\n")] + Suggest { + /// run suggested tests + #[arg(long)] + run: bool, + }, +} + +impl Subcommand { + pub fn kind(&self) -> Kind { + match self { + Subcommand::Bench { .. } => Kind::Bench, + Subcommand::Build { .. } => Kind::Build, + Subcommand::Check { .. } => Kind::Check, + Subcommand::Clippy { .. } => Kind::Clippy, + Subcommand::Doc { .. } => Kind::Doc, + Subcommand::Fix { .. } => Kind::Fix, + Subcommand::Format { .. } => Kind::Format, + Subcommand::Test { .. } => Kind::Test, + Subcommand::Clean { .. } => Kind::Clean, + Subcommand::Dist { .. } => Kind::Dist, + Subcommand::Install { .. } => Kind::Install, + Subcommand::Run { .. } => Kind::Run, + Subcommand::Setup { .. } => Kind::Setup, + Subcommand::Suggest { .. } => Kind::Suggest, + } + } + + pub fn rustc_args(&self) -> Vec<&str> { + match *self { + Subcommand::Test { ref rustc_args, .. } => { + rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + } + } + + pub fn fail_fast(&self) -> bool { + match *self { + Subcommand::Test { no_fail_fast, .. } => !no_fail_fast, + _ => false, + } + } + + pub fn doc_tests(&self) -> DocTests { + match *self { + Subcommand::Test { doc, no_doc, .. } => { + if doc { + DocTests::Only + } else if no_doc { + DocTests::No + } else { + DocTests::Yes + } + } + _ => DocTests::Yes, + } + } + + pub fn bless(&self) -> bool { + match *self { + Subcommand::Test { bless, .. } => bless, + _ => false, + } + } + + pub fn extra_checks(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), + _ => None, + } + } + + pub fn only_modified(&self) -> bool { + match *self { + Subcommand::Test { only_modified, .. } => only_modified, + _ => false, + } + } + + pub fn force_rerun(&self) -> bool { + match *self { + Subcommand::Test { force_rerun, .. } => force_rerun, + _ => false, + } + } + + pub fn rustfix_coverage(&self) -> bool { + match *self { + Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, + _ => false, + } + } + + pub fn compare_mode(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn pass(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn run(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn open(&self) -> bool { + match *self { + Subcommand::Doc { open, .. } => open, + _ => false, + } + } + + pub fn json(&self) -> bool { + match *self { + Subcommand::Doc { json, .. } => json, + _ => false, + } + } +} + +/// Returns the shell completion for a given shell, if the result differs from the current +/// content of `path`. If `path` does not exist, always returns `Some`. +pub fn get_completion(shell: G, path: &Path) -> Option { + let mut cmd = Flags::command(); + let current = if !path.exists() { + String::new() + } else { + std::fs::read_to_string(path).unwrap_or_else(|_| { + eprintln!("couldn't read {}", path.display()); + crate::exit!(1) + }) + }; + let mut buf = Vec::new(); + clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); + if buf == current.as_bytes() { + return None; + } + Some(String::from_utf8(buf).expect("completion script should be UTF-8")) +} diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs new file mode 100644 index 00000000000..9c6861826d6 --- /dev/null +++ b/src/bootstrap/src/core/config/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod config; +pub(crate) mod flags; + +pub use config::*; diff --git a/src/bootstrap/download.rs b/src/bootstrap/src/core/download.rs similarity index 99% rename from src/bootstrap/download.rs rename to src/bootstrap/src/core/download.rs index 2a0dec75599..5541a2f3e35 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -11,13 +11,10 @@ use build_helper::ci::CiEnv; use once_cell::sync::OnceCell; use xz2::bufread::XzDecoder; -use crate::{ - config::RustfmtMetadata, - llvm::detect_llvm_sha, - t, - util::{check_run, exe, program_out_of_date}, - Config, -}; +use crate::core::build_steps::llvm::detect_llvm_sha; +use crate::core::config::RustfmtMetadata; +use crate::utils::helpers::{check_run, exe, program_out_of_date}; +use crate::{t, Config}; static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/src/core/metadata.rs similarity index 98% rename from src/bootstrap/metadata.rs rename to src/bootstrap/src/core/metadata.rs index 3b20ceac875..5802082326a 100644 --- a/src/bootstrap/metadata.rs +++ b/src/bootstrap/src/core/metadata.rs @@ -3,8 +3,8 @@ use std::process::Command; use serde_derive::Deserialize; -use crate::cache::INTERNER; -use crate::util::output; +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; use crate::{t, Build, Crate}; /// For more information, see the output of diff --git a/src/bootstrap/src/core/mod.rs b/src/bootstrap/src/core/mod.rs new file mode 100644 index 00000000000..9e18d6704d4 --- /dev/null +++ b/src/bootstrap/src/core/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod build_steps; +pub(crate) mod builder; +pub(crate) mod config; +pub(crate) mod download; +pub(crate) mod metadata; +pub(crate) mod sanity; diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/src/core/sanity.rs similarity index 98% rename from src/bootstrap/sanity.rs rename to src/bootstrap/src/core/sanity.rs index 0febdf250d3..eec3be66a12 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -15,9 +15,9 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -use crate::cache::INTERNER; -use crate::config::Target; -use crate::util::output; +use crate::core::config::Target; +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; use crate::Build; pub struct Finder { From 9f381fe3452a0b938e91dc1d10c492d0635b8bad Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 7 Oct 2023 23:37:55 +0300 Subject: [PATCH 4/6] move bootstrap utils into bootstrap/src/utils module Signed-off-by: onur-ozkan --- src/bootstrap/Cargo.toml | 10 +- src/bootstrap/flags.rs | 568 ------------------ src/bootstrap/job.rs | 143 ----- src/bootstrap/src/core/config/flags.rs | 3 + src/bootstrap/{ => src}/lib.rs | 148 ++--- src/bootstrap/src/utils/bin_helpers.rs | 28 + src/bootstrap/{ => src/utils}/cache.rs | 2 +- src/bootstrap/{ => src/utils}/cc_detect.rs | 4 +- src/bootstrap/{ => src/utils}/channel.rs | 3 +- src/bootstrap/{ => src/utils}/dylib_util.rs | 11 +- .../{util.rs => src/utils/helpers.rs} | 36 +- src/bootstrap/src/utils/job.rs | 161 +++++ src/bootstrap/{ => src/utils}/metrics.rs | 4 +- src/bootstrap/src/utils/mod.rs | 14 + src/bootstrap/{ => src/utils}/render_tests.rs | 2 +- src/bootstrap/{ => src/utils}/tarball.rs | 19 +- 16 files changed, 296 insertions(+), 860 deletions(-) delete mode 100644 src/bootstrap/flags.rs delete mode 100644 src/bootstrap/job.rs rename src/bootstrap/{ => src}/lib.rs (95%) create mode 100644 src/bootstrap/src/utils/bin_helpers.rs rename src/bootstrap/{ => src/utils}/cache.rs (99%) rename src/bootstrap/{ => src/utils}/cc_detect.rs (99%) rename src/bootstrap/{ => src/utils}/channel.rs (99%) rename src/bootstrap/{ => src/utils}/dylib_util.rs (62%) rename src/bootstrap/{util.rs => src/utils/helpers.rs} (95%) create mode 100644 src/bootstrap/src/utils/job.rs rename src/bootstrap/{ => src/utils}/metrics.rs (99%) create mode 100644 src/bootstrap/src/utils/mod.rs rename src/bootstrap/{ => src/utils}/render_tests.rs (99%) rename src/bootstrap/{ => src/utils}/tarball.rs (95%) diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 006a992eb3c..e236421b7f5 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -9,27 +9,27 @@ default-run = "bootstrap" build-metrics = ["sysinfo"] [lib] -path = "lib.rs" +path = "src/lib.rs" doctest = false [[bin]] name = "bootstrap" -path = "bin/main.rs" +path = "src/bin/main.rs" test = false [[bin]] name = "rustc" -path = "bin/rustc.rs" +path = "src/bin/rustc.rs" test = false [[bin]] name = "rustdoc" -path = "bin/rustdoc.rs" +path = "src/bin/rustdoc.rs" test = false [[bin]] name = "sccache-plus-cl" -path = "bin/sccache-plus-cl.rs" +path = "src/bin/sccache-plus-cl.rs" test = false [dependencies] diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs deleted file mode 100644 index 5a6a5f37fc6..00000000000 --- a/src/bootstrap/flags.rs +++ /dev/null @@ -1,568 +0,0 @@ -//! Command-line interface of the rustbuild build system. -//! -//! This module implements the command-line parsing of the build system which -//! has various flags to configure how it's run. - -use std::path::{Path, PathBuf}; - -use clap::{CommandFactory, Parser, ValueEnum}; - -use crate::builder::{Builder, Kind}; -use crate::config::{target_selection_list, Config, TargetSelectionList}; -use crate::setup::Profile; -use crate::{Build, DocTests}; - -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Color { - Always, - Never, - #[default] - Auto, -} - -/// Whether to deny warnings, emit them as warnings, or use the default behavior -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Warnings { - Deny, - Warn, - #[default] - Default, -} - -/// Deserialized version of all flags for this compile. -#[derive(Debug, Parser)] -#[clap( - override_usage = "x.py [options] [...]", - disable_help_subcommand(true), - about = "", - next_line_help(false) -)] -pub struct Flags { - #[command(subcommand)] - pub cmd: Subcommand, - - #[arg(global(true), short, long, action = clap::ArgAction::Count)] - /// use verbose output (-vv for very verbose) - pub verbose: u8, // each extra -v after the first is passed to Cargo - #[arg(global(true), short, long)] - /// use incremental compilation - pub incremental: bool, - #[arg(global(true), long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] - /// TOML configuration file for build - pub config: Option, - #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// Build directory, overrides `build.build-dir` in `config.toml` - pub build_dir: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] - /// build target of the stage0 compiler - pub build: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] - /// host targets to build - pub host: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] - /// target targets to build - pub target: Option, - - #[arg(global(true), long, value_name = "PATH")] - /// build paths to exclude - pub exclude: Vec, // keeping for client backward compatibility - #[arg(global(true), long, value_name = "PATH")] - /// build paths to skip - pub skip: Vec, - #[arg(global(true), long)] - /// include default paths in addition to the provided ones - pub include_default_paths: bool, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long)] - pub rustc_error_format: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] - /// command to run on failure - pub on_fail: Option, - #[arg(global(true), long)] - /// dry run; don't build anything - pub dry_run: bool, - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the - /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) - pub stage: Option, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage: Vec, - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) of the standard library to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage_std: Vec, - #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// path to the root of the rust checkout - pub src: Option, - - #[arg( - global(true), - short, - long, - value_hint = clap::ValueHint::Other, - default_value_t = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), - value_name = "JOBS" - )] - /// number of jobs to run in parallel - pub jobs: usize, - // This overrides the deny-warnings configuration option, - // which passes -Dwarnings to the compiler invocations. - #[arg(global(true), long)] - #[clap(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] - /// if value is deny, will deny warnings - /// if value is warn, will emit warnings - /// otherwise, use the default configured behaviour - pub warnings: Warnings, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] - /// rustc error format - pub error_format: Option, - #[arg(global(true), long)] - /// use message-format=json - pub json_output: bool, - - #[arg(global(true), long, value_name = "STYLE")] - #[clap(value_enum, default_value_t = Color::Auto)] - /// whether to use color in cargo and rustc output - pub color: Color, - - /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml - #[arg(global(true), long, value_name = "VALUE")] - pub llvm_skip_rebuild: Option, - /// generate PGO profile with rustc build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_generate: Option, - /// use PGO profile for rustc build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_use: Option, - /// use PGO profile for LLVM build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub llvm_profile_use: Option, - // LLVM doesn't support a custom location for generating profile - // information. - // - // llvm_out/build/profiles/ is the location this writes to. - /// generate PGO profile with llvm built for rustc - #[arg(global(true), long)] - pub llvm_profile_generate: bool, - /// Enable BOLT link flags - #[arg(global(true), long)] - pub enable_bolt_settings: bool, - /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. - #[arg(global(true), long)] - pub reproducible_artifact: Vec, - #[arg(global(true))] - /// paths for the subcommand - pub paths: Vec, - /// override options in config.toml - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] - pub set: Vec, - /// arguments passed to subcommands - #[arg(global(true), last(true), value_name = "ARGS")] - pub free_args: Vec, -} - -impl Flags { - pub fn parse(args: &[String]) -> Self { - let first = String::from("x.py"); - let it = std::iter::once(&first).chain(args.iter()); - // We need to check for ` -h -v`, in which case we list the paths - #[derive(Parser)] - #[clap(disable_help_flag(true))] - struct HelpVerboseOnly { - #[arg(short, long)] - help: bool, - #[arg(global(true), short, long, action = clap::ArgAction::Count)] - pub verbose: u8, - #[arg(value_enum)] - cmd: Kind, - } - if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = - HelpVerboseOnly::try_parse_from(it.clone()) - { - println!("note: updating submodules before printing available paths"); - let config = Config::parse(&[String::from("build")]); - let build = Build::new(config); - let paths = Builder::get_help(&build, subcommand); - if let Some(s) = paths { - println!("{s}"); - } else { - panic!("No paths available for subcommand `{}`", subcommand.as_str()); - } - crate::exit!(0); - } - - Flags::parse_from(it) - } -} - -#[derive(Debug, Clone, Default, clap::Subcommand)] -pub enum Subcommand { - #[clap(aliases = ["b"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example, for a quick build of a usable - compiler: - ./x.py build --stage 1 library/std - This will build a compiler and standard library from the local source code. - Once this is done, build/$ARCH/stage1 contains a usable compiler. - If no arguments are passed then the default artifacts for that stage are - compiled. For example: - ./x.py build --stage 0 - ./x.py build ")] - /// Compile either the compiler or libraries - #[default] - Build, - #[clap(aliases = ["c"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example: - ./x.py check library/std - If no arguments are passed then many artifacts are checked.")] - /// Compile either the compiler or libraries, using cargo check - Check { - #[arg(long)] - /// Check all targets - all_targets: bool, - }, - /// Run Clippy (uses rustup/cargo-installed clippy binary) - #[clap(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run clippy against. For example: - ./x.py clippy library/core - ./x.py clippy library/core library/proc_macro")] - Clippy { - #[arg(long)] - fix: bool, - /// clippy lints to allow - #[arg(global(true), short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] - allow: Vec, - /// clippy lints to deny - #[arg(global(true), short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] - deny: Vec, - /// clippy lints to warn on - #[arg(global(true), short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] - warn: Vec, - /// clippy lints to forbid - #[arg(global(true), short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] - forbid: Vec, - }, - /// Run cargo fix - #[clap(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run `cargo fix` against. For example: - ./x.py fix library/core - ./x.py fix library/core library/proc_macro")] - Fix, - #[clap( - name = "fmt", - long_about = "\n - Arguments: - This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and - fails if it is not. For example: - ./x.py fmt - ./x.py fmt --check" - )] - /// Run rustfmt - Format { - /// check formatting instead of applying - #[arg(long)] - check: bool, - }, - #[clap(aliases = ["d"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories of documentation - to build. For example: - ./x.py doc src/doc/book - ./x.py doc src/doc/nomicon - ./x.py doc src/doc/book library/std - ./x.py doc library/std --json - ./x.py doc library/std --open - If no arguments are passed then everything is documented: - ./x.py doc - ./x.py doc --stage 1")] - /// Build documentation - Doc { - #[arg(long)] - /// open the docs in a browser - open: bool, - #[arg(long)] - /// render the documentation in JSON format in addition to the usual HTML format - json: bool, - }, - #[clap(aliases = ["t"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to test directories that - should be compiled and run. For example: - ./x.py test tests/ui - ./x.py test library/std --test-args hash_map - ./x.py test library/std --stage 0 --no-doc - ./x.py test tests/ui --bless - ./x.py test tests/ui --compare-mode next-solver - Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; - just like `build library/std --stage N` it tests the compiler produced by the previous - stage. - Execute tool tests with a tool name argument: - ./x.py test tidy - If no arguments are passed then the complete artifacts for that stage are - compiled and tested. - ./x.py test - ./x.py test --stage 1")] - /// Build and run some test suites - Test { - #[arg(long)] - /// run all tests regardless of failure - no_fail_fast: bool, - #[arg(long, value_name = "SUBSTRING")] - /// skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times - skip: Vec, - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - /// extra arguments to be passed for the test tool being used - /// (e.g. libtest, compiletest or rustdoc) - test_args: Vec, - /// extra options to pass the compiler when running tests - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - rustc_args: Vec, - #[arg(long)] - /// do not run doc tests - no_doc: bool, - #[arg(long)] - /// only run doc tests - doc: bool, - #[arg(long)] - /// whether to automatically update stderr/stdout files - bless: bool, - #[arg(long)] - /// comma-separated list of other files types to check (accepts py, py:lint, - /// py:fmt, shell) - extra_checks: Option, - #[arg(long)] - /// rerun tests even if the inputs are unchanged - force_rerun: bool, - #[arg(long)] - /// only run tests that result has been changed - only_modified: bool, - #[arg(long, value_name = "COMPARE MODE")] - /// mode describing what file the actual ui output will be compared to - compare_mode: Option, - #[arg(long, value_name = "check | build | run")] - /// force {check,build,run}-pass tests to this mode. - pass: Option, - #[arg(long, value_name = "auto | always | never")] - /// whether to execute run-* tests - run: Option, - #[arg(long)] - /// enable this to generate a Rustfix coverage file, which is saved in - /// `//rustfix_missing_coverage.txt` - rustfix_coverage: bool, - }, - /// Build and run some benchmarks - Bench { - #[arg(long, allow_hyphen_values(true))] - test_args: Vec, - }, - /// Clean out build directories - Clean { - #[arg(long)] - /// Clean the entire build directory (not used by default) - all: bool, - #[arg(long, value_name = "N")] - /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. - stage: Option, - }, - /// Build distribution artifacts - Dist, - /// Install distribution artifacts - Install, - #[clap(aliases = ["r"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to tools to build and run. For - example: - ./x.py run src/tools/expand-yaml-anchors - At least a tool needs to be called.")] - /// Run tools contained in this repository - Run { - /// arguments for the tool - #[arg(long, allow_hyphen_values(true))] - args: Vec, - }, - /// Set up the environment for development - #[clap(long_about = format!( - "\n -x.py setup creates a `config.toml` which changes the defaults for x.py itself, -as well as setting up a git pre-push hook, VS Code config and toolchain link. -Arguments: - This subcommand accepts a 'profile' to use for builds. For example: - ./x.py setup library - The profile is optional and you will be prompted interactively if it is not given. - The following profiles are available: -{} - To only set up the git hook, VS Code config or toolchain link, you may use - ./x.py setup hook - ./x.py setup vscode - ./x.py setup link", Profile::all_for_help(" ").trim_end()))] - Setup { - /// Either the profile for `config.toml` or another setup action. - /// May be omitted to set up interactively - #[arg(value_name = "|hook|vscode|link")] - profile: Option, - }, - /// Suggest a subset of tests to run, based on modified files - #[clap(long_about = "\n")] - Suggest { - /// run suggested tests - #[arg(long)] - run: bool, - }, -} - -impl Subcommand { - pub fn kind(&self) -> Kind { - match self { - Subcommand::Bench { .. } => Kind::Bench, - Subcommand::Build { .. } => Kind::Build, - Subcommand::Check { .. } => Kind::Check, - Subcommand::Clippy { .. } => Kind::Clippy, - Subcommand::Doc { .. } => Kind::Doc, - Subcommand::Fix { .. } => Kind::Fix, - Subcommand::Format { .. } => Kind::Format, - Subcommand::Test { .. } => Kind::Test, - Subcommand::Clean { .. } => Kind::Clean, - Subcommand::Dist { .. } => Kind::Dist, - Subcommand::Install { .. } => Kind::Install, - Subcommand::Run { .. } => Kind::Run, - Subcommand::Setup { .. } => Kind::Setup, - Subcommand::Suggest { .. } => Kind::Suggest, - } - } - - pub fn rustc_args(&self) -> Vec<&str> { - match *self { - Subcommand::Test { ref rustc_args, .. } => { - rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - } - } - - pub fn fail_fast(&self) -> bool { - match *self { - Subcommand::Test { no_fail_fast, .. } => !no_fail_fast, - _ => false, - } - } - - pub fn doc_tests(&self) -> DocTests { - match *self { - Subcommand::Test { doc, no_doc, .. } => { - if doc { - DocTests::Only - } else if no_doc { - DocTests::No - } else { - DocTests::Yes - } - } - _ => DocTests::Yes, - } - } - - pub fn bless(&self) -> bool { - match *self { - Subcommand::Test { bless, .. } => bless, - _ => false, - } - } - - pub fn extra_checks(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), - _ => None, - } - } - - pub fn only_modified(&self) -> bool { - match *self { - Subcommand::Test { only_modified, .. } => only_modified, - _ => false, - } - } - - pub fn force_rerun(&self) -> bool { - match *self { - Subcommand::Test { force_rerun, .. } => force_rerun, - _ => false, - } - } - - pub fn rustfix_coverage(&self) -> bool { - match *self { - Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, - _ => false, - } - } - - pub fn compare_mode(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn pass(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn run(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn open(&self) -> bool { - match *self { - Subcommand::Doc { open, .. } => open, - _ => false, - } - } - - pub fn json(&self) -> bool { - match *self { - Subcommand::Doc { json, .. } => json, - _ => false, - } - } -} - -/// Returns the shell completion for a given shell, if the result differs from the current -/// content of `path`. If `path` does not exist, always returns `Some`. -pub fn get_completion(shell: G, path: &Path) -> Option { - let mut cmd = Flags::command(); - let current = if !path.exists() { - String::new() - } else { - std::fs::read_to_string(path).unwrap_or_else(|_| { - eprintln!("couldn't read {}", path.display()); - crate::exit!(1) - }) - }; - let mut buf = Vec::new(); - clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); - if buf == current.as_bytes() { - return None; - } - Some(String::from_utf8(buf).expect("completion script should be UTF-8")) -} diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs deleted file mode 100644 index b0a97b540ec..00000000000 --- a/src/bootstrap/job.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Job management on Windows for bootstrapping -//! -//! Most of the time when you're running a build system (e.g., make) you expect -//! Ctrl-C or abnormal termination to actually terminate the entire tree of -//! process in play, not just the one at the top. This currently works "by -//! default" on Unix platforms because Ctrl-C actually sends a signal to the -//! *process group* rather than the parent process, so everything will get torn -//! down. On Windows, however, this does not happen and Ctrl-C just kills the -//! parent process. -//! -//! To achieve the same semantics on Windows we use Job Objects to ensure that -//! all processes die at the same time. Job objects have a mode of operation -//! where when all handles to the object are closed it causes all child -//! processes associated with the object to be terminated immediately. -//! Conveniently whenever a process in the job object spawns a new process the -//! child will be associated with the job object as well. This means if we add -//! ourselves to the job object we create then everything will get torn down! -//! -//! Unfortunately most of the time the build system is actually called from a -//! python wrapper (which manages things like building the build system) so this -//! all doesn't quite cut it so far. To go the last mile we duplicate the job -//! object handle into our parent process (a python process probably) and then -//! close our own handle. This means that the only handle to the job object -//! resides in the parent python process, so when python dies the whole build -//! system dies (as one would probably expect!). -//! -//! Note that this module has a #[cfg(windows)] above it as none of this logic -//! is required on Unix. - -use crate::Build; -use std::env; -use std::ffi::c_void; -use std::io; -use std::mem; - -use windows::{ - core::PCWSTR, - Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, - Win32::System::Diagnostics::Debug::{SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE}, - Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, - SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, - }, - Win32::System::Threading::{ - GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, - }, -}; - -pub unsafe fn setup(build: &mut Build) { - // Enable the Windows Error Reporting dialog which msys disables, - // so we can JIT debug rustc - let mode = SetErrorMode(THREAD_ERROR_MODE::default()); - let mode = THREAD_ERROR_MODE(mode); - SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); - - // Create a new job object for us to use - let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); - - // Indicate that when all handles to the job object are gone that all - // process in the object should be killed. Note that this includes our - // entire process tree by default because we've added ourselves and our - // children will reside in the job by default. - let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if build.config.low_priority { - info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; - info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; - } - let r = SetInformationJobObject( - job, - JobObjectExtendedLimitInformation, - &info as *const _ as *const c_void, - mem::size_of_val(&info) as u32, - ) - .ok(); - assert!(r.is_ok(), "{}", io::Error::last_os_error()); - - // Assign our process to this job object. Note that if this fails, one very - // likely reason is that we are ourselves already in a job object! This can - // happen on the build bots that we've got for Windows, or if just anyone - // else is instrumenting the build. In this case we just bail out - // immediately and assume that they take care of it. - // - // Also note that nested jobs (why this might fail) are supported in recent - // versions of Windows, but the version of Windows that our bots are running - // at least don't support nested job objects. - let r = AssignProcessToJobObject(job, GetCurrentProcess()).ok(); - if r.is_err() { - CloseHandle(job); - return; - } - - // If we've got a parent process (e.g., the python script that called us) - // then move ownership of this job object up to them. That way if the python - // script is killed (e.g., via ctrl-c) then we'll all be torn down. - // - // If we don't have a parent (e.g., this was run directly) then we - // intentionally leak the job object handle. When our process exits - // (normally or abnormally) it will close the handle implicitly, causing all - // processes in the job to be cleaned up. - let pid = match env::var("BOOTSTRAP_PARENT_ID") { - Ok(s) => s, - Err(..) => return, - }; - - let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { - Some(parent) => parent, - _ => { - // If we get a null parent pointer here, it is possible that either - // we have an invalid pid or the parent process has been closed. - // Since the first case rarely happens - // (only when wrongly setting the environmental variable), - // it might be better to improve the experience of the second case - // when users have interrupted the parent process and we haven't finish - // duplicating the handle yet. We just need close the job object if that occurs. - CloseHandle(job); - return; - } - }; - - let mut parent_handle = HANDLE::default(); - let r = DuplicateHandle( - GetCurrentProcess(), - job, - parent, - &mut parent_handle, - 0, - false, - DUPLICATE_SAME_ACCESS, - ) - .ok(); - - // If this failed, well at least we tried! An example of DuplicateHandle - // failing in the past has been when the wrong python2 package spawned this - // build system (e.g., the `python2` package in MSYS instead of - // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure - // mode" here is that we only clean everything up when the build system - // dies, not when the python parent does, so not too bad. - if r.is_err() { - CloseHandle(job); - } -} diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs index 92aea76382a..dea55303544 100644 --- a/src/bootstrap/src/core/config/flags.rs +++ b/src/bootstrap/src/core/config/flags.rs @@ -152,6 +152,9 @@ pub struct Flags { /// generate PGO profile with llvm built for rustc #[arg(global(true), long)] pub llvm_profile_generate: bool, + /// Enable BOLT link flags + #[arg(global(true), long)] + pub enable_bolt_settings: bool, /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. #[arg(global(true), long)] pub reproducible_artifact: Vec, diff --git a/src/bootstrap/lib.rs b/src/bootstrap/src/lib.rs similarity index 95% rename from src/bootstrap/lib.rs rename to src/bootstrap/src/lib.rs index a9a81cb25f6..97c743074af 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -28,70 +28,28 @@ use std::str; use build_helper::ci::{gha, CiEnv}; use build_helper::exit; -use channel::GitInfo; -use config::{DryRun, Target}; use filetime::FileTime; use once_cell::sync::OnceCell; +use termcolor::{ColorChoice, StandardStream, WriteColor}; +use utils::channel::GitInfo; -use crate::builder::Kind; -use crate::config::{LlvmLibunwind, TargetSelection}; -use crate::util::{ - dir_is_empty, exe, libdir, mtime, output, run, run_suppressed, symlink_dir, try_run_suppressed, +use crate::core::builder; +use crate::core::builder::Kind; +use crate::core::config::flags; +use crate::core::config::{DryRun, Target}; +use crate::core::config::{LlvmLibunwind, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{ + self, dir_is_empty, exe, libdir, mtime, output, run, run_suppressed, symlink_dir, + try_run_suppressed, }; -mod builder; -mod cache; -mod cc_detect; -mod channel; -mod check; -mod clean; -mod compile; -mod config; -mod dist; -mod doc; -mod download; -mod flags; -mod format; -mod install; -mod llvm; -mod metadata; -mod render_tests; -mod run; -mod sanity; -mod setup; -mod suggest; -mod synthetic_targets; -mod tarball; -mod test; -mod tool; -mod toolstate; -pub mod util; +mod core; +mod utils; -#[cfg(feature = "build-metrics")] -mod metrics; - -#[cfg(windows)] -mod job; - -#[cfg(all(unix, not(target_os = "haiku")))] -mod job { - pub unsafe fn setup(build: &mut crate::Build) { - if build.config.low_priority { - libc::setpriority(libc::PRIO_PGRP as _, 0, 10); - } - } -} - -#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] -mod job { - pub unsafe fn setup(_build: &mut crate::Build) {} -} - -pub use crate::builder::PathSet; -use crate::cache::{Interned, INTERNER}; -pub use crate::config::Config; -pub use crate::flags::Subcommand; -use termcolor::{ColorChoice, StandardStream, WriteColor}; +pub use crate::core::builder::PathSet; +pub use crate::core::config::flags::Subcommand; +pub use crate::core::config::Config; const LLVM_TOOLS: &[&str] = &[ "llvm-cov", // used to generate coverage report @@ -210,12 +168,12 @@ pub struct Build { src: PathBuf, out: PathBuf, bootstrap_out: PathBuf, - cargo_info: channel::GitInfo, - rust_analyzer_info: channel::GitInfo, - clippy_info: channel::GitInfo, - miri_info: channel::GitInfo, - rustfmt_info: channel::GitInfo, - in_tree_llvm_info: channel::GitInfo, + cargo_info: GitInfo, + rust_analyzer_info: GitInfo, + clippy_info: GitInfo, + miri_info: GitInfo, + rustfmt_info: GitInfo, + in_tree_llvm_info: GitInfo, local_rebuild: bool, fail_fast: bool, doc_tests: DocTests, @@ -248,7 +206,7 @@ pub struct Build { prerelease_version: Cell>, #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics, + metrics: crate::utils::metrics::BuildMetrics, } #[derive(Debug, Clone)] @@ -372,16 +330,15 @@ impl Build { let is_sudo = false; let omit_git_hash = config.omit_git_hash; - let rust_info = channel::GitInfo::new(omit_git_hash, &src); - let cargo_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); - let rust_analyzer_info = - channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); - let clippy_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); - let miri_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); - let rustfmt_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); + let rust_info = GitInfo::new(omit_git_hash, &src); + let cargo_info = GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); + let rust_analyzer_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); + let clippy_info = GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); + let miri_info = GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); + let rustfmt_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); // we always try to use git for LLVM builds - let in_tree_llvm_info = channel::GitInfo::new(false, &src.join("src/llvm-project")); + let in_tree_llvm_info = GitInfo::new(false, &src.join("src/llvm-project")); let initial_target_libdir_str = if config.dry_run() { "/dummy/lib/path/to/lib/".to_string() @@ -474,7 +431,7 @@ impl Build { prerelease_version: Cell::new(None), #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics::init(), + metrics: crate::utils::metrics::BuildMetrics::init(), }; // If local-rust is the same major.minor as the current version, then force a @@ -493,7 +450,7 @@ impl Build { } build.verbose("finding compilers"); - cc_detect::find(&build); + utils::cc_detect::find(&build); // When running `setup`, the profile is about to change, so any requirements we have now may // be different on the next invocation. Don't check for them until the next time x.py is // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. @@ -501,7 +458,7 @@ impl Build { // Similarly, for `setup` we don't actually need submodules or cargo metadata. if !matches!(build.config.cmd, Subcommand::Setup { .. }) { build.verbose("running sanity check"); - sanity::check(&mut build); + crate::core::sanity::check(&mut build); // Make sure we update these before gathering metadata so we don't get an error about missing // Cargo.toml files. @@ -513,7 +470,7 @@ impl Build { build.update_existing_submodules(); build.verbose("learning about cargo"); - metadata::build(&mut build); + crate::core::metadata::build(&mut build); } // Make a symbolic link so we can use a consistent directory in the documentation. @@ -549,7 +506,7 @@ impl Build { // NOTE: The check for the empty directory is here because when running x.py the first time, // the submodule won't be checked out. Check it out now so we can build it. - if !channel::GitInfo::new(false, &absolute_path).is_managed_git_subrepository() + if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository() && !dir_is_empty(&absolute_path) { return; @@ -663,7 +620,7 @@ impl Build { // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap()); // Don't update the submodule unless it's already been cloned. - if channel::GitInfo::new(false, submodule).is_managed_git_subrepository() { + if GitInfo::new(false, submodule).is_managed_git_subrepository() { self.update_submodule(submodule); } } @@ -672,7 +629,7 @@ impl Build { /// Executes the entire build, as configured by the flags and configuration. pub fn build(&mut self) { unsafe { - job::setup(self); + crate::utils::job::setup(self); } // Download rustfmt early so that it can be used in rust-analyzer configs. @@ -681,10 +638,14 @@ impl Build { // hardcoded subcommands match &self.config.cmd { Subcommand::Format { check } => { - return format::format(&builder::Builder::new(&self), *check, &self.config.paths); + return core::build_steps::format::format( + &builder::Builder::new(&self), + *check, + &self.config.paths, + ); } Subcommand::Suggest { run } => { - return suggest::suggest(&builder::Builder::new(&self), *run); + return core::build_steps::suggest::suggest(&builder::Builder::new(&self), *run); } _ => (), } @@ -1065,7 +1026,7 @@ impl Build { /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. /// - /// [`Step`]: crate::builder::Step + /// [`Step`]: crate::core::builder::Step #[must_use = "Groups should not be dropped until the Step finishes running"] #[track_caller] fn msg( @@ -1093,7 +1054,7 @@ impl Build { /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. /// - /// [`Step`]: crate::builder::Step + /// [`Step`]: crate::core::builder::Step #[must_use = "Groups should not be dropped until the Step finishes running"] #[track_caller] fn msg_unstaged( @@ -1253,7 +1214,7 @@ impl Build { // that are only existed in CXX libraries Some(self.cxx.borrow()[&target].path().into()) } else if target != self.config.build - && util::use_host_linker(target) + && helpers::use_host_linker(target) && !target.contains("msvc") { Some(self.cc(target)) @@ -1278,7 +1239,7 @@ impl Build { options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); } - let no_threads = util::lld_flag_no_threads(target.contains("windows")); + let no_threads = helpers::lld_flag_no_threads(target.contains("windows")); options[1] = Some(format!("-Clink-arg=-Wl,{no_threads}")); } @@ -1418,7 +1379,7 @@ impl Build { fn extract_beta_rev_from_file>(version_file: P) -> Option { let version = fs::read_to_string(version_file).ok()?; - extract_beta_rev(&version) + helpers::extract_beta_rev(&version) } if let Some(s) = self.prerelease_version.get() { @@ -1732,7 +1693,7 @@ impl Build { /// Returns if config.ninja is enabled, and checks for ninja existence, /// exiting with a nicer error message if not. fn ninja(&self) -> bool { - let mut cmd_finder = crate::sanity::Finder::new(); + let mut cmd_finder = crate::core::sanity::Finder::new(); if self.config.ninja_in_file { // Some Linux distros rename `ninja` to `ninja-build`. @@ -1798,17 +1759,6 @@ to download LLVM rather than building it. } } -/// Extract the beta revision from the full version string. -/// -/// The full version string looks like "a.b.c-beta.y". And we need to extract -/// the "y" part from the string. -pub fn extract_beta_rev(version: &str) -> Option { - let parts = version.splitn(2, "-beta.").collect::>(); - let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); - - count -} - #[cfg(unix)] fn chmod(path: &Path, perms: u32) { use std::os::unix::fs::*; diff --git a/src/bootstrap/src/utils/bin_helpers.rs b/src/bootstrap/src/utils/bin_helpers.rs new file mode 100644 index 00000000000..ab41a6b9600 --- /dev/null +++ b/src/bootstrap/src/utils/bin_helpers.rs @@ -0,0 +1,28 @@ +//! This file is meant to be included directly from bootstrap shims to avoid a +//! dependency on the bootstrap library. This reduces the binary size and +//! improves compilation time by reducing the linking time. + +/// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. +/// If it was not defined, returns 0 by default. +/// +/// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. +pub(crate) fn parse_rustc_verbose() -> usize { + use std::str::FromStr; + + match std::env::var("RUSTC_VERBOSE") { + Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), + Err(_) => 0, + } +} + +/// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. +/// +/// If "RUSTC_STAGE" was not set, the program will be terminated with 101. +pub(crate) fn parse_rustc_stage() -> String { + std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { + // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. + eprintln!("rustc shim: fatal: RUSTC_STAGE was not set"); + eprintln!("rustc shim: note: use `x.py build -vvv` to see all environment variables set by bootstrap"); + std::process::exit(101); + }) +} diff --git a/src/bootstrap/cache.rs b/src/bootstrap/src/utils/cache.rs similarity index 99% rename from src/bootstrap/cache.rs rename to src/bootstrap/src/utils/cache.rs index 53e4ff03431..1b2aa9c234b 100644 --- a/src/bootstrap/cache.rs +++ b/src/bootstrap/src/utils/cache.rs @@ -14,7 +14,7 @@ use std::sync::Mutex; // FIXME: replace with std::lazy after it gets stabilized and reaches beta use once_cell::sync::Lazy; -use crate::builder::Step; +use crate::core::builder::Step; pub struct Interned(usize, PhantomData<*const T>); diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/src/utils/cc_detect.rs similarity index 99% rename from src/bootstrap/cc_detect.rs rename to src/bootstrap/src/utils/cc_detect.rs index 2496c2a9db5..3d3f93f7720 100644 --- a/src/bootstrap/cc_detect.rs +++ b/src/bootstrap/src/utils/cc_detect.rs @@ -26,8 +26,8 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, iter}; -use crate::config::{Target, TargetSelection}; -use crate::util::output; +use crate::core::config::{Target, TargetSelection}; +use crate::utils::helpers::output; use crate::{Build, CLang, GitRepo}; // The `cc` crate doesn't provide a way to obtain a path to the detected archiver, diff --git a/src/bootstrap/channel.rs b/src/bootstrap/src/utils/channel.rs similarity index 99% rename from src/bootstrap/channel.rs rename to src/bootstrap/src/utils/channel.rs index 87018574048..e59d7f22aaa 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -9,8 +9,7 @@ use std::fs; use std::path::Path; use std::process::Command; -use crate::util::output; -use crate::util::t; +use crate::utils::helpers::{output, t}; use crate::Build; #[derive(Clone, Default)] diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/src/utils/dylib_util.rs similarity index 62% rename from src/bootstrap/dylib_util.rs rename to src/bootstrap/src/utils/dylib_util.rs index b14c0bed66c..279a6a010f1 100644 --- a/src/bootstrap/dylib_util.rs +++ b/src/bootstrap/src/utils/dylib_util.rs @@ -1,7 +1,4 @@ -// Various utilities for working with dylib paths. -// -// This file is meant to be included directly to avoid a dependency on the bootstrap library from -// the rustc and rustdoc wrappers. This improves compilation time by reducing the linking time. +//! Various utilities for working with dylib paths. /// Returns the environment variable which the dynamic library lookup path /// resides in for this platform. @@ -21,10 +18,10 @@ pub fn dylib_path_var() -> &'static str { /// Parses the `dylib_path_var()` environment variable, returning a list of /// paths that are members of this lookup path. -pub fn dylib_path() -> Vec { - let var = match env::var_os(dylib_path_var()) { +pub fn dylib_path() -> Vec { + let var = match std::env::var_os(dylib_path_var()) { Some(v) => v, None => return vec![], }; - env::split_paths(&var).collect() + std::env::split_paths(&var).collect() } diff --git a/src/bootstrap/util.rs b/src/bootstrap/src/utils/helpers.rs similarity index 95% rename from src/bootstrap/util.rs rename to src/bootstrap/src/utils/helpers.rs index 3c4a21434c0..90db0a89654 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -12,10 +12,12 @@ use std::process::{Command, Stdio}; use std::str; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::builder::Builder; -use crate::config::{Config, TargetSelection}; +use crate::core::builder::Builder; +use crate::core::config::{Config, TargetSelection}; use crate::OnceCell; +pub use crate::utils::dylib_util::{dylib_path, dylib_path_var}; + /// A helper macro to `unwrap` a result except also print out details like: /// /// * The file/line of the panic @@ -81,8 +83,6 @@ pub fn add_dylib_path(path: Vec, cmd: &mut Command) { cmd.env(dylib_path_var(), t!(env::join_paths(list))); } -include!("dylib_util.rs"); - /// Adds a list of lookup paths to `cmd`'s link library lookup path. pub fn add_link_lib_path(path: Vec, cmd: &mut Command) { let mut list = link_lib_path(); @@ -293,23 +293,6 @@ pub fn output(cmd: &mut Command) -> String { String::from_utf8(output.stdout).unwrap() } -pub fn output_result(cmd: &mut Command) -> Result { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => return Err(format!("failed to run command: {cmd:?}: {e}")), - }; - if !output.status.success() { - return Err(format!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}\n{}", - cmd, - output.status, - String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? - )); - } - Ok(String::from_utf8(output.stdout).map_err(|err| format!("{err:?}"))?) -} - /// Returns the last-modified time for `path`, or zero if it doesn't exist. pub fn mtime(path: &Path) -> SystemTime { fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) @@ -495,3 +478,14 @@ pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { pub fn dir_is_empty(dir: &Path) -> bool { t!(std::fs::read_dir(dir)).next().is_none() } + +/// Extract the beta revision from the full version string. +/// +/// The full version string looks like "a.b.c-beta.y". And we need to extract +/// the "y" part from the string. +pub fn extract_beta_rev(version: &str) -> Option { + let parts = version.splitn(2, "-beta.").collect::>(); + let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); + + count +} diff --git a/src/bootstrap/src/utils/job.rs b/src/bootstrap/src/utils/job.rs new file mode 100644 index 00000000000..37235134a28 --- /dev/null +++ b/src/bootstrap/src/utils/job.rs @@ -0,0 +1,161 @@ +#[cfg(windows)] +pub use for_windows::*; + +#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] +pub unsafe fn setup(_build: &mut crate::Build) {} + +#[cfg(all(unix, not(target_os = "haiku")))] +pub unsafe fn setup(build: &mut crate::Build) { + if build.config.low_priority { + libc::setpriority(libc::PRIO_PGRP as _, 0, 10); + } +} + +#[cfg(windows)] +mod for_windows { + //! Job management on Windows for bootstrapping + //! + //! Most of the time when you're running a build system (e.g., make) you expect + //! Ctrl-C or abnormal termination to actually terminate the entire tree of + //! process in play, not just the one at the top. This currently works "by + //! default" on Unix platforms because Ctrl-C actually sends a signal to the + //! *process group* rather than the parent process, so everything will get torn + //! down. On Windows, however, this does not happen and Ctrl-C just kills the + //! parent process. + //! + //! To achieve the same semantics on Windows we use Job Objects to ensure that + //! all processes die at the same time. Job objects have a mode of operation + //! where when all handles to the object are closed it causes all child + //! processes associated with the object to be terminated immediately. + //! Conveniently whenever a process in the job object spawns a new process the + //! child will be associated with the job object as well. This means if we add + //! ourselves to the job object we create then everything will get torn down! + //! + //! Unfortunately most of the time the build system is actually called from a + //! python wrapper (which manages things like building the build system) so this + //! all doesn't quite cut it so far. To go the last mile we duplicate the job + //! object handle into our parent process (a python process probably) and then + //! close our own handle. This means that the only handle to the job object + //! resides in the parent python process, so when python dies the whole build + //! system dies (as one would probably expect!). + //! + //! Note that this module has a #[cfg(windows)] above it as none of this logic + //! is required on Unix. + + use crate::Build; + use std::env; + use std::ffi::c_void; + use std::io; + use std::mem; + + use windows::{ + core::PCWSTR, + Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, + Win32::System::Diagnostics::Debug::{ + SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE, + }, + Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, + SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, + }, + Win32::System::Threading::{ + GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, + }, + }; + + pub unsafe fn setup(build: &mut Build) { + // Enable the Windows Error Reporting dialog which msys disables, + // so we can JIT debug rustc + let mode = SetErrorMode(THREAD_ERROR_MODE::default()); + let mode = THREAD_ERROR_MODE(mode); + SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); + + // Create a new job object for us to use + let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); + + // Indicate that when all handles to the job object are gone that all + // process in the object should be killed. Note that this includes our + // entire process tree by default because we've added ourselves and our + // children will reside in the job by default. + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if build.config.low_priority { + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; + info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; + } + let r = SetInformationJobObject( + job, + JobObjectExtendedLimitInformation, + &info as *const _ as *const c_void, + mem::size_of_val(&info) as u32, + ) + .ok(); + assert!(r.is_ok(), "{}", io::Error::last_os_error()); + + // Assign our process to this job object. Note that if this fails, one very + // likely reason is that we are ourselves already in a job object! This can + // happen on the build bots that we've got for Windows, or if just anyone + // else is instrumenting the build. In this case we just bail out + // immediately and assume that they take care of it. + // + // Also note that nested jobs (why this might fail) are supported in recent + // versions of Windows, but the version of Windows that our bots are running + // at least don't support nested job objects. + let r = AssignProcessToJobObject(job, GetCurrentProcess()).ok(); + if r.is_err() { + CloseHandle(job); + return; + } + + // If we've got a parent process (e.g., the python script that called us) + // then move ownership of this job object up to them. That way if the python + // script is killed (e.g., via ctrl-c) then we'll all be torn down. + // + // If we don't have a parent (e.g., this was run directly) then we + // intentionally leak the job object handle. When our process exits + // (normally or abnormally) it will close the handle implicitly, causing all + // processes in the job to be cleaned up. + let pid = match env::var("BOOTSTRAP_PARENT_ID") { + Ok(s) => s, + Err(..) => return, + }; + + let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { + Some(parent) => parent, + _ => { + // If we get a null parent pointer here, it is possible that either + // we have an invalid pid or the parent process has been closed. + // Since the first case rarely happens + // (only when wrongly setting the environmental variable), + // it might be better to improve the experience of the second case + // when users have interrupted the parent process and we haven't finish + // duplicating the handle yet. We just need close the job object if that occurs. + CloseHandle(job); + return; + } + }; + + let mut parent_handle = HANDLE::default(); + let r = DuplicateHandle( + GetCurrentProcess(), + job, + parent, + &mut parent_handle, + 0, + false, + DUPLICATE_SAME_ACCESS, + ) + .ok(); + + // If this failed, well at least we tried! An example of DuplicateHandle + // failing in the past has been when the wrong python2 package spawned this + // build system (e.g., the `python2` package in MSYS instead of + // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure + // mode" here is that we only clean everything up when the build system + // dies, not when the python parent does, so not too bad. + if r.is_err() { + CloseHandle(job); + } + } +} diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/src/utils/metrics.rs similarity index 99% rename from src/bootstrap/metrics.rs rename to src/bootstrap/src/utils/metrics.rs index cf8d33dfcb0..65794f05d2d 100644 --- a/src/bootstrap/metrics.rs +++ b/src/bootstrap/src/utils/metrics.rs @@ -4,8 +4,8 @@ //! As this module requires additional dependencies not present during local builds, it's cfg'd //! away whenever the `build.metrics` config option is not set to `true`. -use crate::builder::{Builder, Step}; -use crate::util::t; +use crate::core::builder::{Builder, Step}; +use crate::utils::helpers::t; use crate::Build; use build_helper::metrics::{ JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs new file mode 100644 index 00000000000..4f531f70cd6 --- /dev/null +++ b/src/bootstrap/src/utils/mod.rs @@ -0,0 +1,14 @@ +//! This module contains integral components of the build and configuration process, providing +//! support for a wide range of tasks and operations such as caching, tarballs, release +//! channels, job management, etc. + +pub(crate) mod cache; +pub(crate) mod cc_detect; +pub(crate) mod channel; +pub(crate) mod dylib_util; +pub(crate) mod helpers; +pub(crate) mod job; +#[cfg(feature = "build-metrics")] +pub(crate) mod metrics; +pub(crate) mod render_tests; +pub(crate) mod tarball; diff --git a/src/bootstrap/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs similarity index 99% rename from src/bootstrap/render_tests.rs rename to src/bootstrap/src/utils/render_tests.rs index 6802bf4511b..f97aa958513 100644 --- a/src/bootstrap/render_tests.rs +++ b/src/bootstrap/src/utils/render_tests.rs @@ -6,7 +6,7 @@ //! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had //! to reimplement all the rendering logic in this module because of that. -use crate::builder::Builder; +use crate::core::builder::Builder; use std::io::{BufRead, BufReader, Read, Write}; use std::process::{ChildStdout, Command, Stdio}; use std::time::Duration; diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/src/utils/tarball.rs similarity index 95% rename from src/bootstrap/tarball.rs rename to src/bootstrap/src/utils/tarball.rs index 95d909c5730..b437456f8a1 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/src/utils/tarball.rs @@ -3,9 +3,10 @@ use std::{ process::Command, }; -use crate::builder::Builder; -use crate::channel; -use crate::util::t; +use crate::core::build_steps::dist::distdir; +use crate::core::builder::Builder; +use crate::utils::channel; +use crate::utils::helpers::t; #[derive(Copy, Clone)] pub(crate) enum OverlayKind { @@ -112,7 +113,7 @@ impl<'a> Tarball<'a> { } fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { - let pkgname = crate::dist::pkgname(builder, component); + let pkgname = crate::core::build_steps::dist::pkgname(builder, component); let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); if let Some(target) = &target { @@ -265,7 +266,7 @@ impl<'a> Tarball<'a> { t!(std::fs::rename(&self.image_dir, &dest)); self.run(|this, cmd| { - let distdir = crate::dist::distdir(this.builder); + let distdir = distdir(this.builder); t!(std::fs::create_dir_all(&distdir)); cmd.arg("tarball") .arg("--input") @@ -292,7 +293,7 @@ impl<'a> Tarball<'a> { .arg("--non-installed-overlay") .arg(&self.overlay_dir) .arg("--output-dir") - .arg(crate::dist::distdir(self.builder)); + .arg(distdir(self.builder)); } fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { @@ -306,11 +307,11 @@ impl<'a> Tarball<'a> { self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); } - let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); + let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller); let package_name = self.package_name(); self.builder.info(&format!("Dist {package_name}")); - let _time = crate::util::timeit(self.builder); + let _time = crate::utils::helpers::timeit(self.builder); build_cli(&self, &mut cmd); cmd.arg("--work-dir").arg(&self.temp_dir); @@ -344,7 +345,7 @@ impl<'a> Tarball<'a> { .unwrap_or("gz"); GeneratedTarball { - path: crate::dist::distdir(self.builder).join(format!("{package_name}.tar.{ext}")), + path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")), decompressed_output, work: self.temp_dir, } From f1e3e75f6c046796541495da2e7d8a7c852b3e78 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 8 Oct 2023 00:07:59 +0300 Subject: [PATCH 5/6] rename bootstrap module utils/dylib_util->utils/dylib Signed-off-by: onur-ozkan --- src/bootstrap/src/bin/rustc.rs | 2 +- src/bootstrap/src/bin/rustdoc.rs | 2 +- src/bootstrap/src/utils/{dylib_util.rs => dylib.rs} | 0 src/bootstrap/src/utils/helpers.rs | 2 +- src/bootstrap/src/utils/mod.rs | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/bootstrap/src/utils/{dylib_util.rs => dylib.rs} (100%) diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index b050713265b..241ae16e595 100644 --- a/src/bootstrap/src/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -25,7 +25,7 @@ use dylib_util::{dylib_path, dylib_path_var}; #[path = "../utils/bin_helpers.rs"] mod bin_helpers; -#[path = "../utils/dylib_util.rs"] +#[path = "../utils/dylib.rs"] mod dylib_util; fn main() { diff --git a/src/bootstrap/src/bin/rustdoc.rs b/src/bootstrap/src/bin/rustdoc.rs index 85d685475b4..f5f80ba2a0b 100644 --- a/src/bootstrap/src/bin/rustdoc.rs +++ b/src/bootstrap/src/bin/rustdoc.rs @@ -12,7 +12,7 @@ use dylib_util::{dylib_path, dylib_path_var}; #[path = "../utils/bin_helpers.rs"] mod bin_helpers; -#[path = "../utils/dylib_util.rs"] +#[path = "../utils/dylib.rs"] mod dylib_util; fn main() { diff --git a/src/bootstrap/src/utils/dylib_util.rs b/src/bootstrap/src/utils/dylib.rs similarity index 100% rename from src/bootstrap/src/utils/dylib_util.rs rename to src/bootstrap/src/utils/dylib.rs diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 90db0a89654..bb84b70d987 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -16,7 +16,7 @@ use crate::core::builder::Builder; use crate::core::config::{Config, TargetSelection}; use crate::OnceCell; -pub use crate::utils::dylib_util::{dylib_path, dylib_path_var}; +pub use crate::utils::dylib::{dylib_path, dylib_path_var}; /// A helper macro to `unwrap` a result except also print out details like: /// diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs index 4f531f70cd6..7dcb6a82862 100644 --- a/src/bootstrap/src/utils/mod.rs +++ b/src/bootstrap/src/utils/mod.rs @@ -5,7 +5,7 @@ pub(crate) mod cache; pub(crate) mod cc_detect; pub(crate) mod channel; -pub(crate) mod dylib_util; +pub(crate) mod dylib; pub(crate) mod helpers; pub(crate) mod job; #[cfg(feature = "build-metrics")] From 3ecff1b760152191f41bb4872c02f818b80e7d10 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 12 Oct 2023 14:07:36 +0300 Subject: [PATCH 6/6] bootstrap: fix warnings Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/run.rs | 2 -- src/bootstrap/src/utils/bin_helpers.rs | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 29272bebf99..d1d6b7e869e 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -1,8 +1,6 @@ use std::path::PathBuf; use std::process::Command; -use clap_complete::shells; - use crate::core::build_steps::dist::distdir; use crate::core::build_steps::test; use crate::core::build_steps::tool::{self, SourceType, Tool}; diff --git a/src/bootstrap/src/utils/bin_helpers.rs b/src/bootstrap/src/utils/bin_helpers.rs index ab41a6b9600..b9177c490ac 100644 --- a/src/bootstrap/src/utils/bin_helpers.rs +++ b/src/bootstrap/src/utils/bin_helpers.rs @@ -18,6 +18,7 @@ pub(crate) fn parse_rustc_verbose() -> usize { /// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. /// /// If "RUSTC_STAGE" was not set, the program will be terminated with 101. +#[allow(unused)] pub(crate) fn parse_rustc_stage() -> String { std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead.