From 0be4046bf91ea3726098e7714642697c43af7e98 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 31 Jan 2021 12:20:30 -0500 Subject: [PATCH] Move llvm submodule updates to rustbuild This enables better caching, since LLVM is only updated when needed, not whenever x.py is run. Before, bootstrap.py had to use heuristics to guess if LLVM would be needed, and updated the module more often than necessary as a result. This syncs the LLVM submodule only just before building the compiler, so people working on the standard library never have to worry about it. Example output: ``` Copying stage0 std from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) Updating submodule src/llvm-project Submodule 'src/llvm-project' (https://github.com/rust-lang/llvm-project.git) registered for path 'src/llvm-project' Submodule path 'src/llvm-project': checked out 'f9a8d70b6e0365ac2172ca6b7f1de0341297458d' ``` - Don't try to update the LLVM submodule when using system LLVM Previously, this would try to update LLVM unconditionally. Now the submodule is only initialized if `llvm-config` is not set. - Don't update LLVM submodule in dry runs This prevents the following test failures: ``` running 17 tests fatal: invalid gitfile format: /checkout/src/llvm-project/.git test builder::tests::defaults::build_cross_compile ... FAILED ---- builder::tests::defaults::build_default stdout ---- thread 'main' panicked at 'command did not execute successfully: "git" "rev-parse" "HEAD" expected success, got: exit code: 128', src/build_helper/lib.rs:139:9 ``` - Try running git without --progress if it fails the first time This avoids having to do version detection to see if --progress is supported or not. - Don't try to update submodules when the source repository isn't managed by git - Update LLVM submodules that have already been checked out - Only check for whether the submodule should be updated in lib.rs; update it unconditionally in native.rs --- src/bootstrap/bootstrap.py | 16 ++------ src/bootstrap/lib.rs | 10 +++++ src/bootstrap/native.rs | 84 +++++++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 13 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 149a899cef7..bd5b3797ea8 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -991,28 +991,20 @@ class RustBuild(object): ).decode(default_encoding).splitlines()] filtered_submodules = [] submodules_names = [] - llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git")) - external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm() - llvm_needed = not self.get_toml('codegen-backends', 'rust') \ - or "llvm" in self.get_toml('codegen-backends', 'rust') for module in submodules: + # This is handled by native::Llvm in rustbuild, not here if module.endswith("llvm-project"): - # Don't sync the llvm-project submodule if an external LLVM was - # provided, if we are downloading LLVM or if the LLVM backend is - # not being built. Also, if the submodule has been initialized - # already, sync it anyways so that it doesn't mess up contributor - # pull requests. - if external_llvm_provided or not llvm_needed: - if self.get_toml('lld') != 'true' and not llvm_checked_out: - continue + continue check = self.check_submodule(module, slow_submodules) filtered_submodules.append((module, check)) submodules_names.append(module) recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, cwd=self.rust_root, stdout=subprocess.PIPE) recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() + # { filename: hash } recorded_submodules = {} for data in recorded: + # [mode, kind, hash, filename] data = data.split() recorded_submodules[data[3]] = data[2] for module in filtered_submodules: diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 2960dd3df6b..a351290a420 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -472,12 +472,22 @@ impl Build { slice::from_ref(&self.build.triple) } + /// If the LLVM submodule has been initialized already, sync it unconditionally. This avoids + /// contributors checking in a submodule change by accident. + pub fn maybe_update_llvm_submodule(&self) { + if self.in_tree_llvm_info.is_git() { + native::update_llvm_submodule(self); + } + } + /// Executes the entire build, as configured by the flags and configuration. pub fn build(&mut self) { unsafe { job::setup(self); } + self.maybe_update_llvm_submodule(); + if let Subcommand::Format { check, paths } = &self.config.cmd { return format::format(self, *check, &paths); } diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index bde0a96f030..44c281efe22 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -21,7 +21,7 @@ use build_helper::{output, t}; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::TargetSelection; use crate::util::{self, exe}; -use crate::GitRepo; +use crate::{Build, GitRepo}; use build_helper::up_to_date; pub struct Meta { @@ -91,6 +91,85 @@ pub fn prebuilt_llvm_config( Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) } +// modified from `check_submodule` and `update_submodule` in bootstrap.py +pub(crate) fn update_llvm_submodule(build: &Build) { + let llvm_project = &Path::new("src").join("llvm-project"); + + fn dir_is_empty(dir: &Path) -> bool { + t!(std::fs::read_dir(dir)).next().is_none() + } + + // NOTE: The check for the empty directory is here because when running x.py + // the first time, the llvm submodule won't be checked out. Check it out + // now so we can build it. + if !build.in_tree_llvm_info.is_git() && !dir_is_empty(&build.config.src.join(llvm_project)) { + return; + } + + // check_submodule + let checked_out = if build.config.fast_submodules { + Some(output( + Command::new("git") + .args(&["rev-parse", "HEAD"]) + .current_dir(build.config.src.join(llvm_project)), + )) + } else { + None + }; + + // update_submodules + let recorded = output( + Command::new("git") + .args(&["ls-tree", "HEAD"]) + .arg(llvm_project) + .current_dir(&build.config.src), + ); + let hash = + recorded.split(' ').nth(2).unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + + // update_submodule + if let Some(llvm_hash) = checked_out { + if hash == llvm_hash { + // already checked out + return; + } + } + + println!("Updating submodule {}", llvm_project.display()); + build.run( + Command::new("git") + .args(&["submodule", "-q", "sync"]) + .arg(llvm_project) + .current_dir(&build.config.src), + ); + + // Try passing `--progress` to start, then run git again without if that fails. + let update = |progress: bool| { + let mut git = Command::new("git"); + git.args(&["submodule", "update", "--init", "--recursive"]); + if progress { + git.arg("--progress"); + } + git.arg(llvm_project).current_dir(&build.config.src); + git + }; + // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. + if !update(true).status().map_or(false, |status| status.success()) { + build.run(&mut update(false)); + } + + build.run( + Command::new("git") + .args(&["reset", "-q", "--hard"]) + .current_dir(build.config.src.join(llvm_project)), + ); + build.run( + Command::new("git") + .args(&["clean", "-qdfx"]) + .current_dir(build.config.src.join(llvm_project)), + ); +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Llvm { pub target: TargetSelection, @@ -128,6 +207,9 @@ impl Step for Llvm { Err(m) => m, }; + if !builder.config.dry_run { + update_llvm_submodule(builder); + } if builder.config.llvm_link_shared && (target.contains("windows") || target.contains("apple-darwin")) {