mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #98483 - dvtkrlbs:bootstrap-dist, r=jyn514

Browse Source 
Distribute bootstrap in CI

This pre-compiles bootstrap from source and adds it to the existing `rust-dev` component. There are two main goals here:
1. Make it faster to build rust from source, both the first time and incrementally
2. Make it easier to add non-python entrypoints, since they can call out to bootstrap directly rather than having to figure out the right flags to pre-compile it. This second part is still in a bit of flux, see the tracking issue below for more information.

There are also several changes to make bootstrap able to run on a machine other than the one it was built (particularly around `config.src` and `config.out` detection). I (`@jyn514)` am slightly concerned these will regress unless tested - maybe we should add an automated test that runs bootstrap in a chroot or something? Unclear whether the effort is worth the test coverage.

Helps with https://github.com/rust-lang/rust/issues/94829.
This commit is contained in:
bors 2022-09-24 20:00:51 +00:00
commit 3f83906b30
7 changed files with 122 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/ci.yml vendored
View File

@ -297,7 +297,7 @@ jobs:
os: ubuntu-20.04-xl
- name: dist-x86_64-apple
env:
SCRIPT: "./x.py dist --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
@ -308,7 +308,7 @@ jobs:
os: macos-latest
- name: dist-apple-various
env:
SCRIPT: "./x.py dist --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
@ -318,7 +318,7 @@ jobs:
os: macos-latest
- name: dist-x86_64-apple-alt
env:
SCRIPT: "./x.py dist"
SCRIPT: "./x.py dist bootstrap --include-default-paths"
RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
@ -350,7 +350,7 @@ jobs:
os: macos-latest
- name: dist-aarch64-apple
env:
SCRIPT: "./x.py dist --stage 2"
SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2"
RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false"
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
USE_XCODE_CLANG: 1
@ -424,19 +424,19 @@ jobs:
- name: dist-x86_64-msvc
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist
SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl
- name: dist-i686-msvc
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl
- name: dist-aarch64-msvc
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
WINDOWS_SDK_20348_HACK: 1
os: windows-latest-xl
@ -444,13 +444,13 @@ jobs:
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
NO_DOWNLOAD_CI_LLVM: 1
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl
- name: dist-x86_64-mingw
env:
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
@ -459,7 +459,7 @@ jobs:
- name: dist-x86_64-msvc-alt
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
os: windows-latest-xl
timeout-minutes: 600
runs-on: "${{ matrix.os }}"

1
src/bootstrap/builder.rs
View File

@ -724,6 +724,7 @@ macro_rules! describe {
dist::Miri,
dist::LlvmTools,
dist::RustDev,
dist::Bootstrap,
dist::Extended,
// It seems that PlainSourceTarball somehow changes how some of the tools
// perceive their dependencies (see #93033) which would invalidate fingerprints

56
src/bootstrap/config.rs
View File

@ -772,21 +772,20 @@ pub fn default_opts() -> Config {
// set by build.rs
config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE"));
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
// Undo `src/bootstrap`
config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned();
config.out = PathBuf::from("build");
config.initial_cargo = PathBuf::from(env!("CARGO"));
config.initial_rustc = PathBuf::from(env!("RUSTC"));
config
}
pub fn parse(args: &[String]) -> Config {
let flags = Flags::parse(&args);
let mut config = Config::default_opts();
// Set flags.
config.exclude = flags.exclude.into_iter().map(|path| TaskPath::parse(path)).collect();
config.include_default_paths = flags.include_default_paths;
config.rustc_error_format = flags.rustc_error_format;
@ -805,7 +804,49 @@ pub fn parse(args: &[String]) -> Config {
config.llvm_profile_use = flags.llvm_profile_use;
config.llvm_profile_generate = flags.llvm_profile_generate;
// Infer the rest of the configuration.
// Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary,
// running on a completely machine from where it was compiled.
let mut cmd = Command::new("git");
// NOTE: we cannot support running from outside the repository because the only path we have available
// is set at compile time, which can be wrong if bootstrap was downloaded from source.
// We still support running outside the repository if we find we aren't in a git directory.
cmd.arg("rev-parse").arg("--show-toplevel");
// Discard stderr because we expect this to fail when building from a tarball.
let output = cmd
.stderr(std::process::Stdio::null())
.output()
.ok()
.and_then(|output| if output.status.success() { Some(output) } else { None });
if let Some(output) = output {
let git_root = String::from_utf8(output.stdout).unwrap();
// We need to canonicalize this path to make sure it uses backslashes instead of forward slashes.
let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap();
let s = git_root.to_str().unwrap();
// Bootstrap is quite bad at handling /? in front of paths
config.src = match s.strip_prefix("\\\\?\\") {
Some(p) => PathBuf::from(p),
None => PathBuf::from(git_root),
};
} else {
// We're building from a tarball, not git sources.
// We don't support pre-downloaded bootstrap in this case.
}
if cfg!(test) {
// Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly.
config.out = Path::new(
&env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"),
)
.parent()
.unwrap()
.to_path_buf();
}
let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
#[cfg(test)]
@ -860,6 +901,7 @@ pub fn parse(args: &[String]) -> Config {
config.config = toml_path;
let build = toml.build.unwrap_or_default();
let has_custom_rustc = build.rustc.is_some();
set(&mut config.initial_rustc, build.rustc.map(PathBuf::from));
set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from)));
@ -870,6 +912,12 @@ pub fn parse(args: &[String]) -> Config {
config.out = crate::util::absolute(&config.out);
}
if !has_custom_rustc && !config.initial_rustc.starts_with(&config.out) {
config.initial_rustc = config.out.join(config.build.triple).join("stage0/bin/rustc");
config.initial_cargo = config.out.join(config.build.triple).join("stage0/bin/cargo");
}
// NOTE: it's important this comes *after* we set `initial_rustc` just above.
if config.dry_run {
let dir = config.out.join("tmp-dry-run");
t!(fs::create_dir_all(&dir));

37
src/bootstrap/dist.rs
View File

@ -1879,7 +1879,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
let mut cmd = Command::new(llvm_config);
cmd.arg("--libfiles");
builder.verbose(&format!("running {:?}", cmd));
let files = output(&mut cmd);
let files = if builder.config.dry_run { "".into() } else { output(&mut cmd) };
let build_llvm_out = &builder.llvm_out(builder.config.build);
let target_llvm_out = &builder.llvm_out(target);
for file in files.trim_end().split(' ') {
@ -2057,6 +2057,41 @@ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
}
}
// Tarball intended for internal consumption to ease rustc/std development.
//
// Should not be considered stable by end users.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct Bootstrap {
pub target: TargetSelection,
}
impl Step for Bootstrap {
type Output = Option<GeneratedTarball>;
const DEFAULT: bool = false;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.alias("bootstrap")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Bootstrap { target: run.target });
}
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let target = self.target;
let tarball = Tarball::new(builder, "bootstrap", &target.triple);
let bootstrap_outdir = &builder.bootstrap_out;
for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] {
tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755);
}
Some(tarball.generate())
}
}
/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
/// release process to avoid cloning the monorepo and building stuff.
///

21
src/bootstrap/lib.rs
View File

@ -458,19 +458,18 @@ pub fn new(config: Config) -> Build {
.expect("failed to read src/version");
let version = version.trim();
let bootstrap_out = if std::env::var("BOOTSTRAP_PYTHON").is_ok() {
out.join("bootstrap").join("debug")
} else {
let workspace_target_dir = std::env::var("CARGO_TARGET_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| src.join("target"));
let bootstrap_out = workspace_target_dir.join("debug");
if !bootstrap_out.join("rustc").exists() && !cfg!(test) {
let bootstrap_out = std::env::current_exe()
.expect("could not determine path to running process")
.parent()
.unwrap()
.to_path_buf();
if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) {
// this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented
panic!("run `cargo build --bins` before `cargo run`")
panic!(
"`rustc` not found in {}, run `cargo build --bins` before `cargo run`",
bootstrap_out.display()
)
}
bootstrap_out
};
let mut build = Build {
initial_rustc: config.initial_rustc.clone(),

2
src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
View File

@ -82,7 +82,7 @@ ENV RUST_CONFIGURE_ARGS \
ENV SCRIPT ../src/ci/pgo.sh python3 ../x.py dist \
--host $HOSTS --target $HOSTS \
--include-default-paths \
build-manifest
build-manifest bootstrap
ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
# This is the only builder which will create source tarballs

20
src/ci/github-actions/ci.yml
View File

@ -462,7 +462,7 @@ jobs:
- name: dist-x86_64-apple
env:
SCRIPT: ./x.py dist --host=x86_64-apple-darwin --target=x86_64-apple-darwin
SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
@ -474,7 +474,7 @@ jobs:
- name: dist-apple-various
env:
SCRIPT: ./x.py dist --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
@ -485,7 +485,7 @@ jobs:
- name: dist-x86_64-apple-alt
env:
SCRIPT: ./x.py dist
SCRIPT: ./x.py dist bootstrap --include-default-paths
RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
@ -515,7 +515,7 @@ jobs:
# This target only needs to support 11.0 and up as nothing else supports the hardware
- name: dist-aarch64-apple
env:
SCRIPT: ./x.py dist --stage 2
SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2
RUST_CONFIGURE_ARGS: >-
--build=x86_64-apple-darwin
--host=aarch64-apple-darwin
@ -659,7 +659,7 @@ jobs:
--target=x86_64-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist
SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl
@ -671,7 +671,7 @@ jobs:
--target=i686-pc-windows-msvc,i586-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl
@ -682,7 +682,7 @@ jobs:
--host=aarch64-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
# Hack around this SDK version, because it doesn't work with clang.
# See https://github.com/rust-lang/rust/issues/88796
@ -699,14 +699,14 @@ jobs:
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
NO_DOWNLOAD_CI_LLVM: 1
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl
- name: dist-x86_64-mingw
env:
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
RUST_CONFIGURE_ARGS: >-
--build=x86_64-pc-windows-gnu
--enable-full-tools
@ -722,7 +722,7 @@ jobs:
- name: dist-x86_64-msvc-alt
env:
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
SCRIPT: python x.py dist
SCRIPT: python x.py dist bootstrap --include-default-paths
<<: *job-windows-xl
try: