From 5e2de479fa07a365d89bacac473f241960970b21 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 15:01:34 +0300 Subject: [PATCH 1/8] Ensure compiler crate paths are generated before linking Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 628a4ece8e9..b74f6f26fea 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -685,18 +685,7 @@ fn run(self, builder: &Builder<'_>) { target, ); - // This uses a shared directory so that librustdoc documentation gets - // correctly built and merged with the rustc documentation. This is - // needed because rustdoc is built in a different directory from - // rustc. rustdoc needs to be able to see everything, for example when - // merging the search index, or generating local (relative) links. let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); - t!(fs::create_dir_all(out_dir.parent().unwrap())); - symlink_dir_force(&builder.config, &out, &out_dir); - // Cargo puts proc macros in `target/doc` even if you pass `--target` - // explicitly (https://github.com/rust-lang/cargo/issues/7677). - let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); // Build cargo command. let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); @@ -736,6 +725,18 @@ fn run(self, builder: &Builder<'_>) { } } + // This uses a shared directory so that librustdoc documentation gets + // correctly built and merged with the rustc documentation. + // + // This is needed because rustdoc is built in a different directory from + // rustc. rustdoc needs to be able to see everything, for example when + // merging the search index, or generating local (relative) links. + symlink_dir_force(&builder.config, &out, &out_dir); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + builder.run(&mut cargo.into()); if builder.paths.iter().any(|path| path.ends_with("compiler")) { From fda009c858fe2039f981be56ce4b177da1fb8703 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 15:02:38 +0300 Subject: [PATCH 2/8] add sanity check for compiler crate docs Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index b74f6f26fea..99440482adc 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -739,6 +739,14 @@ fn run(self, builder: &Builder<'_>) { builder.run(&mut cargo.into()); + if !builder.config.dry_run() { + // Sanity check on linked compiler crates + for krate in &*self.crates { + let dir_name = krate.replace("-", "_"); + assert!(out.join(&*dir_name).exists()); + } + } + if builder.paths.iter().any(|path| path.ends_with("compiler")) { // For `x.py doc compiler --open`, open `rustc_middle` by default. let index = out.join("rustc_middle").join("index.html"); From d82fdb6616207dff00af563508c6e6cc5aaa4ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 28 Oct 2023 19:00:49 +0000 Subject: [PATCH 3/8] pass `CODEGEN_BACKENDS` to docker --- src/ci/docker/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index a2891ef9563..cedbc0390f8 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -302,6 +302,7 @@ docker \ --env DIST_TRY_BUILD \ --env PR_CI_JOB \ --env OBJDIR_ON_HOST="$objdir" \ + --env CODEGEN_BACKENDS \ --init \ --rm \ rust-ci \ From a9913ebed0befb09ed21725e1e30cdbd6a9c33ea Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:11:51 +0000 Subject: [PATCH 4/8] Ensure the rustc-codegen-cranelift-preview component is never empty Either generate it with the actual codegen backend dylib or omit it entirely when the cranelift backend is disabled for this build. --- src/bootstrap/src/core/build_steps/dist.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index b578c5ec295..197ce78d885 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1305,6 +1305,10 @@ fn run(self, builder: &Builder<'_>) -> Option { return None; } + if !builder.config.rust_codegen_backends.contains(&self.backend) { + return None; + } + if self.backend == "cranelift" { if !target_supports_cranelift_backend(self.compiler.host) { builder.info("target not supported by rustc_codegen_cranelift. skipping"); @@ -1343,12 +1347,15 @@ fn run(self, builder: &Builder<'_>) -> Option { let backends_dst = PathBuf::from("lib").join(&backends_rel); let backend_name = format!("rustc_codegen_{}", backend); + let mut found_backend = false; for backend in fs::read_dir(&backends_src).unwrap() { let file_name = backend.unwrap().file_name(); if file_name.to_str().unwrap().contains(&backend_name) { tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); + found_backend = true; } } + assert!(found_backend); Some(tarball.generate()) } From 451778b5517a7017eda2fea79ebb5078f11da3fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sun, 29 Oct 2023 13:12:02 +0000 Subject: [PATCH 5/8] handle dry runs in `dist::CodegenBackend` self.number_of_times_dry_runs_have_caused_issues += 1; --- src/bootstrap/src/core/build_steps/dist.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 197ce78d885..6e80c55c8ce 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1298,6 +1298,10 @@ fn make_run(run: RunConfig<'_>) { } fn run(self, builder: &Builder<'_>) -> Option { + if builder.config.dry_run() { + return None; + } + // This prevents rustc_codegen_cranelift from being built for "dist" // or "install" on the stable/beta channels. It is not yet stable and // should not be included. From 55b49455e79bf43ad07e3d1f1482b07ca8cf3bdf Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 15:43:09 +0300 Subject: [PATCH 6/8] ensure doc directory is not empty Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 99440482adc..1ef481408f9 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -743,7 +743,8 @@ fn run(self, builder: &Builder<'_>) { // Sanity check on linked compiler crates for krate in &*self.crates { let dir_name = krate.replace("-", "_"); - assert!(out.join(&*dir_name).exists()); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); } } From 52933e0bd2006b948f174a256cca65197d2ff4a6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 4 Nov 2023 16:49:10 +0300 Subject: [PATCH 7/8] Don't disable inline asm usage in compiler-builtins when the cranelift backend is enabled This was a leftover from when inline asm wasn't universally supported by the cranelift backend yet. It would cause the optimized inline asm intrinsics to be avoided for the standard library distributed with rustup now that we build the cranelift backend by default on nightly for several tier 1 targets. --- src/bootstrap/src/core/build_steps/compile.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 1eed534150b..8ab0ef61c50 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -413,11 +413,6 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car let mut features = String::new(); - // Cranelift doesn't support `asm`. - if stage != 0 && builder.config.default_codegen_backend().unwrap_or_default() == "cranelift" { - features += " compiler-builtins-no-asm"; - } - if builder.no_std(target) == Some(true) { features += " compiler-builtins-mem"; if !target.starts_with("bpf") { From b0df8216ed4ab2d59e8cb94974ca5d2e01c8fc63 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 4 Nov 2023 19:35:58 +0300 Subject: [PATCH 8/8] bootstrap: improve linking of tool docs Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/doc.rs | 81 +++++++++++------------ 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 1ef481408f9..f2a185f70fe 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -685,8 +685,6 @@ fn run(self, builder: &Builder<'_>) { target, ); - let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); - // Build cargo command. let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); cargo.rustdocflag("--document-private-items"); @@ -713,6 +711,7 @@ fn run(self, builder: &Builder<'_>) { let mut to_open = None; + let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); for krate in &*self.crates { // Create all crate output directories first to make sure rustdoc uses // relative links. @@ -766,10 +765,10 @@ macro_rules! tool_doc { $should_run: literal, $path: literal, $(rustc_tool = $rustc_tool:literal, )? - $(in_tree = $in_tree:literal, )? - [$($extra_arg: literal),+ $(,)?] - $(,)? - ) => { + $(in_tree = $in_tree:literal ,)? + $(is_library = $is_library:expr,)? + $(crates = $crates:expr)? + ) => { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct $tool { target: TargetSelection, @@ -822,17 +821,6 @@ fn run(self, builder: &Builder<'_>) { SourceType::Submodule }; - // Symlink compiler docs to the output directory of rustdoc documentation. - let out_dirs = [ - builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"), - // Cargo uses a different directory for proc macros. - builder.stage_out(compiler, Mode::ToolRustc).join("doc"), - ]; - for out_dir in out_dirs { - t!(fs::create_dir_all(&out_dir)); - symlink_dir_force(&builder.config, &out, &out_dir); - } - // Build cargo command. let mut cargo = prepare_tool_cargo( builder, @@ -849,9 +837,13 @@ fn run(self, builder: &Builder<'_>) { // Only include compiler crates, no dependencies of those, such as `libc`. cargo.arg("--no-deps"); - $( - cargo.arg($extra_arg); - )+ + if false $(|| $is_library)? { + cargo.arg("--lib"); + } + + $(for krate in $crates { + cargo.arg("-p").arg(krate); + })? cargo.rustdocflag("--document-private-items"); // Since we always pass --document-private-items, there's no need to warn about linking to private items. @@ -861,62 +853,69 @@ fn run(self, builder: &Builder<'_>) { cargo.rustdocflag("--generate-link-to-definition"); cargo.rustdocflag("-Zunstable-options"); + let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"); + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); + })? + + // Symlink compiler docs to the output directory of rustdoc documentation. + symlink_dir_force(&builder.config, &out, &out_dir); + let proc_macro_out_dir = builder.stage_out(compiler, Mode::ToolRustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); builder.run(&mut cargo.into()); + + if !builder.config.dry_run() { + // Sanity check on linked doc directories + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + })? + } } } } } -tool_doc!( - Rustdoc, - "rustdoc-tool", - "src/tools/rustdoc", - ["-p", "rustdoc", "-p", "rustdoc-json-types"] -); +tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", crates = ["rustdoc", "rustdoc-json-types"]); tool_doc!( Rustfmt, "rustfmt-nightly", "src/tools/rustfmt", - ["-p", "rustfmt-nightly", "-p", "rustfmt-config_proc_macro"], + crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"] ); -tool_doc!(Clippy, "clippy", "src/tools/clippy", ["-p", "clippy_utils"]); -tool_doc!(Miri, "miri", "src/tools/miri", ["-p", "miri"]); +tool_doc!(Clippy, "clippy", "src/tools/clippy", crates = ["clippy_utils"]); +tool_doc!(Miri, "miri", "src/tools/miri", crates = ["miri"]); tool_doc!( Cargo, "cargo", "src/tools/cargo", rustc_tool = false, in_tree = false, - [ - "-p", + crates = [ "cargo", - "-p", "cargo-platform", - "-p", "cargo-util", - "-p", "crates-io", - "-p", "cargo-test-macro", - "-p", "cargo-test-support", - "-p", "cargo-credential", - "-p", "mdman", // FIXME: this trips a license check in tidy. - // "-p", // "resolver-tests", ] ); -tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, ["-p", "tidy"]); +tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, crates = ["tidy"]); tool_doc!( Bootstrap, "bootstrap", "src/bootstrap", rustc_tool = false, - ["--lib", "-p", "bootstrap"] + is_library = true, + crates = ["bootstrap"] ); #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]