Auto merge of #64328 - Mark-Simulacrum:rustdoc-find-rustc, r=GuillaumeGomez

rustdoc: change doctests locating rustc binary

We previously used the "naive" approach of replacing the `current_exe()`'s file name with rustc, but now load from the sysroot by default (`$sysroot/bin/rustc`). The functionality of locating the sysroot overlaps/is the same as the functionality used by codegen backend loading; this ensures that any failure cases we've introduced are not exceeding those, and that improvements to finding the sysroot for loading codegen backends likewise enhance rustdoc.

The second commit adds an unstable `--test-builder` flag to rustdoc, and is largely separate (I can split into separate PR, but it's a simple and related change). This is largely intended for "advanced" users at this point (I'm not sure if we'll ever stabilize it); it permits use of a different rustc binary for rustdoc compilation of doctests than the rustdoc binary used when loading. Note, that this may not be what you want as the parsers and such differ (and rustdoc uses its own libsyntax, etc.). However, I've been told that running doctests in miri may be assisted by this change, so I've implemented it; I'll file a tracking issue for it if there's interest in it (and we land this PR).

Cargo.lock

@@ -2137,6 +2137,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6a04cb71e910d0034815600180f62a95bf6e67942d7ab52a166a68c7d7e9cd0"
+
 [[package]]
 name = "open"
 version = "1.2.1"
@@ -3425,6 +3431,7 @@ name = "rustc_interface"
 version = "0.0.0"
 dependencies = [
  "log",
+ "once_cell",
  "rustc",
  "rustc-rayon",
  "rustc_ast_borrowck",

src/bootstrap/builder.rs

@@ -1167,6 +1167,8 @@ pub fn cargo(
             cargo.arg("--frozen");
         }
 
+        cargo.env("RUSTC_INSTALL_BINDIR", &self.config.bindir);
+
         self.ci_env.force_coloring_in_ci(&mut cargo);
 
         cargo

src/bootstrap/config.rs

@@ -137,7 +137,7 @@ pub struct Config {
     pub sysconfdir: Option<PathBuf>,
     pub datadir: Option<PathBuf>,
     pub docdir: Option<PathBuf>,
-    pub bindir: Option<PathBuf>,
+    pub bindir: PathBuf,
     pub libdir: Option<PathBuf>,
     pub mandir: Option<PathBuf>,
     pub codegen_tests: bool,
@@ -400,6 +400,7 @@ pub fn parse(args: &[String]) -> Config {
         config.incremental = flags.incremental;
         config.dry_run = flags.dry_run;
         config.keep_stage = flags.keep_stage;
+        config.bindir = "bin".into(); // default
         if let Some(value) = flags.deny_warnings {
             config.deny_warnings = value;
         }
@@ -482,7 +483,7 @@ pub fn parse(args: &[String]) -> Config {
             config.sysconfdir = install.sysconfdir.clone().map(PathBuf::from);
             config.datadir = install.datadir.clone().map(PathBuf::from);
             config.docdir = install.docdir.clone().map(PathBuf::from);
-            config.bindir = install.bindir.clone().map(PathBuf::from);
+            set(&mut config.bindir, install.bindir.clone().map(PathBuf::from));
             config.libdir = install.libdir.clone().map(PathBuf::from);
             config.mandir = install.mandir.clone().map(PathBuf::from);
         }

src/bootstrap/install.rs

@@ -67,7 +67,6 @@ fn install_sh(
     let sysconfdir_default = PathBuf::from("/etc");
     let datadir_default = PathBuf::from("share");
     let docdir_default = datadir_default.join("doc/rust");
-    let bindir_default = PathBuf::from("bin");
     let libdir_default = PathBuf::from("lib");
     let mandir_default = datadir_default.join("man");
     let prefix = builder.config.prefix.as_ref().map_or(prefix_default, |p| {
@@ -76,7 +75,7 @@ fn install_sh(
     let sysconfdir = builder.config.sysconfdir.as_ref().unwrap_or(&sysconfdir_default);
     let datadir = builder.config.datadir.as_ref().unwrap_or(&datadir_default);
     let docdir = builder.config.docdir.as_ref().unwrap_or(&docdir_default);
-    let bindir = builder.config.bindir.as_ref().unwrap_or(&bindir_default);
+    let bindir = &builder.config.bindir;
     let libdir = builder.config.libdir.as_ref().unwrap_or(&libdir_default);
     let mandir = builder.config.mandir.as_ref().unwrap_or(&mandir_default);
 

src/librustc_interface/Cargo.toml

@@ -34,3 +34,4 @@ rustc_plugin = { path = "../librustc_plugin", package = "rustc_plugin_impl" }
 rustc_privacy = { path = "../librustc_privacy" }
 rustc_resolve = { path = "../librustc_resolve" }
 tempfile = "3.0.5"
+once_cell = "1"

src/librustc_interface/build.rs

@@ -0,0 +1,4 @@
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=RUSTC_INSTALL_BINDIR");
+}

src/librustc_interface/util.rs

@@ -289,20 +289,39 @@ pub fn get_codegen_backend(sess: &Session) -> Box<dyn CodegenBackend> {
     backend
 }
 
-pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
-    // For now we only allow this function to be called once as it'll dlopen a
-    // few things, which seems to work best if we only do that once. In
-    // general this assertion never trips due to the once guard in `get_codegen_backend`,
-    // but there's a few manual calls to this function in this file we protect
-    // against.
-    static LOADED: AtomicBool = AtomicBool::new(false);
-    assert!(!LOADED.fetch_or(true, Ordering::SeqCst),
-            "cannot load the default codegen backend twice");
+// This is used for rustdoc, but it uses similar machinery to codegen backend
+// loading, so we leave the code here. It is potentially useful for other tools
+// that want to invoke the rustc binary while linking to rustc as well.
+pub fn rustc_path<'a>() -> Option<&'a Path> {
+    static RUSTC_PATH: once_cell::sync::OnceCell<Option<PathBuf>> =
+        once_cell::sync::OnceCell::new();
 
+    const BIN_PATH: &str = env!("RUSTC_INSTALL_BINDIR");
+
+    RUSTC_PATH.get_or_init(|| get_rustc_path_inner(BIN_PATH)).as_ref().map(|v| &**v)
+}
+
+fn get_rustc_path_inner(bin_path: &str) -> Option<PathBuf> {
+    sysroot_candidates().iter()
+        .filter_map(|sysroot| {
+            let candidate = sysroot.join(bin_path).join(if cfg!(target_os = "windows") {
+                "rustc.exe"
+            } else {
+                "rustc"
+            });
+            if candidate.exists() {
+                Some(candidate)
+            } else {
+                None
+            }
+        })
+        .next()
+}
+
+fn sysroot_candidates() -> Vec<PathBuf> {
     let target = session::config::host_triple();
     let mut sysroot_candidates = vec![filesearch::get_or_default_sysroot()];
-    let path = current_dll_path()
-        .and_then(|s| s.canonicalize().ok());
+    let path = current_dll_path().and_then(|s| s.canonicalize().ok());
     if let Some(dll) = path {
         // use `parent` twice to chop off the file name and then also the
         // directory containing the dll which should be either `lib` or `bin`.
@@ -327,69 +346,7 @@ pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend
         }
     }
 
-    let sysroot = sysroot_candidates.iter()
-        .map(|sysroot| {
-            let libdir = filesearch::relative_target_lib_path(&sysroot, &target);
-            sysroot.join(libdir).with_file_name(
-                option_env!("CFG_CODEGEN_BACKENDS_DIR").unwrap_or("codegen-backends"))
-        })
-        .filter(|f| {
-            info!("codegen backend candidate: {}", f.display());
-            f.exists()
-        })
-        .next();
-    let sysroot = sysroot.unwrap_or_else(|| {
-        let candidates = sysroot_candidates.iter()
-            .map(|p| p.display().to_string())
-            .collect::<Vec<_>>()
-            .join("\n* ");
-        let err = format!("failed to find a `codegen-backends` folder \
-                           in the sysroot candidates:\n* {}", candidates);
-        early_error(ErrorOutputType::default(), &err);
-    });
-    info!("probing {} for a codegen backend", sysroot.display());
-
-    let d = sysroot.read_dir().unwrap_or_else(|e| {
-        let err = format!("failed to load default codegen backend, couldn't \
-                           read `{}`: {}", sysroot.display(), e);
-        early_error(ErrorOutputType::default(), &err);
-    });
-
-    let mut file: Option<PathBuf> = None;
-
-    let expected_name = format!("rustc_codegen_llvm-{}", backend_name);
-    for entry in d.filter_map(|e| e.ok()) {
-        let path = entry.path();
-        let filename = match path.file_name().and_then(|s| s.to_str()) {
-            Some(s) => s,
-            None => continue,
-        };
-        if !(filename.starts_with(DLL_PREFIX) && filename.ends_with(DLL_SUFFIX)) {
-            continue
-        }
-        let name = &filename[DLL_PREFIX.len() .. filename.len() - DLL_SUFFIX.len()];
-        if name != expected_name {
-            continue
-        }
-        if let Some(ref prev) = file {
-            let err = format!("duplicate codegen backends found\n\
-                               first:  {}\n\
-                               second: {}\n\
-            ", prev.display(), path.display());
-            early_error(ErrorOutputType::default(), &err);
-        }
-        file = Some(path.clone());
-    }
-
-    match file {
-        Some(ref s) => return load_backend_from_dylib(s),
-        None => {
-            let err = format!("failed to load default codegen backend for `{}`, \
-                               no appropriate codegen dylib found in `{}`",
-                              backend_name, sysroot.display());
-            early_error(ErrorOutputType::default(), &err);
-        }
-    }
+    return sysroot_candidates;
 
     #[cfg(unix)]
     fn current_dll_path() -> Option<PathBuf> {
@@ -459,6 +416,85 @@ fn GetModuleFileNameW(hModule: usize,
     }
 }
 
+pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
+    // For now we only allow this function to be called once as it'll dlopen a
+    // few things, which seems to work best if we only do that once. In
+    // general this assertion never trips due to the once guard in `get_codegen_backend`,
+    // but there's a few manual calls to this function in this file we protect
+    // against.
+    static LOADED: AtomicBool = AtomicBool::new(false);
+    assert!(!LOADED.fetch_or(true, Ordering::SeqCst),
+            "cannot load the default codegen backend twice");
+
+    let target = session::config::host_triple();
+    let sysroot_candidates = sysroot_candidates();
+
+    let sysroot = sysroot_candidates.iter()
+        .map(|sysroot| {
+            let libdir = filesearch::relative_target_lib_path(&sysroot, &target);
+            sysroot.join(libdir).with_file_name(
+                option_env!("CFG_CODEGEN_BACKENDS_DIR").unwrap_or("codegen-backends"))
+        })
+        .filter(|f| {
+            info!("codegen backend candidate: {}", f.display());
+            f.exists()
+        })
+        .next();
+    let sysroot = sysroot.unwrap_or_else(|| {
+        let candidates = sysroot_candidates.iter()
+            .map(|p| p.display().to_string())
+            .collect::<Vec<_>>()
+            .join("\n* ");
+        let err = format!("failed to find a `codegen-backends` folder \
+                           in the sysroot candidates:\n* {}", candidates);
+        early_error(ErrorOutputType::default(), &err);
+    });
+    info!("probing {} for a codegen backend", sysroot.display());
+
+    let d = sysroot.read_dir().unwrap_or_else(|e| {
+        let err = format!("failed to load default codegen backend, couldn't \
+                           read `{}`: {}", sysroot.display(), e);
+        early_error(ErrorOutputType::default(), &err);
+    });
+
+    let mut file: Option<PathBuf> = None;
+
+    let expected_name = format!("rustc_codegen_llvm-{}", backend_name);
+    for entry in d.filter_map(|e| e.ok()) {
+        let path = entry.path();
+        let filename = match path.file_name().and_then(|s| s.to_str()) {
+            Some(s) => s,
+            None => continue,
+        };
+        if !(filename.starts_with(DLL_PREFIX) && filename.ends_with(DLL_SUFFIX)) {
+            continue
+        }
+        let name = &filename[DLL_PREFIX.len() .. filename.len() - DLL_SUFFIX.len()];
+        if name != expected_name {
+            continue
+        }
+        if let Some(ref prev) = file {
+            let err = format!("duplicate codegen backends found\n\
+                               first:  {}\n\
+                               second: {}\n\
+            ", prev.display(), path.display());
+            early_error(ErrorOutputType::default(), &err);
+        }
+        file = Some(path.clone());
+    }
+
+    match file {
+        Some(ref s) => return load_backend_from_dylib(s),
+        None => {
+            let err = format!("failed to load default codegen backend for `{}`, \
+                               no appropriate codegen dylib found in `{}`",
+                              backend_name, sysroot.display());
+            early_error(ErrorOutputType::default(), &err);
+        }
+    }
+
+}
+
 pub(crate) fn compute_crate_disambiguator(session: &Session) -> CrateDisambiguator {
     use std::hash::Hasher;
 

src/librustdoc/config.rs

@@ -86,6 +86,10 @@ pub struct Options {
     /// contains "foo" as a substring
     pub enable_per_target_ignores: bool,
 
+    /// The path to a rustc-like binary to build tests with. If not set, we
+    /// default to loading from $sysroot/bin/rustc.
+    pub test_builder: Option<PathBuf>,
+
     // Options that affect the documentation process
 
     /// The selected default set of passes to use.
@@ -476,6 +480,7 @@ pub fn from_matches(matches: &getopts::Matches) -> Result<Options, i32> {
         let generate_search_filter = !matches.opt_present("disable-per-crate-search");
         let persist_doctests = matches.opt_str("persist-doctests").map(PathBuf::from);
         let generate_redirect_pages = matches.opt_present("generate-redirect-pages");
+        let test_builder = matches.opt_str("test-builder").map(PathBuf::from);
         let codegen_options_strs = matches.opt_strs("C");
         let lib_strs = matches.opt_strs("L");
         let extern_strs = matches.opt_strs("extern");
@@ -515,6 +520,7 @@ pub fn from_matches(matches: &getopts::Matches) -> Result<Options, i32> {
             runtool,
             runtool_args,
             enable_per_target_ignores,
+            test_builder,
             render_options: RenderOptions {
                 output,
                 external_html,

src/librustdoc/lib.rs

@@ -373,6 +373,11 @@ fn opts() -> Vec<RustcOptGroup> {
                        "",
                        "One (of possibly many) arguments to pass to the runtool")
         }),
+        unstable("test-builder", |o| {
+            o.optflag("",
+                      "test-builder",
+                      "specified the rustc-like binary to use as the test builder")
+        }),
     ]
 }
 

src/librustdoc/test.rs

@@ -248,7 +248,10 @@ fn path(&self) -> &std::path::Path {
     };
     let output_file = outdir.path().join("rust_out");
 
-    let mut compiler = Command::new(std::env::current_exe().unwrap().with_file_name("rustc"));
+    let rustc_binary = options.test_builder.as_ref().map(|v| &**v).unwrap_or_else(|| {
+        rustc_interface::util::rustc_path().expect("found rustc")
+    });
+    let mut compiler = Command::new(&rustc_binary);
     compiler.arg("--crate-type").arg("bin");
     for cfg in &options.cfgs {
         compiler.arg("--cfg").arg(&cfg);