Auto merge of #12858 - fasterthanlime:proc-macro-srv-bin, r=Veykril

Add `rust-analyzer-proc-macro-srv` binary, use it if found in sysroot

This adds a `bin` crate which simply runs `proc_macro_srv::cli::run()` (it does no CLI argument parsing, nothing).

The intent is to build that crate in Rust CI as part of the `dist::Rustc` component, then ship it in the sysroot: it would probably land in something like `~/.rustup/toolchains/nightly-2022-07-23-x86_64-unknown-linux-gnu/libexec/proc-macro-srv-cli`.

This makes https://github.com/rust-lang/rustup/pull/3022 less pressing. (Instead of teaching RA about rustup components, we simply teach it to look in the sysroot via `rustc --print sysroot`. If it can't find `proc-macro-srv-cli`, it falls back to its own `proc-macro` subcommand).

This is closely related to https://github.com/rust-lang/rust-analyzer/issues/12803 (but doesn't close it yet).

Things to address now:

  * [ ] What should the binary be named? What should the crate be named? We can pick different names with `[bin]` in the `Cargo.toml`

Things to address later:

  * Disable the "multi ABI compatibility scheme" when building that binary in Rust CI (that'll probably happen in `rust-lang/rust`)
  * Teaching RA to look in the sysroot

Things to address much, much later:

  * Is JSON a good fit here
  * Do we want to add versioning to future-proof it?
  * Other bikesheds

When built with `--features sysroot` on `nightly-2022-07-23-x86_64-unknown-linux-gnu`, the binary is 7.4MB. After stripping debuginfo, it's 2.6MB. When compressed to `.tar.xz`, it's 619KB.

In a Zulip discussion, `@jyn514` and `@Mark-Simulacrum` seemed to think that those sizes weren't a stopper for including the binary in the rustc component, even before we shrink it down further.

Cargo.lock

@@ -1198,6 +1198,13 @@ dependencies = [
  "tt",
 ]
 
+[[package]]
+name = "proc-macro-srv-cli"
+version = "0.0.0"
+dependencies = [
+ "proc-macro-srv",
+]
+
 [[package]]
 name = "proc-macro-test"
 version = "0.0.0"

crates/proc-macro-api/src/process.rs

@@ -86,6 +86,7 @@ fn mk_child(
 ) -> io::Result<Child> {
     Command::new(path.as_os_str())
         .args(args)
+        .env("RUST_ANALYZER_INTERNALS_DO_NOT_USE", "this is unstable")
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::inherit())

crates/proc-macro-srv-cli/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "proc-macro-srv-cli"
+version = "0.0.0"
+description = "TBD"
+license = "MIT OR Apache-2.0"
+edition = "2021"
+rust-version = "1.57"
+
+[dependencies]
+proc-macro-srv = { version = "0.0.0", path = "../proc-macro-srv" }
+
+[features]
+sysroot-abi = ["proc-macro-srv/sysroot-abi"]
+
+[[bin]]
+name = "rust-analyzer-proc-macro-srv"
+path = "src/main.rs"

crates/proc-macro-srv-cli/src/main.rs

@@ -0,0 +1,19 @@
+//! A standalone binary for `proc-macro-srv`.
+
+use proc_macro_srv::cli;
+
+fn main() -> std::io::Result<()> {
+    let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE");
+    match v.as_deref() {
+        Ok("this is unstable") => {
+            // very well, if you must
+        }
+        _ => {
+            eprintln!("If you're rust-analyzer, you can use this tool by exporting RUST_ANALYZER_INTERNALS_DO_NOT_USE='this is unstable'.");
+            eprintln!("If not, you probably shouldn't use this tool. But do what you want: I'm an error message, not a cop.");
+            std::process::exit(122);
+        }
+    }
+
+    cli::run()
+}

crates/project-model/src/project_json.rs

@@ -17,6 +17,9 @@
 /// Roots and crates that compose this Rust project.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ProjectJson {
+    /// e.g. `path/to/sysroot`
+    pub(crate) sysroot: Option<AbsPathBuf>,
+    /// e.g. `path/to/sysroot/lib/rustlib/src/rust`
     pub(crate) sysroot_src: Option<AbsPathBuf>,
     project_root: AbsPathBuf,
     crates: Vec<Crate>,
@@ -52,6 +55,7 @@ impl ProjectJson {
     ///            configuration.
     pub fn new(base: &AbsPath, data: ProjectJsonData) -> ProjectJson {
         ProjectJson {
+            sysroot: data.sysroot.map(|it| base.join(it)),
             sysroot_src: data.sysroot_src.map(|it| base.join(it)),
             project_root: base.to_path_buf(),
             crates: data
@@ -122,6 +126,7 @@ pub fn path(&self) -> &AbsPath {
 
 #[derive(Deserialize, Debug, Clone)]
 pub struct ProjectJsonData {
+    sysroot: Option<PathBuf>,
     sysroot_src: Option<PathBuf>,
     crates: Vec<CrateData>,
 }

crates/project-model/src/sysroot.rs

@@ -15,6 +15,7 @@
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct Sysroot {
     root: AbsPathBuf,
+    src_root: AbsPathBuf,
     crates: Arena<SysrootCrateData>,
 }
 
@@ -35,10 +36,19 @@ fn index(&self, index: SysrootCrate) -> &SysrootCrateData {
 }
 
 impl Sysroot {
+    /// Returns sysroot "root" directory, where `bin/`, `etc/`, `lib/`, `libexec/`
+    /// subfolder live, like:
+    /// `$HOME/.rustup/toolchains/nightly-2022-07-23-x86_64-unknown-linux-gnu`
     pub fn root(&self) -> &AbsPath {
         &self.root
     }
 
+    /// Returns the sysroot "source" directory, where stdlib sources are located, like:
+    /// `$HOME/.rustup/toolchains/nightly-2022-07-23-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library`
+    pub fn src_root(&self) -> &AbsPath {
+        &self.src_root
+    }
+
     pub fn public_deps(&self) -> impl Iterator<Item = (&'static str, SysrootCrate, bool)> + '_ {
         // core is added as a dependency before std in order to
         // mimic rustcs dependency order
@@ -61,7 +71,7 @@ pub fn discover(dir: &AbsPath) -> Result<Sysroot> {
         tracing::debug!("Discovering sysroot for {}", dir.display());
         let sysroot_dir = discover_sysroot_dir(dir)?;
         let sysroot_src_dir = discover_sysroot_src_dir(&sysroot_dir, dir)?;
-        let res = Sysroot::load(sysroot_src_dir)?;
+        let res = Sysroot::load(sysroot_dir, sysroot_src_dir)?;
         Ok(res)
     }
 
@@ -71,14 +81,15 @@ pub fn discover_rustc(cargo_toml: &ManifestPath) -> Option<ManifestPath> {
         discover_sysroot_dir(current_dir).ok().and_then(|sysroot_dir| get_rustc_src(&sysroot_dir))
     }
 
-    pub fn load(sysroot_src_dir: AbsPathBuf) -> Result<Sysroot> {
-        let mut sysroot = Sysroot { root: sysroot_src_dir, crates: Arena::default() };
+    pub fn load(sysroot_dir: AbsPathBuf, sysroot_src_dir: AbsPathBuf) -> Result<Sysroot> {
+        let mut sysroot =
+            Sysroot { root: sysroot_dir, src_root: sysroot_src_dir, crates: Arena::default() };
 
         for path in SYSROOT_CRATES.trim().lines() {
             let name = path.split('/').last().unwrap();
             let root = [format!("{}/src/lib.rs", path), format!("lib{}/lib.rs", path)]
                 .into_iter()
-                .map(|it| sysroot.root.join(it))
+                .map(|it| sysroot.src_root.join(it))
                 .filter_map(|it| ManifestPath::try_from(it).ok())
                 .find(|it| fs::metadata(it).is_ok());
 
@@ -119,7 +130,7 @@ pub fn load(sysroot_src_dir: AbsPathBuf) -> Result<Sysroot> {
             };
             anyhow::bail!(
                 "could not find libcore in sysroot path `{}`{}",
-                sysroot.root.as_path().display(),
+                sysroot.src_root.as_path().display(),
                 var_note,
             );
         }

crates/project-model/src/tests.rs

@@ -75,8 +75,11 @@ fn get_test_path(file: &str) -> PathBuf {
 
 fn get_fake_sysroot() -> Sysroot {
     let sysroot_path = get_test_path("fake-sysroot");
-    let sysroot_src_dir = AbsPathBuf::assert(sysroot_path);
-    Sysroot::load(sysroot_src_dir).unwrap()
+    // there's no `libexec/` directory with a `proc-macro-srv` binary in that
+    // fake sysroot, so we give them both the same path:
+    let sysroot_dir = AbsPathBuf::assert(sysroot_path);
+    let sysroot_src_dir = sysroot_dir.clone();
+    Sysroot::load(sysroot_dir, sysroot_src_dir).unwrap()
 }
 
 fn rooted_project_json(data: ProjectJsonData) -> ProjectJson {

crates/project-model/src/workspace.rs

@@ -230,10 +230,26 @@ pub fn load_inline(
         project_json: ProjectJson,
         target: Option<&str>,
     ) -> Result<ProjectWorkspace> {
-        let sysroot = match &project_json.sysroot_src {
-            Some(path) => Some(Sysroot::load(path.clone())?),
-            None => None,
+        let sysroot = match (project_json.sysroot.clone(), project_json.sysroot_src.clone()) {
+            (Some(sysroot), Some(sysroot_src)) => Some(Sysroot::load(sysroot, sysroot_src)?),
+            (Some(sysroot), None) => {
+                // assume sysroot is structured like rustup's and guess `sysroot_src`
+                let sysroot_src =
+                    sysroot.join("lib").join("rustlib").join("src").join("rust").join("library");
+
+                Some(Sysroot::load(sysroot, sysroot_src)?)
+            }
+            (None, Some(sysroot_src)) => {
+                // assume sysroot is structured like rustup's and guess `sysroot`
+                let mut sysroot = sysroot_src.clone();
+                for _ in 0..5 {
+                    sysroot.pop();
+                }
+                Some(Sysroot::load(sysroot, sysroot_src)?)
+            }
+            (None, None) => None,
         };
+
         let rustc_cfg = rustc_cfg::get(None, target);
         Ok(ProjectWorkspace::Json { project: project_json, sysroot, rustc_cfg })
     }
@@ -345,7 +361,7 @@ pub fn to_roots(&self) -> Vec<PackageRoot> {
                     })
                     .chain(sysroot.iter().map(|sysroot| PackageRoot {
                         is_local: false,
-                        include: vec![sysroot.root().to_path_buf()],
+                        include: vec![sysroot.src_root().to_path_buf()],
                         exclude: Vec::new(),
                     }))
                     .chain(rustc.iter().flat_map(|rustc| {

crates/rust-analyzer/src/reload.rs

@@ -305,8 +305,42 @@ fn eq_ignore_build_data<'a>(
 
         if self.proc_macro_clients.is_empty() {
             if let Some((path, args)) = self.config.proc_macro_srv() {
-                self.proc_macro_clients = (0..self.workspaces.len())
-                    .map(|_| {
+                self.proc_macro_clients = self
+                    .workspaces
+                    .iter()
+                    .map(|ws| {
+                        let mut args = args.clone();
+                        let mut path = path.clone();
+
+                        if let ProjectWorkspace::Cargo { sysroot, .. } = ws {
+                            tracing::info!("Found a cargo workspace...");
+                            if let Some(sysroot) = sysroot.as_ref() {
+                                tracing::info!("Found a cargo workspace with a sysroot...");
+                                let server_path = sysroot
+                                    .root()
+                                    .join("libexec")
+                                    .join("rust-analyzer-proc-macro-srv");
+                                if std::fs::metadata(&server_path).is_ok() {
+                                    tracing::info!(
+                                        "And the server exists at {}",
+                                        server_path.display()
+                                    );
+                                    path = server_path;
+                                    args = vec![];
+                                } else {
+                                    tracing::info!(
+                                        "And the server does not exist at {}",
+                                        server_path.display()
+                                    );
+                                }
+                            }
+                        }
+
+                        tracing::info!(
+                            "Using proc-macro server at {} with args {:?}",
+                            path.display(),
+                            args
+                        );
                         ProcMacroServer::spawn(path.clone(), args.clone()).map_err(|err| {
                             let error = format!(
                                 "Failed to run proc_macro_srv from path {}, error: {:?}",