This commit add Cargo-style project discovery for Buck and Bazel users.

This feature requires the user to add a command that generates a
`rust-project.json` from a set of files. Project discovery can be invoked
in two ways:

1. At extension activation time, which includes the generated
   `rust-project.json` as part of the linkedProjects argument in
    InitializeParams
2. Through a new command titled "Add current file to workspace", which
   makes use of a new, rust-analyzer specific LSP request that adds
   the workspace without erasing any existing workspaces.

I think that the command-running functionality _could_ merit being
placed into its own extension (and expose it via extension contribution
points), if only provide build-system idiomatic progress reporting and
status handling, but I haven't (yet) made an extension that does this.

crates/project-model/src/cfg_flag.rs

@@ -4,6 +4,7 @@
 use std::{fmt, str::FromStr};
 
 use cfg::CfgOptions;
+use serde::Serialize;
 
 #[derive(Clone, Eq, PartialEq, Debug)]
 pub enum CfgFlag {
@@ -38,6 +39,18 @@ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     }
 }
 
+impl Serialize for CfgFlag {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        match self {
+            CfgFlag::Atom(s) => serializer.serialize_str(s),
+            CfgFlag::KeyValue { .. } => serializer.serialize_str(&format!("{}", &self)),
+        }
+    }
+}
+
 impl Extend<CfgFlag> for CfgOptions {
     fn extend<T: IntoIterator<Item = CfgFlag>>(&mut self, iter: T) {
         for cfg_flag in iter {

crates/project-model/src/project_json.rs

@@ -54,7 +54,7 @@
 use base_db::{CrateDisplayName, CrateId, CrateName, Dependency, Edition};
 use paths::{AbsPath, AbsPathBuf};
 use rustc_hash::FxHashMap;
-use serde::{de, Deserialize};
+use serde::{de, ser, Deserialize, Serialize};
 
 use crate::cfg_flag::CfgFlag;
 
@@ -171,14 +171,14 @@ pub fn path(&self) -> &AbsPath {
     }
 }
 
-#[derive(Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ProjectJsonData {
     sysroot: Option<PathBuf>,
     sysroot_src: Option<PathBuf>,
     crates: Vec<CrateData>,
 }
 
-#[derive(Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 struct CrateData {
     display_name: Option<String>,
     root_module: PathBuf,
@@ -200,7 +200,7 @@ struct CrateData {
     repository: Option<String>,
 }
 
-#[derive(Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename = "edition")]
 enum EditionData {
     #[serde(rename = "2015")]
@@ -221,16 +221,16 @@ fn from(data: EditionData) -> Self {
     }
 }
 
-#[derive(Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 struct DepData {
     /// Identifies a crate by position in the crates array.
     #[serde(rename = "crate")]
     krate: usize,
-    #[serde(deserialize_with = "deserialize_crate_name")]
+    #[serde(deserialize_with = "deserialize_crate_name", serialize_with = "serialize_crate_name")]
     name: CrateName,
 }
 
-#[derive(Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 struct CrateSource {
     include_dirs: Vec<PathBuf>,
     exclude_dirs: Vec<PathBuf>,
@@ -243,3 +243,10 @@ fn deserialize_crate_name<'de, D>(de: D) -> Result<CrateName, D::Error>
     let name = String::deserialize(de)?;
     CrateName::new(&name).map_err(|err| de::Error::custom(format!("invalid crate name: {err:?}")))
 }
+
+fn serialize_crate_name<S>(crate_name: &CrateName, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: ser::Serializer,
+{
+    crate_name.serialize(serializer)
+}

crates/rust-analyzer/src/config.rs

@@ -272,7 +272,6 @@ struct ConfigData {
         /// The warnings will be indicated by a blue squiggly underline in code
         /// and a blue icon in the `Problems Panel`.
         diagnostics_warningsAsInfo: Vec<String> = "[]",
-
         /// These directories will be ignored by rust-analyzer. They are
         /// relative to the workspace root, and globs are not supported. You may
         /// also need to add the folders to Code's `files.watcherExclude`.
@@ -895,6 +894,15 @@ pub fn linked_projects(&self) -> Vec<LinkedProject> {
         }
     }
 
+    pub fn add_linked_projects(&mut self, linked_projects: Vec<ProjectJsonData>) {
+        let mut linked_projects = linked_projects
+            .into_iter()
+            .map(ManifestOrProjectJson::ProjectJson)
+            .collect::<Vec<ManifestOrProjectJson>>();
+
+        self.data.linkedProjects.append(&mut linked_projects);
+    }
+
     pub fn did_save_text_document_dynamic_registration(&self) -> bool {
         let caps = try_or_def!(self.caps.text_document.as_ref()?.synchronization.clone()?);
         caps.did_save == Some(true) && caps.dynamic_registration == Some(true)

crates/rust-analyzer/src/handlers.rs

@@ -5,6 +5,7 @@
 use std::{
     io::Write as _,
     process::{self, Stdio},
+    sync::Arc,
 };
 
 use anyhow::Context;
@@ -46,6 +47,22 @@
 pub(crate) fn handle_workspace_reload(state: &mut GlobalState, _: ()) -> Result<()> {
     state.proc_macro_clients.clear();
     state.proc_macro_changed = false;
+
+    state.fetch_workspaces_queue.request_op("reload workspace request".to_string());
+    state.fetch_build_data_queue.request_op("reload workspace request".to_string());
+    Ok(())
+}
+
+pub(crate) fn handle_add_project(
+    state: &mut GlobalState,
+    params: lsp_ext::AddProjectParams,
+) -> Result<()> {
+    state.proc_macro_clients.clear();
+    state.proc_macro_changed = false;
+
+    let config = Arc::make_mut(&mut state.config);
+    config.add_linked_projects(params.project);
+
     state.fetch_workspaces_queue.request_op("reload workspace request".to_string());
     state.fetch_build_data_queue.request_op("reload workspace request".to_string());
     Ok(())

crates/rust-analyzer/src/lsp_ext.rs

@@ -9,6 +9,7 @@
     notification::Notification, CodeActionKind, DocumentOnTypeFormattingParams,
     PartialResultParams, Position, Range, TextDocumentIdentifier, WorkDoneProgressParams,
 };
+use project_model::ProjectJsonData;
 use serde::{Deserialize, Serialize};
 
 use crate::line_index::PositionEncoding;
@@ -51,6 +52,20 @@ impl Request for ReloadWorkspace {
     const METHOD: &'static str = "rust-analyzer/reloadWorkspace";
 }
 
+pub enum AddProject {}
+
+impl Request for AddProject {
+    type Params = AddProjectParams;
+    type Result = ();
+    const METHOD: &'static str = "rust-analyzer/addProject";
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct AddProjectParams {
+    pub project: Vec<ProjectJsonData>,
+}
+
 pub enum SyntaxTree {}
 
 impl Request for SyntaxTree {

crates/rust-analyzer/src/main_loop.rs

@@ -625,6 +625,7 @@ fn on_request(&mut self, req: Request) {
             .on_sync_mut::<lsp_ext::ReloadWorkspace>(handlers::handle_workspace_reload)
             .on_sync_mut::<lsp_ext::MemoryUsage>(handlers::handle_memory_usage)
             .on_sync_mut::<lsp_ext::ShuffleCrateGraph>(handlers::handle_shuffle_crate_graph)
+            .on_sync_mut::<lsp_ext::AddProject>(handlers::handle_add_project)
             .on_sync::<lsp_ext::JoinLines>(handlers::handle_join_lines)
             .on_sync::<lsp_ext::OnEnter>(handlers::handle_on_enter)
             .on_sync::<lsp_types::request::SelectionRangeRequest>(handlers::handle_selection_range)

editors/code/package.json

@@ -199,6 +199,11 @@
                 "title": "Reload workspace",
                 "category": "rust-analyzer"
             },
+            {
+                "command": "rust-analyzer.addProject",
+                "title": "Add current file to workspace",
+                "category": "rust-analyzer"
+            },
             {
                 "command": "rust-analyzer.reload",
                 "title": "Restart server",
@@ -447,6 +452,17 @@
                         "Fill missing expressions with reasonable defaults, `new` or `default` constructors."
                     ]
                 },
+                "rust-analyzer.discoverProjectCommand": {
+                    "markdownDescription": "Sets the command that rust-analyzer uses to generate `rust-project.json` files. This command is\n only suggested if a build system like Buck or Bazel is used. The command must accept files as arguements and return \n a rust-project.json over stdout.",
+                    "default": null,
+                    "type": [
+                        "null",
+                        "array"
+                    ],
+                    "items": {
+                        "type": "string"
+                    }
+                },
                 "rust-analyzer.cachePriming.enable": {
                     "markdownDescription": "Warm up caches on project load.",
                     "default": true,

editors/code/src/commands.ts

@@ -3,7 +3,7 @@ import * as lc from "vscode-languageclient";
 import * as ra from "./lsp_ext";
 import * as path from "path";
 
-import { Ctx, Cmd, CtxInit } from "./ctx";
+import { Ctx, Cmd, CtxInit, discoverWorkspace } from "./ctx";
 import { applySnippetWorkspaceEdit, applySnippetTextEdits } from "./snippets";
 import { spawnSync } from "child_process";
 import { RunnableQuickPick, selectRunnable, createTask, createArgs } from "./run";
@@ -749,6 +749,23 @@ export function reloadWorkspace(ctx: CtxInit): Cmd {
     return async () => ctx.client.sendRequest(ra.reloadWorkspace);
 }
 
+export function addProject(ctx: CtxInit): Cmd {
+    return async () => {
+        const discoverProjectCommand = ctx.config.discoverProjectCommand;
+        if (!discoverProjectCommand) {
+            return;
+        }
+
+        let workspaces: JsonProject[] = await Promise.all(vscode.workspace.workspaceFolders!.map(async (folder): Promise<JsonProject> => {
+            return discoverWorkspace(vscode.workspace.textDocuments, discoverProjectCommand, { cwd: folder.uri.fsPath });
+        }));
+
+        await ctx.client.sendRequest(ra.addProject, {
+            project: workspaces
+        });
+    }
+}
+
 async function showReferencesImpl(
     client: LanguageClient | undefined,
     uri: string,

editors/code/src/config.ts

@@ -214,6 +214,10 @@ export class Config {
         return this.get<boolean>("trace.extension");
     }
 
+    get discoverProjectCommand() {
+        return this.get<string[] | undefined>("discoverProjectCommand")
+    }
+
     get cargoRunner() {
         return this.get<string | undefined>("cargoRunner");
     }

editors/code/src/ctx.ts

@@ -4,10 +4,11 @@ import * as ra from "./lsp_ext";
 
 import { Config, substituteVSCodeVariables } from "./config";
 import { createClient } from "./client";
-import { isRustDocument, isRustEditor, LazyOutputChannel, log, RustEditor } from "./util";
+import { executeDiscoverProject, isRustDocument, isRustEditor, LazyOutputChannel, log, RustEditor } from "./util";
 import { ServerStatusParams } from "./lsp_ext";
 import { PersistentState } from "./persistent_state";
 import { bootstrap } from "./bootstrap";
+import { ExecOptions } from "child_process";
 
 // We only support local folders, not eg. Live Share (`vlsl:` scheme), so don't activate if
 // only those are in use. We use "Empty" to represent these scenarios
@@ -41,6 +42,13 @@ export function fetchWorkspace(): Workspace {
         : { kind: "Workspace Folder" };
 }
 
+export async function discoverWorkspace(files: readonly vscode.TextDocument[], command: string[], options: ExecOptions): Promise<JsonProject> {
+    const paths = files.map((f) => f.uri.fsPath).join(" ");
+    const joinedCommand = command.join(" ");
+    const data = await executeDiscoverProject(`${joinedCommand} -- ${paths}`, options);
+    return JSON.parse(data) as JsonProject;
+}
+
 export type CommandFactory = {
     enabled: (ctx: CtxInit) => Cmd;
     disabled?: (ctx: Ctx) => Cmd;
@@ -63,6 +71,7 @@ export class Ctx {
     private state: PersistentState;
     private commandFactories: Record<string, CommandFactory>;
     private commandDisposables: Disposable[];
+    private discoveredWorkspaces: JsonProject[] | undefined;
 
     get client() {
         return this._client;
@@ -71,7 +80,7 @@ export class Ctx {
     constructor(
         readonly extCtx: vscode.ExtensionContext,
         commandFactories: Record<string, CommandFactory>,
-        workspace: Workspace
+        workspace: Workspace,
     ) {
         extCtx.subscriptions.push(this);
         this.statusBar = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left);
@@ -169,7 +178,18 @@ export class Ctx {
                 };
             }
 
-            const initializationOptions = substituteVSCodeVariables(rawInitializationOptions);
+            const discoverProjectCommand = this.config.discoverProjectCommand;
+            if (discoverProjectCommand) {
+                let workspaces: JsonProject[] = await Promise.all(vscode.workspace.workspaceFolders!.map(async (folder): Promise<JsonProject> => {
+                    return discoverWorkspace(vscode.workspace.textDocuments, discoverProjectCommand, { cwd: folder.uri.fsPath });
+                }));
+
+                this.discoveredWorkspaces = workspaces;
+            }
+
+            let initializationOptions = substituteVSCodeVariables(rawInitializationOptions);
+            // this appears to be load-bearing, for better or worse.
+            await initializationOptions.update('linkedProjects', this.discoveredWorkspaces)
 
             this._client = await createClient(
                 this.traceOutputChannel,

editors/code/src/lsp_ext.ts

@@ -43,6 +43,10 @@ export const relatedTests = new lc.RequestType<lc.TextDocumentPositionParams, Te
     "rust-analyzer/relatedTests"
 );
 export const reloadWorkspace = new lc.RequestType0<null, void>("rust-analyzer/reloadWorkspace");
+export const addProject = new lc.RequestType<AddProjectParams, string, void>(
+    "rust-analyzer/addProject"
+)
+
 export const runFlycheck = new lc.NotificationType<{
     textDocument: lc.TextDocumentIdentifier | null;
 }>("rust-analyzer/runFlycheck");
@@ -68,6 +72,8 @@ export const viewItemTree = new lc.RequestType<ViewItemTreeParams, string, void>
 
 export type AnalyzerStatusParams = { textDocument?: lc.TextDocumentIdentifier };
 
+export type AddProjectParams = { project: JsonProject[] };
+
 export type ExpandMacroParams = {
     textDocument: lc.TextDocumentIdentifier;
     position: lc.Position;

editors/code/src/main.ts

@@ -153,6 +153,7 @@ function createCommands(): Record<string, CommandFactory> {
         memoryUsage: { enabled: commands.memoryUsage },
         shuffleCrateGraph: { enabled: commands.shuffleCrateGraph },
         reloadWorkspace: { enabled: commands.reloadWorkspace },
+        addProject: { enabled: commands.addProject },
         matchingBrace: { enabled: commands.matchingBrace },
         joinLines: { enabled: commands.joinLines },
         parentModule: { enabled: commands.parentModule },

editors/code/src/rust_project.ts

@@ -0,0 +1,91 @@
+interface JsonProject {
+    /// Path to the directory with *source code* of
+    /// sysroot crates.
+    ///
+    /// It should point to the directory where std,
+    /// core, and friends can be found:
+    ///
+    /// https://github.com/rust-lang/rust/tree/master/library.
+    ///
+    /// If provided, rust-analyzer automatically adds
+    /// dependencies on sysroot crates. Conversely,
+    /// if you omit this path, you can specify sysroot
+    /// dependencies yourself and, for example, have
+    /// several different "sysroots" in one graph of
+    /// crates.
+    sysroot_src?: string;
+    /// The set of crates comprising the current
+    /// project. Must include all transitive
+    /// dependencies as well as sysroot crate (libstd,
+    /// libcore and such).
+    crates: Crate[];
+}
+
+interface Crate {
+    /// Optional crate name used for display purposes,
+    /// without affecting semantics. See the `deps`
+    /// key for semantically-significant crate names.
+    display_name?: string;
+    /// Path to the root module of the crate.
+    root_module: string;
+    /// Edition of the crate.
+    edition: "2015" | "2018" | "2021";
+    /// Dependencies
+    deps: Dep[];
+    /// Should this crate be treated as a member of
+    /// current "workspace".
+    ///
+    /// By default, inferred from the `root_module`
+    /// (members are the crates which reside inside
+    /// the directory opened in the editor).
+    ///
+    /// Set this to `false` for things like standard
+    /// library and 3rd party crates to enable
+    /// performance optimizations (rust-analyzer
+    /// assumes that non-member crates don't change).
+    is_workspace_member?: boolean;
+    /// Optionally specify the (super)set of `.rs`
+    /// files comprising this crate.
+    ///
+    /// By default, rust-analyzer assumes that only
+    /// files under `root_module.parent` can belong
+    /// to a crate. `include_dirs` are included
+    /// recursively, unless a subdirectory is in
+    /// `exclude_dirs`.
+    ///
+    /// Different crates can share the same `source`.
+    ///
+    /// If two crates share an `.rs` file in common,
+    /// they *must* have the same `source`.
+    /// rust-analyzer assumes that files from one
+    /// source can't refer to files in another source.
+    source?: {
+        include_dirs: string[],
+        exclude_dirs: string[],
+    },
+    /// The set of cfgs activated for a given crate, like
+    /// `["unix", "feature=\"foo\"", "feature=\"bar\""]`.
+    cfg: string[];
+    /// Target triple for this Crate.
+    ///
+    /// Used when running `rustc --print cfg`
+    /// to get target-specific cfgs.
+    target?: string;
+    /// Environment variables, used for
+    /// the `env!` macro
+    env: { [key: string]: string; },
+
+    /// Whether the crate is a proc-macro crate.
+    is_proc_macro: boolean;
+    /// For proc-macro crates, path to compiled
+    /// proc-macro (.so file).
+    proc_macro_dylib_path?: string;
+}
+
+interface Dep {
+    /// Index of a crate in the `crates` array.
+    crate: number,
+    /// Name as should appear in the (implicit)
+    /// `extern crate name` declaration.
+    name: string,
+}

editors/code/src/util.ts

@@ -150,9 +150,11 @@ export function memoizeAsync<Ret, TThis, Param extends string>(
 
 /** Awaitable wrapper around `child_process.exec` */
 export function execute(command: string, options: ExecOptions): Promise<string> {
+    log.info(`running command: ${command}`)
     return new Promise((resolve, reject) => {
         exec(command, options, (err, stdout, stderr) => {
             if (err) {
+                log.error(err);
                 reject(err);
                 return;
             }
@@ -167,6 +169,21 @@ export function execute(command: string, options: ExecOptions): Promise<string>
     });
 }
 
+export function executeDiscoverProject(command: string, options: ExecOptions): Promise<string> {
+    log.info(`running command: ${command}`)
+    return new Promise((resolve, reject) => {
+        exec(command, options, (err, stdout, _) => {
+            if (err) {
+                log.error(err);
+                reject(err);
+                return;
+            }
+
+            resolve(stdout.trimEnd());
+        });
+    });
+}
+
 export class LazyOutputChannel implements vscode.OutputChannel {
     constructor(name: string) {
         this.name = name;