Merge #6033

6033: Make name resolution resolve proc macros instead of relying purely on the build system r=matklad a=jonas-schievink

This makes name resolution look at proc-macro declaration attributes like `#[proc_macro_derive]` and defines the right proc macro in the macro namespace, fixing unresolved custom derives like `thiserror::Error` (which can cause false positives, now that we emit diagnostics for unresolved imports).

This works even when proc-macro support is turned off, in which case we fall back to a dummy expander that always returns an error. IMO this is the right way to handle at least the name resolution part of proc. macros, while the *expansion* itself should rely on the build system to build and provide the macro DLL. It does mean that they may go out of sync, but we can provide diagnostics if that happens (something like "could not find macro X in crate Y – ensure that all files of crate Y are saved").

I think it is valuable to be able to reason about proc macros even when we can't expand them, since proc macro expansion can break between Rust releases or users might not want to turn it on for performance reasons. It allows us to provide better diagnostics on any proc macro invocation we're not expanding (like a weak warning that informs the user that proc macro support is turned off, or that it has been disabled because the server crashed).

Fixes https://github.com/rust-analyzer/rust-analyzer/issues/5763

Co-authored-by: Jonas Schievink <jonas.schievink@ferrous-systems.com>

crates/hir_def/src/item_scope.rs

@@ -5,10 +5,12 @@
 
 use base_db::CrateId;
 use hir_expand::name::Name;
+use hir_expand::MacroDefKind;
 use once_cell::sync::Lazy;
 use rustc_hash::{FxHashMap, FxHashSet};
 use test_utils::mark;
 
+use crate::ModuleId;
 use crate::{
     db::DefDatabase, per_ns::PerNs, visibility::Visibility, AdtId, BuiltinType, HasModule, ImplId,
     LocalModuleId, Lookup, MacroDefId, ModuleDefId, TraitId,
@@ -265,6 +267,26 @@ pub(crate) fn resolutions<'a>(&'a self) -> impl Iterator<Item = (Option<Name>, P
     pub(crate) fn collect_legacy_macros(&self) -> FxHashMap<Name, MacroDefId> {
         self.legacy_macros.clone()
     }
+
+    /// Marks everything that is not a procedural macro as private to `this_module`.
+    pub(crate) fn censor_non_proc_macros(&mut self, this_module: ModuleId) {
+        self.types
+            .values_mut()
+            .chain(self.values.values_mut())
+            .map(|(_, v)| v)
+            .chain(self.unnamed_trait_imports.values_mut())
+            .for_each(|vis| *vis = Visibility::Module(this_module));
+
+        for (mac, vis) in self.macros.values_mut() {
+            if let MacroDefKind::ProcMacro(_) = mac.kind {
+                // FIXME: Technically this is insufficient since reexports of proc macros are also
+                // forbidden. Practically nobody does that.
+                continue;
+            }
+
+            *vis = Visibility::Module(this_module);
+        }
+    }
 }
 
 impl PerNs {

crates/hir_def/src/nameres/collector.rs

@@ -16,10 +16,10 @@
     proc_macro::ProcMacroExpander,
     HirFileId, MacroCallId, MacroDefId, MacroDefKind,
 };
-use rustc_hash::FxHashMap;
-use rustc_hash::FxHashSet;
+use rustc_hash::{FxHashMap, FxHashSet};
 use syntax::ast;
 use test_utils::mark;
+use tt::{Leaf, TokenTree};
 
 use crate::{
     attr::Attrs,
@@ -87,6 +87,7 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: CrateDefMap) -> Cr
         mod_dirs: FxHashMap::default(),
         cfg_options,
         proc_macros,
+        exports_proc_macros: false,
         from_glob_import: Default::default(),
     };
     collector.collect();
@@ -202,7 +203,12 @@ struct DefCollector<'a> {
     unexpanded_attribute_macros: Vec<DeriveDirective>,
     mod_dirs: FxHashMap<LocalModuleId, ModDir>,
     cfg_options: &'a CfgOptions,
+    /// List of procedural macros defined by this crate. This is read from the dynamic library
+    /// built by the build system, and is the list of proc. macros we can actually expand. It is
+    /// empty when proc. macro support is disabled (in which case we still do name resolution for
+    /// them).
     proc_macros: Vec<(Name, ProcMacroExpander)>,
+    exports_proc_macros: bool,
     from_glob_import: PerNsGlobImports,
 }
 
@@ -261,24 +267,56 @@ fn collect(&mut self) {
         }
         self.unresolved_imports = unresolved_imports;
 
-        // Record proc-macros
-        self.collect_proc_macro();
+        // FIXME: This condition should instead check if this is a `proc-macro` type crate.
+        if self.exports_proc_macros {
+            // A crate exporting procedural macros is not allowed to export anything else.
+            //
+            // Additionally, while the proc macro entry points must be `pub`, they are not publicly
+            // exported in type/value namespace. This function reduces the visibility of all items
+            // in the crate root that aren't proc macros.
+            let root = self.def_map.root;
+            let root = &mut self.def_map.modules[root];
+            root.scope.censor_non_proc_macros(ModuleId {
+                krate: self.def_map.krate,
+                local_id: self.def_map.root,
+            });
+        }
     }
 
-    fn collect_proc_macro(&mut self) {
-        let proc_macros = std::mem::take(&mut self.proc_macros);
-        for (name, expander) in proc_macros {
-            let krate = self.def_map.krate;
-
-            let macro_id = MacroDefId {
+    /// Adds a definition of procedural macro `name` to the root module.
+    ///
+    /// # Notes on procedural macro resolution
+    ///
+    /// Procedural macro functionality is provided by the build system: It has to build the proc
+    /// macro and pass the resulting dynamic library to rust-analyzer.
+    ///
+    /// When procedural macro support is enabled, the list of proc macros exported by a crate is
+    /// known before we resolve names in the crate. This list is stored in `self.proc_macros` and is
+    /// derived from the dynamic library.
+    ///
+    /// However, we *also* would like to be able to at least *resolve* macros on our own, without
+    /// help by the build system. So, when the macro isn't found in `self.proc_macros`, we instead
+    /// use a dummy expander that always errors. This comes with the drawback of macros potentially
+    /// going out of sync with what the build system sees (since we resolve using VFS state, but
+    /// Cargo builds only on-disk files). We could and probably should add diagnostics for that.
+    fn resolve_proc_macro(&mut self, name: &Name) {
+        self.exports_proc_macros = true;
+        let macro_def = match self.proc_macros.iter().find(|(n, _)| n == name) {
+            Some((_, expander)) => MacroDefId {
                 ast_id: None,
-                krate: Some(krate),
-                kind: MacroDefKind::ProcMacro(expander),
+                krate: Some(self.def_map.krate),
+                kind: MacroDefKind::ProcMacro(*expander),
                 local_inner: false,
-            };
+            },
+            None => MacroDefId {
+                ast_id: None,
+                krate: Some(self.def_map.krate),
+                kind: MacroDefKind::ProcMacro(ProcMacroExpander::dummy(self.def_map.krate)),
+                local_inner: false,
+            },
+        };
 
-            self.define_proc_macro(name.clone(), macro_id);
-        }
+        self.define_proc_macro(name.clone(), macro_def);
     }
 
     /// Define a macro with `macro_rules`.
@@ -917,6 +955,9 @@ fn collect(&mut self, items: &[ModItem]) {
                 }
                 ModItem::Function(id) => {
                     let func = &self.item_tree[id];
+
+                    self.collect_proc_macro_def(&func.name, attrs);
+
                     def = Some(DefData {
                         id: FunctionLoc {
                             container: container.into(),
@@ -1177,6 +1218,30 @@ fn collect_derives(&mut self, attrs: &Attrs, ast_id: FileAstId<ast::Item>) {
         }
     }
 
+    /// If `attrs` registers a procedural macro, collects its definition.
+    fn collect_proc_macro_def(&mut self, func_name: &Name, attrs: &Attrs) {
+        // FIXME: this should only be done in the root module of `proc-macro` crates, not everywhere
+        // FIXME: distinguish the type of macro
+        let macro_name = if attrs.by_key("proc_macro").exists()
+            || attrs.by_key("proc_macro_attribute").exists()
+        {
+            func_name.clone()
+        } else {
+            let derive = attrs.by_key("proc_macro_derive");
+            if let Some(arg) = derive.tt_values().next() {
+                if let [TokenTree::Leaf(Leaf::Ident(trait_name))] = &*arg.token_trees {
+                    trait_name.as_name()
+                } else {
+                    return;
+                }
+            } else {
+                return;
+            }
+        };
+
+        self.def_collector.resolve_proc_macro(&macro_name);
+    }
+
     fn collect_macro(&mut self, mac: &MacroCall) {
         let mut ast_id = AstIdWithPath::new(self.file_id, mac.ast_id, mac.path.clone());
 
@@ -1283,6 +1348,7 @@ fn do_collect_defs(db: &dyn DefDatabase, def_map: CrateDefMap) -> CrateDefMap {
             mod_dirs: FxHashMap::default(),
             cfg_options: &CfgOptions::default(),
             proc_macros: Default::default(),
+            exports_proc_macros: false,
             from_glob_import: Default::default(),
         };
         collector.collect();

crates/hir_def/src/nameres/tests/macros.rs

@@ -667,3 +667,76 @@ macro_rules! b {
         "#]],
     );
 }
+
+#[test]
+fn resolves_proc_macros() {
+    check(
+        r"
+        struct TokenStream;
+
+        #[proc_macro]
+        pub fn function_like_macro(args: TokenStream) -> TokenStream {
+            args
+        }
+
+        #[proc_macro_attribute]
+        pub fn attribute_macro(_args: TokenStream, item: TokenStream) -> TokenStream {
+            item
+        }
+
+        #[proc_macro_derive(DummyTrait)]
+        pub fn derive_macro(_item: TokenStream) -> TokenStream {
+            TokenStream
+        }
+        ",
+        expect![[r#"
+            crate
+            DummyTrait: m
+            TokenStream: t v
+            attribute_macro: v m
+            derive_macro: v
+            function_like_macro: v m
+        "#]],
+    );
+}
+
+#[test]
+fn proc_macro_censoring() {
+    // Make sure that only proc macros are publicly exported from proc-macro crates.
+
+    check(
+        r"
+        //- /main.rs crate:main deps:macros
+        pub use macros::*;
+
+        //- /macros.rs crate:macros
+        pub struct TokenStream;
+
+        #[proc_macro]
+        pub fn function_like_macro(args: TokenStream) -> TokenStream {
+            args
+        }
+
+        #[proc_macro_attribute]
+        pub fn attribute_macro(_args: TokenStream, item: TokenStream) -> TokenStream {
+            item
+        }
+
+        #[proc_macro_derive(DummyTrait)]
+        pub fn derive_macro(_item: TokenStream) -> TokenStream {
+            TokenStream
+        }
+
+        #[macro_export]
+        macro_rules! mbe {
+            () => {};
+        }
+        ",
+        expect![[r#"
+            crate
+            DummyTrait: m
+            attribute_macro: m
+            function_like_macro: m
+        "#]],
+    );
+}

crates/hir_expand/src/proc_macro.rs

@@ -7,7 +7,7 @@
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub struct ProcMacroExpander {
     krate: CrateId,
-    proc_macro_id: ProcMacroId,
+    proc_macro_id: Option<ProcMacroId>,
 }
 
 macro_rules! err {
@@ -20,8 +20,14 @@ macro_rules! err {
 }
 
 impl ProcMacroExpander {
-    pub fn new(krate: CrateId, proc_macro_id: ProcMacroId) -> ProcMacroExpander {
-        ProcMacroExpander { krate, proc_macro_id }
+    pub fn new(krate: CrateId, proc_macro_id: ProcMacroId) -> Self {
+        Self { krate, proc_macro_id: Some(proc_macro_id) }
+    }
+
+    pub fn dummy(krate: CrateId) -> Self {
+        // FIXME: Should store the name for better errors
+        // FIXME: I think this is the second layer of "dummy" expansion, we should reduce that
+        Self { krate, proc_macro_id: None }
     }
 
     pub fn expand(
@@ -30,17 +36,22 @@ pub fn expand(
         _id: LazyMacroId,
         tt: &tt::Subtree,
     ) -> Result<tt::Subtree, mbe::ExpandError> {
-        let krate_graph = db.crate_graph();
-        let proc_macro = krate_graph[self.krate]
-            .proc_macro
-            .get(self.proc_macro_id.0 as usize)
-            .clone()
-            .ok_or_else(|| err!("No derive macro found."))?;
+        match self.proc_macro_id {
+            Some(id) => {
+                let krate_graph = db.crate_graph();
+                let proc_macro = krate_graph[self.krate]
+                    .proc_macro
+                    .get(id.0 as usize)
+                    .clone()
+                    .ok_or_else(|| err!("No derive macro found."))?;
 
-        let tt = remove_derive_attrs(tt)
-            .ok_or_else(|| err!("Fail to remove derive for custom derive"))?;
+                let tt = remove_derive_attrs(tt)
+                    .ok_or_else(|| err!("Fail to remove derive for custom derive"))?;
 
-        proc_macro.expander.expand(&tt, None).map_err(mbe::ExpandError::from)
+                proc_macro.expander.expand(&tt, None).map_err(mbe::ExpandError::from)
+            }
+            None => Err(err!("Unresolved proc macro")),
+        }
     }
 }