Merge #9169

9169: internal: steps towards attribute macro token mapping r=jonas-schievink a=jonas-schievink

This doesn't work yet, but we seem to be getting a bit further along (for example, we now stop highlighting `use` items inside item with attribute macros as if they were written verbatim).

bors r+

Co-authored-by: Jonas Schievink <jonasschievink@gmail.com>

crates/hir/src/semantics.rs

@@ -117,6 +117,16 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
     pub fn expand(&self, macro_call: &ast::MacroCall) -> Option<SyntaxNode> {
         self.imp.expand(macro_call)
     }
+
+    /// If `item` has an attribute macro attached to it, expands it.
+    pub fn expand_attr_macro(&self, item: &ast::Item) -> Option<SyntaxNode> {
+        self.imp.expand_attr_macro(item)
+    }
+
+    pub fn is_attr_macro_call(&self, item: &ast::Item) -> bool {
+        self.imp.is_attr_macro_call(item)
+    }
+
     pub fn speculative_expand(
         &self,
         actual_macro_call: &ast::MacroCall,
@@ -332,6 +342,22 @@ impl<'db> SemanticsImpl<'db> {
         Some(node)
     }
 
+    fn expand_attr_macro(&self, item: &ast::Item) -> Option<SyntaxNode> {
+        let sa = self.analyze(item.syntax());
+        let src = InFile::new(sa.file_id, item.clone());
+        let macro_call_id = self.with_ctx(|ctx| ctx.item_to_macro_call(src))?;
+        let file_id = macro_call_id.as_file();
+        let node = self.db.parse_or_expand(file_id)?;
+        self.cache(node.clone(), file_id);
+        Some(node)
+    }
+
+    fn is_attr_macro_call(&self, item: &ast::Item) -> bool {
+        let sa = self.analyze(item.syntax());
+        let src = InFile::new(sa.file_id, item.clone());
+        self.with_ctx(|ctx| ctx.item_to_macro_call(src).is_some())
+    }
+
     fn speculative_expand(
         &self,
         actual_macro_call: &ast::MacroCall,
@@ -362,25 +388,57 @@ impl<'db> SemanticsImpl<'db> {
 
         let token = successors(Some(InFile::new(sa.file_id, token)), |token| {
             self.db.unwind_if_cancelled();
-            let macro_call = token.value.ancestors().find_map(ast::MacroCall::cast)?;
-            let tt = macro_call.token_tree()?;
-            if !tt.syntax().text_range().contains_range(token.value.text_range()) {
-                return None;
-            }
-            let file_id = sa.expand(self.db, token.with_value(&macro_call))?;
-            let token = self
-                .expansion_info_cache
-                .borrow_mut()
-                .entry(file_id)
-                .or_insert_with(|| file_id.expansion_info(self.db.upcast()))
-                .as_ref()?
-                .map_token_down(token.as_ref())?;
 
-            if let Some(parent) = token.value.parent() {
-                self.cache(find_root(&parent), token.file_id);
+            for node in token.value.ancestors() {
+                match_ast! {
+                    match node {
+                        ast::MacroCall(macro_call) => {
+                            let tt = macro_call.token_tree()?;
+                            if !tt.syntax().text_range().contains_range(token.value.text_range()) {
+                                return None;
+                            }
+                            let file_id = sa.expand(self.db, token.with_value(&macro_call))?;
+                            let token = self
+                                .expansion_info_cache
+                                .borrow_mut()
+                                .entry(file_id)
+                                .or_insert_with(|| file_id.expansion_info(self.db.upcast()))
+                                .as_ref()?
+                                .map_token_down(token.as_ref())?;
+
+                            if let Some(parent) = token.value.parent() {
+                                self.cache(find_root(&parent), token.file_id);
+                            }
+
+                            return Some(token);
+                        },
+                        ast::Item(item) => {
+                            match self.with_ctx(|ctx| ctx.item_to_macro_call(token.with_value(item))) {
+                                Some(call_id) => {
+                                    let file_id = call_id.as_file();
+                                    let token = self
+                                        .expansion_info_cache
+                                        .borrow_mut()
+                                        .entry(file_id)
+                                        .or_insert_with(|| file_id.expansion_info(self.db.upcast()))
+                                        .as_ref()?
+                                        .map_token_down(token.as_ref())?;
+
+                                    if let Some(parent) = token.value.parent() {
+                                        self.cache(find_root(&parent), token.file_id);
+                                    }
+
+                                    return Some(token);
+                                }
+                                None => {}
+                            }
+                        },
+                        _ => {}
+                    }
+                }
             }
 
-            Some(token)
+            None
         })
         .last()
         .unwrap();

crates/hir/src/semantics/source_to_def.rs

@@ -10,7 +10,7 @@ use hir_def::{
     ImplId, LifetimeParamId, ModuleId, StaticId, StructId, TraitId, TypeAliasId, TypeParamId,
     UnionId, VariantId,
 };
-use hir_expand::{name::AsName, AstId, MacroDefKind};
+use hir_expand::{name::AsName, AstId, MacroCallId, MacroDefKind};
 use rustc_hash::FxHashMap;
 use smallvec::SmallVec;
 use stdx::impl_from;
@@ -145,16 +145,25 @@ impl SourceToDefCtx<'_, '_> {
         Some((container, label_id))
     }
 
+    pub(super) fn item_to_macro_call(&mut self, src: InFile<ast::Item>) -> Option<MacroCallId> {
+        let map = self.dyn_map(src.as_ref())?;
+        map[keys::ATTR_MACRO].get(&src).copied()
+    }
+
     fn to_def<Ast: AstNode + 'static, ID: Copy + 'static>(
         &mut self,
         src: InFile<Ast>,
         key: Key<Ast, ID>,
     ) -> Option<ID> {
-        let container = self.find_container(src.as_ref().map(|it| it.syntax()))?;
+        self.dyn_map(src.as_ref())?[key].get(&src).copied()
+    }
+
+    fn dyn_map<Ast: AstNode + 'static>(&mut self, src: InFile<&Ast>) -> Option<&DynMap> {
+        let container = self.find_container(src.map(|it| it.syntax()))?;
         let db = self.db;
         let dyn_map =
             &*self.cache.entry(container).or_insert_with(|| container.child_by_source(db));
-        dyn_map[key].get(&src).copied()
+        Some(dyn_map)
     }
 
     pub(super) fn type_param_to_def(&mut self, src: InFile<ast::TypeParam>) -> Option<TypeParamId> {

crates/hir_def/src/child_by_source.rs

@@ -85,6 +85,10 @@ impl ChildBySource for ItemScope {
             res[keys::CONST].insert(src, konst);
         });
         self.impls().for_each(|imp| add_impl(db, res, imp));
+        self.attr_macro_invocs().for_each(|(ast_id, call_id)| {
+            let item = ast_id.with_value(ast_id.to_node(db.upcast()));
+            res[keys::ATTR_MACRO].insert(item, call_id);
+        });
 
         fn add_module_def(db: &dyn DefDatabase, map: &mut DynMap, item: ModuleDefId) {
             match item {

crates/hir_def/src/item_scope.rs

@@ -4,11 +4,11 @@
 use std::collections::hash_map::Entry;
 
 use base_db::CrateId;
-use hir_expand::name::Name;
-use hir_expand::MacroDefKind;
+use hir_expand::{name::Name, AstId, MacroCallId, MacroDefKind};
 use once_cell::sync::Lazy;
 use rustc_hash::{FxHashMap, FxHashSet};
 use stdx::format_to;
+use syntax::ast;
 
 use crate::{
     db::DefDatabase, per_ns::PerNs, visibility::Visibility, AdtId, BuiltinType, ConstId, ImplId,
@@ -53,6 +53,7 @@ pub struct ItemScope {
     // FIXME: Macro shadowing in one module is not properly handled. Non-item place macros will
     // be all resolved to the last one defined if shadowing happens.
     legacy_macros: FxHashMap<Name, MacroDefId>,
+    attr_macros: FxHashMap<AstId<ast::Item>, MacroCallId>,
 }
 
 pub(crate) static BUILTIN_SCOPE: Lazy<FxHashMap<Name, PerNs>> = Lazy::new(|| {
@@ -169,6 +170,16 @@ impl ItemScope {
         self.legacy_macros.insert(name, mac);
     }
 
+    pub(crate) fn add_attr_macro_invoc(&mut self, item: AstId<ast::Item>, call: MacroCallId) {
+        self.attr_macros.insert(item, call);
+    }
+
+    pub(crate) fn attr_macro_invocs(
+        &self,
+    ) -> impl Iterator<Item = (AstId<ast::Item>, MacroCallId)> + '_ {
+        self.attr_macros.iter().map(|(k, v)| (*k, *v))
+    }
+
     pub(crate) fn unnamed_trait_vis(&self, tr: TraitId) -> Option<Visibility> {
         self.unnamed_trait_imports.get(&tr).copied()
     }
@@ -307,6 +318,7 @@ impl ItemScope {
             unnamed_consts,
             unnamed_trait_imports,
             legacy_macros,
+            attr_macros,
         } = self;
         types.shrink_to_fit();
         values.shrink_to_fit();
@@ -317,6 +329,7 @@ impl ItemScope {
         unnamed_consts.shrink_to_fit();
         unnamed_trait_imports.shrink_to_fit();
         legacy_macros.shrink_to_fit();
+        attr_macros.shrink_to_fit();
     }
 }
 

crates/hir_def/src/keys.rs

@@ -2,7 +2,7 @@
 
 use std::marker::PhantomData;
 
-use hir_expand::{InFile, MacroDefId};
+use hir_expand::{InFile, MacroCallId, MacroDefId};
 use rustc_hash::FxHashMap;
 use syntax::{ast, AstNode, AstPtr};
 
@@ -32,6 +32,7 @@ pub const LIFETIME_PARAM: Key<ast::LifetimeParam, LifetimeParamId> = Key::new();
 pub const CONST_PARAM: Key<ast::ConstParam, ConstParamId> = Key::new();
 
 pub const MACRO: Key<ast::MacroCall, MacroDefId> = Key::new();
+pub const ATTR_MACRO: Key<ast::Item, MacroCallId> = Key::new();
 
 /// XXX: AST Nodes and SyntaxNodes have identity equality semantics: nodes are
 /// equal if they point to exactly the same object.

crates/hir_def/src/nameres/collector.rs

@@ -1112,6 +1112,11 @@ impl DefCollector<'_> {
                                     return false;
                                 }
                             }
+
+                            self.def_map.modules[directive.module_id]
+                                .scope
+                                .add_attr_macro_invoc(ast_id.ast_id, call_id);
+
                             resolved.push((directive.module_id, call_id, directive.depth));
                             res = ReachedFixedPoint::No;
                             return false;

crates/ide/src/expand_macro.rs

@@ -3,8 +3,7 @@ use std::iter;
 use hir::Semantics;
 use ide_db::RootDatabase;
 use syntax::{
-    algo::find_node_at_offset, ast, ted, AstNode, NodeOrToken, SyntaxKind, SyntaxKind::*,
-    SyntaxNode, WalkEvent, T,
+    ast, match_ast, ted, AstNode, NodeOrToken, SyntaxKind, SyntaxKind::*, SyntaxNode, WalkEvent, T,
 };
 
 use crate::FilePosition;
@@ -28,16 +27,37 @@ pub struct ExpandedMacro {
 pub(crate) fn expand_macro(db: &RootDatabase, position: FilePosition) -> Option<ExpandedMacro> {
     let sema = Semantics::new(db);
     let file = sema.parse(position.file_id);
-    let mac = find_node_at_offset::<ast::MacroCall>(file.syntax(), position.offset)?;
-    let name = mac.path()?.segment()?.name_ref()?;
 
-    let expanded = expand_macro_recur(&sema, &mac)?;
+    let tok = file.syntax().token_at_offset(position.offset).left_biased()?;
+    let mut expanded = None;
+    let mut name = None;
+    for node in tok.ancestors() {
+        match_ast! {
+            match node {
+                ast::MacroCall(mac) => {
+                    name = Some(mac.path()?.segment()?.name_ref()?.to_string());
+                    expanded = expand_macro_recur(&sema, &mac);
+                    break;
+                },
+                ast::Item(item) => {
+                    // FIXME: add the macro name
+                    // FIXME: make this recursive too
+                    name = Some("?".to_string());
+                    expanded = sema.expand_attr_macro(&item);
+                    if expanded.is_some() {
+                        break;
+                    }
+                },
+                _ => {}
+            }
+        }
+    }
 
     // FIXME:
     // macro expansion may lose all white space information
     // But we hope someday we can use ra_fmt for that
-    let expansion = insert_whitespaces(expanded);
-    Some(ExpandedMacro { name: name.to_string(), expansion })
+    let expansion = insert_whitespaces(expanded?);
+    Some(ExpandedMacro { name: name?, expansion })
 }
 
 fn expand_macro_recur(

crates/ide/src/syntax_highlighting.rs

@@ -192,6 +192,7 @@ fn traverse(
     let mut bindings_shadow_count: FxHashMap<Name, u32> = FxHashMap::default();
 
     let mut current_macro_call: Option<ast::MacroCall> = None;
+    let mut current_attr_macro_call = None;
     let mut current_macro: Option<ast::Macro> = None;
     let mut macro_highlighter = MacroHighlighter::default();
     let mut inside_attribute = false;
@@ -227,6 +228,19 @@ fn traverse(
             }
             _ => (),
         }
+        match event.clone().map(|it| it.into_node().and_then(ast::Item::cast)) {
+            WalkEvent::Enter(Some(item)) => {
+                if sema.is_attr_macro_call(&item) {
+                    current_attr_macro_call = Some(item);
+                }
+            }
+            WalkEvent::Leave(Some(item)) => {
+                if current_attr_macro_call == Some(item) {
+                    current_attr_macro_call = None;
+                }
+            }
+            _ => (),
+        }
 
         match event.clone().map(|it| it.into_node().and_then(ast::Macro::cast)) {
             WalkEvent::Enter(Some(mac)) => {
@@ -286,6 +300,22 @@ fn traverse(
                 }
                 None => token.into(),
             }
+        } else if current_attr_macro_call.is_some() {
+            let token = match element.clone().into_token() {
+                Some(it) => it,
+                _ => continue,
+            };
+            let token = sema.descend_into_macros(token.clone());
+            match token.parent() {
+                Some(parent) => {
+                    // We only care Name and Name_ref
+                    match (token.kind(), parent.kind()) {
+                        (IDENT, NAME) | (IDENT, NAME_REF) => parent.into(),
+                        _ => token.into(),
+                    }
+                }
+                None => token.into(),
+            }
         } else {
             element.clone()
         };