From 3a525c831ffb1326dfaab4dcff078c21697af801 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 2 Jan 2022 19:10:10 +0100 Subject: [PATCH] internal: Handle macro calls better in highlighting --- crates/ide/src/syntax_highlighting.rs | 167 ++++++++---------- .../ide/src/syntax_highlighting/highlight.rs | 2 +- crates/ide/src/syntax_highlighting/macro_.rs | 4 +- crates/ide/src/syntax_highlighting/tags.rs | 3 + .../test_data/highlight_doctest.html | 6 +- .../test_data/highlight_injection.html | 2 +- .../test_data/highlight_strings.html | 98 +++++----- .../test_data/highlighting.html | 18 +- crates/rust-analyzer/src/semantic_tokens.rs | 1 + crates/rust-analyzer/src/to_proto.rs | 1 + editors/code/package.json | 8 + 11 files changed, 151 insertions(+), 159 deletions(-) diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 0741e1ffbfa..a908cdb718b 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -13,11 +13,11 @@ mod html; mod tests; use hir::{InFile, Name, Semantics}; -use ide_db::{RootDatabase, SymbolKind}; +use ide_db::RootDatabase; use rustc_hash::FxHashMap; use syntax::{ ast::{self, HasFormatSpecifier}, - match_ast, AstNode, AstToken, Direction, NodeOrToken, + AstNode, AstToken, NodeOrToken, SyntaxKind::*, SyntaxNode, TextRange, WalkEvent, T, }; @@ -100,7 +100,8 @@ pub struct HlRange { // colon:: Emitted for the `:` token. // comma:: Emitted for the `,` token. // dot:: Emitted for the `.` token. -// Semi:: Emitted for the `;` token. +// semi:: Emitted for the `;` token. +// macroBang:: Emitted for the `!` token in macro calls. // // //- // @@ -209,107 +210,94 @@ fn traverse( // Walk all nodes, keeping track of whether we are inside a macro or not. // If in macro, expand it first and highlight the expanded code. for event in root.value.preorder_with_tokens() { - let event_range = match &event { + let range = match &event { WalkEvent::Enter(it) | WalkEvent::Leave(it) => it.text_range(), }; // Element outside of the viewport, no need to highlight - if range_to_highlight.intersect(event_range).is_none() { + if range_to_highlight.intersect(range).is_none() { continue; } + // set macro and attribute highlighting states match event.clone() { - WalkEvent::Enter(NodeOrToken::Node(node)) => { - match_ast! { - match node { - ast::MacroCall(mcall) => { - if let Some(range) = macro_call_range(&mcall) { - hl.add(HlRange { - range, - highlight: HlTag::Symbol(SymbolKind::Macro).into(), - binding_hash: None, - }); - } - current_macro_call = Some(mcall); - continue; - }, - ast::Macro(mac) => { - macro_highlighter.init(); - current_macro = Some(mac); - continue; - }, - ast::Item(item) => { - if sema.is_attr_macro_call(&item) { - current_attr_call = Some(item); - } - }, - ast::Attr(__) => inside_attribute = true, - _ => () - } + WalkEvent::Enter(NodeOrToken::Node(node)) => match ast::Item::cast(node.clone()) { + Some(ast::Item::MacroCall(mcall)) => { + current_macro_call = Some(mcall); + continue; } - } - WalkEvent::Leave(NodeOrToken::Node(node)) => { - match_ast! { - match node { - ast::MacroCall(mcall) => { - assert_eq!(current_macro_call, Some(mcall)); - current_macro_call = None; - }, - ast::Macro(mac) => { - assert_eq!(current_macro, Some(mac)); - current_macro = None; - macro_highlighter = MacroHighlighter::default(); - }, - ast::Item(item) => { - if current_attr_call == Some(item) { - current_attr_call = None; - } - }, - ast::Attr(__) => inside_attribute = false, - _ => () - } + Some(ast::Item::MacroRules(mac)) => { + macro_highlighter.init(); + current_macro = Some(mac.into()); + continue; } - } + Some(ast::Item::MacroDef(mac)) => { + macro_highlighter.init(); + current_macro = Some(mac.into()); + continue; + } + Some(item) if sema.is_attr_macro_call(&item) => current_attr_call = Some(item), + None if ast::Attr::can_cast(node.kind()) => inside_attribute = true, + _ => (), + }, + WalkEvent::Leave(NodeOrToken::Node(node)) => match ast::Item::cast(node.clone()) { + Some(ast::Item::MacroCall(mcall)) => { + assert_eq!(current_macro_call, Some(mcall)); + current_macro_call = None; + } + Some(ast::Item::MacroRules(mac)) => { + assert_eq!(current_macro, Some(mac.into())); + current_macro = None; + macro_highlighter = MacroHighlighter::default(); + } + Some(ast::Item::MacroDef(mac)) => { + assert_eq!(current_macro, Some(mac.into())); + current_macro = None; + macro_highlighter = MacroHighlighter::default(); + } + Some(item) if current_attr_call.as_ref().map_or(false, |it| *it == item) => { + current_attr_call = None + } + None if ast::Attr::can_cast(node.kind()) => inside_attribute = false, + _ => (), + }, _ => (), } let element = match event { WalkEvent::Enter(it) => it, - WalkEvent::Leave(it) => { - if let Some(node) = it.as_node() { - inject::doc_comment(hl, sema, root.with_value(node)); - } + WalkEvent::Leave(NodeOrToken::Token(_)) => continue, + WalkEvent::Leave(NodeOrToken::Node(node)) => { + inject::doc_comment(hl, sema, root.with_value(&node)); continue; } }; - let range = element.text_range(); - if current_macro.is_some() { if let Some(tok) = element.as_token() { macro_highlighter.advance(tok); } } - let descend_token = (current_macro_call.is_some() || current_attr_call.is_some()) - && element.kind() != COMMENT; + // only attempt to descend if we are inside a macro call or attribute + // as calling `descend_into_macros_single` gets rather expensive if done for every single token + let descend_token = current_macro_call.is_some() || current_attr_call.is_some(); let element_to_highlight = if descend_token { - // Inside a macro -- expand it first - let token = match element.clone().into_token() { - Some(it) if current_macro_call.is_some() => { - let not_in_tt = it.parent().map_or(true, |it| it.kind() != TOKEN_TREE); - if not_in_tt { - continue; - } - it - } - Some(it) => it, - _ => continue, + let token = match &element { + NodeOrToken::Node(_) => continue, + NodeOrToken::Token(tok) => tok.clone(), + }; + let in_mcall_outside_tt = current_macro_call.is_some() + && token.parent().as_ref().map(SyntaxNode::kind) != Some(TOKEN_TREE); + let token = match in_mcall_outside_tt { + // not in the macros token tree, don't attempt to descend + true => token, + false => sema.descend_into_macros_single(token), }; - let token = sema.descend_into_macros_single(token); match token.parent() { Some(parent) => { - // We only care Name and Name_ref + // Names and NameRefs have special semantics, use them instead of the tokens + // as otherwise we won't ever visit them match (token.kind(), parent.kind()) { (T![ident], NAME | NAME_REF) => parent.into(), (T![self] | T![super] | T![crate], NAME_REF) => parent.into(), @@ -323,10 +311,14 @@ fn traverse( element.clone() }; - if macro_highlighter.highlight(element_to_highlight.clone()).is_some() { + // FIXME: do proper macro def highlighting https://github.com/rust-analyzer/rust-analyzer/issues/6232 + // Skip metavariables from being highlighted to prevent keyword highlighting in them + if macro_highlighter.highlight(&element_to_highlight).is_some() { continue; } + // string highlight injections, note this does not use the descended element as proc-macros + // can rewrite string literals which invalidates our indices if let (Some(token), Some(token_to_highlight)) = (element.into_token(), element_to_highlight.as_token()) { @@ -354,13 +346,15 @@ fn traverse( } } - if let Some((mut highlight, binding_hash)) = highlight::element( + // do the normal highlighting + let element = highlight::element( sema, krate, &mut bindings_shadow_count, syntactic_name_ref_highlighting, - element_to_highlight.clone(), - ) { + element_to_highlight, + ); + if let Some((mut highlight, binding_hash)) = element { if inside_attribute { highlight |= HlMod::Attribute } @@ -369,18 +363,3 @@ fn traverse( } } } - -fn macro_call_range(macro_call: &ast::MacroCall) -> Option { - let path = macro_call.path()?; - let name_ref = path.segment()?.name_ref()?; - - let range_start = name_ref.syntax().text_range().start(); - let mut range_end = name_ref.syntax().text_range().end(); - for sibling in path.syntax().siblings_with_tokens(Direction::Next) { - if let T![!] | T![ident] = sibling.kind() { - range_end = sibling.text_range().end(); - } - } - - Some(TextRange::new(range_start, range_end)) -} diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index 05d2732251f..54b4bcf7c36 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -81,7 +81,7 @@ fn token( T![::] | T![->] | T![=>] | T![..] | T![=] | T![@] | T![.] => { HlOperator::Other.into() } - T![!] if parent_matches::(&token) => SymbolKind::Macro.into(), + T![!] if parent_matches::(&token) => HlPunct::MacroBang.into(), T![!] if parent_matches::(&token) => HlTag::BuiltinType.into(), T![!] if parent_matches::(&token) => HlOperator::Logical.into(), T![*] if parent_matches::(&token) => HlTag::Keyword.into(), diff --git a/crates/ide/src/syntax_highlighting/macro_.rs b/crates/ide/src/syntax_highlighting/macro_.rs index 8197042942e..7f2d61a0bd3 100644 --- a/crates/ide/src/syntax_highlighting/macro_.rs +++ b/crates/ide/src/syntax_highlighting/macro_.rs @@ -19,7 +19,7 @@ impl MacroHighlighter { } } - pub(super) fn highlight(&self, element: SyntaxElement) -> Option { + pub(super) fn highlight(&self, element: &SyntaxElement) -> Option { if let Some(state) = self.state.as_ref() { if matches!(state.rule_state, RuleState::Matcher | RuleState::Expander) { if let Some(range) = is_metavariable(element) { @@ -115,7 +115,7 @@ fn update_macro_state(state: &mut MacroMatcherParseState, tok: &SyntaxToken) { } } -fn is_metavariable(element: SyntaxElement) -> Option { +fn is_metavariable(element: &SyntaxElement) -> Option { let tok = element.as_token()?; match tok.kind() { kind if kind == SyntaxKind::IDENT || kind.is_keyword() => { diff --git a/crates/ide/src/syntax_highlighting/tags.rs b/crates/ide/src/syntax_highlighting/tags.rs index 7216eae0ebc..bdf484e01f7 100644 --- a/crates/ide/src/syntax_highlighting/tags.rs +++ b/crates/ide/src/syntax_highlighting/tags.rs @@ -102,6 +102,8 @@ pub enum HlPunct { Colon, /// ; Semi, + /// ! (only for macro calls) + MacroBang, /// Other, } @@ -167,6 +169,7 @@ impl HlTag { HlPunct::Dot => "dot", HlPunct::Colon => "colon", HlPunct::Semi => "semicolon", + HlPunct::MacroBang => "macro_bang", HlPunct::Other => "punctuation", }, HlTag::NumericLiteral => "numeric_literal", diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html index 0b1c5470ee9..a3f3b4d6b82 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html @@ -89,7 +89,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd /// let foo = Foo::new(); /// /// // calls bar on foo - /// assert!(foo.bar()); + /// assert!(foo.bar()); /// /// let bar = foo.bar || Foo::bar; /// @@ -129,7 +129,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd /// ``` /// macro_rules! noop { ($expr:expr) => { $expr }} -/// noop!(1); +/// noop!(1); /// ``` macro_rules! noop { ($expr:expr) => { @@ -149,7 +149,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd /// #[cfg_attr(feature = "alloc", doc = "```rust")] #[cfg_attr(not(feature = "alloc"), doc = "```ignore")] -/// let _ = example(&alloc::vec![1, 2, 3]); +/// let _ = example(&alloc::vec![1, 2, 3]); /// ``` pub fn mix_and_match() {} diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html b/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html index 88380bdfe12..f4134fee4aa 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html @@ -46,7 +46,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd fixture(r#" trait Foo { fn foo() { - println!("2 + 2 = {}", 4); + println!("2 + 2 = {}", 4); } }"# ); diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index e457069e1f6..41cb3903d67 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -92,58 +92,58 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd fn main() { // from https://doc.rust-lang.org/std/fmt/index.html - println!("Hello"); // => "Hello" - println!("Hello, {}!", "world"); // => "Hello, world!" - println!("The number is {}", 1); // => "The number is 1" - println!("{:?}", (3, 4)); // => "(3, 4)" - println!("{value}", value=4); // => "4" - println!("{} {}", 1, 2); // => "1 2" - println!("{:04}", 42); // => "0042" with leading zerosV - println!("{1} {} {0} {}", 1, 2); // => "2 1 1 2" - println!("{argument}", argument = "test"); // => "test" - println!("{name} {}", 1, name = 2); // => "2 1" - println!("{a} {c} {b}", a="a", b='b', c=3); // => "a 3 b" - println!("{{{}}}", 2); // => "{2}" - println!("Hello {:5}!", "x"); - println!("Hello {:1$}!", "x", 5); - println!("Hello {1:0$}!", 5, "x"); - println!("Hello {:width$}!", "x", width = 5); - println!("Hello {:<5}!", "x"); - println!("Hello {:-<5}!", "x"); - println!("Hello {:^5}!", "x"); - println!("Hello {:>5}!", "x"); - println!("Hello {:+}!", 5); - println!("{:#x}!", 27); - println!("Hello {:05}!", 5); - println!("Hello {:05}!", -5); - println!("{:#010x}!", 27); - println!("Hello {0} is {1:.5}", "x", 0.01); - println!("Hello {1} is {2:.0$}", 5, "x", 0.01); - println!("Hello {0} is {2:.1$}", "x", 5, 0.01); - println!("Hello {} is {:.*}", "x", 5, 0.01); - println!("Hello {} is {2:.*}", "x", 5, 0.01); - println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); - println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); - println!("{}, `{name:.*}` has 3 characters", "Hello", 3, name="1234.56"); - println!("{}, `{name:>8.*}` has 3 right-aligned characters", "Hello", 3, name="1234.56"); - println!("Hello {{}}"); - println!("{{ Hello"); + println!("Hello"); // => "Hello" + println!("Hello, {}!", "world"); // => "Hello, world!" + println!("The number is {}", 1); // => "The number is 1" + println!("{:?}", (3, 4)); // => "(3, 4)" + println!("{value}", value=4); // => "4" + println!("{} {}", 1, 2); // => "1 2" + println!("{:04}", 42); // => "0042" with leading zerosV + println!("{1} {} {0} {}", 1, 2); // => "2 1 1 2" + println!("{argument}", argument = "test"); // => "test" + println!("{name} {}", 1, name = 2); // => "2 1" + println!("{a} {c} {b}", a="a", b='b', c=3); // => "a 3 b" + println!("{{{}}}", 2); // => "{2}" + println!("Hello {:5}!", "x"); + println!("Hello {:1$}!", "x", 5); + println!("Hello {1:0$}!", 5, "x"); + println!("Hello {:width$}!", "x", width = 5); + println!("Hello {:<5}!", "x"); + println!("Hello {:-<5}!", "x"); + println!("Hello {:^5}!", "x"); + println!("Hello {:>5}!", "x"); + println!("Hello {:+}!", 5); + println!("{:#x}!", 27); + println!("Hello {:05}!", 5); + println!("Hello {:05}!", -5); + println!("{:#010x}!", 27); + println!("Hello {0} is {1:.5}", "x", 0.01); + println!("Hello {1} is {2:.0$}", 5, "x", 0.01); + println!("Hello {0} is {2:.1$}", "x", 5, 0.01); + println!("Hello {} is {:.*}", "x", 5, 0.01); + println!("Hello {} is {2:.*}", "x", 5, 0.01); + println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); + println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); + println!("{}, `{name:.*}` has 3 characters", "Hello", 3, name="1234.56"); + println!("{}, `{name:>8.*}` has 3 right-aligned characters", "Hello", 3, name="1234.56"); + println!("Hello {{}}"); + println!("{{ Hello"); - println!(r"Hello, {}!", "world"); + println!(r"Hello, {}!", "world"); // escape sequences - println!("Hello\nWorld"); - println!("\u{48}\x65\x6C\x6C\x6F World"); + println!("Hello\nWorld"); + println!("\u{48}\x65\x6C\x6C\x6F World"); - println!("{\x41}", A = 92); - println!("{ничоси}", ничоси = 92); + println!("{\x41}", A = 92); + println!("{ничоси}", ничоси = 92); - println!("{:x?} {} ", thingy, n2); - panic!("{}", 0); - panic!("more {}", 1); - assert!(true, "{}", 1); - assert!(true, "{} asdasd", 1); - toho!("{}fmt", 0); - asm!("mov eax, {0}"); - format_args!(concat!("{}"), "{}"); + println!("{:x?} {} ", thingy, n2); + panic!("{}", 0); + panic!("more {}", 1); + assert!(true, "{}", 1); + assert!(true, "{} asdasd", 1); + toho!("{}fmt", 0); + asm!("mov eax, {0}"); + format_args!(concat!("{}"), "{}"); } \ No newline at end of file diff --git a/crates/ide/src/syntax_highlighting/test_data/highlighting.html b/crates/ide/src/syntax_highlighting/test_data/highlighting.html index 5de621ce078..185f137b8a5 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlighting.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlighting.html @@ -56,7 +56,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd pub trait Fn<Args>: FnMut<Args> {} } -proc_macros::mirror! { +proc_macros::mirror! { { ,i32 :x pub ,i32 :y pub @@ -138,7 +138,7 @@ proc_macros::mirror! { ($($tt:tt)*) => {$($tt)*} } -def_fn! { +def_fn! { fn bar() -> u32 { 100 } @@ -170,8 +170,8 @@ proc_macros::mirror! { // comment fn main() { - println!("Hello, {}!", 92); - dont_color_me_braces!(); + println!("Hello, {}!", 92); + dont_color_me_braces!(); let mut vec = Vec::new(); if true { @@ -183,7 +183,7 @@ proc_macros::mirror! { // Do nothing } - noop!(noop!(1)); + noop!(noop!(1)); let mut x = 42; x += 1; @@ -228,7 +228,7 @@ proc_macros::mirror! { impl<T> Option<T> { fn and<U>(self, other: Option<U>) -> Option<(T, U)> { match other { - None => unimplemented!(), + None => unimplemented!(), Nope => Nope, } } @@ -242,7 +242,7 @@ proc_macros::mirror! { async fn async_main() { let f1 = learn_and_sing(); let f2 = dance(); - futures::join!(f1, f2); + futures::join!(f1, f2); } fn use_foo_items() { @@ -254,7 +254,7 @@ proc_macros::mirror! { let control_flow = foo::identity(foo::ControlFlow::Continue); if control_flow.should_die() { - foo::die!(); + foo::die!(); } } @@ -262,7 +262,7 @@ proc_macros::mirror! { impl Bool { pub const fn to_primitive(self) -> bool { - matches!(self, Self::True) + matches!(self, Self::True) } } const USAGE_OF_BOOL:bool = Bool::True.to_primitive(); diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs index 1097a77afec..c6322327a5e 100644 --- a/crates/rust-analyzer/src/semantic_tokens.rs +++ b/crates/rust-analyzer/src/semantic_tokens.rs @@ -61,6 +61,7 @@ define_semantic_token_types![ (LABEL, "label"), (LIFETIME, "lifetime"), (LOGICAL, "logical"), + (MACRO_BANG, "macroBang"), (OPERATOR, "operator"), (PARENTHESIS, "parenthesis"), (PUNCTUATION, "punctuation"), diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs index f8c46b92fa8..f0de166d8b5 100644 --- a/crates/rust-analyzer/src/to_proto.rs +++ b/crates/rust-analyzer/src/to_proto.rs @@ -539,6 +539,7 @@ fn semantic_token_type_and_modifiers( HlPunct::Colon => semantic_tokens::COLON, HlPunct::Semi => semantic_tokens::SEMICOLON, HlPunct::Other => semantic_tokens::PUNCTUATION, + HlPunct::MacroBang => semantic_tokens::MACRO_BANG, }, }; diff --git a/editors/code/package.json b/editors/code/package.json index 550b28e1cdb..57757553a7c 100644 --- a/editors/code/package.json +++ b/editors/code/package.json @@ -1240,6 +1240,11 @@ "description": "Style for logic operators", "superType": "operator" }, + { + "id": "macroBang", + "description": "Style for the ! token of macro calls", + "superType": "punctuation" + }, { "id": "operator", "description": "Style for operators", @@ -1378,6 +1383,9 @@ "lifetime": [ "storage.modifier.lifetime.rust" ], + "macroBang": [ + "entity.name.function.macro.rust" + ], "method": [ "entity.name.function.rust" ],