From 59295854f892b0a8f42a6fbc80b04d1f1c695828 Mon Sep 17 00:00:00 2001
From: Edwin Cheng <edwin0cheng@gmail.com>
Date: Fri, 13 Dec 2019 01:41:44 +0800
Subject: [PATCH] Add token id to delims
---
crates/ra_hir_expand/src/quote.rs | 14 ++-
crates/ra_mbe/src/mbe_expander/matcher.rs | 2 +-
crates/ra_mbe/src/subtree_source.rs | 8 +-
crates/ra_mbe/src/syntax_bridge.rs | 144 +++++++++++++++-------
crates/ra_mbe/src/tests.rs | 14 ++-
crates/ra_tt/src/lib.rs | 16 ++-
6 files changed, 134 insertions(+), 64 deletions(-)
diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs
index bce38cc679b..49155fe6262 100644
--- a/crates/ra_hir_expand/src/quote.rs
+++ b/crates/ra_hir_expand/src/quote.rs
@@ -16,7 +16,10 @@ macro_rules! __quote {
{
let children = $crate::__quote!($($tt)*);
let subtree = tt::Subtree {
- delimiter: Some(tt::Delimiter::$delim),
+ delimiter: Some(tt::Delimiter {
+ kind: tt::DelimiterKind::$delim,
+ id: tt::TokenId::unspecified(),
+ }),
token_trees: $crate::quote::IntoTt::to_tokens(children),
};
subtree
@@ -257,8 +260,13 @@ mod tests {
let fields =
fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten();
- let list =
- tt::Subtree { delimiter: Some(tt::Delimiter::Brace), token_trees: fields.collect() };
+ let list = tt::Subtree {
+ delimiter: Some(tt::Delimiter {
+ kind: tt::DelimiterKind::Brace,
+ id: tt::TokenId::unspecified(),
+ }),
+ token_trees: fields.collect(),
+ };
let quoted = quote! {
impl Clone for #struct_name {
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 3f51364780a..c67ae41109c 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -106,7 +106,7 @@ fn match_subtree(
}
Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
- if lhs.delimiter != rhs.delimiter {
+ if lhs.delimiter.map(|it| it.kind) != rhs.delimiter.map(|it| it.kind) {
bail!("mismatched delimiter")
}
let mut src = TtIter::new(rhs);
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 061e9f20b2b..5a03a372a33 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -115,10 +115,10 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
}
fn convert_delim(d: Option<tt::Delimiter>, closing: bool) -> TtToken {
- let (kinds, texts) = match d {
- Some(tt::Delimiter::Parenthesis) => ([T!['('], T![')']], "()"),
- Some(tt::Delimiter::Brace) => ([T!['{'], T!['}']], "{}"),
- Some(tt::Delimiter::Bracket) => ([T!['['], T![']']], "[]"),
+ let (kinds, texts) = match d.map(|it| it.kind) {
+ Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"),
+ Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"),
+ Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"),
None => ([L_DOLLAR, R_DOLLAR], ""),
};
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 8f65ff125ef..a85bb058b7c 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -5,6 +5,7 @@ use ra_syntax::{
ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
SyntaxTreeBuilder, TextRange, TextUnit, T,
};
+use rustc_hash::FxHashMap;
use std::iter::successors;
use tt::buffer::{Cursor, TokenBuffer};
@@ -83,6 +84,15 @@ impl TokenMap {
fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) {
self.entries.push((token_id, relative_range));
}
+
+ fn insert_delim(
+ &mut self,
+ _token_id: tt::TokenId,
+ _open_relative_range: TextRange,
+ _close_relative_range: TextRange,
+ ) {
+ // FIXME: Add entries for delimiter
+ }
}
/// Returns the textual content of a doc comment block as a quoted string
@@ -121,7 +131,10 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr
token_trees.push(mk_punct('!'));
}
token_trees.push(tt::TokenTree::from(tt::Subtree {
- delimiter: Some(tt::Delimiter::Bracket),
+ delimiter: Some(tt::Delimiter {
+ kind: tt::DelimiterKind::Bracket,
+ id: tt::TokenId::unspecified(),
+ }),
token_trees: meta_tkns,
}));
@@ -190,12 +203,16 @@ impl Convertor {
.last()
.unwrap();
- let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) {
- (T!['('], T![')']) => (Some(tt::Delimiter::Parenthesis), true),
- (T!['{'], T!['}']) => (Some(tt::Delimiter::Brace), true),
- (T!['['], T![']']) => (Some(tt::Delimiter::Bracket), true),
+ let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) {
+ (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true),
+ (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true),
+ (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true),
_ => (None, false),
};
+ let delimiter = delimiter_kind.map(|kind| tt::Delimiter {
+ kind,
+ id: self.alloc_delim(first_child.text_range(), last_child.text_range()),
+ });
let mut token_trees = Vec::new();
let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable();
@@ -232,25 +249,31 @@ impl Convertor {
.push(tt::Leaf::from(tt::Punct { char, spacing, id }).into());
}
} else {
- let child: tt::TokenTree = if token.kind() == T![true]
- || token.kind() == T![false]
- {
- let id = self.alloc(token.text_range());
- tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into()
- } else if token.kind().is_keyword()
- || token.kind() == IDENT
- || token.kind() == LIFETIME
- {
- let id = self.alloc(token.text_range());
- let text = token.text().clone();
- tt::Leaf::from(tt::Ident { text, id }).into()
- } else if token.kind().is_literal() {
- let id = self.alloc(token.text_range());
- tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into()
- } else {
- return None;
+ let child: tt::Leaf = match token.kind() {
+ T![true] | T![false] => {
+ let id = self.alloc(token.text_range());
+ let text = token.text().clone();
+ tt::Literal { text, id }.into()
+ }
+ IDENT | LIFETIME => {
+ let id = self.alloc(token.text_range());
+ let text = token.text().clone();
+ tt::Ident { text, id }.into()
+ }
+ k if k.is_keyword() => {
+ let id = self.alloc(token.text_range());
+ let text = token.text().clone();
+ tt::Ident { text, id }.into()
+ }
+ k if k.is_literal() => {
+ let id = self.alloc(token.text_range());
+ let text = token.text().clone();
+ tt::Literal { text, id }.into()
+ }
+ _ => return None,
};
- token_trees.push(child);
+
+ token_trees.push(child.into());
}
}
NodeOrToken::Node(node) => {
@@ -275,11 +298,26 @@ impl Convertor {
self.map.insert(token_id, relative_range);
token_id
}
+
+ fn alloc_delim(
+ &mut self,
+ open_abs_range: TextRange,
+ close_abs_range: TextRange,
+ ) -> tt::TokenId {
+ let open_relative_range = open_abs_range - self.global_offset;
+ let close_relative_range = close_abs_range - self.global_offset;
+ let token_id = tt::TokenId(self.next_id);
+ self.next_id += 1;
+
+ self.map.insert_delim(token_id, open_relative_range, close_relative_range);
+ token_id
+ }
}
struct TtTreeSink<'a> {
buf: String,
cursor: Cursor<'a>,
+ open_delims: FxHashMap<tt::TokenId, TextUnit>,
text_pos: TextUnit,
inner: SyntaxTreeBuilder,
token_map: TokenMap,
@@ -294,6 +332,7 @@ impl<'a> TtTreeSink<'a> {
TtTreeSink {
buf: String::new(),
cursor,
+ open_delims: FxHashMap::default(),
text_pos: 0.into(),
inner: SyntaxTreeBuilder::default(),
roots: smallvec::SmallVec::new(),
@@ -307,10 +346,10 @@ impl<'a> TtTreeSink<'a> {
}
fn delim_to_str(d: Option<tt::Delimiter>, closing: bool) -> SmolStr {
- let texts = match d {
- Some(tt::Delimiter::Parenthesis) => "()",
- Some(tt::Delimiter::Brace) => "{}",
- Some(tt::Delimiter::Bracket) => "[]",
+ let texts = match d.map(|it| it.kind) {
+ Some(tt::DelimiterKind::Parenthesis) => "()",
+ Some(tt::DelimiterKind::Brace) => "{}",
+ Some(tt::DelimiterKind::Bracket) => "[]",
None => return "".into(),
};
@@ -331,34 +370,49 @@ impl<'a> TreeSink for TtTreeSink<'a> {
break;
}
- match self.cursor.token_tree() {
+ let text: Option<SmolStr> = match self.cursor.token_tree() {
Some(tt::TokenTree::Leaf(leaf)) => {
// Mark the range if needed
- if let tt::Leaf::Ident(ident) = leaf {
- if kind == IDENT {
- let range =
- TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text));
- self.token_map.insert(ident.id, range);
- }
- }
-
+ let id = match leaf {
+ tt::Leaf::Ident(ident) => ident.id,
+ tt::Leaf::Punct(punct) => punct.id,
+ tt::Leaf::Literal(lit) => lit.id,
+ };
+ let text = SmolStr::new(format!("{}", leaf));
+ let range = TextRange::offset_len(self.text_pos, TextUnit::of_str(&text));
+ self.token_map.insert(id, range);
self.cursor = self.cursor.bump();
- self.buf += &format!("{}", leaf);
+ Some(text)
}
Some(tt::TokenTree::Subtree(subtree)) => {
self.cursor = self.cursor.subtree().unwrap();
- self.buf += &delim_to_str(subtree.delimiter, false);
- }
- None => {
- if let Some(parent) = self.cursor.end() {
- self.cursor = self.cursor.bump();
- self.buf += &delim_to_str(parent.delimiter, true);
+ if let Some(id) = subtree.delimiter.map(|it| it.id) {
+ self.open_delims.insert(id, self.text_pos);
}
+ Some(delim_to_str(subtree.delimiter, false))
}
+ None => self.cursor.end().and_then(|parent| {
+ self.cursor = self.cursor.bump();
+ if let Some(id) = parent.delimiter.map(|it| it.id) {
+ if let Some(open_delim) = self.open_delims.get(&id) {
+ let open_range =
+ TextRange::offset_len(*open_delim, TextUnit::from_usize(1));
+ let close_range =
+ TextRange::offset_len(self.text_pos, TextUnit::from_usize(1));
+ self.token_map.insert_delim(id, open_range, close_range);
+ }
+ }
+
+ Some(delim_to_str(parent.delimiter, true))
+ }),
};
+
+ if let Some(text) = text {
+ self.buf += &text;
+ self.text_pos += TextUnit::of_str(&text);
+ }
}
- self.text_pos += TextUnit::of_str(&self.buf);
let text = SmolStr::new(self.buf.as_str());
self.buf.clear();
self.inner.token(kind, text);
@@ -504,7 +558,7 @@ mod tests {
let token_tree = ast::TokenTree::cast(token_tree).unwrap();
let tt = ast_to_token_tree(&token_tree).unwrap().0;
- assert_eq!(tt.delimiter, Some(tt::Delimiter::Brace));
+ assert_eq!(tt.delimiter.map(|it| it.kind), Some(tt::DelimiterKind::Brace));
}
#[test]
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 70e65bc74c4..6bcfedcac26 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -77,13 +77,15 @@ macro_rules! foobar {
}
assert_eq!(expansion.token_trees.len(), 3);
- // ($e:ident) => { foo bar $e }
- // 0123 45 6 7 89
- assert_eq!(get_id(&expansion.token_trees[0]), Some(6));
- assert_eq!(get_id(&expansion.token_trees[1]), Some(7));
+ // {($e:ident) => { foo bar $e }}
+ // 012345 67 8 9 T 12
+ assert_eq!(get_id(&expansion.token_trees[0]), Some(9));
+ assert_eq!(get_id(&expansion.token_trees[1]), Some(10));
- // So baz should be 10
- assert_eq!(get_id(&expansion.token_trees[2]), Some(10));
+ // The input args of macro call include parentheses:
+ // (baz)
+ // So baz should be 12+1+1
+ assert_eq!(get_id(&expansion.token_trees[2]), Some(14));
}
#[test]
diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs
index 209ca404844..73d8395a8e5 100644
--- a/crates/ra_tt/src/lib.rs
+++ b/crates/ra_tt/src/lib.rs
@@ -55,7 +55,13 @@ pub struct Subtree {
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-pub enum Delimiter {
+pub struct Delimiter {
+ pub id: TokenId,
+ pub kind: DelimiterKind,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum DelimiterKind {
Parenthesis,
Brace,
Bracket,
@@ -97,10 +103,10 @@ impl fmt::Display for TokenTree {
impl fmt::Display for Subtree {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- let (l, r) = match self.delimiter {
- Some(Delimiter::Parenthesis) => ("(", ")"),
- Some(Delimiter::Brace) => ("{", "}"),
- Some(Delimiter::Bracket) => ("[", "]"),
+ let (l, r) = match self.delimiter.map(|it| it.kind) {
+ Some(DelimiterKind::Parenthesis) => ("(", ")"),
+ Some(DelimiterKind::Brace) => ("{", "}"),
+ Some(DelimiterKind::Bracket) => ("[", "]"),
None => ("", ""),
};
f.write_str(l)?;