Simplify SubtreeTokenSource
This commit is contained in:
Edwin Cheng
parent
af3d75ad2e
commit
d387bfdc4a
@ -1,131 +1,104 @@
|
||||
//! FIXME: write short doc here
|
||||
|
||||
use parser::{Token, TokenSource};
|
||||
use std::cell::{Cell, Ref, RefCell};
|
||||
use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
|
||||
use tt::buffer::{Cursor, TokenBuffer};
|
||||
use tt::buffer::TokenBuffer;
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
struct TtToken {
|
||||
kind: SyntaxKind,
|
||||
is_joint_to_next: bool,
|
||||
tt: Token,
|
||||
text: SmolStr,
|
||||
}
|
||||
|
||||
pub(crate) struct SubtreeTokenSource<'a> {
|
||||
cached_cursor: Cell<Cursor<'a>>,
|
||||
cached: RefCell<Vec<Option<TtToken>>>,
|
||||
pub(crate) struct SubtreeTokenSource {
|
||||
cached: Vec<TtToken>,
|
||||
curr: (Token, usize),
|
||||
}
|
||||
|
||||
impl<'a> SubtreeTokenSource<'a> {
|
||||
impl<'a> SubtreeTokenSource {
|
||||
// Helper function used in test
|
||||
#[cfg(test)]
|
||||
pub(crate) fn text(&self) -> SmolStr {
|
||||
match *self.get(self.curr.1) {
|
||||
match self.cached.get(self.curr.1) {
|
||||
Some(ref tt) => tt.text.clone(),
|
||||
_ => SmolStr::new(""),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> SubtreeTokenSource<'a> {
|
||||
pub(crate) fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
|
||||
let cursor = buffer.begin();
|
||||
impl<'a> SubtreeTokenSource {
|
||||
pub(crate) fn new(buffer: &TokenBuffer) -> SubtreeTokenSource {
|
||||
let mut current = buffer.begin();
|
||||
let mut cached = Vec::with_capacity(100);
|
||||
|
||||
let mut res = SubtreeTokenSource {
|
||||
curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
|
||||
cached_cursor: Cell::new(cursor),
|
||||
cached: RefCell::new(Vec::with_capacity(10)),
|
||||
};
|
||||
res.curr = (res.mk_token(0), 0);
|
||||
res
|
||||
}
|
||||
while !current.eof() {
|
||||
let cursor = current;
|
||||
let tt = cursor.token_tree();
|
||||
|
||||
fn mk_token(&self, pos: usize) -> Token {
|
||||
match *self.get(pos) {
|
||||
Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
|
||||
None => Token { kind: EOF, is_jointed_to_next: false },
|
||||
}
|
||||
}
|
||||
|
||||
fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
|
||||
fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
|
||||
let tkn = c.token_tree();
|
||||
|
||||
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tkn {
|
||||
// Check if it is lifetime
|
||||
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
|
||||
if punct.char == '\'' {
|
||||
let next = c.bump();
|
||||
let next = cursor.bump();
|
||||
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) =
|
||||
next.token_tree()
|
||||
{
|
||||
let res_cursor = next.bump();
|
||||
let text = SmolStr::new("'".to_string() + &ident.to_string());
|
||||
|
||||
return Some((res_cursor, text));
|
||||
let text = SmolStr::new("'".to_string() + &ident.text);
|
||||
cached.push(TtToken {
|
||||
tt: Token { kind: LIFETIME_IDENT, is_jointed_to_next: false },
|
||||
text,
|
||||
});
|
||||
current = next.bump();
|
||||
continue;
|
||||
} else {
|
||||
panic!("Next token must be ident : {:#?}", next.token_tree());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
if pos < self.cached.borrow().len() {
|
||||
return Ref::map(self.cached.borrow(), |c| &c[pos]);
|
||||
}
|
||||
|
||||
{
|
||||
let mut cached = self.cached.borrow_mut();
|
||||
while pos >= cached.len() {
|
||||
let cursor = self.cached_cursor.get();
|
||||
if cursor.eof() {
|
||||
cached.push(None);
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some((curr, text)) = is_lifetime(cursor) {
|
||||
cached.push(Some(TtToken {
|
||||
kind: LIFETIME_IDENT,
|
||||
is_joint_to_next: false,
|
||||
text,
|
||||
}));
|
||||
self.cached_cursor.set(curr);
|
||||
continue;
|
||||
}
|
||||
|
||||
match cursor.token_tree() {
|
||||
current = match tt {
|
||||
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
|
||||
cached.push(Some(convert_leaf(&leaf)));
|
||||
self.cached_cursor.set(cursor.bump());
|
||||
cached.push(convert_leaf(&leaf));
|
||||
cursor.bump()
|
||||
}
|
||||
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
|
||||
self.cached_cursor.set(cursor.subtree().unwrap());
|
||||
cached.push(Some(convert_delim(subtree.delimiter_kind(), false)));
|
||||
cached.push(convert_delim(subtree.delimiter_kind(), false));
|
||||
cursor.subtree().unwrap()
|
||||
}
|
||||
None => {
|
||||
if let Some(subtree) = cursor.end() {
|
||||
cached.push(Some(convert_delim(subtree.delimiter_kind(), true)));
|
||||
self.cached_cursor.set(cursor.bump());
|
||||
}
|
||||
}
|
||||
cached.push(convert_delim(subtree.delimiter_kind(), true));
|
||||
cursor.bump()
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Ref::map(self.cached.borrow(), |c| &c[pos])
|
||||
let mut res = SubtreeTokenSource {
|
||||
curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
|
||||
cached,
|
||||
};
|
||||
res.curr = (res.token(0), 0);
|
||||
res
|
||||
}
|
||||
|
||||
fn token(&self, pos: usize) -> Token {
|
||||
match self.cached.get(pos) {
|
||||
Some(it) => it.tt,
|
||||
None => Token { kind: EOF, is_jointed_to_next: false },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> TokenSource for SubtreeTokenSource<'a> {
|
||||
impl<'a> TokenSource for SubtreeTokenSource {
|
||||
fn current(&self) -> Token {
|
||||
self.curr.0
|
||||
}
|
||||
|
||||
/// Lookahead n token
|
||||
fn lookahead_nth(&self, n: usize) -> Token {
|
||||
self.mk_token(self.curr.1 + n)
|
||||
self.token(self.curr.1 + n)
|
||||
}
|
||||
|
||||
/// bump cursor to next token
|
||||
@ -133,13 +106,12 @@ fn bump(&mut self) {
|
||||
if self.current().kind == EOF {
|
||||
return;
|
||||
}
|
||||
|
||||
self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1);
|
||||
self.curr = (self.token(self.curr.1 + 1), self.curr.1 + 1);
|
||||
}
|
||||
|
||||
/// Is the current token a specified keyword?
|
||||
fn is_keyword(&self, kw: &str) -> bool {
|
||||
match *self.get(self.curr.1) {
|
||||
match self.cached.get(self.curr.1) {
|
||||
Some(ref t) => t.text == *kw,
|
||||
_ => false,
|
||||
}
|
||||
@ -157,7 +129,7 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
|
||||
let idx = closing as usize;
|
||||
let kind = kinds[idx];
|
||||
let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" };
|
||||
TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }
|
||||
TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) }
|
||||
}
|
||||
|
||||
fn convert_literal(l: &tt::Literal) -> TtToken {
|
||||
@ -171,7 +143,7 @@ fn convert_literal(l: &tt::Literal) -> TtToken {
|
||||
})
|
||||
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
|
||||
|
||||
TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
|
||||
TtToken { tt: Token { kind, is_jointed_to_next: false }, text: l.text.clone() }
|
||||
}
|
||||
|
||||
fn convert_ident(ident: &tt::Ident) -> TtToken {
|
||||
@ -182,7 +154,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
|
||||
_ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
|
||||
};
|
||||
|
||||
TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
|
||||
TtToken { tt: Token { kind, is_jointed_to_next: false }, text: ident.text.clone() }
|
||||
}
|
||||
|
||||
fn convert_punct(p: tt::Punct) -> TtToken {
|
||||
@ -196,7 +168,7 @@ fn convert_punct(p: tt::Punct) -> TtToken {
|
||||
let s: &str = p.char.encode_utf8(&mut buf);
|
||||
SmolStr::new(s)
|
||||
};
|
||||
TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
|
||||
TtToken { tt: Token { kind, is_jointed_to_next: p.spacing == tt::Spacing::Joint }, text }
|
||||
}
|
||||
|
||||
fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
|
||||
@ -210,6 +182,7 @@ fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{convert_literal, TtToken};
|
||||
use parser::Token;
|
||||
use syntax::{SmolStr, SyntaxKind};
|
||||
|
||||
#[test]
|
||||
@ -220,8 +193,7 @@ fn test_negative_literal() {
|
||||
text: SmolStr::new("-42.0")
|
||||
}),
|
||||
TtToken {
|
||||
kind: SyntaxKind::FLOAT_NUMBER,
|
||||
is_joint_to_next: false,
|
||||
tt: Token { kind: SyntaxKind::FLOAT_NUMBER, is_jointed_to_next: false },
|
||||
text: SmolStr::new("-42.0")
|
||||
}
|
||||
);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user