mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove TokenCursorFrame.

Browse Source 
The motivation here is to eliminate the `Option<(Delimiter,
DelimSpan)>`, which is `None` for the outermost token stream and `Some`
for all other token streams.

We are already treating the innermost frame specially -- this is the
`frame` vs `stack` distinction in `TokenCursor`. We can push that
further so that `frame` only contains the cursor, and `stack` elements
contain the delimiters for their children. When we are in the outermost
token stream `stack` is empty, so there are no stored delimiters, which
is what we want because the outermost token stream *has* no delimiters.

This change also shrinks `TokenCursor`, which shrinks `Parser` and
`LazyAttrTokenStreamImpl`, which is nice.
This commit is contained in:
Nicholas Nethercote 2023-02-01 12:43:13 +11:00
parent b23f272db0
commit b5ecbbb998
3 changed files with 34 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
compiler/rustc_parse/src/parser/attr_wrapper.rs
View File

@ -469,6 +469,6 @@ mod size_asserts {
use rustc_data_structures::static_assert_size; use rustc_data_structures::static_assert_size;
// tidy-alphabetical-start // tidy-alphabetical-start
static_assert_size!(AttrWrapper, 16); static_assert_size!(AttrWrapper, 16);
static_assert_size!(LazyAttrTokenStreamImpl, 144); static_assert_size!(LazyAttrTokenStreamImpl, 120);
// tidy-alphabetical-end // tidy-alphabetical-end
} }

2
compiler/rustc_parse/src/parser/expr.rs
View File

@ -2141,7 +2141,7 @@ impl<'a> Parser<'a> {
} }
if self.token.kind == TokenKind::Semi if self.token.kind == TokenKind::Semi
&& matches!(self.token_cursor.frame.delim_sp, Some((Delimiter::Parenthesis, _))) && matches!(self.token_cursor.stack.last(), Some((_, Delimiter::Parenthesis, _)))
&& self.may_recover() && self.may_recover()
{ {
// It is likely that the closure body is a block but where the // It is likely that the closure body is a block but where the

72
compiler/rustc_parse/src/parser/mod.rs
View File

@ -168,7 +168,7 @@ pub struct Parser<'a> {
// This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
// it doesn't unintentionally get bigger. // it doesn't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(Parser<'_>, 336); rustc_data_structures::static_assert_size!(Parser<'_>, 312);
/// Stores span information about a closure. /// Stores span information about a closure.
#[derive(Clone)] #[derive(Clone)]
@ -223,16 +223,21 @@ impl<'a> Drop for Parser<'a> {
#[derive(Clone)] #[derive(Clone)]
struct TokenCursor { struct TokenCursor {
// The current (innermost) frame. `frame` and `stack` could be combined, // Cursor for the current (innermost) token stream. The delimiters for this
// but it's faster to keep them separate and access `frame` directly // token stream are found in `self.stack.last()`; when that is `None` then
// rather than via something like `stack.last().unwrap()` or // we are in the outermost token stream which never has delimiters.
// `stack[stack.len() - 1]`. tree_cursor: tokenstream::Cursor,
frame: TokenCursorFrame,
// Additional frames that enclose `frame`. // Token streams surrounding the current one. The delimiters for stack[n]'s
stack: Vec<TokenCursorFrame>, // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
// because it's the outermost token stream which never has delimiters.
stack: Vec<(tokenstream::Cursor, Delimiter, DelimSpan)>,
desugar_doc_comments: bool, desugar_doc_comments: bool,
// Counts the number of calls to `{,inlined_}next`. // Counts the number of calls to `{,inlined_}next`.
num_next_calls: usize, num_next_calls: usize,
// During parsing, we may sometimes need to 'unglue' a // During parsing, we may sometimes need to 'unglue' a
// glued token into two component tokens // glued token into two component tokens
// (e.g. '>>' into '>' and '>), so that the parser // (e.g. '>>' into '>' and '>), so that the parser
@ -257,19 +262,6 @@ struct TokenCursor {
break_last_token: bool, break_last_token: bool,
} }
#[derive(Clone)]
struct TokenCursorFrame {
// This is `None` only for the outermost frame.
delim_sp: Option<(Delimiter, DelimSpan)>,
tree_cursor: tokenstream::Cursor,
}
impl TokenCursorFrame {
fn new(delim_sp: Option<(Delimiter, DelimSpan)>, tts: TokenStream) -> Self {
TokenCursorFrame { delim_sp, tree_cursor: tts.into_trees() }
}
}
impl TokenCursor { impl TokenCursor {
fn next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) { fn next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
self.inlined_next(desugar_doc_comments) self.inlined_next(desugar_doc_comments)
@ -282,12 +274,12 @@ impl TokenCursor {
// FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will // FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will
// need to, whereupon the `delim != Delimiter::Invisible` conditions below can be // need to, whereupon the `delim != Delimiter::Invisible` conditions below can be
// removed. // removed.
if let Some(tree) = self.frame.tree_cursor.next_ref() { if let Some(tree) = self.tree_cursor.next_ref() {
match tree { match tree {
&TokenTree::Token(ref token, spacing) => match (desugar_doc_comments, token) { &TokenTree::Token(ref token, spacing) => match (desugar_doc_comments, token) {
(true, &Token { kind: token::DocComment(_, attr_style, data), span }) => { (true, &Token { kind: token::DocComment(_, attr_style, data), span }) => {
let desugared = self.desugar(attr_style, data, span); let desugared = self.desugar(attr_style, data, span);
self.frame.tree_cursor.replace_prev_and_rewind(desugared); self.tree_cursor.replace_prev_and_rewind(desugared);
// Continue to get the first token of the desugared doc comment. // Continue to get the first token of the desugared doc comment.
} }
_ => { _ => {
@ -299,25 +291,23 @@ impl TokenCursor {
} }
}, },
&TokenTree::Delimited(sp, delim, ref tts) => { &TokenTree::Delimited(sp, delim, ref tts) => {
// Set `open_delim` to true here because we deal with it immediately. let trees = tts.clone().into_trees();
let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone()); self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp));
self.stack.push(mem::replace(&mut self.frame, frame));
if delim != Delimiter::Invisible { if delim != Delimiter::Invisible {
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone); return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
} }
// No open delimiter to return; continue on to the next iteration. // No open delimiter to return; continue on to the next iteration.
} }
}; };
} else if let Some(frame) = self.stack.pop() { } else if let Some((tree_cursor, delim, span)) = self.stack.pop() {
// We have exhausted this frame. Move back to its parent frame. // We have exhausted this token stream. Move back to its parent token stream.
let (delim, span) = self.frame.delim_sp.unwrap(); self.tree_cursor = tree_cursor;
self.frame = frame;
if delim != Delimiter::Invisible { if delim != Delimiter::Invisible {
return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone); return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
} }
// No close delimiter to return; continue on to the next iteration. // No close delimiter to return; continue on to the next iteration.
} else { } else {
// We have exhausted the outermost frame. // We have exhausted the outermost token stream.
return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone); return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
} }
} }
@ -475,7 +465,7 @@ impl<'a> Parser<'a> {
restrictions: Restrictions::empty(), restrictions: Restrictions::empty(),
expected_tokens: Vec::new(), expected_tokens: Vec::new(),
token_cursor: TokenCursor { token_cursor: TokenCursor {
frame: TokenCursorFrame::new(None, tokens), tree_cursor: tokens.into_trees(),
stack: Vec::new(), stack: Vec::new(),
num_next_calls: 0, num_next_calls: 0,
desugar_doc_comments, desugar_doc_comments,
@ -1142,14 +1132,16 @@ impl<'a> Parser<'a> {
return looker(&self.token); return looker(&self.token);
} }
let frame = &self.token_cursor.frame; let tree_cursor = &self.token_cursor.tree_cursor;
if let Some((delim, span)) = frame.delim_sp && delim != Delimiter::Invisible { if let Some(&(_, delim, span)) = self.token_cursor.stack.last()
&& delim != Delimiter::Invisible
{
let all_normal = (0..dist).all(|i| { let all_normal = (0..dist).all(|i| {
let token = frame.tree_cursor.look_ahead(i); let token = tree_cursor.look_ahead(i);
!matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _))) !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
}); });
if all_normal { if all_normal {
return match frame.tree_cursor.look_ahead(dist - 1) { return match tree_cursor.look_ahead(dist - 1) {
Some(tree) => match tree { Some(tree) => match tree {
TokenTree::Token(token, _) => looker(token), TokenTree::Token(token, _) => looker(token),
TokenTree::Delimited(dspan, delim, _) => { TokenTree::Delimited(dspan, delim, _) => {
@ -1310,10 +1302,10 @@ impl<'a> Parser<'a> {
pub(crate) fn parse_token_tree(&mut self) -> TokenTree { pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
match self.token.kind { match self.token.kind {
token::OpenDelim(..) => { token::OpenDelim(..) => {
// Grab the tokens from this frame. // Grab the tokens within the delimiters.
let frame = &self.token_cursor.frame; let tree_cursor = &self.token_cursor.tree_cursor;
let stream = frame.tree_cursor.stream.clone(); let stream = tree_cursor.stream.clone();
let (delim, span) = frame.delim_sp.unwrap(); let (_, delim, span) = *self.token_cursor.stack.last().unwrap();
// Advance the token cursor through the entire delimited // Advance the token cursor through the entire delimited
// sequence. After getting the `OpenDelim` we are *within* the // sequence. After getting the `OpenDelim` we are *within* the