mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize the parser's last token handling.

Browse Source 
The parser currently makes a heap copy of the last token in four cases:
identifiers, paths, doc comments, and commas. The identifier and
interpolation cases are unused, and for doc comments and commas we only
need to record their presence, not their value.

This commit consolidates the last token handling and avoids the
unnecessary copies by replacing `last_token`, `last_token_eof`, and
`last_token_interpolated` with a new field `last_token_kind`. This
simplifies the parser slightly and speeds up parsing on some files by
3--4%.
This commit is contained in:
Nicholas Nethercote 2016-09-16 15:46:40 +10:00
parent f81f4964d2
commit 8075d54606
1 changed files with 39 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
src/libsyntax/parse/parser.rs
View File

@ -237,6 +237,15 @@ fn maybe_append(mut lhs: Vec<Attribute>, rhs: Option<Vec<Attribute>>)
lhs
}
#[derive(PartialEq)]
enum LastTokenKind {
DocComment,
Comma,
Interpolated,
Eof,
Other,
}
/* ident is handled by common.rs */
pub struct Parser<'a> {
@ -248,10 +257,8 @@ pub struct Parser<'a> {
/// the span of the prior token:
pub last_span: Span,
pub cfg: CrateConfig,
/// the previous token or None (only stashed sometimes).
pub last_token: Option<Box<token::Token>>,
last_token_interpolated: bool,
last_token_eof: bool,
/// the previous token kind
last_token_kind: LastTokenKind,
pub buffer: [TokenAndSpan; 4],
pub buffer_start: isize,
pub buffer_end: isize,
@ -362,9 +369,7 @@ pub fn new(sess: &'a ParseSess,
token: tok0.tok,
span: span,
last_span: span,
last_token: None,
last_token_interpolated: false,
last_token_eof: false,
last_token_kind: LastTokenKind::Other,
buffer: [
placeholder.clone(),
placeholder.clone(),
@ -500,7 +505,7 @@ fn interpolated_or_expr_span(&self,
expr: PResult<'a, P<Expr>>)
-> PResult<'a, (Span, P<Expr>)> {
expr.map(|e| {
if self.last_token_interpolated {
if self.last_token_kind == LastTokenKind::Interpolated {
(self.last_span, e)
} else {
(e.span, e)
@ -520,21 +525,19 @@ pub fn parse_ident(&mut self) -> PResult<'a, ast::Ident> {
self.bug("ident interpolation not converted to real token");
}
_ => {
let last_token = self.last_token.clone().map(|t| *t);
Err(match last_token {
Some(token::DocComment(_)) => self.span_fatal_help(self.last_span,
Err(if self.last_token_kind == LastTokenKind::DocComment {
self.span_fatal_help(self.last_span,
"found a documentation comment that doesn't document anything",
"doc comments must come before what they document, maybe a comment was \
intended with `//`?"),
_ => {
intended with `//`?")
} else {
let mut err = self.fatal(&format!("expected identifier, found `{}`",
self.this_token_to_string()));
if self.token == token::Underscore {
err.note("`_` is a wildcard pattern, not an identifier");
}
err
}
})
})
}
}
}
@ -923,26 +926,22 @@ pub fn parse_seq<T, F>(&mut self,
/// Advance the parser by one token
pub fn bump(&mut self) {
if self.last_token_eof {
if self.last_token_kind == LastTokenKind::Eof {
// Bumping after EOF is a bad sign, usually an infinite loop.
self.bug("attempted to bump the parser past EOF (may be stuck in a loop)");
}
if self.token == token::Eof {
self.last_token_eof = true;
}
self.last_span = self.span;
// Stash token for error recovery (sometimes; clone is not necessarily cheap).
self.last_token = if self.token.is_ident() ||
self.token.is_path() ||
self.token.is_doc_comment() ||
self.token == token::Comma {
Some(Box::new(self.token.clone()))
} else {
None
// Record last token kind for possible error recovery.
self.last_token_kind = match self.token {
token::DocComment(..) => LastTokenKind::DocComment,
token::Comma => LastTokenKind::Comma,
token::Interpolated(..) => LastTokenKind::Interpolated,
token::Eof => LastTokenKind::Eof,
_ => LastTokenKind::Other,
};
self.last_token_interpolated = self.token.is_interpolated();
let next = if self.buffer_start == self.buffer_end {
self.reader.real_token()
} else {
@ -979,11 +978,10 @@ pub fn bump_with(&mut self,
lo: BytePos,
hi: BytePos) {
self.last_span = mk_sp(self.span.lo, lo);
// It would be incorrect to just stash current token, but fortunately
// for tokens currently using `bump_with`, last_token will be of no
// use anyway.
self.last_token = None;
self.last_token_interpolated = false;
// It would be incorrect to record the kind of the current token, but
// fortunately for tokens currently using `bump_with`, the
// last_token_kind will be of no use anyway.
self.last_token_kind = LastTokenKind::Other;
self.span = mk_sp(lo, hi);
self.token = next;
self.expected_tokens.clear();
@ -2974,7 +2972,7 @@ pub fn parse_assoc_expr_with(&mut self,
self.expected_tokens.push(TokenType::Operator);
while let Some(op) = AssocOp::from_token(&self.token) {
let lhs_span = if self.last_token_interpolated {
let lhs_span = if self.last_token_kind == LastTokenKind::Interpolated {
self.last_span
} else {
lhs.span
@ -4036,13 +4034,13 @@ fn parse_stmt_without_recovery(&mut self, macro_expanded: bool) -> PResult<'a, O
None => {
let unused_attrs = |attrs: &[_], s: &mut Self| {
if attrs.len() > 0 {
let last_token = s.last_token.clone().map(|t| *t);
match last_token {
Some(token::DocComment(_)) => s.span_err_help(s.last_span,
if s.last_token_kind == LastTokenKind::DocComment {
s.span_err_help(s.last_span,
"found a documentation comment that doesn't document anything",
"doc comments must come before what they document, maybe a \
comment was intended with `//`?"),
_ => s.span_err(s.span, "expected statement after outer attribute"),
comment was intended with `//`?");
} else {
s.span_err(s.span, "expected statement after outer attribute");
}
}
};
@ -4332,9 +4330,7 @@ fn parse_generic_values_after_lt(&mut self) -> PResult<'a, (Vec<ast::Lifetime>,
let missing_comma = !lifetimes.is_empty() &&
!self.token.is_like_gt() &&
self.last_token
.as_ref().map_or(true,
|x| &**x != &token::Comma);
self.last_token_kind != LastTokenKind::Comma;
if missing_comma {