From d33b3562e5e888eaffd2f8f1af08ca2afdbe542c Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sun, 16 Feb 2020 16:47:24 +0300 Subject: [PATCH] parser: Do not call `bump` recursively Token normalization is merged directly into `bump`. Special "unknown macro variable" diagnostic for unexpected `$`s is removed as preventing legal code from compiling. --- src/librustc_expand/mbe/macro_parser.rs | 2 - src/librustc_expand/mbe/macro_rules.rs | 1 - src/librustc_parse/parser/mod.rs | 75 +++++++++++-------------- src/test/ui/issues/issue-6596-1.rs | 2 +- src/test/ui/issues/issue-6596-1.stderr | 4 +- src/test/ui/issues/issue-6596-2.rs | 2 +- src/test/ui/issues/issue-6596-2.stderr | 4 +- 7 files changed, 38 insertions(+), 52 deletions(-) diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs index 5bf7602ea6e..6599e92222c 100644 --- a/src/librustc_expand/mbe/macro_parser.rs +++ b/src/librustc_expand/mbe/macro_parser.rs @@ -856,8 +856,6 @@ fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { if name == sym::tt { return token::NtTT(p.parse_token_tree()); } - // check at the beginning and the parser checks after each bump - p.process_potential_macro_variable(); match parse_nt_inner(p, sp, name) { Ok(nt) => nt, Err(mut err) => { diff --git a/src/librustc_expand/mbe/macro_rules.rs b/src/librustc_expand/mbe/macro_rules.rs index f3c827b1816..52e581e30f5 100644 --- a/src/librustc_expand/mbe/macro_rules.rs +++ b/src/librustc_expand/mbe/macro_rules.rs @@ -267,7 +267,6 @@ fn generic_extension<'cx>( cx.current_expansion.module.mod_path.last().map(|id| id.to_string()); p.last_type_ascription = cx.current_expansion.prior_type_ascription; - p.process_potential_macro_variable(); // Let the context choose how to interpret the result. // Weird, but useful for X-macros. return Box::new(ParserAnyMacro { diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index 79944dc35e5..4f96d33b83f 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -404,7 +404,8 @@ impl<'a> Parser<'a> { subparser_name, }; - parser.token = parser.next_tok(); + // Make parser point to the first token. + parser.bump(); if let Some(directory) = directory { parser.directory = directory; @@ -418,7 +419,6 @@ impl<'a> Parser<'a> { } } - parser.process_potential_macro_variable(); parser } @@ -430,7 +430,7 @@ impl<'a> Parser<'a> { self.unnormalized_prev_token.as_ref().unwrap_or(&self.prev_token) } - fn next_tok(&mut self) -> Token { + fn next_tok(&mut self, fallback_span: Span) -> Token { let mut next = if self.desugar_doc_comments { self.token_cursor.next_desugared() } else { @@ -438,7 +438,7 @@ impl<'a> Parser<'a> { }; if next.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact. - next.span = self.unnormalized_token().span.with_ctxt(next.span.ctxt()); + next.span = fallback_span.with_ctxt(next.span.ctxt()); } next } @@ -896,6 +896,24 @@ impl<'a> Parser<'a> { self.parse_delim_comma_seq(token::Paren, f) } + // Interpolated identifier (`$i: ident`) and lifetime (`$l: lifetime`) + // tokens are replaced with usual identifier and lifetime tokens, + // so the former are never encountered during normal parsing. + fn normalize_token(token: &Token) -> Option { + match &token.kind { + token::Interpolated(nt) => match **nt { + token::NtIdent(ident, is_raw) => { + Some(Token::new(token::Ident(ident.name, is_raw), ident.span)) + } + token::NtLifetime(ident) => { + Some(Token::new(token::Lifetime(ident.name), ident.span)) + } + _ => None, + }, + _ => None, + } + } + /// Advance the parser by one token. pub fn bump(&mut self) { if self.prev_token.kind == TokenKind::Eof { @@ -905,16 +923,17 @@ impl<'a> Parser<'a> { } // Update the current and previous tokens. - let next_token = self.next_tok(); - self.prev_token = mem::replace(&mut self.token, next_token); + self.prev_token = self.token.take(); self.unnormalized_prev_token = self.unnormalized_token.take(); + self.token = self.next_tok(self.unnormalized_prev_token().span); + if let Some(normalized_token) = Self::normalize_token(&self.token) { + self.unnormalized_token = Some(mem::replace(&mut self.token, normalized_token)); + } // Update fields derived from the previous token. self.prev_span = self.unnormalized_prev_token().span; self.expected_tokens.clear(); - // Check after each token. - self.process_potential_macro_variable(); } /// Advances the parser using provided token as a next one. Use this when @@ -924,9 +943,12 @@ impl<'a> Parser<'a> { /// Correct token kinds and spans need to be calculated instead. fn bump_with(&mut self, next: TokenKind, span: Span) { // Update the current and previous tokens. - let next_token = Token::new(next, span); - self.prev_token = mem::replace(&mut self.token, next_token); + self.prev_token = self.token.take(); self.unnormalized_prev_token = self.unnormalized_token.take(); + self.token = Token::new(next, span); + if let Some(normalized_token) = Self::normalize_token(&self.token) { + self.unnormalized_token = Some(mem::replace(&mut self.token, normalized_token)); + } // Update fields derived from the previous token. self.prev_span = self.unnormalized_prev_token().span.with_hi(span.lo()); @@ -1066,39 +1088,6 @@ impl<'a> Parser<'a> { } } - pub fn process_potential_macro_variable(&mut self) { - let normalized_token = match self.token.kind { - token::Dollar - if self.token.span.from_expansion() && self.look_ahead(1, |t| t.is_ident()) => - { - self.bump(); - let name = match self.token.kind { - token::Ident(name, _) => name, - _ => unreachable!(), - }; - let span = self.prev_span.to(self.token.span); - self.struct_span_err(span, &format!("unknown macro variable `{}`", name)) - .span_label(span, "unknown macro variable") - .emit(); - self.bump(); - return; - } - token::Interpolated(ref nt) => { - // Interpolated identifier and lifetime tokens are replaced with usual identifier - // and lifetime tokens, so the former are never encountered during normal parsing. - match **nt { - token::NtIdent(ident, is_raw) => { - Token::new(token::Ident(ident.name, is_raw), ident.span) - } - token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span), - _ => return, - } - } - _ => return, - }; - self.unnormalized_token = Some(mem::replace(&mut self.token, normalized_token)); - } - /// Parses a single token tree from the input. pub fn parse_token_tree(&mut self) -> TokenTree { match self.token.kind { diff --git a/src/test/ui/issues/issue-6596-1.rs b/src/test/ui/issues/issue-6596-1.rs index 5da54451346..25f1d650072 100644 --- a/src/test/ui/issues/issue-6596-1.rs +++ b/src/test/ui/issues/issue-6596-1.rs @@ -1,7 +1,7 @@ macro_rules! e { ($inp:ident) => ( $nonexistent - //~^ ERROR unknown macro variable `nonexistent` + //~^ ERROR expected expression, found `$` ); } diff --git a/src/test/ui/issues/issue-6596-1.stderr b/src/test/ui/issues/issue-6596-1.stderr index 4f29d8a9274..216fe6472a5 100644 --- a/src/test/ui/issues/issue-6596-1.stderr +++ b/src/test/ui/issues/issue-6596-1.stderr @@ -1,8 +1,8 @@ -error: unknown macro variable `nonexistent` +error: expected expression, found `$` --> $DIR/issue-6596-1.rs:3:9 | LL | $nonexistent - | ^^^^^^^^^^^^ unknown macro variable + | ^^^^^^^^^^^^ expected expression ... LL | e!(foo); | -------- in this macro invocation diff --git a/src/test/ui/issues/issue-6596-2.rs b/src/test/ui/issues/issue-6596-2.rs index b19700efe5a..8f7c98d9a67 100644 --- a/src/test/ui/issues/issue-6596-2.rs +++ b/src/test/ui/issues/issue-6596-2.rs @@ -3,7 +3,7 @@ macro_rules! g { ($inp:ident) => ( { $inp $nonexistent } - //~^ ERROR unknown macro variable `nonexistent` + //~^ ERROR expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `$` ); } diff --git a/src/test/ui/issues/issue-6596-2.stderr b/src/test/ui/issues/issue-6596-2.stderr index 4fcb0176faa..3d13c64f762 100644 --- a/src/test/ui/issues/issue-6596-2.stderr +++ b/src/test/ui/issues/issue-6596-2.stderr @@ -1,8 +1,8 @@ -error: unknown macro variable `nonexistent` +error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `$` --> $DIR/issue-6596-2.rs:5:16 | LL | { $inp $nonexistent } - | ^^^^^^^^^^^^ unknown macro variable + | ^^^^^^^^^^^^ expected one of 8 possible tokens ... LL | g!(foo); | -------- in this macro invocation