From 74599cd362076816755f087876949c85d8ed92fc Mon Sep 17 00:00:00 2001
From: Aaron Hill <aa1ronham@gmail.com>
Date: Mon, 15 Jun 2020 11:50:27 -0400
Subject: [PATCH] Always capture tokens for `macro_rules!` arguments
---
src/librustc_expand/mbe/macro_parser.rs | 29 +++++++++--
src/librustc_parse/parser/mod.rs | 2 +-
.../ui/proc-macro/auxiliary/first-second.rs | 20 ++++++++
src/test/ui/proc-macro/auxiliary/recollect.rs | 12 +++++
.../ui/proc-macro/auxiliary/weird-hygiene.rs | 48 +++++++++++++++++++
.../proc-macro/capture-macro-rules-invoke.rs | 22 +++++++++
src/test/ui/proc-macro/macro-rules-derive.rs | 20 ++++++++
.../ui/proc-macro/macro-rules-derive.stderr | 9 ++++
src/test/ui/proc-macro/weird-hygiene.rs | 48 +++++++++++++++++++
9 files changed, 205 insertions(+), 5 deletions(-)
create mode 100644 src/test/ui/proc-macro/auxiliary/first-second.rs
create mode 100644 src/test/ui/proc-macro/auxiliary/recollect.rs
create mode 100644 src/test/ui/proc-macro/auxiliary/weird-hygiene.rs
create mode 100644 src/test/ui/proc-macro/capture-macro-rules-invoke.rs
create mode 100644 src/test/ui/proc-macro/macro-rules-derive.rs
create mode 100644 src/test/ui/proc-macro/macro-rules-derive.stderr
create mode 100644 src/test/ui/proc-macro/weird-hygiene.rs
diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs
index db8258a7786..968f7c8e273 100644
--- a/src/librustc_expand/mbe/macro_parser.rs
+++ b/src/librustc_expand/mbe/macro_parser.rs
@@ -866,10 +866,23 @@ fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Result<Nonterminal, (
}
fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> {
+ // Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`)
+ // needs to have them force-captured here.
+ // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
+ // which requires having captured tokens available. Since we cannot determine
+ // in advance whether or not a proc-macro will be (transitively) invoked,
+ // we always capture tokens for any `Nonterminal` which needs them.
Ok(match name {
- sym::item => match p.parse_item()? {
- Some(i) => token::NtItem(i),
- None => return Err(p.struct_span_err(p.token.span, "expected an item keyword")),
+ sym::item => match p.collect_tokens(|this| this.parse_item())? {
+ (Some(mut item), tokens) => {
+ // If we captured tokens during parsing (due to outer attributes),
+ // use those.
+ if item.tokens.is_none() {
+ item.tokens = Some(tokens);
+ }
+ token::NtItem(item)
+ }
+ (None, _) => return Err(p.struct_span_err(p.token.span, "expected an item keyword")),
},
sym::block => token::NtBlock(p.parse_block()?),
sym::stmt => match p.parse_stmt()? {
@@ -877,7 +890,15 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
None => return Err(p.struct_span_err(p.token.span, "expected a statement")),
},
sym::pat => token::NtPat(p.parse_pat(None)?),
- sym::expr => token::NtExpr(p.parse_expr()?),
+ sym::expr => {
+ let (mut expr, tokens) = p.collect_tokens(|this| this.parse_expr())?;
+ // If we captured tokens during parsing (due to outer attributes),
+ // use those.
+ if expr.tokens.is_none() {
+ expr.tokens = Some(tokens);
+ }
+ token::NtExpr(expr)
+ }
sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?),
sym::ty => token::NtTy(p.parse_ty()?),
// this could be handled like a token, since it is one
diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs
index 47ae92c48bd..7811d5fb741 100644
--- a/src/librustc_parse/parser/mod.rs
+++ b/src/librustc_parse/parser/mod.rs
@@ -1151,7 +1151,7 @@ impl<'a> Parser<'a> {
/// This restriction shouldn't be an issue in practice,
/// since this function is used to record the tokens for
/// a parsed AST item, which always has matching delimiters.
- fn collect_tokens<R>(
+ pub fn collect_tokens<R>(
&mut self,
f: impl FnOnce(&mut Self) -> PResult<'a, R>,
) -> PResult<'a, (R, TokenStream)> {
diff --git a/src/test/ui/proc-macro/auxiliary/first-second.rs b/src/test/ui/proc-macro/auxiliary/first-second.rs
new file mode 100644
index 00000000000..6331608fbe5
--- /dev/null
+++ b/src/test/ui/proc-macro/auxiliary/first-second.rs
@@ -0,0 +1,20 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::{TokenStream, TokenTree, Group, Delimiter};
+
+#[proc_macro_attribute]
+pub fn first(_attr: TokenStream, item: TokenStream) -> TokenStream {
+ let tokens: TokenStream = "#[derive(Second)]".parse().unwrap();
+ let wrapped = TokenTree::Group(Group::new(Delimiter::None, item.into_iter().collect()));
+ tokens.into_iter().chain(std::iter::once(wrapped)).collect()
+}
+
+#[proc_macro_derive(Second)]
+pub fn second(item: TokenStream) -> TokenStream {
+ TokenStream::new()
+}
diff --git a/src/test/ui/proc-macro/auxiliary/recollect.rs b/src/test/ui/proc-macro/auxiliary/recollect.rs
new file mode 100644
index 00000000000..d4494a5aff2
--- /dev/null
+++ b/src/test/ui/proc-macro/auxiliary/recollect.rs
@@ -0,0 +1,12 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+use proc_macro::TokenStream;
+
+#[proc_macro]
+pub fn recollect(tokens: TokenStream) -> TokenStream {
+ tokens.into_iter().collect()
+}
diff --git a/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs b/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs
new file mode 100644
index 00000000000..338e436df50
--- /dev/null
+++ b/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs
@@ -0,0 +1,48 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::{TokenStream, TokenTree, Group};
+
+fn find_my_ident(tokens: TokenStream) -> Option<TokenStream> {
+ for token in tokens {
+ if let TokenTree::Ident(ident) = &token {
+ if ident.to_string() == "hidden_ident" {
+ return Some(vec![token].into_iter().collect())
+ }
+ } else if let TokenTree::Group(g) = token {
+ if let Some(stream) = find_my_ident(g.stream()) {
+ return Some(stream)
+ }
+ }
+ }
+ return None;
+}
+
+
+#[proc_macro_derive(WeirdDerive)]
+pub fn weird_derive(item: TokenStream) -> TokenStream {
+ let my_ident = find_my_ident(item).expect("Missing 'my_ident'!");
+ let tokens: TokenStream = "call_it!();".parse().unwrap();
+ let final_call = tokens.into_iter().map(|tree| {
+ if let TokenTree::Group(g) = tree {
+ return Group::new(g.delimiter(), my_ident.clone()).into()
+ } else {
+ return tree
+ }
+ }).collect();
+ final_call
+}
+
+#[proc_macro]
+pub fn recollect(item: TokenStream) -> TokenStream {
+ item.into_iter().collect()
+}
+
+#[proc_macro_attribute]
+pub fn recollect_attr(_attr: TokenStream, mut item: TokenStream) -> TokenStream {
+ item.into_iter().collect()
+}
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
new file mode 100644
index 00000000000..a404ddace9b
--- /dev/null
+++ b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
@@ -0,0 +1,22 @@
+// aux-build:test-macros.rs
+// check-pass
+
+extern crate test_macros;
+use test_macros::recollect;
+
+macro_rules! use_expr {
+ ($expr:expr) => {
+ recollect!($expr)
+ }
+}
+
+#[allow(dead_code)]
+struct Foo;
+impl Foo {
+ #[allow(dead_code)]
+ fn use_self(self) {
+ drop(use_expr!(self));
+ }
+}
+
+fn main() {}
diff --git a/src/test/ui/proc-macro/macro-rules-derive.rs b/src/test/ui/proc-macro/macro-rules-derive.rs
new file mode 100644
index 00000000000..5b4d577a1ac
--- /dev/null
+++ b/src/test/ui/proc-macro/macro-rules-derive.rs
@@ -0,0 +1,20 @@
+// aux-build:first-second.rs
+// FIXME: The spans here are bad, see PR #73084
+
+extern crate first_second;
+use first_second::*;
+
+macro_rules! produce_it {
+ ($name:ident) => {
+ #[first] //~ ERROR cannot find type
+ struct $name {
+ field: MissingType
+ }
+ }
+}
+
+produce_it!(MyName);
+
+fn main() {
+ println!("Hello, world!");
+}
diff --git a/src/test/ui/proc-macro/macro-rules-derive.stderr b/src/test/ui/proc-macro/macro-rules-derive.stderr
new file mode 100644
index 00000000000..4b72d29fe8a
--- /dev/null
+++ b/src/test/ui/proc-macro/macro-rules-derive.stderr
@@ -0,0 +1,9 @@
+error[E0412]: cannot find type `MissingType` in this scope
+ --> $DIR/macro-rules-derive.rs:9:9
+ |
+LL | #[first]
+ | ^^^^^^^^ not found in this scope
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0412`.
diff --git a/src/test/ui/proc-macro/weird-hygiene.rs b/src/test/ui/proc-macro/weird-hygiene.rs
new file mode 100644
index 00000000000..3f48191b5b2
--- /dev/null
+++ b/src/test/ui/proc-macro/weird-hygiene.rs
@@ -0,0 +1,48 @@
+// aux-build:weird-hygiene.rs
+// check-pass
+// FIXME: This should actually error, see PR #73084
+
+#![feature(stmt_expr_attributes)]
+#![feature(proc_macro_hygiene)]
+
+extern crate weird_hygiene;
+use weird_hygiene::*;
+
+macro_rules! other {
+ ($tokens:expr) => {
+ macro_rules! call_it {
+ ($outer_ident:ident) => {
+ macro_rules! inner {
+ () => {
+ $outer_ident;
+ }
+ }
+ }
+ }
+
+ #[derive(WeirdDerive)]
+ enum MyEnum {
+ Value = (stringify!($tokens + hidden_ident), 1).1
+ }
+
+ inner!();
+ }
+}
+
+macro_rules! invoke_it {
+ ($token:expr) => {
+ #[recollect_attr] {
+ $token;
+ hidden_ident
+ }
+ }
+}
+
+fn main() {
+ // `other` and `invoke_it` are both macro_rules! macros,
+ // so it should be impossible for them to ever see `hidden_ident`,
+ // even if they invoke a proc macro.
+ let hidden_ident = "Hello1";
+ other!(50);
+ invoke_it!(25);
+}