expand: Stop using nonterminals for passing tokens to attribute and derive macros

This commit is contained in:
Vadim Petrochenkov 2020-06-14 14:30:42 +03:00
parent d462551a86
commit a5764de00b
26 changed files with 118 additions and 88 deletions

View File

@ -11,7 +11,7 @@ use crate::tokenstream::TokenTree;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::Lrc;
use rustc_macros::HashStable_Generic;
use rustc_span::symbol::kw;
use rustc_span::symbol::{kw, sym};
use rustc_span::symbol::{Ident, Symbol};
use rustc_span::{self, Span, DUMMY_SP};
use std::borrow::Cow;
@ -785,6 +785,26 @@ impl Nonterminal {
NtTT(tt) => tt.span(),
}
}
/// This nonterminal looks like some specific enums from
/// `proc-macro-hack` and `procedural-masquerade` crates.
/// We need to maintain some special pretty-printing behavior for them due to incorrect
/// asserts in old versions of those crates and their wide use in the ecosystem.
/// See issue #73345 for more details.
/// FIXME: Remove this eventually.
pub fn pretty_printing_compatibility_hack(&self) -> bool {
if let NtItem(item) = self {
let name = item.ident.name;
if name == sym::ProceduralMasqueradeDummyType || name == sym::ProcMacroHack {
if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
if let [variant] = &*enum_def.variants {
return variant.ident.name == sym::Input;
}
}
}
}
false
}
}
impl PartialEq for Nonterminal {

View File

@ -148,9 +148,14 @@ pub fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
printer.s.eof()
}
// This makes comma-separated lists look slightly nicer,
// and also addresses a specific regression described in issue #63896.
// This makes printed token streams look slightly nicer,
// and also addresses some specific regressions described in #63896 and #73345.
fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
if let TokenTree::Token(token) = prev {
if let token::DocComment(s) = token.kind {
return !s.as_str().starts_with("//");
}
}
match tt {
TokenTree::Token(token) => match token.kind {
token::Comma => false,
@ -163,7 +168,14 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
},
_ => true,
},
_ => true,
TokenTree::Delimited(_, DelimToken::Bracket, _) => match prev {
TokenTree::Token(token) => match token.kind {
token::Pound => false,
_ => true,
},
_ => true,
},
TokenTree::Delimited(..) => true,
}
}

View File

@ -4,14 +4,14 @@ use crate::module::DirectoryOwnership;
use rustc_ast::ast::{self, Attribute, NodeId, PatKind};
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::ptr::P;
use rustc_ast::token::{self, FlattenGroup};
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
use rustc_ast::token;
use rustc_ast::tokenstream::{self, TokenStream};
use rustc_ast::visit::{AssocCtxt, Visitor};
use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::{self, Lrc};
use rustc_errors::{DiagnosticBuilder, ErrorReported};
use rustc_parse::{self, parser, MACRO_ARGUMENTS};
use rustc_parse::{self, nt_to_tokenstream, parser, MACRO_ARGUMENTS};
use rustc_session::{parse::ParseSess, Limit};
use rustc_span::def_id::DefId;
use rustc_span::edition::Edition;
@ -120,10 +120,7 @@ impl Annotatable {
}
}
crate fn into_tokens(self) -> TokenStream {
// `Annotatable` can be converted into tokens directly, but we
// are packing it into a nonterminal as a piece of AST to make
// the produced token stream look nicer in pretty-printed form.
crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
let nt = match self {
Annotatable::Item(item) => token::NtItem(item),
Annotatable::TraitItem(item) | Annotatable::ImplItem(item) => {
@ -142,7 +139,7 @@ impl Annotatable {
| Annotatable::StructField(..)
| Annotatable::Variant(..) => panic!("unexpected annotatable"),
};
TokenTree::token(token::Interpolated(Lrc::new(nt), FlattenGroup::Yes), DUMMY_SP).into()
nt_to_tokenstream(&nt, sess, DUMMY_SP)
}
pub fn expect_item(self) -> P<ast::Item> {

View File

@ -705,7 +705,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
SyntaxExtensionKind::Attr(expander) => {
self.gate_proc_macro_input(&item);
self.gate_proc_macro_attr_item(span, &item);
let tokens = item.into_tokens();
let tokens = item.into_tokens(self.cx.parse_sess);
let attr_item = attr.unwrap_normal_item();
if let MacArgs::Eq(..) = attr_item.args {
self.cx.span_err(span, "key-value macro attributes are not supported");

View File

@ -3,9 +3,10 @@ use crate::proc_macro_server;
use rustc_ast::ast::{self, ItemKind, MetaItemKind, NestedMetaItem};
use rustc_ast::token::{self, FlattenGroup};
use rustc_ast::tokenstream::{self, TokenStream};
use rustc_ast::tokenstream::{TokenStream, TokenTree};
use rustc_data_structures::sync::Lrc;
use rustc_errors::{Applicability, ErrorReported};
use rustc_parse::nt_to_tokenstream;
use rustc_span::symbol::sym;
use rustc_span::{Span, DUMMY_SP};
@ -102,8 +103,13 @@ impl MultiItemModifier for ProcMacroDerive {
}
}
let token = token::Interpolated(Lrc::new(token::NtItem(item)), FlattenGroup::Yes);
let input = tokenstream::TokenTree::token(token, DUMMY_SP).into();
let item = token::NtItem(item);
let input = if item.pretty_printing_compatibility_hack() {
TokenTree::token(token::Interpolated(Lrc::new(item), FlattenGroup::Yes), DUMMY_SP)
.into()
} else {
nt_to_tokenstream(&item, ecx.parse_sess, DUMMY_SP)
};
let server = proc_macro_server::Rustc::new(ecx);
let stream = match self.client.run(&EXEC_STRATEGY, server, input) {

View File

@ -180,13 +180,17 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
tt!(Punct::new('#', false))
}
Interpolated(nt, flatten) => {
Interpolated(nt, _) => {
let stream = nt_to_tokenstream(&nt, sess, span);
TokenTree::Group(Group {
delimiter: Delimiter::None,
stream,
span: DelimSpan::from_single(span),
flatten,
flatten: if nt.pretty_printing_compatibility_hack() {
FlattenGroup::Yes
} else {
FlattenGroup::No
},
})
}

View File

@ -401,6 +401,7 @@ symbols! {
infer_outlives_requirements,
infer_static_outlives_requirements,
inline,
Input,
intel,
into_iter,
IntoIterator,
@ -588,6 +589,8 @@ symbols! {
proc_macro_mod,
proc_macro_non_items,
proc_macro_path_invoc,
ProceduralMasqueradeDummyType,
ProcMacroHack,
profiler_builtins,
profiler_runtime,
ptr_guaranteed_eq,

View File

@ -7,6 +7,6 @@ use proc_macro::TokenStream;
#[proc_macro_derive(A)]
pub fn derive(input: TokenStream) -> TokenStream {
let input = input.to_string();
assert!(input.contains("struct A;"));
assert!(input.contains("struct A ;"));
"struct B;".parse().unwrap()
}

View File

@ -1,3 +1,3 @@
async fn f(mut x: u8) { }
async fn g((mut x, y, mut z): (u8, u8, u8)) { }
async fn g(mut x: u8, (a, mut b, c): (u8, u8, u8), y: u8) { }
async fn f(mut x : u8) { }
async fn g((mut x, y, mut z) : (u8, u8, u8)) { }
async fn g(mut x : u8, (a, mut b, c) : (u8, u8, u8), y : u8) { }

View File

@ -11,11 +11,9 @@ use proc_macro::TokenStream;
pub fn attr_cfg(args: TokenStream, input: TokenStream) -> TokenStream {
let input_str = input.to_string();
assert_eq!(input_str, "fn outer() -> u8 {
#[cfg(foo)]
fn inner() -> u8 { 1 }
#[cfg(bar)]
fn inner() -> u8 { 2 }
assert_eq!(input_str, "fn outer() -> u8
{
#[cfg(foo)] fn inner() -> u8 { 1 } #[cfg(bar)] fn inner() -> u8 { 2 }
inner()
}");

View File

@ -10,14 +10,14 @@ use proc_macro::TokenStream;
#[proc_macro_attribute]
pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
assert!(attr.to_string().is_empty());
assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
assert_eq!(item.to_string(), "let string = \"Hello, world!\" ;");
item
}
#[proc_macro_attribute]
pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
assert!(attr.to_string().is_empty());
assert_eq!(item.to_string(), "println!(\"{}\", string);");
assert_eq!(item.to_string(), "println ! (\"{}\", string) ;");
item
}
@ -31,7 +31,7 @@ pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
#[proc_macro_attribute]
pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
assert!(attr.to_string().is_empty());
assert_eq!(item.to_string(), "println!(\"{}\", string)");
assert_eq!(item.to_string(), "println ! (\"{}\", string)");
item
}

View File

@ -10,14 +10,14 @@ use proc_macro::TokenStream;
#[proc_macro_attribute]
pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
assert!(attr.to_string().is_empty());
assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
assert_eq!(item.to_string(), "let string = \"Hello, world!\" ;");
item
}
#[proc_macro_attribute]
pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
assert!(attr.to_string().is_empty());
assert_eq!(item.to_string(), "println!(\"{}\", string);");
assert_eq!(item.to_string(), "println ! (\"{}\", string) ;");
item
}
@ -31,7 +31,7 @@ pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
#[proc_macro_attribute]
pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
assert!(attr.to_string().is_empty());
assert_eq!(item.to_string(), "println!(\"{}\", string)");
assert_eq!(item.to_string(), "println ! (\"{}\", string)");
item
}

View File

@ -10,6 +10,6 @@ use proc_macro::TokenStream;
#[proc_macro_derive(A)]
pub fn derive(input: TokenStream) -> TokenStream {
let input = input.to_string();
assert!(input.contains("struct A;"));
assert!(input.contains("struct A ;"));
"".parse().unwrap()
}

View File

@ -10,6 +10,6 @@ use proc_macro::TokenStream;
#[proc_macro_derive(AToB)]
pub fn derive(input: TokenStream) -> TokenStream {
let input = input.to_string();
assert_eq!(input, "struct A;");
assert_eq!(input, "struct A ;");
"struct B;".parse().unwrap()
}

View File

@ -10,7 +10,7 @@ use proc_macro::TokenStream;
#[proc_macro_derive(B, attributes(B, C))]
pub fn derive(input: TokenStream) -> TokenStream {
let input = input.to_string();
assert!(input.contains("#[B[arbitrary tokens]]"));
assert!(input.contains("#[B [arbitrary tokens]]"));
assert!(input.contains("struct B {"));
assert!(input.contains("#[C]"));
"".parse().unwrap()

View File

@ -10,6 +10,6 @@ use proc_macro::TokenStream;
#[proc_macro_derive(CToD)]
pub fn derive(input: TokenStream) -> TokenStream {
let input = input.to_string();
assert_eq!(input, "struct C;");
assert_eq!(input, "struct C ;");
"struct D;".parse().unwrap()
}

View File

@ -10,12 +10,12 @@ use proc_macro::TokenStream;
#[proc_macro_derive(AToB)]
pub fn derive1(input: TokenStream) -> TokenStream {
println!("input1: {:?}", input.to_string());
assert_eq!(input.to_string(), "struct A;");
assert_eq!(input.to_string(), "struct A ;");
"#[derive(BToC)] struct B;".parse().unwrap()
}
#[proc_macro_derive(BToC)]
pub fn derive2(input: TokenStream) -> TokenStream {
assert_eq!(input.to_string(), "struct B;");
assert_eq!(input.to_string(), "struct B ;");
"struct C;".parse().unwrap()
}

View File

@ -12,7 +12,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
let input = input.to_string();
assert!(input.contains("#[repr(C)]"));
assert!(input.contains("union Test {"));
assert!(input.contains("a: u8,"));
assert!(input.contains("a : u8,"));
assert!(input.contains("}"));
"".parse().unwrap()
}

View File

@ -1,15 +1,16 @@
// force-host
// no-prefer-dynamic
#![feature(proc_macro_quote)]
#![crate_type = "proc-macro"]
extern crate proc_macro;
use proc_macro::TokenStream;
use proc_macro::*;
// Outputs another copy of the struct. Useful for testing the tokens
// seen by the proc_macro.
#[proc_macro_derive(Double)]
pub fn derive(input: TokenStream) -> TokenStream {
format!("mod foo {{ {} }}", input.to_string()).parse().unwrap()
quote!(mod foo { $input })
}

View File

@ -11,7 +11,7 @@ use proc_macro::TokenStream;
#[proc_macro_derive(A)]
pub fn derive(input: TokenStream) -> TokenStream {
let input = input.to_string();
assert!(input.contains("struct A;"));
assert!(input.contains("struct A ;"));
r#"
impl A {
fn a(&self) {

View File

@ -1 +1 @@
input1: "struct A;"
input1: "struct A ;"

View File

@ -38,8 +38,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
span: #3 bytes(LO..HI),
},
]
PRINT-ATTR INPUT (DISPLAY): struct A(crate::S);
PRINT-ATTR RE-COLLECTED (DISPLAY): struct A($crate :: S) ;
PRINT-ATTR INPUT (DISPLAY): struct A($crate :: S) ;
PRINT-ATTR INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",

View File

@ -1,5 +1,4 @@
PRINT-ATTR INPUT (DISPLAY): struct A(identity!(crate :: S));
PRINT-ATTR RE-COLLECTED (DISPLAY): struct A(identity ! ($crate :: S)) ;
PRINT-ATTR INPUT (DISPLAY): struct A(identity ! ($crate :: S)) ;
PRINT-ATTR INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",
@ -54,8 +53,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
span: #3 bytes(LO..HI),
},
]
PRINT-ATTR INPUT (DISPLAY): struct B(identity!(::dollar_crate_external :: S));
PRINT-ATTR RE-COLLECTED (DISPLAY): struct B(identity ! ($crate :: S)) ;
PRINT-ATTR INPUT (DISPLAY): struct B(identity ! ($crate :: S)) ;
PRINT-ATTR INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",

View File

@ -38,8 +38,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
span: #3 bytes(LO..HI),
},
]
PRINT-ATTR INPUT (DISPLAY): struct A(crate::S);
PRINT-ATTR RE-COLLECTED (DISPLAY): struct A($crate :: S) ;
PRINT-ATTR INPUT (DISPLAY): struct A($crate :: S) ;
PRINT-ATTR INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",
@ -79,8 +78,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
span: #3 bytes(LO..HI),
},
]
PRINT-DERIVE INPUT (DISPLAY): struct D(crate::S);
PRINT-DERIVE RE-COLLECTED (DISPLAY): struct D($crate :: S) ;
PRINT-DERIVE INPUT (DISPLAY): struct D($crate :: S) ;
PRINT-DERIVE INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",
@ -160,8 +158,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
span: #13 bytes(LO..HI),
},
]
PRINT-ATTR INPUT (DISPLAY): struct A(::dollar_crate_external::S);
PRINT-ATTR RE-COLLECTED (DISPLAY): struct A($crate :: S) ;
PRINT-ATTR INPUT (DISPLAY): struct A($crate :: S) ;
PRINT-ATTR INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",
@ -201,8 +198,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
span: #13 bytes(LO..HI),
},
]
PRINT-DERIVE INPUT (DISPLAY): struct D(::dollar_crate_external::S);
PRINT-DERIVE RE-COLLECTED (DISPLAY): struct D($crate :: S) ;
PRINT-DERIVE INPUT (DISPLAY): struct D($crate :: S) ;
PRINT-DERIVE INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",

View File

@ -12,8 +12,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
span: #3 bytes(269..271),
},
]
PRINT-ATTR INPUT (DISPLAY): const A: u8 = 0;
PRINT-ATTR RE-COLLECTED (DISPLAY): const A : u8 = 0 ;
PRINT-ATTR INPUT (DISPLAY): const A : u8 = 0 ;
PRINT-ATTR INPUT (DEBUG): TokenStream [
Ident {
ident: "const",
@ -49,9 +48,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
span: #0 bytes(0..0),
},
]
PRINT-DERIVE INPUT (DISPLAY): struct A {
}
PRINT-DERIVE RE-COLLECTED (DISPLAY): struct A { }
PRINT-DERIVE INPUT (DISPLAY): struct A { }
PRINT-DERIVE INPUT (DEBUG): TokenStream [
Ident {
ident: "struct",

View File

@ -17,28 +17,27 @@ macro_rules! checker {
}
}
checker!(attr_extern, r#"extern "C" {
fn ffi(#[a1] arg1: i32, #[a2] ...);
}"#);
checker!(attr_extern_cvar, r#"unsafe extern "C" fn cvar(arg1: i32, #[a1] mut args: ...) { }"#);
checker!(attr_alias, "type Alias = fn(#[a1] u8, #[a2] ...);");
checker!(attr_free, "fn free(#[a1] arg1: u8) { let lam = |#[a2] W(x), #[a3] y| (); }");
checker!(attr_inherent_1, "fn inherent1(#[a1] self, #[a2] arg1: u8) { }");
checker!(attr_inherent_2, "fn inherent2(#[a1] &self, #[a2] arg1: u8) { }");
checker!(attr_inherent_3, "fn inherent3<'a>(#[a1] &'a mut self, #[a2] arg1: u8) { }");
checker!(attr_inherent_4, "fn inherent4<'a>(#[a1] self: Box<Self>, #[a2] arg1: u8) { }");
checker!(attr_inherent_issue_64682, "fn inherent5(#[a1] #[a2] arg1: u8, #[a3] arg2: u8) { }");
checker!(attr_trait_1, "fn trait1(#[a1] self, #[a2] arg1: u8);");
checker!(attr_trait_2, "fn trait2(#[a1] &self, #[a2] arg1: u8);");
checker!(attr_trait_3, "fn trait3<'a>(#[a1] &'a mut self, #[a2] arg1: u8);");
checker!(attr_trait_4, "fn trait4<'a>(#[a1] self: Box<Self>, #[a2] arg1: u8, #[a3] Vec<u8>);");
checker!(attr_trait_issue_64682, "fn trait5(#[a1] #[a2] arg1: u8, #[a3] arg2: u8);");
checker!(rename_params, r#"impl Foo {
fn hello(#[angery(true)] a: i32, #[a2] b: i32, #[what = "how"] c: u32) { }
fn hello2(#[a1] #[a2] a: i32, #[what = "how"] b: i32,
#[angery(true)] c: u32) {
}
fn hello_self(#[a1] #[a2] &self, #[a1] #[a2] a: i32,
#[what = "how"] b: i32, #[angery(true)] c: u32) {
}
checker!(attr_extern, r#"extern "C" { fn ffi(#[a1] arg1 : i32, #[a2] ...) ; }"#);
checker!(attr_extern_cvar, r#"unsafe extern "C" fn cvar(arg1 : i32, #[a1] mut args : ...) { }"#);
checker!(attr_alias, "type Alias = fn(#[a1] u8, #[a2] ...) ;");
checker!(attr_free, "fn free(#[a1] arg1 : u8) { let lam = | #[a2] W(x), #[a3] y | () ; }");
checker!(attr_inherent_1, "fn inherent1(#[a1] self, #[a2] arg1 : u8) { }");
checker!(attr_inherent_2, "fn inherent2(#[a1] & self, #[a2] arg1 : u8) { }");
checker!(attr_inherent_3, "fn inherent3 < 'a > (#[a1] & 'a mut self, #[a2] arg1 : u8) { }");
checker!(attr_inherent_4, "fn inherent4 < 'a > (#[a1] self : Box < Self >, #[a2] arg1 : u8) { }");
checker!(attr_inherent_issue_64682, "fn inherent5(#[a1] #[a2] arg1 : u8, #[a3] arg2 : u8) { }");
checker!(attr_trait_1, "fn trait1(#[a1] self, #[a2] arg1 : u8) ;");
checker!(attr_trait_2, "fn trait2(#[a1] & self, #[a2] arg1 : u8) ;");
checker!(attr_trait_3, "fn trait3 < 'a > (#[a1] & 'a mut self, #[a2] arg1 : u8) ;");
checker!(attr_trait_4, r#"fn trait4 < 'a >
(#[a1] self : Box < Self >, #[a2] arg1 : u8, #[a3] Vec < u8 >) ;"#);
checker!(attr_trait_issue_64682, "fn trait5(#[a1] #[a2] arg1 : u8, #[a3] arg2 : u8) ;");
checker!(rename_params, r#"impl Foo
{
fn hello(#[angery(true)] a : i32, #[a2] b : i32, #[what = "how"] c : u32)
{ } fn
hello2(#[a1] #[a2] a : i32, #[what = "how"] b : i32, #[angery(true)] c :
u32) { } fn
hello_self(#[a1] #[a2] & self, #[a1] #[a2] a : i32, #[what = "how"] b :
i32, #[angery(true)] c : u32) { }
}"#);