mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use reparsed TokenStream if we captured any inner attributes

Browse Source 
Fixes #78675

We now bail out of `prepend_attrs` if we ended up capturing any inner
attributes (which can happen in several places, due to token capturing
for `macro_rules!` arguments.
This commit is contained in:
Aaron Hill 2020-11-02 13:22:03 -05:00
parent 499ebcfdf3
commit 22383b32b8
No known key found for this signature in database
GPG Key ID: B4087E510E98B164
4 changed files with 132 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
compiler/rustc_parse/src/lib.rs
View File

@ -611,11 +611,11 @@ fn prepend_attrs(
}
let mut builder = tokenstream::TokenStreamBuilder::new();
for attr in attrs {
assert_eq!(
attr.style,
ast::AttrStyle::Outer,
"inner attributes should prevent cached tokens from existing"
);
// FIXME: Correctly handle tokens for inner attributes.
// For now, we fall back to reparsing the original AST node
if attr.style == ast::AttrStyle::Inner {
return None;
}
builder.push(
attr.tokens
.as_ref()

33
compiler/rustc_parse/src/parser/item.rs
View File

@ -7,7 +7,7 @@
use rustc_ast::ptr::P;
use rustc_ast::token::{self, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
use rustc_ast::{self as ast, AttrStyle, AttrVec, Attribute, DUMMY_NODE_ID};
use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID};
use rustc_ast::{AssocItem, AssocItemKind, ForeignItemKind, Item, ItemKind, Mod};
use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind};
use rustc_ast::{BindingMode, Block, FnDecl, FnSig, Param, SelfKind};
@ -127,34 +127,19 @@ pub(super) fn parse_item_common(
let (mut item, tokens) = if needs_tokens {
let (item, tokens) = self.collect_tokens(parse_item)?;
(item, Some(tokens))
(item, tokens)
} else {
(parse_item(self)?, None)
};
self.unclosed_delims.append(&mut unclosed_delims);
// Once we've parsed an item and recorded the tokens we got while
// parsing we may want to store `tokens` into the item we're about to
// return. Note, though, that we specifically didn't capture tokens
// related to outer attributes. The `tokens` field here may later be
// used with procedural macros to convert this item back into a token
// stream, but during expansion we may be removing attributes as we go
// along.
//
// If we've got inner attributes then the `tokens` we've got above holds
// these inner attributes. If an inner attribute is expanded we won't
// actually remove it from the token stream, so we'll just keep yielding
// it (bad!). To work around this case for now we just avoid recording
// `tokens` if we detect any inner attributes. This should help keep
// expansion correct, but we should fix this bug one day!
if let Some(tokens) = tokens {
if let Some(item) = &mut item {
if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) {
item.tokens = tokens;
}
if let Some(item) = &mut item {
// If we captured tokens during parsing (due to encountering an `NtItem`),
// use those instead
if item.tokens.is_none() {
item.tokens = tokens;
}
}
self.unclosed_delims.append(&mut unclosed_delims);
Ok(item)
}

32
src/test/ui/proc-macro/issue-78675-captured-inner-attrs.rs Normal file
View File

@ -0,0 +1,32 @@
// check-pass
// edition:2018
// compile-flags: -Z span-debug
// aux-build:test-macros.rs
#![no_std] // Don't load unnecessary hygiene information from std
extern crate std;
#[macro_use] extern crate test_macros;
macro_rules! foo {(
#[fake_attr]
$item:item
) => (
$item
)}
macro_rules! outer {($item:item) => (
print_bang! { // Identity proc-macro
foo! {
#[fake_attr]
$item
}
}
)}
outer! {
mod bar {
//! Foo
}
}
fn main() {}

86
src/test/ui/proc-macro/issue-78675-captured-inner-attrs.stdout Normal file
View File

@ -0,0 +1,86 @@
PRINT-BANG INPUT (DISPLAY): foo ! { #[fake_attr] mod bar {
#![doc = r" Foo"]
} }
PRINT-BANG INPUT (DEBUG): TokenStream [
Ident {
ident: "foo",
span: $DIR/issue-78675-captured-inner-attrs.rs:20:9: 20:12 (#4),
},
Punct {
ch: '!',
spacing: Alone,
span: $DIR/issue-78675-captured-inner-attrs.rs:20:12: 20:13 (#4),
},
Group {
delimiter: Brace,
stream: TokenStream [
Punct {
ch: '#',
spacing: Alone,
span: $DIR/issue-78675-captured-inner-attrs.rs:21:13: 21:14 (#4),
},
Group {
delimiter: Bracket,
stream: TokenStream [
Ident {
ident: "fake_attr",
span: $DIR/issue-78675-captured-inner-attrs.rs:21:15: 21:24 (#4),
},
],
span: $DIR/issue-78675-captured-inner-attrs.rs:21:14: 21:25 (#4),
},
Group {
delimiter: None,
stream: TokenStream [
Ident {
ident: "mod",
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
Ident {
ident: "bar",
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
Group {
delimiter: Brace,
stream: TokenStream [
Punct {
ch: '#',
spacing: Joint,
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
Punct {
ch: '!',
spacing: Alone,
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
Group {
delimiter: Bracket,
stream: TokenStream [
Ident {
ident: "doc",
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
Punct {
ch: '=',
spacing: Alone,
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
Literal {
kind: StrRaw(0),
symbol: " Foo",
suffix: None,
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
],
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
],
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
],
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
},
],
span: $DIR/issue-78675-captured-inner-attrs.rs:20:14: 23:10 (#4),
},
]