mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rollup merge of #101602 - nnethercote:AttrTokenStream, r=petrochenkov

Browse Source 
Streamline `AttrAnnotatedTokenStream`

r? ```@petrochenkov```
This commit is contained in:
Dylan DPC 2022-09-13 16:51:31 +05:30 committed by GitHub
commit db75d7e14b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 181 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
compiler/rustc_ast/src/ast.rs
View File

@ -24,7 +24,7 @@
use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter};
use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream};
use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream};
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_data_structures::sync::Lrc;
@ -91,7 +91,7 @@ pub struct Path {
/// The segments in the path: the things separated by `::`.
/// Global paths begin with `kw::PathRoot`.
pub segments: Vec<PathSegment>,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
impl PartialEq<Symbol> for Path {
@ -534,7 +534,7 @@ pub struct Block {
/// Distinguishes between `unsafe { ... }` and `{ ... }`.
pub rules: BlockCheckMode,
pub span: Span,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
/// The following *isn't* a parse error, but will cause multiple errors in following stages.
/// ```compile_fail
/// let x = {
@ -553,7 +553,7 @@ pub struct Pat {
pub id: NodeId,
pub kind: PatKind,
pub span: Span,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
impl Pat {
@ -937,8 +937,8 @@ pub fn has_trailing_semicolon(&self) -> bool {
/// a trailing semicolon.
///
/// This only modifies the parsed AST struct, not the attached
/// `LazyTokenStream`. The parser is responsible for calling
/// `CreateTokenStream::add_trailing_semi` when there is actually
/// `LazyAttrTokenStream`. The parser is responsible for calling
/// `ToAttrTokenStream::add_trailing_semi` when there is actually
/// a semicolon in the tokenstream.
pub fn add_trailing_semicolon(mut self) -> Self {
self.kind = match self.kind {
@ -984,7 +984,7 @@ pub struct MacCallStmt {
pub mac: P<MacCall>,
pub style: MacStmtStyle,
pub attrs: AttrVec,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
@ -1009,7 +1009,7 @@ pub struct Local {
pub kind: LocalKind,
pub span: Span,
pub attrs: AttrVec,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
#[derive(Clone, Encodable, Decodable, Debug)]
@ -1108,7 +1108,7 @@ pub struct Expr {
pub kind: ExprKind,
pub span: Span,
pub attrs: AttrVec,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
impl Expr {
@ -1967,7 +1967,7 @@ pub struct Ty {
pub id: NodeId,
pub kind: TyKind,
pub span: Span,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
impl Clone for Ty {
@ -2532,7 +2532,7 @@ fn decode(_: &mut D) -> AttrId {
pub struct AttrItem {
pub path: Path,
pub args: MacArgs,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
/// A list of attributes.
@ -2552,7 +2552,7 @@ pub struct Attribute {
#[derive(Clone, Encodable, Decodable, Debug)]
pub struct NormalAttr {
pub item: AttrItem,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
#[derive(Clone, Encodable, Decodable, Debug)]
@ -2603,7 +2603,7 @@ pub fn new(generic_params: Vec<GenericParam>, path: Path, span: Span) -> Self {
pub struct Visibility {
pub kind: VisibilityKind,
pub span: Span,
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
#[derive(Clone, Encodable, Decodable, Debug)]
@ -2689,7 +2689,7 @@ pub struct Item<K = ItemKind> {
///
/// Note that the tokens here do not include the outer attributes, but will
/// include inner attributes.
pub tokens: Option<LazyTokenStream>,
pub tokens: Option<LazyAttrTokenStream>,
}
impl Item {

38
compiler/rustc_ast/src/ast_traits.rs
View File

@ -4,7 +4,7 @@
use crate::ptr::P;
use crate::token::Nonterminal;
use crate::tokenstream::LazyTokenStream;
use crate::tokenstream::LazyAttrTokenStream;
use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
use crate::{AssocItem, Expr, ForeignItem, Item, NodeId};
use crate::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
@ -124,18 +124,18 @@ fn span(&self) -> Span {
/// A trait for AST nodes having (or not having) collected tokens.
pub trait HasTokens {
fn tokens(&self) -> Option<&LazyTokenStream>;
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
fn tokens(&self) -> Option<&LazyAttrTokenStream>;
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>>;
}
macro_rules! impl_has_tokens {
($($T:ty),+ $(,)?) => {
$(
impl HasTokens for $T {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
self.tokens.as_ref()
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
Some(&mut self.tokens)
}
}
@ -147,10 +147,10 @@ macro_rules! impl_has_tokens_none {
($($T:ty),+ $(,)?) => {
$(
impl HasTokens for $T {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
None
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
None
}
}
@ -162,25 +162,25 @@ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
impl_has_tokens_none!(Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant);
impl<T: AstDeref<Target: HasTokens>> HasTokens for T {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
self.ast_deref().tokens()
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
self.ast_deref_mut().tokens_mut()
}
}
impl<T: HasTokens> HasTokens for Option<T> {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
self.as_ref().and_then(|inner| inner.tokens())
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
self.as_mut().and_then(|inner| inner.tokens_mut())
}
}
impl HasTokens for StmtKind {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
match self {
StmtKind::Local(local) => local.tokens.as_ref(),
StmtKind::Item(item) => item.tokens(),
@ -189,7 +189,7 @@ fn tokens(&self) -> Option<&LazyTokenStream> {
StmtKind::MacCall(mac) => mac.tokens.as_ref(),
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
match self {
StmtKind::Local(local) => Some(&mut local.tokens),
StmtKind::Item(item) => item.tokens_mut(),
@ -201,16 +201,16 @@ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
}
impl HasTokens for Stmt {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
self.kind.tokens()
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
self.kind.tokens_mut()
}
}
impl HasTokens for Attribute {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
match &self.kind {
AttrKind::Normal(normal) => normal.tokens.as_ref(),
kind @ AttrKind::DocComment(..) => {
@ -218,7 +218,7 @@ fn tokens(&self) -> Option<&LazyTokenStream> {
}
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
Some(match &mut self.kind {
AttrKind::Normal(normal) => &mut normal.tokens,
kind @ AttrKind::DocComment(..) => {
@ -229,7 +229,7 @@ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
}
impl HasTokens for Nonterminal {
fn tokens(&self) -> Option<&LazyTokenStream> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
match self {
Nonterminal::NtItem(item) => item.tokens(),
Nonterminal::NtStmt(stmt) => stmt.tokens(),
@ -243,7 +243,7 @@ fn tokens(&self) -> Option<&LazyTokenStream> {
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
match self {
Nonterminal::NtItem(item) => item.tokens_mut(),
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),

19
compiler/rustc_ast/src/attr/mod.rs
View File

@ -7,9 +7,8 @@
use crate::ast::{Path, PathSegment};
use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, Token};
use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
use crate::tokenstream::{LazyTokenStream, TokenStream};
use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
use crate::util::comments;
use rustc_index::bit_set::GrowableBitSet;
@ -296,20 +295,18 @@ pub fn meta_kind(&self) -> Option<MetaItemKind> {
}
}
pub fn tokens(&self) -> AttrAnnotatedTokenStream {
pub fn tokens(&self) -> TokenStream {
match self.kind {
AttrKind::Normal(ref normal) => normal
.tokens
.as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
.create_token_stream(),
AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from((
AttrAnnotatedTokenTree::Token(Token::new(
token::DocComment(comment_kind, self.style, data),
self.span,
)),
.to_attr_token_stream()
.to_tokenstream(),
AttrKind::DocComment(comment_kind, data) => TokenStream::new(vec![TokenTree::Token(
Token::new(token::DocComment(comment_kind, self.style, data), self.span),
Spacing::Alone,
)),
)]),
}
}
}
@ -356,7 +353,7 @@ pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attri
pub fn mk_attr_from_item(
item: AttrItem,
tokens: Option<LazyTokenStream>,
tokens: Option<LazyAttrTokenStream>,
style: AttrStyle,
span: Span,
) -> Attribute {

1
compiler/rustc_ast/src/lib.rs
View File

@ -15,6 +15,7 @@
#![feature(if_let_guard)]
#![cfg_attr(bootstrap, feature(label_break_value))]
#![feature(let_chains)]
#![feature(let_else)]
#![feature(min_specialization)]
#![feature(negative_impls)]
#![feature(slice_internals)]

30
compiler/rustc_ast/src/mut_visit.rs
View File

@ -642,17 +642,17 @@ pub fn noop_visit_meta_item<T: MutVisitor>(mi: &mut MetaItem, vis: &mut T) {
}
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) {
pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
match tt {
AttrAnnotatedTokenTree::Token(token) => {
AttrTokenTree::Token(token, _) => {
visit_token(token, vis);
}
AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
vis.visit_span(open);
vis.visit_span(close);
visit_attr_annotated_tts(tts, vis);
visit_attr_tts(tts, vis);
}
AttrAnnotatedTokenTree::Attributes(data) => {
AttrTokenTree::Attributes(data) => {
for attr in &mut *data.attrs {
match &mut attr.kind {
AttrKind::Normal(normal) => {
@ -690,27 +690,27 @@ pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T)
}
}
pub fn visit_attr_annotated_tts<T: MutVisitor>(
AttrAnnotatedTokenStream(tts): &mut AttrAnnotatedTokenStream,
vis: &mut T,
) {
pub fn visit_attr_tts<T: MutVisitor>(AttrTokenStream(tts): &mut AttrTokenStream, vis: &mut T) {
if T::VISIT_TOKENS && !tts.is_empty() {
let tts = Lrc::make_mut(tts);
visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis));
visit_vec(tts, |tree| visit_attr_tt(tree, vis));
}
}
pub fn visit_lazy_tts_opt_mut<T: MutVisitor>(lazy_tts: Option<&mut LazyTokenStream>, vis: &mut T) {
pub fn visit_lazy_tts_opt_mut<T: MutVisitor>(
lazy_tts: Option<&mut LazyAttrTokenStream>,
vis: &mut T,
) {
if T::VISIT_TOKENS {
if let Some(lazy_tts) = lazy_tts {
let mut tts = lazy_tts.create_token_stream();
visit_attr_annotated_tts(&mut tts, vis);
*lazy_tts = LazyTokenStream::new(tts);
let mut tts = lazy_tts.to_attr_token_stream();
visit_attr_tts(&mut tts, vis);
*lazy_tts = LazyAttrTokenStream::new(tts);
}
}
}
pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyAttrTokenStream>, vis: &mut T) {
visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis);
}

126
compiler/rustc_ast/src/tokenstream.rs
View File

@ -121,12 +121,12 @@ fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
}
}
pub trait CreateTokenStream: sync::Send + sync::Sync {
fn create_token_stream(&self) -> AttrAnnotatedTokenStream;
pub trait ToAttrTokenStream: sync::Send + sync::Sync {
fn to_attr_token_stream(&self) -> AttrTokenStream;
}
impl CreateTokenStream for AttrAnnotatedTokenStream {
fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
impl ToAttrTokenStream for AttrTokenStream {
fn to_attr_token_stream(&self) -> AttrTokenStream {
self.clone()
}
}
@ -135,68 +135,68 @@ fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
/// of an actual `TokenStream` until it is needed.
/// `Box` is here only to reduce the structure size.
#[derive(Clone)]
pub struct LazyTokenStream(Lrc<Box<dyn CreateTokenStream>>);
pub struct LazyAttrTokenStream(Lrc<Box<dyn ToAttrTokenStream>>);
impl LazyTokenStream {
pub fn new(inner: impl CreateTokenStream + 'static) -> LazyTokenStream {
LazyTokenStream(Lrc::new(Box::new(inner)))
impl LazyAttrTokenStream {
pub fn new(inner: impl ToAttrTokenStream + 'static) -> LazyAttrTokenStream {
LazyAttrTokenStream(Lrc::new(Box::new(inner)))
}
pub fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
self.0.create_token_stream()
pub fn to_attr_token_stream(&self) -> AttrTokenStream {
self.0.to_attr_token_stream()
}
}
impl fmt::Debug for LazyTokenStream {
impl fmt::Debug for LazyAttrTokenStream {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "LazyTokenStream({:?})", self.create_token_stream())
write!(f, "LazyAttrTokenStream({:?})", self.to_attr_token_stream())
}
}
impl<S: Encoder> Encodable<S> for LazyTokenStream {
impl<S: Encoder> Encodable<S> for LazyAttrTokenStream {
fn encode(&self, s: &mut S) {
// Used by AST json printing.
Encodable::encode(&self.create_token_stream(), s);
Encodable::encode(&self.to_attr_token_stream(), s);
}
}
impl<D: Decoder> Decodable<D> for LazyTokenStream {
impl<D: Decoder> Decodable<D> for LazyAttrTokenStream {
fn decode(_d: &mut D) -> Self {
panic!("Attempted to decode LazyTokenStream");
panic!("Attempted to decode LazyAttrTokenStream");
}
}
impl<CTX> HashStable<CTX> for LazyTokenStream {
impl<CTX> HashStable<CTX> for LazyAttrTokenStream {
fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
panic!("Attempted to compute stable hash for LazyTokenStream");
panic!("Attempted to compute stable hash for LazyAttrTokenStream");
}
}
/// A `AttrAnnotatedTokenStream` is similar to a `TokenStream`, but with extra
/// An `AttrTokenStream` is similar to a `TokenStream`, but with extra
/// information about the tokens for attribute targets. This is used
/// during expansion to perform early cfg-expansion, and to process attributes
/// during proc-macro invocations.
#[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing)>>);
pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>);
/// Like `TokenTree`, but for `AttrAnnotatedTokenStream`
/// Like `TokenTree`, but for `AttrTokenStream`.
#[derive(Clone, Debug, Encodable, Decodable)]
pub enum AttrAnnotatedTokenTree {
Token(Token),
Delimited(DelimSpan, Delimiter, AttrAnnotatedTokenStream),
pub enum AttrTokenTree {
Token(Token, Spacing),
Delimited(DelimSpan, Delimiter, AttrTokenStream),
/// Stores the attributes for an attribute target,
/// along with the tokens for that attribute target.
/// See `AttributesData` for more information
Attributes(AttributesData),
}
impl AttrAnnotatedTokenStream {
pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream {
AttrAnnotatedTokenStream(Lrc::new(tokens))
impl AttrTokenStream {
pub fn new(tokens: Vec<AttrTokenTree>) -> AttrTokenStream {
AttrTokenStream(Lrc::new(tokens))
}
/// Converts this `AttrAnnotatedTokenStream` to a plain `TokenStream
/// During conversion, `AttrAnnotatedTokenTree::Attributes` get 'flattened'
/// Converts this `AttrTokenStream` to a plain `TokenStream`.
/// During conversion, `AttrTokenTree::Attributes` get 'flattened'
/// back to a `TokenStream` of the form `outer_attr attr_target`.
/// If there are inner attributes, they are inserted into the proper
/// place in the attribute target tokens.
@ -204,31 +204,27 @@ pub fn to_tokenstream(&self) -> TokenStream {
let trees: Vec<_> = self
.0
.iter()
.flat_map(|tree| match &tree.0 {
AttrAnnotatedTokenTree::Token(inner) => {
smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter()
.flat_map(|tree| match &tree {
AttrTokenTree::Token(inner, spacing) => {
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
}
AttrAnnotatedTokenTree::Delimited(span, delim, stream) => {
AttrTokenTree::Delimited(span, delim, stream) => {
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
.into_iter()
}
AttrAnnotatedTokenTree::Attributes(data) => {
AttrTokenTree::Attributes(data) => {
let mut outer_attrs = Vec::new();
let mut inner_attrs = Vec::new();
for attr in &data.attrs {
match attr.style {
crate::AttrStyle::Outer => {
outer_attrs.push(attr);
}
crate::AttrStyle::Inner => {
inner_attrs.push(attr);
}
crate::AttrStyle::Outer => outer_attrs.push(attr),
crate::AttrStyle::Inner => inner_attrs.push(attr),
}
}
let mut target_tokens: Vec<_> = data
.tokens
.create_token_stream()
.to_attr_token_stream()
.to_tokenstream()
.0
.iter()
@ -239,9 +235,9 @@ pub fn to_tokenstream(&self) -> TokenStream {
// Check the last two trees (to account for a trailing semi)
for tree in target_tokens.iter_mut().rev().take(2) {
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
// Inner attributes are only supported on extern blocks, functions, impls,
// and modules. All of these have their inner attributes placed at
// the beginning of the rightmost outermost braced group:
// Inner attributes are only supported on extern blocks, functions,
// impls, and modules. All of these have their inner attributes
// placed at the beginning of the rightmost outermost braced group:
// e.g. fn foo() { #![my_attr} }
//
// Therefore, we can insert them back into the right location
@ -255,7 +251,7 @@ pub fn to_tokenstream(&self) -> TokenStream {
let mut builder = TokenStreamBuilder::new();
for inner_attr in inner_attrs {
builder.push(inner_attr.tokens().to_tokenstream());
builder.push(inner_attr.tokens());
}
builder.push(delim_tokens.clone());
*tree = TokenTree::Delimited(*span, *delim, builder.build());
@ -273,7 +269,7 @@ pub fn to_tokenstream(&self) -> TokenStream {
let mut flat: SmallVec<[_; 1]> = SmallVec::new();
for attr in outer_attrs {
// FIXME: Make this more efficient
flat.extend(attr.tokens().to_tokenstream().0.clone().iter().cloned());
flat.extend(attr.tokens().0.clone().iter().cloned());
}
flat.extend(target_tokens);
flat.into_iter()
@ -300,7 +296,7 @@ pub struct AttributesData {
pub attrs: AttrVec,
/// The underlying tokens for the attribute target that `attrs`
/// are applied to
pub tokens: LazyTokenStream,
pub tokens: LazyAttrTokenStream,
}
/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
@ -363,12 +359,6 @@ pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
}
}
impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream {
fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream {
AttrAnnotatedTokenStream::new(vec![(tree, spacing)])
}
}
impl iter::FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
@ -420,22 +410,6 @@ pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F)
TokenStream(Lrc::new(self.0.iter().enumerate().map(|(i, tree)| f(i, tree)).collect()))
}
fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option<TokenStream> {
let tokens = node.tokens()?;
let attrs = node.attrs();
let attr_annotated = if attrs.is_empty() {
tokens.create_token_stream()
} else {
let attr_data =
AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
AttrAnnotatedTokenStream::new(vec![(
AttrAnnotatedTokenTree::Attributes(attr_data),
Spacing::Alone,
)])
};
Some(attr_annotated.to_tokenstream())
}
// Create a token stream containing a single token with alone spacing.
pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
@ -452,8 +426,18 @@ pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenSt
}
pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
TokenStream::opt_from_ast(node)
.unwrap_or_else(|| panic!("missing tokens for node at {:?}: {:?}", node.span(), node))
let Some(tokens) = node.tokens() else {
panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
};
let attrs = node.attrs();
let attr_stream = if attrs.is_empty() {
tokens.to_attr_token_stream()
} else {
let attr_data =
AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)])
};
attr_stream.to_tokenstream()
}
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {

6
compiler/rustc_builtin_macros/src/cfg_eval.rs
View File

@ -188,14 +188,14 @@ fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option<Anno
let orig_tokens = annotatable.to_tokens().flattened();
// Re-parse the tokens, setting the `capture_cfg` flag to save extra information
// to the captured `AttrAnnotatedTokenStream` (specifically, we capture
// `AttrAnnotatedTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`)
// to the captured `AttrTokenStream` (specifically, we capture
// `AttrTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`)
let mut parser =
rustc_parse::stream_to_parser(&self.cfg.sess.parse_sess, orig_tokens, None);
parser.capture_cfg = true;
annotatable = parse_annotatable_with(&mut parser);
// Now that we have our re-parsed `AttrAnnotatedTokenStream`, recursively configuring
// Now that we have our re-parsed `AttrTokenStream`, recursively configuring
// our attribute target will correctly the tokens as well.
flat_map_annotatable(self, annotatable)
}

58
compiler/rustc_expand/src/config.rs
View File

@ -2,9 +2,9 @@
use rustc_ast::ptr::P;
use rustc_ast::token::{Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree};
use rustc_ast::tokenstream::{DelimSpan, Spacing};
use rustc_ast::tokenstream::{LazyTokenStream, TokenTree};
use rustc_ast::tokenstream::{LazyAttrTokenStream, TokenTree};
use rustc_ast::NodeId;
use rustc_ast::{self as ast, AttrStyle, Attribute, HasAttrs, HasTokens, MetaItem};
use rustc_attr as attr;
@ -259,8 +259,8 @@ pub fn configure<T: HasAttrs + HasTokens>(&self, mut node: T) -> Option<T> {
fn try_configure_tokens<T: HasTokens>(&self, node: &mut T) {
if self.config_tokens {
if let Some(Some(tokens)) = node.tokens_mut() {
let attr_annotated_tokens = tokens.create_token_stream();
*tokens = LazyTokenStream::new(self.configure_tokens(&attr_annotated_tokens));
let attr_stream = tokens.to_attr_token_stream();
*tokens = LazyAttrTokenStream::new(self.configure_tokens(&attr_stream));
}
}
}
@ -270,16 +270,16 @@ fn configure_krate_attrs(&self, mut attrs: ast::AttrVec) -> Option<ast::AttrVec>
if self.in_cfg(&attrs) { Some(attrs) } else { None }
}
/// Performs cfg-expansion on `stream`, producing a new `AttrAnnotatedTokenStream`.
/// Performs cfg-expansion on `stream`, producing a new `AttrTokenStream`.
/// This is only used during the invocation of `derive` proc-macros,
/// which require that we cfg-expand their entire input.
/// Normal cfg-expansion operates on parsed AST nodes via the `configure` method
fn configure_tokens(&self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream {
fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool {
stream.0.iter().all(|(tree, _spacing)| match tree {
AttrAnnotatedTokenTree::Attributes(_) => false,
AttrAnnotatedTokenTree::Token(_) => true,
AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner),
fn configure_tokens(&self, stream: &AttrTokenStream) -> AttrTokenStream {
fn can_skip(stream: &AttrTokenStream) -> bool {
stream.0.iter().all(|tree| match tree {
AttrTokenTree::Attributes(_) => false,
AttrTokenTree::Token(..) => true,
AttrTokenTree::Delimited(_, _, inner) => can_skip(inner),
})
}
@ -290,36 +290,36 @@ fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool {
let trees: Vec<_> = stream
.0
.iter()
.flat_map(|(tree, spacing)| match tree.clone() {
AttrAnnotatedTokenTree::Attributes(mut data) => {
.flat_map(|tree| match tree.clone() {
AttrTokenTree::Attributes(mut data) => {
data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
if self.in_cfg(&data.attrs) {
data.tokens = LazyTokenStream::new(
self.configure_tokens(&data.tokens.create_token_stream()),
data.tokens = LazyAttrTokenStream::new(
self.configure_tokens(&data.tokens.to_attr_token_stream()),
);
Some((AttrAnnotatedTokenTree::Attributes(data), *spacing)).into_iter()
Some(AttrTokenTree::Attributes(data)).into_iter()
} else {
None.into_iter()
}
}
AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => {
AttrTokenTree::Delimited(sp, delim, mut inner) => {
inner = self.configure_tokens(&inner);
Some((AttrAnnotatedTokenTree::Delimited(sp, delim, inner), *spacing))
Some(AttrTokenTree::Delimited(sp, delim, inner))
.into_iter()
}
AttrAnnotatedTokenTree::Token(ref token) if let TokenKind::Interpolated(ref nt) = token.kind => {
AttrTokenTree::Token(ref token, _) if let TokenKind::Interpolated(ref nt) = token.kind => {
panic!(
"Nonterminal should have been flattened at {:?}: {:?}",
token.span, nt
);
}
AttrAnnotatedTokenTree::Token(token) => {
Some((AttrAnnotatedTokenTree::Token(token), *spacing)).into_iter()
AttrTokenTree::Token(token, spacing) => {
Some(AttrTokenTree::Token(token, spacing)).into_iter()
}
})
.collect();
AttrAnnotatedTokenStream::new(trees)
AttrTokenStream::new(trees)
}
/// Parse and expand all `cfg_attr` attributes into a list of attributes
@ -388,7 +388,7 @@ fn expand_cfg_attr_item(
attr: &Attribute,
(item, item_span): (ast::AttrItem, Span),
) -> Attribute {
let orig_tokens = attr.tokens().to_tokenstream();
let orig_tokens = attr.tokens();
// We are taking an attribute of the form `#[cfg_attr(pred, attr)]`
// and producing an attribute of the form `#[attr]`. We
@ -404,26 +404,26 @@ fn expand_cfg_attr_item(
};
let pound_span = pound_token.span;
let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
let mut trees = vec![AttrTokenTree::Token(pound_token, Spacing::Alone)];
if attr.style == AttrStyle::Inner {
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else {
panic!("Bad tokens for attribute {:?}", attr);
};
trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));
trees.push(AttrTokenTree::Token(bang_token, Spacing::Alone));
}
// We don't really have a good span to use for the synthesized `[]`
// in `#[attr]`, so just use the span of the `#` token.
let bracket_group = AttrAnnotatedTokenTree::Delimited(
let bracket_group = AttrTokenTree::Delimited(
DelimSpan::from_single(pound_span),
Delimiter::Bracket,
item.tokens
.as_ref()
.unwrap_or_else(|| panic!("Missing tokens for {:?}", item))
.create_token_stream(),
.to_attr_token_stream(),
);
trees.push((bracket_group, Spacing::Alone));
let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees)));
trees.push(bracket_group);
let tokens = Some(LazyAttrTokenStream::new(AttrTokenStream::new(trees)));
let attr = attr::mk_attr_from_item(item, tokens, attr.style, item_span);
if attr.has_name(sym::crate_type) {
self.sess.parse_sess.buffer_lint(

4
compiler/rustc_parse/src/parser/attr.rs
View File

@ -301,9 +301,9 @@ pub(crate) fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> {
if let Some(attr) = attr {
let end_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap();
// If we are currently capturing tokens, mark the location of this inner attribute.
// If capturing ends up creating a `LazyTokenStream`, we will include
// If capturing ends up creating a `LazyAttrTokenStream`, we will include
// this replace range with it, removing the inner attribute from the final
// `AttrAnnotatedTokenStream`. Inner attributes are stored in the parsed AST note.
// `AttrTokenStream`. Inner attributes are stored in the parsed AST note.
// During macro expansion, they are selectively inserted back into the
// token stream (the first inner attribute is removed each time we invoke the
// corresponding macro).

51
compiler/rustc_parse/src/parser/attr_wrapper.rs
View File

@ -1,7 +1,7 @@
use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttributesData, CreateTokenStream};
use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream, Spacing};
use rustc_ast::tokenstream::{AttrTokenStream, AttributesData, ToAttrTokenStream};
use rustc_ast::tokenstream::{AttrTokenTree, DelimSpan, LazyAttrTokenStream, Spacing};
use rustc_ast::{self as ast};
use rustc_ast::{AttrVec, Attribute, HasAttrs, HasTokens};
use rustc_errors::PResult;
@ -88,7 +88,7 @@ fn has_cfg_or_cfg_attr(attrs: &[Attribute]) -> bool {
// This also makes `Parser` very cheap to clone, since
// there is no intermediate collection buffer to clone.
#[derive(Clone)]
struct LazyTokenStreamImpl {
struct LazyAttrTokenStreamImpl {
start_token: (Token, Spacing),
cursor_snapshot: TokenCursor,
num_calls: usize,
@ -97,10 +97,10 @@ struct LazyTokenStreamImpl {
}
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 144);
rustc_data_structures::static_assert_size!(LazyAttrTokenStreamImpl, 144);
impl CreateTokenStream for LazyTokenStreamImpl {
fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
impl ToAttrTokenStream for LazyAttrTokenStreamImpl {
fn to_attr_token_stream(&self) -> AttrTokenStream {
// The token produced by the final call to `{,inlined_}next` was not
// actually consumed by the callback. The combination of chaining the
// initial token and using `take` produces the desired result - we
@ -179,7 +179,7 @@ fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
impl<'a> Parser<'a> {
/// Records all tokens consumed by the provided callback,
/// including the current token. These tokens are collected
/// into a `LazyTokenStream`, and returned along with the result
/// into a `LazyAttrTokenStream`, and returned along with the result
/// of the callback.
///
/// Note: If your callback consumes an opening delimiter
@ -297,8 +297,8 @@ pub fn collect_tokens_trailing_token<R: HasAttrs + HasTokens>(
// If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens),
// then extend the range of captured tokens to include it, since the parser
// was not actually bumped past it. When the `LazyTokenStream` gets converted
// into an `AttrAnnotatedTokenStream`, we will create the proper token.
// was not actually bumped past it. When the `LazyAttrTokenStream` gets converted
// into an `AttrTokenStream`, we will create the proper token.
if self.token_cursor.break_last_token {
assert_eq!(
trailing,
@ -316,8 +316,8 @@ pub fn collect_tokens_trailing_token<R: HasAttrs + HasTokens>(
Box::new([])
} else {
// Grab any replace ranges that occur *inside* the current AST node.
// We will perform the actual replacement when we convert the `LazyTokenStream`
// to an `AttrAnnotatedTokenStream`
// We will perform the actual replacement when we convert the `LazyAttrTokenStream`
// to an `AttrTokenStream`.
let start_calls: u32 = cursor_snapshot_next_calls.try_into().unwrap();
self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end]
.iter()
@ -329,7 +329,7 @@ pub fn collect_tokens_trailing_token<R: HasAttrs + HasTokens>(
.collect()
};
let tokens = LazyTokenStream::new(LazyTokenStreamImpl {
let tokens = LazyAttrTokenStream::new(LazyAttrTokenStreamImpl {
start_token,
num_calls,
cursor_snapshot,
@ -392,12 +392,12 @@ pub fn collect_tokens_trailing_token<R: HasAttrs + HasTokens>(
fn make_token_stream(
mut iter: impl Iterator<Item = (FlatToken, Spacing)>,
break_last_token: bool,
) -> AttrAnnotatedTokenStream {
) -> AttrTokenStream {
#[derive(Debug)]
struct FrameData {
// This is `None` for the first frame, `Some` for all others.
open_delim_sp: Option<(Delimiter, Span)>,
inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
inner: Vec<AttrTokenTree>,
}
let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
let mut token_and_spacing = iter.next();
@ -418,48 +418,47 @@ struct FrameData {
open_delim, span
);
let dspan = DelimSpan::from_pair(open_sp, span);
let stream = AttrAnnotatedTokenStream::new(frame_data.inner);
let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream);
let stream = AttrTokenStream::new(frame_data.inner);
let delimited = AttrTokenTree::Delimited(dspan, delim, stream);
stack
.last_mut()
.unwrap_or_else(|| {
panic!("Bottom token frame is missing for token: {:?}", token)
})
.inner
.push((delimited, Spacing::Alone));
.push(delimited);
}
FlatToken::Token(token) => stack
.last_mut()
.expect("Bottom token frame is missing!")
.inner
.push((AttrAnnotatedTokenTree::Token(token), spacing)),
.push(AttrTokenTree::Token(token, spacing)),
FlatToken::AttrTarget(data) => stack
.last_mut()
.expect("Bottom token frame is missing!")
.inner
.push((AttrAnnotatedTokenTree::Attributes(data), spacing)),
.push(AttrTokenTree::Attributes(data)),
FlatToken::Empty => {}
}
token_and_spacing = iter.next();
}
let mut final_buf = stack.pop().expect("Missing final buf!");
if break_last_token {
let (last_token, spacing) = final_buf.inner.pop().unwrap();
if let AttrAnnotatedTokenTree::Token(last_token) = last_token {
let last_token = final_buf.inner.pop().unwrap();
if let AttrTokenTree::Token(last_token, spacing) = last_token {
let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
// An 'unglued' token is always two ASCII characters
let mut first_span = last_token.span.shrink_to_lo();
first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
final_buf.inner.push((
AttrAnnotatedTokenTree::Token(Token::new(unglued_first, first_span)),
spacing,
));
final_buf
.inner
.push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing));
} else {
panic!("Unexpected last token {:?}", last_token)
}
}
assert!(stack.is_empty(), "Stack should be empty: final_buf={:?} stack={:?}", final_buf, stack);
AttrAnnotatedTokenStream::new(final_buf.inner)
AttrTokenStream::new(final_buf.inner)
}

20
compiler/rustc_parse/src/parser/mod.rs
View File

@ -170,7 +170,7 @@ pub struct ClosureSpans {
/// attribute, we parse a nested AST node that has `#[cfg]` or `#[cfg_attr]`
/// In this case, we use a `ReplaceRange` to replace the entire inner AST node
/// with `FlatToken::AttrTarget`, allowing us to perform eager cfg-expansion
/// on an `AttrAnnotatedTokenStream`
/// on an `AttrTokenStream`.
///
/// 2. When we parse an inner attribute while collecting tokens. We
/// remove inner attributes from the token stream entirely, and
@ -183,7 +183,7 @@ pub struct ClosureSpans {
/// Controls how we capture tokens. Capturing can be expensive,
/// so we try to avoid performing capturing in cases where
/// we will never need an `AttrAnnotatedTokenStream`
/// we will never need an `AttrTokenStream`.
#[derive(Copy, Clone)]
pub enum Capturing {
/// We aren't performing any capturing - this is the default mode.
@ -237,7 +237,7 @@ struct TokenCursor {
// the trailing `>>` token. The `break_last_token`
// field is used to track this token - it gets
// appended to the captured stream when
// we evaluate a `LazyTokenStream`
// we evaluate a `LazyAttrTokenStream`.
break_last_token: bool,
}
@ -1464,11 +1464,11 @@ pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedBrace>, sess: &Pa
}
}
/// A helper struct used when building an `AttrAnnotatedTokenStream` from
/// a `LazyTokenStream`. Both delimiter and non-delimited tokens
/// A helper struct used when building an `AttrTokenStream` from
/// a `LazyAttrTokenStream`. Both delimiter and non-delimited tokens
/// are stored as `FlatToken::Token`. A vector of `FlatToken`s
/// is then 'parsed' to build up an `AttrAnnotatedTokenStream` with nested
/// `AttrAnnotatedTokenTree::Delimited` tokens
/// is then 'parsed' to build up an `AttrTokenStream` with nested
/// `AttrTokenTree::Delimited` tokens.
#[derive(Debug, Clone)]
pub enum FlatToken {
/// A token - this holds both delimiter (e.g. '{' and '}')
@ -1476,11 +1476,11 @@ pub enum FlatToken {
Token(Token),
/// Holds the `AttributesData` for an AST node. The
/// `AttributesData` is inserted directly into the
/// constructed `AttrAnnotatedTokenStream` as
/// an `AttrAnnotatedTokenTree::Attributes`
/// constructed `AttrTokenStream` as
/// an `AttrTokenTree::Attributes`.
AttrTarget(AttributesData),
/// A special 'empty' token that is ignored during the conversion
/// to an `AttrAnnotatedTokenStream`. This is used to simplify the
/// to an `AttrTokenStream`. This is used to simplify the
/// handling of replace ranges.
Empty,
}