From f28dfdf1c777ff83660cec0b432acabde1a2f7a4 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 17:36:38 -0400 Subject: [PATCH] proc_macro: stop using a remote object handle for Group This greatly reduces round-trips to fetch relevant extra information about the token in proc macro code, and avoids RPC messages to create Group tokens. --- .../rustc_expand/src/proc_macro_server.rs | 372 ++++++++---------- library/proc_macro/src/bridge/client.rs | 7 - library/proc_macro/src/bridge/mod.rs | 47 ++- library/proc_macro/src/bridge/server.rs | 1 - library/proc_macro/src/lib.rs | 26 +- 5 files changed, 204 insertions(+), 249 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 12d7338c39b..75731b9071d 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -2,7 +2,7 @@ use rustc_ast as ast; use rustc_ast::token; -use rustc_ast::tokenstream::{self, DelimSpan, Spacing::*, TokenStream, TreeAndSpacing}; +use rustc_ast::tokenstream::{self, Spacing::*, TokenStream}; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -14,7 +14,7 @@ use rustc_span::symbol::{self, kw, sym, Symbol}; use rustc_span::{BytePos, FileName, Pos, SourceFile, Span}; -use pm::bridge::{server, ExpnGlobals, Punct, TokenTree}; +use pm::bridge::{server, DelimSpan, ExpnGlobals, Group, Punct, TokenTree}; use pm::{Delimiter, Level, LineColumn}; use std::ops::Bound; use std::{ascii, panic}; @@ -49,158 +49,179 @@ fn to_internal(self) -> token::Delimiter { } } -impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> - for TokenTree +impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> + for Vec> { - fn from_internal( - ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec, &mut Rustc<'_, '_>), - ) -> Self { + fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self { use rustc_ast::token::*; - let joint = spacing == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = pm::Delimiter::from_internal(delim); - return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false }); - } - tokenstream::TokenTree::Token(token) => token, - }; + let mut cursor = stream.into_trees(); + let mut trees = Vec::new(); - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); - } - macro_rules! op { - ($a:expr) => { - tt!(Punct { ch: $a, joint }) + while let Some((tree, spacing)) = cursor.next_with_spacing() { + let joint = spacing == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = pm::Delimiter::from_internal(delim); + trees.push(TokenTree::Group(Group { + delimiter, + stream: Some(tts), + span: DelimSpan { + open: span.open, + close: span.close, + entire: span.entire(), + }, + })); + continue; + } + tokenstream::TokenTree::Token(token) => token, }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct { ch: $b, joint })); - tt!(Punct { ch: $a, joint: true }) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct { ch: $c, joint })); - stack.push(tt!(Punct { ch: $b, joint: true })); - tt!(Punct { ch: $a, joint: true }) - }}; - } - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)), - Lifetime(name) => { - let ident = symbol::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(rustc.sess(), ident.name, false))); - tt!(Punct { ch: '\'', joint: true }) + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + trees.push(TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + })) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + trees.push(TokenTree::$ty(self::$ty::$method($($value,)* span))) + ); } - Literal(lit) => tt!(Literal { lit }), - DocComment(_, attr_style, data) => { - let mut escaped = String::new(); - for ch in data.as_str().chars() { - escaped.extend(ch.escape_debug()); + macro_rules! op { + ($a:expr) => {{ + tt!(Punct { ch: $a, joint }); + }}; + ($a:expr, $b:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint }); + }}; + ($a:expr, $b:expr, $c:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint: true }); + tt!(Punct { ch: $c, joint }); + }}; + } + + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)), + Lifetime(name) => { + let ident = symbol::Ident::new(name, span).without_first_quote(); + tt!(Punct { ch: '\'', joint: true }); + tt!(Ident::new(rustc.sess(), ident.name, false)); } - let stream = [ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: pm::Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - flatten: false, - })); - if attr_style == ast::AttrStyle::Inner { - stack.push(tt!(Punct { ch: '!', joint: false })); + Literal(lit) => tt!(Literal { lit }), + DocComment(_, attr_style, data) => { + let mut escaped = String::new(); + for ch in data.as_str().chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + tt!(Punct { ch: '#', joint: false }); + if attr_style == ast::AttrStyle::Inner { + tt!(Punct { ch: '!', joint: false }); + } + trees.push(TokenTree::Group(Group { + delimiter: pm::Delimiter::Bracket, + stream: Some(stream), + span: DelimSpan::from_single(span), + })); } - tt!(Punct { ch: '#', joint: false }) - } - Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => { - TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span)) - } - Interpolated(nt) => { - TokenTree::Group(Group { - delimiter: pm::Delimiter::None, - stream: TokenStream::from_nonterminal_ast(&nt), - span: DelimSpan::from_single(span), - flatten: crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()), - }) - } + Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => { + trees.push(TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span))) + } - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Eof => unreachable!(), + Interpolated(nt) => { + let stream = TokenStream::from_nonterminal_ast(&nt); + if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()) { + trees.extend(Self::from_internal((stream, rustc))); + } else { + trees.push(TokenTree::Group(Group { + delimiter: pm::Delimiter::None, + stream: Some(stream), + span: DelimSpan::from_single(span), + })) + } + } + + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Eof => unreachable!(), + } } + trees } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; let (ch, joint, span) = match self { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { delimiter, stream, span, .. }) => { - return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream) - .into(); + TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { + return tokenstream::TokenTree::Delimited( + tokenstream::DelimSpan { open, close }, + delimiter.to_internal(), + stream.unwrap_or_default(), + ) + .into(); } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); @@ -277,17 +298,6 @@ fn to_internal(self) -> rustc_errors::Level { pub struct FreeFunctions; -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, - /// A hack used to pass AST fragments to attribute and derive macros - /// as a single nonterminal token instead of a token stream. - /// FIXME: It needs to be removed, but there are some compatibility issues (see #73345). - flatten: bool, -} - #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct Ident { sym: Symbol, @@ -356,7 +366,6 @@ fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) impl server::Types for Rustc<'_, '_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Group = Group; type Ident = Ident; type Literal = Literal; type SourceFile = Lrc; @@ -449,7 +458,7 @@ fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } @@ -457,7 +466,7 @@ fn from_token_tree( fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -487,71 +496,8 @@ fn concat_streams( fn into_trees( &mut self, stream: Self::TokenStream, - ) -> Vec> { - // FIXME: This is a raw port of the previous approach (which had a - // `TokenStreamIter` server-side object with a single `next` method), - // and can probably be optimized (for bulk conversion). - let mut cursor = stream.into_trees(); - let mut stack = Vec::new(); - let mut tts = Vec::new(); - loop { - let next = stack.pop().or_else(|| { - let next = cursor.next_with_spacing()?; - Some(TokenTree::from_internal((next, &mut stack, self))) - }); - match next { - Some(TokenTree::Group(group)) => { - // A hack used to pass AST fragments to attribute and derive - // macros as a single nonterminal token instead of a token - // stream. Such token needs to be "unwrapped" and not - // represented as a delimited group. - // FIXME: It needs to be removed, but there are some - // compatibility issues (see #73345). - if group.flatten { - tts.append(&mut self.into_trees(group.stream)); - } else { - tts.push(TokenTree::Group(group)); - } - } - Some(tt) => tts.push(tt), - None => return tts, - } - } - } -} - -impl server::Group for Rustc<'_, '_> { - fn new(&mut self, delimiter: Delimiter, stream: Option) -> Self::Group { - Group { - delimiter, - stream: stream.unwrap_or_default(), - span: DelimSpan::from_single(self.call_site), - flatten: false, - } - } - - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); + ) -> Vec> { + FromInternal::from_internal((stream, self)) } } diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index e42427c30fe..8254bd6e502 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -175,7 +175,6 @@ fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { 'owned: FreeFunctions, TokenStream, - Group, Literal, SourceFile, MultiSpan, @@ -198,12 +197,6 @@ fn clone(&self) -> Self { } } -impl Clone for Group { - fn clone(&self) -> Self { - self.clone() - } -} - impl Clone for Literal { fn clone(&self) -> Self { self.clone() diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 4c081396c6f..8995b5de475 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -65,11 +65,11 @@ macro_rules! with_api { fn from_str(src: &str) -> $S::TokenStream; fn to_string($self: &$S::TokenStream) -> String; fn from_token_tree( - tree: TokenTree<$S::Span, $S::Group, $S::Ident, $S::Literal>, + tree: TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>, ) -> $S::TokenStream; fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; fn concat_streams( base: Option<$S::TokenStream>, @@ -77,18 +77,7 @@ fn concat_streams( ) -> $S::TokenStream; fn into_trees( $self: $S::TokenStream - ) -> Vec>; - }, - Group { - fn drop($self: $S::Group); - fn clone($self: &$S::Group) -> $S::Group; - fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group; - fn delimiter($self: &$S::Group) -> Delimiter; - fn stream($self: &$S::Group) -> $S::TokenStream; - fn span($self: &$S::Group) -> $S::Span; - fn span_open($self: &$S::Group) -> $S::Span; - fn span_close($self: &$S::Group) -> $S::Span; - fn set_span($self: &mut $S::Group, span: $S::Span); + ) -> Vec>; }, Ident { fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; @@ -441,6 +430,30 @@ enum Result { } ); +#[derive(Copy, Clone)] +pub struct DelimSpan { + pub open: S, + pub close: S, + pub entire: S, +} + +impl DelimSpan { + pub fn from_single(span: S) -> Self { + DelimSpan { open: span, close: span, entire: span } + } +} + +compound_traits!(struct DelimSpan { open, close, entire }); + +#[derive(Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option, + pub span: DelimSpan, +} + +compound_traits!(struct Group { delimiter, stream, span }); + #[derive(Clone)] pub struct Punct { pub ch: char, @@ -451,15 +464,15 @@ pub struct Punct { compound_traits!(struct Punct { ch, joint, span }); #[derive(Clone)] -pub enum TokenTree { - Group(G), +pub enum TokenTree { + Group(Group), Punct(Punct), Ident(I), Literal(L), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index d30b60d189f..ea8b833b48f 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -8,7 +8,6 @@ pub trait Types { type FreeFunctions: 'static; type TokenStream: 'static + Clone; - type Group: 'static + Clone; type Ident: 'static + Copy + Eq + Hash; type Literal: 'static + Clone; type SourceFile: 'static + Clone; diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 05f45e5a38f..7f6356953ca 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -212,8 +212,8 @@ fn default() -> Self { fn tree_to_bridge_tree( tree: TokenTree, ) -> bridge::TokenTree< + bridge::client::TokenStream, bridge::client::Span, - bridge::client::Group, bridge::client::Ident, bridge::client::Literal, > { @@ -238,8 +238,8 @@ fn from(tree: TokenTree) -> TokenStream { struct ConcatTreesHelper { trees: Vec< bridge::TokenTree< + bridge::client::TokenStream, bridge::client::Span, - bridge::client::Group, bridge::client::Ident, bridge::client::Literal, >, @@ -365,8 +365,8 @@ pub mod token_stream { pub struct IntoIter( std::vec::IntoIter< bridge::TokenTree< + bridge::client::TokenStream, bridge::client::Span, - bridge::client::Group, bridge::client::Ident, bridge::client::Literal, >, @@ -788,7 +788,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Group(bridge::client::Group); +pub struct Group(bridge::Group); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Group {} @@ -825,13 +825,17 @@ impl Group { /// method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { - Group(bridge::client::Group::new(delimiter, stream.0)) + Group(bridge::Group { + delimiter, + stream: stream.0, + span: bridge::DelimSpan::from_single(Span::call_site().0), + }) } /// Returns the delimiter of this `Group` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn delimiter(&self) -> Delimiter { - self.0.delimiter() + self.0.delimiter } /// Returns the `TokenStream` of tokens that are delimited in this `Group`. @@ -840,7 +844,7 @@ pub fn delimiter(&self) -> Delimiter { /// returned above. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { - TokenStream(Some(self.0.stream())) + TokenStream(self.0.stream.clone()) } /// Returns the span for the delimiters of this token stream, spanning the @@ -852,7 +856,7 @@ pub fn stream(&self) -> TokenStream { /// ``` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span.entire) } /// Returns the span pointing to the opening delimiter of this group. @@ -863,7 +867,7 @@ pub fn span(&self) -> Span { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_open(&self) -> Span { - Span(self.0.span_open()) + Span(self.0.span.open) } /// Returns the span pointing to the closing delimiter of this group. @@ -874,7 +878,7 @@ pub fn span_open(&self) -> Span { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_close(&self) -> Span { - Span(self.0.span_close()) + Span(self.0.span.close) } /// Configures the span for this `Group`'s delimiters, but not its internal @@ -885,7 +889,7 @@ pub fn span_close(&self) -> Span { /// tokens at the level of the `Group`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0.set_span(span.0); + self.0.span = bridge::DelimSpan::from_single(span.0); } }