mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rollup merge of #65641 - nnethercote:derive-TokenStream-Encodable-Decodable, r=petrochenkov

Browse Source 
Derive `Rustc{En,De}codable` for `TokenStream`.

`TokenStream` used to be a complex type, but it is now just a newtype
around a `Lrc<Vec<TreeAndJoint>>`. Currently it uses custom encoding
that discards the `IsJoint` and custom decoding that adds `NonJoint`
back in for every token tree. This requires building intermediate
`Vec<TokenTree>`s.

This commit makes `TokenStream` derive `Rustc{En,De}codable`. This
simplifies the code, and avoids the creation of the intermediate
vectors, saving up to 3% on various benchmarks. It also changes the AST
JSON output in one test.

r? @petrochenkov
This commit is contained in:
Mazdak Farrokhzad 2019-10-23 22:19:16 +02:00 committed by GitHub
commit 4a870aaee4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 3 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/libsyntax/tokenstream.rs
View File

@ -19,7 +19,6 @@
#[cfg(target_arch = "x86_64")]
use rustc_data_structures::static_assert_size;
use rustc_data_structures::sync::Lrc;
use rustc_serialize::{Decoder, Decodable, Encoder, Encodable};
use smallvec::{SmallVec, smallvec};
use std::{iter, mem};
@ -136,7 +135,7 @@ pub fn close_tt(span: Span, delim: DelimToken) -> TokenTree {
/// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
/// instead of a representation of the abstract syntax tree.
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, RustcEncodable, RustcDecodable)]
pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>);
pub type TreeAndJoint = (TokenTree, IsJoint);
@ -145,7 +144,7 @@ pub fn close_tt(span: Span, delim: DelimToken) -> TokenTree {
#[cfg(target_arch = "x86_64")]
static_assert_size!(TokenStream, 8);
#[derive(Clone, Copy, Debug, PartialEq)]
#[derive(Clone, Copy, Debug, PartialEq, RustcEncodable, RustcDecodable)]
pub enum IsJoint {
Joint,
NonJoint
@ -460,18 +459,6 @@ pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
}
}
impl Encodable for TokenStream {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), E::Error> {
self.trees().collect::<Vec<_>>().encode(encoder)
}
}
impl Decodable for TokenStream {
fn decode<D: Decoder>(decoder: &mut D) -> Result<TokenStream, D::Error> {
Vec::<TokenTree>::decode(decoder).map(|vec| vec.into_iter().collect())
}
}
#[derive(Debug, Copy, Clone, PartialEq, RustcEncodable, RustcDecodable)]
pub struct DelimSpan {
pub open: Span,

2
src/test/ui/ast-json/ast-json-output.stdout
View File

@ -1 +1 @@
{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"kind":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0}}]}]}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}}
{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"kind":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":{"_field0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0}}]},"NonJoint"]]}}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}}