From 971d776aa5a678672eb3d37f2f507664aacd2440 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Wed, 22 Oct 2014 16:37:20 +1100 Subject: [PATCH 1/7] Add Span and separate open/close delims to TTDelim This came up when working [on the gl-rs generator extension](https://github.com/bjz/gl-rs/blob/990383de801bd2e233159d5be07c9b5622827620/src/gl_generator/lib.rs#L135-L146). The new definition of `TTDelim` adds an associated `Span` that covers the whole token tree and enforces the invariant that a delimited sequence of token trees must have an opening and closing delimiter. A `get_span` method has also been added to `TokenTree` type to make it easier to implement better error messages for syntax extensions. --- src/libsyntax/ast.rs | 31 +++++++++++++++-- src/libsyntax/ext/log_syntax.rs | 8 ++--- src/libsyntax/ext/quote.rs | 17 +++++---- src/libsyntax/ext/tt/macro_rules.rs | 20 +++-------- src/libsyntax/ext/tt/transcribe.rs | 54 +++++++++++++++++------------ src/libsyntax/fold.rs | 12 ++++++- src/libsyntax/parse/mod.rs | 37 ++++++++++---------- src/libsyntax/parse/parser.rs | 36 +++++++++++-------- src/libsyntax/print/pprust.rs | 8 ++++- 9 files changed, 135 insertions(+), 88 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 8eaee7282d1..be316ba9f4d 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -592,6 +592,20 @@ pub enum CaptureClause { CaptureByRef, } +/// A token that delimits a sequence of token trees +#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] +pub struct Delimiter { + pub span: Span, + pub token: ::parse::token::Token, +} + +impl Delimiter { + /// Convert the delimiter to a `TTTok` + pub fn to_tt(&self) -> TokenTree { + TTTok(self.span, self.token.clone()) + } +} + /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -611,10 +625,9 @@ pub enum CaptureClause { pub enum TokenTree { /// A single token TTTok(Span, ::parse::token::Token), - /// A delimited sequence (the delimiters appear as the first - /// and last elements of the vector) + /// A delimited sequence of token trees // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TTDelim(Rc>), + TTDelim(Span, Delimiter, Rc>, Delimiter), // These only make sense for right-hand-sides of MBE macros: @@ -628,6 +641,18 @@ pub enum TokenTree { TTNonterminal(Span, Ident) } +impl TokenTree { + /// Returns the `Span` corresponding to this token tree. + pub fn get_span(&self) -> Span { + match *self { + TTTok(span, _) => span, + TTDelim(span, _, _, _) => span, + TTSeq(span, _, _, _) => span, + TTNonterminal(span, _) => span, + } + } +} + // Matchers are nodes defined-by and recognized-by the main rust parser and // language, but they're only ever found inside syntax-extension invocations; // indeed, the only thing that ever _activates_ the rules in the rust parser diff --git a/src/libsyntax/ext/log_syntax.rs b/src/libsyntax/ext/log_syntax.rs index 8df5746e412..30301e3b8cc 100644 --- a/src/libsyntax/ext/log_syntax.rs +++ b/src/libsyntax/ext/log_syntax.rs @@ -13,16 +13,14 @@ use codemap; use ext::base; use print; -use std::rc::Rc; - pub fn expand_syntax_ext<'cx>(cx: &'cx mut base::ExtCtxt, sp: codemap::Span, - tt: &[ast::TokenTree]) + tts: &[ast::TokenTree]) -> Box { cx.print_backtrace(); - println!("{}", print::pprust::tt_to_string(&ast::TTDelim( - Rc::new(tt.iter().map(|x| (*x).clone()).collect())))); + + println!("{}", print::pprust::tts_to_string(tts)); // any so that `log_syntax` can be invoked as an expression and item. base::DummyResult::any(sp) diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 84775c12d64..783c08a4443 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -637,7 +637,7 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { } -fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec> { +fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { match *tt { ast::TTTok(sp, ref tok) => { let e_sp = cx.expr_ident(sp, id_ext("_sp")); @@ -650,13 +650,16 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec> { id_ext("push"), vec!(e_tok)); vec!(cx.stmt_expr(e_push)) - } - - ast::TTDelim(ref tts) => mk_tts(cx, sp, tts.as_slice()), + }, + ast::TTDelim(sp, ref open, ref tts, ref close) => { + let mut stmts = vec![]; + stmts.extend(mk_tt(cx, sp, &open.to_tt()).into_iter()); + stmts.extend(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter())); + stmts.extend(mk_tt(cx, sp, &close.to_tt()).into_iter()); + stmts + }, ast::TTSeq(..) => fail!("TTSeq in quote!"), - ast::TTNonterminal(sp, ident) => { - // tt.extend($ident.to_tokens(ext_cx).into_iter()) let e_to_toks = @@ -674,7 +677,7 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec> { vec!(e_to_toks)); vec!(cx.stmt_expr(e_push)) - } + }, } } diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 91db3a9d8df..fbfe10d004e 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -147,13 +147,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, rhses: &[Rc]) -> Box { if cx.trace_macros() { - println!("{}! {} {} {}", + println!("{}! {{ {} }}", token::get_ident(name), - "{", - print::pprust::tt_to_string(&TTDelim(Rc::new(arg.iter() - .map(|x| (*x).clone()) - .collect()))), - "}"); + print::pprust::tts_to_string(arg)); } // Which arm's failure should we report? (the one furthest along) @@ -175,15 +171,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, // okay, what's your transcriber? MatchedNonterminal(NtTT(ref tt)) => { match **tt { - // cut off delimiters; don't parse 'em - TTDelim(ref tts) => { - (*tts).slice(1u,(*tts).len()-1u) - .iter() - .map(|x| (*x).clone()) - .collect() - } - _ => cx.span_fatal( - sp, "macro rhs must be delimited") + // ignore delimiters + TTDelim(_, _, ref tts, _) => (**tts).clone(), + _ => cx.span_fatal(sp, "macro rhs must be delimited"), } }, _ => cx.span_bug(sp, "bad thing in rhs") diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 35ec37d842a..472b24be81b 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -18,6 +18,7 @@ use parse::token; use parse::lexer::TokenAndSpan; use std::rc::Rc; +use std::ops::Add; use std::collections::HashMap; ///an unzipping of `TokenTree`s @@ -104,37 +105,41 @@ enum LockstepIterSize { LisContradiction(String), } -fn lis_merge(lhs: LockstepIterSize, rhs: LockstepIterSize) -> LockstepIterSize { - match lhs { - LisUnconstrained => rhs.clone(), - LisContradiction(_) => lhs.clone(), - LisConstraint(l_len, l_id) => match rhs { - LisUnconstrained => lhs.clone(), - LisContradiction(_) => rhs.clone(), - LisConstraint(r_len, _) if l_len == r_len => lhs.clone(), - LisConstraint(r_len, r_id) => { - let l_n = token::get_ident(l_id); - let r_n = token::get_ident(r_id); - LisContradiction(format!("inconsistent lockstep iteration: \ - '{}' has {} items, but '{}' has {}", - l_n, l_len, r_n, r_len).to_string()) - } +impl Add for LockstepIterSize { + fn add(&self, other: &LockstepIterSize) -> LockstepIterSize { + match *self { + LisUnconstrained => other.clone(), + LisContradiction(_) => self.clone(), + LisConstraint(l_len, l_id) => match *other { + LisUnconstrained => self.clone(), + LisContradiction(_) => other.clone(), + LisConstraint(r_len, _) if l_len == r_len => self.clone(), + LisConstraint(r_len, r_id) => { + let l_n = token::get_ident(l_id); + let r_n = token::get_ident(r_id); + LisContradiction(format!("inconsistent lockstep iteration: \ + '{}' has {} items, but '{}' has {}", + l_n, l_len, r_n, r_len).to_string()) + } + }, } } } fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { match *t { - TTDelim(ref tts) | TTSeq(_, ref tts, _, _) => { - tts.iter().fold(LisUnconstrained, |lis, tt| { - lis_merge(lis, lockstep_iter_size(tt, r)) + // The opening and closing delimiters are both tokens, so they are + // treated as `LisUnconstrained`. + TTDelim(_, _, ref tts, _) | TTSeq(_, ref tts, _, _) => { + tts.iter().fold(LisUnconstrained, |size, tt| { + size + lockstep_iter_size(tt, r) }) - } + }, TTTok(..) => LisUnconstrained, TTNonterminal(_, name) => match *lookup_cur_matched(r, name) { MatchedNonterminal(_) => LisUnconstrained, MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name) - } + }, } } @@ -197,9 +202,14 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { (*frame.forest)[frame.idx].clone() }; match t { - TTDelim(tts) => { + TTDelim(_, open, delimed_tts, close) => { + let mut tts = vec![]; + tts.push(open.to_tt()); + tts.extend(delimed_tts.iter().map(|x| (*x).clone())); + tts.push(close.to_tt()); + r.stack.push(TtFrame { - forest: tts, + forest: Rc::new(tts), idx: 0, dotdotdoted: false, sep: None diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index ceef190f5d4..ddb2ab49f8b 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -571,7 +571,17 @@ pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { match *tt { TTTok(span, ref tok) => TTTok(span, fld.fold_token(tok.clone())), - TTDelim(ref tts) => TTDelim(Rc::new(fld.fold_tts(tts.as_slice()))), + TTDelim(span, ref open, ref tts, ref close) => + TTDelim(span, + Delimiter { + span: open.span, + token: fld.fold_token(open.token.clone()) + }, + Rc::new(fld.fold_tts(tts.as_slice())), + Delimiter { + span: close.span, + token: fld.fold_token(close.token.clone()) + }), TTSeq(span, ref pattern, ref sep, is_optional) => TTSeq(span, Rc::new(fld.fold_tts(pattern.as_slice())), diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 2d7d32cd9ea..1c99b608f7a 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -788,35 +788,34 @@ mod test { } // check the token-tree-ization of macros - #[test] fn string_to_tts_macro () { + #[test] + fn string_to_tts_macro () { let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string()); let tts: &[ast::TokenTree] = tts.as_slice(); match tts { - [ast::TTTok(_,_), - ast::TTTok(_,token::NOT), - ast::TTTok(_,_), - ast::TTDelim(ref delim_elts)] => { + [ast::TTTok(_, _), + ast::TTTok(_, token::NOT), + ast::TTTok(_, _), + ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + ref delim_elts, + ast::TTTok(_, token::RPAREN))] => { let delim_elts: &[ast::TokenTree] = delim_elts.as_slice(); match delim_elts { - [ast::TTTok(_,token::LPAREN), - ast::TTDelim(ref first_set), - ast::TTTok(_,token::FAT_ARROW), - ast::TTDelim(ref second_set), - ast::TTTok(_,token::RPAREN)] => { + [ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + ref first_set, + ast::TTTok(_, token::RPAREN)), + ast::TTTok(_, token::FAT_ARROW), + ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + ref second_set, + ast::TTTok(_, token::RPAREN))] => { let first_set: &[ast::TokenTree] = first_set.as_slice(); match first_set { - [ast::TTTok(_,token::LPAREN), - ast::TTTok(_,token::DOLLAR), - ast::TTTok(_,_), - ast::TTTok(_,token::RPAREN)] => { + [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => { let second_set: &[ast::TokenTree] = second_set.as_slice(); match second_set { - [ast::TTTok(_,token::LPAREN), - ast::TTTok(_,token::DOLLAR), - ast::TTTok(_,_), - ast::TTTok(_,token::RPAREN)] => { + [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => { assert_eq!("correct","correct") } _ => assert_eq!("wrong 4","correct") @@ -837,7 +836,7 @@ mod test { _ => { error!("failing value: {}",tts); assert_eq!("wrong 1","correct"); - } + }, } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 5abf79836f5..005ed2e7ed3 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -48,7 +48,7 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField}; use ast::{StructVariantKind, BiSub}; use ast::StrStyle; use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue}; -use ast::{TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok}; +use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok}; use ast::{TTNonterminal, TupleVariantKind, Ty, Ty_, TyBot}; use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn}; use ast::{TyTypeof, TyInfer, TypeMethod}; @@ -2574,16 +2574,11 @@ impl<'a> Parser<'a> { } } _ => { - parse_any_tt_tok(p) + TTTok(p.span, p.bump_and_get()) } } } - // turn the next token into a TTTok: - fn parse_any_tt_tok(p: &mut Parser) -> TokenTree { - TTTok(p.span, p.bump_and_get()) - } - match (&self.token, token::close_delimiter_for(&self.token)) { (&token::EOF, _) => { let open_braces = self.open_braces.clone(); @@ -2595,21 +2590,32 @@ impl<'a> Parser<'a> { self.fatal("this file contains an un-closed delimiter "); } (_, Some(close_delim)) => { + // The span for beginning of the delimited section + let pre_span = self.span; + // Parse the open delimiter. self.open_braces.push(self.span); - let mut result = vec!(parse_any_tt_tok(self)); + let open = Delimiter { + span: self.span, + token: self.bump_and_get(), + }; - let trees = - self.parse_seq_to_before_end(&close_delim, - seq_sep_none(), - |p| p.parse_token_tree()); - result.extend(trees.into_iter()); + // Parse the token trees within the delimeters + let tts = self.parse_seq_to_before_end( + &close_delim, seq_sep_none(), |p| p.parse_token_tree() + ); // Parse the close delimiter. - result.push(parse_any_tt_tok(self)); + let close = Delimiter { + span: self.span, + token: self.bump_and_get(), + }; self.open_braces.pop().unwrap(); - TTDelim(Rc::new(result)) + // Expand to cover the entire delimited token tree + let span = Span { hi: self.span.hi, ..pre_span }; + + TTDelim(span, open, Rc::new(tts), close) } _ => parse_non_delim_tt_tok(self) } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index b63f9b0120b..4f4b153d3a9 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1020,7 +1020,13 @@ impl<'a> State<'a> { /// expression arguments as expressions). It can be done! I think. pub fn print_tt(&mut self, tt: &ast::TokenTree) -> IoResult<()> { match *tt { - ast::TTDelim(ref tts) => self.print_tts(tts.as_slice()), + ast::TTDelim(_, ref open, ref tts, ref close) => { + try!(word(&mut self.s, parse::token::to_string(&open.token).as_slice())); + try!(space(&mut self.s)); + try!(self.print_tts(tts.as_slice())); + try!(space(&mut self.s)); + word(&mut self.s, parse::token::to_string(&close.token).as_slice()) + }, ast::TTTok(_, ref tk) => { try!(word(&mut self.s, parse::token::to_string(tk).as_slice())); match *tk { From ec3f0201e76b5cf689f3e8e6418435c3e6d9271c Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Wed, 22 Oct 2014 23:35:32 +1100 Subject: [PATCH 2/7] Rename TokenTree variants for clarity This should be clearer, and fits in better with the `TTNonterminal` variant. Renames: - `TTTok` -> `TTToken` - `TTDelim` -> `TTDelimited` - `TTSeq` -> `TTSequence` --- src/doc/guide-plugin.md | 4 +-- src/libsyntax/ast.rs | 27 ++++++++------- src/libsyntax/diagnostics/plugin.rs | 12 +++---- src/libsyntax/ext/base.rs | 4 +-- src/libsyntax/ext/concat_idents.rs | 4 +-- src/libsyntax/ext/quote.rs | 8 ++--- src/libsyntax/ext/trace_macros.rs | 4 +-- src/libsyntax/ext/tt/macro_rules.rs | 4 +-- src/libsyntax/ext/tt/transcribe.rs | 20 +++++------ src/libsyntax/fold.rs | 36 ++++++++++---------- src/libsyntax/parse/mod.rs | 50 ++++++++++++++-------------- src/libsyntax/parse/parser.rs | 10 +++--- src/libsyntax/print/pprust.rs | 6 ++-- src/test/auxiliary/roman_numerals.rs | 4 +-- 14 files changed, 98 insertions(+), 95 deletions(-) diff --git a/src/doc/guide-plugin.md b/src/doc/guide-plugin.md index 3830a2126e1..9bf1d29569c 100644 --- a/src/doc/guide-plugin.md +++ b/src/doc/guide-plugin.md @@ -56,7 +56,7 @@ extern crate rustc; use syntax::codemap::Span; use syntax::parse::token::{IDENT, get_ident}; -use syntax::ast::{TokenTree, TTTok}; +use syntax::ast::{TokenTree, TTToken}; use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacExpr}; use syntax::ext::build::AstBuilder; // trait for expr_uint use rustc::plugin::Registry; @@ -71,7 +71,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) ("I", 1)]; let text = match args { - [TTTok(_, IDENT(s, _))] => get_ident(s).to_string(), + [TTToken(_, IDENT(s, _))] => get_ident(s).to_string(), _ => { cx.span_err(sp, "argument should be a single identifier"); return DummyResult::any(sp); diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index be316ba9f4d..36373638099 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -24,6 +24,9 @@ use std::fmt::Show; use std::rc::Rc; use serialize::{Encodable, Decodable, Encoder, Decoder}; +#[cfg(stage0)] +pub use self::TTToken as TTTok; + // FIXME #6993: in librustc, uses of "ident" should be replaced // by just "Name". @@ -600,9 +603,9 @@ pub struct Delimiter { } impl Delimiter { - /// Convert the delimiter to a `TTTok` + /// Convert the delimiter to a `TTToken` pub fn to_tt(&self) -> TokenTree { - TTTok(self.span, self.token.clone()) + TTToken(self.span, self.token.clone()) } } @@ -614,9 +617,9 @@ impl Delimiter { /// If the syntax extension is an MBE macro, it will attempt to match its /// LHS "matchers" against the provided token tree, and if it finds a /// match, will transcribe the RHS token tree, splicing in any captured -/// macro_parser::matched_nonterminals into the TTNonterminals it finds. +/// `macro_parser::matched_nonterminals` into the `TTNonterminal`s it finds. /// -/// The RHS of an MBE macro is the only place a TTNonterminal or TTSeq +/// The RHS of an MBE macro is the only place a `TTNonterminal` or `TTSequence` /// makes any real sense. You could write them elsewhere but nothing /// else knows what to do with them, so you'll probably get a syntax /// error. @@ -624,18 +627,18 @@ impl Delimiter { #[doc="For macro invocations; parsing is delegated to the macro"] pub enum TokenTree { /// A single token - TTTok(Span, ::parse::token::Token), + TTToken(Span, ::parse::token::Token), /// A delimited sequence of token trees // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TTDelim(Span, Delimiter, Rc>, Delimiter), + TTDelimited(Span, Delimiter, Rc>, Delimiter), // These only make sense for right-hand-sides of MBE macros: - /// A kleene-style repetition sequence with a span, a TTForest, + /// A kleene-style repetition sequence with a span, a `TTForest`, /// an optional separator, and a boolean where true indicates /// zero or more (..), and false indicates one or more (+). // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TTSeq(Span, Rc>, Option<::parse::token::Token>, bool), + TTSequence(Span, Rc>, Option<::parse::token::Token>, bool), /// A syntactic variable that will be filled in by macro expansion. TTNonterminal(Span, Ident) @@ -645,10 +648,10 @@ impl TokenTree { /// Returns the `Span` corresponding to this token tree. pub fn get_span(&self) -> Span { match *self { - TTTok(span, _) => span, - TTDelim(span, _, _, _) => span, - TTSeq(span, _, _, _) => span, - TTNonterminal(span, _) => span, + TTToken(span, _) => span, + TTDelimited(span, _, _, _) => span, + TTSequence(span, _, _, _) => span, + TTNonterminal(span, _) => span, } } } diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index d3c39284f55..8ea08c58d06 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -50,7 +50,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt, token_tree: &[TokenTree]) -> Box { let code = match token_tree { - [ast::TTTok(_, token::IDENT(code, _))] => code, + [ast::TTToken(_, token::IDENT(code, _))] => code, _ => unreachable!() }; with_registered_diagnostics(|diagnostics| { @@ -82,12 +82,12 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt, token_tree: &[TokenTree]) -> Box { let (code, description) = match token_tree { - [ast::TTTok(_, token::IDENT(ref code, _))] => { + [ast::TTToken(_, token::IDENT(ref code, _))] => { (code, None) }, - [ast::TTTok(_, token::IDENT(ref code, _)), - ast::TTTok(_, token::COMMA), - ast::TTTok(_, token::LIT_STR_RAW(description, _))] => { + [ast::TTToken(_, token::IDENT(ref code, _)), + ast::TTToken(_, token::COMMA), + ast::TTToken(_, token::LIT_STR_RAW(description, _))] => { (code, Some(description)) } _ => unreachable!() @@ -110,7 +110,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt, token_tree: &[TokenTree]) -> Box { let name = match token_tree { - [ast::TTTok(_, token::IDENT(ref name, _))] => name, + [ast::TTToken(_, token::IDENT(ref name, _))] => name, _ => unreachable!() }; diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 5cc2fe03618..b5cc2d95890 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -684,8 +684,8 @@ pub fn get_single_str_from_tts(cx: &ExtCtxt, cx.span_err(sp, format!("{} takes 1 argument.", name).as_slice()); } else { match tts[0] { - ast::TTTok(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())), - ast::TTTok(_, token::LIT_STR_RAW(ident, _)) => { + ast::TTToken(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())), + ast::TTToken(_, token::LIT_STR_RAW(ident, _)) => { return Some(parse::raw_str_lit(ident.as_str())) } _ => { diff --git a/src/libsyntax/ext/concat_idents.rs b/src/libsyntax/ext/concat_idents.rs index 145412caa0b..e6befdd2aac 100644 --- a/src/libsyntax/ext/concat_idents.rs +++ b/src/libsyntax/ext/concat_idents.rs @@ -23,7 +23,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree] for (i, e) in tts.iter().enumerate() { if i & 1 == 1 { match *e { - ast::TTTok(_, token::COMMA) => (), + ast::TTToken(_, token::COMMA) => (), _ => { cx.span_err(sp, "concat_idents! expecting comma."); return DummyResult::expr(sp); @@ -31,7 +31,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree] } } else { match *e { - ast::TTTok(_, token::IDENT(ident,_)) => { + ast::TTToken(_, token::IDENT(ident,_)) => { res_str.push_str(token::get_ident(ident).get()) } _ => { diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 783c08a4443..93bd66d6eeb 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -639,10 +639,10 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { match *tt { - ast::TTTok(sp, ref tok) => { + ast::TTToken(sp, ref tok) => { let e_sp = cx.expr_ident(sp, id_ext("_sp")); let e_tok = cx.expr_call(sp, - mk_ast_path(cx, sp, "TTTok"), + mk_ast_path(cx, sp, "TTToken"), vec!(e_sp, mk_token(cx, sp, tok))); let e_push = cx.expr_method_call(sp, @@ -651,14 +651,14 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { vec!(e_tok)); vec!(cx.stmt_expr(e_push)) }, - ast::TTDelim(sp, ref open, ref tts, ref close) => { + ast::TTDelimited(sp, ref open, ref tts, ref close) => { let mut stmts = vec![]; stmts.extend(mk_tt(cx, sp, &open.to_tt()).into_iter()); stmts.extend(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter())); stmts.extend(mk_tt(cx, sp, &close.to_tt()).into_iter()); stmts }, - ast::TTSeq(..) => fail!("TTSeq in quote!"), + ast::TTSequence(..) => fail!("TTSequence in quote!"), ast::TTNonterminal(sp, ident) => { // tt.extend($ident.to_tokens(ext_cx).into_iter()) diff --git a/src/libsyntax/ext/trace_macros.rs b/src/libsyntax/ext/trace_macros.rs index 1f50eb933bb..4c3846731f4 100644 --- a/src/libsyntax/ext/trace_macros.rs +++ b/src/libsyntax/ext/trace_macros.rs @@ -20,10 +20,10 @@ pub fn expand_trace_macros(cx: &mut ExtCtxt, tt: &[ast::TokenTree]) -> Box { match tt { - [ast::TTTok(_, ref tok)] if is_keyword(keywords::True, tok) => { + [ast::TTToken(_, ref tok)] if is_keyword(keywords::True, tok) => { cx.set_trace_macros(true); } - [ast::TTTok(_, ref tok)] if is_keyword(keywords::False, tok) => { + [ast::TTToken(_, ref tok)] if is_keyword(keywords::False, tok) => { cx.set_trace_macros(false); } _ => cx.span_err(sp, "trace_macros! accepts only `true` or `false`"), diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index fbfe10d004e..4a3828a8043 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TTDelim}; +use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TTDelimited}; use ast; use codemap::{Span, Spanned, DUMMY_SP}; use ext::base::{ExtCtxt, MacResult, MacroDef}; @@ -172,7 +172,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, MatchedNonterminal(NtTT(ref tt)) => { match **tt { // ignore delimiters - TTDelim(_, _, ref tts, _) => (**tts).clone(), + TTDelimited(_, _, ref tts, _) => (**tts).clone(), _ => cx.span_fatal(sp, "macro rhs must be delimited"), } }, diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 472b24be81b..e705c4d8b33 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -9,7 +9,7 @@ // except according to those terms. use ast; -use ast::{TokenTree, TTDelim, TTTok, TTSeq, TTNonterminal, Ident}; +use ast::{TokenTree, TTDelimited, TTToken, TTSequence, TTNonterminal, Ident}; use codemap::{Span, DUMMY_SP}; use diagnostic::SpanHandler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; @@ -45,7 +45,7 @@ pub struct TtReader<'a> { } /// This can do Macro-By-Example transcription. On the other hand, if -/// `src` contains no `TTSeq`s and `TTNonterminal`s, `interp` can (and +/// `src` contains no `TTSequence`s and `TTNonterminal`s, `interp` can (and /// should) be none. pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler, interp: Option>>, @@ -130,12 +130,12 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { match *t { // The opening and closing delimiters are both tokens, so they are // treated as `LisUnconstrained`. - TTDelim(_, _, ref tts, _) | TTSeq(_, ref tts, _, _) => { + TTDelimited(_, _, ref tts, _) | TTSequence(_, ref tts, _, _) => { tts.iter().fold(LisUnconstrained, |size, tt| { size + lockstep_iter_size(tt, r) }) }, - TTTok(..) => LisUnconstrained, + TTToken(..) => LisUnconstrained, TTNonterminal(_, name) => match *lookup_cur_matched(r, name) { MatchedNonterminal(_) => LisUnconstrained, MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name) @@ -194,15 +194,15 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } } } - loop { /* because it's easiest, this handles `TTDelim` not starting - with a `TTTok`, even though it won't happen */ + loop { /* because it's easiest, this handles `TTDelimited` not starting + with a `TTToken`, even though it won't happen */ let t = { let frame = r.stack.last().unwrap(); // FIXME(pcwalton): Bad copy. (*frame.forest)[frame.idx].clone() }; match t { - TTDelim(_, open, delimed_tts, close) => { + TTDelimited(_, open, delimed_tts, close) => { let mut tts = vec![]; tts.push(open.to_tt()); tts.extend(delimed_tts.iter().map(|x| (*x).clone())); @@ -216,15 +216,15 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { }); // if this could be 0-length, we'd need to potentially recur here } - TTTok(sp, tok) => { + TTToken(sp, tok) => { r.cur_span = sp; r.cur_tok = tok; r.stack.last_mut().unwrap().idx += 1; return ret_val; } - TTSeq(sp, tts, sep, zerok) => { + TTSequence(sp, tts, sep, zerok) => { // FIXME(pcwalton): Bad copy. - match lockstep_iter_size(&TTSeq(sp, tts.clone(), sep.clone(), zerok), r) { + match lockstep_iter_size(&TTSequence(sp, tts.clone(), sep.clone(), zerok), r) { LisUnconstrained => { r.sp_diag.span_fatal( sp.clone(), /* blame macro writer */ diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index ddb2ab49f8b..9cffce74a09 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -569,24 +569,24 @@ pub fn noop_fold_arg(Arg {id, pat, ty}: Arg, fld: &mut T) -> Arg { pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { match *tt { - TTTok(span, ref tok) => - TTTok(span, fld.fold_token(tok.clone())), - TTDelim(span, ref open, ref tts, ref close) => - TTDelim(span, - Delimiter { - span: open.span, - token: fld.fold_token(open.token.clone()) - }, - Rc::new(fld.fold_tts(tts.as_slice())), - Delimiter { - span: close.span, - token: fld.fold_token(close.token.clone()) - }), - TTSeq(span, ref pattern, ref sep, is_optional) => - TTSeq(span, - Rc::new(fld.fold_tts(pattern.as_slice())), - sep.clone().map(|tok| fld.fold_token(tok)), - is_optional), + TTToken(span, ref tok) => + TTToken(span, fld.fold_token(tok.clone())), + TTDelimited(span, ref open, ref tts, ref close) => + TTDelimited(span, + Delimiter { + span: open.span, + token: fld.fold_token(open.token.clone()) + }, + Rc::new(fld.fold_tts(tts.as_slice())), + Delimiter { + span: close.span, + token: fld.fold_token(close.token.clone()) + }), + TTSequence(span, ref pattern, ref sep, is_optional) => + TTSequence(span, + Rc::new(fld.fold_tts(pattern.as_slice())), + sep.clone().map(|tok| fld.fold_token(tok)), + is_optional), TTNonterminal(sp,ref ident) => TTNonterminal(sp,fld.fold_ident(*ident)) } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 1c99b608f7a..a2e40282321 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -793,29 +793,29 @@ mod test { let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string()); let tts: &[ast::TokenTree] = tts.as_slice(); match tts { - [ast::TTTok(_, _), - ast::TTTok(_, token::NOT), - ast::TTTok(_, _), - ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + [ast::TTToken(_, _), + ast::TTToken(_, token::NOT), + ast::TTToken(_, _), + ast::TTDelimited(_, ast::TTToken(_, token::LPAREN), ref delim_elts, - ast::TTTok(_, token::RPAREN))] => { + ast::TTToken(_, token::RPAREN))] => { let delim_elts: &[ast::TokenTree] = delim_elts.as_slice(); match delim_elts { - [ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + [ast::TTDelimited(_, ast::TTToken(_, token::LPAREN), ref first_set, - ast::TTTok(_, token::RPAREN)), - ast::TTTok(_, token::FAT_ARROW), - ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + ast::TTToken(_, token::RPAREN)), + ast::TTToken(_, token::FAT_ARROW), + ast::TTDelimited(_, ast::TTToken(_, token::LPAREN), ref second_set, - ast::TTTok(_, token::RPAREN))] => { + ast::TTToken(_, token::RPAREN))] => { let first_set: &[ast::TokenTree] = first_set.as_slice(); match first_set { - [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => { + [ast::TTToken(_, token::DOLLAR), ast::TTToken(_, _)] => { let second_set: &[ast::TokenTree] = second_set.as_slice(); match second_set { - [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => { + [ast::TTToken(_, token::DOLLAR), ast::TTToken(_, _)] => { assert_eq!("correct","correct") } _ => assert_eq!("wrong 4","correct") @@ -845,7 +845,7 @@ mod test { assert_eq!(json::encode(&tts), "[\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ {\ @@ -858,7 +858,7 @@ mod test { ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ {\ @@ -871,18 +871,18 @@ mod test { ]\ },\ {\ - \"variant\":\"TTDelim\",\ + \"variant\":\"TTDelimited\",\ \"fields\":[\ [\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ \"LPAREN\"\ ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ {\ @@ -895,14 +895,14 @@ mod test { ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ \"COLON\"\ ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ {\ @@ -915,7 +915,7 @@ mod test { ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ \"RPAREN\"\ @@ -925,18 +925,18 @@ mod test { ]\ },\ {\ - \"variant\":\"TTDelim\",\ + \"variant\":\"TTDelimited\",\ \"fields\":[\ [\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ \"LBRACE\"\ ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ {\ @@ -949,14 +949,14 @@ mod test { ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ \"SEMI\"\ ]\ },\ {\ - \"variant\":\"TTTok\",\ + \"variant\":\"TTToken\",\ \"fields\":[\ null,\ \"RBRACE\"\ diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 005ed2e7ed3..1ed7baa13b4 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -48,7 +48,7 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField}; use ast::{StructVariantKind, BiSub}; use ast::StrStyle; use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue}; -use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok}; +use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TTDelimited, TTSequence, TTToken}; use ast::{TTNonterminal, TupleVariantKind, Ty, Ty_, TyBot}; use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn}; use ast::{TyTypeof, TyInfer, TypeMethod}; @@ -2526,7 +2526,7 @@ impl<'a> Parser<'a> { /// parse a single token tree from the input. pub fn parse_token_tree(&mut self) -> TokenTree { // FIXME #6994: currently, this is too eager. It - // parses token trees but also identifies TTSeq's + // parses token trees but also identifies TTSequence's // and TTNonterminal's; it's too early to know yet // whether something will be a nonterminal or a seq // yet. @@ -2568,13 +2568,13 @@ impl<'a> Parser<'a> { let seq = match seq { Spanned { node, .. } => node, }; - TTSeq(mk_sp(sp.lo, p.span.hi), Rc::new(seq), s, z) + TTSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), s, z) } else { TTNonterminal(sp, p.parse_ident()) } } _ => { - TTTok(p.span, p.bump_and_get()) + TTToken(p.span, p.bump_and_get()) } } } @@ -2615,7 +2615,7 @@ impl<'a> Parser<'a> { // Expand to cover the entire delimited token tree let span = Span { hi: self.span.hi, ..pre_span }; - TTDelim(span, open, Rc::new(tts), close) + TTDelimited(span, open, Rc::new(tts), close) } _ => parse_non_delim_tt_tok(self) } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 4f4b153d3a9..9a102d22971 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1020,14 +1020,14 @@ impl<'a> State<'a> { /// expression arguments as expressions). It can be done! I think. pub fn print_tt(&mut self, tt: &ast::TokenTree) -> IoResult<()> { match *tt { - ast::TTDelim(_, ref open, ref tts, ref close) => { + ast::TTDelimited(_, ref open, ref tts, ref close) => { try!(word(&mut self.s, parse::token::to_string(&open.token).as_slice())); try!(space(&mut self.s)); try!(self.print_tts(tts.as_slice())); try!(space(&mut self.s)); word(&mut self.s, parse::token::to_string(&close.token).as_slice()) }, - ast::TTTok(_, ref tk) => { + ast::TTToken(_, ref tk) => { try!(word(&mut self.s, parse::token::to_string(tk).as_slice())); match *tk { parse::token::DOC_COMMENT(..) => { @@ -1036,7 +1036,7 @@ impl<'a> State<'a> { _ => Ok(()) } } - ast::TTSeq(_, ref tts, ref sep, zerok) => { + ast::TTSequence(_, ref tts, ref sep, zerok) => { try!(word(&mut self.s, "$(")); for tt_elt in (*tts).iter() { try!(self.print_tt(tt_elt)); diff --git a/src/test/auxiliary/roman_numerals.rs b/src/test/auxiliary/roman_numerals.rs index 43842fae70f..0d5abb8fb5d 100644 --- a/src/test/auxiliary/roman_numerals.rs +++ b/src/test/auxiliary/roman_numerals.rs @@ -18,7 +18,7 @@ extern crate rustc; use syntax::codemap::Span; use syntax::parse::token::{IDENT, get_ident}; -use syntax::ast::{TokenTree, TTTok}; +use syntax::ast::{TokenTree, TTToken}; use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacExpr}; use syntax::ext::build::AstBuilder; // trait for expr_uint use rustc::plugin::Registry; @@ -39,7 +39,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) ("I", 1)]; let text = match args { - [TTTok(_, IDENT(s, _))] => get_ident(s).to_string(), + [TTToken(_, IDENT(s, _))] => get_ident(s).to_string(), _ => { cx.span_err(sp, "argument should be a single identifier"); return DummyResult::any(sp); From 6a50b4d018b0e44b9e12560030ca7fb240107a68 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Thu, 23 Oct 2014 01:42:47 +1100 Subject: [PATCH 3/7] Prevent some vector reallocations --- src/libsyntax/ext/quote.rs | 9 ++++----- src/libsyntax/ext/tt/transcribe.rs | 12 ++++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 93bd66d6eeb..baba38d8cbb 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -652,11 +652,10 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { vec!(cx.stmt_expr(e_push)) }, ast::TTDelimited(sp, ref open, ref tts, ref close) => { - let mut stmts = vec![]; - stmts.extend(mk_tt(cx, sp, &open.to_tt()).into_iter()); - stmts.extend(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter())); - stmts.extend(mk_tt(cx, sp, &close.to_tt()).into_iter()); - stmts + mk_tt(cx, sp, &open.to_tt()).into_iter() + .chain(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter())) + .chain(mk_tt(cx, sp, &close.to_tt()).into_iter()) + .collect() }, ast::TTSequence(..) => fail!("TTSequence in quote!"), ast::TTNonterminal(sp, ident) => { diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index e705c4d8b33..c0b66851dfe 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -202,14 +202,14 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { (*frame.forest)[frame.idx].clone() }; match t { - TTDelimited(_, open, delimed_tts, close) => { - let mut tts = vec![]; - tts.push(open.to_tt()); - tts.extend(delimed_tts.iter().map(|x| (*x).clone())); - tts.push(close.to_tt()); + TTDelimited(_, open, tts, close) => { + let mut forest = Vec::with_capacity(1 + tts.len() + 1); + forest.push(open.to_tt()); + forest.extend(tts.iter().map(|x| (*x).clone())); + forest.push(close.to_tt()); r.stack.push(TtFrame { - forest: Rc::new(tts), + forest: Rc::new(forest), idx: 0, dotdotdoted: false, sep: None From dfb4163f8380e9a1aaf64a7474de30634bca4034 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Thu, 23 Oct 2014 04:39:58 +1100 Subject: [PATCH 4/7] Use standard capitalisation for TokenTree variants --- src/doc/guide-plugin.md | 4 +-- src/libsyntax/ast.rs | 26 +++++++-------- src/libsyntax/diagnostics/plugin.rs | 12 +++---- src/libsyntax/ext/base.rs | 4 +-- src/libsyntax/ext/concat_idents.rs | 4 +-- src/libsyntax/ext/quote.rs | 14 ++++---- src/libsyntax/ext/trace_macros.rs | 4 +-- src/libsyntax/ext/tt/macro_rules.rs | 4 +-- src/libsyntax/ext/tt/transcribe.rs | 24 ++++++------- src/libsyntax/fold.rs | 16 ++++----- src/libsyntax/parse/mod.rs | 50 ++++++++++++++-------------- src/libsyntax/parse/parser.rs | 16 ++++----- src/libsyntax/print/pprust.rs | 8 ++--- src/test/auxiliary/roman_numerals.rs | 4 +-- 14 files changed, 95 insertions(+), 95 deletions(-) diff --git a/src/doc/guide-plugin.md b/src/doc/guide-plugin.md index 9bf1d29569c..83a5697f75a 100644 --- a/src/doc/guide-plugin.md +++ b/src/doc/guide-plugin.md @@ -56,7 +56,7 @@ extern crate rustc; use syntax::codemap::Span; use syntax::parse::token::{IDENT, get_ident}; -use syntax::ast::{TokenTree, TTToken}; +use syntax::ast::{TokenTree, TtToken}; use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacExpr}; use syntax::ext::build::AstBuilder; // trait for expr_uint use rustc::plugin::Registry; @@ -71,7 +71,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) ("I", 1)]; let text = match args { - [TTToken(_, IDENT(s, _))] => get_ident(s).to_string(), + [TtToken(_, IDENT(s, _))] => get_ident(s).to_string(), _ => { cx.span_err(sp, "argument should be a single identifier"); return DummyResult::any(sp); diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 36373638099..f87c7cf0215 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -25,7 +25,7 @@ use std::rc::Rc; use serialize::{Encodable, Decodable, Encoder, Decoder}; #[cfg(stage0)] -pub use self::TTToken as TTTok; +pub use self::TtToken as TTTok; // FIXME #6993: in librustc, uses of "ident" should be replaced // by just "Name". @@ -603,9 +603,9 @@ pub struct Delimiter { } impl Delimiter { - /// Convert the delimiter to a `TTToken` + /// Convert the delimiter to a `TtToken` pub fn to_tt(&self) -> TokenTree { - TTToken(self.span, self.token.clone()) + TtToken(self.span, self.token.clone()) } } @@ -617,9 +617,9 @@ impl Delimiter { /// If the syntax extension is an MBE macro, it will attempt to match its /// LHS "matchers" against the provided token tree, and if it finds a /// match, will transcribe the RHS token tree, splicing in any captured -/// `macro_parser::matched_nonterminals` into the `TTNonterminal`s it finds. +/// `macro_parser::matched_nonterminals` into the `TtNonterminal`s it finds. /// -/// The RHS of an MBE macro is the only place a `TTNonterminal` or `TTSequence` +/// The RHS of an MBE macro is the only place a `TtNonterminal` or `TtSequence` /// makes any real sense. You could write them elsewhere but nothing /// else knows what to do with them, so you'll probably get a syntax /// error. @@ -627,10 +627,10 @@ impl Delimiter { #[doc="For macro invocations; parsing is delegated to the macro"] pub enum TokenTree { /// A single token - TTToken(Span, ::parse::token::Token), + TtToken(Span, ::parse::token::Token), /// A delimited sequence of token trees // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TTDelimited(Span, Delimiter, Rc>, Delimiter), + TtDelimited(Span, Delimiter, Rc>, Delimiter), // These only make sense for right-hand-sides of MBE macros: @@ -638,20 +638,20 @@ pub enum TokenTree { /// an optional separator, and a boolean where true indicates /// zero or more (..), and false indicates one or more (+). // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TTSequence(Span, Rc>, Option<::parse::token::Token>, bool), + TtSequence(Span, Rc>, Option<::parse::token::Token>, bool), /// A syntactic variable that will be filled in by macro expansion. - TTNonterminal(Span, Ident) + TtNonterminal(Span, Ident) } impl TokenTree { /// Returns the `Span` corresponding to this token tree. pub fn get_span(&self) -> Span { match *self { - TTToken(span, _) => span, - TTDelimited(span, _, _, _) => span, - TTSequence(span, _, _, _) => span, - TTNonterminal(span, _) => span, + TtToken(span, _) => span, + TtDelimited(span, _, _, _) => span, + TtSequence(span, _, _, _) => span, + TtNonterminal(span, _) => span, } } } diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 8ea08c58d06..b8795ad5be8 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -50,7 +50,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt, token_tree: &[TokenTree]) -> Box { let code = match token_tree { - [ast::TTToken(_, token::IDENT(code, _))] => code, + [ast::TtToken(_, token::IDENT(code, _))] => code, _ => unreachable!() }; with_registered_diagnostics(|diagnostics| { @@ -82,12 +82,12 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt, token_tree: &[TokenTree]) -> Box { let (code, description) = match token_tree { - [ast::TTToken(_, token::IDENT(ref code, _))] => { + [ast::TtToken(_, token::IDENT(ref code, _))] => { (code, None) }, - [ast::TTToken(_, token::IDENT(ref code, _)), - ast::TTToken(_, token::COMMA), - ast::TTToken(_, token::LIT_STR_RAW(description, _))] => { + [ast::TtToken(_, token::IDENT(ref code, _)), + ast::TtToken(_, token::COMMA), + ast::TtToken(_, token::LIT_STR_RAW(description, _))] => { (code, Some(description)) } _ => unreachable!() @@ -110,7 +110,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt, token_tree: &[TokenTree]) -> Box { let name = match token_tree { - [ast::TTToken(_, token::IDENT(ref name, _))] => name, + [ast::TtToken(_, token::IDENT(ref name, _))] => name, _ => unreachable!() }; diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index b5cc2d95890..64c8068607a 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -684,8 +684,8 @@ pub fn get_single_str_from_tts(cx: &ExtCtxt, cx.span_err(sp, format!("{} takes 1 argument.", name).as_slice()); } else { match tts[0] { - ast::TTToken(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())), - ast::TTToken(_, token::LIT_STR_RAW(ident, _)) => { + ast::TtToken(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())), + ast::TtToken(_, token::LIT_STR_RAW(ident, _)) => { return Some(parse::raw_str_lit(ident.as_str())) } _ => { diff --git a/src/libsyntax/ext/concat_idents.rs b/src/libsyntax/ext/concat_idents.rs index e6befdd2aac..e12f9ee133a 100644 --- a/src/libsyntax/ext/concat_idents.rs +++ b/src/libsyntax/ext/concat_idents.rs @@ -23,7 +23,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree] for (i, e) in tts.iter().enumerate() { if i & 1 == 1 { match *e { - ast::TTToken(_, token::COMMA) => (), + ast::TtToken(_, token::COMMA) => (), _ => { cx.span_err(sp, "concat_idents! expecting comma."); return DummyResult::expr(sp); @@ -31,7 +31,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree] } } else { match *e { - ast::TTToken(_, token::IDENT(ident,_)) => { + ast::TtToken(_, token::IDENT(ident,_)) => { res_str.push_str(token::get_ident(ident).get()) } _ => { diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index baba38d8cbb..5c4290d217b 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -23,7 +23,7 @@ use ptr::P; * * This is registered as a set of expression syntax extension called quote! * that lifts its argument token-tree to an AST representing the -* construction of the same token tree, with ast::TTNonterminal nodes +* construction of the same token tree, with ast::TtNonterminal nodes * interpreted as antiquotes (splices). * */ @@ -639,10 +639,10 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { match *tt { - ast::TTToken(sp, ref tok) => { + ast::TtToken(sp, ref tok) => { let e_sp = cx.expr_ident(sp, id_ext("_sp")); let e_tok = cx.expr_call(sp, - mk_ast_path(cx, sp, "TTToken"), + mk_ast_path(cx, sp, "TtToken"), vec!(e_sp, mk_token(cx, sp, tok))); let e_push = cx.expr_method_call(sp, @@ -651,14 +651,14 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { vec!(e_tok)); vec!(cx.stmt_expr(e_push)) }, - ast::TTDelimited(sp, ref open, ref tts, ref close) => { + ast::TtDelimited(sp, ref open, ref tts, ref close) => { mk_tt(cx, sp, &open.to_tt()).into_iter() .chain(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter())) .chain(mk_tt(cx, sp, &close.to_tt()).into_iter()) .collect() }, - ast::TTSequence(..) => fail!("TTSequence in quote!"), - ast::TTNonterminal(sp, ident) => { + ast::TtSequence(..) => fail!("TtSequence in quote!"), + ast::TtNonterminal(sp, ident) => { // tt.extend($ident.to_tokens(ext_cx).into_iter()) let e_to_toks = @@ -692,7 +692,7 @@ fn mk_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree]) fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree]) -> (P, P) { // NB: It appears that the main parser loses its mind if we consider - // $foo as a TTNonterminal during the main parse, so we have to re-parse + // $foo as a TtNonterminal during the main parse, so we have to re-parse // under quote_depth > 0. This is silly and should go away; the _guess_ is // it has to do with transition away from supporting old-style macros, so // try removing it when enough of them are gone. diff --git a/src/libsyntax/ext/trace_macros.rs b/src/libsyntax/ext/trace_macros.rs index 4c3846731f4..abf798ddacb 100644 --- a/src/libsyntax/ext/trace_macros.rs +++ b/src/libsyntax/ext/trace_macros.rs @@ -20,10 +20,10 @@ pub fn expand_trace_macros(cx: &mut ExtCtxt, tt: &[ast::TokenTree]) -> Box { match tt { - [ast::TTToken(_, ref tok)] if is_keyword(keywords::True, tok) => { + [ast::TtToken(_, ref tok)] if is_keyword(keywords::True, tok) => { cx.set_trace_macros(true); } - [ast::TTToken(_, ref tok)] if is_keyword(keywords::False, tok) => { + [ast::TtToken(_, ref tok)] if is_keyword(keywords::False, tok) => { cx.set_trace_macros(false); } _ => cx.span_err(sp, "trace_macros! accepts only `true` or `false`"), diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 4a3828a8043..75ad2e0fde8 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TTDelimited}; +use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TtDelimited}; use ast; use codemap::{Span, Spanned, DUMMY_SP}; use ext::base::{ExtCtxt, MacResult, MacroDef}; @@ -172,7 +172,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, MatchedNonterminal(NtTT(ref tt)) => { match **tt { // ignore delimiters - TTDelimited(_, _, ref tts, _) => (**tts).clone(), + TtDelimited(_, _, ref tts, _) => (**tts).clone(), _ => cx.span_fatal(sp, "macro rhs must be delimited"), } }, diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index c0b66851dfe..59b87afe0ee 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -9,7 +9,7 @@ // except according to those terms. use ast; -use ast::{TokenTree, TTDelimited, TTToken, TTSequence, TTNonterminal, Ident}; +use ast::{TokenTree, TtDelimited, TtToken, TtSequence, TtNonterminal, Ident}; use codemap::{Span, DUMMY_SP}; use diagnostic::SpanHandler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; @@ -45,7 +45,7 @@ pub struct TtReader<'a> { } /// This can do Macro-By-Example transcription. On the other hand, if -/// `src` contains no `TTSequence`s and `TTNonterminal`s, `interp` can (and +/// `src` contains no `TtSequence`s and `TtNonterminal`s, `interp` can (and /// should) be none. pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler, interp: Option>>, @@ -130,13 +130,13 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { match *t { // The opening and closing delimiters are both tokens, so they are // treated as `LisUnconstrained`. - TTDelimited(_, _, ref tts, _) | TTSequence(_, ref tts, _, _) => { + TtDelimited(_, _, ref tts, _) | TtSequence(_, ref tts, _, _) => { tts.iter().fold(LisUnconstrained, |size, tt| { size + lockstep_iter_size(tt, r) }) }, - TTToken(..) => LisUnconstrained, - TTNonterminal(_, name) => match *lookup_cur_matched(r, name) { + TtToken(..) => LisUnconstrained, + TtNonterminal(_, name) => match *lookup_cur_matched(r, name) { MatchedNonterminal(_) => LisUnconstrained, MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name) }, @@ -194,15 +194,15 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } } } - loop { /* because it's easiest, this handles `TTDelimited` not starting - with a `TTToken`, even though it won't happen */ + loop { /* because it's easiest, this handles `TtDelimited` not starting + with a `TtToken`, even though it won't happen */ let t = { let frame = r.stack.last().unwrap(); // FIXME(pcwalton): Bad copy. (*frame.forest)[frame.idx].clone() }; match t { - TTDelimited(_, open, tts, close) => { + TtDelimited(_, open, tts, close) => { let mut forest = Vec::with_capacity(1 + tts.len() + 1); forest.push(open.to_tt()); forest.extend(tts.iter().map(|x| (*x).clone())); @@ -216,15 +216,15 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { }); // if this could be 0-length, we'd need to potentially recur here } - TTToken(sp, tok) => { + TtToken(sp, tok) => { r.cur_span = sp; r.cur_tok = tok; r.stack.last_mut().unwrap().idx += 1; return ret_val; } - TTSequence(sp, tts, sep, zerok) => { + TtSequence(sp, tts, sep, zerok) => { // FIXME(pcwalton): Bad copy. - match lockstep_iter_size(&TTSequence(sp, tts.clone(), sep.clone(), zerok), r) { + match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), zerok), r) { LisUnconstrained => { r.sp_diag.span_fatal( sp.clone(), /* blame macro writer */ @@ -259,7 +259,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } } // FIXME #2887: think about span stuff here - TTNonterminal(sp, ident) => { + TtNonterminal(sp, ident) => { r.stack.last_mut().unwrap().idx += 1; match *lookup_cur_matched(r, ident) { /* sidestep the interpolation tricks for ident because diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 9cffce74a09..2dfa69b1f38 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -569,10 +569,10 @@ pub fn noop_fold_arg(Arg {id, pat, ty}: Arg, fld: &mut T) -> Arg { pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { match *tt { - TTToken(span, ref tok) => - TTToken(span, fld.fold_token(tok.clone())), - TTDelimited(span, ref open, ref tts, ref close) => - TTDelimited(span, + TtToken(span, ref tok) => + TtToken(span, fld.fold_token(tok.clone())), + TtDelimited(span, ref open, ref tts, ref close) => + TtDelimited(span, Delimiter { span: open.span, token: fld.fold_token(open.token.clone()) @@ -582,13 +582,13 @@ pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { span: close.span, token: fld.fold_token(close.token.clone()) }), - TTSequence(span, ref pattern, ref sep, is_optional) => - TTSequence(span, + TtSequence(span, ref pattern, ref sep, is_optional) => + TtSequence(span, Rc::new(fld.fold_tts(pattern.as_slice())), sep.clone().map(|tok| fld.fold_token(tok)), is_optional), - TTNonterminal(sp,ref ident) => - TTNonterminal(sp,fld.fold_ident(*ident)) + TtNonterminal(sp,ref ident) => + TtNonterminal(sp,fld.fold_ident(*ident)) } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index a2e40282321..d7438f11a94 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -793,29 +793,29 @@ mod test { let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string()); let tts: &[ast::TokenTree] = tts.as_slice(); match tts { - [ast::TTToken(_, _), - ast::TTToken(_, token::NOT), - ast::TTToken(_, _), - ast::TTDelimited(_, ast::TTToken(_, token::LPAREN), + [ast::TtToken(_, _), + ast::TtToken(_, token::NOT), + ast::TtToken(_, _), + ast::TtDelimited(_, ast::TtToken(_, token::LPAREN), ref delim_elts, - ast::TTToken(_, token::RPAREN))] => { + ast::TtToken(_, token::RPAREN))] => { let delim_elts: &[ast::TokenTree] = delim_elts.as_slice(); match delim_elts { - [ast::TTDelimited(_, ast::TTToken(_, token::LPAREN), + [ast::TtDelimited(_, ast::TtToken(_, token::LPAREN), ref first_set, - ast::TTToken(_, token::RPAREN)), - ast::TTToken(_, token::FAT_ARROW), - ast::TTDelimited(_, ast::TTToken(_, token::LPAREN), + ast::TtToken(_, token::RPAREN)), + ast::TtToken(_, token::FAT_ARROW), + ast::TtDelimited(_, ast::TtToken(_, token::LPAREN), ref second_set, - ast::TTToken(_, token::RPAREN))] => { + ast::TtToken(_, token::RPAREN))] => { let first_set: &[ast::TokenTree] = first_set.as_slice(); match first_set { - [ast::TTToken(_, token::DOLLAR), ast::TTToken(_, _)] => { + [ast::TtToken(_, token::DOLLAR), ast::TtToken(_, _)] => { let second_set: &[ast::TokenTree] = second_set.as_slice(); match second_set { - [ast::TTToken(_, token::DOLLAR), ast::TTToken(_, _)] => { + [ast::TtToken(_, token::DOLLAR), ast::TtToken(_, _)] => { assert_eq!("correct","correct") } _ => assert_eq!("wrong 4","correct") @@ -845,7 +845,7 @@ mod test { assert_eq!(json::encode(&tts), "[\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ {\ @@ -858,7 +858,7 @@ mod test { ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ {\ @@ -871,18 +871,18 @@ mod test { ]\ },\ {\ - \"variant\":\"TTDelimited\",\ + \"variant\":\"TtDelimited\",\ \"fields\":[\ [\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ \"LPAREN\"\ ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ {\ @@ -895,14 +895,14 @@ mod test { ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ \"COLON\"\ ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ {\ @@ -915,7 +915,7 @@ mod test { ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ \"RPAREN\"\ @@ -925,18 +925,18 @@ mod test { ]\ },\ {\ - \"variant\":\"TTDelimited\",\ + \"variant\":\"TtDelimited\",\ \"fields\":[\ [\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ \"LBRACE\"\ ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ {\ @@ -949,14 +949,14 @@ mod test { ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ \"SEMI\"\ ]\ },\ {\ - \"variant\":\"TTToken\",\ + \"variant\":\"TtToken\",\ \"fields\":[\ null,\ \"RBRACE\"\ diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 1ed7baa13b4..ebca362b9d8 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -48,8 +48,8 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField}; use ast::{StructVariantKind, BiSub}; use ast::StrStyle; use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue}; -use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TTDelimited, TTSequence, TTToken}; -use ast::{TTNonterminal, TupleVariantKind, Ty, Ty_, TyBot}; +use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TtDelimited, TtSequence, TtToken}; +use ast::{TtNonterminal, TupleVariantKind, Ty, Ty_, TyBot}; use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn}; use ast::{TyTypeof, TyInfer, TypeMethod}; use ast::{TyNil, TyParam, TyParamBound, TyParen, TyPath, TyPtr, TyQPath}; @@ -2526,8 +2526,8 @@ impl<'a> Parser<'a> { /// parse a single token tree from the input. pub fn parse_token_tree(&mut self) -> TokenTree { // FIXME #6994: currently, this is too eager. It - // parses token trees but also identifies TTSequence's - // and TTNonterminal's; it's too early to know yet + // parses token trees but also identifies TtSequence's + // and TtNonterminal's; it's too early to know yet // whether something will be a nonterminal or a seq // yet. maybe_whole!(deref self, NtTT); @@ -2568,13 +2568,13 @@ impl<'a> Parser<'a> { let seq = match seq { Spanned { node, .. } => node, }; - TTSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), s, z) + TtSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), s, z) } else { - TTNonterminal(sp, p.parse_ident()) + TtNonterminal(sp, p.parse_ident()) } } _ => { - TTToken(p.span, p.bump_and_get()) + TtToken(p.span, p.bump_and_get()) } } } @@ -2615,7 +2615,7 @@ impl<'a> Parser<'a> { // Expand to cover the entire delimited token tree let span = Span { hi: self.span.hi, ..pre_span }; - TTDelimited(span, open, Rc::new(tts), close) + TtDelimited(span, open, Rc::new(tts), close) } _ => parse_non_delim_tt_tok(self) } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 9a102d22971..e3b7a164108 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1020,14 +1020,14 @@ impl<'a> State<'a> { /// expression arguments as expressions). It can be done! I think. pub fn print_tt(&mut self, tt: &ast::TokenTree) -> IoResult<()> { match *tt { - ast::TTDelimited(_, ref open, ref tts, ref close) => { + ast::TtDelimited(_, ref open, ref tts, ref close) => { try!(word(&mut self.s, parse::token::to_string(&open.token).as_slice())); try!(space(&mut self.s)); try!(self.print_tts(tts.as_slice())); try!(space(&mut self.s)); word(&mut self.s, parse::token::to_string(&close.token).as_slice()) }, - ast::TTToken(_, ref tk) => { + ast::TtToken(_, ref tk) => { try!(word(&mut self.s, parse::token::to_string(tk).as_slice())); match *tk { parse::token::DOC_COMMENT(..) => { @@ -1036,7 +1036,7 @@ impl<'a> State<'a> { _ => Ok(()) } } - ast::TTSequence(_, ref tts, ref sep, zerok) => { + ast::TtSequence(_, ref tts, ref sep, zerok) => { try!(word(&mut self.s, "$(")); for tt_elt in (*tts).iter() { try!(self.print_tt(tt_elt)); @@ -1051,7 +1051,7 @@ impl<'a> State<'a> { } word(&mut self.s, if zerok { "*" } else { "+" }) } - ast::TTNonterminal(_, name) => { + ast::TtNonterminal(_, name) => { try!(word(&mut self.s, "$")); self.print_ident(name) } diff --git a/src/test/auxiliary/roman_numerals.rs b/src/test/auxiliary/roman_numerals.rs index 0d5abb8fb5d..40ed3a35ddf 100644 --- a/src/test/auxiliary/roman_numerals.rs +++ b/src/test/auxiliary/roman_numerals.rs @@ -18,7 +18,7 @@ extern crate rustc; use syntax::codemap::Span; use syntax::parse::token::{IDENT, get_ident}; -use syntax::ast::{TokenTree, TTToken}; +use syntax::ast::{TokenTree, TtToken}; use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacExpr}; use syntax::ext::build::AstBuilder; // trait for expr_uint use rustc::plugin::Registry; @@ -39,7 +39,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) ("I", 1)]; let text = match args { - [TTToken(_, IDENT(s, _))] => get_ident(s).to_string(), + [TtToken(_, IDENT(s, _))] => get_ident(s).to_string(), _ => { cx.span_err(sp, "argument should be a single identifier"); return DummyResult::any(sp); From 34dacb80cea4071233fb74b479e1f8c148a0be03 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Thu, 23 Oct 2014 04:58:48 +1100 Subject: [PATCH 5/7] Reduce the size of the TokenTree --- src/libsyntax/ast.rs | 5 ++--- src/libsyntax/ext/quote.rs | 3 ++- src/libsyntax/ext/tt/macro_rules.rs | 5 ++++- src/libsyntax/ext/tt/transcribe.rs | 13 +++++++++---- src/libsyntax/fold.rs | 25 ++++++++++++++----------- src/libsyntax/parse/parser.rs | 2 +- src/libsyntax/print/pprust.rs | 3 ++- 7 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index f87c7cf0215..a6156bfa496 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -629,8 +629,7 @@ pub enum TokenTree { /// A single token TtToken(Span, ::parse::token::Token), /// A delimited sequence of token trees - // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TtDelimited(Span, Delimiter, Rc>, Delimiter), + TtDelimited(Span, Rc<(Delimiter, Vec, Delimiter)>), // These only make sense for right-hand-sides of MBE macros: @@ -649,7 +648,7 @@ impl TokenTree { pub fn get_span(&self) -> Span { match *self { TtToken(span, _) => span, - TtDelimited(span, _, _, _) => span, + TtDelimited(span, _) => span, TtSequence(span, _, _, _) => span, TtNonterminal(span, _) => span, } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 5c4290d217b..6f1fd90adfa 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -651,7 +651,8 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { vec!(e_tok)); vec!(cx.stmt_expr(e_push)) }, - ast::TtDelimited(sp, ref open, ref tts, ref close) => { + ast::TtDelimited(sp, ref delimed) => { + let (ref open, ref tts, ref close) = **delimed; mk_tt(cx, sp, &open.to_tt()).into_iter() .chain(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter())) .chain(mk_tt(cx, sp, &close.to_tt()).into_iter()) diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 75ad2e0fde8..8b45cf34e80 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -172,7 +172,10 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, MatchedNonterminal(NtTT(ref tt)) => { match **tt { // ignore delimiters - TtDelimited(_, _, ref tts, _) => (**tts).clone(), + TtDelimited(_, ref delimed) => { + let (_, ref tts, _) = **delimed; + tts.clone() + }, _ => cx.span_fatal(sp, "macro rhs must be delimited"), } }, diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 59b87afe0ee..fde950e4999 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -128,9 +128,13 @@ impl Add for LockstepIterSize { fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { match *t { - // The opening and closing delimiters are both tokens, so they are - // treated as `LisUnconstrained`. - TtDelimited(_, _, ref tts, _) | TtSequence(_, ref tts, _, _) => { + TtDelimited(_, ref delimed) => { + let (_, ref tts, _) = **delimed; + tts.iter().fold(LisUnconstrained, |size, tt| { + size + lockstep_iter_size(tt, r) + }) + }, + TtSequence(_, ref tts, _, _) => { tts.iter().fold(LisUnconstrained, |size, tt| { size + lockstep_iter_size(tt, r) }) @@ -202,7 +206,8 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { (*frame.forest)[frame.idx].clone() }; match t { - TtDelimited(_, open, tts, close) => { + TtDelimited(_, ref delimed) => { + let (ref open, ref tts, ref close) = **delimed; let mut forest = Vec::with_capacity(1 + tts.len() + 1); forest.push(open.to_tt()); forest.extend(tts.iter().map(|x| (*x).clone())); diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 2dfa69b1f38..0f9ab5c6b26 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -571,17 +571,20 @@ pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { match *tt { TtToken(span, ref tok) => TtToken(span, fld.fold_token(tok.clone())), - TtDelimited(span, ref open, ref tts, ref close) => - TtDelimited(span, - Delimiter { - span: open.span, - token: fld.fold_token(open.token.clone()) - }, - Rc::new(fld.fold_tts(tts.as_slice())), - Delimiter { - span: close.span, - token: fld.fold_token(close.token.clone()) - }), + TtDelimited(span, ref delimed) => { + let (ref open, ref tts, ref close) = **delimed; + TtDelimited(span, Rc::new(( + Delimiter { + span: open.span, + token: fld.fold_token(open.token.clone()) + }, + fld.fold_tts(tts.as_slice()), + Delimiter { + span: close.span, + token: fld.fold_token(close.token.clone()) + }, + ))) + }, TtSequence(span, ref pattern, ref sep, is_optional) => TtSequence(span, Rc::new(fld.fold_tts(pattern.as_slice())), diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index ebca362b9d8..f8fa053b7ae 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2615,7 +2615,7 @@ impl<'a> Parser<'a> { // Expand to cover the entire delimited token tree let span = Span { hi: self.span.hi, ..pre_span }; - TtDelimited(span, open, Rc::new(tts), close) + TtDelimited(span, Rc::new((open, tts, close))) } _ => parse_non_delim_tt_tok(self) } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index e3b7a164108..97c177b696c 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1020,7 +1020,8 @@ impl<'a> State<'a> { /// expression arguments as expressions). It can be done! I think. pub fn print_tt(&mut self, tt: &ast::TokenTree) -> IoResult<()> { match *tt { - ast::TtDelimited(_, ref open, ref tts, ref close) => { + ast::TtDelimited(_, ref delimed) => { + let (ref open, ref tts, ref close) = **delimed; try!(word(&mut self.s, parse::token::to_string(&open.token).as_slice())); try!(space(&mut self.s)); try!(self.print_tts(tts.as_slice())); From 94d6eee3357e24913d1331b1fe0bd4e4524bdab6 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Thu, 23 Oct 2014 11:24:20 +1100 Subject: [PATCH 6/7] Add a KleeneOp enum for clarity --- src/libsyntax/ast.rs | 19 +++++++++++------- src/libsyntax/ext/tt/macro_parser.rs | 4 ++-- src/libsyntax/ext/tt/macro_rules.rs | 5 +++-- src/libsyntax/ext/tt/transcribe.rs | 6 +++--- src/libsyntax/parse/parser.rs | 29 +++++++++++++++------------- src/libsyntax/print/pprust.rs | 9 ++++++--- 6 files changed, 42 insertions(+), 30 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index a6156bfa496..580b93eb4c6 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -609,6 +609,14 @@ impl Delimiter { } } +/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) +/// for token sequences. +#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] +pub enum KleeneOp { + ZeroOrMore, + OneOrMore, +} + /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -633,12 +641,9 @@ pub enum TokenTree { // These only make sense for right-hand-sides of MBE macros: - /// A kleene-style repetition sequence with a span, a `TTForest`, - /// an optional separator, and a boolean where true indicates - /// zero or more (..), and false indicates one or more (+). + /// A Kleene-style repetition sequence with an optional separator. // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TtSequence(Span, Rc>, Option<::parse::token::Token>, bool), - + TtSequence(Span, Rc>, Option<::parse::token::Token>, KleeneOp), /// A syntactic variable that will be filled in by macro expansion. TtNonterminal(Span, Ident) } @@ -711,9 +716,9 @@ pub type Matcher = Spanned; pub enum Matcher_ { /// Match one token MatchTok(::parse::token::Token), - /// Match repetitions of a sequence: body, separator, zero ok?, + /// Match repetitions of a sequence: body, separator, Kleene operator, /// lo, hi position-in-match-array used: - MatchSeq(Vec , Option<::parse::token::Token>, bool, uint, uint), + MatchSeq(Vec , Option<::parse::token::Token>, KleeneOp, uint, uint), /// Parse a Rust NT: name to bind, name of NT, position in match array: MatchNonterminal(Ident, Ident, uint) } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index f2081674fb7..cea8cab5265 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -323,9 +323,9 @@ pub fn parse(sess: &ParseSess, } else { match ei.elts[idx].node.clone() { /* need to descend into sequence */ - MatchSeq(ref matchers, ref sep, zero_ok, + MatchSeq(ref matchers, ref sep, kleene_op, match_idx_lo, match_idx_hi) => { - if zero_ok { + if kleene_op == ast::ZeroOrMore { let mut new_ei = ei.clone(); new_ei.idx += 1u; //we specifically matched zero repeats. diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 8b45cf34e80..3b51fb380b8 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -232,10 +232,11 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt, ms(MatchSeq(vec!( ms(MatchNonterminal(lhs_nm, special_idents::matchers, 0u)), ms(MatchTok(FAT_ARROW)), - ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), Some(SEMI), false, 0u, 2u)), + ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), Some(SEMI), + ast::OneOrMore, 0u, 2u)), //to phase into semicolon-termination instead of //semicolon-separation - ms(MatchSeq(vec!(ms(MatchTok(SEMI))), None, true, 2u, 2u))); + ms(MatchSeq(vec!(ms(MatchTok(SEMI))), None, ast::ZeroOrMore, 2u, 2u))); // Parse the macro_rules! invocation (`none` is for no interpolations): diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index fde950e4999..1bb519f66cd 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -227,9 +227,9 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { r.stack.last_mut().unwrap().idx += 1; return ret_val; } - TtSequence(sp, tts, sep, zerok) => { + TtSequence(sp, tts, sep, kleene_op) => { // FIXME(pcwalton): Bad copy. - match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), zerok), r) { + match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), kleene_op), r) { LisUnconstrained => { r.sp_diag.span_fatal( sp.clone(), /* blame macro writer */ @@ -243,7 +243,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } LisConstraint(len, _) => { if len == 0 { - if !zerok { + if kleene_op == ast::OneOrMore { // FIXME #2887 blame invoker r.sp_diag.span_fatal(sp.clone(), "this must repeat at least once"); diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index f8fa053b7ae..7bf751c2d5e 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2497,27 +2497,30 @@ impl<'a> Parser<'a> { return e; } - /// Parse an optional separator followed by a kleene-style + /// Parse an optional separator followed by a Kleene-style /// repetition token (+ or *). - pub fn parse_sep_and_zerok(&mut self) -> (Option, bool) { - fn parse_zerok(parser: &mut Parser) -> Option { + pub fn parse_sep_and_kleene_op(&mut self) -> (Option, ast::KleeneOp) { + fn parse_kleene_op(parser: &mut Parser) -> Option { match parser.token { - token::BINOP(token::STAR) | token::BINOP(token::PLUS) => { - let zerok = parser.token == token::BINOP(token::STAR); + token::BINOP(token::STAR) => { parser.bump(); - Some(zerok) + Some(ast::ZeroOrMore) + }, + token::BINOP(token::PLUS) => { + parser.bump(); + Some(ast::OneOrMore) }, _ => None } }; - match parse_zerok(self) { - Some(zerok) => return (None, zerok), + match parse_kleene_op(self) { + Some(kleene_op) => return (None, kleene_op), None => {} } let separator = self.bump_and_get(); - match parse_zerok(self) { + match parse_kleene_op(self) { Some(zerok) => (Some(separator), zerok), None => self.fatal("expected `*` or `+`") } @@ -2564,11 +2567,11 @@ impl<'a> Parser<'a> { seq_sep_none(), |p| p.parse_token_tree() ); - let (s, z) = p.parse_sep_and_zerok(); + let (sep, repeat) = p.parse_sep_and_kleene_op(); let seq = match seq { Spanned { node, .. } => node, }; - TtSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), s, z) + TtSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), sep, repeat) } else { TtNonterminal(sp, p.parse_ident()) } @@ -2679,8 +2682,8 @@ impl<'a> Parser<'a> { if ms.len() == 0u { self.fatal("repetition body must be nonempty"); } - let (sep, zerok) = self.parse_sep_and_zerok(); - MatchSeq(ms, sep, zerok, name_idx_lo, *name_idx) + let (sep, kleene_op) = self.parse_sep_and_kleene_op(); + MatchSeq(ms, sep, kleene_op, name_idx_lo, *name_idx) } else { let bound_to = self.parse_ident(); self.expect(&token::COLON); diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 97c177b696c..0a77343547b 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1037,20 +1037,23 @@ impl<'a> State<'a> { _ => Ok(()) } } - ast::TtSequence(_, ref tts, ref sep, zerok) => { + ast::TtSequence(_, ref tts, ref separator, kleene_op) => { try!(word(&mut self.s, "$(")); for tt_elt in (*tts).iter() { try!(self.print_tt(tt_elt)); } try!(word(&mut self.s, ")")); - match *sep { + match *separator { Some(ref tk) => { try!(word(&mut self.s, parse::token::to_string(tk).as_slice())); } None => () } - word(&mut self.s, if zerok { "*" } else { "+" }) + match kleene_op { + ast::ZeroOrMore => word(&mut self.s, "*"), + ast::OneOrMore => word(&mut self.s, "+"), + } } ast::TtNonterminal(_, name) => { try!(word(&mut self.s, "$")); From 6598d33bd0edf22adb24423851bf2761cae0ada0 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Sun, 26 Oct 2014 10:51:41 +1100 Subject: [PATCH 7/7] Update parse::test::string_to_tts_1 test --- src/libsyntax/parse/mod.rs | 213 +++++++++++++++++-------------------- 1 file changed, 100 insertions(+), 113 deletions(-) diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index d7438f11a94..2965094f236 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -793,54 +793,47 @@ mod test { let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string()); let tts: &[ast::TokenTree] = tts.as_slice(); match tts { - [ast::TtToken(_, _), + [ast::TtToken(_, token::IDENT(name_macro_rules, false)), ast::TtToken(_, token::NOT), - ast::TtToken(_, _), - ast::TtDelimited(_, ast::TtToken(_, token::LPAREN), - ref delim_elts, - ast::TtToken(_, token::RPAREN))] => { - let delim_elts: &[ast::TokenTree] = delim_elts.as_slice(); - match delim_elts { - [ast::TtDelimited(_, ast::TtToken(_, token::LPAREN), - ref first_set, - ast::TtToken(_, token::RPAREN)), - ast::TtToken(_, token::FAT_ARROW), - ast::TtDelimited(_, ast::TtToken(_, token::LPAREN), - ref second_set, - ast::TtToken(_, token::RPAREN))] => { - let first_set: &[ast::TokenTree] = - first_set.as_slice(); - match first_set { - [ast::TtToken(_, token::DOLLAR), ast::TtToken(_, _)] => { - let second_set: &[ast::TokenTree] = - second_set.as_slice(); - match second_set { - [ast::TtToken(_, token::DOLLAR), ast::TtToken(_, _)] => { - assert_eq!("correct","correct") - } - _ => assert_eq!("wrong 4","correct") - } - }, - _ => { - error!("failing value 3: {}",first_set); - assert_eq!("wrong 3","correct") - } + ast::TtToken(_, token::IDENT(name_zip, false)), + ast::TtDelimited(_, ref macro_delimed)] + if name_macro_rules.as_str() == "macro_rules" + && name_zip.as_str() == "zip" => { + let (ref macro_open, ref macro_tts, ref macro_close) = **macro_delimed; + match (macro_open, macro_tts.as_slice(), macro_close) { + (&ast::Delimiter { token: token::LPAREN, .. }, + [ast::TtDelimited(_, ref first_delimed), + ast::TtToken(_, token::FAT_ARROW), + ast::TtDelimited(_, ref second_delimed)], + &ast::Delimiter { token: token::RPAREN, .. }) => { + let (ref first_open, ref first_tts, ref first_close) = **first_delimed; + match (first_open, first_tts.as_slice(), first_close) { + (&ast::Delimiter { token: token::LPAREN, .. }, + [ast::TtToken(_, token::DOLLAR), + ast::TtToken(_, token::IDENT(name, false))], + &ast::Delimiter { token: token::RPAREN, .. }) + if name.as_str() == "a" => {}, + _ => fail!("value 3: {}", **first_delimed), + } + let (ref second_open, ref second_tts, ref second_close) = **second_delimed; + match (second_open, second_tts.as_slice(), second_close) { + (&ast::Delimiter { token: token::LPAREN, .. }, + [ast::TtToken(_, token::DOLLAR), + ast::TtToken(_, token::IDENT(name, false))], + &ast::Delimiter { token: token::RPAREN, .. }) + if name.as_str() == "a" => {}, + _ => fail!("value 4: {}", **second_delimed), } }, - _ => { - error!("failing value 2: {}",delim_elts); - assert_eq!("wrong","correct"); - } + _ => fail!("value 2: {}", **macro_delimed), } }, - _ => { - error!("failing value: {}",tts); - assert_eq!("wrong 1","correct"); - }, + _ => fail!("value: {}",tts), } } - #[test] fn string_to_tts_1 () { + #[test] + fn string_to_tts_1 () { let tts = string_to_tts("fn a (b : int) { b; }".to_string()); assert_eq!(json::encode(&tts), "[\ @@ -873,53 +866,50 @@ mod test { {\ \"variant\":\"TtDelimited\",\ \"fields\":[\ + null,\ [\ {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"LPAREN\"\ - ]\ + \"span\":null,\ + \"token\":\"LPAREN\"\ },\ + [\ + {\ + \"variant\":\"TtToken\",\ + \"fields\":[\ + null,\ + {\ + \"variant\":\"IDENT\",\ + \"fields\":[\ + \"b\",\ + false\ + ]\ + }\ + ]\ + },\ + {\ + \"variant\":\"TtToken\",\ + \"fields\":[\ + null,\ + \"COLON\"\ + ]\ + },\ + {\ + \"variant\":\"TtToken\",\ + \"fields\":[\ + null,\ + {\ + \"variant\":\"IDENT\",\ + \"fields\":[\ + \"int\",\ + false\ + ]\ + }\ + ]\ + }\ + ],\ {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"IDENT\",\ - \"fields\":[\ - \"b\",\ - false\ - ]\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"COLON\"\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"IDENT\",\ - \"fields\":[\ - \"int\",\ - false\ - ]\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"RPAREN\"\ - ]\ + \"span\":null,\ + \"token\":\"RPAREN\"\ }\ ]\ ]\ @@ -927,40 +917,37 @@ mod test { {\ \"variant\":\"TtDelimited\",\ \"fields\":[\ + null,\ [\ {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"LBRACE\"\ - ]\ + \"span\":null,\ + \"token\":\"LBRACE\"\ },\ + [\ + {\ + \"variant\":\"TtToken\",\ + \"fields\":[\ + null,\ + {\ + \"variant\":\"IDENT\",\ + \"fields\":[\ + \"b\",\ + false\ + ]\ + }\ + ]\ + },\ + {\ + \"variant\":\"TtToken\",\ + \"fields\":[\ + null,\ + \"SEMI\"\ + ]\ + }\ + ],\ {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - {\ - \"variant\":\"IDENT\",\ - \"fields\":[\ - \"b\",\ - false\ - ]\ - }\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"SEMI\"\ - ]\ - },\ - {\ - \"variant\":\"TtToken\",\ - \"fields\":[\ - null,\ - \"RBRACE\"\ - ]\ + \"span\":null,\ + \"token\":\"RBRACE\"\ }\ ]\ ]\