From 971d776aa5a678672eb3d37f2f507664aacd2440 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Wed, 22 Oct 2014 16:37:20 +1100 Subject: [PATCH] Add Span and separate open/close delims to TTDelim This came up when working [on the gl-rs generator extension](https://github.com/bjz/gl-rs/blob/990383de801bd2e233159d5be07c9b5622827620/src/gl_generator/lib.rs#L135-L146). The new definition of `TTDelim` adds an associated `Span` that covers the whole token tree and enforces the invariant that a delimited sequence of token trees must have an opening and closing delimiter. A `get_span` method has also been added to `TokenTree` type to make it easier to implement better error messages for syntax extensions. --- src/libsyntax/ast.rs | 31 +++++++++++++++-- src/libsyntax/ext/log_syntax.rs | 8 ++--- src/libsyntax/ext/quote.rs | 17 +++++---- src/libsyntax/ext/tt/macro_rules.rs | 20 +++-------- src/libsyntax/ext/tt/transcribe.rs | 54 +++++++++++++++++------------ src/libsyntax/fold.rs | 12 ++++++- src/libsyntax/parse/mod.rs | 37 ++++++++++---------- src/libsyntax/parse/parser.rs | 36 +++++++++++-------- src/libsyntax/print/pprust.rs | 8 ++++- 9 files changed, 135 insertions(+), 88 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 8eaee7282d1..be316ba9f4d 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -592,6 +592,20 @@ pub enum CaptureClause { CaptureByRef, } +/// A token that delimits a sequence of token trees +#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] +pub struct Delimiter { + pub span: Span, + pub token: ::parse::token::Token, +} + +impl Delimiter { + /// Convert the delimiter to a `TTTok` + pub fn to_tt(&self) -> TokenTree { + TTTok(self.span, self.token.clone()) + } +} + /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -611,10 +625,9 @@ pub enum CaptureClause { pub enum TokenTree { /// A single token TTTok(Span, ::parse::token::Token), - /// A delimited sequence (the delimiters appear as the first - /// and last elements of the vector) + /// A delimited sequence of token trees // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TTDelim(Rc>), + TTDelim(Span, Delimiter, Rc>, Delimiter), // These only make sense for right-hand-sides of MBE macros: @@ -628,6 +641,18 @@ pub enum TokenTree { TTNonterminal(Span, Ident) } +impl TokenTree { + /// Returns the `Span` corresponding to this token tree. + pub fn get_span(&self) -> Span { + match *self { + TTTok(span, _) => span, + TTDelim(span, _, _, _) => span, + TTSeq(span, _, _, _) => span, + TTNonterminal(span, _) => span, + } + } +} + // Matchers are nodes defined-by and recognized-by the main rust parser and // language, but they're only ever found inside syntax-extension invocations; // indeed, the only thing that ever _activates_ the rules in the rust parser diff --git a/src/libsyntax/ext/log_syntax.rs b/src/libsyntax/ext/log_syntax.rs index 8df5746e412..30301e3b8cc 100644 --- a/src/libsyntax/ext/log_syntax.rs +++ b/src/libsyntax/ext/log_syntax.rs @@ -13,16 +13,14 @@ use codemap; use ext::base; use print; -use std::rc::Rc; - pub fn expand_syntax_ext<'cx>(cx: &'cx mut base::ExtCtxt, sp: codemap::Span, - tt: &[ast::TokenTree]) + tts: &[ast::TokenTree]) -> Box { cx.print_backtrace(); - println!("{}", print::pprust::tt_to_string(&ast::TTDelim( - Rc::new(tt.iter().map(|x| (*x).clone()).collect())))); + + println!("{}", print::pprust::tts_to_string(tts)); // any so that `log_syntax` can be invoked as an expression and item. base::DummyResult::any(sp) diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 84775c12d64..783c08a4443 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -637,7 +637,7 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { } -fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec> { +fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { match *tt { ast::TTTok(sp, ref tok) => { let e_sp = cx.expr_ident(sp, id_ext("_sp")); @@ -650,13 +650,16 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec> { id_ext("push"), vec!(e_tok)); vec!(cx.stmt_expr(e_push)) - } - - ast::TTDelim(ref tts) => mk_tts(cx, sp, tts.as_slice()), + }, + ast::TTDelim(sp, ref open, ref tts, ref close) => { + let mut stmts = vec![]; + stmts.extend(mk_tt(cx, sp, &open.to_tt()).into_iter()); + stmts.extend(tts.iter().flat_map(|tt| mk_tt(cx, sp, tt).into_iter())); + stmts.extend(mk_tt(cx, sp, &close.to_tt()).into_iter()); + stmts + }, ast::TTSeq(..) => fail!("TTSeq in quote!"), - ast::TTNonterminal(sp, ident) => { - // tt.extend($ident.to_tokens(ext_cx).into_iter()) let e_to_toks = @@ -674,7 +677,7 @@ fn mk_tt(cx: &ExtCtxt, sp: Span, tt: &ast::TokenTree) -> Vec> { vec!(e_to_toks)); vec!(cx.stmt_expr(e_push)) - } + }, } } diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 91db3a9d8df..fbfe10d004e 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -147,13 +147,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, rhses: &[Rc]) -> Box { if cx.trace_macros() { - println!("{}! {} {} {}", + println!("{}! {{ {} }}", token::get_ident(name), - "{", - print::pprust::tt_to_string(&TTDelim(Rc::new(arg.iter() - .map(|x| (*x).clone()) - .collect()))), - "}"); + print::pprust::tts_to_string(arg)); } // Which arm's failure should we report? (the one furthest along) @@ -175,15 +171,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, // okay, what's your transcriber? MatchedNonterminal(NtTT(ref tt)) => { match **tt { - // cut off delimiters; don't parse 'em - TTDelim(ref tts) => { - (*tts).slice(1u,(*tts).len()-1u) - .iter() - .map(|x| (*x).clone()) - .collect() - } - _ => cx.span_fatal( - sp, "macro rhs must be delimited") + // ignore delimiters + TTDelim(_, _, ref tts, _) => (**tts).clone(), + _ => cx.span_fatal(sp, "macro rhs must be delimited"), } }, _ => cx.span_bug(sp, "bad thing in rhs") diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 35ec37d842a..472b24be81b 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -18,6 +18,7 @@ use parse::token; use parse::lexer::TokenAndSpan; use std::rc::Rc; +use std::ops::Add; use std::collections::HashMap; ///an unzipping of `TokenTree`s @@ -104,37 +105,41 @@ enum LockstepIterSize { LisContradiction(String), } -fn lis_merge(lhs: LockstepIterSize, rhs: LockstepIterSize) -> LockstepIterSize { - match lhs { - LisUnconstrained => rhs.clone(), - LisContradiction(_) => lhs.clone(), - LisConstraint(l_len, l_id) => match rhs { - LisUnconstrained => lhs.clone(), - LisContradiction(_) => rhs.clone(), - LisConstraint(r_len, _) if l_len == r_len => lhs.clone(), - LisConstraint(r_len, r_id) => { - let l_n = token::get_ident(l_id); - let r_n = token::get_ident(r_id); - LisContradiction(format!("inconsistent lockstep iteration: \ - '{}' has {} items, but '{}' has {}", - l_n, l_len, r_n, r_len).to_string()) - } +impl Add for LockstepIterSize { + fn add(&self, other: &LockstepIterSize) -> LockstepIterSize { + match *self { + LisUnconstrained => other.clone(), + LisContradiction(_) => self.clone(), + LisConstraint(l_len, l_id) => match *other { + LisUnconstrained => self.clone(), + LisContradiction(_) => other.clone(), + LisConstraint(r_len, _) if l_len == r_len => self.clone(), + LisConstraint(r_len, r_id) => { + let l_n = token::get_ident(l_id); + let r_n = token::get_ident(r_id); + LisContradiction(format!("inconsistent lockstep iteration: \ + '{}' has {} items, but '{}' has {}", + l_n, l_len, r_n, r_len).to_string()) + } + }, } } } fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { match *t { - TTDelim(ref tts) | TTSeq(_, ref tts, _, _) => { - tts.iter().fold(LisUnconstrained, |lis, tt| { - lis_merge(lis, lockstep_iter_size(tt, r)) + // The opening and closing delimiters are both tokens, so they are + // treated as `LisUnconstrained`. + TTDelim(_, _, ref tts, _) | TTSeq(_, ref tts, _, _) => { + tts.iter().fold(LisUnconstrained, |size, tt| { + size + lockstep_iter_size(tt, r) }) - } + }, TTTok(..) => LisUnconstrained, TTNonterminal(_, name) => match *lookup_cur_matched(r, name) { MatchedNonterminal(_) => LisUnconstrained, MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name) - } + }, } } @@ -197,9 +202,14 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { (*frame.forest)[frame.idx].clone() }; match t { - TTDelim(tts) => { + TTDelim(_, open, delimed_tts, close) => { + let mut tts = vec![]; + tts.push(open.to_tt()); + tts.extend(delimed_tts.iter().map(|x| (*x).clone())); + tts.push(close.to_tt()); + r.stack.push(TtFrame { - forest: tts, + forest: Rc::new(tts), idx: 0, dotdotdoted: false, sep: None diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index ceef190f5d4..ddb2ab49f8b 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -571,7 +571,17 @@ pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { match *tt { TTTok(span, ref tok) => TTTok(span, fld.fold_token(tok.clone())), - TTDelim(ref tts) => TTDelim(Rc::new(fld.fold_tts(tts.as_slice()))), + TTDelim(span, ref open, ref tts, ref close) => + TTDelim(span, + Delimiter { + span: open.span, + token: fld.fold_token(open.token.clone()) + }, + Rc::new(fld.fold_tts(tts.as_slice())), + Delimiter { + span: close.span, + token: fld.fold_token(close.token.clone()) + }), TTSeq(span, ref pattern, ref sep, is_optional) => TTSeq(span, Rc::new(fld.fold_tts(pattern.as_slice())), diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 2d7d32cd9ea..1c99b608f7a 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -788,35 +788,34 @@ mod test { } // check the token-tree-ization of macros - #[test] fn string_to_tts_macro () { + #[test] + fn string_to_tts_macro () { let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string()); let tts: &[ast::TokenTree] = tts.as_slice(); match tts { - [ast::TTTok(_,_), - ast::TTTok(_,token::NOT), - ast::TTTok(_,_), - ast::TTDelim(ref delim_elts)] => { + [ast::TTTok(_, _), + ast::TTTok(_, token::NOT), + ast::TTTok(_, _), + ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + ref delim_elts, + ast::TTTok(_, token::RPAREN))] => { let delim_elts: &[ast::TokenTree] = delim_elts.as_slice(); match delim_elts { - [ast::TTTok(_,token::LPAREN), - ast::TTDelim(ref first_set), - ast::TTTok(_,token::FAT_ARROW), - ast::TTDelim(ref second_set), - ast::TTTok(_,token::RPAREN)] => { + [ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + ref first_set, + ast::TTTok(_, token::RPAREN)), + ast::TTTok(_, token::FAT_ARROW), + ast::TTDelim(_, ast::TTTok(_, token::LPAREN), + ref second_set, + ast::TTTok(_, token::RPAREN))] => { let first_set: &[ast::TokenTree] = first_set.as_slice(); match first_set { - [ast::TTTok(_,token::LPAREN), - ast::TTTok(_,token::DOLLAR), - ast::TTTok(_,_), - ast::TTTok(_,token::RPAREN)] => { + [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => { let second_set: &[ast::TokenTree] = second_set.as_slice(); match second_set { - [ast::TTTok(_,token::LPAREN), - ast::TTTok(_,token::DOLLAR), - ast::TTTok(_,_), - ast::TTTok(_,token::RPAREN)] => { + [ast::TTTok(_, token::DOLLAR), ast::TTTok(_, _)] => { assert_eq!("correct","correct") } _ => assert_eq!("wrong 4","correct") @@ -837,7 +836,7 @@ mod test { _ => { error!("failing value: {}",tts); assert_eq!("wrong 1","correct"); - } + }, } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 5abf79836f5..005ed2e7ed3 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -48,7 +48,7 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField}; use ast::{StructVariantKind, BiSub}; use ast::StrStyle; use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue}; -use ast::{TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok}; +use ast::{Delimiter, TokenTree, TraitItem, TraitRef, TTDelim, TTSeq, TTTok}; use ast::{TTNonterminal, TupleVariantKind, Ty, Ty_, TyBot}; use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn}; use ast::{TyTypeof, TyInfer, TypeMethod}; @@ -2574,16 +2574,11 @@ impl<'a> Parser<'a> { } } _ => { - parse_any_tt_tok(p) + TTTok(p.span, p.bump_and_get()) } } } - // turn the next token into a TTTok: - fn parse_any_tt_tok(p: &mut Parser) -> TokenTree { - TTTok(p.span, p.bump_and_get()) - } - match (&self.token, token::close_delimiter_for(&self.token)) { (&token::EOF, _) => { let open_braces = self.open_braces.clone(); @@ -2595,21 +2590,32 @@ impl<'a> Parser<'a> { self.fatal("this file contains an un-closed delimiter "); } (_, Some(close_delim)) => { + // The span for beginning of the delimited section + let pre_span = self.span; + // Parse the open delimiter. self.open_braces.push(self.span); - let mut result = vec!(parse_any_tt_tok(self)); + let open = Delimiter { + span: self.span, + token: self.bump_and_get(), + }; - let trees = - self.parse_seq_to_before_end(&close_delim, - seq_sep_none(), - |p| p.parse_token_tree()); - result.extend(trees.into_iter()); + // Parse the token trees within the delimeters + let tts = self.parse_seq_to_before_end( + &close_delim, seq_sep_none(), |p| p.parse_token_tree() + ); // Parse the close delimiter. - result.push(parse_any_tt_tok(self)); + let close = Delimiter { + span: self.span, + token: self.bump_and_get(), + }; self.open_braces.pop().unwrap(); - TTDelim(Rc::new(result)) + // Expand to cover the entire delimited token tree + let span = Span { hi: self.span.hi, ..pre_span }; + + TTDelim(span, open, Rc::new(tts), close) } _ => parse_non_delim_tt_tok(self) } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index b63f9b0120b..4f4b153d3a9 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1020,7 +1020,13 @@ impl<'a> State<'a> { /// expression arguments as expressions). It can be done! I think. pub fn print_tt(&mut self, tt: &ast::TokenTree) -> IoResult<()> { match *tt { - ast::TTDelim(ref tts) => self.print_tts(tts.as_slice()), + ast::TTDelim(_, ref open, ref tts, ref close) => { + try!(word(&mut self.s, parse::token::to_string(&open.token).as_slice())); + try!(space(&mut self.s)); + try!(self.print_tts(tts.as_slice())); + try!(space(&mut self.s)); + word(&mut self.s, parse::token::to_string(&close.token).as_slice()) + }, ast::TTTok(_, ref tk) => { try!(word(&mut self.s, parse::token::to_string(tk).as_slice())); match *tk {