// Copyright 2018 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. use {Delimiter, Level, Spacing, Span, __internal}; use {Group, Ident, Literal, Punct, TokenTree}; use rustc_errors as errors; use syntax::ast; use syntax::parse::lexer::comments; use syntax::parse::token; use syntax::tokenstream; use syntax_pos::symbol::{keywords, Symbol}; impl Ident { pub(crate) fn new_maybe_raw(string: &str, span: Span, is_raw: bool) -> Ident { let sym = Symbol::intern(string); if is_raw && (sym == keywords::Underscore.name() || ast::Ident::with_empty_ctxt(sym).is_path_segment_keyword()) { panic!("`{:?}` is not a valid raw identifier", string) } Ident { sym, span, is_raw } } } impl Delimiter { pub(crate) fn from_internal(delim: token::DelimToken) -> Delimiter { match delim { token::Paren => Delimiter::Parenthesis, token::Brace => Delimiter::Brace, token::Bracket => Delimiter::Bracket, token::NoDelim => Delimiter::None, } } pub(crate) fn to_internal(self) -> token::DelimToken { match self { Delimiter::Parenthesis => token::Paren, Delimiter::Brace => token::Brace, Delimiter::Bracket => token::Bracket, Delimiter::None => token::NoDelim, } } } impl TokenTree { pub(crate) fn from_internal( stream: tokenstream::TokenStream, stack: &mut Vec, ) -> TokenTree { use syntax::parse::token::*; let (tree, is_joint) = stream.as_tree(); let (span, token) = match tree { tokenstream::TokenTree::Token(span, token) => (span, token), tokenstream::TokenTree::Delimited(span, delimed) => { let delimiter = Delimiter::from_internal(delimed.delim); let mut g = Group::new(delimiter, ::TokenStream(delimed.tts.into())); g.set_span(Span(span)); return g.into(); } }; let op_kind = if is_joint { Spacing::Joint } else { Spacing::Alone }; macro_rules! tt { ($e:expr) => {{ let mut x = TokenTree::from($e); x.set_span(Span(span)); x }}; } macro_rules! op { ($a:expr) => { tt!(Punct::new($a, op_kind)) }; ($a:expr, $b:expr) => {{ stack.push(tt!(Punct::new($b, op_kind))); tt!(Punct::new($a, Spacing::Joint)) }}; ($a:expr, $b:expr, $c:expr) => {{ stack.push(tt!(Punct::new($c, op_kind))); stack.push(tt!(Punct::new($b, Spacing::Joint))); tt!(Punct::new($a, Spacing::Joint)) }}; } match token { Eq => op!('='), Lt => op!('<'), Le => op!('<', '='), EqEq => op!('=', '='), Ne => op!('!', '='), Ge => op!('>', '='), Gt => op!('>'), AndAnd => op!('&', '&'), OrOr => op!('|', '|'), Not => op!('!'), Tilde => op!('~'), BinOp(Plus) => op!('+'), BinOp(Minus) => op!('-'), BinOp(Star) => op!('*'), BinOp(Slash) => op!('/'), BinOp(Percent) => op!('%'), BinOp(Caret) => op!('^'), BinOp(And) => op!('&'), BinOp(Or) => op!('|'), BinOp(Shl) => op!('<', '<'), BinOp(Shr) => op!('>', '>'), BinOpEq(Plus) => op!('+', '='), BinOpEq(Minus) => op!('-', '='), BinOpEq(Star) => op!('*', '='), BinOpEq(Slash) => op!('/', '='), BinOpEq(Percent) => op!('%', '='), BinOpEq(Caret) => op!('^', '='), BinOpEq(And) => op!('&', '='), BinOpEq(Or) => op!('|', '='), BinOpEq(Shl) => op!('<', '<', '='), BinOpEq(Shr) => op!('>', '>', '='), At => op!('@'), Dot => op!('.'), DotDot => op!('.', '.'), DotDotDot => op!('.', '.', '.'), DotDotEq => op!('.', '.', '='), Comma => op!(','), Semi => op!(';'), Colon => op!(':'), ModSep => op!(':', ':'), RArrow => op!('-', '>'), LArrow => op!('<', '-'), FatArrow => op!('=', '>'), Pound => op!('#'), Dollar => op!('$'), Question => op!('?'), SingleQuote => op!('\''), Ident(ident, false) => tt!(self::Ident::new(&ident.as_str(), Span(span))), Ident(ident, true) => tt!(self::Ident::new_raw(&ident.as_str(), Span(span))), Lifetime(ident) => { let ident = ident.without_first_quote(); stack.push(tt!(self::Ident::new(&ident.as_str(), Span(span)))); tt!(Punct::new('\'', Spacing::Joint)) } Literal(lit, suffix) => tt!(self::Literal { lit, suffix, span: Span(span) }), DocComment(c) => { let style = comments::doc_comment_style(&c.as_str()); let stripped = comments::strip_doc_comment_decoration(&c.as_str()); let stream = vec![ tt!(self::Ident::new("doc", Span(span))), tt!(Punct::new('=', Spacing::Alone)), tt!(self::Literal::string(&stripped)), ].into_iter() .collect(); stack.push(tt!(Group::new(Delimiter::Bracket, stream))); if style == ast::AttrStyle::Inner { stack.push(tt!(Punct::new('!', Spacing::Alone))); } tt!(Punct::new('#', Spacing::Alone)) } Interpolated(_) => __internal::with_sess(|sess, _| { let tts = token.interpolated_to_tokenstream(sess, span); tt!(Group::new(Delimiter::None, ::TokenStream(tts))) }), DotEq => op!('.', '='), OpenDelim(..) | CloseDelim(..) => unreachable!(), Whitespace | Comment | Shebang(..) | Eof => unreachable!(), } } pub(crate) fn to_internal(self) -> tokenstream::TokenStream { use syntax::parse::token::*; use syntax::tokenstream::{Delimited, TokenTree}; let (ch, kind, span) = match self { self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()), self::TokenTree::Group(tt) => { return TokenTree::Delimited( tt.span.0, Delimited { delim: tt.delimiter.to_internal(), tts: tt.stream.0.into(), }, ).into(); } self::TokenTree::Ident(tt) => { let token = Ident(ast::Ident::new(tt.sym, tt.span.0), tt.is_raw); return TokenTree::Token(tt.span.0, token).into(); } self::TokenTree::Literal(self::Literal { lit: Lit::Integer(ref a), suffix, span, }) if a.as_str().starts_with("-") => { let minus = BinOp(BinOpToken::Minus); let integer = Symbol::intern(&a.as_str()[1..]); let integer = Literal(Lit::Integer(integer), suffix); let a = TokenTree::Token(span.0, minus); let b = TokenTree::Token(span.0, integer); return vec![a, b].into_iter().collect(); } self::TokenTree::Literal(self::Literal { lit: Lit::Float(ref a), suffix, span, }) if a.as_str().starts_with("-") => { let minus = BinOp(BinOpToken::Minus); let float = Symbol::intern(&a.as_str()[1..]); let float = Literal(Lit::Float(float), suffix); let a = TokenTree::Token(span.0, minus); let b = TokenTree::Token(span.0, float); return vec![a, b].into_iter().collect(); } self::TokenTree::Literal(tt) => { let token = Literal(tt.lit, tt.suffix); return TokenTree::Token(tt.span.0, token).into(); } }; let token = match ch { '=' => Eq, '<' => Lt, '>' => Gt, '!' => Not, '~' => Tilde, '+' => BinOp(Plus), '-' => BinOp(Minus), '*' => BinOp(Star), '/' => BinOp(Slash), '%' => BinOp(Percent), '^' => BinOp(Caret), '&' => BinOp(And), '|' => BinOp(Or), '@' => At, '.' => Dot, ',' => Comma, ';' => Semi, ':' => Colon, '#' => Pound, '$' => Dollar, '?' => Question, '\'' => SingleQuote, _ => unreachable!(), }; let tree = TokenTree::Token(span.0, token); match kind { Spacing::Alone => tree.into(), Spacing::Joint => tree.joint(), } } } impl Level { pub(crate) fn to_internal(self) -> errors::Level { match self { Level::Error => errors::Level::Error, Level::Warning => errors::Level::Warning, Level::Note => errors::Level::Note, Level::Help => errors::Level::Help, } } }