// Copyright 2012 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. use core::prelude::*; use ast; use ast_util; use parse::token; use util::interner::StrInterner; use util::interner; use core::cmp::Equiv; use core::to_bytes; #[deriving(Encodable, Decodable, Eq)] pub enum binop { PLUS, MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, } #[deriving(Encodable, Decodable, Eq)] pub enum Token { /* Expression-operator symbols. */ EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, BINOP(binop), BINOPEQ(binop), /* Structural symbols */ AT, DOT, DOTDOT, COMMA, SEMI, COLON, MOD_SEP, RARROW, LARROW, DARROW, FAT_ARROW, LPAREN, RPAREN, LBRACKET, RBRACKET, LBRACE, RBRACE, POUND, DOLLAR, /* Literals */ LIT_INT(i64, ast::int_ty), LIT_UINT(u64, ast::uint_ty), LIT_INT_UNSUFFIXED(i64), LIT_FLOAT(ast::ident, ast::float_ty), LIT_FLOAT_UNSUFFIXED(ast::ident), LIT_STR(ast::ident), /* Name components */ // an identifier contains an "is_mod_name" boolean, // indicating whether :: follows this token with no // whitespace in between. IDENT(ast::ident, bool), UNDERSCORE, LIFETIME(ast::ident), /* For interpolation */ INTERPOLATED(nonterminal), DOC_COMMENT(ast::ident), EOF, } #[deriving(Encodable, Decodable, Eq)] /// For interpolation during macro expansion. pub enum nonterminal { nt_item(@ast::item), nt_block(ast::blk), nt_stmt(@ast::stmt), nt_pat( @ast::pat), nt_expr(@ast::expr), nt_ty( @ast::Ty), nt_ident(ast::ident, bool), nt_path(@ast::Path), nt_tt( @ast::token_tree), //needs @ed to break a circularity nt_matchers(~[ast::matcher]) } pub fn binop_to_str(o: binop) -> ~str { match o { PLUS => ~"+", MINUS => ~"-", STAR => ~"*", SLASH => ~"/", PERCENT => ~"%", CARET => ~"^", AND => ~"&", OR => ~"|", SHL => ~"<<", SHR => ~">>" } } pub fn to_str(in: @ident_interner, t: &Token) -> ~str { match *t { EQ => ~"=", LT => ~"<", LE => ~"<=", EQEQ => ~"==", NE => ~"!=", GE => ~">=", GT => ~">", NOT => ~"!", TILDE => ~"~", OROR => ~"||", ANDAND => ~"&&", BINOP(op) => binop_to_str(op), BINOPEQ(op) => binop_to_str(op) + "=", /* Structural symbols */ AT => ~"@", DOT => ~".", DOTDOT => ~"..", COMMA => ~",", SEMI => ~";", COLON => ~":", MOD_SEP => ~"::", RARROW => ~"->", LARROW => ~"<-", DARROW => ~"<->", FAT_ARROW => ~"=>", LPAREN => ~"(", RPAREN => ~")", LBRACKET => ~"[", RBRACKET => ~"]", LBRACE => ~"{", RBRACE => ~"}", POUND => ~"#", DOLLAR => ~"$", /* Literals */ LIT_INT(c, ast::ty_char) => { ~"'" + char::escape_default(c as char) + "'" } LIT_INT(i, t) => { i.to_str() + ast_util::int_ty_to_str(t) } LIT_UINT(u, t) => { u.to_str() + ast_util::uint_ty_to_str(t) } LIT_INT_UNSUFFIXED(i) => { i.to_str() } LIT_FLOAT(s, t) => { let mut body = copy *in.get(s); if body.ends_with(".") { body += "0"; // `10.f` is not a float literal } body + ast_util::float_ty_to_str(t) } LIT_FLOAT_UNSUFFIXED(s) => { let mut body = copy *in.get(s); if body.ends_with(".") { body += "0"; // `10.f` is not a float literal } body } LIT_STR(s) => { ~"\"" + str::escape_default(*in.get(s)) + "\"" } /* Name components */ IDENT(s, _) => copy *in.get(s), LIFETIME(s) => fmt!("'%s", *in.get(s)), UNDERSCORE => ~"_", /* Other */ DOC_COMMENT(s) => copy *in.get(s), EOF => ~"", INTERPOLATED(ref nt) => { match nt { &nt_expr(e) => ::print::pprust::expr_to_str(e, in), _ => { ~"an interpolated " + match (*nt) { nt_item(*) => ~"item", nt_block(*) => ~"block", nt_stmt(*) => ~"statement", nt_pat(*) => ~"pattern", nt_expr(*) => fail!("should have been handled above"), nt_ty(*) => ~"type", nt_ident(*) => ~"identifier", nt_path(*) => ~"path", nt_tt(*) => ~"tt", nt_matchers(*) => ~"matcher sequence" } } } } } } pub fn can_begin_expr(t: &Token) -> bool { match *t { LPAREN => true, LBRACE => true, LBRACKET => true, IDENT(_, _) => true, UNDERSCORE => true, TILDE => true, LIT_INT(_, _) => true, LIT_UINT(_, _) => true, LIT_INT_UNSUFFIXED(_) => true, LIT_FLOAT(_, _) => true, LIT_FLOAT_UNSUFFIXED(_) => true, LIT_STR(_) => true, POUND => true, AT => true, NOT => true, BINOP(MINUS) => true, BINOP(STAR) => true, BINOP(AND) => true, BINOP(OR) => true, // in lambda syntax OROR => true, // in lambda syntax MOD_SEP => true, INTERPOLATED(nt_expr(*)) | INTERPOLATED(nt_ident(*)) | INTERPOLATED(nt_block(*)) | INTERPOLATED(nt_path(*)) => true, _ => false } } /// what's the opposite delimiter? pub fn flip_delimiter(t: &token::Token) -> token::Token { match *t { LPAREN => RPAREN, LBRACE => RBRACE, LBRACKET => RBRACKET, RPAREN => LPAREN, RBRACE => LBRACE, RBRACKET => LBRACKET, _ => fail!() } } pub fn is_lit(t: &Token) -> bool { match *t { LIT_INT(_, _) => true, LIT_UINT(_, _) => true, LIT_INT_UNSUFFIXED(_) => true, LIT_FLOAT(_, _) => true, LIT_FLOAT_UNSUFFIXED(_) => true, LIT_STR(_) => true, _ => false } } pub fn is_ident(t: &Token) -> bool { match *t { IDENT(_, _) => true, _ => false } } pub fn is_ident_or_path(t: &Token) -> bool { match *t { IDENT(_, _) | INTERPOLATED(nt_path(*)) => true, _ => false } } pub fn is_plain_ident(t: &Token) -> bool { match *t { IDENT(_, false) => true, _ => false } } pub fn is_bar(t: &Token) -> bool { match *t { BINOP(OR) | OROR => true, _ => false } } pub mod special_idents { use ast::ident; pub static underscore : ident = ident { repr: 0, ctxt: 0}; pub static anon : ident = ident { repr: 1, ctxt: 0}; pub static invalid : ident = ident { repr: 2, ctxt: 0}; // '' pub static unary : ident = ident { repr: 3, ctxt: 0}; pub static not_fn : ident = ident { repr: 4, ctxt: 0}; pub static idx_fn : ident = ident { repr: 5, ctxt: 0}; pub static unary_minus_fn : ident = ident { repr: 6, ctxt: 0}; pub static clownshoes_extensions : ident = ident { repr: 7, ctxt: 0}; pub static self_ : ident = ident { repr: 8, ctxt: 0}; // 'self' /* for matcher NTs */ pub static item : ident = ident { repr: 9, ctxt: 0}; pub static block : ident = ident { repr: 10, ctxt: 0}; pub static stmt : ident = ident { repr: 11, ctxt: 0}; pub static pat : ident = ident { repr: 12, ctxt: 0}; pub static expr : ident = ident { repr: 13, ctxt: 0}; pub static ty : ident = ident { repr: 14, ctxt: 0}; pub static ident : ident = ident { repr: 15, ctxt: 0}; pub static path : ident = ident { repr: 16, ctxt: 0}; pub static tt : ident = ident { repr: 17, ctxt: 0}; pub static matchers : ident = ident { repr: 18, ctxt: 0}; pub static str : ident = ident { repr: 19, ctxt: 0}; // for the type /* outside of libsyntax */ pub static ty_visitor : ident = ident { repr: 20, ctxt: 0}; pub static arg : ident = ident { repr: 21, ctxt: 0}; pub static descrim : ident = ident { repr: 22, ctxt: 0}; pub static clownshoe_abi : ident = ident { repr: 23, ctxt: 0}; pub static clownshoe_stack_shim : ident = ident { repr: 24, ctxt: 0}; pub static tydesc : ident = ident { repr: 25, ctxt: 0}; pub static main : ident = ident { repr: 26, ctxt: 0}; pub static opaque : ident = ident { repr: 27, ctxt: 0}; pub static blk : ident = ident { repr: 28, ctxt: 0}; pub static statik : ident = ident { repr: 29, ctxt: 0}; pub static intrinsic : ident = ident { repr: 30, ctxt: 0}; pub static clownshoes_foreign_mod: ident = ident { repr: 31, ctxt: 0}; pub static unnamed_field: ident = ident { repr: 32, ctxt: 0}; pub static c_abi: ident = ident { repr: 33, ctxt: 0}; pub static type_self: ident = ident { repr: 34, ctxt: 0}; // `Self` } pub struct StringRef<'self>(&'self str); impl<'self> Equiv<@~str> for StringRef<'self> { #[inline(always)] fn equiv(&self, other: &@~str) -> bool { str::eq_slice(**self, **other) } } impl<'self> to_bytes::IterBytes for StringRef<'self> { #[inline(always)] fn iter_bytes(&self, lsb0: bool, f: to_bytes::Cb) -> bool { (**self).iter_bytes(lsb0, f) } } /** * Maps a token to a record specifying the corresponding binary * operator */ pub fn token_to_binop(tok: Token) -> Option { match tok { BINOP(STAR) => Some(ast::mul), BINOP(SLASH) => Some(ast::div), BINOP(PERCENT) => Some(ast::rem), BINOP(PLUS) => Some(ast::add), BINOP(MINUS) => Some(ast::subtract), BINOP(SHL) => Some(ast::shl), BINOP(SHR) => Some(ast::shr), BINOP(AND) => Some(ast::bitand), BINOP(CARET) => Some(ast::bitxor), BINOP(OR) => Some(ast::bitor), LT => Some(ast::lt), LE => Some(ast::le), GE => Some(ast::ge), GT => Some(ast::gt), EQEQ => Some(ast::eq), NE => Some(ast::ne), ANDAND => Some(ast::and), OROR => Some(ast::or), _ => None } } pub struct ident_interner { priv interner: StrInterner, } pub impl ident_interner { fn intern(&self, val: &str) -> ast::ident { ast::ident { repr: self.interner.intern(val), ctxt: 0 } } fn gensym(&self, val: &str) -> ast::ident { ast::ident { repr: self.interner.gensym(val), ctxt: 0 } } fn get(&self, idx: ast::ident) -> @~str { self.interner.get(idx.repr) } fn len(&self) -> uint { self.interner.len() } fn find_equiv>(&self, val: &Q) -> Option { match self.interner.find_equiv(val) { Some(v) => Some(ast::ident { repr: v, ctxt: 0 }), None => None, } } } // return a fresh interner, preloaded with special identifiers. fn mk_fresh_ident_interner() -> @ident_interner { // the indices here must correspond to the numbers in // special_idents. let init_vec = ~[ "_", // 0 "anon", // 1 "", // 2 "unary", // 3 "!", // 4 "[]", // 5 "unary-", // 6 "__extensions__", // 7 "self", // 8 "item", // 9 "block", // 10 "stmt", // 11 "pat", // 12 "expr", // 13 "ty", // 14 "ident", // 15 "path", // 16 "tt", // 17 "matchers", // 18 "str", // 19 "TyVisitor", // 20 "arg", // 21 "descrim", // 22 "__rust_abi", // 23 "__rust_stack_shim", // 24 "TyDesc", // 25 "main", // 26 "", // 27 "blk", // 28 "static", // 29 "intrinsic", // 30 "__foreign_mod__", // 31 "__field__", // 32 "C", // 33 "Self", // 34 "as", // 35 "break", // 36 "const", // 37 "copy", // 38 "do", // 39 "drop", // 40 "else", // 41 "enum", // 42 "extern", // 43 "false", // 44 "fn", // 45 "for", // 46 "if", // 47 "impl", // 48 "let", // 49 "__log", // 50 "loop", // 51 "match", // 52 "mod", // 53 "mut", // 54 "once", // 55 "priv", // 56 "pub", // 57 "pure", // 58 "ref", // 59 "return", // 60 "static", // 29 -- also a special ident "self", // 8 -- also a special ident "struct", // 61 "super", // 62 "true", // 63 "trait", // 64 "type", // 65 "unsafe", // 66 "use", // 67 "while", // 68 "be", // 69 ]; @ident_interner { interner: interner::StrInterner::prefill(init_vec) } } // if an interner exists in TLS, return it. Otherwise, prepare a // fresh one. pub fn get_ident_interner() -> @ident_interner { unsafe { let key = (cast::transmute::<(uint, uint), &fn(v: @@::parse::token::ident_interner)>( (-3 as uint, 0u))); match local_data::local_data_get(key) { Some(interner) => *interner, None => { let interner = mk_fresh_ident_interner(); unsafe { local_data::local_data_set(key, @interner); } interner } } } } /* for when we don't care about the contents; doesn't interact with TLD or serialization */ pub fn mk_fake_ident_interner() -> @ident_interner { @ident_interner { interner: interner::StrInterner::new() } } // maps a string to its interned representation pub fn intern(str : &str) -> ast::ident { let interner = get_ident_interner(); interner.intern(str) } /** * All the valid words that have meaning in the Rust language. * * Rust keywords are either 'strict' or 'reserved'. Strict keywords may not * appear as identifiers at all. Reserved keywords are not used anywhere in * the language and may not appear as identifiers. */ pub mod keywords { use ast::ident; pub enum Keyword { // Strict keywords As, Break, Const, Copy, Do, Else, Enum, Extern, False, Fn, For, If, Impl, Let, __Log, Loop, Match, Mod, Mut, Once, Priv, Pub, Pure, Ref, Return, Static, Self, Struct, Super, True, Trait, Type, Unsafe, Use, While, // Reserved keywords Be, } pub impl Keyword { fn to_ident(&self) -> ident { match *self { As => ident { repr: 35, ctxt: 0 }, Break => ident { repr: 36, ctxt: 0 }, Const => ident { repr: 37, ctxt: 0 }, Copy => ident { repr: 38, ctxt: 0 }, Do => ident { repr: 39, ctxt: 0 }, Else => ident { repr: 41, ctxt: 0 }, Enum => ident { repr: 42, ctxt: 0 }, Extern => ident { repr: 43, ctxt: 0 }, False => ident { repr: 44, ctxt: 0 }, Fn => ident { repr: 45, ctxt: 0 }, For => ident { repr: 46, ctxt: 0 }, If => ident { repr: 47, ctxt: 0 }, Impl => ident { repr: 48, ctxt: 0 }, Let => ident { repr: 49, ctxt: 0 }, __Log => ident { repr: 50, ctxt: 0 }, Loop => ident { repr: 51, ctxt: 0 }, Match => ident { repr: 52, ctxt: 0 }, Mod => ident { repr: 53, ctxt: 0 }, Mut => ident { repr: 54, ctxt: 0 }, Once => ident { repr: 55, ctxt: 0 }, Priv => ident { repr: 56, ctxt: 0 }, Pub => ident { repr: 57, ctxt: 0 }, Pure => ident { repr: 58, ctxt: 0 }, Ref => ident { repr: 59, ctxt: 0 }, Return => ident { repr: 60, ctxt: 0 }, Static => ident { repr: 29, ctxt: 0 }, Self => ident { repr: 8, ctxt: 0 }, Struct => ident { repr: 61, ctxt: 0 }, Super => ident { repr: 62, ctxt: 0 }, True => ident { repr: 63, ctxt: 0 }, Trait => ident { repr: 64, ctxt: 0 }, Type => ident { repr: 65, ctxt: 0 }, Unsafe => ident { repr: 66, ctxt: 0 }, Use => ident { repr: 67, ctxt: 0 }, While => ident { repr: 68, ctxt: 0 }, Be => ident { repr: 69, ctxt: 0 }, } } } } pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool { match *tok { token::IDENT(sid, false) => { kw.to_ident().repr == sid.repr } _ => { false } } } pub fn is_any_keyword(tok: &Token) -> bool { match *tok { token::IDENT(sid, false) => match sid.repr { 8 | 29 | 35 .. 69 => true, _ => false, }, _ => false } } pub fn is_strict_keyword(tok: &Token) -> bool { match *tok { token::IDENT(sid, false) => match sid.repr { 8 | 29 | 35 .. 68 => true, _ => false, }, _ => false, } } pub fn is_reserved_keyword(tok: &Token) -> bool { match *tok { token::IDENT(sid, false) => match sid.repr { 69 => true, _ => false, }, _ => false, } }