mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch to an independent enum for Lit* subtokens.

Browse Source
This commit is contained in:
Huon Wilson 2014-11-19 10:17:40 +11:00
parent c8d6e3b2c2
commit 5b5638f686
9 changed files with 115 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/grammar/verify.rs
View File

@ -61,7 +61,7 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"SHL" => token::BinOp(token::Shl),
"LBRACE" => token::OpenDelim(token::Brace),
"RARROW" => token::Rarrow,
"LIT_STR" => token::LitStr(Name(0)),
"LIT_STR" => token::Literal(token::Str_(Name(0))),
"DOTDOT" => token::DotDot,
"MOD_SEP" => token::ModSep,
"DOTDOTDOT" => token::DotDotDot,
@ -71,7 +71,7 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"ANDAND" => token::AndAnd,
"AT" => token::At,
"LBRACKET" => token::OpenDelim(token::Bracket),
"LIT_STR_RAW" => token::LitStrRaw(Name(0), 0),
"LIT_STR_RAW" => token::Literal(token::StrRaw(Name(0), 0)),
"RPAREN" => token::CloseDelim(token::Paren),
"SLASH" => token::BinOp(token::Slash),
"COMMA" => token::Comma,
@ -80,8 +80,8 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"TILDE" => token::Tilde,
"IDENT" => token::Id(),
"PLUS" => token::BinOp(token::Plus),
"LIT_CHAR" => token::LitChar(Name(0)),
"LIT_BYTE" => token::LitByte(Name(0)),
"LIT_CHAR" => token::Literal(token::Char(Name(0))),
"LIT_BYTE" => token::Literal(token::Byte(Name(0))),
"EQ" => token::Eq,
"RBRACKET" => token::CloseDelim(token::Bracket),
"COMMENT" => token::Comment,
@ -95,9 +95,9 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"BINOP" => token::BinOp(token::Plus),
"POUND" => token::Pound,
"OROR" => token::OrOr,
"LIT_INTEGER" => token::LitInteger(Name(0)),
"LIT_INTEGER" => token::Literal(token::Integer(Name(0))),
"BINOPEQ" => token::BinOpEq(token::Plus),
"LIT_FLOAT" => token::LitFloat(Name(0)),
"LIT_FLOAT" => token::Literal(token::Float(Name(0))),
"WHITESPACE" => token::Whitespace,
"UNDERSCORE" => token::Underscore,
"MINUS" => token::BinOp(token::Minus),
@ -107,8 +107,8 @@ fn parse_token_list(file: &str) -> HashMap<String, Token> {
"OR" => token::BinOp(token::Or),
"GT" => token::Gt,
"LE" => token::Le,
"LIT_BINARY" => token::LitBinary(Name(0)),
"LIT_BINARY_RAW" => token::LitBinaryRaw(Name(0), 0),
"LIT_BINARY" => token::Literal(token::Binary(Name(0))),
"LIT_BINARY_RAW" => token::Literal(token::BinaryRaw(Name(0), 0)),
_ => continue,
};
@ -189,15 +189,17 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, Token>) -> TokenAndSpan {
token::BinOp(..) => token::BinOp(str_to_binop(content)),
token::BinOpEq(..) => token::BinOpEq(str_to_binop(content.slice_to(
content.len() - 1))),
token::LitStr(..) => token::LitStr(fix(content)),
token::LitStrRaw(..) => token::LitStrRaw(fix(content), count(content)),
token::LitChar(..) => token::LitChar(fixchar(content)),
token::LitByte(..) => token::LitByte(fixchar(content)),
token::Literal(token::Str_(..)) => token::Literal(token::Str_(fix(content))),
token::Literal(token::StrRaw(..)) => token::Literal(token::StrRaw(fix(content),
count(content))),
token::Literal(token::Char(..)) => token::Literal(token::Char(fixchar(content))),
token::Literal(token::Byte(..)) => token::Literal(token::Byte(fixchar(content))),
token::DocComment(..) => token::DocComment(nm),
token::LitInteger(..) => token::LitInteger(nm),
token::LitFloat(..) => token::LitFloat(nm),
token::LitBinary(..) => token::LitBinary(nm),
token::LitBinaryRaw(..) => token::LitBinaryRaw(fix(content), count(content)),
token::Literal(token::Integer(..)) => token::Literal(token::Integer(nm)),
token::Literal(token::Float(..)) => token::Literal(token::Float(nm)),
token::Literal(token::Binary(..)) => token::Literal(token::Binary(nm)),
token::Literal(token::BinaryRaw(..)) => token::Literal(token::BinaryRaw(fix(content),
count(content))),
token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 },
token::ModName),
token::Lifetime(..) => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
@ -284,14 +286,14 @@ fn main() {
)
matches!(
LitByte(..),
LitChar(..),
LitInteger(..),
LitFloat(..),
LitStr(..),
LitStrRaw(..),
LitBinary(..),
LitBinaryRaw(..),
token::Literal(token::Byte(..)),
token::Literal(token::Char(..)),
token::Literal(token::Integer(..)),
token::Literal(token::Float(..)),
token::Literal(token::Str_(..)),
token::Literal(token::StrRaw(..)),
token::Literal(token::Binary(..)),
token::Literal(token::BinaryRaw(..)),
Ident(..),
Lifetime(..),
Interpolated(..),

7
src/librustdoc/html/highlight.rs
View File

@ -129,11 +129,12 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
}
// text literals
token::LitByte(..) | token::LitBinary(..) | token::LitBinaryRaw(..) |
token::LitChar(..) | token::LitStr(..) | token::LitStrRaw(..) => "string",
token::Literal(token::Byte(..)) | token::Literal(token::Char(..)) |
token::Literal(token::Binary(..)) | token::Literal(token::BinaryRaw(..)) |
token::Literal(token::Str_(..)) | token::Literal(token::StrRaw(..)) => "string",
// number literals
token::LitInteger(..) | token::LitFloat(..) => "number",
token::Literal(token::Integer(..)) | token::Literal(token::Float(..)) => "number",
// keywords are also included in the identifier set
token::Ident(ident, _is_mod_sep) => {

2
src/libsyntax/ast.rs
View File

@ -838,7 +838,7 @@ impl TokenTree {
tts: vec![TtToken(sp, token::Ident(token::str_to_ident("doc"),
token::Plain)),
TtToken(sp, token::Eq),
TtToken(sp, token::LitStr(name))],
TtToken(sp, token::Literal(token::Str_(name)))],
close_span: sp,
}))
}

2
src/libsyntax/diagnostics/plugin.rs
View File

@ -87,7 +87,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
},
[ast::TtToken(_, token::Ident(ref code, _)),
ast::TtToken(_, token::Comma),
ast::TtToken(_, token::LitStrRaw(description, _))] => {
ast::TtToken(_, token::Literal(token::StrRaw(description, _)))] => {
(code, Some(description))
}
_ => unreachable!()

37
src/libsyntax/ext/quote.rs
View File

@ -542,6 +542,13 @@ fn mk_delim(cx: &ExtCtxt, sp: Span, delim: token::DelimToken) -> P<ast::Expr> {
#[allow(non_upper_case_globals)]
fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
macro_rules! mk_lit {
($name: expr, $($args: expr),*) => {{
let inner = cx.expr_call(sp, mk_token_path(cx, sp, $name), vec![$($args),*]);
cx.expr_call(sp, mk_token_path(cx, sp, "Literal"), vec![inner])
}}
}
match *tok {
token::BinOp(binop) => {
return cx.expr_call(sp, mk_token_path(cx, sp, "BinOp"), vec!(mk_binop(cx, sp, binop)));
@ -560,38 +567,32 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
vec![mk_delim(cx, sp, delim)]);
}
token::LitByte(i) => {
token::Literal(token::Byte(i)) => {
let e_byte = mk_name(cx, sp, i.ident());
return cx.expr_call(sp, mk_token_path(cx, sp, "LitByte"), vec!(e_byte));
return mk_lit!("Byte", e_byte);
}
token::LitChar(i) => {
token::Literal(token::Char(i)) => {
let e_char = mk_name(cx, sp, i.ident());
return cx.expr_call(sp, mk_token_path(cx, sp, "LitChar"), vec!(e_char));
return mk_lit!("Char", e_char);
}
token::LitInteger(i) => {
token::Literal(token::Integer(i)) => {
let e_int = mk_name(cx, sp, i.ident());
return cx.expr_call(sp, mk_token_path(cx, sp, "LitInteger"), vec!(e_int));
return mk_lit!("Integer", e_int);
}
token::LitFloat(fident) => {
token::Literal(token::Float(fident)) => {
let e_fident = mk_name(cx, sp, fident.ident());
return cx.expr_call(sp, mk_token_path(cx, sp, "LitFloat"), vec!(e_fident));
return mk_lit!("Float", e_fident);
}
token::LitStr(ident) => {
return cx.expr_call(sp,
mk_token_path(cx, sp, "LitStr"),
vec!(mk_name(cx, sp, ident.ident())));
token::Literal(token::Str_(ident)) => {
return mk_lit!("Str_", mk_name(cx, sp, ident.ident()))
}
token::LitStrRaw(ident, n) => {
return cx.expr_call(sp,
mk_token_path(cx, sp, "LitStrRaw"),
vec!(mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n)));
token::Literal(token::StrRaw(ident, n)) => {
return mk_lit!("StrRaw", mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))
}
token::Ident(ident, style) => {

41
src/libsyntax/parse/lexer/mod.rs
View File

@ -655,17 +655,17 @@ impl<'a> StringReader<'a> {
}
'u' | 'i' => {
self.scan_int_suffix();
return token::LitInteger(self.name_from(start_bpos));
return token::Literal(token::Integer(self.name_from(start_bpos)));
},
'f' => {
let last_pos = self.last_pos;
self.scan_float_suffix();
self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos));
return token::Literal(token::Float(self.name_from(start_bpos)));
}
_ => {
// just a 0
return token::LitInteger(self.name_from(start_bpos));
return token::Literal(token::Integer(self.name_from(start_bpos)));
}
}
} else if c.is_digit_radix(10) {
@ -678,7 +678,7 @@ impl<'a> StringReader<'a> {
self.err_span_(start_bpos, self.last_pos, "no valid digits found for number");
// eat any suffix
self.scan_int_suffix();
return token::LitInteger(token::intern("0"));
return token::Literal(token::Integer(token::intern("0")));
}
// might be a float, but don't be greedy if this is actually an
@ -696,13 +696,13 @@ impl<'a> StringReader<'a> {
}
let last_pos = self.last_pos;
self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos));
return token::Literal(token::Float(self.name_from(start_bpos)));
} else if self.curr_is('f') {
// or it might be an integer literal suffixed as a float
self.scan_float_suffix();
let last_pos = self.last_pos;
self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos));
return token::Literal(token::Float(self.name_from(start_bpos)));
} else {
// it might be a float if it has an exponent
if self.curr_is('e') || self.curr_is('E') {
@ -710,11 +710,11 @@ impl<'a> StringReader<'a> {
self.scan_float_suffix();
let last_pos = self.last_pos;
self.check_float_base(start_bpos, last_pos, base);
return token::LitFloat(self.name_from(start_bpos));
return token::Literal(token::Float(self.name_from(start_bpos)));
}
// but we certainly have an integer!
self.scan_int_suffix();
return token::LitInteger(self.name_from(start_bpos));
return token::Literal(token::Integer(self.name_from(start_bpos)));
}
}
@ -1126,7 +1126,7 @@ impl<'a> StringReader<'a> {
}
let id = if valid { self.name_from(start) } else { token::intern("0") };
self.bump(); // advance curr past token
return token::LitChar(id);
return token::Literal(token::Char(id));
}
'b' => {
self.bump();
@ -1157,7 +1157,7 @@ impl<'a> StringReader<'a> {
let id = if valid { self.name_from(start_bpos + BytePos(1)) }
else { token::intern("??") };
self.bump();
return token::LitStr(id);
return token::Literal(token::Str_(id));
}
'r' => {
let start_bpos = self.last_pos;
@ -1224,7 +1224,7 @@ impl<'a> StringReader<'a> {
} else {
token::intern("??")
};
return token::LitStrRaw(id, hash_count);
return token::Literal(token::StrRaw(id, hash_count));
}
'-' => {
if self.nextch_is('>') {
@ -1314,7 +1314,7 @@ impl<'a> StringReader<'a> {
let id = if valid { self.name_from(start) } else { token::intern("??") };
self.bump(); // advance curr past token
return token::LitByte(id);
return token::Literal(token::Byte(id));
}
fn scan_byte_string(&mut self) -> token::Token {
@ -1336,7 +1336,7 @@ impl<'a> StringReader<'a> {
}
let id = if valid { self.name_from(start) } else { token::intern("??") };
self.bump();
return token::LitBinary(id);
return token::Literal(token::Binary(id));
}
fn scan_raw_byte_string(&mut self) -> token::Token {
@ -1387,8 +1387,9 @@ impl<'a> StringReader<'a> {
self.bump();
}
self.bump();
return token::LitBinaryRaw(self.name_from_to(content_start_bpos, content_end_bpos),
hash_count);
return token::Literal(token::BinaryRaw(self.name_from_to(content_start_bpos,
content_end_bpos),
hash_count));
}
}
@ -1535,17 +1536,17 @@ mod test {
#[test] fn character_a() {
assert_eq!(setup(&mk_sh(), "'a'".to_string()).next_token().tok,
token::LitChar(token::intern("a")));
token::Literal(token::Char(token::intern("a"))));
}
#[test] fn character_space() {
assert_eq!(setup(&mk_sh(), "' '".to_string()).next_token().tok,
token::LitChar(token::intern(" ")));
token::Literal(token::Char(token::intern(" "))));
}
#[test] fn character_escaped() {
assert_eq!(setup(&mk_sh(), "'\\n'".to_string()).next_token().tok,
token::LitChar(token::intern("\\n")));
token::Literal(token::Char(token::intern("\\n"))));
}
#[test] fn lifetime_name() {
@ -1557,7 +1558,7 @@ mod test {
assert_eq!(setup(&mk_sh(),
"r###\"\"#a\\b\x00c\"\"###".to_string()).next_token()
.tok,
token::LitStrRaw(token::intern("\"#a\\b\x00c\""), 3));
token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3)));
}
#[test] fn line_doc_comments() {
@ -1573,7 +1574,7 @@ mod test {
token::Comment => { },
_ => panic!("expected a comment!")
}
assert_eq!(lexer.next_token().tok, token::LitChar(token::intern("a")));
assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a"))));
}
}

28
src/libsyntax/parse/parser.rs
View File

@ -1640,23 +1640,23 @@ impl<'a> Parser<'a> {
/// Matches token_lit = LIT_INTEGER | ...
pub fn lit_from_token(&mut self, tok: &token::Token) -> Lit_ {
match *tok {
token::LitByte(i) => LitByte(parse::byte_lit(i.as_str()).val0()),
token::LitChar(i) => LitChar(parse::char_lit(i.as_str()).val0()),
token::LitInteger(s) => parse::integer_lit(s.as_str(),
token::Literal(token::Byte(i)) => LitByte(parse::byte_lit(i.as_str()).val0()),
token::Literal(token::Char(i)) => LitChar(parse::char_lit(i.as_str()).val0()),
token::Literal(token::Integer(s)) => parse::integer_lit(s.as_str(),
&self.sess.span_diagnostic,
self.last_span),
token::LitFloat(s) => parse::float_lit(s.as_str()),
token::LitStr(s) => {
token::Literal(token::Float(s)) => parse::float_lit(s.as_str()),
token::Literal(token::Str_(s)) => {
LitStr(token::intern_and_get_ident(parse::str_lit(s.as_str()).as_slice()),
ast::CookedStr)
}
token::LitStrRaw(s, n) => {
token::Literal(token::StrRaw(s, n)) => {
LitStr(token::intern_and_get_ident(parse::raw_str_lit(s.as_str()).as_slice()),
ast::RawStr(n))
}
token::LitBinary(i) =>
token::Literal(token::Binary(i)) =>
LitBinary(parse::binary_lit(i.as_str())),
token::LitBinaryRaw(i, _) =>
token::Literal(token::BinaryRaw(i, _)) =>
LitBinary(Rc::new(i.as_str().as_bytes().iter().map(|&x| x).collect())),
_ => { self.unexpected_last(tok); }
}
@ -2424,7 +2424,7 @@ impl<'a> Parser<'a> {
}
}
}
token::LitInteger(n) => {
token::Literal(token::Integer(n)) => {
let index = n.as_str();
let dot = self.last_span.hi;
hi = self.span.hi;
@ -2449,7 +2449,7 @@ impl<'a> Parser<'a> {
}
}
}
token::LitFloat(n) => {
token::Literal(token::Float(n)) => {
self.bump();
let last_span = self.last_span;
let fstr = n.as_str();
@ -5085,7 +5085,7 @@ impl<'a> Parser<'a> {
self.expect(&token::Semi);
(path, the_ident)
},
token::LitStr(..) | token::LitStrRaw(..) => {
token::Literal(token::Str_(..)) | token::Literal(token::StrRaw(..)) => {
let path = self.parse_str();
self.expect_keyword(keywords::As);
let the_ident = self.parse_ident();
@ -5267,7 +5267,7 @@ impl<'a> Parser<'a> {
/// the `extern` keyword, if one is found.
fn parse_opt_abi(&mut self) -> Option<abi::Abi> {
match self.token {
token::LitStr(s) | token::LitStrRaw(s, _) => {
token::Literal(token::Str_(s)) | token::Literal(token::StrRaw(s, _)) => {
self.bump();
let the_string = s.as_str();
match abi::lookup(the_string) {
@ -5904,8 +5904,8 @@ impl<'a> Parser<'a> {
pub fn parse_optional_str(&mut self)
-> Option<(InternedString, ast::StrStyle)> {
let (s, style) = match self.token {
token::LitStr(s) => (self.id_to_interned_str(s.ident()), ast::CookedStr),
token::LitStrRaw(s, n) => {
token::Literal(token::Str_(s)) => (self.id_to_interned_str(s.ident()), ast::CookedStr),
token::Literal(token::StrRaw(s, n)) => {
(self.id_to_interned_str(s.ident()), ast::RawStr(n))
}
_ => return None

42
src/libsyntax/parse/token.rs
View File

@ -12,6 +12,7 @@ pub use self::BinOpToken::*;
pub use self::Nonterminal::*;
pub use self::DelimToken::*;
pub use self::IdentStyle::*;
pub use self::Lit::*;
pub use self::Token::*;
use ast;
@ -59,6 +60,18 @@ pub enum IdentStyle {
Plain,
}
#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)]
pub enum Lit {
Byte(ast::Name),
Char(ast::Name),
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, uint), /* raw str delimited by n hash symbols */
Binary(ast::Name),
BinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */
}
#[allow(non_camel_case_types)]
#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)]
pub enum Token {
@ -98,14 +111,7 @@ pub enum Token {
CloseDelim(DelimToken),
/* Literals */
LitByte(ast::Name),
LitChar(ast::Name),
LitInteger(ast::Name),
LitFloat(ast::Name),
LitStr(ast::Name),
LitStrRaw(ast::Name, uint), /* raw str delimited by n hash symbols */
LitBinary(ast::Name),
LitBinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */
Literal(Lit),
/* Name components */
Ident(ast::Ident, IdentStyle),
@ -145,14 +151,7 @@ impl Token {
Ident(_, _) => true,
Underscore => true,
Tilde => true,
LitByte(_) => true,
LitChar(_) => true,
LitInteger(_) => true,
LitFloat(_) => true,
LitStr(_) => true,
LitStrRaw(_, _) => true,
LitBinary(_) => true,
LitBinaryRaw(_, _) => true,
Literal(_) => true,
Pound => true,
At => true,
Not => true,
@ -173,15 +172,8 @@ impl Token {
/// Returns `true` if the token is any literal
pub fn is_lit(&self) -> bool {
match *self {
LitByte(_) => true,
LitChar(_) => true,
LitInteger(_) => true,
LitFloat(_) => true,
LitStr(_) => true,
LitStrRaw(_, _) => true,
LitBinary(_) => true,
LitBinaryRaw(_, _) => true,
_ => false,
Literal(_) => true,
_ => false,
}
}

24
src/libsyntax/print/pprust.rs
View File

@ -236,18 +236,18 @@ pub fn token_to_string(tok: &Token) -> String {
token::Question => "?".into_string(),
/* Literals */
token::LitByte(b) => format!("b'{}'", b.as_str()),
token::LitChar(c) => format!("'{}'", c.as_str()),
token::LitFloat(c) => c.as_str().into_string(),
token::LitInteger(c) => c.as_str().into_string(),
token::LitStr(s) => format!("\"{}\"", s.as_str()),
token::LitStrRaw(s, n) => format!("r{delim}\"{string}\"{delim}",
delim="#".repeat(n),
string=s.as_str()),
token::LitBinary(v) => format!("b\"{}\"", v.as_str()),
token::LitBinaryRaw(s, n) => format!("br{delim}\"{string}\"{delim}",
delim="#".repeat(n),
string=s.as_str()),
token::Literal(token::Byte(b)) => format!("b'{}'", b.as_str()),
token::Literal(token::Char(c)) => format!("'{}'", c.as_str()),
token::Literal(token::Float(c)) => c.as_str().into_string(),
token::Literal(token::Integer(c)) => c.as_str().into_string(),
token::Literal(token::Str_(s)) => format!("\"{}\"", s.as_str()),
token::Literal(token::StrRaw(s, n)) => format!("r{delim}\"{string}\"{delim}",
delim="#".repeat(n),
string=s.as_str()),
token::Literal(token::Binary(v)) => format!("b\"{}\"", v.as_str()),
token::Literal(token::BinaryRaw(s, n)) => format!("br{delim}\"{string}\"{delim}",
delim="#".repeat(n),
string=s.as_str()),
/* Name components */
token::Ident(s, _) => token::get_ident(s).get().into_string(),