2012-12-03 18:48:01 -06:00
|
|
|
// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
|
|
|
|
// file at the top-level directory of this distribution and at
|
|
|
|
// http://rust-lang.org/COPYRIGHT.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
|
|
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
|
|
// option. This file may not be copied, modified, or distributed
|
|
|
|
// except according to those terms.
|
|
|
|
|
2012-09-04 13:37:29 -05:00
|
|
|
use diagnostic::span_handler;
|
2012-11-16 16:35:43 -06:00
|
|
|
use codemap::{span, CodeMap, CharPos, BytePos};
|
2012-09-04 13:37:29 -05:00
|
|
|
use ext::tt::transcribe::{tt_reader, new_tt_reader, dup_tt_reader,
|
2012-06-27 17:29:35 -05:00
|
|
|
tt_next_token};
|
2011-06-15 13:19:50 -05:00
|
|
|
|
2012-05-30 13:36:30 -05:00
|
|
|
export reader, string_reader, new_string_reader, is_whitespace;
|
2012-06-27 17:29:35 -05:00
|
|
|
export tt_reader, new_tt_reader;
|
2012-06-15 11:32:17 -05:00
|
|
|
export nextch, is_eof, bump, get_str_from, new_low_level_string_reader;
|
2012-06-04 20:18:47 -05:00
|
|
|
export string_reader_as_reader, tt_reader_as_reader;
|
2012-05-30 13:36:30 -05:00
|
|
|
|
2012-07-31 12:27:51 -05:00
|
|
|
trait reader {
|
2012-05-30 13:36:30 -05:00
|
|
|
fn is_eof() -> bool;
|
2012-10-15 16:56:42 -05:00
|
|
|
fn next_token() -> {tok: token::Token, sp: span};
|
2012-07-14 00:57:48 -05:00
|
|
|
fn fatal(~str) -> !;
|
2012-06-27 17:29:35 -05:00
|
|
|
fn span_diag() -> span_handler;
|
2012-09-19 20:50:24 -05:00
|
|
|
pure fn interner() -> @token::ident_interner;
|
2012-10-15 16:56:42 -05:00
|
|
|
fn peek() -> {tok: token::Token, sp: span};
|
2012-06-12 12:59:50 -05:00
|
|
|
fn dup() -> reader;
|
2012-05-30 13:36:30 -05:00
|
|
|
}
|
2012-04-15 05:49:59 -05:00
|
|
|
|
2012-05-30 13:36:30 -05:00
|
|
|
type string_reader = @{
|
2012-06-27 17:29:35 -05:00
|
|
|
span_diagnostic: span_handler,
|
2012-07-14 00:57:48 -05:00
|
|
|
src: @~str,
|
2012-11-15 16:34:23 -06:00
|
|
|
// The absolute offset within the codemap of the next character to read
|
2012-11-16 16:10:17 -06:00
|
|
|
mut pos: BytePos,
|
2012-11-15 21:37:29 -06:00
|
|
|
// The absolute offset within the codemap of the last character read(curr)
|
2012-11-16 16:10:17 -06:00
|
|
|
mut last_pos: BytePos,
|
2012-11-15 16:34:23 -06:00
|
|
|
// The column of the next character to read
|
2012-11-12 21:32:48 -06:00
|
|
|
mut col: CharPos,
|
2012-11-15 16:34:23 -06:00
|
|
|
// The last character to be read
|
2012-03-26 20:35:18 -05:00
|
|
|
mut curr: char,
|
2012-11-12 20:59:37 -06:00
|
|
|
filemap: @codemap::FileMap,
|
2012-09-19 20:50:24 -05:00
|
|
|
interner: @token::ident_interner,
|
2012-06-12 12:59:50 -05:00
|
|
|
/* cached: */
|
2012-10-15 16:56:42 -05:00
|
|
|
mut peek_tok: token::Token,
|
2012-06-12 12:59:50 -05:00
|
|
|
mut peek_span: span
|
2012-01-13 02:56:53 -06:00
|
|
|
};
|
|
|
|
|
2012-06-27 17:29:35 -05:00
|
|
|
fn new_string_reader(span_diagnostic: span_handler,
|
2012-11-12 20:59:37 -06:00
|
|
|
filemap: @codemap::FileMap,
|
2012-09-19 20:50:24 -05:00
|
|
|
itr: @token::ident_interner) -> string_reader {
|
2012-06-15 11:32:17 -05:00
|
|
|
let r = new_low_level_string_reader(span_diagnostic, filemap, itr);
|
2012-06-12 12:59:50 -05:00
|
|
|
string_advance_token(r); /* fill in peek_* */
|
2012-08-01 19:30:05 -05:00
|
|
|
return r;
|
2012-06-15 11:32:17 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
/* For comments.rs, which hackily pokes into 'pos' and 'curr' */
|
2012-06-27 17:29:35 -05:00
|
|
|
fn new_low_level_string_reader(span_diagnostic: span_handler,
|
2012-11-12 20:59:37 -06:00
|
|
|
filemap: @codemap::FileMap,
|
2012-09-19 20:50:24 -05:00
|
|
|
itr: @token::ident_interner)
|
2012-06-15 11:32:17 -05:00
|
|
|
-> string_reader {
|
2012-11-14 19:03:07 -06:00
|
|
|
// Force the initial reader bump to start on a fresh line
|
|
|
|
let initial_char = '\n';
|
2012-06-04 20:18:47 -05:00
|
|
|
let r = @{span_diagnostic: span_diagnostic, src: filemap.src,
|
2012-11-15 00:27:53 -06:00
|
|
|
mut pos: filemap.start_pos,
|
|
|
|
mut last_pos: filemap.start_pos,
|
|
|
|
mut col: CharPos(0),
|
2012-11-14 19:03:07 -06:00
|
|
|
mut curr: initial_char,
|
2012-06-12 12:59:50 -05:00
|
|
|
filemap: filemap, interner: itr,
|
|
|
|
/* dummy values; not read */
|
|
|
|
mut peek_tok: token::EOF,
|
2012-11-12 21:32:48 -06:00
|
|
|
mut peek_span: ast_util::dummy_sp()};
|
2012-11-14 19:03:07 -06:00
|
|
|
bump(r);
|
2012-08-01 19:30:05 -05:00
|
|
|
return r;
|
2012-06-04 20:18:47 -05:00
|
|
|
}
|
|
|
|
|
2012-06-12 12:59:50 -05:00
|
|
|
fn dup_string_reader(&&r: string_reader) -> string_reader {
|
|
|
|
@{span_diagnostic: r.span_diagnostic, src: r.src,
|
2012-11-15 00:27:53 -06:00
|
|
|
mut pos: r.pos,
|
|
|
|
mut last_pos: r.last_pos,
|
|
|
|
mut col: r.col, mut curr: r.curr,
|
2012-06-12 12:59:50 -05:00
|
|
|
filemap: r.filemap, interner: r.interner,
|
|
|
|
mut peek_tok: r.peek_tok, mut peek_span: r.peek_span}
|
|
|
|
}
|
|
|
|
|
2012-08-07 20:10:06 -05:00
|
|
|
impl string_reader: reader {
|
2012-05-30 13:36:30 -05:00
|
|
|
fn is_eof() -> bool { is_eof(self) }
|
2012-10-15 16:56:42 -05:00
|
|
|
fn next_token() -> {tok: token::Token, sp: span} {
|
2012-06-12 12:59:50 -05:00
|
|
|
let ret_val = {tok: self.peek_tok, sp: self.peek_span};
|
|
|
|
string_advance_token(self);
|
2012-08-01 19:30:05 -05:00
|
|
|
return ret_val;
|
2012-01-13 02:56:53 -06:00
|
|
|
}
|
2012-07-14 00:57:48 -05:00
|
|
|
fn fatal(m: ~str) -> ! {
|
2012-06-12 12:59:50 -05:00
|
|
|
self.span_diagnostic.span_fatal(copy self.peek_span, m)
|
2012-01-13 02:56:53 -06:00
|
|
|
}
|
2012-06-27 17:29:35 -05:00
|
|
|
fn span_diag() -> span_handler { self.span_diagnostic }
|
2012-09-19 20:50:24 -05:00
|
|
|
pure fn interner() -> @token::ident_interner { self.interner }
|
2012-10-15 16:56:42 -05:00
|
|
|
fn peek() -> {tok: token::Token, sp: span} {
|
2012-06-12 12:59:50 -05:00
|
|
|
{tok: self.peek_tok, sp: self.peek_span}
|
|
|
|
}
|
|
|
|
fn dup() -> reader { dup_string_reader(self) as reader }
|
2012-06-01 17:52:07 -05:00
|
|
|
}
|
|
|
|
|
2012-08-07 20:10:06 -05:00
|
|
|
impl tt_reader: reader {
|
2012-06-01 17:52:07 -05:00
|
|
|
fn is_eof() -> bool { self.cur_tok == token::EOF }
|
2012-10-15 16:56:42 -05:00
|
|
|
fn next_token() -> {tok: token::Token, sp: span} {
|
2012-06-07 20:01:58 -05:00
|
|
|
/* weird resolve bug: if the following `if`, or any of its
|
|
|
|
statements are removed, we get resolution errors */
|
|
|
|
if false {
|
|
|
|
let _ignore_me = 0;
|
|
|
|
let _me_too = self.cur.readme[self.cur.idx];
|
2012-06-01 17:52:07 -05:00
|
|
|
}
|
2012-06-07 20:01:58 -05:00
|
|
|
tt_next_token(self)
|
2012-06-01 17:52:07 -05:00
|
|
|
}
|
2012-07-14 00:57:48 -05:00
|
|
|
fn fatal(m: ~str) -> ! {
|
2012-07-06 20:04:28 -05:00
|
|
|
self.sp_diag.span_fatal(copy self.cur_span, m);
|
2012-06-01 17:52:07 -05:00
|
|
|
}
|
2012-07-06 20:04:28 -05:00
|
|
|
fn span_diag() -> span_handler { self.sp_diag }
|
2012-09-19 20:50:24 -05:00
|
|
|
pure fn interner() -> @token::ident_interner { self.interner }
|
2012-10-15 16:56:42 -05:00
|
|
|
fn peek() -> {tok: token::Token, sp: span} {
|
2012-06-12 12:59:50 -05:00
|
|
|
{ tok: self.cur_tok, sp: self.cur_span }
|
|
|
|
}
|
|
|
|
fn dup() -> reader { dup_tt_reader(self) as reader }
|
2012-01-13 02:56:53 -06:00
|
|
|
}
|
2011-06-15 13:19:50 -05:00
|
|
|
|
2012-06-15 11:32:17 -05:00
|
|
|
fn string_advance_token(&&r: string_reader) {
|
2012-07-02 16:44:31 -05:00
|
|
|
for consume_whitespace_and_comments(r).each |comment| {
|
2012-06-30 05:54:54 -05:00
|
|
|
r.peek_tok = comment.tok;
|
|
|
|
r.peek_span = comment.sp;
|
2012-08-01 19:30:05 -05:00
|
|
|
return;
|
2012-06-30 05:54:54 -05:00
|
|
|
}
|
2012-06-15 11:32:17 -05:00
|
|
|
|
2012-06-12 12:59:50 -05:00
|
|
|
if is_eof(r) {
|
|
|
|
r.peek_tok = token::EOF;
|
|
|
|
} else {
|
2012-11-16 16:10:17 -06:00
|
|
|
let start_bytepos = r.last_pos;
|
2012-06-12 12:59:50 -05:00
|
|
|
r.peek_tok = next_token_inner(r);
|
2012-11-16 16:10:17 -06:00
|
|
|
r.peek_span = ast_util::mk_sp(start_bytepos, r.last_pos);
|
2012-06-12 12:59:50 -05:00
|
|
|
};
|
|
|
|
|
2012-06-15 11:32:17 -05:00
|
|
|
}
|
|
|
|
|
2012-11-15 00:27:53 -06:00
|
|
|
fn byte_offset(rdr: string_reader) -> BytePos {
|
2012-11-16 16:10:17 -06:00
|
|
|
(rdr.pos - rdr.filemap.start_pos)
|
2012-11-15 00:27:53 -06:00
|
|
|
}
|
|
|
|
|
2012-11-12 21:32:48 -06:00
|
|
|
fn get_str_from(rdr: string_reader, start: BytePos) -> ~str unsafe {
|
2012-05-30 13:36:30 -05:00
|
|
|
// I'm pretty skeptical about this subtraction. What if there's a
|
|
|
|
// multi-byte character before the mark?
|
2012-11-15 00:27:53 -06:00
|
|
|
return str::slice(*rdr.src, start.to_uint() - 1u,
|
|
|
|
byte_offset(rdr).to_uint() - 1u);
|
2012-05-30 13:36:30 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
fn bump(rdr: string_reader) {
|
2012-11-15 00:27:53 -06:00
|
|
|
rdr.last_pos = rdr.pos;
|
|
|
|
let current_byte_offset = byte_offset(rdr).to_uint();;
|
|
|
|
if current_byte_offset < (*rdr.src).len() {
|
2012-11-18 16:58:57 -06:00
|
|
|
assert rdr.curr != -1 as char;
|
2012-11-15 00:27:53 -06:00
|
|
|
let last_char = rdr.curr;
|
|
|
|
let next = str::char_range_at(*rdr.src, current_byte_offset);
|
|
|
|
let byte_offset_diff = next.next - current_byte_offset;
|
2012-11-16 16:10:17 -06:00
|
|
|
rdr.pos = rdr.pos + BytePos(byte_offset_diff);
|
2012-05-30 13:36:30 -05:00
|
|
|
rdr.curr = next.ch;
|
2012-11-12 21:32:48 -06:00
|
|
|
rdr.col += CharPos(1u);
|
2012-11-15 00:27:53 -06:00
|
|
|
if last_char == '\n' {
|
|
|
|
rdr.filemap.next_line(rdr.last_pos);
|
2012-11-12 21:32:48 -06:00
|
|
|
rdr.col = CharPos(0u);
|
2012-05-30 13:36:30 -05:00
|
|
|
}
|
2012-11-15 21:37:29 -06:00
|
|
|
|
|
|
|
if byte_offset_diff > 1 {
|
|
|
|
rdr.filemap.record_multibyte_char(
|
|
|
|
BytePos(current_byte_offset), byte_offset_diff);
|
|
|
|
}
|
2012-05-30 13:36:30 -05:00
|
|
|
} else {
|
2012-11-18 16:58:57 -06:00
|
|
|
rdr.curr = -1 as char;
|
2012-05-30 13:36:30 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
fn is_eof(rdr: string_reader) -> bool {
|
|
|
|
rdr.curr == -1 as char
|
|
|
|
}
|
|
|
|
fn nextch(rdr: string_reader) -> char {
|
2012-11-15 00:27:53 -06:00
|
|
|
let offset = byte_offset(rdr).to_uint();
|
|
|
|
if offset < (*rdr.src).len() {
|
|
|
|
return str::char_at(*rdr.src, offset);
|
2012-08-01 19:30:05 -05:00
|
|
|
} else { return -1 as char; }
|
2012-05-30 13:36:30 -05:00
|
|
|
}
|
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
fn dec_digit_val(c: char) -> int { return (c as int) - ('0' as int); }
|
2010-08-31 13:36:36 -05:00
|
|
|
|
2011-07-27 07:19:39 -05:00
|
|
|
fn hex_digit_val(c: char) -> int {
|
2012-08-01 19:30:05 -05:00
|
|
|
if in_range(c, '0', '9') { return (c as int) - ('0' as int); }
|
|
|
|
if in_range(c, 'a', 'f') { return (c as int) - ('a' as int) + 10; }
|
|
|
|
if in_range(c, 'A', 'F') { return (c as int) - ('A' as int) + 10; }
|
2010-08-31 13:36:36 -05:00
|
|
|
fail;
|
|
|
|
}
|
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
fn bin_digit_value(c: char) -> int { if c == '0' { return 0; } return 1; }
|
2010-08-31 13:36:36 -05:00
|
|
|
|
2011-07-27 07:19:39 -05:00
|
|
|
fn is_whitespace(c: char) -> bool {
|
2012-08-01 19:30:05 -05:00
|
|
|
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
2010-08-19 20:42:17 -05:00
|
|
|
}
|
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
fn may_begin_ident(c: char) -> bool { return is_alpha(c) || c == '_'; }
|
2011-07-05 04:48:19 -05:00
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
fn in_range(c: char, lo: char, hi: char) -> bool {
|
|
|
|
return lo <= c && c <= hi
|
|
|
|
}
|
2011-07-05 04:48:19 -05:00
|
|
|
|
2011-07-27 07:19:39 -05:00
|
|
|
fn is_alpha(c: char) -> bool {
|
2012-08-01 19:30:05 -05:00
|
|
|
return in_range(c, 'a', 'z') || in_range(c, 'A', 'Z');
|
2011-07-05 04:48:19 -05:00
|
|
|
}
|
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
fn is_dec_digit(c: char) -> bool { return in_range(c, '0', '9'); }
|
2011-07-05 04:48:19 -05:00
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
fn is_alnum(c: char) -> bool { return is_alpha(c) || is_dec_digit(c); }
|
2011-07-05 04:48:19 -05:00
|
|
|
|
2011-07-27 07:19:39 -05:00
|
|
|
fn is_hex_digit(c: char) -> bool {
|
2012-08-01 19:30:05 -05:00
|
|
|
return in_range(c, '0', '9') || in_range(c, 'a', 'f') ||
|
2011-07-05 04:48:19 -05:00
|
|
|
in_range(c, 'A', 'F');
|
|
|
|
}
|
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
fn is_bin_digit(c: char) -> bool { return c == '0' || c == '1'; }
|
2011-07-05 04:48:19 -05:00
|
|
|
|
2012-06-30 05:54:54 -05:00
|
|
|
// might return a sugared-doc-attr
|
|
|
|
fn consume_whitespace_and_comments(rdr: string_reader)
|
2012-10-15 16:56:42 -05:00
|
|
|
-> Option<{tok: token::Token, sp: span}> {
|
2012-05-30 13:36:30 -05:00
|
|
|
while is_whitespace(rdr.curr) { bump(rdr); }
|
2012-08-01 19:30:05 -05:00
|
|
|
return consume_any_line_comment(rdr);
|
2010-08-23 21:17:04 -05:00
|
|
|
}
|
|
|
|
|
2012-06-30 05:54:54 -05:00
|
|
|
// might return a sugared-doc-attr
|
|
|
|
fn consume_any_line_comment(rdr: string_reader)
|
2012-10-15 16:56:42 -05:00
|
|
|
-> Option<{tok: token::Token, sp: span}> {
|
2012-01-13 02:56:53 -06:00
|
|
|
if rdr.curr == '/' {
|
2012-08-06 14:34:08 -05:00
|
|
|
match nextch(rdr) {
|
2012-08-03 21:59:04 -05:00
|
|
|
'/' => {
|
2012-06-30 05:54:54 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
|
|
|
// line comments starting with "///" or "//!" are doc-comments
|
|
|
|
if rdr.curr == '/' || rdr.curr == '!' {
|
2012-11-16 16:10:17 -06:00
|
|
|
let start_bpos = rdr.pos - BytePos(2u);
|
2012-07-14 00:57:48 -05:00
|
|
|
let mut acc = ~"//";
|
2012-06-30 05:54:54 -05:00
|
|
|
while rdr.curr != '\n' && !is_eof(rdr) {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut acc, rdr.curr);
|
2012-06-30 05:54:54 -05:00
|
|
|
bump(rdr);
|
|
|
|
}
|
2012-08-20 14:23:37 -05:00
|
|
|
return Some({
|
2012-08-14 17:27:06 -05:00
|
|
|
tok: token::DOC_COMMENT(rdr.interner.intern(@acc)),
|
2012-11-16 16:10:17 -06:00
|
|
|
sp: ast_util::mk_sp(start_bpos, rdr.pos)
|
2012-06-30 05:54:54 -05:00
|
|
|
});
|
|
|
|
} else {
|
|
|
|
while rdr.curr != '\n' && !is_eof(rdr) { bump(rdr); }
|
|
|
|
// Restart whitespace munch.
|
2012-08-01 19:30:05 -05:00
|
|
|
return consume_whitespace_and_comments(rdr);
|
2012-06-30 05:54:54 -05:00
|
|
|
}
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'*' => { bump(rdr); bump(rdr); return consume_block_comment(rdr); }
|
|
|
|
_ => ()
|
2010-08-23 21:17:04 -05:00
|
|
|
}
|
2012-05-22 19:27:20 -05:00
|
|
|
} else if rdr.curr == '#' {
|
2012-05-30 13:36:30 -05:00
|
|
|
if nextch(rdr) == '!' {
|
2012-11-12 20:24:56 -06:00
|
|
|
let cmap = @CodeMap::new();
|
2012-05-22 19:27:20 -05:00
|
|
|
(*cmap).files.push(rdr.filemap);
|
2012-11-16 16:10:17 -06:00
|
|
|
let loc = cmap.lookup_char_pos_adj(rdr.last_pos);
|
2012-11-12 21:32:48 -06:00
|
|
|
if loc.line == 1u && loc.col == CharPos(0u) {
|
2012-05-30 13:36:30 -05:00
|
|
|
while rdr.curr != '\n' && !is_eof(rdr) { bump(rdr); }
|
2012-08-01 19:30:05 -05:00
|
|
|
return consume_whitespace_and_comments(rdr);
|
2012-05-22 19:27:20 -05:00
|
|
|
}
|
|
|
|
}
|
2010-08-23 21:17:04 -05:00
|
|
|
}
|
2012-08-20 14:23:37 -05:00
|
|
|
return None;
|
2010-08-23 21:17:04 -05:00
|
|
|
}
|
|
|
|
|
2012-06-30 05:54:54 -05:00
|
|
|
// might return a sugared-doc-attr
|
|
|
|
fn consume_block_comment(rdr: string_reader)
|
2012-10-15 16:56:42 -05:00
|
|
|
-> Option<{tok: token::Token, sp: span}> {
|
2012-06-30 05:54:54 -05:00
|
|
|
|
|
|
|
// block comments starting with "/**" or "/*!" are doc-comments
|
|
|
|
if rdr.curr == '*' || rdr.curr == '!' {
|
2012-11-16 16:10:17 -06:00
|
|
|
let start_bpos = rdr.pos - BytePos(2u);
|
2012-07-14 00:57:48 -05:00
|
|
|
let mut acc = ~"/*";
|
2012-06-30 05:54:54 -05:00
|
|
|
while !(rdr.curr == '*' && nextch(rdr) == '/') && !is_eof(rdr) {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut acc, rdr.curr);
|
2012-06-30 05:54:54 -05:00
|
|
|
bump(rdr);
|
|
|
|
}
|
|
|
|
if is_eof(rdr) {
|
2012-07-14 00:57:48 -05:00
|
|
|
rdr.fatal(~"unterminated block doc-comment");
|
2012-06-30 05:54:54 -05:00
|
|
|
} else {
|
2012-07-14 00:57:48 -05:00
|
|
|
acc += ~"*/";
|
2012-06-30 05:54:54 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-20 14:23:37 -05:00
|
|
|
return Some({
|
2012-08-14 17:27:06 -05:00
|
|
|
tok: token::DOC_COMMENT(rdr.interner.intern(@acc)),
|
2012-11-16 16:10:17 -06:00
|
|
|
sp: ast_util::mk_sp(start_bpos, rdr.pos)
|
2012-06-30 05:54:54 -05:00
|
|
|
});
|
|
|
|
}
|
2012-09-07 18:58:27 -05:00
|
|
|
} else {
|
|
|
|
loop {
|
|
|
|
if is_eof(rdr) { rdr.fatal(~"unterminated block comment"); }
|
2012-05-30 13:36:30 -05:00
|
|
|
if rdr.curr == '*' && nextch(rdr) == '/' {
|
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-09-07 18:58:27 -05:00
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
bump(rdr);
|
|
|
|
}
|
2010-08-31 15:29:00 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// restart whitespace munch.
|
2011-06-15 13:19:50 -05:00
|
|
|
|
2012-08-01 19:30:05 -05:00
|
|
|
return consume_whitespace_and_comments(rdr);
|
2010-08-31 15:29:00 -05:00
|
|
|
}
|
|
|
|
|
2012-08-20 14:23:37 -05:00
|
|
|
fn scan_exponent(rdr: string_reader) -> Option<~str> {
|
2012-03-15 08:47:03 -05:00
|
|
|
let mut c = rdr.curr;
|
2012-07-14 00:57:48 -05:00
|
|
|
let mut rslt = ~"";
|
2011-07-27 07:19:39 -05:00
|
|
|
if c == 'e' || c == 'E' {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut rslt, c);
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
c = rdr.curr;
|
2011-07-27 07:19:39 -05:00
|
|
|
if c == '-' || c == '+' {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut rslt, c);
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2011-03-22 19:25:40 -05:00
|
|
|
}
|
2011-12-07 14:06:12 -06:00
|
|
|
let exponent = scan_digits(rdr, 10u);
|
2012-02-23 03:44:04 -06:00
|
|
|
if str::len(exponent) > 0u {
|
2012-08-20 14:23:37 -05:00
|
|
|
return Some(rslt + exponent);
|
2012-07-14 00:57:48 -05:00
|
|
|
} else { rdr.fatal(~"scan_exponent: bad fp literal"); }
|
2012-08-20 14:23:37 -05:00
|
|
|
} else { return None::<~str>; }
|
2011-03-24 19:12:24 -05:00
|
|
|
}
|
|
|
|
|
2012-07-14 00:57:48 -05:00
|
|
|
fn scan_digits(rdr: string_reader, radix: uint) -> ~str {
|
|
|
|
let mut rslt = ~"";
|
2012-03-10 22:34:17 -06:00
|
|
|
loop {
|
2012-01-13 02:56:53 -06:00
|
|
|
let c = rdr.curr;
|
2012-09-07 17:32:04 -05:00
|
|
|
if c == '_' { bump(rdr); loop; }
|
2012-08-06 14:34:08 -05:00
|
|
|
match char::to_digit(c, radix) {
|
2012-08-20 14:23:37 -05:00
|
|
|
Some(_) => {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut rslt, c);
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2011-12-07 14:06:12 -06:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
_ => return rslt
|
2011-12-07 14:06:12 -06:00
|
|
|
}
|
2012-03-10 22:34:17 -06:00
|
|
|
};
|
2011-03-22 19:25:40 -05:00
|
|
|
}
|
|
|
|
|
2012-10-15 16:56:42 -05:00
|
|
|
fn scan_number(c: char, rdr: string_reader) -> token::Token {
|
2012-05-30 13:36:30 -05:00
|
|
|
let mut num_str, base = 10u, c = c, n = nextch(rdr);
|
2011-07-27 07:19:39 -05:00
|
|
|
if c == '0' && n == 'x' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2011-12-07 14:06:12 -06:00
|
|
|
base = 16u;
|
2011-08-19 17:16:48 -05:00
|
|
|
} else if c == '0' && n == 'b' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2011-12-07 14:06:12 -06:00
|
|
|
base = 2u;
|
2011-09-02 00:39:44 -05:00
|
|
|
}
|
2011-12-07 14:06:12 -06:00
|
|
|
num_str = scan_digits(rdr, base);
|
2012-01-13 02:56:53 -06:00
|
|
|
c = rdr.curr;
|
2012-05-30 13:36:30 -05:00
|
|
|
nextch(rdr);
|
2011-07-27 07:19:39 -05:00
|
|
|
if c == 'u' || c == 'i' {
|
2012-03-15 08:47:03 -05:00
|
|
|
let signed = c == 'i';
|
|
|
|
let mut tp = {
|
2012-08-14 18:54:13 -05:00
|
|
|
if signed { either::Left(ast::ty_i) }
|
|
|
|
else { either::Right(ast::ty_u) }
|
2012-03-15 08:47:03 -05:00
|
|
|
};
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
c = rdr.curr;
|
2011-07-27 07:19:39 -05:00
|
|
|
if c == '8' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-14 18:54:13 -05:00
|
|
|
tp = if signed { either::Left(ast::ty_i8) }
|
|
|
|
else { either::Right(ast::ty_u8) };
|
2010-11-22 19:41:26 -06:00
|
|
|
}
|
2012-05-30 13:36:30 -05:00
|
|
|
n = nextch(rdr);
|
2011-07-27 07:19:39 -05:00
|
|
|
if c == '1' && n == '6' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-14 18:54:13 -05:00
|
|
|
tp = if signed { either::Left(ast::ty_i16) }
|
|
|
|
else { either::Right(ast::ty_u16) };
|
2011-12-07 14:06:12 -06:00
|
|
|
} else if c == '3' && n == '2' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-14 18:54:13 -05:00
|
|
|
tp = if signed { either::Left(ast::ty_i32) }
|
|
|
|
else { either::Right(ast::ty_u32) };
|
2011-12-07 14:06:12 -06:00
|
|
|
} else if c == '6' && n == '4' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-14 18:54:13 -05:00
|
|
|
tp = if signed { either::Left(ast::ty_i64) }
|
|
|
|
else { either::Right(ast::ty_u64) };
|
2010-11-22 19:41:26 -06:00
|
|
|
}
|
2012-02-23 03:44:04 -06:00
|
|
|
if str::len(num_str) == 0u {
|
2012-07-14 00:57:48 -05:00
|
|
|
rdr.fatal(~"no valid digits found for number");
|
2012-02-10 02:50:16 -06:00
|
|
|
}
|
2012-09-21 21:37:57 -05:00
|
|
|
let parsed = u64::from_str_radix(num_str, base as u64).get();
|
2012-08-06 14:34:08 -05:00
|
|
|
match tp {
|
2012-08-14 18:54:13 -05:00
|
|
|
either::Left(t) => return token::LIT_INT(parsed as i64, t),
|
|
|
|
either::Right(t) => return token::LIT_UINT(parsed, t)
|
2010-11-22 19:41:26 -06:00
|
|
|
}
|
|
|
|
}
|
2012-03-15 08:47:03 -05:00
|
|
|
let mut is_float = false;
|
2012-08-13 19:37:08 -05:00
|
|
|
if rdr.curr == '.' && !(is_alpha(nextch(rdr)) || nextch(rdr) == '_' ||
|
|
|
|
nextch(rdr) == '.') {
|
2011-09-13 10:43:01 -05:00
|
|
|
is_float = true;
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2011-12-07 14:06:12 -06:00
|
|
|
let dec_part = scan_digits(rdr, 10u);
|
2012-07-14 00:57:48 -05:00
|
|
|
num_str += ~"." + dec_part;
|
2011-09-13 10:43:01 -05:00
|
|
|
}
|
2012-08-06 14:34:08 -05:00
|
|
|
match scan_exponent(rdr) {
|
2012-12-04 12:50:00 -06:00
|
|
|
Some(ref s) => {
|
2011-09-13 10:43:01 -05:00
|
|
|
is_float = true;
|
2012-12-04 12:50:00 -06:00
|
|
|
num_str += (*s);
|
2011-09-13 10:43:01 -05:00
|
|
|
}
|
2012-08-20 14:23:37 -05:00
|
|
|
None => ()
|
2011-09-13 10:43:01 -05:00
|
|
|
}
|
2012-11-07 20:40:34 -06:00
|
|
|
|
|
|
|
let mut is_machine_float = false;
|
2012-01-13 02:56:53 -06:00
|
|
|
if rdr.curr == 'f' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
c = rdr.curr;
|
2012-05-30 13:36:30 -05:00
|
|
|
n = nextch(rdr);
|
2011-09-13 10:43:01 -05:00
|
|
|
if c == '3' && n == '2' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-14 17:27:06 -05:00
|
|
|
return token::LIT_FLOAT(rdr.interner.intern(@num_str),
|
2012-07-17 19:05:38 -05:00
|
|
|
ast::ty_f32);
|
2011-09-13 10:43:01 -05:00
|
|
|
} else if c == '6' && n == '4' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-14 17:27:06 -05:00
|
|
|
return token::LIT_FLOAT(rdr.interner.intern(@num_str),
|
2012-07-17 19:05:38 -05:00
|
|
|
ast::ty_f64);
|
2012-06-21 18:44:10 -05:00
|
|
|
/* FIXME (#2252): if this is out of range for either a
|
|
|
|
32-bit or 64-bit float, it won't be noticed till the
|
|
|
|
back-end. */
|
2011-06-15 13:19:50 -05:00
|
|
|
} else {
|
2011-09-13 10:43:01 -05:00
|
|
|
is_float = true;
|
2012-11-07 20:40:34 -06:00
|
|
|
is_machine_float = true;
|
2011-03-22 19:25:40 -05:00
|
|
|
}
|
2011-03-21 19:12:05 -05:00
|
|
|
}
|
2011-09-13 10:43:01 -05:00
|
|
|
if is_float {
|
2012-11-07 20:40:34 -06:00
|
|
|
if is_machine_float {
|
|
|
|
return token::LIT_FLOAT(rdr.interner.intern(@num_str), ast::ty_f);
|
|
|
|
}
|
|
|
|
return token::LIT_FLOAT_UNSUFFIXED(rdr.interner.intern(@num_str));
|
2011-09-13 10:43:01 -05:00
|
|
|
} else {
|
2012-02-23 03:44:04 -06:00
|
|
|
if str::len(num_str) == 0u {
|
2012-07-14 00:57:48 -05:00
|
|
|
rdr.fatal(~"no valid digits found for number");
|
2012-02-10 02:50:16 -06:00
|
|
|
}
|
2012-09-21 21:37:57 -05:00
|
|
|
let parsed = u64::from_str_radix(num_str, base as u64).get();
|
2012-06-11 18:31:03 -05:00
|
|
|
|
2012-08-22 19:24:52 -05:00
|
|
|
debug!("lexing %s as an unsuffixed integer literal",
|
|
|
|
num_str);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::LIT_INT_UNSUFFIXED(parsed as i64);
|
2011-03-21 19:12:05 -05:00
|
|
|
}
|
2010-11-22 19:41:26 -06:00
|
|
|
}
|
|
|
|
|
2012-05-30 13:36:30 -05:00
|
|
|
fn scan_numeric_escape(rdr: string_reader, n_hex_digits: uint) -> char {
|
2012-03-15 08:47:03 -05:00
|
|
|
let mut accum_int = 0, i = n_hex_digits;
|
2011-11-03 04:57:54 -05:00
|
|
|
while i != 0u {
|
2012-01-13 02:56:53 -06:00
|
|
|
let n = rdr.curr;
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2011-07-27 07:19:39 -05:00
|
|
|
if !is_hex_digit(n) {
|
2012-08-22 19:24:52 -05:00
|
|
|
rdr.fatal(fmt!("illegal numeric character escape: %d", n as int));
|
2011-04-13 13:25:01 -05:00
|
|
|
}
|
|
|
|
accum_int *= 16;
|
|
|
|
accum_int += hex_digit_val(n);
|
2011-11-03 04:57:54 -05:00
|
|
|
i -= 1u;
|
2011-04-13 13:25:01 -05:00
|
|
|
}
|
2012-08-01 19:30:05 -05:00
|
|
|
return accum_int as char;
|
2011-04-13 13:25:01 -05:00
|
|
|
}
|
|
|
|
|
2012-10-15 16:56:42 -05:00
|
|
|
fn next_token_inner(rdr: string_reader) -> token::Token {
|
2012-07-14 00:57:48 -05:00
|
|
|
let mut accum_str = ~"";
|
2012-03-15 08:47:03 -05:00
|
|
|
let mut c = rdr.curr;
|
2012-03-02 20:45:10 -06:00
|
|
|
if (c >= 'a' && c <= 'z')
|
|
|
|
|| (c >= 'A' && c <= 'Z')
|
|
|
|
|| c == '_'
|
|
|
|
|| (c > 'z' && char::is_XID_start(c)) {
|
|
|
|
while (c >= 'a' && c <= 'z')
|
|
|
|
|| (c >= 'A' && c <= 'Z')
|
|
|
|
|| (c >= '0' && c <= '9')
|
|
|
|
|| c == '_'
|
|
|
|
|| (c > 'z' && char::is_XID_continue(c)) {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut accum_str, c);
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
c = rdr.curr;
|
2010-08-20 13:41:34 -05:00
|
|
|
}
|
2012-08-02 17:42:56 -05:00
|
|
|
if accum_str == ~"_" { return token::UNDERSCORE; }
|
2012-05-30 13:36:30 -05:00
|
|
|
let is_mod_name = c == ':' && nextch(rdr) == ':';
|
2011-12-29 16:45:18 -06:00
|
|
|
|
2012-04-19 17:24:49 -05:00
|
|
|
// FIXME: perform NFKC normalization here. (Issue #2253)
|
2012-08-14 17:27:06 -05:00
|
|
|
return token::IDENT(rdr.interner.intern(@accum_str), is_mod_name);
|
2010-08-20 12:03:31 -05:00
|
|
|
}
|
2011-12-02 06:57:12 -06:00
|
|
|
if is_dec_digit(c) {
|
2012-08-01 19:30:05 -05:00
|
|
|
return scan_number(c, rdr);
|
2011-09-02 00:39:44 -05:00
|
|
|
}
|
2012-10-15 16:56:42 -05:00
|
|
|
fn binop(rdr: string_reader, op: token::binop) -> token::Token {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
if rdr.curr == '=' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::BINOPEQ(op);
|
|
|
|
} else { return token::BINOP(op); }
|
2010-08-23 21:17:04 -05:00
|
|
|
}
|
2012-08-06 14:34:08 -05:00
|
|
|
match c {
|
2011-07-27 07:19:39 -05:00
|
|
|
|
2011-08-19 17:16:48 -05:00
|
|
|
|
2011-09-02 17:34:58 -05:00
|
|
|
|
2011-09-12 04:27:30 -05:00
|
|
|
|
2011-09-12 05:39:38 -05:00
|
|
|
|
2011-07-27 07:19:39 -05:00
|
|
|
// One-byte tokens.
|
2012-08-03 21:59:04 -05:00
|
|
|
';' => { bump(rdr); return token::SEMI; }
|
|
|
|
',' => { bump(rdr); return token::COMMA; }
|
|
|
|
'.' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-03 20:01:30 -05:00
|
|
|
if rdr.curr == '.' && nextch(rdr) != '.' {
|
|
|
|
bump(rdr);
|
|
|
|
return token::DOTDOT;
|
|
|
|
}
|
2012-05-30 13:36:30 -05:00
|
|
|
if rdr.curr == '.' && nextch(rdr) == '.' {
|
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::ELLIPSIS;
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::DOT;
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'(' => { bump(rdr); return token::LPAREN; }
|
|
|
|
')' => { bump(rdr); return token::RPAREN; }
|
|
|
|
'{' => { bump(rdr); return token::LBRACE; }
|
|
|
|
'}' => { bump(rdr); return token::RBRACE; }
|
|
|
|
'[' => { bump(rdr); return token::LBRACKET; }
|
|
|
|
']' => { bump(rdr); return token::RBRACKET; }
|
|
|
|
'@' => { bump(rdr); return token::AT; }
|
|
|
|
'#' => { bump(rdr); return token::POUND; }
|
|
|
|
'~' => { bump(rdr); return token::TILDE; }
|
|
|
|
':' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
if rdr.curr == ':' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::MOD_SEP;
|
|
|
|
} else { return token::COLON; }
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
|
|
|
|
2012-08-03 21:59:04 -05:00
|
|
|
'$' => { bump(rdr); return token::DOLLAR; }
|
2012-01-25 17:38:09 -06:00
|
|
|
|
2011-08-19 17:16:48 -05:00
|
|
|
|
2011-09-02 17:34:58 -05:00
|
|
|
|
2011-09-12 04:27:30 -05:00
|
|
|
|
2011-09-12 05:39:38 -05:00
|
|
|
|
2011-07-27 07:19:39 -05:00
|
|
|
// Multi-byte tokens.
|
2012-08-03 21:59:04 -05:00
|
|
|
'=' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
if rdr.curr == '=' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::EQEQ;
|
2012-06-04 20:34:10 -05:00
|
|
|
} else if rdr.curr == '>' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::FAT_ARROW;
|
2012-06-04 20:34:10 -05:00
|
|
|
} else {
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::EQ;
|
2012-06-04 20:34:10 -05:00
|
|
|
}
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'!' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
if rdr.curr == '=' {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::NE;
|
|
|
|
} else { return token::NOT; }
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'<' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-06 14:34:08 -05:00
|
|
|
match rdr.curr {
|
2012-08-03 21:59:04 -05:00
|
|
|
'=' => { bump(rdr); return token::LE; }
|
|
|
|
'<' => { return binop(rdr, token::SHL); }
|
|
|
|
'-' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-06 14:34:08 -05:00
|
|
|
match rdr.curr {
|
2012-08-03 21:59:04 -05:00
|
|
|
'>' => { bump(rdr); return token::DARROW; }
|
|
|
|
_ => { return token::LARROW; }
|
2010-08-31 15:29:00 -05:00
|
|
|
}
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
_ => { return token::LT; }
|
2010-08-31 15:29:00 -05:00
|
|
|
}
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'>' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-06 14:34:08 -05:00
|
|
|
match rdr.curr {
|
2012-08-03 21:59:04 -05:00
|
|
|
'=' => { bump(rdr); return token::GE; }
|
|
|
|
'>' => { return binop(rdr, token::SHR); }
|
|
|
|
_ => { return token::GT; }
|
2010-08-20 17:36:48 -05:00
|
|
|
}
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'\'' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-03-15 08:47:03 -05:00
|
|
|
let mut c2 = rdr.curr;
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2011-07-27 07:19:39 -05:00
|
|
|
if c2 == '\\' {
|
2012-01-13 02:56:53 -06:00
|
|
|
let escaped = rdr.curr;
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-06 14:34:08 -05:00
|
|
|
match escaped {
|
2012-08-03 21:59:04 -05:00
|
|
|
'n' => { c2 = '\n'; }
|
|
|
|
'r' => { c2 = '\r'; }
|
|
|
|
't' => { c2 = '\t'; }
|
|
|
|
'\\' => { c2 = '\\'; }
|
|
|
|
'\'' => { c2 = '\''; }
|
|
|
|
'"' => { c2 = '"'; }
|
|
|
|
'x' => { c2 = scan_numeric_escape(rdr, 2u); }
|
|
|
|
'u' => { c2 = scan_numeric_escape(rdr, 4u); }
|
|
|
|
'U' => { c2 = scan_numeric_escape(rdr, 8u); }
|
|
|
|
c2 => {
|
2012-08-22 19:24:52 -05:00
|
|
|
rdr.fatal(fmt!("unknown character escape: %d", c2 as int));
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2010-08-24 10:56:42 -05:00
|
|
|
}
|
|
|
|
}
|
2012-01-13 02:56:53 -06:00
|
|
|
if rdr.curr != '\'' {
|
2012-07-14 00:57:48 -05:00
|
|
|
rdr.fatal(~"unterminated character constant");
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr); // advance curr past token
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::LIT_INT(c2 as i64, ast::ty_char);
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'"' => {
|
2012-11-15 00:27:53 -06:00
|
|
|
let n = byte_offset(rdr);
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-01-13 02:56:53 -06:00
|
|
|
while rdr.curr != '"' {
|
2012-05-30 13:36:30 -05:00
|
|
|
if is_eof(rdr) {
|
2012-08-22 19:24:52 -05:00
|
|
|
rdr.fatal(fmt!("unterminated double quote string: %s",
|
|
|
|
get_str_from(rdr, n)));
|
2011-09-27 13:15:24 -05:00
|
|
|
}
|
|
|
|
|
2012-01-13 02:56:53 -06:00
|
|
|
let ch = rdr.curr;
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-06 14:34:08 -05:00
|
|
|
match ch {
|
2012-08-03 21:59:04 -05:00
|
|
|
'\\' => {
|
2012-01-13 02:56:53 -06:00
|
|
|
let escaped = rdr.curr;
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-06 14:34:08 -05:00
|
|
|
match escaped {
|
2012-09-21 20:36:32 -05:00
|
|
|
'n' => str::push_char(&mut accum_str, '\n'),
|
|
|
|
'r' => str::push_char(&mut accum_str, '\r'),
|
|
|
|
't' => str::push_char(&mut accum_str, '\t'),
|
|
|
|
'\\' => str::push_char(&mut accum_str, '\\'),
|
|
|
|
'\'' => str::push_char(&mut accum_str, '\''),
|
|
|
|
'"' => str::push_char(&mut accum_str, '"'),
|
2012-08-03 21:59:04 -05:00
|
|
|
'\n' => consume_whitespace(rdr),
|
|
|
|
'x' => {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut accum_str,
|
|
|
|
scan_numeric_escape(rdr, 2u));
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'u' => {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut accum_str,
|
|
|
|
scan_numeric_escape(rdr, 4u));
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'U' => {
|
2012-09-21 20:36:32 -05:00
|
|
|
str::push_char(&mut accum_str,
|
|
|
|
scan_numeric_escape(rdr, 8u));
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
c2 => {
|
2012-08-22 19:24:52 -05:00
|
|
|
rdr.fatal(fmt!("unknown string escape: %d", c2 as int));
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2010-08-24 10:56:42 -05:00
|
|
|
}
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-09-21 20:36:32 -05:00
|
|
|
_ => str::push_char(&mut accum_str, ch)
|
2010-08-24 10:56:42 -05:00
|
|
|
}
|
2011-04-13 13:25:01 -05:00
|
|
|
}
|
2012-05-30 13:36:30 -05:00
|
|
|
bump(rdr);
|
2012-08-14 17:27:06 -05:00
|
|
|
return token::LIT_STR(rdr.interner.intern(@accum_str));
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'-' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
if nextch(rdr) == '>' {
|
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::RARROW;
|
|
|
|
} else { return binop(rdr, token::MINUS); }
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'&' => {
|
2012-05-30 13:36:30 -05:00
|
|
|
if nextch(rdr) == '&' {
|
|
|
|
bump(rdr);
|
|
|
|
bump(rdr);
|
2012-08-01 19:30:05 -05:00
|
|
|
return token::ANDAND;
|
|
|
|
} else { return binop(rdr, token::AND); }
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'|' => {
|
2012-08-06 14:34:08 -05:00
|
|
|
match nextch(rdr) {
|
2012-08-03 21:59:04 -05:00
|
|
|
'|' => { bump(rdr); bump(rdr); return token::OROR; }
|
|
|
|
_ => { return binop(rdr, token::OR); }
|
2011-07-27 07:19:39 -05:00
|
|
|
}
|
|
|
|
}
|
2012-08-03 21:59:04 -05:00
|
|
|
'+' => { return binop(rdr, token::PLUS); }
|
|
|
|
'*' => { return binop(rdr, token::STAR); }
|
|
|
|
'/' => { return binop(rdr, token::SLASH); }
|
|
|
|
'^' => { return binop(rdr, token::CARET); }
|
|
|
|
'%' => { return binop(rdr, token::PERCENT); }
|
2012-08-22 19:24:52 -05:00
|
|
|
c => { rdr.fatal(fmt!("unknown start of token: %d", c as int)); }
|
2010-08-20 16:34:48 -05:00
|
|
|
}
|
2010-07-14 11:41:36 -05:00
|
|
|
}
|
|
|
|
|
2012-05-30 13:36:30 -05:00
|
|
|
fn consume_whitespace(rdr: string_reader) {
|
|
|
|
while is_whitespace(rdr.curr) && !is_eof(rdr) { bump(rdr); }
|
2011-03-24 10:33:20 -05:00
|
|
|
}
|
|
|
|
|
2011-10-07 09:22:53 -05:00
|
|
|
|
2010-08-12 12:29:23 -05:00
|
|
|
//
|
|
|
|
// Local Variables:
|
|
|
|
// mode: rust
|
|
|
|
// fill-column: 78;
|
|
|
|
// indent-tabs-mode: nil
|
|
|
|
// c-basic-offset: 4
|
|
|
|
// buffer-file-coding-system: utf-8-unix
|
|
|
|
// End:
|
|
|
|
//
|