2011-03-10 16:02:53 +01:00
|
|
|
import std.io;
|
2010-08-24 09:59:41 -07:00
|
|
|
import std._str;
|
2011-03-24 16:33:20 +01:00
|
|
|
import std._vec;
|
2011-03-21 17:12:05 -07:00
|
|
|
import std._int;
|
2010-08-24 09:59:41 -07:00
|
|
|
import std.map;
|
|
|
|
import std.map.hashmap;
|
2011-03-22 17:25:40 -07:00
|
|
|
import std.option;
|
|
|
|
import std.option.some;
|
|
|
|
import std.option.none;
|
2010-08-31 11:36:36 -07:00
|
|
|
import util.common;
|
2010-09-21 16:22:32 -07:00
|
|
|
import util.common.new_str_hash;
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2010-09-01 13:24:14 -07:00
|
|
|
state type reader = state obj {
|
2011-04-08 16:48:17 +02:00
|
|
|
fn is_eof() -> bool;
|
|
|
|
fn curr() -> char;
|
|
|
|
fn next() -> char;
|
2011-04-19 13:35:49 -07:00
|
|
|
fn init();
|
|
|
|
fn bump();
|
2011-04-08 16:48:17 +02:00
|
|
|
fn mark();
|
2011-04-08 18:44:20 +02:00
|
|
|
fn get_mark_chpos() -> uint;
|
|
|
|
fn get_chpos() -> uint;
|
2011-04-08 16:48:17 +02:00
|
|
|
fn get_keywords() -> hashmap[str,token.token];
|
|
|
|
fn get_reserved() -> hashmap[str,()];
|
2011-04-08 18:44:20 +02:00
|
|
|
fn get_filemap() -> codemap.filemap;
|
2010-08-24 09:59:41 -07:00
|
|
|
};
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn new_reader(io.reader rdr, str filename, codemap.filemap filemap)
|
2011-04-08 18:44:20 +02:00
|
|
|
-> reader {
|
2011-04-08 16:48:17 +02:00
|
|
|
state obj reader(str file,
|
|
|
|
uint len,
|
|
|
|
mutable uint pos,
|
|
|
|
mutable char ch,
|
2011-04-08 18:44:20 +02:00
|
|
|
mutable uint mark_chpos,
|
|
|
|
mutable uint chpos,
|
2010-09-01 13:24:14 -07:00
|
|
|
hashmap[str,token.token] keywords,
|
2011-04-08 18:44:20 +02:00
|
|
|
hashmap[str,()] reserved,
|
|
|
|
codemap.filemap fm) {
|
2010-09-01 13:24:14 -07:00
|
|
|
|
2011-04-08 16:48:17 +02:00
|
|
|
fn is_eof() -> bool {
|
|
|
|
ret ch == -1 as char;
|
|
|
|
}
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2011-04-08 18:44:20 +02:00
|
|
|
fn mark() { mark_chpos = chpos; }
|
|
|
|
fn get_mark_chpos() -> uint { ret mark_chpos; }
|
|
|
|
fn get_chpos() -> uint { ret chpos; }
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2011-04-08 16:48:17 +02:00
|
|
|
fn curr() -> char {
|
|
|
|
ret ch;
|
|
|
|
}
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2011-04-08 16:48:17 +02:00
|
|
|
fn next() -> char {
|
|
|
|
if (pos < len) {ret _str.char_at(file, pos);}
|
|
|
|
else {ret -1 as char;}
|
|
|
|
}
|
2011-04-13 11:25:01 -07:00
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn init() {
|
2011-04-08 16:48:17 +02:00
|
|
|
if (pos < len) {
|
|
|
|
auto next = _str.char_range_at(file, pos);
|
|
|
|
pos = next._1;
|
|
|
|
ch = next._0;
|
2010-08-27 12:36:57 -07:00
|
|
|
}
|
2011-04-08 16:48:17 +02:00
|
|
|
}
|
2010-08-27 12:36:57 -07:00
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn bump() {
|
2011-04-08 16:48:17 +02:00
|
|
|
if (pos < len) {
|
2011-04-08 18:44:20 +02:00
|
|
|
chpos += 1u;
|
2011-04-08 16:48:17 +02:00
|
|
|
if (ch == '\n') {
|
2011-04-08 18:44:20 +02:00
|
|
|
codemap.next_line(fm, chpos);
|
2010-08-24 09:59:41 -07:00
|
|
|
}
|
2011-04-08 16:48:17 +02:00
|
|
|
auto next = _str.char_range_at(file, pos);
|
|
|
|
pos = next._1;
|
|
|
|
ch = next._0;
|
|
|
|
} else {
|
|
|
|
ch = -1 as char;
|
2010-08-24 09:59:41 -07:00
|
|
|
}
|
2011-04-08 16:48:17 +02:00
|
|
|
}
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2011-04-08 16:48:17 +02:00
|
|
|
fn get_keywords() -> hashmap[str,token.token] {
|
|
|
|
ret keywords;
|
|
|
|
}
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2011-04-08 16:48:17 +02:00
|
|
|
fn get_reserved() -> hashmap[str,()] {
|
|
|
|
ret reserved;
|
2010-08-24 09:59:41 -07:00
|
|
|
}
|
2011-04-08 18:44:20 +02:00
|
|
|
|
|
|
|
fn get_filemap() -> codemap.filemap {
|
|
|
|
ret fm;
|
|
|
|
}
|
2011-04-08 16:48:17 +02:00
|
|
|
}
|
2011-04-08 18:44:20 +02:00
|
|
|
auto file = _str.unsafe_from_bytes(rdr.read_whole_stream());
|
|
|
|
auto rd = reader(file, _str.byte_len(file), 0u, -1 as char,
|
|
|
|
filemap.start_pos, filemap.start_pos,
|
|
|
|
keyword_table(),
|
|
|
|
reserved_word_table(),
|
|
|
|
filemap);
|
|
|
|
rd.init();
|
|
|
|
ret rd;
|
|
|
|
}
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2011-04-08 18:44:20 +02:00
|
|
|
fn keyword_table() -> std.map.hashmap[str, token.token] {
|
2010-08-24 09:59:41 -07:00
|
|
|
auto keywords = new_str_hash[token.token]();
|
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("mod", token.MOD);
|
|
|
|
keywords.insert("use", token.USE);
|
|
|
|
keywords.insert("meta", token.META);
|
|
|
|
keywords.insert("auth", token.AUTH);
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("syntax", token.SYNTAX);
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("if", token.IF);
|
|
|
|
keywords.insert("else", token.ELSE);
|
|
|
|
keywords.insert("while", token.WHILE);
|
|
|
|
keywords.insert("do", token.DO);
|
|
|
|
keywords.insert("alt", token.ALT);
|
|
|
|
keywords.insert("case", token.CASE);
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("for", token.FOR);
|
|
|
|
keywords.insert("each", token.EACH);
|
2011-03-25 16:28:16 +01:00
|
|
|
keywords.insert("break", token.BREAK);
|
|
|
|
keywords.insert("cont", token.CONT);
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("put", token.PUT);
|
|
|
|
keywords.insert("ret", token.RET);
|
|
|
|
keywords.insert("be", token.BE);
|
2010-08-24 09:59:41 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("fail", token.FAIL);
|
|
|
|
keywords.insert("drop", token.DROP);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("type", token.TYPE);
|
|
|
|
keywords.insert("check", token.CHECK);
|
2011-05-02 17:47:24 -07:00
|
|
|
keywords.insert("assert", token.ASSERT);
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("claim", token.CLAIM);
|
|
|
|
keywords.insert("prove", token.PROVE);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("state", token.STATE);
|
2010-11-03 10:29:48 -07:00
|
|
|
keywords.insert("gc", token.GC);
|
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("unsafe", token.UNSAFE);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("native", token.NATIVE);
|
|
|
|
keywords.insert("mutable", token.MUTABLE);
|
|
|
|
keywords.insert("auto", token.AUTO);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("fn", token.FN);
|
|
|
|
keywords.insert("iter", token.ITER);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("import", token.IMPORT);
|
|
|
|
keywords.insert("export", token.EXPORT);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("let", token.LET);
|
|
|
|
keywords.insert("const", token.CONST);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("log", token.LOG);
|
2011-04-19 11:21:23 +02:00
|
|
|
keywords.insert("log_err", token.LOG_ERR);
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("spawn", token.SPAWN);
|
|
|
|
keywords.insert("thread", token.THREAD);
|
|
|
|
keywords.insert("yield", token.YIELD);
|
|
|
|
keywords.insert("join", token.JOIN);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("bool", token.BOOL);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("int", token.INT);
|
|
|
|
keywords.insert("uint", token.UINT);
|
|
|
|
keywords.insert("float", token.FLOAT);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("char", token.CHAR);
|
|
|
|
keywords.insert("str", token.STR);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("rec", token.REC);
|
|
|
|
keywords.insert("tup", token.TUP);
|
|
|
|
keywords.insert("tag", token.TAG);
|
|
|
|
keywords.insert("vec", token.VEC);
|
|
|
|
keywords.insert("any", token.ANY);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("obj", token.OBJ);
|
2011-03-29 15:51:53 -07:00
|
|
|
keywords.insert("self", token.SELF);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("port", token.PORT);
|
|
|
|
keywords.insert("chan", token.CHAN);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("task", token.TASK);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
|
|
|
keywords.insert("true", token.LIT_BOOL(true));
|
|
|
|
keywords.insert("false", token.LIT_BOOL(false));
|
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("in", token.IN);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("as", token.AS);
|
|
|
|
keywords.insert("with", token.WITH);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("bind", token.BIND);
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
keywords.insert("u8", token.MACH(common.ty_u8));
|
|
|
|
keywords.insert("u16", token.MACH(common.ty_u16));
|
|
|
|
keywords.insert("u32", token.MACH(common.ty_u32));
|
|
|
|
keywords.insert("u64", token.MACH(common.ty_u64));
|
|
|
|
keywords.insert("i8", token.MACH(common.ty_i8));
|
|
|
|
keywords.insert("i16", token.MACH(common.ty_i16));
|
|
|
|
keywords.insert("i32", token.MACH(common.ty_i32));
|
|
|
|
keywords.insert("i64", token.MACH(common.ty_i64));
|
|
|
|
keywords.insert("f32", token.MACH(common.ty_f32));
|
|
|
|
keywords.insert("f64", token.MACH(common.ty_f64));
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2011-04-08 18:44:20 +02:00
|
|
|
ret keywords;
|
|
|
|
}
|
2011-02-22 20:23:38 -05:00
|
|
|
|
2011-04-08 18:44:20 +02:00
|
|
|
fn reserved_word_table() -> std.map.hashmap[str, ()] {
|
|
|
|
auto reserved = new_str_hash[()]();
|
2011-02-22 20:23:38 -05:00
|
|
|
reserved.insert("f16", ()); // IEEE 754-2008 'binary16' interchange fmt
|
|
|
|
reserved.insert("f80", ()); // IEEE 754-1985 'extended'
|
|
|
|
reserved.insert("f128", ()); // IEEE 754-2008 'binary128'
|
|
|
|
reserved.insert("m32", ()); // IEEE 754-2008 'decimal32'
|
|
|
|
reserved.insert("m64", ()); // IEEE 754-2008 'decimal64'
|
|
|
|
reserved.insert("m128", ()); // IEEE 754-2008 'decimal128'
|
|
|
|
reserved.insert("dec", ()); // One of m32, m64, m128
|
2011-04-08 18:44:20 +02:00
|
|
|
ret reserved;
|
2010-08-24 09:59:41 -07:00
|
|
|
}
|
|
|
|
|
2010-08-19 18:42:17 -07:00
|
|
|
fn in_range(char c, char lo, char hi) -> bool {
|
2010-08-20 12:12:37 -07:00
|
|
|
ret lo <= c && c <= hi;
|
2010-08-19 18:42:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn is_alpha(char c) -> bool {
|
|
|
|
ret in_range(c, 'a', 'z') ||
|
|
|
|
in_range(c, 'A', 'Z');
|
|
|
|
}
|
|
|
|
|
|
|
|
fn is_dec_digit(char c) -> bool {
|
|
|
|
ret in_range(c, '0', '9');
|
|
|
|
}
|
|
|
|
|
2010-10-13 11:02:56 -07:00
|
|
|
fn is_alnum(char c) -> bool {
|
|
|
|
ret is_alpha(c) || is_dec_digit(c);
|
|
|
|
}
|
|
|
|
|
2010-08-19 18:42:17 -07:00
|
|
|
fn is_hex_digit(char c) -> bool {
|
|
|
|
ret in_range(c, '0', '9') ||
|
|
|
|
in_range(c, 'a', 'f') ||
|
|
|
|
in_range(c, 'A', 'F');
|
|
|
|
}
|
|
|
|
|
|
|
|
fn is_bin_digit(char c) -> bool {
|
|
|
|
ret c == '0' || c == '1';
|
|
|
|
}
|
|
|
|
|
2010-08-31 11:36:36 -07:00
|
|
|
fn dec_digit_val(char c) -> int {
|
|
|
|
ret (c as int) - ('0' as int);
|
|
|
|
}
|
|
|
|
|
|
|
|
fn hex_digit_val(char c) -> int {
|
|
|
|
if (in_range(c, '0', '9')) {
|
|
|
|
ret (c as int) - ('0' as int);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (in_range(c, 'a', 'f')) {
|
2010-08-31 12:54:24 -07:00
|
|
|
ret ((c as int) - ('a' as int)) + 10;
|
2010-08-31 11:36:36 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
if (in_range(c, 'A', 'F')) {
|
2010-08-31 12:54:24 -07:00
|
|
|
ret ((c as int) - ('A' as int)) + 10;
|
2010-08-31 11:36:36 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
fn bin_digit_value(char c) -> int {
|
2010-08-31 12:54:24 -07:00
|
|
|
if (c == '0') { ret 0; }
|
2010-08-31 11:36:36 -07:00
|
|
|
ret 1;
|
|
|
|
}
|
|
|
|
|
2010-08-19 18:42:17 -07:00
|
|
|
fn is_whitespace(char c) -> bool {
|
2010-08-20 15:36:48 -07:00
|
|
|
ret c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
2010-08-19 18:42:17 -07:00
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn consume_any_whitespace(reader rdr) {
|
2010-08-27 12:36:57 -07:00
|
|
|
while (is_whitespace(rdr.curr())) {
|
|
|
|
rdr.bump();
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
2010-08-27 12:36:57 -07:00
|
|
|
be consume_any_line_comment(rdr);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn consume_any_line_comment(reader rdr) {
|
2010-08-27 12:36:57 -07:00
|
|
|
if (rdr.curr() == '/') {
|
2010-08-31 13:29:00 -07:00
|
|
|
alt (rdr.next()) {
|
|
|
|
case ('/') {
|
2011-03-29 14:07:41 +02:00
|
|
|
while (rdr.curr() != '\n' && !rdr.is_eof()) {
|
2010-08-31 13:29:00 -07:00
|
|
|
rdr.bump();
|
|
|
|
}
|
|
|
|
// Restart whitespace munch.
|
|
|
|
be consume_any_whitespace(rdr);
|
|
|
|
}
|
|
|
|
case ('*') {
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-08-31 13:29:00 -07:00
|
|
|
rdr.bump();
|
|
|
|
be consume_block_comment(rdr);
|
|
|
|
}
|
|
|
|
case (_) {
|
|
|
|
ret;
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-08-31 13:29:00 -07:00
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn consume_block_comment(reader rdr) {
|
2010-08-31 13:29:00 -07:00
|
|
|
let int level = 1;
|
|
|
|
while (level > 0) {
|
|
|
|
if (rdr.curr() == '/' && rdr.next() == '*') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
|
|
|
level += 1;
|
|
|
|
} else {
|
|
|
|
if (rdr.curr() == '*' && rdr.next() == '/') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
|
|
|
level -= 1;
|
|
|
|
} else {
|
|
|
|
rdr.bump();
|
|
|
|
}
|
|
|
|
}
|
2011-03-29 14:07:41 +02:00
|
|
|
if (rdr.is_eof()) {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err "unterminated block comment";
|
2011-03-29 14:07:41 +02:00
|
|
|
fail;
|
|
|
|
}
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
// restart whitespace munch.
|
|
|
|
be consume_any_whitespace(rdr);
|
|
|
|
}
|
|
|
|
|
2011-03-24 17:12:24 -07:00
|
|
|
fn digits_to_string(str s) -> int {
|
2011-03-21 17:12:05 -07:00
|
|
|
|
|
|
|
let int accum_int = 0;
|
2011-03-24 17:12:24 -07:00
|
|
|
let int i = 0;
|
2011-03-21 17:12:05 -07:00
|
|
|
|
2011-03-24 17:12:24 -07:00
|
|
|
for (u8 c in s) {
|
|
|
|
accum_int *= 10;
|
|
|
|
accum_int += dec_digit_val(c as char);
|
2011-03-21 17:12:05 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
ret accum_int;
|
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn scan_exponent(reader rdr) -> option.t[str] {
|
2011-03-22 17:25:40 -07:00
|
|
|
auto c = rdr.curr();
|
2011-03-24 17:12:24 -07:00
|
|
|
auto res = "";
|
2011-03-22 17:25:40 -07:00
|
|
|
|
|
|
|
if (c == 'e' || c == 'E') {
|
2011-03-24 17:12:24 -07:00
|
|
|
res += _str.from_bytes(vec(c as u8));
|
2011-03-22 17:25:40 -07:00
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
2011-03-24 17:12:24 -07:00
|
|
|
if (c == '-' || c == '+') {
|
|
|
|
res += _str.from_bytes(vec(c as u8));
|
2011-03-22 17:25:40 -07:00
|
|
|
rdr.bump();
|
|
|
|
}
|
|
|
|
auto exponent = scan_dec_digits(rdr);
|
2011-03-24 17:12:24 -07:00
|
|
|
if (_str.byte_len(exponent) > 0u) {
|
|
|
|
ret(some(res + exponent));
|
|
|
|
}
|
|
|
|
else {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err ("scan_exponent: bad fp literal");
|
2011-03-24 17:12:24 -07:00
|
|
|
fail;
|
|
|
|
}
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
|
|
|
else {
|
2011-03-24 17:12:24 -07:00
|
|
|
ret none[str];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn scan_dec_digits(reader rdr) -> str {
|
2011-04-13 11:25:01 -07:00
|
|
|
|
2011-03-24 17:12:24 -07:00
|
|
|
auto c = rdr.curr();
|
|
|
|
let str res = "";
|
|
|
|
|
|
|
|
while (is_dec_digit (c) || c == '_') {
|
|
|
|
if (c != '_') {
|
|
|
|
res += _str.from_bytes(vec(c as u8));
|
|
|
|
}
|
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
2011-03-24 17:12:24 -07:00
|
|
|
|
|
|
|
ret res;
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn scan_number(mutable char c, reader rdr) -> token.token {
|
2010-11-22 17:41:26 -08:00
|
|
|
auto accum_int = 0;
|
2011-03-24 17:12:24 -07:00
|
|
|
let str dec_str = "";
|
|
|
|
let bool is_dec_integer = false;
|
2010-11-22 17:41:26 -08:00
|
|
|
auto n = rdr.next();
|
|
|
|
|
|
|
|
if (c == '0' && n == 'x') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
|
|
|
while (is_hex_digit(c) || c == '_') {
|
|
|
|
if (c != '_') {
|
|
|
|
accum_int *= 16;
|
|
|
|
accum_int += hex_digit_val(c);
|
|
|
|
}
|
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
|
|
|
}
|
2011-03-21 17:12:05 -07:00
|
|
|
} else if (c == '0' && n == 'b') {
|
2010-11-22 17:41:26 -08:00
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
|
|
|
while (is_bin_digit(c) || c == '_') {
|
|
|
|
if (c != '_') {
|
|
|
|
accum_int *= 2;
|
|
|
|
accum_int += bin_digit_value(c);
|
|
|
|
}
|
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
|
|
|
}
|
2011-03-21 17:12:05 -07:00
|
|
|
} else {
|
2011-03-24 17:12:24 -07:00
|
|
|
dec_str = scan_dec_digits(rdr);
|
|
|
|
is_dec_integer = true;
|
|
|
|
}
|
2011-04-13 11:25:01 -07:00
|
|
|
|
2011-03-24 17:12:24 -07:00
|
|
|
if (is_dec_integer) {
|
|
|
|
accum_int = digits_to_string(dec_str);
|
2010-11-22 17:41:26 -08:00
|
|
|
}
|
2011-04-13 11:25:01 -07:00
|
|
|
|
2011-03-21 17:12:05 -07:00
|
|
|
c = rdr.curr();
|
|
|
|
n = rdr.next();
|
2010-11-22 17:41:26 -08:00
|
|
|
|
|
|
|
if (c == 'u' || c == 'i') {
|
|
|
|
let bool signed = (c == 'i');
|
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
|
|
|
if (c == '8') {
|
|
|
|
rdr.bump();
|
|
|
|
if (signed) {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_i8, accum_int);
|
|
|
|
} else {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_u8, accum_int);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
n = rdr.next();
|
|
|
|
if (c == '1' && n == '6') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
|
|
|
if (signed) {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_i16, accum_int);
|
|
|
|
} else {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_u16, accum_int);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (c == '3' && n == '2') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
|
|
|
if (signed) {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_i32, accum_int);
|
|
|
|
} else {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_u32, accum_int);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (c == '6' && n == '4') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
|
|
|
if (signed) {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_i64, accum_int);
|
|
|
|
} else {
|
|
|
|
ret token.LIT_MACH_INT(common.ty_u64, accum_int);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (signed) {
|
|
|
|
ret token.LIT_INT(accum_int);
|
|
|
|
} else {
|
|
|
|
// FIXME: should cast in the target bit-width.
|
|
|
|
ret token.LIT_UINT(accum_int as uint);
|
|
|
|
}
|
|
|
|
}
|
2011-03-22 17:25:40 -07:00
|
|
|
c = rdr.curr();
|
2011-03-24 17:12:24 -07:00
|
|
|
|
2011-03-22 17:25:40 -07:00
|
|
|
if (c == '.') {
|
2011-03-21 17:12:05 -07:00
|
|
|
// Parse a floating-point number.
|
|
|
|
rdr.bump();
|
2011-03-24 17:12:24 -07:00
|
|
|
auto dec_part = scan_dec_digits(rdr);
|
|
|
|
auto float_str = dec_str + "." + dec_part;
|
2011-03-22 17:25:40 -07:00
|
|
|
c = rdr.curr();
|
2011-03-24 17:12:24 -07:00
|
|
|
auto exponent_str = scan_exponent(rdr);
|
|
|
|
alt (exponent_str) {
|
|
|
|
case (some[str](?s)) {
|
|
|
|
float_str += s;
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
2011-03-24 17:12:24 -07:00
|
|
|
case (none[str]) {
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
c = rdr.curr();
|
|
|
|
if (c == 'f') {
|
|
|
|
rdr.bump();
|
|
|
|
c = rdr.curr();
|
|
|
|
n = rdr.next();
|
|
|
|
if (c == '3' && n == '2') {
|
|
|
|
rdr.bump(); rdr.bump();
|
|
|
|
ret token.LIT_MACH_FLOAT(util.common.ty_f32,
|
2011-03-24 17:12:24 -07:00
|
|
|
float_str);
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
|
|
|
else if (c == '6' && n == '4') {
|
|
|
|
rdr.bump(); rdr.bump();
|
|
|
|
ret token.LIT_MACH_FLOAT(util.common.ty_f64,
|
2011-03-24 17:12:24 -07:00
|
|
|
float_str);
|
|
|
|
/* FIXME: if this is out of range for either a 32-bit or
|
|
|
|
64-bit float, it won't be noticed till the back-end */
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else {
|
2011-03-24 17:12:24 -07:00
|
|
|
ret token.LIT_FLOAT(float_str);
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
2011-03-21 17:12:05 -07:00
|
|
|
}
|
2011-03-22 17:25:40 -07:00
|
|
|
|
|
|
|
auto maybe_exponent = scan_exponent(rdr);
|
|
|
|
alt(maybe_exponent) {
|
2011-03-24 17:12:24 -07:00
|
|
|
case(some[str](?s)) {
|
|
|
|
ret token.LIT_FLOAT(dec_str + s);
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
2011-03-24 17:12:24 -07:00
|
|
|
case(none[str]) {
|
|
|
|
ret token.LIT_INT(accum_int);
|
2011-03-22 17:25:40 -07:00
|
|
|
}
|
2011-03-21 17:12:05 -07:00
|
|
|
}
|
2010-11-22 17:41:26 -08:00
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn scan_numeric_escape(reader rdr) -> char {
|
2011-04-13 11:25:01 -07:00
|
|
|
|
|
|
|
auto n_hex_digits = 0;
|
|
|
|
|
2011-05-02 17:47:24 -07:00
|
|
|
assert (rdr.curr() == '\\');
|
2011-04-13 11:25:01 -07:00
|
|
|
|
|
|
|
alt (rdr.next()) {
|
|
|
|
case ('x') { n_hex_digits = 2; }
|
|
|
|
case ('u') { n_hex_digits = 4; }
|
|
|
|
case ('U') { n_hex_digits = 8; }
|
|
|
|
case (?c) {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err #fmt("unknown numeric character escape: %d", c as int);
|
2011-04-13 11:25:01 -07:00
|
|
|
fail;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
rdr.bump(); // advance curr past \
|
|
|
|
|
|
|
|
auto n = rdr.next();
|
|
|
|
auto accum_int = 0;
|
|
|
|
|
|
|
|
while (n_hex_digits != 0) {
|
|
|
|
if (!is_hex_digit(n)) {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err #fmt("illegal numeric character escape: %d", n as int);
|
2011-04-13 11:25:01 -07:00
|
|
|
fail;
|
|
|
|
}
|
|
|
|
accum_int *= 16;
|
|
|
|
accum_int += hex_digit_val(n);
|
|
|
|
rdr.bump();
|
|
|
|
n = rdr.next();
|
|
|
|
n_hex_digits -= 1;
|
|
|
|
}
|
|
|
|
ret accum_int as char;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn next_token(reader rdr) -> token.token {
|
2010-08-20 15:36:48 -07:00
|
|
|
auto accum_str = "";
|
2010-08-19 18:42:17 -07:00
|
|
|
|
2010-08-27 12:36:57 -07:00
|
|
|
consume_any_whitespace(rdr);
|
2010-08-24 08:50:56 -07:00
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
if (rdr.is_eof()) { ret token.EOF; }
|
2010-08-19 18:42:17 -07:00
|
|
|
|
2010-12-09 17:11:05 -08:00
|
|
|
rdr.mark();
|
2010-08-27 12:36:57 -07:00
|
|
|
auto c = rdr.curr();
|
2010-08-20 12:12:37 -07:00
|
|
|
|
2010-10-13 11:02:56 -07:00
|
|
|
if (is_alpha(c) || c == '_') {
|
|
|
|
while (is_alnum(c) || c == '_') {
|
2011-03-24 16:07:43 +01:00
|
|
|
_str.push_char(accum_str, c);
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-08-31 11:36:36 -07:00
|
|
|
c = rdr.curr();
|
2010-08-20 11:41:34 -07:00
|
|
|
}
|
2010-08-31 11:36:36 -07:00
|
|
|
|
2010-12-15 09:44:48 -08:00
|
|
|
if (_str.eq(accum_str, "_")) {
|
|
|
|
ret token.UNDERSCORE;
|
|
|
|
}
|
|
|
|
|
2010-08-31 11:36:36 -07:00
|
|
|
auto kwds = rdr.get_keywords();
|
|
|
|
if (kwds.contains_key(accum_str)) {
|
|
|
|
ret kwds.get(accum_str);
|
|
|
|
}
|
|
|
|
|
2011-02-22 20:23:38 -05:00
|
|
|
auto rsvd = rdr.get_reserved();
|
|
|
|
if (rsvd.contains_key(accum_str)) {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err #fmt("reserved keyword: %s", accum_str);
|
2011-02-22 20:23:38 -05:00
|
|
|
fail;
|
|
|
|
}
|
|
|
|
|
2010-08-20 15:36:48 -07:00
|
|
|
ret token.IDENT(accum_str);
|
2010-08-20 10:03:31 -07:00
|
|
|
}
|
2010-08-20 11:41:34 -07:00
|
|
|
|
|
|
|
if (is_dec_digit(c)) {
|
2010-11-22 17:41:26 -08:00
|
|
|
ret scan_number(c, rdr);
|
2010-08-20 11:41:34 -07:00
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn binop(reader rdr, token.binop op) -> token.token {
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-12-08 14:50:47 -08:00
|
|
|
if (rdr.curr() == '=') {
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-08-31 13:29:00 -07:00
|
|
|
ret token.BINOPEQ(op);
|
2010-08-23 19:17:04 -07:00
|
|
|
} else {
|
2010-08-31 13:29:00 -07:00
|
|
|
ret token.BINOP(op);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-08-20 14:34:48 -07:00
|
|
|
alt (c) {
|
2010-08-23 19:17:04 -07:00
|
|
|
// One-byte tokens.
|
2010-09-20 23:56:43 -07:00
|
|
|
case (':') { rdr.bump(); ret token.COLON; }
|
2010-09-21 08:30:06 -07:00
|
|
|
case ('?') { rdr.bump(); ret token.QUES; }
|
2010-09-20 23:56:43 -07:00
|
|
|
case (';') { rdr.bump(); ret token.SEMI; }
|
|
|
|
case (',') { rdr.bump(); ret token.COMMA; }
|
|
|
|
case ('.') { rdr.bump(); ret token.DOT; }
|
|
|
|
case ('(') { rdr.bump(); ret token.LPAREN; }
|
|
|
|
case (')') { rdr.bump(); ret token.RPAREN; }
|
|
|
|
case ('{') { rdr.bump(); ret token.LBRACE; }
|
|
|
|
case ('}') { rdr.bump(); ret token.RBRACE; }
|
|
|
|
case ('[') { rdr.bump(); ret token.LBRACKET; }
|
|
|
|
case (']') { rdr.bump(); ret token.RBRACKET; }
|
|
|
|
case ('@') { rdr.bump(); ret token.AT; }
|
|
|
|
case ('#') { rdr.bump(); ret token.POUND; }
|
|
|
|
case ('~') { rdr.bump(); ret token.TILDE; }
|
2010-08-31 13:29:00 -07:00
|
|
|
|
2010-08-23 19:17:04 -07:00
|
|
|
|
|
|
|
// Multi-byte tokens.
|
2010-08-20 15:36:48 -07:00
|
|
|
case ('=') {
|
2010-08-31 13:29:00 -07:00
|
|
|
rdr.bump();
|
|
|
|
if (rdr.curr() == '=') {
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.EQEQ;
|
2010-08-20 15:36:48 -07:00
|
|
|
} else {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.EQ;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('!') {
|
|
|
|
rdr.bump();
|
|
|
|
if (rdr.curr() == '=') {
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.NE;
|
2010-08-31 13:29:00 -07:00
|
|
|
} else {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.NOT;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('<') {
|
|
|
|
rdr.bump();
|
|
|
|
alt (rdr.curr()) {
|
|
|
|
case ('=') {
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.LE;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
case ('<') {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.LSL);
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
case ('-') {
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.LARROW;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
case ('|') {
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.SEND;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
case (_) {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.LT;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('>') {
|
|
|
|
rdr.bump();
|
|
|
|
alt (rdr.curr()) {
|
|
|
|
case ('=') {
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.GE;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
case ('>') {
|
|
|
|
if (rdr.next() == '>') {
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.ASR);
|
2010-08-31 13:29:00 -07:00
|
|
|
} else {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.LSR);
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
case (_) {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.GT;
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
2010-08-20 15:36:48 -07:00
|
|
|
}
|
|
|
|
}
|
2010-08-23 19:17:04 -07:00
|
|
|
|
2010-08-24 08:56:42 -07:00
|
|
|
case ('\'') {
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
|
|
|
auto c2 = rdr.curr();
|
2010-08-24 08:56:42 -07:00
|
|
|
if (c2 == '\\') {
|
2010-08-27 12:36:57 -07:00
|
|
|
alt (rdr.next()) {
|
2011-04-13 11:25:01 -07:00
|
|
|
case ('n') { c2 = '\n'; }
|
|
|
|
case ('r') { c2 = '\r'; }
|
|
|
|
case ('t') { c2 = '\t'; }
|
|
|
|
case ('\\') { c2 = '\\'; }
|
|
|
|
case ('\'') { c2 = '\''; }
|
|
|
|
|
|
|
|
case ('x') { c2 = scan_numeric_escape(rdr); }
|
|
|
|
case ('u') { c2 = scan_numeric_escape(rdr); }
|
|
|
|
case ('U') { c2 = scan_numeric_escape(rdr); }
|
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
case (?c2) {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err #fmt("unknown character escape: %d",
|
|
|
|
c2 as int);
|
2010-08-24 08:56:42 -07:00
|
|
|
fail;
|
|
|
|
}
|
|
|
|
}
|
2011-04-13 11:25:01 -07:00
|
|
|
rdr.bump();
|
2010-08-24 08:56:42 -07:00
|
|
|
}
|
2010-08-27 12:36:57 -07:00
|
|
|
|
|
|
|
if (rdr.next() != '\'') {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err "unterminated character constant";
|
2010-08-24 08:56:42 -07:00
|
|
|
fail;
|
|
|
|
}
|
2011-04-13 11:25:01 -07:00
|
|
|
rdr.bump(); // advance curr to closing '
|
|
|
|
rdr.bump(); // advance curr past token
|
2010-08-24 08:56:42 -07:00
|
|
|
ret token.LIT_CHAR(c2);
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('"') {
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
|
|
|
while (rdr.curr() != '"') {
|
|
|
|
alt (rdr.curr()) {
|
2010-08-24 08:56:42 -07:00
|
|
|
case ('\\') {
|
2010-08-27 12:36:57 -07:00
|
|
|
alt (rdr.next()) {
|
2010-08-27 13:36:18 -07:00
|
|
|
case ('n') {
|
|
|
|
rdr.bump();
|
2011-03-16 14:58:02 -07:00
|
|
|
_str.push_byte(accum_str, '\n' as u8);
|
2010-08-27 13:36:18 -07:00
|
|
|
}
|
|
|
|
case ('r') {
|
|
|
|
rdr.bump();
|
2011-03-16 14:58:02 -07:00
|
|
|
_str.push_byte(accum_str, '\r' as u8);
|
2010-08-27 13:36:18 -07:00
|
|
|
}
|
|
|
|
case ('t') {
|
|
|
|
rdr.bump();
|
2011-03-16 14:58:02 -07:00
|
|
|
_str.push_byte(accum_str, '\t' as u8);
|
2010-08-27 13:36:18 -07:00
|
|
|
}
|
|
|
|
case ('\\') {
|
|
|
|
rdr.bump();
|
2011-03-16 14:58:02 -07:00
|
|
|
_str.push_byte(accum_str, '\\' as u8);
|
2010-08-27 13:36:18 -07:00
|
|
|
}
|
|
|
|
case ('"') {
|
|
|
|
rdr.bump();
|
2011-03-16 14:58:02 -07:00
|
|
|
_str.push_byte(accum_str, '"' as u8);
|
2010-08-27 13:36:18 -07:00
|
|
|
}
|
2011-04-13 11:25:01 -07:00
|
|
|
|
|
|
|
case ('x') {
|
|
|
|
_str.push_char(accum_str,
|
|
|
|
scan_numeric_escape(rdr));
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('u') {
|
|
|
|
_str.push_char(accum_str,
|
|
|
|
scan_numeric_escape(rdr));
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('U') {
|
|
|
|
_str.push_char(accum_str,
|
|
|
|
scan_numeric_escape(rdr));
|
|
|
|
}
|
|
|
|
|
2010-09-20 23:56:43 -07:00
|
|
|
case (?c2) {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err #fmt("unknown string escape: %d",
|
|
|
|
c2 as int);
|
2010-08-24 08:56:42 -07:00
|
|
|
fail;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
case (_) {
|
2011-03-24 16:07:43 +01:00
|
|
|
_str.push_char(accum_str, rdr.curr());
|
2010-08-24 08:56:42 -07:00
|
|
|
}
|
|
|
|
}
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-08-24 08:56:42 -07:00
|
|
|
}
|
2010-08-27 12:36:57 -07:00
|
|
|
rdr.bump();
|
2010-08-24 08:56:42 -07:00
|
|
|
ret token.LIT_STR(accum_str);
|
|
|
|
}
|
|
|
|
|
2010-08-23 19:17:04 -07:00
|
|
|
case ('-') {
|
2010-08-27 12:36:57 -07:00
|
|
|
if (rdr.next() == '>') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.RARROW;
|
2010-08-23 19:17:04 -07:00
|
|
|
} else {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.MINUS);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('&') {
|
2010-08-27 12:36:57 -07:00
|
|
|
if (rdr.next() == '&') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.ANDAND;
|
2010-08-23 19:17:04 -07:00
|
|
|
} else {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.AND);
|
2010-08-31 13:29:00 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('|') {
|
|
|
|
if (rdr.next() == '|') {
|
|
|
|
rdr.bump();
|
|
|
|
rdr.bump();
|
2010-09-20 23:56:43 -07:00
|
|
|
ret token.OROR;
|
2010-08-31 13:29:00 -07:00
|
|
|
} else {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.OR);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
case ('+') {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.PLUS);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
case ('*') {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.STAR);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
case ('/') {
|
2010-11-22 11:35:00 -08:00
|
|
|
ret binop(rdr, token.SLASH);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
case ('^') {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.CARET);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
case ('%') {
|
2010-09-20 23:56:43 -07:00
|
|
|
ret binop(rdr, token.PERCENT);
|
2010-08-23 19:17:04 -07:00
|
|
|
}
|
2011-04-13 11:25:01 -07:00
|
|
|
|
|
|
|
case (?c) {
|
2011-04-19 12:02:06 +02:00
|
|
|
log_err #fmt("unkown start of token: %d", c as int);
|
2011-04-13 11:25:01 -07:00
|
|
|
fail;
|
|
|
|
}
|
2010-08-20 14:34:48 -07:00
|
|
|
}
|
2010-08-20 11:41:34 -07:00
|
|
|
|
2011-03-05 20:05:02 +00:00
|
|
|
fail;
|
2010-07-14 09:41:36 -07:00
|
|
|
}
|
|
|
|
|
2011-03-24 16:33:20 +01:00
|
|
|
tag cmnt_ {
|
|
|
|
cmnt_line(str);
|
|
|
|
cmnt_block(vec[str]);
|
|
|
|
}
|
2011-04-08 18:44:20 +02:00
|
|
|
|
|
|
|
type cmnt = rec(cmnt_ val, uint pos, bool space_after);
|
2011-03-24 16:33:20 +01:00
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn consume_whitespace(reader rdr) -> uint {
|
2011-03-24 16:33:20 +01:00
|
|
|
auto lines = 0u;
|
|
|
|
while (is_whitespace(rdr.curr())) {
|
|
|
|
if (rdr.curr() == '\n') {lines += 1u;}
|
|
|
|
rdr.bump();
|
|
|
|
}
|
|
|
|
ret lines;
|
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn read_line_comment(reader rdr) -> cmnt {
|
2011-04-08 18:44:20 +02:00
|
|
|
auto p = rdr.get_chpos();
|
2011-03-24 16:33:20 +01:00
|
|
|
rdr.bump(); rdr.bump();
|
2011-03-29 14:07:41 +02:00
|
|
|
while (rdr.curr() == ' ') {rdr.bump();}
|
2011-03-24 16:33:20 +01:00
|
|
|
auto val = "";
|
2011-03-29 14:07:41 +02:00
|
|
|
while (rdr.curr() != '\n' && !rdr.is_eof()) {
|
2011-03-24 16:33:20 +01:00
|
|
|
_str.push_char(val, rdr.curr());
|
|
|
|
rdr.bump();
|
|
|
|
}
|
|
|
|
ret rec(val=cmnt_line(val),
|
|
|
|
pos=p,
|
|
|
|
space_after=consume_whitespace(rdr) > 1u);
|
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn read_block_comment(reader rdr) -> cmnt {
|
2011-04-08 18:44:20 +02:00
|
|
|
auto p = rdr.get_chpos();
|
2011-03-24 16:33:20 +01:00
|
|
|
rdr.bump(); rdr.bump();
|
2011-03-29 14:07:41 +02:00
|
|
|
while (rdr.curr() == ' ') {rdr.bump();}
|
2011-03-24 16:33:20 +01:00
|
|
|
let vec[str] lines = vec();
|
|
|
|
auto val = "";
|
|
|
|
auto level = 1;
|
|
|
|
while (true) {
|
|
|
|
if (rdr.curr() == '\n') {
|
|
|
|
_vec.push[str](lines, val);
|
|
|
|
val = "";
|
|
|
|
consume_whitespace(rdr);
|
|
|
|
} else {
|
|
|
|
if (rdr.curr() == '*' && rdr.next() == '/') {
|
|
|
|
level -= 1;
|
|
|
|
if (level == 0) {
|
|
|
|
rdr.bump(); rdr.bump();
|
|
|
|
_vec.push[str](lines, val);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else if (rdr.curr() == '/' && rdr.next() == '*') {
|
|
|
|
level += 1;
|
|
|
|
}
|
|
|
|
_str.push_char(val, rdr.curr());
|
|
|
|
rdr.bump();
|
|
|
|
}
|
2011-03-29 14:07:41 +02:00
|
|
|
if (rdr.is_eof()) {fail;}
|
2011-03-24 16:33:20 +01:00
|
|
|
}
|
|
|
|
ret rec(val=cmnt_block(lines),
|
|
|
|
pos=p,
|
|
|
|
space_after=consume_whitespace(rdr) > 1u);
|
|
|
|
}
|
|
|
|
|
2011-04-19 13:35:49 -07:00
|
|
|
fn gather_comments(str path) -> vec[cmnt] {
|
2011-03-24 16:33:20 +01:00
|
|
|
auto srdr = io.file_reader(path);
|
2011-04-08 18:44:20 +02:00
|
|
|
auto rdr = new_reader(srdr, path, codemap.new_filemap(path, 0u));
|
2011-03-24 16:33:20 +01:00
|
|
|
let vec[cmnt] comments = vec();
|
|
|
|
while (!rdr.is_eof()) {
|
|
|
|
while (true) {
|
|
|
|
consume_whitespace(rdr);
|
|
|
|
if (rdr.curr() == '/' && rdr.next() == '/') {
|
|
|
|
_vec.push[cmnt](comments, read_line_comment(rdr));
|
|
|
|
} else if (rdr.curr() == '/' && rdr.next() == '*') {
|
|
|
|
_vec.push[cmnt](comments, read_block_comment(rdr));
|
|
|
|
} else { break; }
|
|
|
|
}
|
|
|
|
next_token(rdr);
|
|
|
|
}
|
|
|
|
ret comments;
|
|
|
|
}
|
|
|
|
|
2010-08-18 11:35:34 -07:00
|
|
|
|
2010-08-12 10:29:23 -07:00
|
|
|
//
|
|
|
|
// Local Variables:
|
|
|
|
// mode: rust
|
|
|
|
// fill-column: 78;
|
|
|
|
// indent-tabs-mode: nil
|
|
|
|
// c-basic-offset: 4
|
|
|
|
// buffer-file-coding-system: utf-8-unix
|
2011-03-25 15:07:27 -07:00
|
|
|
// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
|
2010-08-12 10:29:23 -07:00
|
|
|
// End:
|
|
|
|
//
|