diff --git a/src/libsyntax/parse.rs b/src/libsyntax/parse.rs index c08ad9927ad..29cbb9db839 100644 --- a/src/libsyntax/parse.rs +++ b/src/libsyntax/parse.rs @@ -18,7 +18,8 @@ import common::parser_common; import ast::node_id; import util::interner; -import lexer::{string_reader_as_reader, reader, string_reader}; +import lexer::{string_reader_as_reader, tt_reader_as_reader, + reader, string_reader, tt_reader}; type parse_sess = @{ cm: codemap::codemap, @@ -178,3 +179,10 @@ fn new_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg, +path: str, let (p, _) = new_parser_etc_from_file(sess, cfg, path, ftype); ret p; } + +fn new_parser_from_tt(sess: parse_sess, cfg: ast::crate_cfg, + itr: @interner::interner<@str>, tt: ast::token_tree) + -> parser { + let trdr = lexer::new_tt_reader(sess.span_diagnostic, itr, tt); + ret parser(sess, cfg, trdr as reader, parser::SOURCE_FILE) +} diff --git a/src/libsyntax/parse/lexer.rs b/src/libsyntax/parse/lexer.rs index bf151ca4a68..51ab3085eaf 100644 --- a/src/libsyntax/parse/lexer.rs +++ b/src/libsyntax/parse/lexer.rs @@ -4,8 +4,9 @@ import ast::{tt_delim,tt_flat}; export reader, string_reader, new_string_reader, is_whitespace; +export tt_reader, new_tt_reader; export nextch, is_eof, bump, get_str_from; -export string_reader_as_reader; +export string_reader_as_reader, tt_reader_as_reader; iface reader { fn is_eof() -> bool; @@ -27,14 +28,27 @@ enum tt_frame_up { /* to break a circularity */ }; type tt_reader = @{ - mut cur: tt_frame, - interner: @interner::interner<@str>, span_diagnostic: diagnostic::span_handler, + interner: @interner::interner<@str>, + mut cur: tt_frame, /* cached: */ mut cur_tok: token::token, mut cur_chpos: uint }; +fn new_tt_reader(span_diagnostic: diagnostic::span_handler, + itr: @interner::interner<@str>, src: ast::token_tree) + -> tt_reader { + let r = @{span_diagnostic: span_diagnostic, interner: itr, + mut cur: @{readme: [src], mut idx: 0u, + up: tt_frame_up(option::none)}, + mut cur_tok: token::EOF, /* dummy value, never read */ + mut cur_chpos: 0u /* dummy value, never read */ + }; + (r as reader).next_token(); /* get cur_tok and cur_chpos set up */ + ret r; +} + type string_reader = @{ span_diagnostic: diagnostic::span_handler, src: @str, @@ -46,6 +60,21 @@ enum tt_frame_up { /* to break a circularity */ interner: @interner::interner<@str> }; +fn new_string_reader(span_diagnostic: diagnostic::span_handler, + filemap: codemap::filemap, + itr: @interner::interner<@str>) -> string_reader { + let r = @{span_diagnostic: span_diagnostic, src: filemap.src, + mut col: 0u, mut pos: 0u, mut curr: -1 as char, + mut chpos: filemap.start_pos.ch, + filemap: filemap, interner: itr}; + if r.pos < (*filemap.src).len() { + let next = str::char_range_at(*r.src, r.pos); + r.pos = next.next; + r.curr = next.ch; + } + ret r; +} + impl string_reader_as_reader of reader for string_reader { fn is_eof() -> bool { is_eof(self) } fn next_token() -> {tok: token::token, chpos: uint} { @@ -70,7 +99,6 @@ impl tt_reader_as_reader of reader for tt_reader { fn is_eof() -> bool { self.cur_tok == token::EOF } fn next_token() -> {tok: token::token, chpos: uint} { let ret_val = { tok: self.cur_tok, chpos: self.cur_chpos }; - self.cur.idx += 1u; if self.cur.idx >= vec::len(self.cur.readme) { /* done with this set; pop */ alt self.cur.up { @@ -97,6 +125,7 @@ fn next_token() -> {tok: token::token, chpos: uint} { } tt_flat(chpos, tok) { self.cur_chpos = chpos; self.cur_tok = tok; + self.cur.idx += 1u; ret ret_val; } } @@ -144,21 +173,6 @@ fn nextch(rdr: string_reader) -> char { } else { ret -1 as char; } } -fn new_string_reader(span_diagnostic: diagnostic::span_handler, - filemap: codemap::filemap, - itr: @interner::interner<@str>) -> string_reader { - let r = @{span_diagnostic: span_diagnostic, src: filemap.src, - mut col: 0u, mut pos: 0u, mut curr: -1 as char, - mut chpos: filemap.start_pos.ch, - filemap: filemap, interner: itr}; - if r.pos < (*filemap.src).len() { - let next = str::char_range_at(*r.src, r.pos); - r.pos = next.next; - r.curr = next.ch; - } - ret r; -} - fn dec_digit_val(c: char) -> int { ret (c as int) - ('0' as int); } fn hex_digit_val(c: char) -> int {