mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Track character and byte positions together in the parser

Browse Source
This commit is contained in:
Brian Anderson 2012-11-14 16:34:18 -08:00
parent f67bfe9738
commit 8069d2f266
4 changed files with 42 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/libsyntax/codemap.rs
View File

@ -184,20 +184,20 @@ pub struct FileMap {
pub impl FileMap {
static fn new_w_substr(+filename: FileName, +substr: FileSubstr,
src: @~str,
+start_pos_ch: CharPos, +start_pos_byte: BytePos)
+start_pos: FilePos)
-> FileMap {
return FileMap {
name: filename, substr: substr, src: src,
start_pos: FilePos {ch: start_pos_ch, byte: start_pos_byte},
mut lines: ~[FilePos {ch: start_pos_ch, byte: start_pos_byte}]
start_pos: start_pos,
mut lines: ~[start_pos]
};
}
static fn new(+filename: FileName, src: @~str,
+start_pos_ch: CharPos, +start_pos_byte: BytePos)
+start_pos: FilePos)
-> FileMap {
return FileMap::new_w_substr(filename, FssNone, src,
start_pos_ch, start_pos_byte);
start_pos);
}
fn next_line(@self, +chpos: CharPos, +byte_pos: BytePos) {

42
src/libsyntax/parse.rs
View File

@ -20,16 +20,15 @@
use diagnostic::{span_handler, mk_span_handler, mk_handler, emitter};
use lexer::{reader, string_reader};
use parse::token::{ident_interner, mk_ident_interner};
use codemap::{CodeMap, FileMap, CharPos, BytePos};
use codemap::{CodeMap, FileMap, CharPos, BytePos, FilePos};
type parse_sess = @{
cm: @codemap::CodeMap,
mut next_id: node_id,
span_diagnostic: span_handler,
interner: @ident_interner,
// these two must be kept up to date
mut chpos: CharPos,
mut byte_pos: BytePos
// must be kept up to date
mut pos: FilePos
};
fn new_parse_sess(demitter: Option<emitter>) -> parse_sess {
@ -38,7 +37,10 @@ fn new_parse_sess(demitter: Option<emitter>) -> parse_sess {
mut next_id: 1,
span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
interner: mk_ident_interner(),
mut chpos: CharPos(0u), mut byte_pos: BytePos(0u)};
mut pos: FilePos {
ch: CharPos(0u),
byte: BytePos(0u)
}};
}
fn new_parse_sess_special_handler(sh: span_handler, cm: @codemap::CodeMap)
@ -47,7 +49,10 @@ fn new_parse_sess_special_handler(sh: span_handler, cm: @codemap::CodeMap)
mut next_id: 1,
span_diagnostic: sh,
interner: mk_ident_interner(),
mut chpos: CharPos(0u), mut byte_pos: BytePos(0u)};
mut pos: FilePos {
ch: CharPos(0u),
byte: BytePos(0u)
}};
}
fn parse_crate_from_file(input: &Path, cfg: ast::crate_cfg,
@ -71,8 +76,7 @@ fn parse_crate_from_crate_file(input: &Path, cfg: ast::crate_cfg,
let leading_attrs = p.parse_inner_attrs_and_next();
let { inner: crate_attrs, next: first_cdir_attr } = leading_attrs;
let cdirs = p.parse_crate_directives(token::EOF, first_cdir_attr);
sess.chpos = rdr.chpos;
sess.byte_pos = sess.byte_pos + rdr.pos;
eval::update_parse_sess_position(&sess, &rdr);
let cx = @{sess: sess, cfg: /* FIXME (#2543) */ copy p.cfg};
let companionmod = input.filestem().map(|s| Path(*s));
let (m, attrs) = eval::eval_crate_directives_to_mod(
@ -92,8 +96,7 @@ fn parse_crate_from_source_file(input: &Path, cfg: ast::crate_cfg,
let (p, rdr) = new_parser_etc_from_file(sess, cfg, input,
parser::SOURCE_FILE);
let r = p.parse_crate_mod(cfg);
sess.chpos = rdr.chpos;
sess.byte_pos = sess.byte_pos + rdr.pos;
eval::update_parse_sess_position(&sess, &rdr);
return r;
}
@ -103,8 +106,7 @@ fn parse_crate_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg,
codemap::FssNone, source);
let r = p.parse_crate_mod(cfg);
p.abort_if_errors();
sess.chpos = rdr.chpos;
sess.byte_pos = sess.byte_pos + rdr.pos;
eval::update_parse_sess_position(&sess, &rdr);
return r;
}
@ -114,8 +116,7 @@ fn parse_expr_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg,
codemap::FssNone, source);
let r = p.parse_expr();
p.abort_if_errors();
sess.chpos = rdr.chpos;
sess.byte_pos = sess.byte_pos + rdr.pos;
eval::update_parse_sess_position(&sess, &rdr);
return r;
}
@ -126,8 +127,7 @@ fn parse_item_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg,
codemap::FssNone, source);
let r = p.parse_item(attrs);
p.abort_if_errors();
sess.chpos = rdr.chpos;
sess.byte_pos = sess.byte_pos + rdr.pos;
eval::update_parse_sess_position(&sess, &rdr);
return r;
}
@ -138,8 +138,7 @@ fn parse_stmt_from_source_str(name: ~str, source: @~str, cfg: ast::crate_cfg,
codemap::FssNone, source);
let r = p.parse_stmt(attrs);
p.abort_if_errors();
sess.chpos = rdr.chpos;
sess.byte_pos = sess.byte_pos + rdr.pos;
eval::update_parse_sess_position(&sess, &rdr);
return r;
}
@ -156,8 +155,7 @@ fn parse_from_source_str<T>(f: fn (p: Parser) -> T,
p.reader.fatal(~"expected end-of-string");
}
p.abort_if_errors();
sess.chpos = rdr.chpos;
sess.byte_pos = sess.byte_pos + rdr.pos;
eval::update_parse_sess_position(&sess, &rdr);
move r
}
@ -174,7 +172,7 @@ fn new_parser_etc_from_source_str(sess: parse_sess, cfg: ast::crate_cfg,
source: @~str) -> (Parser, string_reader) {
let ftype = parser::SOURCE_FILE;
let filemap = @FileMap::new_w_substr
(name, ss, source, sess.chpos, sess.byte_pos);
(name, ss, source, sess.pos);
sess.cm.files.push(filemap);
let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap,
sess.interner);
@ -199,7 +197,7 @@ fn new_parser_etc_from_file(sess: parse_sess, cfg: ast::crate_cfg,
}
let src = @result::unwrap(res);
let filemap = @FileMap::new(path.to_str(), src,
sess.chpos, sess.byte_pos);
sess.pos);
sess.cm.files.push(filemap);
let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap,
sess.interner);

10
src/libsyntax/parse/comments.rs
View File

@ -292,9 +292,13 @@ fn gather_comments_and_literals(span_diagnostic: diagnostic::span_handler,
{cmnts: ~[cmnt], lits: ~[lit]} {
let src = @str::from_bytes(srdr.read_whole_stream());
let itr = parse::token::mk_fake_ident_interner();
let rdr = lexer::new_low_level_string_reader
(span_diagnostic, @FileMap::new(path, src,
CharPos(0u), BytePos(0u)), itr);
let filemap = @FileMap::new(path, src,
FilePos {
ch: CharPos(0u),
byte: BytePos(0u)
});
let rdr = lexer::new_low_level_string_reader(
span_diagnostic, filemap, itr);
let mut comments: ~[cmnt] = ~[];
let mut literals: ~[lit] = ~[];

14
src/libsyntax/parse/eval.rs
View File

@ -2,6 +2,7 @@
use attr::parser_attr;
export eval_crate_directives_to_mod;
export update_parse_sess_position;
type ctx =
@{sess: parse::parse_sess,
@ -66,14 +67,20 @@ fn file_exists(path: &Path) -> bool {
modpath, SOURCE_FILE);
let inner_attrs = p0.parse_inner_attrs_and_next();
let m0 = p0.parse_mod_items(token::EOF, inner_attrs.next);
cx.sess.chpos = r0.chpos;
cx.sess.byte_pos = cx.sess.byte_pos + r0.pos;
update_parse_sess_position(&cx.sess, &r0);
return (m0.view_items, m0.items, inner_attrs.inner);
} else {
return (~[], ~[], ~[]);
}
}
fn update_parse_sess_position(sess: &parse_sess, r: &lexer::string_reader) {
sess.pos = FilePos {
ch: r.chpos,
byte: sess.pos.byte + r.pos
};
}
fn cdir_path_opt(default: ~str, attrs: ~[ast::attribute]) -> ~str {
match ::attr::first_attr_value_str_by_name(attrs, ~"path") {
Some(d) => d,
@ -105,8 +112,7 @@ fn eval_crate_directive(cx: ctx, cdir: @ast::crate_directive, prefix: &Path,
/* FIXME (#2543) */ copy id,
ast::item_mod(m0), vis, mod_attrs);
// Thread defids, chpos and byte_pos through the parsers
cx.sess.chpos = r0.chpos;
cx.sess.byte_pos = cx.sess.byte_pos + r0.pos;
update_parse_sess_position(&cx.sess, &r0);
items.push(i);
}
ast::cdir_dir_mod(vis, id, cdirs, attrs) => {