From a7c362a6093b389e3a5f77e2ca91b05c57032498 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 10 Feb 2012 03:03:21 -0700 Subject: [PATCH 1/6] Change file_substr to allow for external strings. --- src/comp/syntax/codemap.rs | 17 ++++++++++------- src/comp/syntax/ext/qquote.rs | 7 +++---- src/comp/syntax/parse/parser.rs | 6 +++--- src/rustdoc/attr_parser.rs | 2 +- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/comp/syntax/codemap.rs b/src/comp/syntax/codemap.rs index 5d8d7ff5633..963f3d02573 100644 --- a/src/comp/syntax/codemap.rs +++ b/src/comp/syntax/codemap.rs @@ -8,8 +8,11 @@ type file_pos = {ch: uint, byte: uint}; * compiler. */ -type file_substr_ = {lo: uint, hi: uint, col: uint, line: uint}; -type file_substr = option; +enum file_substr { + fss_none, + fss_internal(span), + fss_external({filename: str, line: uint, col: uint}) +} type filemap = @{name: filename, substr: file_substr, src: @str, @@ -33,16 +36,16 @@ fn new_filemap_w_substr(filename: filename, substr: file_substr, fn new_filemap(filename: filename, src: @str, start_pos_ch: uint, start_pos_byte: uint) -> filemap { - ret new_filemap_w_substr(filename, none, src, + ret new_filemap_w_substr(filename, fss_none, src, start_pos_ch, start_pos_byte); } -fn get_substr_info(cm: codemap, lo: uint, hi: uint) - -> (filename, file_substr_) +fn get_substr_info(cm: codemap, sp: span) + -> (filename, file_substr) { - let pos = lookup_char_pos(cm, lo); + let pos = lookup_char_pos(cm, sp.lo); let name = #fmt("<%s:%u:%u>", pos.file.name, pos.line, pos.col); - ret (name, {lo: lo, hi: hi, col: pos.col, line: pos.line}); + ret (name, fss_internal(sp)); } fn next_line(file: filemap, chpos: uint, byte_pos: uint) { diff --git a/src/comp/syntax/ext/qquote.rs b/src/comp/syntax/ext/qquote.rs index 39bd34785ba..d983ff94f15 100644 --- a/src/comp/syntax/ext/qquote.rs +++ b/src/comp/syntax/ext/qquote.rs @@ -149,10 +149,9 @@ fn expand_ast(ecx: ext_ctxt, _sp: span, { let cm = ecx.session().parse_sess.cm; let str = @codemap::span_to_snippet(body.span, cm); - let (fname, ss) = codemap::get_substr_info - (cm, body.span.lo, body.span.hi); + let (fname, ss) = codemap::get_substr_info(cm, body.span); let node = parse_from_source_str - (f, fname, some(ss), str, + (f, fname, ss, str, ecx.session().opts.cfg, ecx.session().parse_sess); ret expand_qquote(ecx, node.span(), *str, node); } @@ -229,7 +228,7 @@ fn expand_qquote "parse_from_source_str"], [node.mk_parse_fn(cx,sp), mk_str(cx,sp, ""), - mk_path(cx,sp, ["option","none"]), + mk_path(cx,sp, ["syntax", "codemap", "fss_none"]), mk_unary(cx,sp, ast::box(ast::imm), mk_str(cx,sp, str2)), mk_access_(cx,sp, diff --git a/src/comp/syntax/parse/parser.rs b/src/comp/syntax/parse/parser.rs index 8e2897407a3..2352ac2f9db 100644 --- a/src/comp/syntax/parse/parser.rs +++ b/src/comp/syntax/parse/parser.rs @@ -2,7 +2,7 @@ import std::{io, fs}; import either::{left, right}; import std::map::{hashmap, new_str_hash}; import token::can_begin_expr; -import codemap::span; +import codemap::{span,fss_none}; import util::interner; import ast::{node_id, spanned}; import ast_util::{mk_sp, ident_to_path}; @@ -2607,7 +2607,7 @@ fn parse_crate_from_source_file(input: str, cfg: ast::crate_cfg, fn parse_expr_from_source_str(name: str, source: @str, cfg: ast::crate_cfg, sess: parse_sess) -> @ast::expr { - let p = new_parser_from_source_str(sess, cfg, name, none, source); + let p = new_parser_from_source_str(sess, cfg, name, fss_none, source); let r = parse_expr(p); sess.chpos = p.reader.chpos; sess.byte_pos = sess.byte_pos + p.reader.pos; @@ -2629,7 +2629,7 @@ fn parse_from_source_str(f: fn (p: parser) -> T, fn parse_crate_from_source_str(name: str, source: @str, cfg: ast::crate_cfg, sess: parse_sess) -> @ast::crate { - let p = new_parser_from_source_str(sess, cfg, name, none, source); + let p = new_parser_from_source_str(sess, cfg, name, fss_none, source); let r = parse_crate_mod(p, cfg); sess.chpos = p.reader.chpos; sess.byte_pos = sess.byte_pos + p.reader.pos; diff --git a/src/rustdoc/attr_parser.rs b/src/rustdoc/attr_parser.rs index c44af3dd523..066fad91f37 100644 --- a/src/rustdoc/attr_parser.rs +++ b/src/rustdoc/attr_parser.rs @@ -93,7 +93,7 @@ mod test { mutable byte_pos: 0u }; let parser = parser::new_parser_from_source_str( - parse_sess, [], "-", none, @source); + parse_sess, [], "-", codemap::fss_none, @source); parser::parse_outer_attributes(parser) } From 4026053e99b6554bd7d996839e04dfeb9d40aa4d Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 10 Feb 2012 11:28:43 -0700 Subject: [PATCH 2/6] Do a better job of reporting source location for files (i.e. filemap) that are really a substr of another file. --- src/comp/driver/diagnostic.rs | 1 + src/comp/syntax/codemap.rs | 46 +++++++++++++++++++++++++++-- src/test/compile-fail/qquote.rs | 51 +++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 src/test/compile-fail/qquote.rs diff --git a/src/comp/driver/diagnostic.rs b/src/comp/driver/diagnostic.rs index ebda3956bfd..6bc5f91a16a 100644 --- a/src/comp/driver/diagnostic.rs +++ b/src/comp/driver/diagnostic.rs @@ -175,6 +175,7 @@ fn emit(cmsp: option<(codemap::codemap, span)>, msg: str, lvl: level) { alt cmsp { some((cm, sp)) { + let sp = codemap::adjust_span(cm,sp); let ss = codemap::span_to_str(sp, cm); let lines = codemap::span_to_lines(sp, cm); print_diagnostic(ss, lvl, msg); diff --git a/src/comp/syntax/codemap.rs b/src/comp/syntax/codemap.rs index 963f3d02573..3f848cc893c 100644 --- a/src/comp/syntax/codemap.rs +++ b/src/comp/syntax/codemap.rs @@ -46,6 +46,8 @@ fn get_substr_info(cm: codemap, sp: span) let pos = lookup_char_pos(cm, sp.lo); let name = #fmt("<%s:%u:%u>", pos.file.name, pos.line, pos.col); ret (name, fss_internal(sp)); + //ret (name, fss_external({filename: pos.file.name, + // line: pos.line, col: pos.col})); } fn next_line(file: filemap, chpos: uint, byte_pos: uint) { @@ -92,6 +94,40 @@ fn lookup_byte_pos(map: codemap, pos: uint) -> loc { ret lookup_pos(map, pos, lookup); } +fn lookup_char_pos_adj(map: codemap, pos: uint) + -> {filename: str, line: uint, col: uint, file: option} +{ + let loc = lookup_char_pos(map, pos); + alt (loc.file.substr) { + fss_none { + {filename: loc.file.name, line: loc.line, col: loc.col, + file: some(loc.file)} + } + fss_internal(sp) { + lookup_char_pos_adj(map, sp.lo + (pos - loc.file.start_pos.ch)) + } + fss_external(eloc) { + {filename: eloc.filename, + line: eloc.line + loc.line - 1u, + col: if loc.line == 1u {eloc.col + loc.col} else {loc.col}, + file: none} + } + } +} + +fn adjust_span(map: codemap, sp: span) -> span { + fn lookup(pos: file_pos) -> uint { ret pos.ch; } + let line = lookup_line(map, sp.lo, lookup); + alt (line.fm.substr) { + fss_none {sp} + fss_internal(s) { + adjust_span(map, {lo: s.lo + (sp.lo - line.fm.start_pos.ch), + hi: s.lo + (sp.hi - line.fm.start_pos.ch), + expn_info: sp.expn_info})} + fss_external(_) {sp} + } +} + enum expn_info_ { expanded_from({call_site: span, callie: {name: str, span: option}}) @@ -99,19 +135,25 @@ enum expn_info_ { type expn_info = option<@expn_info_>; type span = {lo: uint, hi: uint, expn_info: expn_info}; -fn span_to_str(sp: span, cm: codemap) -> str { +fn span_to_str_no_adj(sp: span, cm: codemap) -> str { let lo = lookup_char_pos(cm, sp.lo); let hi = lookup_char_pos(cm, sp.hi); ret #fmt("%s:%u:%u: %u:%u", lo.file.name, lo.line, lo.col, hi.line, hi.col) } +fn span_to_str(sp: span, cm: codemap) -> str { + let lo = lookup_char_pos_adj(cm, sp.lo); + let hi = lookup_char_pos_adj(cm, sp.hi); + ret #fmt("%s:%u:%u: %u:%u", lo.filename, + lo.line, lo.col, hi.line, hi.col) +} + type file_lines = {file: filemap, lines: [uint]}; fn span_to_lines(sp: span, cm: codemap::codemap) -> @file_lines { let lo = lookup_char_pos(cm, sp.lo); let hi = lookup_char_pos(cm, sp.hi); - // FIXME: Check for filemap? let lines = []; uint::range(lo.line - 1u, hi.line as uint) {|i| lines += [i]; }; ret @{file: lo.file, lines: lines}; diff --git a/src/test/compile-fail/qquote.rs b/src/test/compile-fail/qquote.rs new file mode 100644 index 00000000000..243e90c35df --- /dev/null +++ b/src/test/compile-fail/qquote.rs @@ -0,0 +1,51 @@ +// xfail-pretty + +use std; +use rustc; + +import rustc::*; +import std::io::*; + +import rustc::driver::diagnostic; +import rustc::syntax::ast; +import rustc::syntax::codemap; +import rustc::syntax::parse::parser; +import rustc::syntax::print::*; + +fn new_parse_sess() -> parser::parse_sess { + fail; +} + +iface fake_ext_ctxt { + fn session() -> fake_session; +} + +type fake_options = {cfg: ast::crate_cfg}; + +type fake_session = {opts: @fake_options, + parse_sess: parser::parse_sess}; + +impl of fake_ext_ctxt for fake_session { + fn session() -> fake_session {self} +} + +fn mk_ctxt() -> fake_ext_ctxt { + let opts : fake_options = {cfg: []}; + {opts: @opts, parse_sess: new_parse_sess()} as fake_ext_ctxt +} + + +fn main() { + let ext_cx = mk_ctxt(); + + let abc = #ast{23}; + check_pp(abc, pprust::print_expr, "23"); + + let expr3 = #ast{2 - $(abcd) + 7}; //! ERROR unresolved name: abcd + check_pp(expr3, pprust::print_expr, "2 - 23 + 7"); +} + +fn check_pp(expr: T, f: fn(pprust::ps, T), expect: str) { + fail; +} + From d808df893a085205cb6e113ce98ac09accbc595a Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 10 Feb 2012 12:23:40 -0700 Subject: [PATCH 3/6] Use file_substr rather than when re-parsing quasi-quotes for better error messages. --- src/comp/syntax/codemap.rs | 8 ++----- src/comp/syntax/ext/expand.rs | 2 +- src/comp/syntax/ext/qquote.rs | 39 ++++++++++++++++++++--------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/comp/syntax/codemap.rs b/src/comp/syntax/codemap.rs index 3f848cc893c..d96ed502258 100644 --- a/src/comp/syntax/codemap.rs +++ b/src/comp/syntax/codemap.rs @@ -40,14 +40,10 @@ fn new_filemap(filename: filename, src: @str, start_pos_ch, start_pos_byte); } -fn get_substr_info(cm: codemap, sp: span) - -> (filename, file_substr) +fn mk_substr_filename(cm: codemap, sp: span) -> str { let pos = lookup_char_pos(cm, sp.lo); - let name = #fmt("<%s:%u:%u>", pos.file.name, pos.line, pos.col); - ret (name, fss_internal(sp)); - //ret (name, fss_external({filename: pos.file.name, - // line: pos.line, col: pos.col})); + ret #fmt("<%s:%u:%u>", pos.file.name, pos.line, pos.col); } fn next_line(file: filemap, chpos: uint, byte_pos: uint) { diff --git a/src/comp/syntax/ext/expand.rs b/src/comp/syntax/ext/expand.rs index 8fc0fdcdf23..104f49aebac 100644 --- a/src/comp/syntax/ext/expand.rs +++ b/src/comp/syntax/ext/expand.rs @@ -5,7 +5,7 @@ import std::map::hashmap; import syntax::ast::{crate, expr_, expr_mac, mac_invoc}; import syntax::fold::*; import syntax::ext::base::*; -import syntax::ext::qquote::{expand_qquote,qq_helper}; +import syntax::ext::qquote::{qq_helper}; import syntax::parse::parser::parse_expr_from_source_str; import codemap::{span, expanded_from}; diff --git a/src/comp/syntax/ext/qquote.rs b/src/comp/syntax/ext/qquote.rs index d983ff94f15..8899ff64f57 100644 --- a/src/comp/syntax/ext/qquote.rs +++ b/src/comp/syntax/ext/qquote.rs @@ -143,18 +143,6 @@ fn expand_ast(ecx: ext_ctxt, _sp: span, } } let body = get_mac_body(ecx,_sp,body); - fn finish(ecx: ext_ctxt, body: ast::mac_body_, - f: fn (p: parser) -> T) - -> @ast::expr - { - let cm = ecx.session().parse_sess.cm; - let str = @codemap::span_to_snippet(body.span, cm); - let (fname, ss) = codemap::get_substr_info(cm, body.span); - let node = parse_from_source_str - (f, fname, ss, str, - ecx.session().opts.cfg, ecx.session().parse_sess); - ret expand_qquote(ecx, node.span(), *str, node); - } ret alt what { "expr" {finish(ecx, body, parser::parse_expr)} @@ -181,10 +169,19 @@ fn parse_item(p: parser) -> @ast::item { } } -fn expand_qquote - (ecx: ext_ctxt, sp: span, str: str, node: N) +fn finish + (ecx: ext_ctxt, body: ast::mac_body_, f: fn (p: parser) -> T) -> @ast::expr { + let cm = ecx.session().parse_sess.cm; + let str = @codemap::span_to_snippet(body.span, cm); + let fname = codemap::mk_substr_filename(cm, body.span); + let node = parse_from_source_str + (f, fname, codemap::fss_internal(body.span), str, + ecx.session().opts.cfg, ecx.session().parse_sess); + let loc = codemap::lookup_char_pos(cm, body.span.lo); + + let sp = node.span(); let qcx = gather_anti_quotes(sp.lo, node); let cx = qcx; let prev = 0u; @@ -197,7 +194,7 @@ fn expand_qquote let state = active; let i = 0u, j = 0u; let g_len = vec::len(cx.gather); - str::chars_iter(str) {|ch| + str::chars_iter(*str) {|ch| if (j < g_len && i == cx.gather[j].lo) { assert ch == '$'; let repl = #fmt("$%u ", j); @@ -227,8 +224,12 @@ fn expand_qquote ["syntax", "parse", "parser", "parse_from_source_str"], [node.mk_parse_fn(cx,sp), - mk_str(cx,sp, ""), - mk_path(cx,sp, ["syntax", "codemap", "fss_none"]), + mk_str(cx,sp, fname), + mk_call(cx,sp, + ["syntax","ext","qquote", "mk_file_substr"], + [mk_str(cx,sp, loc.file.name), + mk_uint(cx,sp, loc.line), + mk_uint(cx,sp, loc.col)]), mk_unary(cx,sp, ast::box(ast::imm), mk_str(cx,sp, str2)), mk_access_(cx,sp, @@ -305,6 +306,10 @@ fn print_expr(expr: @ast::expr) { stdout.write_str("\n"); } +fn mk_file_substr(fname: str, line: uint, col: uint) -> codemap::file_substr { + codemap::fss_external({filename: fname, line: line, col: col}) +} + // Local Variables: // mode: rust // fill-column: 78; From f9a63efb8214a36b13545b158bf27c1b6540b650 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Wed, 8 Feb 2012 17:45:02 -0700 Subject: [PATCH 4/6] Correctly handle the character position at the EOF. Fixes issue #1785. --- src/comp/syntax/parse/lexer.rs | 8 +++++++- src/test/run-pass/qquote.rs | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/comp/syntax/parse/lexer.rs b/src/comp/syntax/parse/lexer.rs index 42cc5400b0c..2902ad513a1 100644 --- a/src/comp/syntax/parse/lexer.rs +++ b/src/comp/syntax/parse/lexer.rs @@ -43,7 +43,13 @@ impl reader for reader { let next = str::char_range_at(*self.src, self.pos); self.pos = next.next; self.curr = next.ch; - } else { self.curr = -1 as char; } + } else { + if (self.curr != -1 as char) { + self.col += 1u; + self.chpos += 1u; + self.curr = -1 as char; + } + } } fn fatal(m: str) -> ! { self.span_diagnostic.span_fatal( diff --git a/src/test/run-pass/qquote.rs b/src/test/run-pass/qquote.rs index 50751a66081..ef10f03129b 100644 --- a/src/test/run-pass/qquote.rs +++ b/src/test/run-pass/qquote.rs @@ -76,6 +76,11 @@ fn main() { let pat = #ast(pat){some(_)}; check_pp(pat, pprust::print_pat, "some(_)"); + + // issue #1785 + let x = #ast{1}; + let test1 = #ast{1+$(x)}; + check_pp(test1, pprust::print_expr, "1 + 1"); } fn check_pp(expr: T, f: fn(pprust::ps, T), expect: str) { From 48eda22835e58ec1f14f6333e444ccc8a32244b7 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 10 Feb 2012 17:27:35 -0700 Subject: [PATCH 5/6] Bug fix to accept $ in 0th pos, (ie #ast{$(x) + ...}). Note: part from Niko Matsakis commit: rewrite assert to accept a $ in 0th pos. --- src/comp/syntax/ext/qquote.rs | 9 +++++---- src/test/run-pass/qquote.rs | 7 +++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/comp/syntax/ext/qquote.rs b/src/comp/syntax/ext/qquote.rs index 8899ff64f57..1d5022d1903 100644 --- a/src/comp/syntax/ext/qquote.rs +++ b/src/comp/syntax/ext/qquote.rs @@ -184,11 +184,12 @@ fn finish let sp = node.span(); let qcx = gather_anti_quotes(sp.lo, node); let cx = qcx; - let prev = 0u; - for {lo: lo, _} in cx.gather { - assert lo > prev; - prev = lo; + + // assert that the vector is sorted by position: + uint::range(1u, vec::len(cx.gather)) {|i| + assert cx.gather[i-1u].lo < cx.gather[i].lo; } + let str2 = ""; enum state {active, skip(uint), blank}; let state = active; diff --git a/src/test/run-pass/qquote.rs b/src/test/run-pass/qquote.rs index ef10f03129b..46330d119b6 100644 --- a/src/test/run-pass/qquote.rs +++ b/src/test/run-pass/qquote.rs @@ -81,6 +81,13 @@ fn main() { let x = #ast{1}; let test1 = #ast{1+$(x)}; check_pp(test1, pprust::print_expr, "1 + 1"); + + let test2 = #ast{$(x)+1}; + check_pp(test2, pprust::print_expr, "1 + 1"); + + let y = #ast{2}; + let test3 = #ast{$(x) + $(y)}; + check_pp(test3, pprust::print_expr, "1 + 2"); } fn check_pp(expr: T, f: fn(pprust::ps, T), expect: str) { From 54875908607aeba57477e117a08e5f8f3611bf76 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 10 Feb 2012 13:22:15 -0700 Subject: [PATCH 6/6] When parsing a source string, fail when the entire string is not parsed. (For now only fail when parse_from_source_str is used to avoid possible compatibility problems; parse_expr_from_source_str still does not check.) --- src/comp/syntax/parse/parser.rs | 3 ++ .../compile-fail/{qquote.rs => qquote-1.rs} | 0 src/test/compile-fail/qquote-2.rs | 48 +++++++++++++++++++ 3 files changed, 51 insertions(+) rename src/test/compile-fail/{qquote.rs => qquote-1.rs} (100%) create mode 100644 src/test/compile-fail/qquote-2.rs diff --git a/src/comp/syntax/parse/parser.rs b/src/comp/syntax/parse/parser.rs index 2352ac2f9db..1cfd96c97cb 100644 --- a/src/comp/syntax/parse/parser.rs +++ b/src/comp/syntax/parse/parser.rs @@ -2622,6 +2622,9 @@ fn parse_from_source_str(f: fn (p: parser) -> T, { let p = new_parser_from_source_str(sess, cfg, name, ss, source); let r = f(p); + if !p.reader.is_eof() { + p.reader.fatal("expected end-of-string"); + } sess.chpos = p.reader.chpos; sess.byte_pos = sess.byte_pos + p.reader.pos; ret r; diff --git a/src/test/compile-fail/qquote.rs b/src/test/compile-fail/qquote-1.rs similarity index 100% rename from src/test/compile-fail/qquote.rs rename to src/test/compile-fail/qquote-1.rs diff --git a/src/test/compile-fail/qquote-2.rs b/src/test/compile-fail/qquote-2.rs new file mode 100644 index 00000000000..a29543e29d8 --- /dev/null +++ b/src/test/compile-fail/qquote-2.rs @@ -0,0 +1,48 @@ +// xfail-pretty + +use std; +use rustc; + +import rustc::*; +import std::io::*; + +import rustc::driver::diagnostic; +import rustc::syntax::ast; +import rustc::syntax::codemap; +import rustc::syntax::parse::parser; +import rustc::syntax::print::*; + +fn new_parse_sess() -> parser::parse_sess { + fail; +} + +iface fake_ext_ctxt { + fn session() -> fake_session; +} + +type fake_options = {cfg: ast::crate_cfg}; + +type fake_session = {opts: @fake_options, + parse_sess: parser::parse_sess}; + +impl of fake_ext_ctxt for fake_session { + fn session() -> fake_session {self} +} + +fn mk_ctxt() -> fake_ext_ctxt { + let opts : fake_options = {cfg: []}; + {opts: @opts, parse_sess: new_parse_sess()} as fake_ext_ctxt +} + + +fn main() { + let ext_cx = mk_ctxt(); + + let stmt = #ast(stmt){let x int = 20;}; //! ERROR expected end-of-string + check_pp(*stmt, pprust::print_stmt, ""); +} + +fn check_pp(expr: T, f: fn(pprust::ps, T), expect: str) { + fail; +} +