rust/src/grammar/verify.rs

355 lines
14 KiB
Rust
Raw Normal View History

2014-07-21 15:04:35 -05:00
// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
2015-04-21 05:02:12 -05:00
#![feature(plugin, rustc_private, str_char, collections)]
2014-07-14 03:52:18 -05:00
extern crate syntax;
extern crate rustc;
#[macro_use]
2014-07-14 03:52:18 -05:00
extern crate log;
use std::collections::HashMap;
2015-04-21 05:02:12 -05:00
use std::env;
use std::fs::File;
use std::io::{BufRead, Read};
use std::path::Path;
2014-07-14 03:52:18 -05:00
use syntax::parse;
use syntax::parse::lexer;
2015-01-03 21:42:21 -06:00
use rustc::session::{self, config};
2014-07-14 03:52:18 -05:00
use syntax::ast;
use syntax::ast::Name;
2015-04-21 05:02:12 -05:00
use syntax::codemap;
use syntax::codemap::Pos;
2014-10-27 03:22:52 -05:00
use syntax::parse::token;
2014-07-14 03:52:18 -05:00
use syntax::parse::lexer::TokenAndSpan;
fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
fn id() -> token::Token {
token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain)
2014-07-14 03:52:18 -05:00
}
let mut res = HashMap::new();
res.insert("-1".to_string(), token::Eof);
2014-07-14 03:52:18 -05:00
for line in file.split('\n') {
let eq = match line.trim().rfind('=') {
Some(val) => val,
None => continue
};
let val = &line[..eq];
let num = &line[eq + 1..];
2014-07-14 03:52:18 -05:00
let tok = match val {
2014-10-27 03:22:52 -05:00
"SHR" => token::BinOp(token::Shr),
"DOLLAR" => token::Dollar,
"LT" => token::Lt,
"STAR" => token::BinOp(token::Star),
"FLOAT_SUFFIX" => id(),
"INT_SUFFIX" => id(),
"SHL" => token::BinOp(token::Shl),
"LBRACE" => token::OpenDelim(token::Brace),
"RARROW" => token::RArrow,
"LIT_STR" => token::Literal(token::Str_(Name(0)), None),
2014-10-27 03:22:52 -05:00
"DOTDOT" => token::DotDot,
"MOD_SEP" => token::ModSep,
"DOTDOTDOT" => token::DotDotDot,
"NOT" => token::Not,
"AND" => token::BinOp(token::And),
"LPAREN" => token::OpenDelim(token::Paren),
2014-10-27 03:22:52 -05:00
"ANDAND" => token::AndAnd,
"AT" => token::At,
"LBRACKET" => token::OpenDelim(token::Bracket),
"LIT_STR_RAW" => token::Literal(token::StrRaw(Name(0), 0), None),
"RPAREN" => token::CloseDelim(token::Paren),
2014-10-27 03:22:52 -05:00
"SLASH" => token::BinOp(token::Slash),
"COMMA" => token::Comma,
"LIFETIME" => token::Lifetime(ast::Ident { name: Name(0), ctxt: 0 }),
"CARET" => token::BinOp(token::Caret),
"TILDE" => token::Tilde,
"IDENT" => id(),
2014-10-27 03:22:52 -05:00
"PLUS" => token::BinOp(token::Plus),
"LIT_CHAR" => token::Literal(token::Char(Name(0)), None),
"LIT_BYTE" => token::Literal(token::Byte(Name(0)), None),
2014-10-27 03:22:52 -05:00
"EQ" => token::Eq,
"RBRACKET" => token::CloseDelim(token::Bracket),
2014-10-27 03:22:52 -05:00
"COMMENT" => token::Comment,
"DOC_COMMENT" => token::DocComment(Name(0)),
"DOT" => token::Dot,
"EQEQ" => token::EqEq,
"NE" => token::Ne,
"GE" => token::Ge,
"PERCENT" => token::BinOp(token::Percent),
"RBRACE" => token::CloseDelim(token::Brace),
2014-10-27 03:22:52 -05:00
"BINOP" => token::BinOp(token::Plus),
"POUND" => token::Pound,
"OROR" => token::OrOr,
"LIT_INTEGER" => token::Literal(token::Integer(Name(0)), None),
2014-10-27 03:22:52 -05:00
"BINOPEQ" => token::BinOpEq(token::Plus),
"LIT_FLOAT" => token::Literal(token::Float(Name(0)), None),
2014-10-27 03:22:52 -05:00
"WHITESPACE" => token::Whitespace,
"UNDERSCORE" => token::Underscore,
"MINUS" => token::BinOp(token::Minus),
"SEMI" => token::Semi,
"COLON" => token::Colon,
"FAT_ARROW" => token::FatArrow,
"OR" => token::BinOp(token::Or),
"GT" => token::Gt,
"LE" => token::Le,
"LIT_BINARY" => token::Literal(token::Binary(Name(0)), None),
"LIT_BINARY_RAW" => token::Literal(token::BinaryRaw(Name(0), 0), None),
"QUESTION" => token::Question,
2015-04-21 05:02:12 -05:00
"SHEBANG" => token::Shebang(Name(0)),
2014-10-27 03:22:52 -05:00
_ => continue,
2014-07-14 03:52:18 -05:00
};
res.insert(num.to_string(), tok);
}
debug!("Token map: {:?}", res);
2014-07-14 03:52:18 -05:00
res
}
fn str_to_binop(s: &str) -> token::BinOpToken {
2014-07-14 03:52:18 -05:00
match s {
2014-10-27 03:22:52 -05:00
"+" => token::Plus,
"/" => token::Slash,
"-" => token::Minus,
"*" => token::Star,
"%" => token::Percent,
"^" => token::Caret,
"&" => token::And,
"|" => token::Or,
"<<" => token::Shl,
">>" => token::Shr,
_ => panic!("Bad binop str `{}`", s),
2014-07-14 19:27:28 -05:00
}
}
2014-07-14 22:45:39 -05:00
/// Assuming a string/binary literal, strip out the leading/trailing
2014-07-14 19:27:28 -05:00
/// hashes and surrounding quotes/raw/binary prefix.
fn fix(mut lit: &str) -> ast::Name {
if lit.char_at(0) == 'r' {
if lit.char_at(1) == 'b' {
lit = &lit[2..]
2014-07-14 19:27:28 -05:00
} else {
lit = &lit[1..];
2014-07-14 19:27:28 -05:00
}
} else if lit.char_at(0) == 'b' {
lit = &lit[1..];
2014-07-14 03:52:18 -05:00
}
2014-07-14 19:27:28 -05:00
let leading_hashes = count(lit);
// +1/-1 to adjust for single quotes
parse::token::intern(&lit[leading_hashes + 1..lit.len() - leading_hashes - 1])
2014-07-14 19:27:28 -05:00
}
2014-07-14 22:45:39 -05:00
/// Assuming a char/byte literal, strip the 'b' prefix and the single quotes.
fn fixchar(mut lit: &str) -> ast::Name {
if lit.char_at(0) == 'b' {
lit = &lit[1..];
2014-07-14 22:45:39 -05:00
}
parse::token::intern(&lit[1..lit.len() - 1])
2014-07-14 22:45:39 -05:00
}
fn count(lit: &str) -> usize {
2014-07-14 19:27:28 -05:00
lit.chars().take_while(|c| *c == '#').count()
2014-07-14 03:52:18 -05:00
}
2015-04-21 05:02:12 -05:00
fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_pairs_pos: &[usize],
has_bom: bool)
-> TokenAndSpan {
// old regex:
// \[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]
2015-04-21 05:02:12 -05:00
let start = s.find("[@").unwrap();
let comma = start + s[start..].find(",").unwrap();
let colon = comma + s[comma..].find(":").unwrap();
let content_start = colon + s[colon..].find("='").unwrap();
// Use rfind instead of find, because we don't want to stop at the content
let content_end = content_start + s[content_start..].rfind("',<").unwrap();
let toknum_end = content_end + s[content_end..].find(">,").unwrap();
let start = &s[comma + 1 .. colon];
let end = &s[colon + 1 .. content_start];
let content = &s[content_start + 2 .. content_end];
let toknum = &s[content_end + 3 .. toknum_end];
2014-07-14 03:52:18 -05:00
2015-04-21 05:02:12 -05:00
let not_found = format!("didn't find token {:?} in the map", toknum);
let proto_tok = tokens.get(toknum).expect(&not_found[..]);
2014-07-14 19:27:28 -05:00
let nm = parse::token::intern(content);
debug!("What we got: content (`{}`), proto: {:?}", content, proto_tok);
2014-07-14 19:27:28 -05:00
2014-07-14 03:52:18 -05:00
let real_tok = match *proto_tok {
2014-10-27 03:22:52 -05:00
token::BinOp(..) => token::BinOp(str_to_binop(content)),
token::BinOpEq(..) => token::BinOpEq(str_to_binop(&content[..content.len() - 1])),
token::Literal(token::Str_(..), n) => token::Literal(token::Str_(fix(content)), n),
token::Literal(token::StrRaw(..), n) => token::Literal(token::StrRaw(fix(content),
count(content)), n),
token::Literal(token::Char(..), n) => token::Literal(token::Char(fixchar(content)), n),
token::Literal(token::Byte(..), n) => token::Literal(token::Byte(fixchar(content)), n),
2014-10-27 03:22:52 -05:00
token::DocComment(..) => token::DocComment(nm),
token::Literal(token::Integer(..), n) => token::Literal(token::Integer(nm), n),
token::Literal(token::Float(..), n) => token::Literal(token::Float(nm), n),
token::Literal(token::Binary(..), n) => token::Literal(token::Binary(nm), n),
token::Literal(token::BinaryRaw(..), n) => token::Literal(token::BinaryRaw(fix(content),
count(content)), n),
token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 },
token::ModName),
2014-10-27 03:22:52 -05:00
token::Lifetime(..) => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
2014-07-14 03:52:18 -05:00
ref t => t.clone()
};
2015-04-21 05:02:12 -05:00
let start_offset = if real_tok == token::Eof {
2014-07-14 03:52:18 -05:00
1
} else {
0
};
2015-04-21 05:02:12 -05:00
let offset = if has_bom { 1 } else { 0 };
let mut lo = start.parse::<u32>().unwrap() - start_offset - offset;
let mut hi = end.parse::<u32>().unwrap() + 1 - offset;
// Adjust the span: For each surrogate pair already encountered, subtract one position.
lo -= surrogate_pairs_pos.binary_search(&(lo as usize)).unwrap_or_else(|x| x) as u32;
hi -= surrogate_pairs_pos.binary_search(&(hi as usize)).unwrap_or_else(|x| x) as u32;
2015-04-21 05:02:12 -05:00
let sp = codemap::Span {
lo: codemap::BytePos(lo),
hi: codemap::BytePos(hi),
expn_id: codemap::NO_EXPANSION
2014-07-14 03:52:18 -05:00
};
TokenAndSpan {
tok: real_tok,
sp: sp
}
}
fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
2014-07-14 19:27:28 -05:00
match a {
2014-10-27 03:22:52 -05:00
&token::Ident(id, _) => match b {
&token::Ident(id2, _) => id == id2,
2014-07-14 19:27:28 -05:00
_ => false
},
_ => a == b
}
}
2015-04-21 05:02:12 -05:00
fn span_cmp(antlr_sp: codemap::Span, rust_sp: codemap::Span, cm: &codemap::CodeMap) -> bool {
antlr_sp.expn_id == rust_sp.expn_id &&
2015-04-21 05:02:12 -05:00
antlr_sp.lo.to_usize() == cm.bytepos_to_file_charpos(rust_sp.lo).to_usize() &&
antlr_sp.hi.to_usize() == cm.bytepos_to_file_charpos(rust_sp.hi).to_usize()
}
2014-07-14 03:52:18 -05:00
fn main() {
fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
use syntax::parse::lexer::Reader;
r.next_token()
}
2015-04-21 05:02:12 -05:00
let mut args = env::args().skip(1);
let filename = args.next().unwrap();
if filename.find("parse-fail").is_some() {
return;
}
2014-07-15 02:18:17 -05:00
// Rust's lexer
2015-04-21 05:02:12 -05:00
let mut code = String::new();
File::open(&Path::new(&filename)).unwrap().read_to_string(&mut code).unwrap();
2014-07-21 14:59:25 -05:00
let surrogate_pairs_pos: Vec<usize> = code.chars().enumerate()
.filter(|&(_, c)| c as usize > 0xFFFF)
.map(|(n, _)| n)
.enumerate()
.map(|(x, n)| x + n)
.collect();
2015-04-21 05:02:12 -05:00
let has_bom = code.starts_with("\u{feff}");
debug!("Pairs: {:?}", surrogate_pairs_pos);
2014-07-14 03:52:18 -05:00
let options = config::basic_options();
2014-07-14 19:27:28 -05:00
let session = session::build_session(options, None,
syntax::diagnostics::registry::Registry::new(&[]));
2014-07-14 03:52:18 -05:00
let filemap = parse::string_to_filemap(&session.parse_sess,
code,
String::from_str("<n/a>"));
let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
let ref cm = lexer.span_diagnostic.cm;
2014-07-14 03:52:18 -05:00
// ANTLR
2015-04-21 05:02:12 -05:00
let mut token_file = File::open(&Path::new(&args.next().unwrap())).unwrap();
let mut token_list = String::new();
token_file.read_to_string(&mut token_list).unwrap();
let token_map = parse_token_list(&token_list[..]);
2015-04-21 05:02:12 -05:00
let stdin = std::io::stdin();
let lock = stdin.lock();
let lines = lock.lines();
2015-04-21 05:02:12 -05:00
let antlr_tokens = lines.map(|l| parse_antlr_token(l.unwrap().trim(),
&token_map,
&surrogate_pairs_pos[..],
has_bom));
2014-07-14 03:52:18 -05:00
for antlr_tok in antlr_tokens {
let rustc_tok = next(&mut lexer);
if rustc_tok.tok == token::Eof && antlr_tok.tok == token::Eof {
2014-07-14 03:52:18 -05:00
continue
}
assert!(span_cmp(antlr_tok.sp, rustc_tok.sp, cm), "{:?} and {:?} have different spans",
rustc_tok,
2014-07-21 15:04:35 -05:00
antlr_tok);
2014-07-14 03:52:18 -05:00
macro_rules! matches {
2014-07-14 03:52:18 -05:00
( $($x:pat),+ ) => (
match rustc_tok.tok {
$($x => match antlr_tok.tok {
2014-07-14 19:27:28 -05:00
$x => {
if !tok_cmp(&rustc_tok.tok, &antlr_tok.tok) {
// FIXME #15677: needs more robust escaping in
// antlr
warn!("Different names for {:?} and {:?}", rustc_tok, antlr_tok);
2014-07-14 19:27:28 -05:00
}
}
_ => panic!("{:?} is not {:?}", antlr_tok, rustc_tok)
2014-07-14 03:52:18 -05:00
},)*
2015-04-21 05:02:12 -05:00
ref c => assert!(c == &antlr_tok.tok, "{:?} is not {:?}", antlr_tok, rustc_tok)
2014-07-14 03:52:18 -05:00
}
)
}
2014-07-14 03:52:18 -05:00
2014-10-27 03:22:52 -05:00
matches!(
token::Literal(token::Byte(..), _),
token::Literal(token::Char(..), _),
token::Literal(token::Integer(..), _),
token::Literal(token::Float(..), _),
token::Literal(token::Str_(..), _),
token::Literal(token::StrRaw(..), _),
token::Literal(token::Binary(..), _),
token::Literal(token::BinaryRaw(..), _),
token::Ident(..),
token::Lifetime(..),
token::Interpolated(..),
token::DocComment(..),
token::Shebang(..)
2014-07-14 03:52:18 -05:00
);
}
}