rust/tests/lexer.rs

32 lines
718 B
Rust
Raw Normal View History

2017-12-28 15:27:56 -06:00
extern crate file;
2017-12-28 15:56:36 -06:00
extern crate libsyntax2;
2018-01-07 05:56:08 -06:00
extern crate testutils;
2017-12-28 15:27:56 -06:00
2017-12-28 15:56:36 -06:00
use std::fmt::Write;
2017-12-31 08:54:33 -06:00
use libsyntax2::{Token, tokenize};
2018-01-07 06:34:11 -06:00
use testutils::dir_tests;
2017-12-28 15:27:56 -06:00
#[test]
fn lexer_tests() {
2018-01-07 06:34:11 -06:00
dir_tests(
&["lexer"],
|text| {
let tokens = tokenize(text);
dump_tokens(&tokens, text)
}
)
2017-12-28 15:27:56 -06:00
}
2017-12-30 06:29:09 -06:00
fn dump_tokens(tokens: &[Token], text: &str) -> String {
2017-12-28 15:56:36 -06:00
let mut acc = String::new();
2017-12-30 06:29:09 -06:00
let mut offset = 0;
2017-12-28 15:56:36 -06:00
for token in tokens {
2017-12-30 06:29:09 -06:00
let len: u32 = token.len.into();
let len = len as usize;
let token_text = &text[offset..offset + len];
offset += len;
write!(acc, "{:?} {} {:?}\n", token.kind, token.len, token_text).unwrap()
2017-12-28 15:56:36 -06:00
}
acc
2017-12-28 15:27:56 -06:00
}