mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce code duplication in lexer by introducing small macro

Browse Source
This commit is contained in:
Evgenii P 2019-08-09 14:08:34 +07:00
parent 87608904f6
commit f4a6d92050
1 changed files with 17 additions and 20 deletions

37
crates/ra_syntax/src/parsing/lexer.rs

@ -12,6 +12,21 @@ pub struct Token {
pub len: TextUnit,
}
macro_rules! match_literal_kind {
($kind:expr) => {
match $kind {
ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
}
}
}
/// Break a string up into its component tokens
pub fn tokenize(text: &str) -> Vec<Token> {
if text.is_empty() {
@ -53,16 +68,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
}
}
ra_rustc_lexer::TokenKind::RawIdent => IDENT,
ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind {
ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
},
ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME,
ra_rustc_lexer::TokenKind::Semi => SEMI,
ra_rustc_lexer::TokenKind::Comma => COMMA,
@ -131,16 +137,7 @@ pub fn classify_literal(text: &str) -> Option<Token> {
return None;
}
let kind = match t.kind {
ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind {
ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
},
ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
_ => return None,
};
Some(Token { kind, len: TextUnit::from_usize(t.len) })