mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Indicate the number of float tokens in the first token

Browse Source
This commit is contained in:
Jonas Schievink 2022-05-05 16:06:44 +02:00
parent 34dc8e9383
commit 2fe38d3b63
16 changed files with 194 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
crates/parser/src/grammar.rs
View File

@ -39,6 +39,7 @@
mod types;
use crate::{
grammar::expressions::FLOAT_LITERAL_FIRST,
parser::{CompletedMarker, Marker, Parser},
SyntaxKind::{self, *},
TokenSet, T,
@ -318,9 +319,15 @@ fn name_ref(p: &mut Parser) {
}
fn name_ref_or_index(p: &mut Parser) {
assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART));
assert!(
p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST)
);
let m = p.start();
p.bump_any();
if p.at_ts(FLOAT_LITERAL_FIRST) {
p.bump_remap(FLOAT_NUMBER_PART);
} else {
p.bump_any();
}
m.complete(p, NAME_REF);
}

7
crates/parser/src/grammar/expressions.rs
View File

@ -3,7 +3,7 @@
use super::*;
pub(crate) use self::atom::{block_expr, match_arm_list};
pub(super) use self::atom::{float_literal, literal, LITERAL_FIRST};
pub(super) use self::atom::{float_literal, literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST};
#[derive(PartialEq, Eq)]
pub(super) enum Semicolon {
@ -452,6 +452,9 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo();
// y.bar::<T>(1, 2,);
//
// 0e0.sin();
// 0e0f32.sin();
// }
fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
@ -477,7 +480,7 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]));
let m = lhs.precede(p);
p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) {
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) {
name_ref_or_index(p);
} else {
p.error("expected field name or number");

36
crates/parser/src/grammar/expressions/atom.rs
View File

@ -17,19 +17,24 @@
T![true],
T![false],
INT_NUMBER,
FLOAT_NUMBER_PART,
FLOAT_NUMBER_START_0,
FLOAT_NUMBER_START_1,
FLOAT_NUMBER_START_2,
BYTE,
CHAR,
STRING,
BYTE_STRING,
]);
pub(crate) const FLOAT_LITERAL_FIRST: TokenSet =
TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]);
pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
if !p.at_ts(LITERAL_FIRST) {
return None;
}
let m = p.start();
if p.at(FLOAT_NUMBER_PART) {
if p.at_ts(FLOAT_LITERAL_FIRST) {
float_literal(p);
} else {
// Everything else is just one token.
@ -38,15 +43,30 @@ pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
Some(m.complete(p, LITERAL))
}
// test float_literal
// fn f() {
// 0.0;
// 1.;
// 0e0;
// 0e0f32;
// 1.23f64;
// }
pub(crate) fn float_literal(p: &mut Parser) {
// Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT`
// Floats can be up to 3 tokens. The first token indicates how many there are.
// We remap the first token to `FLOAT_NUMBER_PART` so that no subsequent code has to deal with
// this awful, awful hack.
let f = p.start();
p.bump(FLOAT_NUMBER_PART);
if p.at(DOT) {
if p.at(FLOAT_NUMBER_START_0) {
p.bump_remap(FLOAT_NUMBER_PART);
} else if p.at(FLOAT_NUMBER_START_1) {
p.bump_remap(FLOAT_NUMBER_PART);
p.bump(DOT);
if p.at(FLOAT_NUMBER_PART) {
p.bump(FLOAT_NUMBER_PART);
}
} else if p.at(FLOAT_NUMBER_START_2) {
p.bump_remap(FLOAT_NUMBER_PART);
p.bump(DOT);
p.bump(FLOAT_NUMBER_PART);
} else {
unreachable!();
}
f.complete(p, FLOAT_LITERAL);
}

2
crates/parser/src/grammar/items.rs
View File

@ -457,7 +457,7 @@ pub(crate) fn token_tree(p: &mut Parser) {
return;
}
T![')'] | T![']'] => p.err_and_bump("unmatched brace"),
FLOAT_NUMBER_PART => {
FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => {
float_literal(p);
}
_ => p.bump_any(),

13
crates/parser/src/lexed_str.rs
View File

@ -240,11 +240,16 @@ fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind)
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
// To ensure that later stages can always reconstruct the token correctly, the first
// token in the sequence indicates the number of following tokens that are part of
// the float literal.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
if !before.is_empty() {
self.push(FLOAT_NUMBER_PART, before.len(), None);
}
assert!(!before.is_empty());
let tok =
if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 };
self.push(tok, before.len(), None);
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
@ -252,7 +257,7 @@ fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind)
return;
}
FLOAT_NUMBER_PART
FLOAT_NUMBER_START_0
}
rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated {

8
crates/parser/src/syntax_kind/generated.rs
View File

File diff suppressed because one or more lines are too long

28
crates/parser/test_data/lexer/err/empty_exponent.rast
View File

@ -1,14 +1,14 @@
FLOAT_NUMBER_PART "0e" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "0E" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER_PART "42e+" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42e-" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42E+" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42E-" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
INT_NUMBER "42"
DOT "."
@ -30,35 +30,35 @@ DOT "."
IDENT "E"
MINUS "-"
WHITESPACE "\n\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_PART "42"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"

16
crates/parser/test_data/lexer/ok/numbers.rast
View File

@ -4,7 +4,7 @@ INT_NUMBER "00"
WHITESPACE " "
INT_NUMBER "0_"
WHITESPACE " "
FLOAT_NUMBER_PART "0"
FLOAT_NUMBER_START_1 "0"
DOT "."
WHITESPACE " "
INT_NUMBER "0z"
@ -21,13 +21,13 @@ INT_NUMBER "001279"
WHITESPACE " "
INT_NUMBER "0_1279"
WHITESPACE " "
FLOAT_NUMBER_PART "0"
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "1279"
WHITESPACE " "
FLOAT_NUMBER_PART "0e1279"
FLOAT_NUMBER_START_0 "0e1279"
WHITESPACE " "
FLOAT_NUMBER_PART "0E1279"
FLOAT_NUMBER_START_0 "0E1279"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -40,7 +40,7 @@ IDENT "foo"
L_PAREN "("
R_PAREN ")"
WHITESPACE "\n"
FLOAT_NUMBER_PART "0e+1"
FLOAT_NUMBER_START_0 "0e+1"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -48,17 +48,17 @@ IDENT "e"
PLUS "+"
INT_NUMBER "1"
WHITESPACE "\n"
FLOAT_NUMBER_PART "0"
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0E-2"
WHITESPACE "\n"
FLOAT_NUMBER_PART "0___0"
FLOAT_NUMBER_START_2 "0___0"
DOT "."
FLOAT_NUMBER_PART "10000____0000e+111__"
WHITESPACE "\n"
INT_NUMBER "1i64"
WHITESPACE " "
FLOAT_NUMBER_PART "92"
FLOAT_NUMBER_START_2 "92"
DOT "."
FLOAT_NUMBER_PART "0f32"
WHITESPACE " "

26
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast
View File

@ -58,6 +58,32 @@ SOURCE_FILE
COMMA ","
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_PART "0e0"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_PART "0e0f32"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

3
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs
View File

@ -1,4 +1,7 @@
fn foo() {
x.foo();
y.bar::<T>(1, 2,);
0e0.sin();
0e0f32.sin();
}

51
crates/parser/test_data/parser/inline/ok/0201_float_literal.rast Normal file
View File

@ -0,0 +1,51 @@
SOURCE_FILE
FN
FN_KW "fn"
WHITESPACE " "
NAME
IDENT "f"
PARAM_LIST
L_PAREN "("
R_PAREN ")"
WHITESPACE " "
BLOCK_EXPR
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_PART "0"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_PART "1"
DOT "."
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_PART "0e0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_PART "0e0f32"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_PART "1"
DOT "."
FLOAT_NUMBER_PART "23f64"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

7
crates/parser/test_data/parser/inline/ok/0201_float_literal.rs Normal file
View File

@ -0,0 +1,7 @@
fn f() {
0.0;
1.;
0e0;
0e0f32;
1.23f64;
}

4
crates/syntax/rust.ungram
View File

@ -373,7 +373,9 @@ Literal =
)
FloatLiteral =
'float_number_part'
'float_number_start_0'?
'float_number_start_1'?
'float_number_start_2'?
'.'?
'float_number_part'?

11
crates/syntax/src/ast/generated/nodes.rs
View File

@ -1090,10 +1090,19 @@ pub struct FloatLiteral {
pub(crate) syntax: SyntaxNode,
}
impl FloatLiteral {
pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_0])
}
pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_1])
}
pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_2])
}
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_part])
}
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]

12
crates/syntax/src/tests/ast_src.rs
View File

@ -71,7 +71,17 @@ pub(crate) struct KindsSrc<'a> {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
literals: &["INT_NUMBER", "FLOAT_NUMBER_PART", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
literals: &[
"INT_NUMBER",
"FLOAT_NUMBER_START_0",
"FLOAT_NUMBER_START_1",
"FLOAT_NUMBER_START_2",
"FLOAT_NUMBER_PART",
"CHAR",
"BYTE",
"STRING",
"BYTE_STRING",
],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[
"SOURCE_FILE",

3
crates/syntax/src/tests/sourcegen_ast.rs
View File

@ -463,6 +463,9 @@ macro_rules! T {
[ident] => { $crate::SyntaxKind::IDENT };
[shebang] => { $crate::SyntaxKind::SHEBANG };
[float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
[float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 };
[float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 };
[float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 };
}
pub use T;
};