From 2fe38d3b63cf4b6f2278a656b7ffed40a524f0ff Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Thu, 5 May 2022 16:06:44 +0200 Subject: [PATCH] Indicate the number of float tokens in the first token --- crates/parser/src/grammar.rs | 11 +++- crates/parser/src/grammar/expressions.rs | 7 ++- crates/parser/src/grammar/expressions/atom.rs | 36 ++++++++++--- crates/parser/src/grammar/items.rs | 2 +- crates/parser/src/lexed_str.rs | 13 +++-- crates/parser/src/syntax_kind/generated.rs | 8 ++- .../test_data/lexer/err/empty_exponent.rast | 28 +++++----- crates/parser/test_data/lexer/ok/numbers.rast | 16 +++--- .../inline/ok/0107_method_call_expr.rast | 26 ++++++++++ .../parser/inline/ok/0107_method_call_expr.rs | 3 ++ .../parser/inline/ok/0201_float_literal.rast | 51 +++++++++++++++++++ .../parser/inline/ok/0201_float_literal.rs | 7 +++ crates/syntax/rust.ungram | 4 +- crates/syntax/src/ast/generated/nodes.rs | 11 +++- crates/syntax/src/tests/ast_src.rs | 12 ++++- crates/syntax/src/tests/sourcegen_ast.rs | 3 ++ 16 files changed, 194 insertions(+), 44 deletions(-) create mode 100644 crates/parser/test_data/parser/inline/ok/0201_float_literal.rast create mode 100644 crates/parser/test_data/parser/inline/ok/0201_float_literal.rs diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 45d9b2e4e0d..92f33025387 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -39,6 +39,7 @@ mod types; use crate::{ + grammar::expressions::FLOAT_LITERAL_FIRST, parser::{CompletedMarker, Marker, Parser}, SyntaxKind::{self, *}, TokenSet, T, @@ -318,9 +319,15 @@ fn name_ref(p: &mut Parser) { } fn name_ref_or_index(p: &mut Parser) { - assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART)); + assert!( + p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) + ); let m = p.start(); - p.bump_any(); + if p.at_ts(FLOAT_LITERAL_FIRST) { + p.bump_remap(FLOAT_NUMBER_PART); + } else { + p.bump_any(); + } m.complete(p, NAME_REF); } diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index cb384cce815..15dba362a94 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -3,7 +3,7 @@ use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{float_literal, literal, LITERAL_FIRST}; +pub(super) use self::atom::{float_literal, literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; #[derive(PartialEq, Eq)] pub(super) enum Semicolon { @@ -452,6 +452,9 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo(); // y.bar::(1, 2,); +// +// 0e0.sin(); +// 0e0f32.sin(); // } fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); @@ -477,7 +480,7 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.])); let m = lhs.precede(p); p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) { + if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { name_ref_or_index(p); } else { p.error("expected field name or number"); diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index d7a06917178..c191b5593db 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -17,19 +17,24 @@ T![true], T![false], INT_NUMBER, - FLOAT_NUMBER_PART, + FLOAT_NUMBER_START_0, + FLOAT_NUMBER_START_1, + FLOAT_NUMBER_START_2, BYTE, CHAR, STRING, BYTE_STRING, ]); +pub(crate) const FLOAT_LITERAL_FIRST: TokenSet = + TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]); + pub(crate) fn literal(p: &mut Parser) -> Option { if !p.at_ts(LITERAL_FIRST) { return None; } let m = p.start(); - if p.at(FLOAT_NUMBER_PART) { + if p.at_ts(FLOAT_LITERAL_FIRST) { float_literal(p); } else { // Everything else is just one token. @@ -38,15 +43,30 @@ pub(crate) fn literal(p: &mut Parser) -> Option { Some(m.complete(p, LITERAL)) } +// test float_literal +// fn f() { +// 0.0; +// 1.; +// 0e0; +// 0e0f32; +// 1.23f64; +// } pub(crate) fn float_literal(p: &mut Parser) { - // Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT` + // Floats can be up to 3 tokens. The first token indicates how many there are. + // We remap the first token to `FLOAT_NUMBER_PART` so that no subsequent code has to deal with + // this awful, awful hack. let f = p.start(); - p.bump(FLOAT_NUMBER_PART); - if p.at(DOT) { + if p.at(FLOAT_NUMBER_START_0) { + p.bump_remap(FLOAT_NUMBER_PART); + } else if p.at(FLOAT_NUMBER_START_1) { + p.bump_remap(FLOAT_NUMBER_PART); p.bump(DOT); - if p.at(FLOAT_NUMBER_PART) { - p.bump(FLOAT_NUMBER_PART); - } + } else if p.at(FLOAT_NUMBER_START_2) { + p.bump_remap(FLOAT_NUMBER_PART); + p.bump(DOT); + p.bump(FLOAT_NUMBER_PART); + } else { + unreachable!(); } f.complete(p, FLOAT_LITERAL); } diff --git a/crates/parser/src/grammar/items.rs b/crates/parser/src/grammar/items.rs index 3d2f788e9a4..65166034023 100644 --- a/crates/parser/src/grammar/items.rs +++ b/crates/parser/src/grammar/items.rs @@ -457,7 +457,7 @@ pub(crate) fn token_tree(p: &mut Parser) { return; } T![')'] | T![']'] => p.err_and_bump("unmatched brace"), - FLOAT_NUMBER_PART => { + FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => { float_literal(p); } _ => p.bump_any(), diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index fae5d8884be..9887960871f 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -240,11 +240,16 @@ fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0` // is lexed as a float, we split floats that contain a `.` into 3 tokens. + // To ensure that later stages can always reconstruct the token correctly, the first + // token in the sequence indicates the number of following tokens that are part of + // the float literal. if let Some((before, after)) = token_text.split_once('.') { let err = if err.is_empty() { None } else { Some(err) }; - if !before.is_empty() { - self.push(FLOAT_NUMBER_PART, before.len(), None); - } + + assert!(!before.is_empty()); + let tok = + if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 }; + self.push(tok, before.len(), None); self.push(DOT, 1, None); if !after.is_empty() { self.push(FLOAT_NUMBER_PART, after.len(), err); @@ -252,7 +257,7 @@ fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) return; } - FLOAT_NUMBER_PART + FLOAT_NUMBER_START_0 } rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 31fe1b2aba2..5ddcd6ad6a4 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -110,6 +110,9 @@ pub enum SyntaxKind { RAW_KW, MACRO_RULES_KW, INT_NUMBER, + FLOAT_NUMBER_START_0, + FLOAT_NUMBER_START_1, + FLOAT_NUMBER_START_2, FLOAT_NUMBER_PART, CHAR, BYTE, @@ -287,7 +290,8 @@ pub fn is_punct(self) -> bool { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 + | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } @@ -387,5 +391,5 @@ pub fn from_char(c: char) -> Option { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; [float_number_start_0] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_0 } ; [float_number_start_1] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_1 } ; [float_number_start_2] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_2 } ; } pub use T; diff --git a/crates/parser/test_data/lexer/err/empty_exponent.rast b/crates/parser/test_data/lexer/err/empty_exponent.rast index d0a268b8c14..73de4cac243 100644 --- a/crates/parser/test_data/lexer/err/empty_exponent.rast +++ b/crates/parser/test_data/lexer/err/empty_exponent.rast @@ -1,14 +1,14 @@ -FLOAT_NUMBER_PART "0e" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "0E" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_PART "42e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" INT_NUMBER "42" DOT "." @@ -30,35 +30,35 @@ DOT "." IDENT "E" MINUS "-" WHITESPACE "\n\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" diff --git a/crates/parser/test_data/lexer/ok/numbers.rast b/crates/parser/test_data/lexer/ok/numbers.rast index 689422933c4..428bdf8a1f9 100644 --- a/crates/parser/test_data/lexer/ok/numbers.rast +++ b/crates/parser/test_data/lexer/ok/numbers.rast @@ -4,7 +4,7 @@ INT_NUMBER "00" WHITESPACE " " INT_NUMBER "0_" WHITESPACE " " -FLOAT_NUMBER_PART "0" +FLOAT_NUMBER_START_1 "0" DOT "." WHITESPACE " " INT_NUMBER "0z" @@ -21,13 +21,13 @@ INT_NUMBER "001279" WHITESPACE " " INT_NUMBER "0_1279" WHITESPACE " " -FLOAT_NUMBER_PART "0" +FLOAT_NUMBER_START_2 "0" DOT "." FLOAT_NUMBER_PART "1279" WHITESPACE " " -FLOAT_NUMBER_PART "0e1279" +FLOAT_NUMBER_START_0 "0e1279" WHITESPACE " " -FLOAT_NUMBER_PART "0E1279" +FLOAT_NUMBER_START_0 "0E1279" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -40,7 +40,7 @@ IDENT "foo" L_PAREN "(" R_PAREN ")" WHITESPACE "\n" -FLOAT_NUMBER_PART "0e+1" +FLOAT_NUMBER_START_0 "0e+1" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -48,17 +48,17 @@ IDENT "e" PLUS "+" INT_NUMBER "1" WHITESPACE "\n" -FLOAT_NUMBER_PART "0" +FLOAT_NUMBER_START_2 "0" DOT "." FLOAT_NUMBER_PART "0E-2" WHITESPACE "\n" -FLOAT_NUMBER_PART "0___0" +FLOAT_NUMBER_START_2 "0___0" DOT "." FLOAT_NUMBER_PART "10000____0000e+111__" WHITESPACE "\n" INT_NUMBER "1i64" WHITESPACE " " -FLOAT_NUMBER_PART "92" +FLOAT_NUMBER_START_2 "92" DOT "." FLOAT_NUMBER_PART "0f32" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast index dcbcfe1231e..0a14e20da85 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast @@ -58,6 +58,32 @@ SOURCE_FILE COMMA "," R_PAREN ")" SEMICOLON ";" + WHITESPACE "\n\n " + EXPR_STMT + METHOD_CALL_EXPR + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0" + DOT "." + NAME_REF + IDENT "sin" + ARG_LIST + L_PAREN "(" + R_PAREN ")" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + METHOD_CALL_EXPR + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0f32" + DOT "." + NAME_REF + IDENT "sin" + ARG_LIST + L_PAREN "(" + R_PAREN ")" + SEMICOLON ";" WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs index 1a3aa35ae8e..3e5d464e238 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs @@ -1,4 +1,7 @@ fn foo() { x.foo(); y.bar::(1, 2,); + + 0e0.sin(); + 0e0f32.sin(); } diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast new file mode 100644 index 00000000000..a5cd2dffa4a --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast @@ -0,0 +1,51 @@ +SOURCE_FILE + FN + FN_KW "fn" + WHITESPACE " " + NAME + IDENT "f" + PARAM_LIST + L_PAREN "(" + R_PAREN ")" + WHITESPACE " " + BLOCK_EXPR + STMT_LIST + L_CURLY "{" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0" + DOT "." + FLOAT_NUMBER_PART "0" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "1" + DOT "." + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0f32" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "1" + DOT "." + FLOAT_NUMBER_PART "23f64" + SEMICOLON ";" + WHITESPACE "\n" + R_CURLY "}" + WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs new file mode 100644 index 00000000000..0d51ec1252f --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs @@ -0,0 +1,7 @@ +fn f() { + 0.0; + 1.; + 0e0; + 0e0f32; + 1.23f64; +} diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 8e4f07d3b5b..85270dee624 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -373,7 +373,9 @@ Literal = ) FloatLiteral = - 'float_number_part' + 'float_number_start_0'? + 'float_number_start_1'? + 'float_number_start_2'? '.'? 'float_number_part'? diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index 7c82372aa2d..9d5af8e63ca 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1090,10 +1090,19 @@ pub struct FloatLiteral { pub(crate) syntax: SyntaxNode, } impl FloatLiteral { + pub fn float_number_start_0_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_0]) + } + pub fn float_number_start_1_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_1]) + } + pub fn float_number_start_2_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_2]) + } + pub fn dot_token(&self) -> Option { support::token(&self.syntax, T![.]) } pub fn float_number_part_token(&self) -> Option { support::token(&self.syntax, T![float_number_part]) } - pub fn dot_token(&self) -> Option { support::token(&self.syntax, T![.]) } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index 0a0632da74e..f5a78e4119d 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -71,7 +71,17 @@ pub(crate) struct KindsSrc<'a> { "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], - literals: &["INT_NUMBER", "FLOAT_NUMBER_PART", "CHAR", "BYTE", "STRING", "BYTE_STRING"], + literals: &[ + "INT_NUMBER", + "FLOAT_NUMBER_START_0", + "FLOAT_NUMBER_START_1", + "FLOAT_NUMBER_START_2", + "FLOAT_NUMBER_PART", + "CHAR", + "BYTE", + "STRING", + "BYTE_STRING", + ], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index eb6a5f63eaa..33bf2c0cbad 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -463,6 +463,9 @@ macro_rules! T { [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART }; + [float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 }; + [float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 }; + [float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 }; } pub use T; };