mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Auto merge of #12149 - jonas-schievink:literally-just-a-literal, r=jonas-schievink"

Browse Source 
This reverts commit cc9ae2b89e, reversing
changes made to 7dfd1cb572.
This commit is contained in:
Jonas Schievink 2022-05-13 15:08:14 +02:00
parent 2287ae22c6
commit 9bd11459ba
36 changed files with 121 additions and 502 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/hir-def/src/body/lower.rs
View File

@ -972,7 +972,7 @@ fn from(ast_lit_kind: ast::LiteralKind) -> Self {
}
}
LiteralKind::FloatNumber(lit) => {
let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s));
let ty = lit.suffix().and_then(BuiltinFloat::from_suffix);
Literal::Float(Default::default(), ty)
}
LiteralKind::ByteString(bs) => {

36
crates/hir-expand/src/builtin_fn_macro.rs
View File

@ -4,7 +4,10 @@
use cfg::CfgExpr;
use either::Either;
use mbe::{parse_exprs_with_sep, parse_to_token_tree};
use syntax::{ast, SmolStr};
use syntax::{
ast::{self, AstToken},
SmolStr,
};
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
@ -355,7 +358,14 @@ fn unreachable_expand(
}
fn unquote_str(lit: &tt::Literal) -> Option<String> {
let token = ast::make::literal(&lit.to_string()).as_string()?;
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::String::cast(lit)?;
token.value().map(|it| it.into_owned())
}
fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> {
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::ByteString::cast(lit)?;
token.value().map(|it| it.into_owned())
}
@ -432,16 +442,12 @@ fn concat_bytes_expand(
for (i, t) in tt.token_trees.iter().enumerate() {
match t {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string());
match lit.kind() {
ast::LiteralKind::ByteString(s) => {
s.value()
.unwrap_or_default()
.into_iter()
.for_each(|x| bytes.push(x.to_string()));
}
ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
let token = ast::make::tokens::literal(&lit.to_string());
match token.kind() {
syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()),
syntax::SyntaxKind::BYTE_STRING => {
let components = unquote_byte_string(lit).unwrap_or_else(Vec::new);
components.into_iter().for_each(|x| bytes.push(x.to_string()));
}
_ => {
err.get_or_insert(mbe::ExpandError::UnexpectedToken.into());
@ -475,10 +481,10 @@ fn concat_bytes_expand_subtree(
for (ti, tt) in tree.token_trees.iter().enumerate() {
match tt {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string());
let lit = ast::make::tokens::literal(&lit.to_string());
match lit.kind() {
ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => {
bytes.push(lit.text().to_string())
}
_ => {
return Err(mbe::ExpandError::UnexpectedToken.into());

11
crates/hir-ty/src/tests/simple.rs
View File

@ -2733,14 +2733,3 @@ fn f() {
"#,
);
}
#[test]
fn nested_tuple_index() {
check_no_mismatches(
r#"
fn main() {
let fld: i32 = ((0,),).0.0;
}
"#,
);
}

20
crates/ide-completion/src/completions/dot.rs
View File

@ -793,24 +793,4 @@ fn main() {
",
)
}
#[test]
fn tuple_index_completion() {
check(
r#"
struct I;
impl I {
fn i_method(&self) {}
}
struct S((), I);
fn f(s: S) {
s.1.$0
}
"#,
expect![[r#"
me i_method() fn(&self)
"#]],
);
}
}

4
crates/ide-completion/src/completions/postfix.rs
View File

@ -5,7 +5,7 @@
use hir::{Documentation, HasAttrs};
use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap};
use syntax::{
ast::{self, AstNode, LiteralKind},
ast::{self, AstNode, AstToken},
SyntaxKind::{EXPR_STMT, STMT_LIST},
TextRange, TextSize,
};
@ -194,7 +194,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) {
}
if let ast::Expr::Literal(literal) = dot_receiver.clone() {
if let LiteralKind::String(literal_text) = literal.kind() {
if let Some(literal_text) = ast::String::cast(literal.token()) {
add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text);
}
}

10
crates/ide/src/syntax_highlighting/highlight.rs
View File

@ -30,15 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio
INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
SymbolKind::Field.into()
}
INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1
| FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(),
DOT if matches!(
token.prev_token().map(|n| n.kind()),
Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2)
) =>
{
HlTag::NumericLiteral.into()
}
INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(),
BYTE => HlTag::ByteLiteral.into(),
CHAR => HlTag::CharLiteral.into(),
IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => {

14
crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
View File

@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="numeric_literal">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span>

25
crates/mbe/src/syntax_bridge.rs
View File

@ -260,31 +260,6 @@ macro_rules! make_leaf {
IDENT => make_leaf!(Ident),
UNDERSCORE => make_leaf!(Ident),
k if k.is_keyword() => make_leaf!(Ident),
FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => {
// Reassemble a split-up float token.
let mut range = range;
let mut text = token.to_text(conv).to_string();
if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 {
let (dot, dot_range) = conv.bump().unwrap();
text += &*dot.to_text(conv);
range = TextRange::new(range.start(), dot_range.end());
if kind == FLOAT_NUMBER_START_2 {
let (tail, tail_range) = conv.bump().unwrap();
text += &*tail.to_text(conv);
range = TextRange::new(range.start(), tail_range.end());
}
}
result.push(
tt::Leaf::from(tt::Literal {
id: conv.id_alloc().alloc(range, synth_id),
text: text.into(),
})
.into(),
);
continue;
}
k if k.is_literal() => make_leaf!(Literal),
LIFETIME_IDENT => {
let char_unit = TextSize::of('\'');

16
crates/mbe/src/to_parser_input.rs
View File

@ -35,13 +35,15 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
let is_negated = lit.text.starts_with('-');
let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
let lexed_str = parser::LexedStr::new(inner_text);
if lexed_str.is_empty() {
panic!("failed to convert literal: {:?}", lit);
}
for i in 0..lexed_str.len() {
res.push(lexed_str.kind(i));
}
let kind = parser::LexedStr::single_token(inner_text)
.map(|(kind, _error)| kind)
.filter(|kind| {
kind.is_literal()
&& (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
})
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
res.push(kind);
}
tt::Leaf::Ident(ident) => match ident.text.as_ref() {
"_" => res.push(T![_]),

11
crates/parser/src/grammar.rs
View File

@ -39,7 +39,6 @@
mod types;
use crate::{
grammar::expressions::FLOAT_LITERAL_FIRST,
parser::{CompletedMarker, Marker, Parser},
SyntaxKind::{self, *},
TokenSet, T,
@ -319,15 +318,9 @@ fn name_ref(p: &mut Parser) {
}
fn name_ref_or_index(p: &mut Parser) {
assert!(
p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST)
);
assert!(p.at(IDENT) || p.at(INT_NUMBER));
let m = p.start();
if p.at_ts(FLOAT_LITERAL_FIRST) {
p.bump_remap(FLOAT_NUMBER_PART);
} else {
p.bump_any();
}
p.bump_any();
m.complete(p, NAME_REF);
}

12
crates/parser/src/grammar/expressions.rs
View File

@ -3,7 +3,7 @@
use super::*;
pub(crate) use self::atom::{block_expr, match_arm_list};
pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST};
pub(super) use self::atom::{literal, LITERAL_FIRST};
#[derive(PartialEq, Eq)]
pub(super) enum Semicolon {
@ -452,9 +452,6 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo();
// y.bar::<T>(1, 2,);
//
// 0e0.sin();
// 0e0f32.sin();
// }
fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
@ -472,16 +469,17 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo;
// x.0.bar;
// x.0. bar;
// x.0.1;
// x.0();
// }
fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]));
let m = lhs.precede(p);
p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) {
if p.at(IDENT) || p.at(INT_NUMBER) {
name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
} else {
p.error("expected field name or number");
}

40
crates/parser/src/grammar/expressions/atom.rs
View File

@ -17,58 +17,22 @@
T![true],
T![false],
INT_NUMBER,
FLOAT_NUMBER_START_0,
FLOAT_NUMBER_START_1,
FLOAT_NUMBER_START_2,
FLOAT_NUMBER,
BYTE,
CHAR,
STRING,
BYTE_STRING,
]);
pub(crate) const FLOAT_LITERAL_FIRST: TokenSet =
TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]);
pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
if !p.at_ts(LITERAL_FIRST) {
return None;
}
let m = p.start();
if p.at_ts(FLOAT_LITERAL_FIRST) {
float_literal(p);
} else {
// Everything else is just one token.
p.bump_any();
}
p.bump_any();
Some(m.complete(p, LITERAL))
}
// test float_literal
// fn f() {
// 0.0;
// 1.;
// 0e0;
// 0e0f32;
// 1.23f64;
// }
pub(crate) fn float_literal(p: &mut Parser) {
// Floats can be up to 3 tokens. The first token indicates how many there are.
let f = p.start();
if p.at(FLOAT_NUMBER_START_0) {
p.bump(FLOAT_NUMBER_START_0);
} else if p.at(FLOAT_NUMBER_START_1) {
p.bump(FLOAT_NUMBER_START_1);
p.bump(DOT);
} else if p.at(FLOAT_NUMBER_START_2) {
p.bump(FLOAT_NUMBER_START_2);
p.bump(DOT);
p.bump(FLOAT_NUMBER_PART);
} else {
unreachable!();
}
f.complete(p, FLOAT_LITERAL);
}
// E.g. for after the break in `if break {}`, this should not match
pub(super) const ATOM_EXPR_FIRST: TokenSet =
LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[

2
crates/parser/src/grammar/patterns.rs
View File

@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
}
fn is_literal_pat_start(p: &Parser) -> bool {
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART)
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
|| p.at_ts(expressions::LITERAL_FIRST)
}

28
crates/parser/src/lexed_str.rs
View File

@ -177,7 +177,7 @@ fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, token_text: &str) {
rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => {
self.extend_literal(token_text, kind);
self.extend_literal(token_text.len(), kind);
return;
}
@ -223,7 +223,7 @@ fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, token_text: &str) {
self.push(syntax_kind, token_text.len(), err);
}
fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) {
fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
let mut err = "";
let syntax_kind = match *kind {
@ -237,27 +237,7 @@ fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind)
if empty_exponent {
err = "Missing digits after the exponent symbol";
}
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
// To ensure that later stages can always reconstruct the token correctly, the first
// token in the sequence indicates the number of following tokens that are part of
// the float literal.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
assert!(!before.is_empty());
let tok =
if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 };
self.push(tok, before.len(), None);
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
}
return;
}
FLOAT_NUMBER_START_0
FLOAT_NUMBER
}
rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated {
@ -315,6 +295,6 @@ fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind)
};
let err = if err.is_empty() { None } else { Some(err) };
self.push(syntax_kind, token_text.len(), err);
self.push(syntax_kind, len, err);
}
}

11
crates/parser/src/syntax_kind/generated.rs
View File

File diff suppressed because one or more lines are too long

44
crates/parser/test_data/lexer/err/empty_exponent.rast
View File

@ -1,14 +1,14 @@
FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol
FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol
FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
INT_NUMBER "42"
DOT "."
@ -30,35 +30,19 @@ DOT "."
IDENT "E"
MINUS "-"
WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"

25
crates/parser/test_data/lexer/ok/numbers.rast
View File

@ -4,8 +4,7 @@ INT_NUMBER "00"
WHITESPACE " "
INT_NUMBER "0_"
WHITESPACE " "
FLOAT_NUMBER_START_1 "0"
DOT "."
FLOAT_NUMBER "0."
WHITESPACE " "
INT_NUMBER "0z"
WHITESPACE "\n"
@ -21,13 +20,11 @@ INT_NUMBER "001279"
WHITESPACE " "
INT_NUMBER "0_1279"
WHITESPACE " "
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "1279"
FLOAT_NUMBER "0.1279"
WHITESPACE " "
FLOAT_NUMBER_START_0 "0e1279"
FLOAT_NUMBER "0e1279"
WHITESPACE " "
FLOAT_NUMBER_START_0 "0E1279"
FLOAT_NUMBER "0E1279"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -40,7 +37,7 @@ IDENT "foo"
L_PAREN "("
R_PAREN ")"
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0e+1"
FLOAT_NUMBER "0e+1"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -48,19 +45,13 @@ IDENT "e"
PLUS "+"
INT_NUMBER "1"
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0E-2"
FLOAT_NUMBER "0.0E-2"
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0___0"
DOT "."
FLOAT_NUMBER_PART "10000____0000e+111__"
FLOAT_NUMBER "0___0.10000____0000e+111__"
WHITESPACE "\n"
INT_NUMBER "1i64"
WHITESPACE " "
FLOAT_NUMBER_START_2 "92"
DOT "."
FLOAT_NUMBER_PART "0f32"
FLOAT_NUMBER "92.0f32"
WHITESPACE " "
INT_NUMBER "11__s"
WHITESPACE "\n"

4
crates/parser/test_data/parser/err/0023_mismatched_paren.rast
View File

@ -32,9 +32,7 @@ SOURCE_FILE
INT_NUMBER "1"
COMMA ","
WHITESPACE " "
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
FLOAT_NUMBER "2.0"
WHITESPACE "\n "
R_CURLY "}"
WHITESPACE " "

33
crates/parser/test_data/parser/inline/ok/0011_field_expr.rast
View File

@ -40,39 +40,6 @@ SOURCE_FILE
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
WHITESPACE " "
NAME_REF
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "1"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
CALL_EXPR
FIELD_EXPR

2
crates/parser/test_data/parser/inline/ok/0011_field_expr.rs
View File

@ -1,7 +1,5 @@
fn foo() {
x.foo;
x.0.bar;
x.0. bar;
x.0.1;
x.0();
}

5
crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast
View File

@ -57,10 +57,7 @@ SOURCE_FILE
EQ "="
WHITESPACE " "
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
FLOAT_NUMBER "2.0"
SEMICOLON ";"
WHITESPACE "\n "
LET_STMT

26
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast
View File

@ -58,32 +58,6 @@ SOURCE_FILE
COMMA ","
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

3
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs
View File

@ -1,7 +1,4 @@
fn foo() {
x.foo();
y.bar::<T>(1, 2,);
0e0.sin();
0e0f32.sin();
}

51
crates/parser/test_data/parser/inline/ok/0201_float_literal.rast
View File

@ -1,51 +0,0 @@
SOURCE_FILE
FN
FN_KW "fn"
WHITESPACE " "
NAME
IDENT "f"
PARAM_LIST
L_PAREN "("
R_PAREN ")"
WHITESPACE " "
BLOCK_EXPR
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_1 "1"
DOT "."
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "23f64"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

7
crates/parser/test_data/parser/inline/ok/0201_float_literal.rs
View File

@ -1,7 +0,0 @@
fn f() {
0.0;
1.;
0e0;
0e0f32;
1.23f64;
}

10
crates/parser/test_data/parser/ok/0056_neq_in_type.rast
View File

@ -19,10 +19,7 @@ SOURCE_FILE
CAST_EXPR
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
FLOAT_NUMBER "1.0f32"
DOT "."
NAME_REF
IDENT "floor"
@ -43,10 +40,7 @@ SOURCE_FILE
CAST_EXPR
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
FLOAT_NUMBER "1.0f32"
DOT "."
NAME_REF
IDENT "floor"

9
crates/syntax/rust.ungram
View File

@ -365,20 +365,13 @@ MacroExpr =
Literal =
Attr* value:(
'int_number' | FloatLiteral
'int_number' | 'float_number'
| 'string' | 'raw_string'
| 'byte_string' | 'raw_byte_string'
| 'true' | 'false'
| 'char' | 'byte'
)
FloatLiteral =
'float_number_start_0'?
'float_number_start_1'?
'float_number_start_2'?
'.'?
'float_number_part'?
PathExpr =
Attr* Path

42
crates/syntax/src/ast/expr_ext.rs
View File

@ -8,7 +8,7 @@
operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp},
support, AstChildren, AstNode,
},
AstToken, SyntaxElement,
AstToken,
SyntaxKind::*,
SyntaxNode, SyntaxToken, T,
};
@ -282,32 +282,30 @@ pub enum LiteralKind {
String(ast::String),
ByteString(ast::ByteString),
IntNumber(ast::IntNumber),
FloatNumber(ast::FloatLiteral),
FloatNumber(ast::FloatNumber),
Char(ast::Char),
Byte(ast::Byte),
Bool(bool),
}
impl ast::Literal {
pub fn value(&self) -> SyntaxElement {
pub fn token(&self) -> SyntaxToken {
self.syntax()
.children_with_tokens()
.find(|e| e.kind() != ATTR && !e.kind().is_trivia())
.and_then(|e| e.into_token())
.unwrap()
}
pub fn kind(&self) -> LiteralKind {
let token = match self.value() {
rowan::NodeOrToken::Node(node) => {
return LiteralKind::FloatNumber(
ast::FloatLiteral::cast(node).expect("unreachable"),
);
}
rowan::NodeOrToken::Token(token) => token,
};
let token = self.token();
if let Some(t) = ast::IntNumber::cast(token.clone()) {
return LiteralKind::IntNumber(t);
}
if let Some(t) = ast::FloatNumber::cast(token.clone()) {
return LiteralKind::FloatNumber(t);
}
if let Some(t) = ast::String::cast(token.clone()) {
return LiteralKind::String(t);
}
@ -327,26 +325,6 @@ pub fn kind(&self) -> LiteralKind {
_ => unreachable!(),
}
}
pub fn as_string(&self) -> Option<ast::String> {
match self.kind() {
LiteralKind::String(it) => Some(it),
_ => None,
}
}
pub fn as_byte_string(&self) -> Option<ast::ByteString> {
match self.kind() {
LiteralKind::ByteString(it) => Some(it),
_ => None,
}
}
}
impl ast::FloatLiteral {
pub fn suffix(&self) -> Option<String> {
ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string())
}
}
pub enum BlockModifier {
@ -386,7 +364,7 @@ pub fn is_standalone(&self) -> bool {
fn test_literal_with_attr() {
let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#);
let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap();
assert_eq!(lit.value().to_string(), r#""Hello""#);
assert_eq!(lit.token().text(), r#""Hello""#);
}
impl ast::RecordExprField {

36
crates/syntax/src/ast/generated/nodes.rs
View File

@ -1085,26 +1085,6 @@ impl UnderscoreExpr {
pub fn underscore_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![_]) }
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatLiteral {
pub(crate) syntax: SyntaxNode,
}
impl FloatLiteral {
pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_0])
}
pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_1])
}
pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_2])
}
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_part])
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct StmtList {
pub(crate) syntax: SyntaxNode,
@ -2739,17 +2719,6 @@ fn cast(syntax: SyntaxNode) -> Option<Self> {
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for FloatLiteral {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL }
fn cast(syntax: SyntaxNode) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for StmtList {
fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST }
fn cast(syntax: SyntaxNode) -> Option<Self> {
@ -4639,11 +4608,6 @@ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for FloatLiteral {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for StmtList {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)

8
crates/syntax/src/ast/generated/tokens.rs
View File

@ -112,16 +112,16 @@ fn syntax(&self) -> &SyntaxToken { &self.syntax }
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatNumberPart {
pub struct FloatNumber {
pub(crate) syntax: SyntaxToken,
}
impl std::fmt::Display for FloatNumberPart {
impl std::fmt::Display for FloatNumber {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.syntax, f)
}
}
impl AstToken for FloatNumberPart {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART }
impl AstToken for FloatNumber {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER }
fn cast(syntax: SyntaxToken) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })

13
crates/syntax/src/ast/make.rs
View File

@ -799,11 +799,6 @@ pub fn struct_(
))
}
pub fn literal(text: &str) -> ast::Literal {
assert_eq!(text.trim(), text);
ast_from_text(&format!("fn f() {{ let _ = {}; }}", text))
}
#[track_caller]
fn ast_from_text<N: AstNode>(text: &str) -> N {
let parse = SourceFile::parse(text);
@ -832,7 +827,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken {
pub mod tokens {
use once_cell::sync::Lazy;
use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| {
SourceFile::parse(
@ -863,6 +858,12 @@ pub fn doc_comment(text: &str) -> SyntaxToken {
sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn literal(text: &str) -> SyntaxToken {
assert_eq!(text.trim(), text);
let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
lit.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn single_newline() -> SyntaxToken {
let res = SOURCE_FILE
.tree()

4
crates/syntax/src/ast/node_ext.rs
View File

@ -555,9 +555,7 @@ pub fn index_token(&self) -> Option<SyntaxToken> {
self.syntax
.children_with_tokens()
// FIXME: Accepting floats here to reject them in validation later
.find(|c| {
c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART
})
.find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER)
.as_ref()
.and_then(SyntaxElement::as_token)
.cloned()

29
crates/syntax/src/ast/token_ext.rs
View File

@ -321,7 +321,7 @@ pub fn suffix(&self) -> Option<&str> {
}
}
impl ast::FloatNumberPart {
impl ast::FloatNumber {
pub fn suffix(&self) -> Option<&str> {
let text = self.text();
let mut indices = text.char_indices();
@ -355,24 +355,14 @@ const fn prefix_len(self) -> usize {
#[cfg(test)]
mod tests {
use crate::ast::{self, make};
use crate::ast::{self, make, FloatNumber, IntNumber};
fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let suffix = match make::literal(lit).kind() {
ast::LiteralKind::FloatNumber(f) => f.suffix(),
// `1f32` lexes as an INT_NUMBER
ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()),
e => unreachable!("{e:?}"),
};
assert_eq!(suffix.as_deref(), expected.into());
assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
}
fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let i = match make::literal(lit).kind() {
ast::LiteralKind::IntNumber(i) => i,
_ => unreachable!(),
};
assert_eq!(i.suffix(), expected.into());
assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
}
#[test]
@ -400,11 +390,12 @@ fn test_int_number_suffix() {
}
fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let s = match make::literal(&format!("\"{}\"", lit)).kind() {
ast::LiteralKind::String(s) => s,
_ => unreachable!(),
};
assert_eq!(s.value().as_deref(), expected.into());
assert_eq!(
ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) }
.value()
.as_deref(),
expected.into()
);
}
#[test]

13
crates/syntax/src/tests/ast_src.rs
View File

@ -71,17 +71,7 @@ pub(crate) struct KindsSrc<'a> {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
literals: &[
"INT_NUMBER",
"FLOAT_NUMBER_START_0",
"FLOAT_NUMBER_START_1",
"FLOAT_NUMBER_START_2",
"FLOAT_NUMBER_PART",
"CHAR",
"BYTE",
"STRING",
"BYTE_STRING",
],
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[
"SOURCE_FILE",
@ -193,7 +183,6 @@ pub(crate) struct KindsSrc<'a> {
"PATH",
"PATH_SEGMENT",
"LITERAL",
"FLOAT_LITERAL",
"RENAME",
"VISIBILITY",
"WHERE_CLAUSE",

6
crates/syntax/src/tests/sourcegen_ast.rs
View File

@ -462,10 +462,6 @@ macro_rules! T {
[lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT };
[ident] => { $crate::SyntaxKind::IDENT };
[shebang] => { $crate::SyntaxKind::SHEBANG };
[float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
[float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 };
[float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 };
[float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 };
}
pub use T;
};
@ -589,7 +585,7 @@ fn ty(&self) -> proc_macro2::Ident {
fn lower(grammar: &Grammar) -> AstSrc {
let mut res = AstSrc {
tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident"
tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident"
.split_ascii_whitespace()
.map(|it| it.to_string())
.collect::<Vec<_>>(),

11
crates/syntax/src/validation.rs
View File

@ -119,15 +119,8 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> {
text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end))
}
let token = literal.value();
let text;
let text = match &token {
rowan::NodeOrToken::Node(node) => {
text = node.text().to_string();
&*text
}
rowan::NodeOrToken::Token(token) => token.text(),
};
let token = literal.token();
let text = token.text();
// FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205)
let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {