parser: Break float tokens into parts in tuple field positions

This commit is contained in:
Vadim Petrochenkov 2020-04-18 21:26:10 +03:00
parent 8a6d4342be
commit 8e256b19d5
6 changed files with 493 additions and 60 deletions

View File

@ -770,10 +770,10 @@ impl<'a> Parser<'a> {
match self.token.uninterpolate().kind {
token::Ident(..) => self.parse_dot_suffix(base, lo),
token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => {
Ok(self.parse_tuple_field_access_expr(lo, base, symbol, suffix))
Ok(self.parse_tuple_field_access_expr(lo, base, symbol, suffix, None))
}
token::Literal(token::Lit { kind: token::Float, symbol, .. }) => {
self.recover_field_access_by_float_lit(lo, base, symbol)
token::Literal(token::Lit { kind: token::Float, symbol, suffix }) => {
Ok(self.parse_tuple_field_access_expr_float(lo, base, symbol, suffix))
}
_ => {
self.error_unexpected_after_dot();
@ -788,45 +788,84 @@ impl<'a> Parser<'a> {
self.struct_span_err(self.token.span, &format!("unexpected token: `{}`", actual)).emit();
}
fn recover_field_access_by_float_lit(
// We need and identifier or integer, but the next token is a float.
// Break the float into components to extract the identifier or integer.
// FIXME: With current `TokenCursor` it's hard to break tokens into more than 2
// parts unless those parts are processed immediately. `TokenCursor` should either
// support pushing "future tokens" (would be also helpful to `break_and_eat`), or
// we should break everything including floats into more basic proc-macro style
// tokens in the lexer (probably preferable).
fn parse_tuple_field_access_expr_float(
&mut self,
lo: Span,
base: P<Expr>,
sym: Symbol,
) -> PResult<'a, P<Expr>> {
self.bump();
let fstr = sym.as_str();
let msg = format!("unexpected token: `{}`", sym);
let mut err = self.struct_span_err(self.prev_token.span, &msg);
err.span_label(self.prev_token.span, "unexpected token");
if fstr.chars().all(|x| "0123456789.".contains(x)) {
let float = match fstr.parse::<f64>() {
Ok(f) => f,
Err(_) => {
err.emit();
return Ok(base);
}
};
let sugg = pprust::to_string(|s| {
s.popen();
s.print_expr(&base);
s.s.word(".");
s.print_usize(float.trunc() as usize);
s.pclose();
s.s.word(".");
s.s.word(fstr.splitn(2, '.').last().unwrap().to_string())
});
err.span_suggestion(
lo.to(self.prev_token.span),
"try parenthesizing the first index",
sugg,
Applicability::MachineApplicable,
);
float: Symbol,
suffix: Option<Symbol>,
) -> P<Expr> {
#[derive(Debug)]
enum FloatComponent {
IdentLike(String),
Punct(char),
}
use FloatComponent::*;
let mut components = Vec::new();
let mut ident_like = String::new();
for c in float.as_str().chars() {
if c == '_' || c.is_ascii_alphanumeric() {
ident_like.push(c);
} else if matches!(c, '.' | '+' | '-') {
if !ident_like.is_empty() {
components.push(IdentLike(mem::take(&mut ident_like)));
}
components.push(Punct(c));
} else {
panic!("unexpected character in a float token: {:?}", c)
}
}
if !ident_like.is_empty() {
components.push(IdentLike(ident_like));
}
// FIXME: Make the span more precise.
let span = self.token.span;
match &*components {
// 1e2
[IdentLike(i)] => {
self.parse_tuple_field_access_expr(lo, base, Symbol::intern(&i), suffix, None)
}
// 1.
[IdentLike(i), Punct('.')] => {
assert!(suffix.is_none());
let symbol = Symbol::intern(&i);
self.token = Token::new(token::Ident(symbol, false), span);
let next_token = Token::new(token::Dot, span);
self.parse_tuple_field_access_expr(lo, base, symbol, None, Some(next_token))
}
// 1.2 | 1.2e3
[IdentLike(i1), Punct('.'), IdentLike(i2)] => {
let symbol1 = Symbol::intern(&i1);
self.token = Token::new(token::Ident(symbol1, false), span);
let next_token1 = Token::new(token::Dot, span);
let base1 =
self.parse_tuple_field_access_expr(lo, base, symbol1, None, Some(next_token1));
let symbol2 = Symbol::intern(&i2);
let next_token2 = Token::new(token::Ident(symbol2, false), span);
self.bump_with(next_token2); // `.`
self.parse_tuple_field_access_expr(lo, base1, symbol2, suffix, None)
}
// 1e+ | 1e- (recovered)
[IdentLike(_), Punct('+' | '-')] |
// 1e+2 | 1e-2
[IdentLike(_), Punct('+' | '-'), IdentLike(_)] |
// 1.2e+3 | 1.2e-3
[IdentLike(_), Punct('.'), IdentLike(_), Punct('+' | '-'), IdentLike(_)] => {
// See the FIXME about `TokenCursor` above.
self.error_unexpected_after_dot();
base
}
_ => panic!("unexpected components in a float token: {:?}", components),
}
Err(err)
}
fn parse_tuple_field_access_expr(
@ -835,8 +874,12 @@ impl<'a> Parser<'a> {
base: P<Expr>,
field: Symbol,
suffix: Option<Symbol>,
next_token: Option<Token>,
) -> P<Expr> {
self.bump();
match next_token {
Some(next_token) => self.bump_with(next_token),
None => self.bump(),
}
let span = self.prev_token.span;
let field = ExprKind::Field(base, Ident::new(field, span));
self.expect_no_suffix(span, "a tuple index", suffix);

View File

@ -0,0 +1,62 @@
struct S(u8, (u8, u8));
fn main() {
let s = S(0, (0, 0));
s.1e1; //~ ERROR no field `1e1` on type `S`
s.1.; //~ ERROR unexpected token: `;`
s.1.1;
s.1.1e1; //~ ERROR no field `1e1` on type `(u8, u8)`
{ s.1e+; } //~ ERROR unexpected token: `1e+`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+`
//~| ERROR expected at least one digit in exponent
{ s.1e-; } //~ ERROR unexpected token: `1e-`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-`
//~| ERROR expected at least one digit in exponent
{ s.1e+1; } //~ ERROR unexpected token: `1e+1`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+1`
{ s.1e-1; } //~ ERROR unexpected token: `1e-1`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-1`
{ s.1.1e+1; } //~ ERROR unexpected token: `1.1e+1`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e+1`
{ s.1.1e-1; } //~ ERROR unexpected token: `1.1e-1`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e-1`
s.0x1e1; //~ ERROR no field `0x1e1` on type `S`
s.0x1.; //~ ERROR no field `0x1` on type `S`
//~| ERROR hexadecimal float literal is not supported
//~| ERROR unexpected token: `;`
s.0x1.1; //~ ERROR no field `0x1` on type `S`
//~| ERROR hexadecimal float literal is not supported
s.0x1.1e1; //~ ERROR no field `0x1` on type `S`
//~| ERROR hexadecimal float literal is not supported
{ s.0x1e+; } //~ ERROR expected expression, found `;`
{ s.0x1e-; } //~ ERROR expected expression, found `;`
s.0x1e+1; //~ ERROR no field `0x1e` on type `S`
s.0x1e-1; //~ ERROR no field `0x1e` on type `S`
{ s.0x1.1e+1; } //~ ERROR unexpected token: `0x1.1e+1`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `0x1.1e+1`
//~| ERROR hexadecimal float literal is not supported
{ s.0x1.1e-1; } //~ ERROR unexpected token: `0x1.1e-1`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `0x1.1e-1`
//~| ERROR hexadecimal float literal is not supported
s.1e1f32; //~ ERROR no field `1e1` on type `S`
//~| ERROR suffixes on a tuple index are invalid
s.1.f32; //~ ERROR no field `f32` on type `(u8, u8)`
s.1.1f32; //~ ERROR suffixes on a tuple index are invalid
s.1.1e1f32; //~ ERROR no field `1e1` on type `(u8, u8)`
//~| ERROR suffixes on a tuple index are invalid
{ s.1e+f32; } //~ ERROR unexpected token: `1e+f32`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+f32`
//~| ERROR expected at least one digit in exponent
{ s.1e-f32; } //~ ERROR unexpected token: `1e-f32`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-f32`
//~| ERROR expected at least one digit in exponent
{ s.1e+1f32; } //~ ERROR unexpected token: `1e+1f32`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+1f32`
{ s.1e-1f32; } //~ ERROR unexpected token: `1e-1f32`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-1f32`
{ s.1.1e+1f32; } //~ ERROR unexpected token: `1.1e+1f32`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e+1f32`
{ s.1.1e-1f32; } //~ ERROR unexpected token: `1.1e-1f32`
//~| ERROR expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e-1f32`
}

View File

@ -0,0 +1,349 @@
error: expected at least one digit in exponent
--> $DIR/float-field.rs:10:9
|
LL | { s.1e+; }
| ^^^
error: expected at least one digit in exponent
--> $DIR/float-field.rs:13:9
|
LL | { s.1e-; }
| ^^^
error: hexadecimal float literal is not supported
--> $DIR/float-field.rs:25:7
|
LL | s.0x1.;
| ^^^^
error: hexadecimal float literal is not supported
--> $DIR/float-field.rs:28:7
|
LL | s.0x1.1;
| ^^^^^
error: hexadecimal float literal is not supported
--> $DIR/float-field.rs:30:7
|
LL | s.0x1.1e1;
| ^^^^^^^
error: hexadecimal float literal is not supported
--> $DIR/float-field.rs:36:9
|
LL | { s.0x1.1e+1; }
| ^^^^^^^^
error: hexadecimal float literal is not supported
--> $DIR/float-field.rs:39:9
|
LL | { s.0x1.1e-1; }
| ^^^^^^^^
error: expected at least one digit in exponent
--> $DIR/float-field.rs:48:9
|
LL | { s.1e+f32; }
| ^^^^^^
error: expected at least one digit in exponent
--> $DIR/float-field.rs:51:9
|
LL | { s.1e-f32; }
| ^^^^^^
error: unexpected token: `;`
--> $DIR/float-field.rs:7:9
|
LL | s.1.;
| ^
error: unexpected token: `1e+`
--> $DIR/float-field.rs:10:9
|
LL | { s.1e+; }
| ^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+`
--> $DIR/float-field.rs:10:9
|
LL | { s.1e+; }
| ^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1e-`
--> $DIR/float-field.rs:13:9
|
LL | { s.1e-; }
| ^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-`
--> $DIR/float-field.rs:13:9
|
LL | { s.1e-; }
| ^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1e+1`
--> $DIR/float-field.rs:16:9
|
LL | { s.1e+1; }
| ^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+1`
--> $DIR/float-field.rs:16:9
|
LL | { s.1e+1; }
| ^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1e-1`
--> $DIR/float-field.rs:18:9
|
LL | { s.1e-1; }
| ^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-1`
--> $DIR/float-field.rs:18:9
|
LL | { s.1e-1; }
| ^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1.1e+1`
--> $DIR/float-field.rs:20:9
|
LL | { s.1.1e+1; }
| ^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e+1`
--> $DIR/float-field.rs:20:9
|
LL | { s.1.1e+1; }
| ^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1.1e-1`
--> $DIR/float-field.rs:22:9
|
LL | { s.1.1e-1; }
| ^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e-1`
--> $DIR/float-field.rs:22:9
|
LL | { s.1.1e-1; }
| ^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `;`
--> $DIR/float-field.rs:25:11
|
LL | s.0x1.;
| ^
error: expected expression, found `;`
--> $DIR/float-field.rs:32:14
|
LL | { s.0x1e+; }
| ^ expected expression
error: expected expression, found `;`
--> $DIR/float-field.rs:33:14
|
LL | { s.0x1e-; }
| ^ expected expression
error: unexpected token: `0x1.1e+1`
--> $DIR/float-field.rs:36:9
|
LL | { s.0x1.1e+1; }
| ^^^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `0x1.1e+1`
--> $DIR/float-field.rs:36:9
|
LL | { s.0x1.1e+1; }
| ^^^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `0x1.1e-1`
--> $DIR/float-field.rs:39:9
|
LL | { s.0x1.1e-1; }
| ^^^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `0x1.1e-1`
--> $DIR/float-field.rs:39:9
|
LL | { s.0x1.1e-1; }
| ^^^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: suffixes on a tuple index are invalid
--> $DIR/float-field.rs:42:7
|
LL | s.1e1f32;
| ^^^^^^ invalid suffix `f32`
error: suffixes on a tuple index are invalid
--> $DIR/float-field.rs:45:7
|
LL | s.1.1f32;
| ^^^^^^ invalid suffix `f32`
error: suffixes on a tuple index are invalid
--> $DIR/float-field.rs:46:7
|
LL | s.1.1e1f32;
| ^^^^^^^^ invalid suffix `f32`
error: unexpected token: `1e+f32`
--> $DIR/float-field.rs:48:9
|
LL | { s.1e+f32; }
| ^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+f32`
--> $DIR/float-field.rs:48:9
|
LL | { s.1e+f32; }
| ^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1e-f32`
--> $DIR/float-field.rs:51:9
|
LL | { s.1e-f32; }
| ^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-f32`
--> $DIR/float-field.rs:51:9
|
LL | { s.1e-f32; }
| ^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1e+1f32`
--> $DIR/float-field.rs:54:9
|
LL | { s.1e+1f32; }
| ^^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e+1f32`
--> $DIR/float-field.rs:54:9
|
LL | { s.1e+1f32; }
| ^^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1e-1f32`
--> $DIR/float-field.rs:56:9
|
LL | { s.1e-1f32; }
| ^^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1e-1f32`
--> $DIR/float-field.rs:56:9
|
LL | { s.1e-1f32; }
| ^^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1.1e+1f32`
--> $DIR/float-field.rs:58:9
|
LL | { s.1.1e+1f32; }
| ^^^^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e+1f32`
--> $DIR/float-field.rs:58:9
|
LL | { s.1.1e+1f32; }
| ^^^^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error: unexpected token: `1.1e-1f32`
--> $DIR/float-field.rs:60:9
|
LL | { s.1.1e-1f32; }
| ^^^^^^^^^
error: expected one of `.`, `;`, `?`, `}`, or an operator, found `1.1e-1f32`
--> $DIR/float-field.rs:60:9
|
LL | { s.1.1e-1f32; }
| ^^^^^^^^^ expected one of `.`, `;`, `?`, `}`, or an operator
error[E0609]: no field `1e1` on type `S`
--> $DIR/float-field.rs:6:7
|
LL | s.1e1;
| ^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `1e1` on type `(u8, u8)`
--> $DIR/float-field.rs:9:7
|
LL | s.1.1e1;
| ^^^^^
error[E0609]: no field `0x1e1` on type `S`
--> $DIR/float-field.rs:24:7
|
LL | s.0x1e1;
| ^^^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `0x1` on type `S`
--> $DIR/float-field.rs:25:7
|
LL | s.0x1.;
| ^^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `0x1` on type `S`
--> $DIR/float-field.rs:28:7
|
LL | s.0x1.1;
| ^^^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `0x1` on type `S`
--> $DIR/float-field.rs:30:7
|
LL | s.0x1.1e1;
| ^^^^^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `0x1e` on type `S`
--> $DIR/float-field.rs:34:7
|
LL | s.0x1e+1;
| ^^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `0x1e` on type `S`
--> $DIR/float-field.rs:35:7
|
LL | s.0x1e-1;
| ^^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `1e1` on type `S`
--> $DIR/float-field.rs:42:7
|
LL | s.1e1f32;
| ^^^^^^ unknown field
|
= note: available fields are: `0`, `1`
error[E0609]: no field `f32` on type `(u8, u8)`
--> $DIR/float-field.rs:44:9
|
LL | s.1.f32;
| ^^^
error[E0609]: no field `1e1` on type `(u8, u8)`
--> $DIR/float-field.rs:46:7
|
LL | s.1.1e1f32;
| ^^^^^^^^
error: aborting due to 55 previous errors
For more information about this error, try `rustc --explain E0609`.

View File

@ -1,5 +0,0 @@
// run-rustfix
fn main () {
((1, (2, 3)).1).1; //~ ERROR unexpected token: `1.1`
}

View File

@ -1,5 +0,0 @@
// run-rustfix
fn main () {
(1, (2, 3)).1.1; //~ ERROR unexpected token: `1.1`
}

View File

@ -1,11 +0,0 @@
error: unexpected token: `1.1`
--> $DIR/tuple-float-index.rs:4:17
|
LL | (1, (2, 3)).1.1;
| ------------^^^
| | |
| | unexpected token
| help: try parenthesizing the first index: `((1, (2, 3)).1).1`
error: aborting due to previous error