mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Recover better for more delimited sequences

Browse Source
This commit is contained in:
Lukas Wirth 2023-02-13 14:47:27 +01:00
parent 244a48d13d
commit 4f6b5f41d4
20 changed files with 153 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crates/hir-def/src/macro_expansion_tests/mbe/regression.rs
View File

@ -830,7 +830,6 @@ pub fn new() {
/* parse error: expected COMMA */
/* parse error: expected R_ANGLE */
/* parse error: expected SEMICOLON */
/* parse error: expected SEMICOLON */
/* parse error: expected expression, item or let statement */
pub fn new() {
let _ = 0as u32<<(8+8);
@ -848,21 +847,21 @@ pub fn new() {
// BLOCK_EXPR@10..31
// STMT_LIST@10..31
// L_CURLY@10..11 "{"
// LET_STMT@11..27
// LET_STMT@11..28
// LET_KW@11..14 "let"
// WILDCARD_PAT@14..15
// UNDERSCORE@14..15 "_"
// EQ@15..16 "="
// CAST_EXPR@16..27
// CAST_EXPR@16..28
// LITERAL@16..17
// INT_NUMBER@16..17 "0"
// AS_KW@17..19 "as"
// PATH_TYPE@19..27
// PATH@19..27
// PATH_SEGMENT@19..27
// PATH_TYPE@19..28
// PATH@19..28
// PATH_SEGMENT@19..28
// NAME_REF@19..22
// IDENT@19..22 "u32"
// GENERIC_ARG_LIST@22..27
// GENERIC_ARG_LIST@22..28
// L_ANGLE@22..23 "<"
// TYPE_ARG@23..27
// DYN_TRAIT_TYPE@23..27
@ -877,9 +876,9 @@ pub fn new() {
// ERROR@25..26
// INT_NUMBER@25..26 "8"
// PLUS@26..27 "+"
// EXPR_STMT@27..28
// LITERAL@27..28
// INT_NUMBER@27..28 "8"
// CONST_ARG@27..28
// LITERAL@27..28
// INT_NUMBER@27..28 "8"
// ERROR@28..29
// R_PAREN@28..29 ")"
// SEMICOLON@29..30 ";"

2
crates/ide/src/syntax_highlighting/tests.rs
View File

@ -1126,5 +1126,5 @@ fn benchmark_syntax_highlighting_parser() {
.filter(|it| it.highlight.tag == HlTag::Symbol(SymbolKind::Function))
.count()
};
assert_eq!(hash, 1609);
assert_eq!(hash, 1608);
}

30
crates/parser/src/grammar.rs
View File

@ -200,6 +200,8 @@ fn is_block(self) -> bool {
}
}
const VISIBILITY_FIRST: TokenSet = TokenSet::new(&[T![pub], T![crate]]);
fn opt_visibility(p: &mut Parser<'_>, in_tuple_field: bool) -> bool {
match p.current() {
T![pub] => {
@ -340,3 +342,31 @@ fn error_block(p: &mut Parser<'_>, message: &str) {
p.eat(T!['}']);
m.complete(p, ERROR);
}
/// The `parser` passed this is required to at least consume one token if it returns `true`.
/// If the `parser` returns false, parsing will stop.
fn delimited(
p: &mut Parser<'_>,
bra: SyntaxKind,
ket: SyntaxKind,
delim: SyntaxKind,
first_set: TokenSet,
mut parser: impl FnMut(&mut Parser<'_>) -> bool,
) {
p.bump(bra);
while !p.at(ket) && !p.at(EOF) {
if !parser(p) {
break;
}
if !p.at(delim) {
if p.at_ts(first_set) {
p.error(format!("expected {:?}", delim));
} else {
break;
}
} else {
p.bump(delim);
}
}
p.expect(ket);
}

2
crates/parser/src/grammar/attributes.rs
View File

@ -1,5 +1,7 @@
use super::*;
pub(super) const ATTRIBUTE_FIRST: TokenSet = TokenSet::new(&[T![#]]);
pub(super) fn inner_attrs(p: &mut Parser<'_>) {
while p.at(T![#]) && p.nth(1) == T![!] {
attr(p, true);

28
crates/parser/src/grammar/expressions.rs
View File

@ -1,5 +1,7 @@
mod atom;
use crate::grammar::attributes::ATTRIBUTE_FIRST;
use super::*;
pub(crate) use self::atom::{block_expr, match_arm_list};
@ -572,27 +574,11 @@ fn cast_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
fn arg_list(p: &mut Parser<'_>) {
assert!(p.at(T!['(']));
let m = p.start();
p.bump(T!['(']);
while !p.at(T![')']) && !p.at(EOF) {
// test arg_with_attr
// fn main() {
// foo(#[attr] 92)
// }
if !expr(p) {
break;
}
if !p.at(T![,]) {
if p.at_ts(EXPR_FIRST) {
p.error("expected `,`");
continue;
} else {
break;
}
} else {
p.bump(T![,]);
}
}
p.eat(T![')']);
// test arg_with_attr
// fn main() {
// foo(#[attr] 92)
// }
delimited(p, T!['('], T![')'], T![,], EXPR_FIRST.union(ATTRIBUTE_FIRST), expr);
m.complete(p, ARG_LIST);
}

2
crates/parser/src/grammar/expressions/atom.rs
View File

@ -118,7 +118,7 @@ pub(super) fn atom_expr(
// fn main() {
// 'loop: impl
// }
p.error("expected a loop");
p.error("expected a loop or block");
m.complete(p, ERROR);
return None;
}

32
crates/parser/src/grammar/generic_args.rs
View File

@ -5,27 +5,35 @@ pub(super) fn opt_generic_arg_list(p: &mut Parser<'_>, colon_colon_required: boo
if p.at(T![::]) && p.nth(2) == T![<] {
m = p.start();
p.bump(T![::]);
p.bump(T![<]);
} else if !colon_colon_required && p.at(T![<]) && p.nth(1) != T![=] {
m = p.start();
p.bump(T![<]);
} else {
return;
}
while !p.at(EOF) && !p.at(T![>]) {
generic_arg(p);
if !p.at(T![>]) && !p.expect(T![,]) {
break;
}
}
p.expect(T![>]);
delimited(p, T![<], T![>], T![,], GENERIC_ARG_FIRST, generic_arg);
m.complete(p, GENERIC_ARG_LIST);
}
const GENERIC_ARG_FIRST: TokenSet = TokenSet::new(&[
LIFETIME_IDENT,
IDENT,
T!['{'],
T![true],
T![false],
T![-],
INT_NUMBER,
FLOAT_NUMBER,
CHAR,
BYTE,
STRING,
BYTE_STRING,
])
.union(types::TYPE_FIRST);
// test generic_arg
// type T = S<i32>;
fn generic_arg(p: &mut Parser<'_>) {
fn generic_arg(p: &mut Parser<'_>) -> bool {
match p.current() {
LIFETIME_IDENT => lifetime_arg(p),
T!['{'] | T![true] | T![false] | T![-] => const_arg(p),
@ -68,8 +76,10 @@ fn generic_arg(p: &mut Parser<'_>) {
}
}
}
_ => type_arg(p),
_ if p.at_ts(types::TYPE_FIRST) => type_arg(p),
_ => return false,
}
true
}
// test lifetime_arg

29
crates/parser/src/grammar/generic_params.rs
View File

@ -1,3 +1,5 @@
use crate::grammar::attributes::ATTRIBUTE_FIRST;
use super::*;
pub(super) fn opt_generic_param_list(p: &mut Parser<'_>) {
@ -11,32 +13,31 @@ pub(super) fn opt_generic_param_list(p: &mut Parser<'_>) {
fn generic_param_list(p: &mut Parser<'_>) {
assert!(p.at(T![<]));
let m = p.start();
p.bump(T![<]);
delimited(p, T![<], T![>], T![,], GENERIC_PARAM_FIRST.union(ATTRIBUTE_FIRST), |p| {
// test generic_param_attribute
// fn foo<#[lt_attr] 'a, #[t_attr] T>() {}
let m = p.start();
attributes::outer_attrs(p);
generic_param(p, m)
});
while !p.at(EOF) && !p.at(T![>]) {
generic_param(p);
if !p.at(T![>]) && !p.expect(T![,]) {
break;
}
}
p.expect(T![>]);
m.complete(p, GENERIC_PARAM_LIST);
}
fn generic_param(p: &mut Parser<'_>) {
let m = p.start();
// test generic_param_attribute
// fn foo<#[lt_attr] 'a, #[t_attr] T>() {}
attributes::outer_attrs(p);
const GENERIC_PARAM_FIRST: TokenSet = TokenSet::new(&[IDENT, LIFETIME_IDENT, T![const]]);
fn generic_param(p: &mut Parser<'_>, m: Marker) -> bool {
match p.current() {
LIFETIME_IDENT => lifetime_param(p, m),
IDENT => type_param(p, m),
T![const] => const_param(p, m),
_ => {
m.abandon(p);
p.err_and_bump("expected type parameter");
p.err_and_bump("expected generic parameter");
return false;
}
}
true
}
// test lifetime_param

25
crates/parser/src/grammar/items/adt.rs
View File

@ -1,3 +1,5 @@
use crate::grammar::attributes::ATTRIBUTE_FIRST;
use super::*;
// test struct_item
@ -141,28 +143,31 @@ fn record_field(p: &mut Parser<'_>) {
}
}
const TUPLE_FIELD_FIRST: TokenSet =
types::TYPE_FIRST.union(ATTRIBUTE_FIRST).union(VISIBILITY_FIRST);
fn tuple_field_list(p: &mut Parser<'_>) {
assert!(p.at(T!['(']));
let m = p.start();
p.bump(T!['(']);
while !p.at(T![')']) && !p.at(EOF) {
delimited(p, T!['('], T![')'], T![,], TUPLE_FIELD_FIRST, |p| {
let m = p.start();
// test tuple_field_attrs
// struct S (#[attr] f32);
attributes::outer_attrs(p);
opt_visibility(p, true);
let has_vis = opt_visibility(p, true);
if !p.at_ts(types::TYPE_FIRST) {
p.error("expected a type");
m.complete(p, ERROR);
break;
if has_vis {
m.complete(p, ERROR);
} else {
m.abandon(p);
}
return false;
}
types::type_(p);
m.complete(p, TUPLE_FIELD);
true
});
if !p.at(T![')']) {
p.expect(T![,]);
}
}
p.expect(T![')']);
m.complete(p, TUPLE_FIELD_LIST);
}

14
crates/parser/src/grammar/params.rs
View File

@ -1,3 +1,5 @@
use crate::grammar::attributes::ATTRIBUTE_FIRST;
use super::*;
// test param_list
@ -66,14 +68,20 @@ fn list_(p: &mut Parser<'_>, flavor: Flavor) {
}
};
if !p.at_ts(PARAM_FIRST) {
if !p.at_ts(PARAM_FIRST.union(ATTRIBUTE_FIRST)) {
p.error("expected value parameter");
m.abandon(p);
break;
}
param(p, m, flavor);
if !p.at(ket) {
p.expect(T![,]);
if !p.at(T![,]) {
if p.at_ts(PARAM_FIRST.union(ATTRIBUTE_FIRST)) {
p.error("expected `,`");
} else {
break;
}
} else {
p.bump(T![,]);
}
}

3
crates/parser/test_data/parser/err/0009_broken_struct_type_parameter.rast
View File

@ -44,8 +44,7 @@ SOURCE_FILE
IDENT "T"
SEMICOLON ";"
WHITESPACE "\n"
error 9: expected type parameter
error 11: expected COMMA
error 9: expected generic parameter
error 11: expected R_ANGLE
error 11: expected `;`, `{`, or `(`
error 12: expected an item

38
crates/parser/test_data/parser/err/0013_invalid_type.rast
View File

@ -43,17 +43,14 @@ SOURCE_FILE
IDENT "Box"
GENERIC_ARG_LIST
L_ANGLE "<"
TYPE_ARG
ERROR
AT "@"
WHITESPACE " "
TUPLE_FIELD
PATH_TYPE
PATH
PATH_SEGMENT
NAME_REF
IDENT "Any"
ERROR
ERROR
AT "@"
WHITESPACE " "
MACRO_CALL
PATH
PATH_SEGMENT
NAME_REF
IDENT "Any"
ERROR
R_ANGLE ">"
ERROR
@ -69,17 +66,14 @@ SOURCE_FILE
ERROR
SEMICOLON ";"
WHITESPACE "\n\n"
error 67: expected type
error 68: expected COMMA
error 68: expected R_ANGLE
error 68: expected COMMA
error 68: expected R_ANGLE
error 68: expected COMMA
error 68: expected R_ANGLE
error 68: expected COMMA
error 72: expected COMMA
error 72: expected a type
error 72: expected R_PAREN
error 67: expected R_ANGLE
error 67: expected R_ANGLE
error 67: expected R_ANGLE
error 67: expected R_PAREN
error 67: expected SEMICOLON
error 67: expected an item
error 72: expected BANG
error 72: expected `{`, `[`, `(`
error 72: expected SEMICOLON
error 72: expected an item
error 73: expected an item

2
crates/parser/test_data/parser/err/0022_bad_exprs.rast
View File

@ -151,6 +151,7 @@ error 26: expected `;`, `{`, or `(`
error 30: expected pattern
error 31: expected SEMICOLON
error 53: expected expression
error 54: expected R_PAREN
error 54: expected SEMICOLON
error 54: expected expression, item or let statement
error 60: expected type
@ -160,6 +161,7 @@ error 65: expected pattern
error 65: expected SEMICOLON
error 65: expected expression, item or let statement
error 92: expected expression
error 93: expected R_PAREN
error 93: expected SEMICOLON
error 93: expected expression, item or let statement
error 95: expected expression, item or let statement

17
crates/parser/test_data/parser/err/0024_many_type_parens.rast
View File

@ -168,12 +168,12 @@ SOURCE_FILE
L_PAREN "("
ERROR
QUESTION "?"
EXPR_STMT
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "Sized"
TYPE_ARG
PATH_TYPE
PATH
PATH_SEGMENT
NAME_REF
IDENT "Sized"
ERROR
R_PAREN ")"
WHITESPACE " "
@ -291,15 +291,13 @@ SOURCE_FILE
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"
error 88: expected COMMA
error 88: expected R_ANGLE
error 121: expected SEMICOLON
error 121: expected expression, item or let statement
error 140: expected type
error 141: expected R_PAREN
error 141: expected COMMA
error 141: expected R_ANGLE
error 141: expected SEMICOLON
error 146: expected R_ANGLE
error 146: expected SEMICOLON
error 146: expected expression, item or let statement
error 148: expected expression, item or let statement
@ -309,7 +307,6 @@ error 165: expected expression
error 168: expected expression
error 179: expected expression
error 180: expected SEMICOLON
error 215: expected COMMA
error 215: expected R_ANGLE
error 235: expected SEMICOLON
error 235: expected expression, item or let statement

7
crates/parser/test_data/parser/err/0025_nope.rast
View File

@ -156,8 +156,7 @@ SOURCE_FILE
PATH_SEGMENT
NAME_REF
IDENT "i32"
WHITESPACE " "
ERROR
WHITESPACE " "
ERROR
L_CURLY "{"
R_CURLY "}"
@ -199,10 +198,8 @@ error 95: expected type
error 95: expected COMMA
error 96: expected field
error 98: expected field declaration
error 371: expected R_PAREN
error 371: expected COMMA
error 372: expected a type
error 372: expected R_PAREN
error 372: expected COMMA
error 372: expected enum variant
error 374: expected enum variant
error 494: expected pattern

2
crates/parser/test_data/parser/err/0042_weird_blocks.rast
View File

@ -72,4 +72,4 @@ SOURCE_FILE
error 24: expected existential, fn, trait or impl
error 41: expected existential, fn, trait or impl
error 56: expected a block
error 75: expected a loop
error 75: expected a loop or block

19
crates/parser/test_data/parser/err/0048_double_fish.rast
View File

@ -12,7 +12,7 @@ SOURCE_FILE
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
BIN_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
@ -41,13 +41,14 @@ SOURCE_FILE
COLON2 "::"
ERROR
L_ANGLE "<"
BIN_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "nope"
SHR ">>"
TYPE_ARG
PATH_TYPE
PATH
PATH_SEGMENT
NAME_REF
IDENT "nope"
R_ANGLE ">"
R_ANGLE ">"
ERROR
SEMICOLON ";"
WHITESPACE "\n"
@ -114,8 +115,6 @@ SOURCE_FILE
WHITESPACE "\n"
error 30: expected identifier
error 31: expected COMMA
error 31: expected R_ANGLE
error 31: expected SEMICOLON
error 37: expected expression
error 75: expected identifier
error 76: expected SEMICOLON

2
crates/parser/test_data/parser/inline/err/0002_misplaced_label_err.rast
View File

@ -23,6 +23,6 @@ SOURCE_FILE
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"
error 22: expected a loop
error 22: expected a loop or block
error 27: expected type
error 27: expected `{`

2
crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rast
View File

@ -72,6 +72,6 @@ SOURCE_FILE
R_CURLY "}"
WHITESPACE "\n"
error 25: expected identifier
error 39: expected `,`
error 39: expected COMMA
error 39: expected expression
error 55: expected expression

2
crates/parser/test_data/parser/inline/err/0015_missing_fn_param_type.rast
View File

@ -49,5 +49,5 @@ SOURCE_FILE
R_CURLY "}"
WHITESPACE "\n"
error 6: missing type for function parameter
error 6: expected COMMA
error 6: expected `,`
error 16: missing type for function parameter