mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix DebugParser.

Browse Source 
It currently doesn't work at all. This commit changes it to a simpler
imperative style that produces a valid `tokens` vec.

(An aside: I find `Iterator::scan` to be a pretty wretched function,
that produces code which is very hard to understand. Probably why this
is just one of two uses of it in the entire compiler.)
This commit is contained in:
Nicholas Nethercote 2024-07-13 15:53:31 +10:00
parent cf2dfb2ced
commit aa0e8e1475
2 changed files with 104 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
compiler/rustc_parse/src/parser/mod.rs
View File

@ -1537,14 +1537,16 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// we don't need N spans, but we want at least one, so print all of prev_token
dbg_fmt.field("prev_token", &parser.prev_token);
// make it easier to peek farther ahead by taking TokenKinds only until EOF
let tokens = (0..*lookahead)
.map(|i| parser.look_ahead(i, |tok| tok.kind.clone()))
.scan(parser.prev_token == TokenKind::Eof, |eof, tok| {
let current = eof.then_some(tok.clone()); // include a trailing EOF token
*eof |= &tok == &TokenKind::Eof;
current
});
let mut tokens = vec![];
for i in 0..*lookahead {
let tok = parser.look_ahead(i, |tok| tok.kind.clone());
let is_eof = tok == TokenKind::Eof;
tokens.push(tok);
if is_eof {
// Don't look ahead past EOF.
break;
}
}
dbg_fmt.field_with("tokens", |field| field.debug_list().entries(tokens).finish());
dbg_fmt.field("approx_token_stream_pos", &parser.num_bump_calls);

100
compiler/rustc_parse/src/parser/tests.rs
View File

@ -1541,11 +1541,36 @@ fn debug_lookahead() {
ctxt: #0,
},
},
tokens: [],
tokens: [
Ident(
\"fn\",
No,
),
Ident(
\"f\",
No,
),
OpenDelim(
Parenthesis,
),
Ident(
\"x\",
No,
),
Colon,
Ident(
\"u32\",
No,
),
CloseDelim(
Parenthesis,
),
],
approx_token_stream_pos: 1,
..
}"
);
// There are 13 tokens. We request 15, get 14; the last one is `Eof`.
assert_eq!(
&format!("{:#?}", p.debug_lookahead(15)),
"Parser {
@ -1561,7 +1586,51 @@ fn debug_lookahead() {
ctxt: #0,
},
},
tokens: [],
tokens: [
Ident(
\"fn\",
No,
),
Ident(
\"f\",
No,
),
OpenDelim(
Parenthesis,
),
Ident(
\"x\",
No,
),
Colon,
Ident(
\"u32\",
No,
),
CloseDelim(
Parenthesis,
),
OpenDelim(
Brace,
),
Ident(
\"x\",
No,
),
CloseDelim(
Brace,
),
Ident(
\"struct\",
No,
),
Ident(
\"S\",
No,
),
Semi,
Eof,
],
approx_token_stream_pos: 1,
..
}"
@ -1588,7 +1657,12 @@ fn debug_lookahead() {
ctxt: #0,
},
},
tokens: [],
tokens: [
Ident(
\"x\",
No,
),
],
approx_token_stream_pos: 9,
..
}"
@ -1610,7 +1684,23 @@ fn debug_lookahead() {
ctxt: #0,
},
},
tokens: [],
tokens: [
Ident(
\"x\",
No,
),
CloseDelim(
Brace,
),
Ident(
\"struct\",
No,
),
Ident(
\"S\",
No,
),
],
approx_token_stream_pos: 9,
..
}"
@ -1637,8 +1727,6 @@ fn debug_lookahead() {
},
tokens: [
Eof,
Eof,
Eof,
],
approx_token_stream_pos: 15,
..