Rollup merge of #127273 - nnethercote:fix-DebugParser, r=workingjubilee
Fix `DebugParser`. I tried using this and it didn't work at all. `prev_token` is never eof, so the accumulator is always false, which means the `then_some` always returns `None`, which means `scan` always returns `None`, and `tokens` always ends up an empty vec. I'm not sure how this code was supposed to work. (An aside: I find `Iterator::scan` to be a pretty wretched function, that produces code which is very hard to understand. Probably why this is just one of two uses of it in the entire compiler.) This commit changes it to a simpler imperative style that produces a valid `tokens` vec. r? `@workingjubilee`
This commit is contained in:
Matthias Krüger
GitHub
commit
febe4423c1
@ -1537,14 +1537,16 @@ impl<'a> Parser<'a> {
|
||||
|
||||
// we don't need N spans, but we want at least one, so print all of prev_token
|
||||
dbg_fmt.field("prev_token", &parser.prev_token);
|
||||
// make it easier to peek farther ahead by taking TokenKinds only until EOF
|
||||
let tokens = (0..*lookahead)
|
||||
.map(|i| parser.look_ahead(i, |tok| tok.kind.clone()))
|
||||
.scan(parser.prev_token == TokenKind::Eof, |eof, tok| {
|
||||
let current = eof.then_some(tok.clone()); // include a trailing EOF token
|
||||
*eof |= &tok == &TokenKind::Eof;
|
||||
current
|
||||
});
|
||||
let mut tokens = vec![];
|
||||
for i in 0..*lookahead {
|
||||
let tok = parser.look_ahead(i, |tok| tok.kind.clone());
|
||||
let is_eof = tok == TokenKind::Eof;
|
||||
tokens.push(tok);
|
||||
if is_eof {
|
||||
// Don't look ahead past EOF.
|
||||
break;
|
||||
}
|
||||
}
|
||||
dbg_fmt.field_with("tokens", |field| field.debug_list().entries(tokens).finish());
|
||||
dbg_fmt.field("approx_token_stream_pos", &parser.num_bump_calls);
|
||||
|
||||
|
||||
@ -1376,12 +1376,13 @@ fn ttdelim_span() {
|
||||
});
|
||||
}
|
||||
|
||||
// Uses a macro rather than a function so that failure messages mention the
|
||||
// correct line in the test function.
|
||||
macro_rules! look {
|
||||
($p:ident, $dist:literal, $kind:expr) => {
|
||||
$p.look_ahead($dist, |tok| assert_eq!($kind, tok.kind));
|
||||
};
|
||||
#[track_caller]
|
||||
fn look(p: &Parser<'_>, dist: usize, kind: rustc_ast::token::TokenKind) {
|
||||
// Do the `assert_eq` outside the closure so that `track_caller` works.
|
||||
// (`#![feature(closure_track_caller)]` + `#[track_caller]` on the closure
|
||||
// doesn't give the line number in the test below if the assertion fails.)
|
||||
let tok = p.look_ahead(dist, |tok| tok.clone());
|
||||
assert_eq!(kind, tok.kind);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@ -1397,63 +1398,63 @@ fn look_ahead() {
|
||||
let mut p = string_to_parser(&psess, "fn f(x: u32) { x } struct S;".to_string());
|
||||
|
||||
// Current position is the `fn`.
|
||||
look!(p, 0, token::Ident(kw::Fn, raw_no));
|
||||
look!(p, 1, token::Ident(sym_f, raw_no));
|
||||
look!(p, 2, token::OpenDelim(Delimiter::Parenthesis));
|
||||
look!(p, 3, token::Ident(sym_x, raw_no));
|
||||
look!(p, 4, token::Colon);
|
||||
look!(p, 5, token::Ident(sym::u32, raw_no));
|
||||
look!(p, 6, token::CloseDelim(Delimiter::Parenthesis));
|
||||
look!(p, 7, token::OpenDelim(Delimiter::Brace));
|
||||
look!(p, 8, token::Ident(sym_x, raw_no));
|
||||
look!(p, 9, token::CloseDelim(Delimiter::Brace));
|
||||
look!(p, 10, token::Ident(kw::Struct, raw_no));
|
||||
look!(p, 11, token::Ident(sym_S, raw_no));
|
||||
look!(p, 12, token::Semi);
|
||||
look(&p, 0, token::Ident(kw::Fn, raw_no));
|
||||
look(&p, 1, token::Ident(sym_f, raw_no));
|
||||
look(&p, 2, token::OpenDelim(Delimiter::Parenthesis));
|
||||
look(&p, 3, token::Ident(sym_x, raw_no));
|
||||
look(&p, 4, token::Colon);
|
||||
look(&p, 5, token::Ident(sym::u32, raw_no));
|
||||
look(&p, 6, token::CloseDelim(Delimiter::Parenthesis));
|
||||
look(&p, 7, token::OpenDelim(Delimiter::Brace));
|
||||
look(&p, 8, token::Ident(sym_x, raw_no));
|
||||
look(&p, 9, token::CloseDelim(Delimiter::Brace));
|
||||
look(&p, 10, token::Ident(kw::Struct, raw_no));
|
||||
look(&p, 11, token::Ident(sym_S, raw_no));
|
||||
look(&p, 12, token::Semi);
|
||||
// Any lookahead past the end of the token stream returns `Eof`.
|
||||
look!(p, 13, token::Eof);
|
||||
look!(p, 14, token::Eof);
|
||||
look!(p, 15, token::Eof);
|
||||
look!(p, 100, token::Eof);
|
||||
look(&p, 13, token::Eof);
|
||||
look(&p, 14, token::Eof);
|
||||
look(&p, 15, token::Eof);
|
||||
look(&p, 100, token::Eof);
|
||||
|
||||
// Move forward to the first `x`.
|
||||
for _ in 0..3 {
|
||||
p.bump();
|
||||
}
|
||||
look!(p, 0, token::Ident(sym_x, raw_no));
|
||||
look!(p, 1, token::Colon);
|
||||
look!(p, 2, token::Ident(sym::u32, raw_no));
|
||||
look!(p, 3, token::CloseDelim(Delimiter::Parenthesis));
|
||||
look!(p, 4, token::OpenDelim(Delimiter::Brace));
|
||||
look!(p, 5, token::Ident(sym_x, raw_no));
|
||||
look!(p, 6, token::CloseDelim(Delimiter::Brace));
|
||||
look!(p, 7, token::Ident(kw::Struct, raw_no));
|
||||
look!(p, 8, token::Ident(sym_S, raw_no));
|
||||
look!(p, 9, token::Semi);
|
||||
look!(p, 10, token::Eof);
|
||||
look!(p, 11, token::Eof);
|
||||
look!(p, 100, token::Eof);
|
||||
look(&p, 0, token::Ident(sym_x, raw_no));
|
||||
look(&p, 1, token::Colon);
|
||||
look(&p, 2, token::Ident(sym::u32, raw_no));
|
||||
look(&p, 3, token::CloseDelim(Delimiter::Parenthesis));
|
||||
look(&p, 4, token::OpenDelim(Delimiter::Brace));
|
||||
look(&p, 5, token::Ident(sym_x, raw_no));
|
||||
look(&p, 6, token::CloseDelim(Delimiter::Brace));
|
||||
look(&p, 7, token::Ident(kw::Struct, raw_no));
|
||||
look(&p, 8, token::Ident(sym_S, raw_no));
|
||||
look(&p, 9, token::Semi);
|
||||
look(&p, 10, token::Eof);
|
||||
look(&p, 11, token::Eof);
|
||||
look(&p, 100, token::Eof);
|
||||
|
||||
// Move forward to the `;`.
|
||||
for _ in 0..9 {
|
||||
p.bump();
|
||||
}
|
||||
look!(p, 0, token::Semi);
|
||||
look(&p, 0, token::Semi);
|
||||
// Any lookahead past the end of the token stream returns `Eof`.
|
||||
look!(p, 1, token::Eof);
|
||||
look!(p, 100, token::Eof);
|
||||
look(&p, 1, token::Eof);
|
||||
look(&p, 100, token::Eof);
|
||||
|
||||
// Move one past the `;`, i.e. past the end of the token stream.
|
||||
p.bump();
|
||||
look!(p, 0, token::Eof);
|
||||
look!(p, 1, token::Eof);
|
||||
look!(p, 100, token::Eof);
|
||||
look(&p, 0, token::Eof);
|
||||
look(&p, 1, token::Eof);
|
||||
look(&p, 100, token::Eof);
|
||||
|
||||
// Bumping after Eof is idempotent.
|
||||
p.bump();
|
||||
look!(p, 0, token::Eof);
|
||||
look!(p, 1, token::Eof);
|
||||
look!(p, 100, token::Eof);
|
||||
look(&p, 0, token::Eof);
|
||||
look(&p, 1, token::Eof);
|
||||
look(&p, 100, token::Eof);
|
||||
});
|
||||
}
|
||||
|
||||
@ -1476,24 +1477,261 @@ fn look_ahead_non_outermost_stream() {
|
||||
for _ in 0..3 {
|
||||
p.bump();
|
||||
}
|
||||
look!(p, 0, token::Ident(kw::Fn, raw_no));
|
||||
look!(p, 1, token::Ident(sym_f, raw_no));
|
||||
look!(p, 2, token::OpenDelim(Delimiter::Parenthesis));
|
||||
look!(p, 3, token::Ident(sym_x, raw_no));
|
||||
look!(p, 4, token::Colon);
|
||||
look!(p, 5, token::Ident(sym::u32, raw_no));
|
||||
look!(p, 6, token::CloseDelim(Delimiter::Parenthesis));
|
||||
look!(p, 7, token::OpenDelim(Delimiter::Brace));
|
||||
look!(p, 8, token::Ident(sym_x, raw_no));
|
||||
look!(p, 9, token::CloseDelim(Delimiter::Brace));
|
||||
look!(p, 10, token::Ident(kw::Struct, raw_no));
|
||||
look!(p, 11, token::Ident(sym_S, raw_no));
|
||||
look!(p, 12, token::Semi);
|
||||
look!(p, 13, token::CloseDelim(Delimiter::Brace));
|
||||
look(&p, 0, token::Ident(kw::Fn, raw_no));
|
||||
look(&p, 1, token::Ident(sym_f, raw_no));
|
||||
look(&p, 2, token::OpenDelim(Delimiter::Parenthesis));
|
||||
look(&p, 3, token::Ident(sym_x, raw_no));
|
||||
look(&p, 4, token::Colon);
|
||||
look(&p, 5, token::Ident(sym::u32, raw_no));
|
||||
look(&p, 6, token::CloseDelim(Delimiter::Parenthesis));
|
||||
look(&p, 7, token::OpenDelim(Delimiter::Brace));
|
||||
look(&p, 8, token::Ident(sym_x, raw_no));
|
||||
look(&p, 9, token::CloseDelim(Delimiter::Brace));
|
||||
look(&p, 10, token::Ident(kw::Struct, raw_no));
|
||||
look(&p, 11, token::Ident(sym_S, raw_no));
|
||||
look(&p, 12, token::Semi);
|
||||
look(&p, 13, token::CloseDelim(Delimiter::Brace));
|
||||
// Any lookahead past the end of the token stream returns `Eof`.
|
||||
look!(p, 14, token::Eof);
|
||||
look!(p, 15, token::Eof);
|
||||
look!(p, 100, token::Eof);
|
||||
look(&p, 14, token::Eof);
|
||||
look(&p, 15, token::Eof);
|
||||
look(&p, 100, token::Eof);
|
||||
});
|
||||
}
|
||||
|
||||
// FIXME(nnethercote) All the output is currently wrong.
|
||||
#[test]
|
||||
fn debug_lookahead() {
|
||||
create_default_session_globals_then(|| {
|
||||
let psess = psess();
|
||||
let mut p = string_to_parser(&psess, "fn f(x: u32) { x } struct S;".to_string());
|
||||
|
||||
// Current position is the `fn`.
|
||||
assert_eq!(
|
||||
&format!("{:#?}", p.debug_lookahead(0)),
|
||||
"Parser {
|
||||
prev_token: Token {
|
||||
kind: Question,
|
||||
span: Span {
|
||||
lo: BytePos(
|
||||
0,
|
||||
),
|
||||
hi: BytePos(
|
||||
0,
|
||||
),
|
||||
ctxt: #0,
|
||||
},
|
||||
},
|
||||
tokens: [],
|
||||
approx_token_stream_pos: 1,
|
||||
..
|
||||
}"
|
||||
);
|
||||
assert_eq!(
|
||||
&format!("{:#?}", p.debug_lookahead(7)),
|
||||
"Parser {
|
||||
prev_token: Token {
|
||||
kind: Question,
|
||||
span: Span {
|
||||
lo: BytePos(
|
||||
0,
|
||||
),
|
||||
hi: BytePos(
|
||||
0,
|
||||
),
|
||||
ctxt: #0,
|
||||
},
|
||||
},
|
||||
tokens: [
|
||||
Ident(
|
||||
\"fn\",
|
||||
No,
|
||||
),
|
||||
Ident(
|
||||
\"f\",
|
||||
No,
|
||||
),
|
||||
OpenDelim(
|
||||
Parenthesis,
|
||||
),
|
||||
Ident(
|
||||
\"x\",
|
||||
No,
|
||||
),
|
||||
Colon,
|
||||
Ident(
|
||||
\"u32\",
|
||||
No,
|
||||
),
|
||||
CloseDelim(
|
||||
Parenthesis,
|
||||
),
|
||||
],
|
||||
approx_token_stream_pos: 1,
|
||||
..
|
||||
}"
|
||||
);
|
||||
// There are 13 tokens. We request 15, get 14; the last one is `Eof`.
|
||||
assert_eq!(
|
||||
&format!("{:#?}", p.debug_lookahead(15)),
|
||||
"Parser {
|
||||
prev_token: Token {
|
||||
kind: Question,
|
||||
span: Span {
|
||||
lo: BytePos(
|
||||
0,
|
||||
),
|
||||
hi: BytePos(
|
||||
0,
|
||||
),
|
||||
ctxt: #0,
|
||||
},
|
||||
},
|
||||
tokens: [
|
||||
Ident(
|
||||
\"fn\",
|
||||
No,
|
||||
),
|
||||
Ident(
|
||||
\"f\",
|
||||
No,
|
||||
),
|
||||
OpenDelim(
|
||||
Parenthesis,
|
||||
),
|
||||
Ident(
|
||||
\"x\",
|
||||
No,
|
||||
),
|
||||
Colon,
|
||||
Ident(
|
||||
\"u32\",
|
||||
No,
|
||||
),
|
||||
CloseDelim(
|
||||
Parenthesis,
|
||||
),
|
||||
OpenDelim(
|
||||
Brace,
|
||||
),
|
||||
Ident(
|
||||
\"x\",
|
||||
No,
|
||||
),
|
||||
CloseDelim(
|
||||
Brace,
|
||||
),
|
||||
Ident(
|
||||
\"struct\",
|
||||
No,
|
||||
),
|
||||
Ident(
|
||||
\"S\",
|
||||
No,
|
||||
),
|
||||
Semi,
|
||||
Eof,
|
||||
],
|
||||
approx_token_stream_pos: 1,
|
||||
..
|
||||
}"
|
||||
);
|
||||
|
||||
// Move forward to the second `x`.
|
||||
for _ in 0..8 {
|
||||
p.bump();
|
||||
}
|
||||
assert_eq!(
|
||||
&format!("{:#?}", p.debug_lookahead(1)),
|
||||
"Parser {
|
||||
prev_token: Token {
|
||||
kind: OpenDelim(
|
||||
Brace,
|
||||
),
|
||||
span: Span {
|
||||
lo: BytePos(
|
||||
13,
|
||||
),
|
||||
hi: BytePos(
|
||||
14,
|
||||
),
|
||||
ctxt: #0,
|
||||
},
|
||||
},
|
||||
tokens: [
|
||||
Ident(
|
||||
\"x\",
|
||||
No,
|
||||
),
|
||||
],
|
||||
approx_token_stream_pos: 9,
|
||||
..
|
||||
}"
|
||||
);
|
||||
assert_eq!(
|
||||
&format!("{:#?}", p.debug_lookahead(4)),
|
||||
"Parser {
|
||||
prev_token: Token {
|
||||
kind: OpenDelim(
|
||||
Brace,
|
||||
),
|
||||
span: Span {
|
||||
lo: BytePos(
|
||||
13,
|
||||
),
|
||||
hi: BytePos(
|
||||
14,
|
||||
),
|
||||
ctxt: #0,
|
||||
},
|
||||
},
|
||||
tokens: [
|
||||
Ident(
|
||||
\"x\",
|
||||
No,
|
||||
),
|
||||
CloseDelim(
|
||||
Brace,
|
||||
),
|
||||
Ident(
|
||||
\"struct\",
|
||||
No,
|
||||
),
|
||||
Ident(
|
||||
\"S\",
|
||||
No,
|
||||
),
|
||||
],
|
||||
approx_token_stream_pos: 9,
|
||||
..
|
||||
}"
|
||||
);
|
||||
|
||||
// Move two past the final token (the `;`).
|
||||
for _ in 0..6 {
|
||||
p.bump();
|
||||
}
|
||||
assert_eq!(
|
||||
&format!("{:#?}", p.debug_lookahead(3)),
|
||||
"Parser {
|
||||
prev_token: Token {
|
||||
kind: Eof,
|
||||
span: Span {
|
||||
lo: BytePos(
|
||||
27,
|
||||
),
|
||||
hi: BytePos(
|
||||
28,
|
||||
),
|
||||
ctxt: #0,
|
||||
},
|
||||
},
|
||||
tokens: [
|
||||
Eof,
|
||||
],
|
||||
approx_token_stream_pos: 15,
|
||||
..
|
||||
}"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user