mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

internal: Bump rustc_lexer

Browse Source
This commit is contained in:
Lukas Wirth 2023-05-18 10:17:40 +02:00
parent 0a806fe7ad
commit 099b5b3b15
26 changed files with 134 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Cargo.lock generated
View File

@ -1193,7 +1193,7 @@ dependencies = [
"drop_bomb",
"expect-test",
"limit",
"rustc-ap-rustc_lexer",
"ra-ap-rustc_lexer",
"sourcegen",
"stdx",
]
@ -1396,6 +1396,16 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "ra-ap-rustc_lexer"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1c145702ed3f237918e512685185dc8a4d0edc3a5326c63d20361d8ba9b45b3"
dependencies = [
"unic-emoji-char",
"unicode-xid",
]
[[package]]
name = "rayon"
version = "1.7.0"
@ -1524,15 +1534,6 @@ dependencies = [
"xshell",
]
[[package]]
name = "rustc-ap-rustc_lexer"
version = "727.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f40f26e7abdcd3b982f36c09a634cc6187988fbf6ec466c91f8d30a12ac0237"
dependencies = [
"unicode-xid",
]
[[package]]
name = "rustc-demangle"
version = "0.1.22"
@ -1753,9 +1754,9 @@ dependencies = [
"proc-macro2",
"profile",
"quote",
"ra-ap-rustc_lexer",
"rayon",
"rowan",
"rustc-ap-rustc_lexer",
"rustc-hash",
"smol_str",
"sourcegen",
@ -1998,6 +1999,47 @@ version = "1.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e5df347f0bf3ec1d670aad6ca5c6a1859cd9ea61d2113125794654ccced68f"
[[package]]
name = "unic-char-property"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
dependencies = [
"unic-char-range",
]
[[package]]
name = "unic-char-range"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
[[package]]
name = "unic-common"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
[[package]]
name = "unic-emoji-char"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
dependencies = [
"unic-char-property",
"unic-char-range",
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-version"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
dependencies = [
"unic-common",
]
[[package]]
name = "unicase"
version = "2.6.0"

2
Cargo.toml
View File

@ -89,3 +89,5 @@ text-size = "1.1.0"
serde = { version = "=1.0.156", features = ["derive"] }
serde_json = "1.0.94"
triomphe = { version = "0.1.8", default-features = false, features = ["std"] }
rustc_lexer = { version = "0.1.0", package = "ra-ap-rustc_lexer" }

2
crates/parser/Cargo.toml
View File

@ -13,7 +13,7 @@ doctest = false
[dependencies]
drop_bomb = "0.1.5"
rustc_lexer = { version = "727.0.0", package = "rustc-ap-rustc_lexer" }
rustc_lexer.workspace = true
limit.workspace = true

56
crates/parser/src/lexed_str.rs
View File

@ -36,7 +36,7 @@ pub fn new(text: &'a str) -> LexedStr<'a> {
};
for token in rustc_lexer::tokenize(&text[conv.offset..]) {
let token_text = &text[conv.offset..][..token.len];
let token_text = &text[conv.offset..][..token.len as usize];
conv.extend_token(&token.kind, token_text);
}
@ -49,8 +49,8 @@ pub fn single_token(text: &'a str) -> Option<(SyntaxKind, Option<String>)> {
return None;
}
let token = rustc_lexer::first_token(text);
if token.len != text.len() {
let token = rustc_lexer::tokenize(text).next()?;
if token.len as usize != text.len() {
return None;
}
@ -175,6 +175,10 @@ fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, token_text: &str) {
rustc_lexer::TokenKind::Ident => {
SyntaxKind::from_keyword(token_text).unwrap_or(IDENT)
}
rustc_lexer::TokenKind::InvalidIdent => {
err = "Ident contains invalid characters";
IDENT
}
rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => {
@ -221,6 +225,7 @@ fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, token_text: &str) {
err = "unknown literal prefix";
IDENT
}
rustc_lexer::TokenKind::Eof => EOF,
}
};
@ -268,35 +273,30 @@ fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
}
BYTE_STRING
}
rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => {
if let Some(raw_str_err) = raw_str_err {
err = match raw_str_err {
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal",
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
"Missing trailing `\"` to terminate the raw string literal"
} else {
"Missing trailing `\"` with `#` symbols to terminate the raw string literal"
},
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
};
};
rustc_lexer::LiteralKind::CStr { terminated } => {
if !terminated {
err = "Missing trailing `\"` symbol to terminate the string literal";
}
STRING
}
rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => {
if let Some(raw_str_err) = raw_str_err {
err = match raw_str_err {
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal",
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
"Missing trailing `\"` to terminate the raw byte string literal"
} else {
"Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"
},
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols",
};
};
rustc_lexer::LiteralKind::RawStr { n_hashes } => {
if n_hashes.is_none() {
err = "Invalid raw string literal";
}
STRING
}
rustc_lexer::LiteralKind::RawByteStr { n_hashes } => {
if n_hashes.is_none() {
err = "Invalid raw string literal";
}
BYTE_STRING
}
rustc_lexer::LiteralKind::RawCStr { n_hashes } => {
if n_hashes.is_none() {
err = "Invalid raw string literal";
}
STRING
}
};
let err = if err.is_empty() { None } else { Some(err) };

3
crates/parser/src/syntax_kind/generated.rs
View File

@ -117,6 +117,7 @@ pub enum SyntaxKind {
BYTE,
STRING,
BYTE_STRING,
C_STRING,
ERROR,
IDENT,
WHITESPACE,
@ -379,7 +380,7 @@ pub fn is_punct(self) -> bool {
)
}
pub fn is_literal(self) -> bool {
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING)
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING | C_STRING)
}
pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
let kw = match ident {

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.rast
View File

@ -1 +1 @@
BYTE_STRING "br##\"" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.rast
View File

@ -1 +1 @@
BYTE_STRING "br##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\x7f" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.rast
View File

@ -1 +1 @@
BYTE_STRING "br##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"🦀" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.rast
View File

@ -1 +1 @@
BYTE_STRING "br##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.rast
View File

@ -1 +1 @@
BYTE_STRING "br##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\n" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.rast
View File

@ -1 +1 @@
BYTE_STRING "br##\" " error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\" " error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.rast
View File

@ -1 +1 @@
BYTE_STRING "br##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\u{20AA}" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.rast
View File

@ -1 +1 @@
STRING "r##\"" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.rast
View File

@ -1 +1 @@
STRING "r##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\x7f" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.rast
View File

@ -1 +1 @@
STRING "r##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"🦀" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.rast
View File

@ -1 +1 @@
STRING "r##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.rast
View File

@ -1 +1 @@
STRING "r##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\n" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.rast
View File

@ -1 +1 @@
STRING "r##\" " error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\" " error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.rast
View File

@ -1 +1 @@
STRING "r##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\u{20AA}" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.rast
View File

@ -1 +1 @@
BYTE_STRING "br##" error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
BYTE_STRING "br##" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.rast
View File

@ -1,4 +1,4 @@
BYTE_STRING "br## " error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
BYTE_STRING "br## " error: Invalid raw string literal
IDENT "I"
WHITESPACE " "
IDENT "lack"

2
crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.rast
View File

@ -1 +1 @@
STRING "r##" error: Missing `"` symbol after `#` symbols to begin the raw string literal
STRING "r##" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.rast
View File

@ -1,4 +1,4 @@
STRING "r## " error: Missing `"` symbol after `#` symbols to begin the raw string literal
STRING "r## " error: Invalid raw string literal
IDENT "I"
WHITESPACE " "
IDENT "lack"

3
crates/syntax/Cargo.toml
View File

@ -17,13 +17,14 @@ cov-mark = "2.0.0-pre.1"
either = "1.7.0"
itertools = "0.10.5"
rowan = "0.15.11"
rustc_lexer = { version = "727.0.0", package = "rustc-ap-rustc_lexer" }
rustc-hash = "1.1.0"
once_cell = "1.17.0"
indexmap = "1.9.1"
smol_str.workspace = true
triomphe.workspace = true
rustc_lexer.workspace = true
parser.workspace = true
profile.workspace = true
stdx.workspace = true

2
crates/syntax/src/tests/ast_src.rs
View File

@ -71,7 +71,7 @@ pub(crate) struct KindsSrc<'a> {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules", "yeet"],
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING", "C_STRING"],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[
"SOURCE_FILE",

39
crates/syntax/src/validation.rs
View File

@ -5,7 +5,7 @@
mod block;
use rowan::Direction;
use rustc_lexer::unescape::{self, unescape_byte, unescape_char, unescape_literal, Mode};
use rustc_lexer::unescape::{self, unescape_literal, Mode};
use crate::{
algo,
@ -44,7 +44,7 @@ pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> {
errors
}
fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> (&'static str, bool) {
use unescape::EscapeError as EE;
#[rustfmt::skip]
@ -103,12 +103,15 @@ fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
EE::UnicodeEscapeInByte => {
"Byte literals must not contain unicode escapes"
}
EE::NonAsciiCharInByte | EE::NonAsciiCharInByteString => {
EE::NonAsciiCharInByte => {
"Byte literals must not contain non-ASCII characters"
}
EE::UnskippedWhitespaceWarning => "Whitespace after this escape is not skipped",
EE::MultipleSkippedLinesWarning => "Multiple lines are skipped by this escape",
};
err_message
(err_message, err.is_fatal())
}
fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
@ -121,9 +124,13 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> {
let text = token.text();
// FIXME: lift this lambda refactor to `fn` (https://github.com/rust-lang/rust-analyzer/pull/2834#discussion_r366199205)
let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {
let mut push_err = |prefix_len, off, err: unescape::EscapeError| {
let off = token.text_range().start() + TextSize::try_from(off + prefix_len).unwrap();
acc.push(SyntaxError::new_at_offset(rustc_unescape_error_to_string(err), off));
let (message, is_err) = rustc_unescape_error_to_string(err);
// FIXME: Emit lexer warnings
if is_err {
acc.push(SyntaxError::new_at_offset(message, off));
}
};
match literal.kind() {
@ -132,7 +139,7 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> {
if let Some(without_quotes) = unquote(text, 1, '"') {
unescape_literal(without_quotes, Mode::Str, &mut |range, char| {
if let Err(err) = char {
push_err(1, (range.start, err));
push_err(1, range.start, err);
}
});
}
@ -143,20 +150,28 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> {
if let Some(without_quotes) = unquote(text, 2, '"') {
unescape_literal(without_quotes, Mode::ByteStr, &mut |range, char| {
if let Err(err) = char {
push_err(2, (range.start, err));
push_err(1, range.start, err);
}
});
}
}
}
ast::LiteralKind::Char(_) => {
if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) {
push_err(1, e);
if let Some(without_quotes) = unquote(text, 1, '\'') {
unescape_literal(without_quotes, Mode::Char, &mut |range, char| {
if let Err(err) = char {
push_err(1, range.start, err);
}
});
}
}
ast::LiteralKind::Byte(_) => {
if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) {
push_err(2, e);
if let Some(without_quotes) = unquote(text, 2, '\'') {
unescape_literal(without_quotes, Mode::Byte, &mut |range, char| {
if let Err(err) = char {
push_err(2, range.start, err);
}
});
}
}
ast::LiteralKind::IntNumber(_)