diff --git a/Cargo.lock b/Cargo.lock index 007f05b4d03..a511e0d2804 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,9 +1398,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_lexer" -version = "656.0.0" +version = "660.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cbba98ec46e96a4663197dfa8c0378752de2006e314e5400c0ca74929d6692f" +checksum = "30760dbcc7667c9e0da561e980e24867ca7f4526ce060a3d7e6d9dcfeaae88d1" dependencies = [ "unicode-xid", ] diff --git a/crates/ra_syntax/Cargo.toml b/crates/ra_syntax/Cargo.toml index c07ff488e4f..a9a5cc7bc6e 100644 --- a/crates/ra_syntax/Cargo.toml +++ b/crates/ra_syntax/Cargo.toml @@ -13,7 +13,7 @@ doctest = false [dependencies] itertools = "0.9.0" rowan = "0.10.0" -rustc_lexer = { version = "656.0.0", package = "rustc-ap-rustc_lexer" } +rustc_lexer = { version = "660.0.0", package = "rustc-ap-rustc_lexer" } rustc-hash = "1.1.0" arrayvec = "0.5.1" once_cell = "1.3.1" diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs index 74906d8a62e..3cd6d99c388 100644 --- a/crates/ra_syntax/src/ast/tokens.rs +++ b/crates/ra_syntax/src/ast/tokens.rs @@ -6,6 +6,7 @@ ast::{AstToken, Comment, RawString, String, Whitespace}, TextRange, TextSize, }; +use rustc_lexer::unescape::{unescape_literal, Mode}; impl Comment { pub fn kind(&self) -> CommentKind { @@ -147,7 +148,7 @@ fn value(&self) -> Option { let mut buf = std::string::String::with_capacity(text.len()); let mut has_error = false; - rustc_lexer::unescape::unescape_str(text, &mut |_, unescaped_char| match unescaped_char { + unescape_literal(text, Mode::Str, &mut |_, unescaped_char| match unescaped_char { Ok(c) => buf.push(c), Err(_) => has_error = true, }); @@ -498,7 +499,7 @@ fn char_ranges( let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start(); let mut res = Vec::with_capacity(text.len()); - rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| { + unescape_literal(text, Mode::Str, &mut |range, unescaped_char| { res.push(( TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap()) + offset, diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs index d68cf0a82bf..fdec48fb0b8 100644 --- a/crates/ra_syntax/src/validation.rs +++ b/crates/ra_syntax/src/validation.rs @@ -2,15 +2,15 @@ mod block; -use std::convert::TryFrom; - -use rustc_lexer::unescape; - use crate::{ ast, match_ast, AstNode, SyntaxError, SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST_DEF, FN_DEF, INT_NUMBER, STRING, TYPE_ALIAS_DEF}, SyntaxNode, SyntaxToken, TextSize, T, }; +use rustc_lexer::unescape::{ + self, unescape_byte, unescape_byte_literal, unescape_char, unescape_literal, Mode, +}; +use std::convert::TryFrom; fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str { use unescape::EscapeError as EE; @@ -81,10 +81,8 @@ fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str { pub(crate) fn validate(root: &SyntaxNode) -> Vec { // FIXME: - // * Add validation of character literal containing only a single char - // * Add validation of `crate` keyword not appearing in the middle of the symbol path + // * Add unescape validation of raw string literals and raw byte string literals // * Add validation of doc comments are being attached to nodes - // * Remove validation of unterminated literals (it is already implemented in `tokenize()`) let mut errors = Vec::new(); for node in root.descendants() { @@ -121,18 +119,18 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> { match token.kind() { BYTE => { - if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape::unescape_byte) { + if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) { push_err(2, e); } } CHAR => { - if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape::unescape_char) { + if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) { push_err(1, e); } } BYTE_STRING => { if let Some(without_quotes) = unquote(text, 2, '"') { - unescape::unescape_byte_str(without_quotes, &mut |range, char| { + unescape_byte_literal(without_quotes, Mode::ByteStr, &mut |range, char| { if let Err(err) = char { push_err(2, (range.start, err)); } @@ -141,7 +139,7 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> { } STRING => { if let Some(without_quotes) = unquote(text, 1, '"') { - unescape::unescape_str(without_quotes, &mut |range, char| { + unescape_literal(without_quotes, Mode::Str, &mut |range, char| { if let Err(err) = char { push_err(1, (range.start, err)); }