Merge pull request #4590 from Julian-Wollersberger/update_unescape

Update to rustc_lexer version 660

Cargo.lock

@@ -1398,9 +1398,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-ap-rustc_lexer"
-version = "656.0.0"
+version = "660.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cbba98ec46e96a4663197dfa8c0378752de2006e314e5400c0ca74929d6692f"
+checksum = "30760dbcc7667c9e0da561e980e24867ca7f4526ce060a3d7e6d9dcfeaae88d1"
 dependencies = [
  "unicode-xid",
 ]

crates/ra_syntax/Cargo.toml

@@ -13,7 +13,7 @@ doctest = false
 [dependencies]
 itertools = "0.9.0"
 rowan = "0.10.0"
-rustc_lexer = { version = "656.0.0", package = "rustc-ap-rustc_lexer" }
+rustc_lexer = { version = "660.0.0", package = "rustc-ap-rustc_lexer" }
 rustc-hash = "1.1.0"
 arrayvec = "0.5.1"
 once_cell = "1.3.1"

crates/ra_syntax/src/ast/tokens.rs

@@ -6,6 +6,7 @@
     ast::{AstToken, Comment, RawString, String, Whitespace},
     TextRange, TextSize,
 };
+use rustc_lexer::unescape::{unescape_literal, Mode};
 
 impl Comment {
     pub fn kind(&self) -> CommentKind {
@@ -147,7 +148,7 @@ fn value(&self) -> Option<std::string::String> {
 
         let mut buf = std::string::String::with_capacity(text.len());
         let mut has_error = false;
-        rustc_lexer::unescape::unescape_str(text, &mut |_, unescaped_char| match unescaped_char {
+        unescape_literal(text, Mode::Str, &mut |_, unescaped_char| match unescaped_char {
             Ok(c) => buf.push(c),
             Err(_) => has_error = true,
         });
@@ -498,7 +499,7 @@ fn char_ranges(
         let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
 
         let mut res = Vec::with_capacity(text.len());
-        rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| {
+        unescape_literal(text, Mode::Str, &mut |range, unescaped_char| {
             res.push((
                 TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap())
                     + offset,

crates/ra_syntax/src/validation.rs

@@ -2,15 +2,15 @@
 
 mod block;
 
-use std::convert::TryFrom;
-
-use rustc_lexer::unescape;
-
 use crate::{
     ast, match_ast, AstNode, SyntaxError,
     SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST_DEF, FN_DEF, INT_NUMBER, STRING, TYPE_ALIAS_DEF},
     SyntaxNode, SyntaxToken, TextSize, T,
 };
+use rustc_lexer::unescape::{
+    self, unescape_byte, unescape_byte_literal, unescape_char, unescape_literal, Mode,
+};
+use std::convert::TryFrom;
 
 fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
     use unescape::EscapeError as EE;
@@ -81,10 +81,8 @@ fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
 
 pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> {
     // FIXME:
-    // * Add validation of character literal containing only a single char
-    // * Add validation of `crate` keyword not appearing in the middle of the symbol path
+    // * Add unescape validation of raw string literals and raw byte string literals
     // * Add validation of doc comments are being attached to nodes
-    // * Remove validation of unterminated literals (it is already implemented in `tokenize()`)
 
     let mut errors = Vec::new();
     for node in root.descendants() {
@@ -121,18 +119,18 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> {
 
     match token.kind() {
         BYTE => {
-            if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape::unescape_byte) {
+            if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) {
                 push_err(2, e);
             }
         }
         CHAR => {
-            if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape::unescape_char) {
+            if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) {
                 push_err(1, e);
             }
         }
         BYTE_STRING => {
             if let Some(without_quotes) = unquote(text, 2, '"') {
-                unescape::unescape_byte_str(without_quotes, &mut |range, char| {
+                unescape_byte_literal(without_quotes, Mode::ByteStr, &mut |range, char| {
                     if let Err(err) = char {
                         push_err(2, (range.start, err));
                     }
@@ -141,7 +139,7 @@ fn unquote(text: &str, prefix_len: usize, end_delimiter: char) -> Option<&str> {
         }
         STRING => {
             if let Some(without_quotes) = unquote(text, 1, '"') {
-                unescape::unescape_str(without_quotes, &mut |range, char| {
+                unescape_literal(without_quotes, Mode::Str, &mut |range, char| {
                     if let Err(err) = char {
                         push_err(1, (range.start, err));
                     }