Merge #5930

5930: Migrate to the latest Semantic Tokens Proposal for LSP 3.16 r=matklad a=kjeremy

This stabilizes call hierarchy and semantic tokens features on the client side and changes the server-side semantic tokens protocol to match the latest proposal for 3.16.

The server-side change will break clients depending on the earlier semantic tokens draft.

Fixes #4942 

Co-authored-by: kjeremy <kjeremy@gmail.com>

Cargo.lock

@@ -778,9 +778,9 @@ dependencies = [
 
 [[package]]
 name = "lsp-types"
-version = "0.79.0"
+version = "0.80.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f1f86677fdbe8df5f88b99131b1424e50aad27bbe3e5900d221bc414bd72e9b"
+checksum = "f4265e2715bdacbb4dad029fce525e420cd66dc0af24ff9cb996a8ab48ac92ef"
 dependencies = [
  "base64",
  "bitflags",

crates/rust-analyzer/Cargo.toml

@@ -21,7 +21,7 @@ env_logger = { version = "0.7.1", default-features = false }
 itertools = "0.9.0"
 jod-thread = "0.1.0"
 log = "0.4.8"
-lsp-types = { version = "0.79.0", features = ["proposed"] }
+lsp-types = { version = "0.80.0", features = ["proposed"] }
 parking_lot = "0.11.0"
 pico-args = "0.3.1"
 oorandom = "11.1.2"

crates/rust-analyzer/src/caps.rs

@@ -6,7 +6,7 @@
     CodeActionProviderCapability, CodeLensOptions, CompletionOptions,
     DocumentOnTypeFormattingOptions, FoldingRangeProviderCapability, HoverProviderCapability,
     ImplementationProviderCapability, RenameOptions, RenameProviderCapability, SaveOptions,
-    SelectionRangeProviderCapability, SemanticTokensDocumentProvider, SemanticTokensLegend,
+    SelectionRangeProviderCapability, SemanticTokensFullOptions, SemanticTokensLegend,
     SemanticTokensOptions, ServerCapabilities, SignatureHelpOptions, TextDocumentSyncCapability,
     TextDocumentSyncKind, TextDocumentSyncOptions, TypeDefinitionProviderCapability,
     WorkDoneProgressOptions,
@@ -76,10 +76,8 @@ pub fn server_capabilities(client_caps: &ClientCapabilities) -> ServerCapabiliti
                     token_modifiers: semantic_tokens::SUPPORTED_MODIFIERS.to_vec(),
                 },
 
-                document_provider: Some(SemanticTokensDocumentProvider::Edits {
+                full: Some(SemanticTokensFullOptions::Delta { delta: Some(true) }),
-                    edits: Some(true),
+                range: Some(true),
-                }),
-                range_provider: Some(true),
                 work_done_progress_options: Default::default(),
             }
             .into(),

crates/rust-analyzer/src/handlers.rs

@@ -17,8 +17,8 @@
     CallHierarchyOutgoingCall, CallHierarchyOutgoingCallsParams, CallHierarchyPrepareParams,
     CodeActionKind, CodeLens, Command, CompletionItem, Diagnostic, DocumentFormattingParams,
     DocumentHighlight, DocumentSymbol, FoldingRange, FoldingRangeParams, HoverContents, Location,
-    Position, PrepareRenameResponse, Range, RenameParams, SemanticTokensEditResult,
+    Position, PrepareRenameResponse, Range, RenameParams, SemanticTokensDeltaParams,
-    SemanticTokensEditsParams, SemanticTokensParams, SemanticTokensRangeParams,
+    SemanticTokensFullDeltaResult, SemanticTokensParams, SemanticTokensRangeParams,
     SemanticTokensRangeResult, SemanticTokensResult, SymbolInformation, SymbolTag,
     TextDocumentIdentifier, Url, WorkspaceEdit,
 };
@@ -1171,11 +1171,11 @@ pub(crate) fn handle_call_hierarchy_outgoing(
     Ok(Some(res))
 }
 
-pub(crate) fn handle_semantic_tokens(
+pub(crate) fn handle_semantic_tokens_full(
     snap: GlobalStateSnapshot,
     params: SemanticTokensParams,
 ) -> Result<Option<SemanticTokensResult>> {
-    let _p = profile::span("handle_semantic_tokens");
+    let _p = profile::span("handle_semantic_tokens_full");
 
     let file_id = from_proto::file_id(&snap, &params.text_document.uri)?;
     let text = snap.analysis.file_text(file_id)?;
@@ -1190,11 +1190,11 @@ pub(crate) fn handle_semantic_tokens(
     Ok(Some(semantic_tokens.into()))
 }
 
-pub(crate) fn handle_semantic_tokens_edits(
+pub(crate) fn handle_semantic_tokens_full_delta(
     snap: GlobalStateSnapshot,
-    params: SemanticTokensEditsParams,
+    params: SemanticTokensDeltaParams,
-) -> Result<Option<SemanticTokensEditResult>> {
+) -> Result<Option<SemanticTokensFullDeltaResult>> {
-    let _p = profile::span("handle_semantic_tokens_edits");
+    let _p = profile::span("handle_semantic_tokens_full_delta");
 
     let file_id = from_proto::file_id(&snap, &params.text_document.uri)?;
     let text = snap.analysis.file_text(file_id)?;
@@ -1209,9 +1209,9 @@ pub(crate) fn handle_semantic_tokens_edits(
 
     if let Some(prev_id) = &cached_tokens.result_id {
         if *prev_id == params.previous_result_id {
-            let edits = to_proto::semantic_token_edits(&cached_tokens, &semantic_tokens);
+            let delta = to_proto::semantic_token_delta(&cached_tokens, &semantic_tokens);
             *cached_tokens = semantic_tokens;
-            return Ok(Some(edits.into()));
+            return Ok(Some(delta.into()));
         }
     }
 

crates/rust-analyzer/src/main_loop.rs

@@ -407,9 +407,11 @@ fn on_request(&mut self, request_received: Instant, req: Request) -> Result<()>
             .on::<lsp_types::request::CallHierarchyOutgoingCalls>(
                 handlers::handle_call_hierarchy_outgoing,
             )?
-            .on::<lsp_types::request::SemanticTokensRequest>(handlers::handle_semantic_tokens)?
+            .on::<lsp_types::request::SemanticTokensFullRequest>(
-            .on::<lsp_types::request::SemanticTokensEditsRequest>(
+                handlers::handle_semantic_tokens_full,
-                handlers::handle_semantic_tokens_edits,
+            )?
+            .on::<lsp_types::request::SemanticTokensFullDeltaRequest>(
+                handlers::handle_semantic_tokens_full_delta,
             )?
             .on::<lsp_types::request::SemanticTokensRangeRequest>(
                 handlers::handle_semantic_tokens_range,

crates/rust-analyzer/src/semantic_tokens.rs

@@ -24,6 +24,7 @@ macro_rules! define_semantic_token_types {
             SemanticTokenType::CLASS,
             SemanticTokenType::INTERFACE,
             SemanticTokenType::ENUM,
+            SemanticTokenType::ENUM_MEMBER,
             SemanticTokenType::TYPE_PARAMETER,
             SemanticTokenType::FUNCTION,
             SemanticTokenType::MEMBER,
@@ -31,7 +32,6 @@ macro_rules! define_semantic_token_types {
             SemanticTokenType::MACRO,
             SemanticTokenType::VARIABLE,
             SemanticTokenType::PARAMETER,
-            SemanticTokenType::LABEL,
             $($ident),*
         ];
     };
@@ -41,7 +41,6 @@ macro_rules! define_semantic_token_types {
     (ATTRIBUTE, "attribute"),
     (BOOLEAN, "boolean"),
     (BUILTIN_TYPE, "builtinType"),
-    (ENUM_MEMBER, "enumMember"),
     (ESCAPE_SEQUENCE, "escapeSequence"),
     (FORMAT_SPECIFIER, "formatSpecifier"),
     (GENERIC, "generic"),

crates/rust-analyzer/src/to_proto.rs

@@ -334,13 +334,13 @@ pub(crate) fn semantic_tokens(
     builder.build()
 }
 
-pub(crate) fn semantic_token_edits(
+pub(crate) fn semantic_token_delta(
     previous: &lsp_types::SemanticTokens,
     current: &lsp_types::SemanticTokens,
-) -> lsp_types::SemanticTokensEdits {
+) -> lsp_types::SemanticTokensDelta {
     let result_id = current.result_id.clone();
     let edits = semantic_tokens::diff_tokens(&previous.data, &current.data);
-    lsp_types::SemanticTokensEdits { result_id, edits }
+    lsp_types::SemanticTokensDelta { result_id, edits }
 }
 
 fn semantic_token_type_and_modifiers(
@@ -369,7 +369,7 @@ fn semantic_token_type_and_modifiers(
             mods |= lsp_types::SemanticTokenModifier::STATIC;
             lsp_types::SemanticTokenType::VARIABLE
         }
-        HighlightTag::EnumVariant => semantic_tokens::ENUM_MEMBER,
+        HighlightTag::EnumVariant => lsp_types::SemanticTokenType::ENUM_MEMBER,
         HighlightTag::Macro => lsp_types::SemanticTokenType::MACRO,
         HighlightTag::ValueParam => lsp_types::SemanticTokenType::PARAMETER,
         HighlightTag::Local => lsp_types::SemanticTokenType::VARIABLE,

editors/code/package-lock.json

@@ -2409,32 +2409,32 @@
             }
         },
         "vscode-jsonrpc": {
-            "version": "5.1.0-next.1",
+            "version": "6.0.0-next.5",
-            "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-5.1.0-next.1.tgz",
+            "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0-next.5.tgz",
-            "integrity": "sha512-mwLDojZkbmpizSJSmp690oa9FB9jig18SIDGZeBCvFc2/LYSRvMm/WwWtMBJuJ1MfFh7rZXfQige4Uje5Z9NzA=="
+            "integrity": "sha512-IAgsltQPwg/pXOPsdXgbUTCaO9VSKZwirZN5SGtkdYQ/R3VjeC4v00WTVvoNayWMZpoC3O9u0ogqmsKzKhVasQ=="
         },
         "vscode-languageclient": {
-            "version": "7.0.0-next.1",
+            "version": "7.0.0-next.9",
-            "resolved": "https://registry.npmjs.org/vscode-languageclient/-/vscode-languageclient-7.0.0-next.1.tgz",
+            "resolved": "https://registry.npmjs.org/vscode-languageclient/-/vscode-languageclient-7.0.0-next.9.tgz",
-            "integrity": "sha512-JrjCUhLpQZxQ5VpWpilOHDMhVsn0fdN5jBh1uFNhSr5c2loJvRdr9Km2EuSQOFfOQsBKx0+xvY8PbsypNEcJ6w==",
+            "integrity": "sha512-lFO+rN/i72CM2va6iKXq1lD7pJg8J93KEXf0w0boWVqU+DJhWzLrV3pXl8Xk1nCv//qOAyhlc/nx2KZCTeRF/A==",
             "requires": {
                 "semver": "^6.3.0",
-                "vscode-languageserver-protocol": "3.16.0-next.2"
+                "vscode-languageserver-protocol": "3.16.0-next.7"
             }
         },
         "vscode-languageserver-protocol": {
-            "version": "3.16.0-next.2",
+            "version": "3.16.0-next.7",
-            "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.16.0-next.2.tgz",
+            "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.16.0-next.7.tgz",
-            "integrity": "sha512-atmkGT/W6tF0cx4SaWFYtFs2UeSeC28RPiap9myv2YZTaTCFvTBEPNWrU5QRKfkyM0tbgtGo6T3UCQ8tkDpjzA==",
+            "integrity": "sha512-tOjrg+K3RddJ547zpC9/LAgTbzadkPuHlqJFFWIcKjVhiJOh73XyY+Ngcu9wukGaTsuSGjJ0W8rlmwanixa0FQ==",
             "requires": {
-                "vscode-jsonrpc": "5.1.0-next.1",
+                "vscode-jsonrpc": "6.0.0-next.5",
-                "vscode-languageserver-types": "3.16.0-next.1"
+                "vscode-languageserver-types": "3.16.0-next.3"
             }
         },
         "vscode-languageserver-types": {
-            "version": "3.16.0-next.1",
+            "version": "3.16.0-next.3",
-            "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0-next.1.tgz",
+            "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0-next.3.tgz",
-            "integrity": "sha512-tZFUSbyjUcrh+qQf13ALX4QDdOfDX0cVaBFgy7ktJ0VwS7AW/yRKgGPSxVqqP9OCMNPdqP57O5q47w2pEwfaUg=="
+            "integrity": "sha512-s/z5ZqSe7VpoXJ6JQcvwRiPPA3nG0nAcJ/HH03zoU6QaFfnkcgPK+HshC3WKPPnC2G08xA0iRB6h7kmyBB5Adg=="
         },
         "vscode-test": {
             "version": "1.4.0",

editors/code/package.json

@@ -36,7 +36,7 @@
     },
     "dependencies": {
         "node-fetch": "^2.6.0",
-        "vscode-languageclient": "7.0.0-next.1"
+        "vscode-languageclient": "7.0.0-next.9"
     },
     "devDependencies": {
         "@rollup/plugin-commonjs": "^13.0.2",

editors/code/src/client.ts

@@ -1,10 +1,7 @@
-import * as lc from 'vscode-languageclient';
+import * as lc from 'vscode-languageclient/node';
 import * as vscode from 'vscode';
 import * as ra from '../src/lsp_ext';
-import * as Is from 'vscode-languageclient/lib/utils/is';
+import * as Is from 'vscode-languageclient/lib/common/utils/is';
-
-import { CallHierarchyFeature } from 'vscode-languageclient/lib/callHierarchy.proposed';
-import { SemanticTokensFeature } from 'vscode-languageclient/lib/semanticTokens.proposed';
 import { assert } from './util';
 
 function renderCommand(cmd: ra.CommandLink) {
@@ -57,7 +54,7 @@ export function createClient(serverPath: string, cwd: string): lc.LanguageClient
                         return hover;
                     },
                     (error) => {
-                        client.logFailedRequest(lc.HoverRequest.type, error);
+                        client.handleFailedRequest(lc.HoverRequest.type, error, null);
                         return Promise.resolve(null);
                     });
             },
@@ -140,12 +137,6 @@ export function createClient(serverPath: string, cwd: string): lc.LanguageClient
     );
 
     // To turn on all proposed features use: client.registerProposedFeatures();
-    // Here we want to enable CallHierarchyFeature and SemanticTokensFeature
-    // since they are available on stable.
-    // Note that while these features are stable in vscode their LSP protocol
-    // implementations are still in the "proposed" category for 3.16.
-    client.registerFeature(new CallHierarchyFeature(client));
-    client.registerFeature(new SemanticTokensFeature(client));
     client.registerFeature(new ExperimentalFeatures());
 
     return client;

editors/code/src/commands.ts

@@ -63,7 +63,7 @@ export function memoryUsage(ctx: Ctx): Cmd {
         provideTextDocumentContent(_uri: vscode.Uri): vscode.ProviderResult<string> {
             if (!vscode.window.activeTextEditor) return '';
 
-            return ctx.client.sendRequest(ra.memoryUsage, null).then((mem) => {
+            return ctx.client.sendRequest(ra.memoryUsage, null).then((mem: any) => {
                 return 'Per-query memory usage:\n' + mem + '\n(note: database has been cleared)';
             });
         }
@@ -121,7 +121,7 @@ export function joinLines(ctx: Ctx): Cmd {
             textDocument: { uri: editor.document.uri.toString() },
         });
         editor.edit((builder) => {
-            client.protocol2CodeConverter.asTextEdits(items).forEach((edit) => {
+            client.protocol2CodeConverter.asTextEdits(items).forEach((edit: any) => {
                 builder.replace(edit.range, edit.newText);
             });
         });
@@ -140,8 +140,8 @@ export function onEnter(ctx: Ctx): Cmd {
             position: client.code2ProtocolConverter.asPosition(
                 editor.selection.active,
             ),
-        }).catch(_error => {
+        }).catch((_error: any) => {
-            // client.logFailedRequest(OnEnterRequest.type, error);
+            // client.handleFailedRequest(OnEnterRequest.type, error, null);
             return null;
         });
         if (!lcEdits) return false;

editors/code/src/ctx.ts

@@ -1,5 +1,5 @@
 import * as vscode from 'vscode';
-import * as lc from 'vscode-languageclient';
+import * as lc from 'vscode-languageclient/node';
 import * as ra from './lsp_ext';
 
 import { Config } from './config';

editors/code/src/util.ts

@@ -1,4 +1,4 @@
-import * as lc from "vscode-languageclient";
+import * as lc from "vscode-languageclient/node";
 import * as vscode from "vscode";
 import { strict as nativeAssert } from "assert";
 import { spawnSync } from "child_process";