[tintd] Implement lsp::TextDocumentSemanticTokensFullRequest Bug: tint:2127 Change-Id: Ibaf0ab66bbd09e491458e4f491f883ad7d06fe33 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/180481 Kokoro: Kokoro <noreply+kokoro@google.com> Commit-Queue: Ben Clayton <bclayton@google.com> Reviewed-by: David Neto <dneto@google.com>
diff --git a/src/tint/lang/wgsl/ls/BUILD.bazel b/src/tint/lang/wgsl/ls/BUILD.bazel index aed5fed..38fbe3f 100644 --- a/src/tint/lang/wgsl/ls/BUILD.bazel +++ b/src/tint/lang/wgsl/ls/BUILD.bazel
@@ -49,6 +49,7 @@ "initialize.cc", "references.cc", "rename.cc", + "sem_tokens.cc", "serve.cc", "server.cc", "set_trace.cc", @@ -56,6 +57,7 @@ ], hdrs = [ "file.h", + "sem_token.h", "serve.h", "server.h", "utils.h", @@ -111,6 +113,7 @@ "hover_test.cc", "references_test.cc", "rename_test.cc", + "sem_tokens_test.cc", "symbols_test.cc", ], deps = [
diff --git a/src/tint/lang/wgsl/ls/BUILD.cmake b/src/tint/lang/wgsl/ls/BUILD.cmake index 8ca125b..43bbd4b 100644 --- a/src/tint/lang/wgsl/ls/BUILD.cmake +++ b/src/tint/lang/wgsl/ls/BUILD.cmake
@@ -52,6 +52,8 @@ lang/wgsl/ls/initialize.cc lang/wgsl/ls/references.cc lang/wgsl/ls/rename.cc + lang/wgsl/ls/sem_token.h + lang/wgsl/ls/sem_tokens.cc lang/wgsl/ls/serve.cc lang/wgsl/ls/serve.h lang/wgsl/ls/server.cc @@ -119,6 +121,7 @@ lang/wgsl/ls/hover_test.cc lang/wgsl/ls/references_test.cc lang/wgsl/ls/rename_test.cc + lang/wgsl/ls/sem_tokens_test.cc lang/wgsl/ls/symbols_test.cc )
diff --git a/src/tint/lang/wgsl/ls/BUILD.gn b/src/tint/lang/wgsl/ls/BUILD.gn index 7d91ca4..89375fc 100644 --- a/src/tint/lang/wgsl/ls/BUILD.gn +++ b/src/tint/lang/wgsl/ls/BUILD.gn
@@ -55,6 +55,8 @@ "initialize.cc", "references.cc", "rename.cc", + "sem_token.h", + "sem_tokens.cc", "serve.cc", "serve.h", "server.cc", @@ -111,6 +113,7 @@ "hover_test.cc", "references_test.cc", "rename_test.cc", + "sem_tokens_test.cc", "symbols_test.cc", ] deps = [
diff --git a/src/tint/lang/wgsl/ls/sem_token.h b/src/tint/lang/wgsl/ls/sem_token.h new file mode 100644 index 0000000..6a10ccc --- /dev/null +++ b/src/tint/lang/wgsl/ls/sem_token.h
@@ -0,0 +1,52 @@ +// Copyright 2024 The Dawn & Tint Authors +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef SRC_TINT_LANG_WGSL_LS_SEM_TOKEN_H_ +#define SRC_TINT_LANG_WGSL_LS_SEM_TOKEN_H_ + +#include <array> + +namespace tint::wgsl::ls { + +/// SemToken is a struct used to hold an enumerator of token types, and their corresponding names. +struct SemToken { + enum Kind { + kEnum, + kEnumMember, + kFunction, + kMember, + kType, + kVariable, + }; + static constexpr std::array kNames{ + "enum", "enumMember", "function", "member", "type", "variable", + }; +}; + +} // namespace tint::wgsl::ls + +#endif // SRC_TINT_LANG_WGSL_LS_SEM_TOKEN_H_
diff --git a/src/tint/lang/wgsl/ls/sem_tokens.cc b/src/tint/lang/wgsl/ls/sem_tokens.cc new file mode 100644 index 0000000..3accd26 --- /dev/null +++ b/src/tint/lang/wgsl/ls/sem_tokens.cc
@@ -0,0 +1,141 @@ +// Copyright 2024 The Dawn & Tint Authors +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "langsvr/lsp/comparators.h" +#include "langsvr/lsp/lsp.h" +#include "langsvr/lsp/primitives.h" +#include "src/tint/lang/core/builtin_fn.h" +#include "src/tint/lang/wgsl/ast/identifier.h" +#include "src/tint/lang/wgsl/ast/identifier_expression.h" +#include "src/tint/lang/wgsl/ast/member_accessor_expression.h" +#include "src/tint/lang/wgsl/ast/struct.h" +#include "src/tint/lang/wgsl/ast/struct_member.h" +#include "src/tint/lang/wgsl/ast/type.h" +#include "src/tint/lang/wgsl/builtin_fn.h" +#include "src/tint/lang/wgsl/ls/sem_token.h" +#include "src/tint/lang/wgsl/ls/server.h" +#include "src/tint/lang/wgsl/ls/utils.h" +#include "src/tint/lang/wgsl/sem/function_expression.h" +#include "src/tint/lang/wgsl/sem/type_expression.h" +#include "src/tint/lang/wgsl/sem/variable.h" +#include "src/tint/utils/rtti/switch.h" + +namespace lsp = langsvr::lsp; + +namespace tint::wgsl::ls { + +namespace { + +struct Token { + lsp::Position position; + size_t kind = 0; + size_t length = 0; +}; + +Token TokenFromRange(const tint::Source::Range& range, SemToken::Kind kind) { + Token tok; + tok.position = Conv(range.begin); + tok.length = range.end.column - range.begin.column; + tok.kind = kind; + return tok; +} + +std::optional<SemToken::Kind> TokenKindFor(const sem::Expression* expr) { + return Switch<std::optional<SemToken::Kind>>( + Unwrap(expr), // + [](const sem::TypeExpression*) { return SemToken::kType; }, + [](const sem::VariableUser*) { return SemToken::kVariable; }, + [](const sem::FunctionExpression*) { return SemToken::kFunction; }, + [](const sem::BuiltinEnumExpression<wgsl::BuiltinFn>*) { return SemToken::kFunction; }, + [](const sem::BuiltinEnumExpressionBase*) { return SemToken::kEnumMember; }, + [](tint::Default) { return std::nullopt; }); +} + +std::vector<Token> Tokens(File& file) { + std::vector<Token> tokens; + auto& sem = file.program.Sem(); + for (auto* node : file.nodes) { + Switch( + node, // + [&](const ast::IdentifierExpression* expr) { + if (auto kind = TokenKindFor(sem.Get(expr))) { + tokens.push_back(TokenFromRange(expr->identifier->source.range, *kind)); + } + }, + [&](const ast::Struct* str) { + tokens.push_back(TokenFromRange(str->name->source.range, SemToken::kType)); + }, + [&](const ast::StructMember* member) { + tokens.push_back(TokenFromRange(member->name->source.range, SemToken::kMember)); + }, + [&](const ast::Variable* var) { + tokens.push_back(TokenFromRange(var->name->source.range, SemToken::kVariable)); + }, + [&](const ast::Function* fn) { + tokens.push_back(TokenFromRange(fn->name->source.range, SemToken::kFunction)); + }, + [&](const ast::MemberAccessorExpression* a) { + tokens.push_back(TokenFromRange(a->member->source.range, SemToken::kMember)); + }); + } + return tokens; +} + +} // namespace + +typename lsp::TextDocumentSemanticTokensFullRequest::ResultType // +Server::Handle(const lsp::TextDocumentSemanticTokensFullRequest& r) { + typename lsp::TextDocumentSemanticTokensFullRequest::SuccessType result; + + if (auto file = files_.Get(r.text_document.uri)) { + lsp::SemanticTokens out; + // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens + Token last; + + auto tokens = Tokens(**file); + std::sort(tokens.begin(), tokens.end(), + [](const Token& a, const Token& b) { return a.position < b.position; }); + + for (auto tok : tokens) { + if (last.position.line != tok.position.line) { + last.position.character = 0; + } + out.data.push_back(tok.position.line - last.position.line); + out.data.push_back(tok.position.character - last.position.character); + out.data.push_back(tok.length); + out.data.push_back(static_cast<langsvr::lsp::Uinteger>(tok.kind)); + out.data.push_back(0); // modifiers + last = tok; + } + + result = out; + } + + return result; +} + +} // namespace tint::wgsl::ls
diff --git a/src/tint/lang/wgsl/ls/sem_tokens_test.cc b/src/tint/lang/wgsl/ls/sem_tokens_test.cc new file mode 100644 index 0000000..4626364 --- /dev/null +++ b/src/tint/lang/wgsl/ls/sem_tokens_test.cc
@@ -0,0 +1,277 @@ +// Copyright 2024 The Dawn & Tint Authors +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <gtest/gtest.h> +#include <cstddef> +#include <sstream> +#include <string_view> + +#include "gmock/gmock.h" + +#include "langsvr/lsp/lsp.h" +#include "langsvr/lsp/primitives.h" +#include "langsvr/lsp/printer.h" +#include "src/tint/lang/wgsl/ls/helpers_test.h" +#include "src/tint/lang/wgsl/ls/sem_token.h" + +namespace tint::wgsl::ls { +namespace { + +namespace lsp = langsvr::lsp; + +struct Case { + std::string_view markup; + std::vector<SemToken::Kind> tokens; +}; + +std::ostream& operator<<(std::ostream& stream, const Case& c) { + return stream << "wgsl: '" << c.markup << "'"; +} + +struct RangeAndToken { + lsp::Range range; + SemToken::Kind token; + + bool operator==(const RangeAndToken& other) const { + return range == other.range && token == other.token; + } +}; + +std::ostream& operator<<(std::ostream& stream, const RangeAndToken& rt) { + return stream << "\n" << SemToken::kNames[rt.token] << ": " << rt.range; +} + +using LsSemTokensTest = LsTestWithParam<Case>; +TEST_P(LsSemTokensTest, SemTokens) { + auto parsed = ParseMarkers(GetParam().markup); + ASSERT_EQ(parsed.positions.size(), 0u); + ASSERT_EQ(parsed.ranges.size(), GetParam().tokens.size()); + + lsp::TextDocumentSemanticTokensFullRequest req{}; + req.text_document.uri = OpenDocument(parsed.clean); + + for (auto& n : diagnostics_) { + for (auto& d : n.diagnostics) { + if (d.severity == lsp::DiagnosticSeverity::kError) { + FAIL() << "Error: " << d.message << "\nWGSL:\n" << parsed.clean; + } + } + } + + auto future = client_session_.Send(req); + ASSERT_EQ(future, langsvr::Success); + auto res = future->get(); + if (parsed.ranges.empty()) { + ASSERT_TRUE(res.Is<lsp::Null>()); + } else { + ASSERT_TRUE(res.Is<lsp::SemanticTokens>()); + std::vector<RangeAndToken> expect; + for (size_t i = 0; i < parsed.ranges.size(); i++) { + expect.push_back(RangeAndToken{parsed.ranges[i], GetParam().tokens[i]}); + } + // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens + auto& data = res.Get<lsp::SemanticTokens>()->data; + ASSERT_EQ(data.size() % 5, 0u); + lsp::Position pos{}; + std::vector<RangeAndToken> got; + for (size_t i = 0; i < data.size(); i += 5) { + const auto delta_line = data[i + 0]; + const auto delta_start = data[i + 1]; + const auto length = data[i + 2]; + const auto token_type = data[i + 3]; + const auto modifiers = data[i + 4]; + + pos.line += delta_line; + pos.character = (delta_line == 0) ? (pos.character + delta_start) : delta_start; + lsp::Range range; + range.start = pos; + range.end = lsp::Position{pos.line, pos.character + length}; + auto token = static_cast<SemToken::Kind>(token_type); + ASSERT_EQ(modifiers, 0u); + got.push_back(RangeAndToken{range, token}); + } + EXPECT_EQ(got, expect); + } +} + +// TODO(bclayton): Type aliases. +INSTANTIATE_TEST_SUITE_P(IncludeDeclaration, + LsSemTokensTest, + ::testing::ValuesIn(std::vector<Case>{ + { + R"( +const「CONST」= 42; +fn 「f」() { _=「CONST」; } +const 「C」=「CONST」; +)", + { + /* 'CONST' */ SemToken::kVariable, + /* 'f' */ SemToken::kFunction, + /* 'CONST' */ SemToken::kVariable, + /* 'C' */ SemToken::kVariable, + /* 'CONST' */ SemToken::kVariable, + }, + }, // ========================================= + { + R"( +var<「private」>「VAR」= 42; +fn「f」() { _ =「VAR」; } +fn「g」() { _ = 「VAR」; } +)", + { + /* 'private' */ SemToken::kEnumMember, + /* 'VAR' */ SemToken::kVariable, + /* 'f' */ SemToken::kFunction, + /* 'VAR' */ SemToken::kVariable, + /* 'g' */ SemToken::kFunction, + /* 'VAR' */ SemToken::kVariable, + }, + }, // ========================================= + { + R"( +override「OVERRIDE」= 42; +fn「f」() { _ =「OVERRIDE」+「OVERRIDE」; } +)", + { + /* 'OVERRIDE' */ SemToken::kVariable, + /* 'f' */ SemToken::kFunction, + /* 'OVERRIDE' */ SemToken::kVariable, + /* 'OVERRIDE' */ SemToken::kVariable, + }, + }, // ========================================= + { + R"( +struct「STRUCT」{「i」:「i32」} +fn「f」(「s」:「STRUCT」) { var「v」:「STRUCT」; } +)", + { + /* 'STRUCT' */ SemToken::kType, + /* 'i' */ SemToken::kMember, + /* 'i32' */ SemToken::kType, + /* 'f' */ SemToken::kFunction, + /* 's' */ SemToken::kVariable, + /* 'STRUCT' */ SemToken::kType, + /* 'v' */ SemToken::kVariable, + /* 'STRUCT' */ SemToken::kType, + }, + }, // ========================================= + { + R"( +struct「S」{「i」: 「i32」 } +fn「f」(「s」 : 「S」) { _ = 「s」.「i」; } +fn「g」(「s」 : 「S」) { _ = 「s」.「i」; } +)", + { + /* 'S' */ SemToken::kType, + /* 'i' */ SemToken::kMember, + /* 'i32' */ SemToken::kType, + /* 'f' */ SemToken::kFunction, + /* 's' */ SemToken::kVariable, + /* 'S' */ SemToken::kType, + /* 's' */ SemToken::kVariable, + /* 'i' */ SemToken::kMember, + /* 'g' */ SemToken::kFunction, + /* 's' */ SemToken::kVariable, + /* 'S' */ SemToken::kType, + /* 's' */ SemToken::kVariable, + /* 'i' */ SemToken::kMember, + }, + }, // ========================================= + { + R"( +fn「f」(「p」:「i32」) { _ =「p」*「p」; } +)", + { + /* 'f' */ SemToken::kFunction, + /* 'p' */ SemToken::kVariable, + /* 'i32' */ SemToken::kType, + /* 'p' */ SemToken::kVariable, + /* 'p' */ SemToken::kVariable, + }, + }, // ========================================= + { + R"( +fn「f」() { + const「i」= 42; + _ =「i」*「i」; +} +)", + { + /* 'f' */ SemToken::kFunction, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + }, + }, // ========================================= + { + R"( +fn「f」() { + let「i」= 42; + _ =「i」+「i」; +} +)", + { + /* 'f' */ SemToken::kFunction, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + }, + }, // ========================================= + { + R"( +fn「f」() { + var「i」= 42; + 「i」=「i」; +} +)", + { + /* 'f' */ SemToken::kFunction, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + }, + }, // ========================================= + { + R"( +fn「f」() { + let「i」= 42; + _ = (「max」(「i」, 「i」) * 「i」); +} +)", + { + /* 'f' */ SemToken::kFunction, + /* 'i' */ SemToken::kVariable, + /* 'max' */ SemToken::kFunction, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + /* 'i' */ SemToken::kVariable, + }, + }, + })); + +} // namespace +} // namespace tint::wgsl::ls
diff --git a/src/tint/lang/wgsl/ls/server.cc b/src/tint/lang/wgsl/ls/server.cc index 503d9f53..f44e7f6 100644 --- a/src/tint/lang/wgsl/ls/server.cc +++ b/src/tint/lang/wgsl/ls/server.cc
@@ -29,6 +29,7 @@ #include "langsvr/session.h" +#include "src/tint/lang/wgsl/ls/sem_token.h" #include "src/tint/lang/wgsl/ls/utils.h" namespace lsp = langsvr::lsp; @@ -59,6 +60,14 @@ opts.prepare_provider = true; return opts; }(); + result.capabilities.semantic_tokens_provider = [] { + lsp::SemanticTokensOptions opts; + opts.full = true; + for (auto name : SemToken::kNames) { + opts.legend.token_types.push_back(name); + } + return opts; + }(); return result; }); @@ -84,6 +93,8 @@ session.Register([&](const lsp::TextDocumentPrepareRenameRequest& r) { return Handle(r); }); session.Register([&](const lsp::TextDocumentReferencesRequest& r) { return Handle(r); }); session.Register([&](const lsp::TextDocumentRenameRequest& r) { return Handle(r); }); + session.Register( + [&](const lsp::TextDocumentSemanticTokensFullRequest& r) { return Handle(r); }); } Server::~Server() = default;
diff --git a/src/tint/lang/wgsl/ls/server.h b/src/tint/lang/wgsl/ls/server.h index bc87a21..acd5226 100644 --- a/src/tint/lang/wgsl/ls/server.h +++ b/src/tint/lang/wgsl/ls/server.h
@@ -111,6 +111,10 @@ langsvr::Result<langsvr::SuccessType> // Handle(const langsvr::lsp::TextDocumentDidChangeNotification&); + /// Handler for langsvr::lsp::TextDocumentSemanticTokensFullRequest + typename langsvr::lsp::TextDocumentSemanticTokensFullRequest::ResultType // + Handle(const langsvr::lsp::TextDocumentSemanticTokensFullRequest&); + /// Handler for langsvr::lsp::WorkspaceDidChangeConfigurationNotification langsvr::Result<langsvr::SuccessType> // Handle(const langsvr::lsp::WorkspaceDidChangeConfigurationNotification&);