reader/wgsl: Optimize tokens by using string_view
Each Token was making a copy of the `val_str_`, despite the token being a slice on the original source.
Bug: tint:1383
Change-Id: I17b2da8f986ba105853aa47afe21bcc75f140f8e
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/78320
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
diff --git a/src/reader/wgsl/lexer.cc b/src/reader/wgsl/lexer.cc
index 94d8ced..04a10b4 100644
--- a/src/reader/wgsl/lexer.cc
+++ b/src/reader/wgsl/lexer.cc
@@ -62,38 +62,31 @@
Lexer::~Lexer() = default;
Token Lexer::next() {
- auto t = skip_whitespace_and_comments();
- if (!t.IsUninitialized()) {
+ if (auto t = skip_whitespace_and_comments(); !t.IsUninitialized()) {
return t;
}
- t = try_hex_float();
- if (!t.IsUninitialized()) {
+ if (auto t = try_hex_float(); !t.IsUninitialized()) {
return t;
}
- t = try_hex_integer();
- if (!t.IsUninitialized()) {
+ if (auto t = try_hex_integer(); !t.IsUninitialized()) {
return t;
}
- t = try_float();
- if (!t.IsUninitialized()) {
+ if (auto t = try_float(); !t.IsUninitialized()) {
return t;
}
- t = try_integer();
- if (!t.IsUninitialized()) {
+ if (auto t = try_integer(); !t.IsUninitialized()) {
return t;
}
- t = try_ident();
- if (!t.IsUninitialized()) {
+ if (auto t = try_ident(); !t.IsUninitialized()) {
return t;
}
- t = try_punctuation();
- if (!t.IsUninitialized()) {
+ if (auto t = try_punctuation(); !t.IsUninitialized()) {
return t;
}
@@ -771,7 +764,7 @@
return t;
}
- return {Token::Type::kIdentifier, source, std::string(str)};
+ return {Token::Type::kIdentifier, source, str};
}
Token Lexer::try_punctuation() {
diff --git a/src/reader/wgsl/parser_impl.cc b/src/reader/wgsl/parser_impl.cc
index f219eba..a1c236f 100644
--- a/src/reader/wgsl/parser_impl.cc
+++ b/src/reader/wgsl/parser_impl.cc
@@ -125,28 +125,22 @@
const char kWorkgroupSizeDecoration[] = "workgroup_size";
bool is_decoration(Token t) {
- if (!t.IsIdentifier()) {
- return false;
- }
-
- auto s = t.to_str();
- return s == kAlignDecoration || s == kBindingDecoration ||
- s == kBlockDecoration || s == kBuiltinDecoration ||
- s == kGroupDecoration || s == kInterpolateDecoration ||
- s == kLocationDecoration || s == kOverrideDecoration ||
- s == kSizeDecoration || s == kStageDecoration ||
- s == kStrideDecoration || s == kWorkgroupSizeDecoration;
+ return t == kAlignDecoration || t == kBindingDecoration ||
+ t == kBlockDecoration || t == kBuiltinDecoration ||
+ t == kGroupDecoration || t == kInterpolateDecoration ||
+ t == kLocationDecoration || t == kOverrideDecoration ||
+ t == kSizeDecoration || t == kStageDecoration ||
+ t == kStrideDecoration || t == kWorkgroupSizeDecoration;
}
// https://gpuweb.github.io/gpuweb/wgsl.html#reserved-keywords
bool is_reserved(Token t) {
- auto s = t.to_str();
- return s == "asm" || s == "bf16" || s == "const" || s == "do" ||
- s == "enum" || s == "f16" || s == "f64" || s == "handle" ||
- s == "i8" || s == "i16" || s == "i64" || s == "mat" ||
- s == "premerge" || s == "regardless" || s == "typedef" || s == "u8" ||
- s == "u16" || s == "u64" || s == "unless" || s == "using" ||
- s == "vec" || s == "void" || s == "while";
+ return t == "asm" || t == "bf16" || t == "const" || t == "do" ||
+ t == "enum" || t == "f16" || t == "f64" || t == "handle" ||
+ t == "i8" || t == "i16" || t == "i64" || t == "mat" ||
+ t == "premerge" || t == "regardless" || t == "typedef" || t == "u8" ||
+ t == "u16" || t == "u64" || t == "unless" || t == "using" ||
+ t == "vec" || t == "void" || t == "while";
}
/// Enter-exit counters for block token types.
@@ -760,59 +754,56 @@
// | 'rgba32sint'
// | 'rgba32float'
Expect<ast::TexelFormat> ParserImpl::expect_texel_format(std::string_view use) {
- auto tok = next();
- if (tok.IsIdentifier()) {
- auto s = tok.to_str();
- if (s == "rgba8unorm") {
- return ast::TexelFormat::kRgba8Unorm;
- }
- if (s == "rgba8snorm") {
- return ast::TexelFormat::kRgba8Snorm;
- }
- if (s == "rgba8uint") {
- return ast::TexelFormat::kRgba8Uint;
- }
- if (s == "rgba8sint") {
- return ast::TexelFormat::kRgba8Sint;
- }
- if (s == "rgba16uint") {
- return ast::TexelFormat::kRgba16Uint;
- }
- if (s == "rgba16sint") {
- return ast::TexelFormat::kRgba16Sint;
- }
- if (s == "rgba16float") {
- return ast::TexelFormat::kRgba16Float;
- }
- if (s == "r32uint") {
- return ast::TexelFormat::kR32Uint;
- }
- if (s == "r32sint") {
- return ast::TexelFormat::kR32Sint;
- }
- if (s == "r32float") {
- return ast::TexelFormat::kR32Float;
- }
- if (s == "rg32uint") {
- return ast::TexelFormat::kRg32Uint;
- }
- if (s == "rg32sint") {
- return ast::TexelFormat::kRg32Sint;
- }
- if (s == "rg32float") {
- return ast::TexelFormat::kRg32Float;
- }
- if (s == "rgba32uint") {
- return ast::TexelFormat::kRgba32Uint;
- }
- if (s == "rgba32sint") {
- return ast::TexelFormat::kRgba32Sint;
- }
- if (s == "rgba32float") {
- return ast::TexelFormat::kRgba32Float;
- }
+ auto t = next();
+ if (t == "rgba8unorm") {
+ return ast::TexelFormat::kRgba8Unorm;
}
- return add_error(tok.source(), "invalid format", use);
+ if (t == "rgba8snorm") {
+ return ast::TexelFormat::kRgba8Snorm;
+ }
+ if (t == "rgba8uint") {
+ return ast::TexelFormat::kRgba8Uint;
+ }
+ if (t == "rgba8sint") {
+ return ast::TexelFormat::kRgba8Sint;
+ }
+ if (t == "rgba16uint") {
+ return ast::TexelFormat::kRgba16Uint;
+ }
+ if (t == "rgba16sint") {
+ return ast::TexelFormat::kRgba16Sint;
+ }
+ if (t == "rgba16float") {
+ return ast::TexelFormat::kRgba16Float;
+ }
+ if (t == "r32uint") {
+ return ast::TexelFormat::kR32Uint;
+ }
+ if (t == "r32sint") {
+ return ast::TexelFormat::kR32Sint;
+ }
+ if (t == "r32float") {
+ return ast::TexelFormat::kR32Float;
+ }
+ if (t == "rg32uint") {
+ return ast::TexelFormat::kRg32Uint;
+ }
+ if (t == "rg32sint") {
+ return ast::TexelFormat::kRg32Sint;
+ }
+ if (t == "rg32float") {
+ return ast::TexelFormat::kRg32Float;
+ }
+ if (t == "rgba32uint") {
+ return ast::TexelFormat::kRgba32Uint;
+ }
+ if (t == "rgba32sint") {
+ return ast::TexelFormat::kRgba32Sint;
+ }
+ if (t == "rgba32float") {
+ return ast::TexelFormat::kRgba32Float;
+ }
+ return add_error(t.source(), "invalid format", use);
}
// variable_ident_decl
@@ -1431,24 +1422,18 @@
// | COMPUTE
Expect<ast::PipelineStage> ParserImpl::expect_pipeline_stage() {
auto t = peek();
- if (!t.IsIdentifier()) {
- return add_error(t, "invalid value for stage decoration");
- }
-
- auto s = t.to_str();
- if (s == kVertexStage) {
+ if (t == kVertexStage) {
next(); // Consume the peek
return {ast::PipelineStage::kVertex, t.source()};
}
- if (s == kFragmentStage) {
+ if (t == kFragmentStage) {
next(); // Consume the peek
return {ast::PipelineStage::kFragment, t.source()};
}
- if (s == kComputeStage) {
+ if (t == kComputeStage) {
next(); // Consume the peek
return {ast::PipelineStage::kCompute, t.source()};
}
-
return add_error(peek(), "invalid value for stage decoration");
}
@@ -2932,9 +2917,7 @@
return Failure::kNoMatch;
}
- auto s = t.to_str();
-
- if (s == kLocationDecoration) {
+ if (t == kLocationDecoration) {
const char* use = "location decoration";
return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use);
@@ -2945,7 +2928,7 @@
});
}
- if (s == kBindingDecoration) {
+ if (t == kBindingDecoration) {
const char* use = "binding decoration";
return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use);
@@ -2956,7 +2939,7 @@
});
}
- if (s == kGroupDecoration) {
+ if (t == kGroupDecoration) {
const char* use = "group decoration";
return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use);
@@ -2967,18 +2950,17 @@
});
}
- if (s == kInterpolateDecoration) {
+ if (t == kInterpolateDecoration) {
return expect_paren_block("interpolate decoration", [&]() -> Result {
ast::InterpolationType type;
ast::InterpolationSampling sampling = ast::InterpolationSampling::kNone;
auto type_tok = next();
- auto type_str = type_tok.to_str();
- if (type_str == "perspective") {
+ if (type_tok == "perspective") {
type = ast::InterpolationType::kPerspective;
- } else if (type_str == "linear") {
+ } else if (type_tok == "linear") {
type = ast::InterpolationType::kLinear;
- } else if (type_str == "flat") {
+ } else if (type_tok == "flat") {
type = ast::InterpolationType::kFlat;
} else {
return add_error(type_tok, "invalid interpolation type");
@@ -2986,12 +2968,11 @@
if (match(Token::Type::kComma)) {
auto sampling_tok = next();
- auto sampling_str = sampling_tok.to_str();
- if (sampling_str == "center") {
+ if (sampling_tok == "center") {
sampling = ast::InterpolationSampling::kCenter;
- } else if (sampling_str == "centroid") {
+ } else if (sampling_tok == "centroid") {
sampling = ast::InterpolationSampling::kCentroid;
- } else if (sampling_str == "sample") {
+ } else if (sampling_tok == "sample") {
sampling = ast::InterpolationSampling::kSample;
} else {
return add_error(sampling_tok, "invalid interpolation sampling");
@@ -3002,11 +2983,11 @@
});
}
- if (s == kInvariantDecoration) {
+ if (t == kInvariantDecoration) {
return create<ast::InvariantDecoration>(t.source());
}
- if (s == kBuiltinDecoration) {
+ if (t == kBuiltinDecoration) {
return expect_paren_block("builtin decoration", [&]() -> Result {
auto builtin = expect_builtin();
if (builtin.errored)
@@ -3016,7 +2997,7 @@
});
}
- if (s == kWorkgroupSizeDecoration) {
+ if (t == kWorkgroupSizeDecoration) {
return expect_paren_block("workgroup_size decoration", [&]() -> Result {
const ast::Expression* x = nullptr;
const ast::Expression* y = nullptr;
@@ -3054,7 +3035,7 @@
});
}
- if (s == kStageDecoration) {
+ if (t == kStageDecoration) {
return expect_paren_block("stage decoration", [&]() -> Result {
auto stage = expect_pipeline_stage();
if (stage.errored)
@@ -3064,12 +3045,12 @@
});
}
- if (s == kBlockDecoration) {
+ if (t == kBlockDecoration) {
deprecated(t.source(), "[[block]] attributes have been removed from WGSL");
return create<ast::StructBlockDecoration>(t.source());
}
- if (s == kStrideDecoration) {
+ if (t == kStrideDecoration) {
const char* use = "stride decoration";
return expect_paren_block(use, [&]() -> Result {
auto val = expect_nonzero_positive_sint(use);
@@ -3082,7 +3063,7 @@
});
}
- if (s == kSizeDecoration) {
+ if (t == kSizeDecoration) {
const char* use = "size decoration";
return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use);
@@ -3093,7 +3074,7 @@
});
}
- if (s == kAlignDecoration) {
+ if (t == kAlignDecoration) {
const char* use = "align decoration";
return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use);
@@ -3104,7 +3085,7 @@
});
}
- if (s == kOverrideDecoration) {
+ if (t == kOverrideDecoration) {
const char* use = "override decoration";
if (peek_is(Token::Type::kParenLeft)) {
diff --git a/src/reader/wgsl/token.cc b/src/reader/wgsl/token.cc
index 82147d5..ce462c2 100644
--- a/src/reader/wgsl/token.cc
+++ b/src/reader/wgsl/token.cc
@@ -254,19 +254,25 @@
Token::Token() : type_(Type::kUninitialized) {}
-Token::Token(Type type, const Source& source, const std::string& val)
- : type_(type), source_(source), val_str_(val) {}
+Token::Token(Type type, const Source& source, const std::string_view& view)
+ : type_(type), source_(source), value_(view) {}
+
+Token::Token(Type type, const Source& source, const std::string& str)
+ : type_(type), source_(source), value_(str) {}
+
+Token::Token(Type type, const Source& source, const char* str)
+ : type_(type), source_(source), value_(std::string_view(str)) {}
Token::Token(const Source& source, uint32_t val)
- : type_(Type::kUintLiteral), source_(source), val_uint_(val) {}
+ : type_(Type::kUintLiteral), source_(source), value_(val) {}
Token::Token(const Source& source, int32_t val)
- : type_(Type::kSintLiteral), source_(source), val_int_(val) {}
+ : type_(Type::kSintLiteral), source_(source), value_(val) {}
Token::Token(const Source& source, float val)
- : type_(Type::kFloatLiteral), source_(source), val_float_(val) {}
+ : type_(Type::kFloatLiteral), source_(source), value_(val) {}
-Token::Token(Type type, const Source& source) : Token(type, source, "") {}
+Token::Token(Type type, const Source& source) : type_(type), source_(source) {}
Token::Token(Token&&) = default;
@@ -274,31 +280,47 @@
Token::~Token() = default;
-Token& Token::operator=(const Token&) = default;
+Token& Token::operator=(const Token& rhs) = default;
+
+bool Token::operator==(std::string_view ident) {
+ if (type_ != Type::kIdentifier) {
+ return false;
+ }
+ if (auto* view = std::get_if<std::string_view>(&value_)) {
+ return *view == ident;
+ }
+ return std::get<std::string>(value_) == ident;
+}
std::string Token::to_str() const {
- if (type_ == Type::kFloatLiteral) {
- return std::to_string(val_float_);
+ switch (type_) {
+ case Type::kFloatLiteral:
+ return std::to_string(std::get<float>(value_));
+ case Type::kSintLiteral:
+ return std::to_string(std::get<int32_t>(value_));
+ case Type::kUintLiteral:
+ return std::to_string(std::get<uint32_t>(value_));
+ case Type::kIdentifier:
+ case Type::kError:
+ if (auto* view = std::get_if<std::string_view>(&value_)) {
+ return std::string(*view);
+ }
+ return std::get<std::string>(value_);
+ default:
+ return "";
}
- if (type_ == Type::kSintLiteral) {
- return std::to_string(val_int_);
- }
- if (type_ == Type::kUintLiteral) {
- return std::to_string(val_uint_);
- }
- return val_str_;
}
float Token::to_f32() const {
- return val_float_;
+ return std::get<float>(value_);
}
uint32_t Token::to_u32() const {
- return val_uint_;
+ return std::get<uint32_t>(value_);
}
int32_t Token::to_i32() const {
- return val_int_;
+ return std::get<int32_t>(value_);
}
} // namespace wgsl
diff --git a/src/reader/wgsl/token.h b/src/reader/wgsl/token.h
index bc955dd..0a62ce6 100644
--- a/src/reader/wgsl/token.h
+++ b/src/reader/wgsl/token.h
@@ -17,6 +17,7 @@
#include <string>
#include <string_view>
+#include <variant> // NOLINT: cpplint doesn't recognise this
#include "src/source.h"
@@ -273,8 +274,18 @@
/// Create a string Token
/// @param type the Token::Type of the token
/// @param source the source of the token
- /// @param val the source string for the token
- Token(Type type, const Source& source, const std::string& val);
+ /// @param view the source string view for the token
+ Token(Type type, const Source& source, const std::string_view& view);
+ /// Create a string Token
+ /// @param type the Token::Type of the token
+ /// @param source the source of the token
+ /// @param str the source string for the token
+ Token(Type type, const Source& source, const std::string& str);
+ /// Create a string Token
+ /// @param type the Token::Type of the token
+ /// @param source the source of the token
+ /// @param str the source string for the token
+ Token(Type type, const Source& source, const char* str);
/// Create a unsigned integer Token
/// @param source the source of the token
/// @param val the source unsigned for the token
@@ -298,6 +309,11 @@
/// @return Token
Token& operator=(const Token& b);
+ /// Equality operator with an identifier
+ /// @param ident the identifier string
+ /// @return true if this token is an identifier and is equal to ident.
+ bool operator==(std::string_view ident);
+
/// Returns true if the token is of the given type
/// @param t the type to check against.
/// @returns true if the token is of type `t`
@@ -378,14 +394,8 @@
Type type_ = Type::kError;
/// The source where the token appeared
Source source_;
- /// The string represented by the token
- std::string val_str_;
- /// The signed integer represented by the token
- int32_t val_int_ = 0;
- /// The unsigned integer represented by the token
- uint32_t val_uint_ = 0;
- /// The float value represented by the token
- float val_float_ = 0.0;
+ /// The value represented by the token
+ std::variant<int32_t, uint32_t, float, std::string, std::string_view> value_;
};
#ifndef NDEBUG
