[tint][fuzz] Generate `dictionary.txt` from .def files
Change-Id: I7f575a0f65cb371183aa8a3543cd762071f62345
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/184163
Reviewed-by: James Price <jrprice@google.com>
Commit-Queue: James Price <jrprice@google.com>
Auto-Submit: Ben Clayton <bclayton@google.com>
diff --git a/src/tint/cmd/fuzz/wgsl/dictionary.txt b/src/tint/cmd/fuzz/wgsl/dictionary.txt
index 59742d7..cc0d9d3 100644
--- a/src/tint/cmd/fuzz/wgsl/dictionary.txt
+++ b/src/tint/cmd/fuzz/wgsl/dictionary.txt
@@ -1,3 +1,39 @@
+# Copyright 2024 The Dawn & Tint Authors
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+################################################################################
+# File generated by 'tools/src/cmd/gen' using the template:
+# src/tint/cmd/fuzz/wgsl/dictionary.txt.tmpl
+#
+# To regenerate run: './tools/run gen'
+#
+# Do not modify this file directly
+################################################################################
+
"!"
"!="
"%"
@@ -38,17 +74,48 @@
"^"
"^="
"_"
-"{"
-"|"
-"|="
-"||"
-"}"
-"~"
+"__atomic_compare_exchange_result"
+"__atomic_compare_exchange_result_i32"
+"__atomic_compare_exchange_result_u32"
+"__frexp_result"
+"__frexp_result_abstract"
+"__frexp_result_f16"
+"__frexp_result_f32"
+"__frexp_result_vec"
+"__frexp_result_vec2_abstract"
+"__frexp_result_vec2_f16"
+"__frexp_result_vec2_f32"
+"__frexp_result_vec3_abstract"
+"__frexp_result_vec3_f16"
+"__frexp_result_vec3_f32"
+"__frexp_result_vec4_abstract"
+"__frexp_result_vec4_f16"
+"__frexp_result_vec4_f32"
+"__in"
+"__modf_result"
+"__modf_result_abstract"
+"__modf_result_f16"
+"__modf_result_f32"
+"__modf_result_vec"
+"__modf_result_vec2_abstract"
+"__modf_result_vec2_f16"
+"__modf_result_vec2_f32"
+"__modf_result_vec3_abstract"
+"__modf_result_vec3_f16"
+"__modf_result_vec3_f32"
+"__modf_result_vec4_abstract"
+"__modf_result_vec4_f16"
+"__modf_result_vec4_f32"
+"__out"
+"__packed_vec3"
+"__point_size"
+"__tint_materialize"
"a"
"abs"
"acos"
"acosh"
-"@align"
+"alias"
+"align"
"all"
"any"
"array"
@@ -61,6 +128,8 @@
"atomic"
"atomicAdd"
"atomicAnd"
+"atomicCompareExchangeWeak"
+"atomicExchange"
"atomicLoad"
"atomicMax"
"atomicMin"
@@ -69,31 +138,35 @@
"atomicSub"
"atomicXor"
"b"
-"@binding"
+"bgra8unorm"
+"binding"
"bitcast"
+"blend_src"
"bool"
"break"
-"@builtin"
-"@builtin(frag_depth)"
-"@builtin(front_facing)"
-"@builtin(global_invocation_id)"
-"@builtin(instance_index)"
-"@builtin(local_invocation_id)"
-"@builtin(local_invocation_index)"
-"@builtin(num_workgroups)"
-"@builtin(position)"
-"@builtin(sample_index)"
-"@builtin(sample_mask)"
-"@builtin(vertex_index)"
-"@builtin(workgroup_id)"
+"builtin"
"case"
"ceil"
"center"
"centroid"
+"chromium_disable_uniformity_analysis"
+"chromium_experimental_framebuffer_fetch"
+"chromium_experimental_pixel_local"
+"chromium_experimental_push_constant"
+"chromium_experimental_subgroups"
+"chromium_internal_dual_source_blending"
+"chromium_internal_graphite"
+"chromium_internal_relaxed_uniform_layout"
+"chromium_testing_experimental"
+"chromium_testing_shipped"
+"chromium_testing_shipped_with_killswitch"
+"chromium_testing_unimplemented"
+"chromium_testing_unsafe_experimental"
"clamp"
-"@compute"
-"@const"
+"color"
+"compute"
"const"
+"const_assert"
"continue"
"continuing"
"cos"
@@ -104,7 +177,9 @@
"cross"
"default"
"degrees"
+"derivative_uniformity"
"determinant"
+"diagnostic"
"discard"
"distance"
"dot"
@@ -118,11 +193,13 @@
"dpdyFine"
"else"
"enable"
+"error"
"exp"
"exp2"
"extractBits"
"f16"
"f32"
+"fa"
"faceForward"
"fallthrough"
"false"
@@ -135,7 +212,7 @@
"for"
"fract"
"frag_depth"
-"@fragment"
+"fragment"
"frexp"
"front_facing"
"function"
@@ -144,14 +221,17 @@
"fwidthFine"
"g"
"global_invocation_id"
-"@group"
+"group"
+"handle"
"i32"
-"@id"
+"ia"
+"id"
"if"
+"info"
"insertBits"
"instance_index"
-"@interpolate"
-"@invariant"
+"interpolate"
+"invariant"
"inverseSqrt"
"ldexp"
"length"
@@ -159,46 +239,80 @@
"linear"
"local_invocation_id"
"local_invocation_index"
-"@location"
+"location"
"log"
"log2"
"loop"
+"mat"
"mat2x2"
+"mat2x2f"
+"mat2x2h"
"mat2x3"
+"mat2x3f"
+"mat2x3h"
"mat2x4"
+"mat2x4f"
+"mat2x4h"
"mat3x2"
+"mat3x2f"
+"mat3x2h"
"mat3x3"
+"mat3x3f"
+"mat3x3h"
"mat3x4"
+"mat3x4f"
+"mat3x4h"
"mat4x2"
+"mat4x2f"
+"mat4x2h"
"mat4x3"
+"mat4x3f"
+"mat4x3h"
"mat4x4"
+"mat4x4f"
+"mat4x4h"
"max"
"min"
"mix"
"modf"
+"must_use"
"normalize"
"num_workgroups"
+"off"
"override"
"pack2x16float"
"pack2x16snorm"
"pack2x16unorm"
"pack4x8snorm"
"pack4x8unorm"
+"pack4xI8"
+"pack4xI8Clamp"
+"pack4xU8"
+"pack4xU8Clamp"
+"packedVec3"
+"packed_4x8_integer_dot_product"
"perspective"
+"pixel_local"
+"pointer_composite_access"
"position"
"pow"
"private"
"ptr"
+"push_constant"
"quantizeToF16"
"r"
"r32float"
"r32sint"
"r32uint"
+"r8unorm"
"radians"
"read"
"read_write"
+"readonly_and_readwrite_storage_textures"
+"ref"
"reflect"
"refract"
+"requires"
"return"
"reverseBits"
"rg32float"
@@ -225,17 +339,36 @@
"sign"
"sin"
"sinh"
-"@size"
+"size"
"smoothstep"
"sqrt"
-"staticAssert"
"step"
"storage"
"storageBarrier"
"struct"
+"subgroupBallot"
+"subgroupBroadcast"
+"subgroup_invocation_id"
+"subgroup_size"
"switch"
"tan"
"tanh"
+"textureBarrier"
+"textureDimensions"
+"textureGather"
+"textureGatherCompare"
+"textureLoad"
+"textureNumLayers"
+"textureNumLevels"
+"textureNumSamples"
+"textureSample"
+"textureSampleBaseClampToEdge"
+"textureSampleBias"
+"textureSampleCompare"
+"textureSampleCompareLevel"
+"textureSampleGrad"
+"textureSampleLevel"
+"textureStore"
"texture_1d"
"texture_2d"
"texture_2d_array"
@@ -247,29 +380,15 @@
"texture_depth_cube"
"texture_depth_cube_array"
"texture_depth_multisampled_2d"
-"textureDimensions"
-"textureGather"
-"textureGatherCompare"
-"textureLoad"
+"texture_external"
"texture_multisampled_2d"
-"textureNumLayers"
-"textureNumLevels"
-"textureNumSamples"
-"textureSample"
-"textureSampleBias"
-"textureSampleCompare"
-"textureSampleCompareLevel"
-"textureSampleGrad"
-"textureSampleLevel"
"texture_storage_1d"
"texture_storage_2d"
"texture_storage_2d_array"
"texture_storage_3d"
-"textureStore"
"transpose"
"true"
"trunc"
-"type"
"u32"
"uniform"
"unpack2x16float"
@@ -277,20 +396,44 @@
"unpack2x16unorm"
"unpack4x8snorm"
"unpack4x8unorm"
+"unpack4xI8"
+"unpack4xU8"
+"unreachable_code"
+"unrestricted_pointer_parameters"
"var"
+"vec"
"vec2"
+"vec2f"
+"vec2h"
+"vec2i"
+"vec2u"
"vec3"
+"vec3f"
+"vec3h"
+"vec3i"
+"vec3u"
"vec4"
-"@vertex"
+"vec4f"
+"vec4h"
+"vec4i"
+"vec4u"
+"vertex"
"vertex_index"
"w"
+"warning"
"while"
"workgroup"
"workgroupBarrier"
"workgroupUniformLoad"
"workgroup_id"
-"@workgroup_size"
+"workgroup_size"
"write"
"x"
"y"
"z"
+"{"
+"|"
+"|="
+"||"
+"}"
+"~"
diff --git a/src/tint/cmd/fuzz/wgsl/dictionary.txt.tmpl b/src/tint/cmd/fuzz/wgsl/dictionary.txt.tmpl
new file mode 100644
index 0000000..5ffad37
--- /dev/null
+++ b/src/tint/cmd/fuzz/wgsl/dictionary.txt.tmpl
@@ -0,0 +1,129 @@
+{{- /*
+--------------------------------------------------------------------------------
+Template file for use with tools/src/cmd/gen to generate dictionary.txt
+
+To update the generated file, run:
+ ./tools/run gen
+
+See:
+* tools/src/cmd/gen for structures used by this template
+* https://golang.org/pkg/text/template/ for documentation on the template syntax
+--------------------------------------------------------------------------------
+*/ -}}
+
+{{- SetCommentPrefix "#" -}}
+{{- $W := LoadIntrinsics "src/tint/lang/wgsl/wgsl.def" -}}
+{{- $C := LoadIntrinsics "src/tint/lang/core/core.def" -}}
+
+{{- $tokens := List}}
+
+{{- /* ============================== Operators ============================== */ -}}
+{{- $tokens = Append $tokens
+ "!"
+ "!="
+ "%"
+ "%="
+ "&"
+ "&&"
+ "&="
+ "("
+ ")"
+ "*"
+ "*="
+ "+"
+ "++"
+ "+="
+ ","
+ "-"
+ "--"
+ "-="
+ "->"
+ "."
+ "/"
+ "/="
+ ":"
+ ";"
+ "<"
+ "<<"
+ "<<="
+ "<="
+ "="
+ "=="
+ ">"
+ ">="
+ ">>"
+ ">>="
+ "@"
+ "["
+ "]"
+ "^"
+ "^="
+ "_"
+ "{"
+ "|"
+ "|="
+ "||"
+ "}"
+ "~"
+ "alias"
+ "break"
+ "case"
+ "const"
+ "const_assert"
+ "continue"
+ "continuing"
+ "diagnostic"
+ "discard"
+ "default"
+ "else"
+ "enable"
+ "fallthrough"
+ "false"
+ "fn"
+ "for"
+ "if"
+ "let"
+ "loop"
+ "override"
+ "return"
+ "requires"
+ "struct"
+ "switch"
+ "true"
+ "var"
+ "while"
+-}}
+
+{{- /* ============================== Swizzles =============================== */ -}}
+{{- $tokens = Append $tokens
+ "x" "y" "z" "w"
+ "r" "g" "b" "a"
+-}}
+
+{{- /* ============================ Boolean values =========================== */ -}}
+{{- $tokens = Append $tokens
+ "true" "false"
+-}}
+
+{{- /* ========================== Builtin functions ========================== */ -}}
+{{- range $W.Sem.Builtins}}{{$tokens = Append $tokens .Name}}{{end -}}
+
+{{- /* ================================ Types ================================ */ -}}
+{{- range $W.Sem.Types}}{{$tokens = Append $tokens .Name}}{{end -}}
+
+{{- /* ============================= Enumerators ============================= */ -}}
+{{- range $W.Sem.Enums}}
+ {{- range .Entries}}
+ {{- $tokens = Append $tokens .Name}}
+ {{- end}}
+{{- end}}
+{{- range $C.Sem.Enums}}
+ {{- range .Entries}}
+ {{- $tokens = Append $tokens .Name}}
+ {{- end}}
+{{- end}}
+
+{{- $tokens = SortUnique $tokens}}
+
+{{- range $tokens}}"{{.}}"
+{{end -}}
diff --git a/tools/src/cmd/gen/templates/templates.go b/tools/src/cmd/gen/templates/templates.go
index 5dd5b16..99158bf 100644
--- a/tools/src/cmd/gen/templates/templates.go
+++ b/tools/src/cmd/gen/templates/templates.go
@@ -31,7 +31,6 @@
"context"
"flag"
"fmt"
- "io"
"math/rand"
"os"
"path/filepath"
@@ -118,9 +117,23 @@
// Create or update the file at relPath if the file content has changed,
// preserving the copyright year in the header.
// relPath is a path relative to the template
- writeFile := func(relPath, body string) error {
+ writeFile := func(relPath, body, commentPrefix string) error {
+ if strings.TrimSpace(body) == "" {
+ // Don't write empty files
+ return nil
+ }
+
outPath := filepath.Join(tmplDir, relPath)
+ switch filepath.Ext(relPath) {
+ case ".cc", ".h", ".inl":
+ var err error
+ body, err = common.ClangFormat(body)
+ if err != nil {
+ return err
+ }
+ }
+
// Load the old file
existing, err := os.ReadFile(outPath)
if err != nil {
@@ -132,7 +145,7 @@
fmt.Println(" writing", outPath)
}
sb := strings.Builder{}
- sb.WriteString(common.Header(string(existing), filepath.ToSlash(relTmplPath), "//"))
+ sb.WriteString(common.Header(string(existing), filepath.ToSlash(relTmplPath), commentPrefix))
sb.WriteString("\n")
sb.WriteString(body)
oldContent, newContent := string(existing), sb.String()
@@ -154,28 +167,11 @@
}
// Write the content generated using the template and semantic info
- sb := strings.Builder{}
- if err := generate(tmplPath, cache, &sb, writeFile); err != nil {
+ _, tmplFileName := filepath.Split(tmplPath)
+ outPath := strings.TrimSuffix(tmplFileName, ".tmpl")
+ if err := generate(tmplPath, outPath, cache, writeFile); err != nil {
return fmt.Errorf("while processing '%v': %w", tmplPath, err)
}
-
- if body := sb.String(); body != "" {
- _, tmplFileName := filepath.Split(tmplPath)
- outFileName := strings.TrimSuffix(tmplFileName, ".tmpl")
-
- switch filepath.Ext(outFileName) {
- case ".cc", ".h", ".inl":
- var err error
- body, err = common.ClangFormat(body)
- if err != nil {
- return err
- }
- }
-
- if err := writeFile(outFileName, body); err != nil {
- return err
- }
- }
}
if len(staleFiles) > 0 {
@@ -274,28 +270,38 @@
}
type generator struct {
- cache *genCache
- writeFile WriteFile
- rnd *rand.Rand
+ cache *genCache
+ writeFile WriteFile
+ rnd *rand.Rand
+ commentPrefix string
+}
+
+// setCommentPrefix sets the prefix used for comments, as used by the template
+func (g *generator) setCommentPrefix(commentPrefix string) string {
+ g.commentPrefix = commentPrefix
+ return ""
}
// WriteFile is a function that Generate() may call to emit a new file from a
// template.
// relPath is the relative path from the currently executing template.
// content is the file content to write.
-type WriteFile func(relPath, content string) error
+// comment is the prefix used for line comments
+type WriteFile func(relPath, content, comment string) error
-// generate executes the template tmpl, writing the output to w.
+// generate executes the template tmpl, calling writeFile with the output.
// See https://golang.org/pkg/text/template/ for documentation on the template
// syntax.
-func generate(tmplPath string, cache *genCache, w io.Writer, writeFile WriteFile) error {
+func generate(tmplPath, outPath string, cache *genCache, writeFile WriteFile) error {
g := generator{
- cache: cache,
- writeFile: writeFile,
- rnd: rand.New(rand.NewSource(4561123)),
+ cache: cache,
+ writeFile: writeFile,
+ rnd: rand.New(rand.NewSource(4561123)),
+ commentPrefix: "//",
}
funcs := map[string]any{
+ "SetCommentPrefix": g.setCommentPrefix,
"SplitDisplayName": gen.SplitDisplayName,
"Scramble": g.scramble,
"IsEnumEntry": is(sem.EnumEntry{}),
@@ -316,13 +322,19 @@
"IsFirstIn": isFirstIn,
"IsLastIn": isLastIn,
"LoadIntrinsics": func(path string) *intrinsicCache { return g.cache.intrinsics(path) },
- "WriteFile": func(relPath, content string) (string, error) { return "", g.writeFile(relPath, content) },
+ "WriteFile": func(relPath, content string) (string, error) {
+ return "", g.writeFile(relPath, content, g.commentPrefix)
+ },
}
t, err := template.FromFile(tmplPath)
if err != nil {
return err
}
- return t.Run(w, nil, funcs)
+ w := &strings.Builder{}
+ if err := t.Run(w, nil, funcs); err != nil {
+ return err
+ }
+ return writeFile(outPath, w.String(), g.commentPrefix)
}
// scramble randomly modifies the input string so that it is no longer equal to
diff --git a/tools/src/template/template.go b/tools/src/template/template.go
index 5af3ef6..313de30 100644
--- a/tools/src/template/template.go
+++ b/tools/src/template/template.go
@@ -25,7 +25,7 @@
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// package template wraps the golang "text/template" package to provide an
+// Package template wraps the golang "text/template" package to provide an
// enhanced template generator.
package template
@@ -39,8 +39,10 @@
"strings"
"text/template"
+ "dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/fileutils"
"dawn.googlesource.com/dawn/tools/src/text"
+ "dawn.googlesource.com/dawn/tools/src/transform"
)
// The template function binding table
@@ -79,30 +81,33 @@
// Add a bunch of generic useful functions
g.funcs = Functions{
+ "Append": listAppend,
+ "Concat": listConcat,
"Contains": strings.Contains,
"Eval": g.eval,
"Globals": func() Map { return globals },
"HasPrefix": strings.HasPrefix,
"HasSuffix": strings.HasSuffix,
"Import": g.importTmpl,
+ "Index": index,
"Is": is,
"Iterate": iterate,
+ "Join": strings.Join,
"List": list,
"Map": newMap,
"PascalCase": text.PascalCase,
- "ToUpper": strings.ToUpper,
- "ToLower": strings.ToLower,
"Repeat": strings.Repeat,
+ "Replace": replace,
+ "SortUnique": listSortUnique,
"Split": strings.Split,
- "Join": strings.Join,
+ "Sum": sum,
"Title": strings.Title,
+ "ToLower": strings.ToLower,
+ "ToUpper": strings.ToUpper,
"TrimLeft": strings.TrimLeft,
"TrimPrefix": strings.TrimPrefix,
"TrimRight": strings.TrimRight,
"TrimSuffix": strings.TrimSuffix,
- "Replace": replace,
- "Index": index,
- "Sum": sum,
"Error": func(err string, args ...any) string { panic(fmt.Errorf(err, args...)) },
}
@@ -235,8 +240,34 @@
return out
}
+// listAppend returns the slice list with items appended
+func listAppend(list any, items ...any) any {
+ itemValues := transform.SliceNoErr(items, reflect.ValueOf)
+ return reflect.Append(reflect.ValueOf(list), itemValues...).Interface()
+}
+
+// listConcat returns a slice formed from concatenating all the elements of all
+// the slice arguments
+func listConcat(firstList any, otherLists ...any) any {
+ out := reflect.ValueOf(firstList)
+ for _, list := range otherLists {
+ out = reflect.AppendSlice(out, reflect.ValueOf(list))
+ }
+ return out.Interface()
+}
+
+// listSortUnique returns items sorted by the string-formatted value of each element, with
+// items with the same strings deduplicated.
+func listSortUnique(items []any) []any {
+ m := make(container.Map[string, any], len(items))
+ for _, item := range items {
+ m.Add(fmt.Sprint(item), item)
+ }
+ return m.Values()
+}
+
// list returns a new slice of elements from the argument list
-// Useful for: {{- range Slice "a" "b" "c" -}}{{.}}{{end}}
+// Useful for: {{- range List "a" "b" "c" -}}{{.}}{{end}}
func list(elements ...any) []any { return elements }
func index(obj any, indices ...any) (any, error) {
