[tools][cts] Add 'cts update-testlist'
To re-generate the local testlist with the list generated from `third_party/webgpu-cts`.
Rename `cts update` to `cts update-expectations`.
Make `cts treemap cases` query the CTS instead of reading from `third_party/gn/webgpu-cts/test_list.txt`. This makes the profiling <-> edit cycle smoother.
Change-Id: I2292443402fe45f2e56fd681d66405f78512574f
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/172660
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Auto-Submit: Ben Clayton <bclayton@google.com>
Commit-Queue: Antonio Maiorano <amaiorano@google.com>
diff --git a/tools/src/cmd/cts/common/gen_test_list.go b/tools/src/cmd/cts/common/gen_test_list.go
new file mode 100644
index 0000000..68226c4
--- /dev/null
+++ b/tools/src/cmd/cts/common/gen_test_list.go
@@ -0,0 +1,42 @@
+package common
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "os/exec"
+ "strings"
+)
+
+// GenTestList queries the CTS for the newline delimited list of test names
+func GenTestList(ctx context.Context, ctsDir, node string) (string, error) {
+ // Run 'src/common/runtime/cmdline.ts' to obtain the full test list
+ cmd := exec.CommandContext(ctx, node,
+ "-e", "require('./src/common/tools/setup-ts-in-node.js');require('./src/common/runtime/cmdline.ts');",
+ "--", // Start of arguments
+ // src/common/runtime/helper/sys.ts expects 'node file.js <args>'
+ // and slices away the first two arguments. When running with '-e', args
+ // start at 1, so just inject a placeholder argument.
+ "placeholder-arg",
+ "--list",
+ "webgpu:*",
+ )
+ cmd.Dir = ctsDir
+
+ stderr := bytes.Buffer{}
+ cmd.Stderr = &stderr
+
+ out, err := cmd.Output()
+ if err != nil {
+ return "", fmt.Errorf("failed to generate test list: %w\n%v", err, stderr.String())
+ }
+
+ tests := []string{}
+ for _, test := range strings.Split(string(out), "\n") {
+ if test != "" {
+ tests = append(tests, test)
+ }
+ }
+
+ return strings.Join(tests, "\n"), nil
+}
diff --git a/tools/src/cmd/cts/common/paths.go b/tools/src/cmd/cts/common/paths.go
index 3cbe03d..17d1bb7 100644
--- a/tools/src/cmd/cts/common/paths.go
+++ b/tools/src/cmd/cts/common/paths.go
@@ -47,9 +47,11 @@
// slow_tests.txt file.
RelativeSlowExpectationsPath = "webgpu-cts/slow_tests.txt"
- // RelativeTestListPath is the dawn-root relative path to the
- // test_list.txt file.
+ // RelativeTestListPath is the dawn-root relative path to the test_list.txt file.
RelativeTestListPath = "third_party/gn/webgpu-cts/test_list.txt"
+
+ // RelativeCTSPath is the dawn-root relative path to the WebGPU CTS directory.
+ RelativeCTSPath = "third_party/webgpu-cts"
)
// DefaultExpectationsPath returns the default path to the expectations.txt
@@ -91,8 +93,8 @@
return path
}
-// DefaultTestListPath returns the default path to the test_list.txt
-// file. Returns an empty string if the file cannot be found.
+// DefaultTestListPath returns the default path to the test_list.txt file.
+// Returns an empty string if the file cannot be found.
func DefaultTestListPath() string {
path := filepath.Join(fileutils.DawnRoot(), RelativeTestListPath)
if _, err := os.Stat(path); err != nil {
@@ -100,3 +102,13 @@
}
return path
}
+
+// DefaultCTSPath returns the default path to the WenGPU CTS directory.
+// Returns an empty string if the file cannot be found.
+func DefaultCTSPath() string {
+ path := filepath.Join(fileutils.DawnRoot(), RelativeCTSPath)
+ if _, err := os.Stat(path); err != nil {
+ return ""
+ }
+ return path
+}
diff --git a/tools/src/cmd/cts/main.go b/tools/src/cmd/cts/main.go
index b8e4f77..8b4f01c 100644
--- a/tools/src/cmd/cts/main.go
+++ b/tools/src/cmd/cts/main.go
@@ -48,7 +48,8 @@
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/roll"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/time"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/treemap"
- _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/update"
+ _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/update/expectations"
+ _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/update/testlist"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/validate"
)
diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go
index bc85448..f1e0734 100644
--- a/tools/src/cmd/cts/roll/roll.go
+++ b/tools/src/cmd/cts/roll/roll.go
@@ -28,7 +28,6 @@
package roll
import (
- "bytes"
"context"
"encoding/json"
"flag"
@@ -809,35 +808,7 @@
// genTestList returns the newline delimited list of test names, for the CTS checkout at r.ctsDir
func (r *roller) genTestList(ctx context.Context) (string, error) {
- // Run 'src/common/runtime/cmdline.ts' to obtain the full test list
- cmd := exec.CommandContext(ctx, r.flags.nodePath,
- "-e", "require('./src/common/tools/setup-ts-in-node.js');require('./src/common/runtime/cmdline.ts');",
- "--", // Start of arguments
- // src/common/runtime/helper/sys.ts expects 'node file.js <args>'
- // and slices away the first two arguments. When running with '-e', args
- // start at 1, so just inject a placeholder argument.
- "placeholder-arg",
- "--list",
- "webgpu:*",
- )
- cmd.Dir = r.ctsDir
-
- stderr := bytes.Buffer{}
- cmd.Stderr = &stderr
-
- out, err := cmd.Output()
- if err != nil {
- return "", fmt.Errorf("failed to generate test list: %w\n%v", err, stderr.String())
- }
-
- tests := []string{}
- for _, test := range strings.Split(string(out), "\n") {
- if test != "" {
- tests = append(tests, test)
- }
- }
-
- return strings.Join(tests, "\n"), nil
+ return common.GenTestList(ctx, r.ctsDir, r.flags.nodePath)
}
// genResourceFilesList returns a list of resource files, for the CTS checkout at r.ctsDir
diff --git a/tools/src/cmd/cts/treemap/treemap.go b/tools/src/cmd/cts/treemap/treemap.go
index ff93e6c..a6f4b8b 100644
--- a/tools/src/cmd/cts/treemap/treemap.go
+++ b/tools/src/cmd/cts/treemap/treemap.go
@@ -155,17 +155,13 @@
// loadCasesData creates the JSON payload for a cases visualization
func loadCasesData() (string, error) {
- testListPath := filepath.Join(fileutils.DawnRoot(), common.TestListRelPath)
-
- file, err := os.Open(testListPath)
+ testlist, err := common.GenTestList(context.Background(), common.DefaultCTSPath(), fileutils.NodePath())
if err != nil {
- return "", fmt.Errorf("failed to open test list: %w", err)
+ return "", err
}
- defer file.Close()
-
queryCounts := container.NewMap[string, int]()
- scanner := bufio.NewScanner(file)
+ scanner := bufio.NewScanner(strings.NewReader(testlist))
for scanner.Scan() {
if name := strings.TrimSpace(scanner.Text()); name != "" {
q := query.Parse(name)
@@ -184,7 +180,7 @@
data := &strings.Builder{}
fmt.Fprint(data, `{`)
fmt.Fprint(data, `"desc":"Treemap visualization of the CTS test cases.<br>Area represents total number of test cases.<br>Color represents the number of parameterized test cases for a single test.",`)
- fmt.Fprint(data, `"limit": 5000,`)
+ fmt.Fprintf(data, `"limit": 1000,`)
fmt.Fprint(data, `"data":[`)
fmt.Fprint(data, `["Query", "Parent", "Number of tests", "Color"],`)
fmt.Fprint(data, `["root", null, 0, 0]`)
diff --git a/tools/src/cmd/cts/update/update.go b/tools/src/cmd/cts/update/expectations/expectations.go
similarity index 98%
rename from tools/src/cmd/cts/update/update.go
rename to tools/src/cmd/cts/update/expectations/expectations.go
index ea3b4fe..1bbf024 100644
--- a/tools/src/cmd/cts/update/update.go
+++ b/tools/src/cmd/cts/update/expectations/expectations.go
@@ -25,7 +25,7 @@
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-package update
+package expectations
import (
"context"
@@ -68,7 +68,7 @@
}
func (cmd) Name() string {
- return "update"
+ return "update-expectations"
}
func (cmd) Desc() string {
diff --git a/tools/src/cmd/cts/update/testlist/testlist.go b/tools/src/cmd/cts/update/testlist/testlist.go
new file mode 100644
index 0000000..395b133
--- /dev/null
+++ b/tools/src/cmd/cts/update/testlist/testlist.go
@@ -0,0 +1,85 @@
+// Copyright 2022 The Dawn & Tint Authors
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package testlist
+
+import (
+ "context"
+ "flag"
+ "os"
+ "strings"
+
+ "dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
+ "dawn.googlesource.com/dawn/tools/src/fileutils"
+)
+
+func init() {
+ common.Register(&cmd{})
+}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+ return strings.Join((*i), " ")
+}
+
+func (i *arrayFlags) Set(value string) error {
+ *i = append(*i, value)
+ return nil
+}
+
+type cmd struct {
+ flags struct {
+ ctsDir string
+ nodePath string
+ output string
+ }
+}
+
+func (cmd) Name() string {
+ return "update-testlist"
+}
+
+func (cmd) Desc() string {
+ return "updates a CTS expectations file"
+}
+
+func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
+ flag.StringVar(&c.flags.ctsDir, "cts", common.DefaultCTSPath(), "path to the CTS")
+ flag.StringVar(&c.flags.nodePath, "node", fileutils.NodePath(), "path to node")
+ flag.StringVar(&c.flags.output, "out", common.DefaultTestListPath(), "output testlist path")
+ return nil, nil
+}
+
+func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
+ list, err := common.GenTestList(ctx, c.flags.ctsDir, c.flags.nodePath)
+ if err != nil {
+ return err
+ }
+
+ return os.WriteFile(c.flags.output, []byte(list), 0666)
+}
