Add -test-query and -test-filter options to cts roll
This lets you run a subset of the CTS on the bots via
the roller script
Examples:
```
tools/run cts roll --test-query "webgpu:api,operation,render_pass,storeOp:*" --max-attempts 0 --repo https://github.com/greggman/cts.git --revision 546ab937546eb4d8af1a98d7c07ff2e6c3c0e120
tools/run cts roll --test-filter "*builtin:texture*" --max-attempts 0 --repo https://github.com/greggman/cts.git --revision 546ab937546eb4d8af1a98d7c07ff2e6c3c0e120
```
Change-Id: I4bcc8ce61cf0fd87d35799b4393ae4e68449269e
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/235314
Reviewed-by: David Neto <dneto@google.com>
Commit-Queue: Gregg Tavares <gman@chromium.org>
diff --git a/tools/src/cmd/cts/common/gen_test_list.go b/tools/src/cmd/cts/common/gen_test_list.go
index 68226c4..05594f0 100644
--- a/tools/src/cmd/cts/common/gen_test_list.go
+++ b/tools/src/cmd/cts/common/gen_test_list.go
@@ -9,7 +9,7 @@
)
// GenTestList queries the CTS for the newline delimited list of test names
-func GenTestList(ctx context.Context, ctsDir, node string) (string, error) {
+func GenTestList(ctx context.Context, ctsDir, node string, testQuery string) (string, error) {
// Run 'src/common/runtime/cmdline.ts' to obtain the full test list
cmd := exec.CommandContext(ctx, node,
"-e", "require('./src/common/tools/setup-ts-in-node.js');require('./src/common/runtime/cmdline.ts');",
@@ -19,7 +19,7 @@
// start at 1, so just inject a placeholder argument.
"placeholder-arg",
"--list",
- "webgpu:*",
+ testQuery,
)
cmd.Dir = ctsDir
diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go
index f379eaf..9d3254b 100644
--- a/tools/src/cmd/cts/roll/roll.go
+++ b/tools/src/cmd/cts/roll/roll.go
@@ -75,6 +75,8 @@
webTestsPath = "webgpu-cts/webtests"
refMain = "refs/heads/main"
noExpectations = `# Clear all expectations to obtain full list of results`
+ testQuery = "webgpu:*"
+ testFilter = ""
)
type rollerFlags struct {
@@ -85,6 +87,8 @@
cacheDir string
ctsGitURL string
ctsRevision string
+ testQuery string
+ testFilter string
force bool // Create a new roll, even if CTS is up to date
rebuild bool // Rebuild the expectations file from scratch
preserve bool // If false, abandon past roll changes
@@ -117,6 +121,8 @@
flag.StringVar(&c.flags.cacheDir, "cache", common.DefaultCacheDir, "path to the results cache")
flag.StringVar(&c.flags.ctsGitURL, "repo", cfg.Git.CTS.HttpsURL(), "the CTS source repo")
flag.StringVar(&c.flags.ctsRevision, "revision", refMain, "revision of the CTS to roll")
+ flag.StringVar(&c.flags.testQuery, "test-query", testQuery, "test query to generate test list")
+ flag.StringVar(&c.flags.testFilter, "test-filter", testFilter, "glob to filter the results of the test query")
flag.BoolVar(&c.flags.force, "force", false, "create a new roll, even if CTS is up to date")
flag.BoolVar(&c.flags.rebuild, "rebuild", false, "rebuild the expectation file from scratch")
flag.BoolVar(&c.flags.preserve, "preserve", false, "do not abandon existing rolls")
@@ -317,7 +323,31 @@
exInfo.expectations = ex
}
- generatedFiles, err := r.generateFiles(ctx, r.cfg.OsWrapper)
+ generatedFiles, err := func(ctx context.Context, osWrapper oswrapper.OSWrapper) (map[string]string, error) {
+ generatedFiles, err := r.generateFiles(ctx, r.cfg.OsWrapper)
+ if err != nil {
+ return nil, err
+ }
+
+ if r.flags.testFilter != "" {
+ log.Printf("filtering test list...")
+ // Filter the test list in place. This way it will get used after
+ // being filtered and written as filtered.
+ newLines := []string{}
+ lines := strings.Split(generatedFiles[common.TestListRelPath], "\n")
+ for _, line := range lines {
+ matched, err := filepath.Match(r.flags.testFilter, line)
+ if err != nil {
+ return nil, fmt.Errorf("error using test-filter '%s': %v", r.flags.testFilter, err)
+ }
+ if matched {
+ newLines = append(newLines, line)
+ }
+ }
+ generatedFiles[common.TestListRelPath] = strings.Join(newLines, "\n")
+ }
+ return generatedFiles, nil
+ }(ctx, r.cfg.OsWrapper)
if err != nil {
return err
}
@@ -790,7 +820,7 @@
func (r *roller) genTestList(
ctx context.Context, fsReader oswrapper.FilesystemReader) (string, error) {
- return common.GenTestList(ctx, r.ctsDir, r.flags.nodePath)
+ return common.GenTestList(ctx, r.ctsDir, r.flags.nodePath, r.flags.testQuery)
}
// genResourceFilesList returns a list of resource files, for the CTS checkout at r.ctsDir
diff --git a/tools/src/cmd/cts/treemap/treemap.go b/tools/src/cmd/cts/treemap/treemap.go
index 4027919..0cca6bf 100644
--- a/tools/src/cmd/cts/treemap/treemap.go
+++ b/tools/src/cmd/cts/treemap/treemap.go
@@ -60,6 +60,7 @@
source common.ResultSource
auth authcli.Flags
keepAlive bool
+ testQuery string
}
}
@@ -84,6 +85,7 @@
c.flags.source.RegisterFlags(cfg)
c.flags.auth.Register(flag.CommandLine, auth.DefaultAuthOptions(cfg.OsWrapper))
flag.BoolVar(&c.flags.keepAlive, "keep-alive", false, "keep the server alive after the page has been closed")
+ flag.StringVar(&c.flags.testQuery, "test-query", "webgpu:*", "cts test query to generate test list")
return []string{"[cases | timing]"}, nil
}
@@ -96,7 +98,7 @@
var err error
switch flag.Arg(0) {
case "case", "cases":
- data, err = loadCasesData()
+ data, err = loadCasesData(c.flags.testQuery)
case "time", "times", "timing":
// Validate command line arguments
@@ -154,8 +156,8 @@
}
// loadCasesData creates the JSON payload for a cases visualization
-func loadCasesData() (string, error) {
- testlist, err := common.GenTestList(context.Background(), common.DefaultCTSPath(), fileutils.NodePath())
+func loadCasesData(testQuery string) (string, error) {
+ testlist, err := common.GenTestList(context.Background(), common.DefaultCTSPath(), fileutils.NodePath(), testQuery)
if err != nil {
return "", err
}
diff --git a/tools/src/cmd/cts/update/testlist/testlist.go b/tools/src/cmd/cts/update/testlist/testlist.go
index 084e7ec..5e1a3ea 100644
--- a/tools/src/cmd/cts/update/testlist/testlist.go
+++ b/tools/src/cmd/cts/update/testlist/testlist.go
@@ -54,9 +54,10 @@
type cmd struct {
flags struct {
- ctsDir string
- nodePath string
- output string
+ ctsDir string
+ nodePath string
+ output string
+ testQuery string
}
}
@@ -72,11 +73,13 @@
flag.StringVar(&c.flags.ctsDir, "cts", common.DefaultCTSPath(), "path to the CTS")
flag.StringVar(&c.flags.nodePath, "node", fileutils.NodePath(), "path to node")
flag.StringVar(&c.flags.output, "out", common.DefaultTestListPath(), "output testlist path")
+ flag.StringVar(&c.flags.testQuery, "test-query", "webgpu:*", "cts test query to generate test list")
+
return nil, nil
}
func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
- list, err := common.GenTestList(ctx, c.flags.ctsDir, c.flags.nodePath)
+ list, err := common.GenTestList(ctx, c.flags.ctsDir, c.flags.nodePath, c.flags.testQuery)
if err != nil {
return err
}
