[tools][cts] Improve 'unimplemented' calculations
All statuses other than 'unimplemented' are the sum across all the bot variants, while the 'unimplemented' were just the count of tests.
This resulted in a 10-15x difference in values, which is misleading.
These were also being double-counted as the 'Unimplemented' were being classed as 'Skip' by the test runner.
To improve this, reclassify 'Skip' cases as 'Unimplemented' by looking at the test queries.
Change-Id: Ic8e161294de94081856779e2f5df31fb98ac9901
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/175220
Auto-Submit: Ben Clayton <bclayton@google.com>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
Kokoro: Ben Clayton <bclayton@google.com>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
diff --git a/tools/src/cmd/cts/common/export.go b/tools/src/cmd/cts/common/export.go
index 0c2123f..9294225 100644
--- a/tools/src/cmd/cts/common/export.go
+++ b/tools/src/cmd/cts/common/export.go
@@ -36,6 +36,7 @@
"time"
"dawn.googlesource.com/dawn/tools/src/container"
+ "dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"go.chromium.org/luci/auth"
"google.golang.org/api/option"
@@ -64,8 +65,8 @@
return fmt.Errorf("failed to get spreadsheet: %w", err)
}
- // Grab the CTS revision to count the number of unimplemented tests
- numUnimplemented, err := CountUnimplementedTests(ctx, ctsDir, node, npmPath)
+ // Get the unimplemented tests
+ unimplemented, err := GetUnimplementedTests(ctx, ctsDir, node, npmPath)
if err != nil {
return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
}
@@ -90,7 +91,16 @@
counts := container.NewMap[string, int]()
for _, r := range results {
- counts[string(r.Status)] = counts[string(r.Status)] + 1
+ if r.Status == "Skip" {
+ // Unimplemented tests are marked as 'Skip' by the CTS runner.
+ // Check the unimplemented query tree to see if this should be classed as
+ // 'Unimplemented' instead.
+ if node := unimplemented.Get(r.Query); node != nil && node.Data != nil {
+ counts[string("Unimplemented")]++
+ continue
+ }
+ }
+ counts[string(r.Status)]++
}
{ // Report statuses that have no column
@@ -107,8 +117,6 @@
switch strings.ToLower(column) {
case "date":
data = append(data, time.Now().UTC().Format("2006-01-02"))
- case "unimplemented":
- data = append(data, numUnimplemented)
default:
count, ok := counts[column]
if !ok {
@@ -194,11 +202,11 @@
return nil
}
-// CountUnimplementedTests returns the number of unimplemented tests in the CTS.
+// GetUnimplementedTests returns the unimplemented tests in the CTS.
// It does this by running the cmdline.ts CTS command with '--list-unimplemented webgpu:*'.
-func CountUnimplementedTests(ctx context.Context, ctsDir, node, npmPath string) (int, error) {
+func GetUnimplementedTests(ctx context.Context, ctsDir, node, npmPath string) (*query.Tree[struct{}], error) {
if err := InstallCTSDeps(ctx, ctsDir, npmPath); err != nil {
- return 0, err
+ return nil, err
}
// Run 'src/common/runtime/cmdline.ts' to obtain the full test list
@@ -216,15 +224,16 @@
out, err := cmd.CombinedOutput()
if err != nil {
- return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
+ return nil, fmt.Errorf("failed to gather unimplemented tests: %w", err)
}
- lines := strings.Split(string(out), "\n")
- count := 0
- for _, line := range lines {
- if strings.TrimSpace(line) != "" {
- count++
+ tree, _ := query.NewTree[struct{}]()
+ for _, line := range strings.Split(string(out), "\n") {
+ line = strings.TrimSpace(line)
+ if line != "" {
+ q := query.Parse(line)
+ tree.Add(q, struct{}{})
}
}
- return count, nil
+ return &tree, nil
}
diff --git a/tools/src/cts/query/tree.go b/tools/src/cts/query/tree.go
index eac5568..91ac323 100644
--- a/tools/src/cts/query/tree.go
+++ b/tools/src/cts/query/tree.go
@@ -28,6 +28,7 @@
package query
import (
+ "errors"
"fmt"
"io"
"sort"
@@ -310,6 +311,22 @@
return node.Data
}
+// errStop is an error used to stop traversal of Query.Walk
+var errStop = errors.New("stop")
+
+// Get returns the closest existing tree node for the given query
+func (t *Tree[Data]) Get(q Query) *TreeNode[Data] {
+ node := &t.TreeNode
+ q.Walk(func(q Query, t Target, n string) error {
+ if n := node.Children[TreeNodeChildKey{n, t}]; n != nil {
+ node = n
+ return nil
+ }
+ return errStop
+ })
+ return node
+}
+
// Reduce reduces the tree using the Merger function f.
// If the Merger function returns a non-nil Data value, then this will be used
// to replace the non-leaf node with a new leaf node holding the returned Data.
diff --git a/tools/src/git/git.go b/tools/src/git/git.go
index 9de5e09..ab30476 100644
--- a/tools/src/git/git.go
+++ b/tools/src/git/git.go
@@ -65,7 +65,7 @@
}
// The timeout for git operations if no other timeout is specified
-var DefaultTimeout = time.Minute
+var DefaultTimeout = 5 * time.Minute
// Git wraps the 'git' executable
type Git struct {
