tools: Add the 'cts export' sub-command
Updates a Google spreadsheet with latest CTS statistics.
Bug: dawn:1342
Change-Id: I01a9c99032c755d6a78af674675d8c9cbfc79019
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/88443
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
diff --git a/go.mod b/go.mod
index b6d7bb1..21d9656 100644
--- a/go.mod
+++ b/go.mod
@@ -14,15 +14,19 @@
github.com/tidwall/jsonc v0.3.2
go.chromium.org/luci v0.0.0-20220412023008-ab2409fe739a
golang.org/x/net v0.0.0-20220403103023-749bd193bc2b
+ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
+ google.golang.org/api v0.63.0
google.golang.org/protobuf v1.28.0
)
require (
cloud.google.com/go v0.99.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
+ github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/klauspost/compress v1.13.5 // indirect
github.com/kr/pretty v0.3.0 // indirect
@@ -34,11 +38,10 @@
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
- golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+ go.opencensus.io v0.23.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
golang.org/x/text v0.3.7 // indirect
- google.golang.org/api v0.63.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8 // indirect
google.golang.org/grpc v1.44.0 // indirect
diff --git a/go.sum b/go.sum
index bdd66d6..0b540d3 100644
--- a/go.sum
+++ b/go.sum
@@ -174,6 +174,7 @@
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -257,6 +258,7 @@
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
+github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20210901121439-eee08aaf2717 h1:V1j4G8AXIJeyzT3ng2Oh4IRo/VEgRWYAsyYwhOz5rko=
@@ -490,6 +492,7 @@
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=
diff --git a/tools/src/cmd/cts/common/deps.go b/tools/src/cmd/cts/common/deps.go
new file mode 100644
index 0000000..27b13d3
--- /dev/null
+++ b/tools/src/cmd/cts/common/deps.go
@@ -0,0 +1,71 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+ "fmt"
+ "regexp"
+ "strings"
+
+ "dawn.googlesource.com/dawn/tools/src/container"
+)
+
+// The regular expression used to search for the CTS hash
+var reCTSHash = regexp.MustCompile(reEscape(ctsHashPrefix) + `[0-9a-fA-F]+`)
+
+const (
+ // The string prefix for the CTS hash in the DEPs file, used for identifying
+ // and updating the DEPS file.
+ ctsHashPrefix = `{chromium_git}/external/github.com/gpuweb/cts@`
+)
+
+func reEscape(s string) string {
+ return strings.ReplaceAll(strings.ReplaceAll(s, `/`, `\/`), `.`, `\.`)
+}
+
+// UpdateCTSHashInDeps replaces the CTS hashes in 'deps' with 'newCTSHash'.
+// Returns:
+// newDEPS - the new DEPS content
+// oldCTSHash - the old CTS hash in the 'deps'
+func UpdateCTSHashInDeps(deps, newCTSHash string) (newDEPS, oldCTSHash string, err error) {
+ // Collect old CTS hashes, and replace these with newCTSHash
+ b := strings.Builder{}
+ oldCTSHashes := []string{}
+ matches := reCTSHash.FindAllStringIndex(deps, -1)
+ if len(matches) == 0 {
+ return "", "", fmt.Errorf("failed to find a CTS hash in DEPS file")
+ }
+ end := 0
+ for _, match := range matches {
+ oldCTSHashes = append(oldCTSHashes, deps[match[0]+len(ctsHashPrefix):match[1]])
+ b.WriteString(deps[end:match[0]])
+ b.WriteString(ctsHashPrefix + newCTSHash)
+ end = match[1]
+ }
+ b.WriteString(deps[end:])
+
+ newDEPS = b.String()
+ if deps == newDEPS {
+ fmt.Println("CTS is already up to date")
+ return "", "", nil
+ }
+
+ if s := container.NewSet(oldCTSHashes...); len(s) > 1 {
+ fmt.Println("DEPS contained multiple hashes for CTS, using first for logs")
+ }
+ oldCTSHash = oldCTSHashes[0]
+
+ return newDEPS, oldCTSHash, nil
+}
diff --git a/tools/src/cmd/cts/export/export.go b/tools/src/cmd/cts/export/export.go
new file mode 100644
index 0000000..2e5826e
--- /dev/null
+++ b/tools/src/cmd/cts/export/export.go
@@ -0,0 +1,307 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package export
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "dawn.googlesource.com/dawn/tools/src/buildbucket"
+ "dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
+ "dawn.googlesource.com/dawn/tools/src/cts/result"
+ "dawn.googlesource.com/dawn/tools/src/gerrit"
+ "dawn.googlesource.com/dawn/tools/src/git"
+ "dawn.googlesource.com/dawn/tools/src/gitiles"
+ "dawn.googlesource.com/dawn/tools/src/resultsdb"
+ "dawn.googlesource.com/dawn/tools/src/utils"
+ "go.chromium.org/luci/auth/client/authcli"
+ "golang.org/x/oauth2"
+ "golang.org/x/oauth2/google"
+ "google.golang.org/api/sheets/v4"
+)
+
+func init() {
+ common.Register(&cmd{})
+}
+
+type cmd struct {
+ flags struct {
+ auth authcli.Flags
+ cacheDir string
+ }
+}
+
+func (cmd) Name() string {
+ return "export"
+}
+
+func (cmd) Desc() string {
+ return "exports the latest CTS results to Google sheets"
+}
+
+func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
+ c.flags.auth.Register(flag.CommandLine, common.DefaultAuthOptions())
+ flag.StringVar(&c.flags.cacheDir, "cache", common.DefaultCacheDir, "path to the results cache")
+ return nil, nil
+}
+
+func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
+ // Validate command line arguments
+ auth, err := c.flags.auth.Options()
+ if err != nil {
+ return fmt.Errorf("failed to obtain authentication options: %w", err)
+ }
+
+ // Load the credentials used for accessing the sheets document
+ authdir := utils.ExpandHome(os.ExpandEnv(auth.SecretsDir))
+ credentialsPath := filepath.Join(authdir, "credentials.json")
+ b, err := ioutil.ReadFile(credentialsPath)
+ if err != nil {
+ return fmt.Errorf("unable to read credentials file '%v'\n"+
+ "Obtain this file from: https://console.developers.google.com/apis/credentials\n%w",
+ credentialsPath, err)
+ }
+ credentials, err := google.CredentialsFromJSON(ctx, b, "https://www.googleapis.com/auth/spreadsheets")
+ if err != nil {
+ return fmt.Errorf("unable to parse client secret file to config: %w", err)
+ }
+
+ // Create the sheets service client
+ s, err := sheets.New(oauth2.NewClient(ctx, credentials.TokenSource))
+ if err != nil {
+ return fmt.Errorf("unable to create sheets client: %w", err)
+ }
+
+ // Get the CTS statistics spreadsheet
+ spreadsheet, err := s.Spreadsheets.Get(cfg.Sheets.ID).Do()
+ if err != nil {
+ return fmt.Errorf("failed to get spreadsheet: %w", err)
+ }
+
+ // Scan the sheets of the spreadsheet (tabs at the bottom) for the 'data'
+ // sheet.
+ var dataSheet *sheets.Sheet
+ for _, sheet := range spreadsheet.Sheets {
+ if strings.ToLower(sheet.Properties.Title) == "data" {
+ dataSheet = sheet
+ break
+ }
+ }
+ if dataSheet == nil {
+ return fmt.Errorf("failed to find data sheet")
+ }
+
+ // Fetch the table column names
+ columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0)
+
+ // Create a gerrit client, and find the latest CTS roll
+ gerrit, err := gerrit.New(cfg.Gerrit.Host, gerrit.Credentials{})
+ if err != nil {
+ return err
+ }
+ latestRoll, err := common.LatestCTSRoll(gerrit)
+ if err != nil {
+ return err
+ }
+
+ // Grab the results from the latest CTS roll
+ bb, err := buildbucket.New(ctx, auth)
+ if err != nil {
+ return err
+ }
+ rdb, err := resultsdb.New(ctx, auth)
+ if err != nil {
+ return err
+ }
+ results, ps, err := common.MostRecentResultsForChange(ctx, cfg, c.flags.cacheDir, gerrit, bb, rdb, latestRoll.Number)
+ if err != nil {
+ return err
+ }
+ if len(results) == 0 {
+ return fmt.Errorf("no results found")
+ }
+
+ // Find the CTS revision from the latest CTS roll
+ dawn, err := gitiles.New(ctx, cfg.Git.Dawn.Host, cfg.Git.Dawn.Project)
+ if err != nil {
+ return fmt.Errorf("failed to open dawn host: %w", err)
+ }
+ deps, err := dawn.DownloadFile(ctx, ps.RefsChanges(), "DEPS")
+ if err != nil {
+ return fmt.Errorf("failed to download DEPS from %v: %w", ps.RefsChanges(), err)
+ }
+ _, ctsHash, err := common.UpdateCTSHashInDeps(deps, "<unused>")
+ if err != nil {
+ return fmt.Errorf("failed to find CTS hash in deps: %w", err)
+ }
+
+ // Grab the CTS revision to count the number of unimplemented tests
+ numUnimplemented, err := countUnimplementedTests(cfg, ctsHash)
+ if err != nil {
+ return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
+ }
+
+ // Generate a new set of counts of test by status
+ log.Printf("exporting results from cl %v ps %v...", ps.Change, ps.Patchset)
+ counts := map[result.Status]int{}
+ for _, r := range results {
+ counts[r.Status] = counts[r.Status] + 1
+ }
+
+ // Generate new cell data based on the table column names
+ data := []any{}
+ for _, column := range columns {
+ switch strings.ToLower(column) {
+ case "date":
+ data = append(data, time.Now().UTC().Format("2006-01-02"))
+ case "change":
+ data = append(data, ps.Change)
+ case "unimplemented":
+ data = append(data, numUnimplemented)
+ default:
+ count, ok := counts[result.Status(column)]
+ if !ok {
+ log.Println("no results with status", column)
+ }
+ data = append(data, count)
+ }
+ }
+
+ // Add a new row to the spreadsheet
+ _, err = s.Spreadsheets.Values.Append(spreadsheet.SpreadsheetId, "Data", &sheets.ValueRange{
+ Values: [][]any{data},
+ }).ValueInputOption("RAW").Do()
+ if err != nil {
+ return fmt.Errorf("failed to update spreadsheet: %v", err)
+ }
+
+ return nil
+}
+
+// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
+// given index.
+func rowRange(index int, sheet *sheets.Sheet) string {
+ return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
+}
+
+// columnRange returns a sheets range ("name!i1:i") for the entire column with
+// the given index.
+func columnRange(index int, sheet *sheets.Sheet) string {
+ col := 'A' + index
+ if index > 25 {
+ panic("UNIMPLEMENTED")
+ }
+ return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
+}
+
+// fetchRow returns all the values in the given sheet's row.
+func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) {
+ rng := rowRange(row, sheet)
+ data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+ if err != nil {
+ return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err)
+ }
+ out := make([]T, len(data.Values[0]))
+ for column, v := range data.Values[0] {
+ val, ok := v.(T)
+ if !ok {
+ return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val)
+ }
+ out[column] = val
+ }
+ return out, nil
+}
+
+// countUnimplementedTests checks out the WebGPU CTS at ctsHash, builds the node
+// command line tool, and runs it with '--list-unimplemented webgpu:*' to count
+// the total number of unimplemented tests, which is returned.
+func countUnimplementedTests(cfg common.Config, ctsHash string) (int, error) {
+ tmpDir, err := os.MkdirTemp("", "dawn-cts-export")
+ if err != nil {
+ return 0, err
+ }
+ defer os.RemoveAll(tmpDir)
+
+ dir := filepath.Join(tmpDir, "cts")
+
+ gitExe, err := exec.LookPath("git")
+ if err != nil {
+ return 0, fmt.Errorf("failed to find git on PATH: %w", err)
+ }
+
+ git, err := git.New(gitExe)
+ if err != nil {
+ return 0, err
+ }
+
+ log.Printf("cloning cts to '%v'...", dir)
+ repo, err := git.Clone(dir, cfg.Git.CTS.HttpsURL(), nil)
+ if err != nil {
+ return 0, fmt.Errorf("failed to clone cts: %v", err)
+ }
+
+ log.Printf("checking out cts @ '%v'...", ctsHash)
+ if _, err := repo.Fetch(ctsHash, nil); err != nil {
+ return 0, fmt.Errorf("failed to fetch cts: %v", err)
+ }
+ if err := repo.Checkout(ctsHash, nil); err != nil {
+ return 0, fmt.Errorf("failed to clone cts: %v", err)
+ }
+
+ {
+ npm, err := exec.LookPath("npm")
+ if err != nil {
+ return 0, fmt.Errorf("failed to find npm on PATH: %w", err)
+ }
+ cmd := exec.Command(npm, "ci")
+ cmd.Dir = dir
+ if out, err := cmd.CombinedOutput(); err != nil {
+ return 0, fmt.Errorf("failed to run npm ci: %w\n%v", err, string(out))
+ }
+ }
+ {
+ npx, err := exec.LookPath("npx")
+ if err != nil {
+ return 0, fmt.Errorf("failed to find npx on PATH: %w", err)
+ }
+ cmd := exec.Command(npx, "grunt", "run:build-out-node")
+ cmd.Dir = dir
+ if out, err := cmd.CombinedOutput(); err != nil {
+ return 0, fmt.Errorf("failed to build CTS typescript: %w\n%v", err, string(out))
+ }
+ }
+ {
+ sh, err := exec.LookPath("node")
+ if err != nil {
+ return 0, fmt.Errorf("failed to find sh on PATH: %w", err)
+ }
+ cmd := exec.Command(sh, "./tools/run_node", "--list-unimplemented", "webgpu:*")
+ cmd.Dir = dir
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
+ }
+ lines := strings.Split(string(out), "\n")
+ return len(lines), nil
+ }
+}
diff --git a/tools/src/cmd/cts/main.go b/tools/src/cmd/cts/main.go
index 0aefd29..a44b15d 100644
--- a/tools/src/cmd/cts/main.go
+++ b/tools/src/cmd/cts/main.go
@@ -28,6 +28,7 @@
"dawn.googlesource.com/dawn/tools/src/utils"
// Register sub-commands
+ _ "dawn.googlesource.com/dawn/tools/src/cmd/cts/export"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/format"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/merge"
_ "dawn.googlesource.com/dawn/tools/src/cmd/cts/results"
