blob: 2f85974a47e7b63e1edf0b0fef9726a548f8a0f4 [file] [log] [blame]
// Copyright 2022 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package export
import (
"context"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"dawn.googlesource.com/dawn/tools/src/fileutils"
"dawn.googlesource.com/dawn/tools/src/git"
"dawn.googlesource.com/dawn/tools/src/gitiles"
"go.chromium.org/luci/auth/client/authcli"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/sheets/v4"
)
func init() {
common.Register(&cmd{})
}
type cmd struct {
flags struct {
auth authcli.Flags
results common.ResultSource
}
}
func (cmd) Name() string {
return "export"
}
func (cmd) Desc() string {
return "exports the latest CTS results to Google sheets"
}
func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
c.flags.auth.Register(flag.CommandLine, common.DefaultAuthOptions())
c.flags.results.RegisterFlags(cfg)
return nil, nil
}
func (c *cmd) Run(ctx context.Context, cfg common.Config) error {
// Validate command line arguments
auth, err := c.flags.auth.Options()
if err != nil {
return fmt.Errorf("failed to obtain authentication options: %w", err)
}
// Load the credentials used for accessing the sheets document
authdir := fileutils.ExpandHome(os.ExpandEnv(auth.SecretsDir))
credentialsPath := filepath.Join(authdir, "credentials.json")
b, err := ioutil.ReadFile(credentialsPath)
if err != nil {
return fmt.Errorf("unable to read credentials file '%v'\n"+
"Obtain this file from: https://console.developers.google.com/apis/credentials\n%w",
credentialsPath, err)
}
credentials, err := google.CredentialsFromJSON(ctx, b, "https://www.googleapis.com/auth/spreadsheets")
if err != nil {
return fmt.Errorf("unable to parse client secret file to config: %w", err)
}
// Create the sheets service client
s, err := sheets.New(oauth2.NewClient(ctx, credentials.TokenSource))
if err != nil {
return fmt.Errorf("unable to create sheets client: %w", err)
}
// Get the CTS statistics spreadsheet
spreadsheet, err := s.Spreadsheets.Get(cfg.Sheets.ID).Do()
if err != nil {
return fmt.Errorf("failed to get spreadsheet: %w", err)
}
// Scan the sheets of the spreadsheet (tabs at the bottom) for the 'data'
// sheet.
var dataSheet *sheets.Sheet
for _, sheet := range spreadsheet.Sheets {
if strings.ToLower(sheet.Properties.Title) == "data" {
dataSheet = sheet
break
}
}
if dataSheet == nil {
return fmt.Errorf("failed to find data sheet")
}
// Fetch the table column names
columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0)
// Grab the results
results, err := c.flags.results.GetResults(ctx, cfg, auth)
if err != nil {
return err
}
if len(results) == 0 {
return fmt.Errorf("no results found")
}
ps := c.flags.results.Patchset
// Find the CTS revision
dawn, err := gitiles.New(ctx, cfg.Git.Dawn.Host, cfg.Git.Dawn.Project)
if err != nil {
return fmt.Errorf("failed to open dawn host: %w", err)
}
deps, err := dawn.DownloadFile(ctx, ps.RefsChanges(), "DEPS")
if err != nil {
return fmt.Errorf("failed to download DEPS from %v: %w", ps.RefsChanges(), err)
}
_, ctsHash, err := common.UpdateCTSHashInDeps(deps, "<unused>")
if err != nil {
return fmt.Errorf("failed to find CTS hash in deps: %w", err)
}
// Grab the CTS revision to count the number of unimplemented tests
numUnimplemented, err := countUnimplementedTests(cfg, ctsHash)
if err != nil {
return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
}
// Generate a new set of counts of test by status
log.Printf("exporting results from cl %v ps %v...", ps.Change, ps.Patchset)
counts := map[result.Status]int{}
for _, r := range results {
counts[r.Status] = counts[r.Status] + 1
}
// Generate new cell data based on the table column names
data := []any{}
for _, column := range columns {
switch strings.ToLower(column) {
case "date":
data = append(data, time.Now().UTC().Format("2006-01-02"))
case "change":
data = append(data, ps.Change)
case "unimplemented":
data = append(data, numUnimplemented)
default:
count, ok := counts[result.Status(column)]
if !ok {
log.Println("no results with status", column)
}
data = append(data, count)
}
}
// Insert a blank row under the column header row
if err := insertBlankRows(s, spreadsheet, dataSheet, 1, 1); err != nil {
return err
}
// Add a new row to the spreadsheet
_, err = s.Spreadsheets.Values.BatchUpdate(spreadsheet.SpreadsheetId,
&sheets.BatchUpdateValuesRequest{
ValueInputOption: "RAW",
Data: []*sheets.ValueRange{{
Range: rowRange(1, dataSheet),
Values: [][]any{data},
}},
}).Do()
if err != nil {
return fmt.Errorf("failed to update spreadsheet: %v", err)
}
return nil
}
// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
// given index.
func rowRange(index int, sheet *sheets.Sheet) string {
return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
}
// columnRange returns a sheets range ("name!i1:i") for the entire column with
// the given index.
func columnRange(index int, sheet *sheets.Sheet) string {
col := 'A' + index
if index > 25 {
panic("UNIMPLEMENTED")
}
return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
}
// fetchRow returns all the values in the given sheet's row.
func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) {
rng := rowRange(row, sheet)
data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
if err != nil {
return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err)
}
out := make([]T, len(data.Values[0]))
for column, v := range data.Values[0] {
val, ok := v.(T)
if !ok {
return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val)
}
out[column] = val
}
return out, nil
}
// insertBlankRows inserts blank rows into the given sheet.
func insertBlankRows(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, aboveRow, count int) error {
req := sheets.BatchUpdateSpreadsheetRequest{
Requests: []*sheets.Request{{
InsertRange: &sheets.InsertRangeRequest{
Range: &sheets.GridRange{
SheetId: sheet.Properties.SheetId,
StartRowIndex: int64(aboveRow),
EndRowIndex: int64(aboveRow + count),
},
ShiftDimension: "ROWS",
}},
},
}
if _, err := srv.Spreadsheets.BatchUpdate(spreadsheet.SpreadsheetId, &req).Do(); err != nil {
return fmt.Errorf("BatchUpdate failed: %v", err)
}
return nil
}
// countUnimplementedTests checks out the WebGPU CTS at ctsHash, builds the node
// command line tool, and runs it with '--list-unimplemented webgpu:*' to count
// the total number of unimplemented tests, which is returned.
func countUnimplementedTests(cfg common.Config, ctsHash string) (int, error) {
tmpDir, err := os.MkdirTemp("", "dawn-cts-export")
if err != nil {
return 0, err
}
defer os.RemoveAll(tmpDir)
dir := filepath.Join(tmpDir, "cts")
gitExe, err := exec.LookPath("git")
if err != nil {
return 0, fmt.Errorf("failed to find git on PATH: %w", err)
}
git, err := git.New(gitExe)
if err != nil {
return 0, err
}
log.Printf("cloning cts to '%v'...", dir)
repo, err := git.Clone(dir, cfg.Git.CTS.HttpsURL(), nil)
if err != nil {
return 0, fmt.Errorf("failed to clone cts: %v", err)
}
log.Printf("checking out cts @ '%v'...", ctsHash)
if _, err := repo.Fetch(ctsHash, nil); err != nil {
return 0, fmt.Errorf("failed to fetch cts: %v", err)
}
if err := repo.Checkout(ctsHash, nil); err != nil {
return 0, fmt.Errorf("failed to clone cts: %v", err)
}
{
npm, err := exec.LookPath("npm")
if err != nil {
return 0, fmt.Errorf("failed to find npm on PATH: %w", err)
}
cmd := exec.Command(npm, "ci")
cmd.Dir = dir
if out, err := cmd.CombinedOutput(); err != nil {
return 0, fmt.Errorf("failed to run npm ci: %w\n%v", err, string(out))
}
}
{
npx, err := exec.LookPath("npx")
if err != nil {
return 0, fmt.Errorf("failed to find npx on PATH: %w", err)
}
cmd := exec.Command(npx, "grunt", "run:build-out-node")
cmd.Dir = dir
if out, err := cmd.CombinedOutput(); err != nil {
return 0, fmt.Errorf("failed to build CTS typescript: %w\n%v", err, string(out))
}
}
{
sh, err := exec.LookPath("node")
if err != nil {
return 0, fmt.Errorf("failed to find sh on PATH: %w", err)
}
cmd := exec.Command(sh, "./tools/run_node", "--list-unimplemented", "webgpu:*")
cmd.Dir = dir
out, err := cmd.CombinedOutput()
if err != nil {
return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
}
lines := strings.Split(string(out), "\n")
return len(lines), nil
}
}