blob: d20294b98fff894fbee6a88e45f5d48c7a8c1bbf [file] [log] [blame]
// Copyright 2021 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// run-cts is a tool used to run the WebGPU CTS using the Dawn module for NodeJS
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"math"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"syscall"
"time"
"unicode/utf8"
"github.com/mattn/go-colorable"
"github.com/mattn/go-isatty"
)
const (
testTimeout = time.Minute
)
func main() {
if err := run(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
func showUsage() {
fmt.Println(`
run-cts is a tool used to run the WebGPU CTS using the Dawn module for NodeJS
Usage:
run-cts --dawn-node=<path to dawn.node> --cts=<path to WebGPU CTS> [test-query]`)
os.Exit(1)
}
var (
colors bool
stdout io.Writer
mainCtx context.Context
)
// ANSI escape sequences
const (
escape = "\u001B["
positionLeft = escape + "0G"
ansiReset = escape + "0m"
bold = escape + "1m"
red = escape + "31m"
green = escape + "32m"
yellow = escape + "33m"
blue = escape + "34m"
magenta = escape + "35m"
cyan = escape + "36m"
white = escape + "37m"
)
type dawnNodeFlags []string
func (f *dawnNodeFlags) String() string {
return fmt.Sprint(strings.Join(*f, ""))
}
func (f *dawnNodeFlags) Set(value string) error {
// Multiple flags must be passed in individually:
// -flag=a=b -dawn_node_flag=c=d
*f = append(*f, value)
return nil
}
func makeMainCtx() context.Context {
ctx, cancel := context.WithCancel(context.Background())
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigs
fmt.Printf("Signal received: %v\n", sig)
cancel()
}()
return ctx
}
func run() error {
mainCtx = makeMainCtx()
colors = os.Getenv("TERM") != "dumb" ||
isatty.IsTerminal(os.Stdout.Fd()) ||
isatty.IsCygwinTerminal(os.Stdout.Fd())
if colors {
if _, disable := os.LookupEnv("NO_COLOR"); disable {
colors = false
}
}
backendDefault := "default"
if vkIcdFilenames := os.Getenv("VK_ICD_FILENAMES"); vkIcdFilenames != "" {
backendDefault = "vulkan"
}
var dawnNode, cts, node, npx, resultsPath, expectationsPath, logFilename, backend string
var verbose, isolated, build bool
var numRunners int
var flags dawnNodeFlags
flag.StringVar(&dawnNode, "dawn-node", "", "path to dawn.node module")
flag.StringVar(&cts, "cts", defaultCtsPath(), "root directory of WebGPU CTS")
flag.StringVar(&node, "node", defaultNodePath(), "path to node executable")
flag.StringVar(&npx, "npx", "", "path to npx executable")
flag.StringVar(&resultsPath, "output", "", "path to write test results file")
flag.StringVar(&expectationsPath, "expect", "", "path to expectations file")
flag.BoolVar(&verbose, "verbose", false, "print extra information while testing")
flag.BoolVar(&build, "build", true, "attempt to build the CTS before running")
flag.BoolVar(&isolated, "isolate", false, "run each test in an isolated process")
flag.BoolVar(&colors, "colors", colors, "enable / disable colors")
flag.IntVar(&numRunners, "j", runtime.NumCPU()/2, "number of concurrent runners. 0 runs serially")
flag.StringVar(&logFilename, "log", "", "path to log file of tests run and result")
flag.Var(&flags, "flag", "flag to pass to dawn-node as flag=value. multiple flags must be passed in individually")
flag.StringVar(&backend, "backend", backendDefault, "backend to use: default|null|webgpu|d3d11|d3d12|metal|vulkan|opengl|opengles."+
" set to 'vulkan' if VK_ICD_FILENAMES environment variable is set, 'default' otherwise")
flag.Parse()
if colors {
stdout = colorable.NewColorableStdout()
} else {
stdout = colorable.NewNonColorable(os.Stdout)
}
// Check mandatory arguments
if dawnNode == "" || cts == "" {
showUsage()
}
if !isFile(dawnNode) {
return fmt.Errorf("'%v' is not a file", dawnNode)
}
if !isDir(cts) {
return fmt.Errorf("'%v' is not a directory", cts)
}
// Make paths absolute
for _, path := range []*string{&dawnNode, &cts} {
abs, err := filepath.Abs(*path)
if err != nil {
return fmt.Errorf("unable to get absolute path for '%v'", *path)
}
*path = abs
}
// The test query is the optional unnamed argument
query := "webgpu:*"
switch len(flag.Args()) {
case 0:
case 1:
query = flag.Args()[0]
default:
return fmt.Errorf("only a single query can be provided")
}
// Find node
if node == "" {
return fmt.Errorf("cannot find path to node. Specify with --node")
}
// Find npx
if npx == "" {
var err error
npx, err = exec.LookPath("npx")
if err != nil {
npx = ""
}
}
// Forward the backend to use, if specified.
if backend != "default" {
fmt.Println("Forcing backend to", backend)
flags = append(flags, fmt.Sprint("dawn-backend=", backend))
}
// While running the CTS, always allow unsafe APIs so they can be tested.
disableDawnFeaturesFound := false
for i, flag := range flags {
if strings.HasPrefix(flag, "disable-dawn-features=") {
flags[i] = flag + ",disallow_unsafe_apis"
disableDawnFeaturesFound = true
}
}
if !disableDawnFeaturesFound {
flags = append(flags, "disable-dawn-features=disallow_unsafe_apis")
}
r := runner{
numRunners: numRunners,
verbose: verbose,
node: node,
npx: npx,
dawnNode: dawnNode,
cts: cts,
flags: flags,
results: testcaseStatuses{},
evalScript: func(main string) string {
return fmt.Sprintf(`require('./src/common/tools/setup-ts-in-node.js');require('./src/common/runtime/%v.ts');`, main)
},
}
if logFilename != "" {
writer, err := os.Create(logFilename)
if err != nil {
return fmt.Errorf("failed to open log '%v': %w", logFilename, err)
}
defer writer.Close()
r.log = newLogger(writer)
}
cache := cache{}
cachePath := dawnNode + ".runcts.cache"
if err := cache.load(cachePath); err != nil && verbose {
fmt.Println("failed to load cache from", cachePath, err)
}
defer cache.save(cachePath)
// Scan the CTS source to determine the most recent change to the CTS source
mostRecentSourceChange, err := r.scanSourceTimestamps(verbose)
if err != nil {
return fmt.Errorf("failed to scan source files for modified timestamps: %w", err)
}
ctsNeedsRebuild := mostRecentSourceChange.After(cache.BuildTimestamp) ||
!isDir(filepath.Join(r.cts, "out-node"))
if build {
if verbose {
fmt.Println("CTS needs rebuild:", ctsNeedsRebuild)
}
if npx != "" {
if ctsNeedsRebuild {
if err := r.buildCTS(verbose); err != nil {
return fmt.Errorf("failed to build CTS: %w", err)
}
cache.BuildTimestamp = mostRecentSourceChange
}
// Use the prebuilt CTS (instead of using the `setup-ts-in-node` transpiler)
r.evalScript = func(main string) string {
return fmt.Sprintf(`require('./out-node/common/runtime/%v.js');`, main)
}
} else {
fmt.Println("npx not found on PATH. Using runtime TypeScript transpilation (slow)")
}
}
// If an expectations file was specified, load it.
if expectationsPath != "" {
if ex, err := loadExpectations(expectationsPath); err == nil {
r.expectations = ex
} else {
return err
}
}
if numRunners > 0 {
// Find all the test cases that match the given queries.
if err := r.gatherTestCases(query, verbose); err != nil {
return fmt.Errorf("failed to gather test cases: %w", err)
}
if isolated {
fmt.Println("Running in parallel isolated...")
fmt.Printf("Testing %d test cases...\n", len(r.testcases))
if err := r.runParallelIsolated(); err != nil {
return err
}
} else {
fmt.Println("Running in parallel with server...")
fmt.Printf("Testing %d test cases...\n", len(r.testcases))
if err := r.runParallelWithServer(); err != nil {
return err
}
}
} else {
fmt.Println("Running serially...")
if err := r.runSerially(query); err != nil {
return err
}
}
if resultsPath != "" {
if err := saveExpectations(resultsPath, r.results); err != nil {
return err
}
}
return nil
}
type logger struct {
writer io.Writer
idx int
resultByIndex map[int]result
}
// newLogger creates a new logger instance.
func newLogger(writer io.Writer) logger {
return logger{writer, 0, map[int]result{}}
}
// logResult writes the test results to the log file in sequential order.
// logResult should be called whenever a new test result becomes available.
func (l *logger) logResults(res result) {
if l.writer == nil {
return
}
l.resultByIndex[res.index] = res
for {
logRes, ok := l.resultByIndex[l.idx]
if !ok {
break
}
fmt.Fprintf(l.writer, "%v [%v]\n", logRes.testcase, logRes.status)
l.idx++
}
}
// Cache holds cached information between runs to optimize runs
type cache struct {
BuildTimestamp time.Time
}
// load loads the cache information from the JSON file at path
func (c *cache) load(path string) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return json.NewDecoder(f).Decode(c)
}
// save saves the cache information to the JSON file at path
func (c *cache) save(path string) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
return json.NewEncoder(f).Encode(c)
}
type runner struct {
numRunners int
verbose bool
node, npx, dawnNode, cts string
flags dawnNodeFlags
evalScript func(string) string
testcases []string
expectations testcaseStatuses
results testcaseStatuses
log logger
}
// scanSourceTimestamps scans all the .js and .ts files in all subdirectories of
// r.cts, and returns the file with the most recent timestamp.
func (r *runner) scanSourceTimestamps(verbose bool) (time.Time, error) {
if verbose {
start := time.Now()
fmt.Println("Scanning .js / .ts files for changes...")
defer func() {
fmt.Println("completed in", time.Since(start))
}()
}
dir := filepath.Join(r.cts, "src")
mostRecentChange := time.Time{}
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
switch filepath.Ext(path) {
case ".ts", ".js":
if info.ModTime().After(mostRecentChange) {
mostRecentChange = info.ModTime()
}
}
return nil
})
if err != nil {
return time.Time{}, err
}
return mostRecentChange, nil
}
// buildCTS calls `npx grunt run:build-out-node` in the CTS directory to compile
// the TypeScript files down to JavaScript. Doing this once ahead of time can be
// much faster than dynamically transpiling when there are many tests to run.
func (r *runner) buildCTS(verbose bool) error {
if verbose {
start := time.Now()
fmt.Println("Building CTS...")
defer func() {
fmt.Println("completed in", time.Since(start))
}()
}
cmd := exec.Command(r.npx, "grunt", "run:build-out-node")
cmd.Dir = r.cts
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%w: %v", err, string(out))
}
return nil
}
// gatherTestCases() queries the CTS for all test cases that match the given
// query. On success, gatherTestCases() populates r.testcases.
func (r *runner) gatherTestCases(query string, verbose bool) error {
if verbose {
start := time.Now()
fmt.Println("Gathering test cases...")
defer func() {
fmt.Println("completed in", time.Since(start))
}()
}
args := append([]string{
"-e", r.evalScript("cmdline"),
"--", // Start of arguments
// src/common/runtime/helper/sys.ts expects 'node file.js <args>'
// and slices away the first two arguments. When running with '-e', args
// start at 1, so just inject a dummy argument.
"dummy-arg",
"--list",
}, query)
cmd := exec.Command(r.node, args...)
cmd.Dir = r.cts
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%w\n%v", err, string(out))
}
tests := filterTestcases(strings.Split(string(out), "\n"))
r.testcases = tests
return nil
}
type portListener struct {
buffer strings.Builder
port chan int
}
func newPortListener() portListener {
return portListener{strings.Builder{}, make(chan int)}
}
var portRE = regexp.MustCompile(`\[\[(\d+)\]\]`)
func (p *portListener) Write(data []byte) (n int, err error) {
if p.port != nil {
p.buffer.Write(data)
match := portRE.FindStringSubmatch(p.buffer.String())
if len(match) == 2 {
port, err := strconv.Atoi(match[1])
if err != nil {
return 0, err
}
p.port <- port
close(p.port)
p.port = nil
}
}
return len(data), nil
}
// runParallelWithServer() starts r.numRunners instances of the CTS server test
// runner, and issues test run requests to those servers, concurrently.
func (r *runner) runParallelWithServer() error {
// Create a chan of test indices.
// This will be read by the test runner goroutines.
caseIndices := make(chan int, len(r.testcases))
for i := range r.testcases {
caseIndices <- i
}
close(caseIndices)
// Create a chan for the test results.
// This will be written to by the test runner goroutines.
results := make(chan result, len(r.testcases))
// Spin up the test runner goroutines
wg := &sync.WaitGroup{}
for i := 0; i < r.numRunners; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if err := r.runServer(caseIndices, results); err != nil {
results <- result{
status: fail,
error: fmt.Errorf("Test server error: %w", err),
}
}
}()
}
r.streamResults(wg, results)
return nil
}
type redirectingWriter struct {
io.Writer
}
// runServer starts a test runner server instance, takes case indices from
// caseIndices, and requests the server run the test with the given index.
// The result of the test run is written to the results chan.
// Once the caseIndices chan has been closed, the server is stopped and
// runServer returns.
func (r *runner) runServer(caseIndices <-chan int, results chan<- result) error {
var port int
var rw redirectingWriter
stopServer := func() {}
startServer := func() error {
args := []string{
"-e", r.evalScript("server"), // Evaluate 'eval'.
"--",
// src/common/runtime/helper/sys.ts expects 'node file.js <args>'
// and slices away the first two arguments. When running with '-e', args
// start at 1, so just inject a dummy argument.
"dummy-arg",
// Actual arguments begin here
"--gpu-provider", r.dawnNode,
}
for _, f := range r.flags {
args = append(args, "--gpu-provider-flag", f)
}
ctx := mainCtx
cmd := exec.CommandContext(ctx, r.node, args...)
serverLog := &bytes.Buffer{}
pl := newPortListener()
cmd.Dir = r.cts
cmd.Stdout = io.MultiWriter(&rw, serverLog, &pl)
cmd.Stderr = io.MultiWriter(&rw, serverLog)
err := cmd.Start()
if err != nil {
return fmt.Errorf("failed to start test runner server: %v", err)
}
select {
case port = <-pl.port:
case <-time.After(time.Second * 10):
return fmt.Errorf("timeout waiting for server port:\n%v", serverLog.String())
case <-ctx.Done():
return ctx.Err()
}
return nil
}
stopServer = func() {
if port > 0 {
go http.Post(fmt.Sprintf("http://localhost:%v/terminate", port), "", &bytes.Buffer{})
time.Sleep(time.Millisecond * 100)
port = 0
}
}
for idx := range caseIndices {
// Redirect the server log per test case
caseServerLog := &bytes.Buffer{}
rw.Writer = caseServerLog
if port == 0 {
if err := startServer(); err != nil {
return err
}
}
res := result{index: idx, testcase: r.testcases[idx]}
type Response struct {
Status string
Message string
}
postResp, err := http.Post(fmt.Sprintf("http://localhost:%v/run?%v", port, r.testcases[idx]), "", &bytes.Buffer{})
if err != nil {
res.error = fmt.Errorf("server POST failure. Restarting server... This can happen when there is a crash. Try running with --isolate.")
res.status = fail
results <- res
stopServer()
continue
}
if postResp.StatusCode == http.StatusOK {
var resp Response
if err := json.NewDecoder(postResp.Body).Decode(&resp); err != nil {
res.error = fmt.Errorf("server response decode failure")
res.status = fail
results <- res
continue
}
switch resp.Status {
case "pass":
res.status = pass
res.message = resp.Message + caseServerLog.String()
case "warn":
res.status = warn
res.message = resp.Message + caseServerLog.String()
case "fail":
res.status = fail
res.message = resp.Message + caseServerLog.String()
case "skip":
res.status = skip
res.message = resp.Message + caseServerLog.String()
default:
res.status = fail
res.error = fmt.Errorf("unknown status: '%v'", resp.Status)
}
} else {
msg, err := ioutil.ReadAll(postResp.Body)
if err != nil {
msg = []byte(err.Error())
}
res.status = fail
res.error = fmt.Errorf("server error: %v", string(msg))
}
results <- res
}
stopServer()
return nil
}
// runParallelIsolated() calls the CTS command-line test runner to run each
// testcase in a separate process. This reduces possibility of state leakage
// between tests.
// Up to r.numRunners tests will be run concurrently.
func (r *runner) runParallelIsolated() error {
// Create a chan of test indices.
// This will be read by the test runner goroutines.
caseIndices := make(chan int, len(r.testcases))
for i := range r.testcases {
caseIndices <- i
}
close(caseIndices)
// Create a chan for the test results.
// This will be written to by the test runner goroutines.
results := make(chan result, len(r.testcases))
// Spin up the test runner goroutines
wg := &sync.WaitGroup{}
for i := 0; i < r.numRunners; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for idx := range caseIndices {
res := r.runTestcase(r.testcases[idx])
res.index = idx
results <- res
}
}()
}
r.streamResults(wg, results)
return nil
}
// streamResults reads from the chan 'results', printing the results in test-id
// sequential order. Once the WaitGroup 'wg' is complete, streamResults() will
// automatically close the 'results' chan.
// Once all the results have been printed, a summary will be printed and the
// function will return.
func (r *runner) streamResults(wg *sync.WaitGroup, results chan result) {
// Create another goroutine to close the results chan when all the runner
// goroutines have finished.
start := time.Now()
var timeTaken time.Duration
go func() {
wg.Wait()
timeTaken = time.Since(start)
close(results)
}()
// Total number of tests, test counts binned by status
numTests, numByExpectedStatus := len(r.testcases), map[expectedStatus]int{}
// Helper function for printing a progress bar.
lastStatusUpdate, animFrame := time.Now(), 0
updateProgress := func() {
printANSIProgressBar(animFrame, numTests, numByExpectedStatus)
animFrame++
lastStatusUpdate = time.Now()
}
// Pull test results as they become available.
// Update the status counts, and print any failures (or all test results if --verbose)
progressUpdateRate := time.Millisecond * 10
if !colors {
// No colors == no cursor control. Reduce progress updates so that
// we're not printing endless progress bars.
progressUpdateRate = time.Second
}
for res := range results {
r.log.logResults(res)
r.results[res.testcase] = res.status
expected := r.expectations[res.testcase]
exStatus := expectedStatus{
status: res.status,
expected: expected == res.status,
}
numByExpectedStatus[exStatus] = numByExpectedStatus[exStatus] + 1
name := res.testcase
if r.verbose ||
res.error != nil ||
(exStatus.status != pass && exStatus.status != skip && !exStatus.expected) {
fmt.Printf("%v - %v: %v", name, res.status, res.message)
if expected != "" {
fmt.Printf(" [%v -> %v]", expected, res.status)
}
fmt.Println()
if res.error != nil {
fmt.Println(res.error)
}
updateProgress()
}
if time.Since(lastStatusUpdate) > progressUpdateRate {
updateProgress()
}
}
printANSIProgressBar(animFrame, numTests, numByExpectedStatus)
// All done. Print final stats.
fmt.Printf("\nCompleted in %v\n", timeTaken)
var numExpectedByStatus map[status]int
if r.expectations != nil {
// The status of each testcase that was run
numExpectedByStatus = map[status]int{}
for t, s := range r.expectations {
if _, wasTested := r.results[t]; wasTested {
numExpectedByStatus[s] = numExpectedByStatus[s] + 1
}
}
}
for _, s := range statuses {
// number of tests, just run, that resulted in the given status
numByStatus := numByExpectedStatus[expectedStatus{s, true}] +
numByExpectedStatus[expectedStatus{s, false}]
// difference in number of tests that had the given status from the
// expected number (taken from the expectations file)
diffFromExpected := 0
if numExpectedByStatus != nil {
diffFromExpected = numByStatus - numExpectedByStatus[s]
}
if numByStatus == 0 && diffFromExpected == 0 {
continue
}
fmt.Print(bold, statusColor[s])
fmt.Print(alignRight(strings.ToUpper(string(s))+": ", 10))
fmt.Print(ansiReset)
if numByStatus > 0 {
fmt.Print(bold)
}
fmt.Print(alignLeft(numByStatus, 10))
fmt.Print(ansiReset)
fmt.Print(alignRight("("+percentage(numByStatus, numTests)+")", 6))
if diffFromExpected != 0 {
fmt.Print(bold, " [")
fmt.Printf("%+d", diffFromExpected)
fmt.Print(ansiReset, "]")
}
fmt.Println()
}
}
// runSerially() calls the CTS test runner to run the test query in a single
// process.
// TODO(bclayton): Support comparing against r.expectations
func (r *runner) runSerially(query string) error {
start := time.Now()
result := r.runTestcase(query)
timeTaken := time.Since(start)
if r.verbose {
fmt.Println(result)
}
fmt.Println("Status:", result.status)
fmt.Println("Completed in", timeTaken)
return nil
}
// status is an enumerator of test result status
type status string
const (
pass status = "pass"
warn status = "warn"
fail status = "fail"
skip status = "skip"
timeout status = "timeout"
)
// All the status types
var statuses = []status{pass, warn, fail, skip, timeout}
var statusColor = map[status]string{
pass: green,
warn: yellow,
skip: blue,
timeout: yellow,
fail: red,
}
// expectedStatus is a test status, along with a boolean to indicate whether the
// status matches the test expectations
type expectedStatus struct {
status status
expected bool
}
// result holds the information about a completed test
type result struct {
index int
testcase string
status status
message string
error error
}
// runTestcase() runs the CTS testcase with the given query, returning the test
// result.
func (r *runner) runTestcase(query string) result {
ctx, cancel := context.WithTimeout(mainCtx, testTimeout)
defer cancel()
args := []string{
"-e", r.evalScript("cmdline"), // Evaluate 'eval'.
"--",
// src/common/runtime/helper/sys.ts expects 'node file.js <args>'
// and slices away the first two arguments. When running with '-e', args
// start at 1, so just inject a dummy argument.
"dummy-arg",
// Actual arguments begin here
"--gpu-provider", r.dawnNode,
"--verbose",
}
for _, f := range r.flags {
args = append(args, "--gpu-provider-flag", f)
}
args = append(args, query)
cmd := exec.CommandContext(ctx, r.node, args...)
cmd.Dir = r.cts
var buf bytes.Buffer
cmd.Stdout = &buf
cmd.Stderr = &buf
err := cmd.Run()
msg := buf.String()
switch {
case errors.Is(err, context.DeadlineExceeded):
return result{testcase: query, status: timeout, message: msg}
case strings.Contains(msg, "[fail]"):
return result{testcase: query, status: fail, message: msg}
case strings.Contains(msg, "[warn]"):
return result{testcase: query, status: warn, message: msg}
case strings.Contains(msg, "[skip]"):
return result{testcase: query, status: skip, message: msg}
case strings.Contains(msg, "[pass]"), err == nil:
return result{testcase: query, status: pass, message: msg}
}
return result{testcase: query, status: fail, message: fmt.Sprint(msg, err), error: err}
}
// filterTestcases returns in with empty strings removed
func filterTestcases(in []string) []string {
out := make([]string, 0, len(in))
for _, c := range in {
if c != "" {
out = append(out, c)
}
}
return out
}
// percentage returns the percentage of n out of total as a string
func percentage(n, total int) string {
if total == 0 {
return "-"
}
f := float64(n) / float64(total)
return fmt.Sprintf("%.1f%c", f*100.0, '%')
}
// isDir returns true if the path resolves to a directory
func isDir(path string) bool {
s, err := os.Stat(path)
if err != nil {
return false
}
return s.IsDir()
}
// isFile returns true if the path resolves to a file
func isFile(path string) bool {
s, err := os.Stat(path)
if err != nil {
return false
}
return !s.IsDir()
}
// alignLeft returns the string of 'val' padded so that it is aligned left in
// a column of the given width
func alignLeft(val interface{}, width int) string {
s := fmt.Sprint(val)
padding := width - utf8.RuneCountInString(s)
if padding < 0 {
return s
}
return s + strings.Repeat(" ", padding)
}
// alignRight returns the string of 'val' padded so that it is aligned right in
// a column of the given width
func alignRight(val interface{}, width int) string {
s := fmt.Sprint(val)
padding := width - utf8.RuneCountInString(s)
if padding < 0 {
return s
}
return strings.Repeat(" ", padding) + s
}
// printANSIProgressBar prints a colored progress bar, providing realtime
// information about the status of the CTS run.
// Note: We'll want to skip this if !isatty or if we're running on windows.
func printANSIProgressBar(animFrame int, numTests int, numByExpectedStatus map[expectedStatus]int) {
const barWidth = 50
animSymbols := []rune{'⣾', '⣽', '⣻', '⢿', '⡿', '⣟', '⣯', '⣷'}
blockSymbols := []rune{'▏', '▎', '▍', '▌', '▋', '▊', '▉'}
numBlocksPrinted := 0
fmt.Fprint(stdout, string(animSymbols[animFrame%len(animSymbols)]), " [")
animFrame++
numFinished := 0
for _, status := range statuses {
for _, expected := range []bool{true, false} {
color := statusColor[status]
if expected {
color += bold
}
num := numByExpectedStatus[expectedStatus{status, expected}]
numFinished += num
statusFrac := float64(num) / float64(numTests)
fNumBlocks := barWidth * statusFrac
fmt.Fprint(stdout, color)
numBlocks := int(math.Ceil(fNumBlocks))
if expected {
if numBlocks > 1 {
fmt.Print(strings.Repeat(string("░"), numBlocks))
}
} else {
if numBlocks > 1 {
fmt.Print(strings.Repeat(string("▉"), numBlocks))
}
if numBlocks > 0 {
frac := fNumBlocks - math.Floor(fNumBlocks)
symbol := blockSymbols[int(math.Round(frac*float64(len(blockSymbols)-1)))]
fmt.Print(string(symbol))
}
}
numBlocksPrinted += numBlocks
}
}
if barWidth > numBlocksPrinted {
fmt.Print(strings.Repeat(string(" "), barWidth-numBlocksPrinted))
}
fmt.Fprint(stdout, ansiReset)
fmt.Print("] ", percentage(numFinished, numTests))
if colors {
// move cursor to start of line so the bar is overridden
fmt.Fprint(stdout, positionLeft)
} else {
// cannot move cursor, so newline
fmt.Println()
}
}
// testcaseStatus is a pair of testcase name and result status
// Intended to be serialized for expectations files.
type testcaseStatus struct {
Testcase string
Status status
}
// testcaseStatuses is a map of testcase to test status
type testcaseStatuses map[string]status
// loadExpectations loads the test expectations from path
func loadExpectations(path string) (testcaseStatuses, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open expectations file: %w", err)
}
defer f.Close()
statuses := []testcaseStatus{}
if err := json.NewDecoder(f).Decode(&statuses); err != nil {
return nil, fmt.Errorf("failed to read expectations file: %w", err)
}
out := make(testcaseStatuses, len(statuses))
for _, s := range statuses {
out[s.Testcase] = s.Status
}
return out, nil
}
// saveExpectations saves the test results 'ex' as an expectations file to path
func saveExpectations(path string, ex testcaseStatuses) error {
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("failed to create expectations file: %w", err)
}
defer f.Close()
statuses := make([]testcaseStatus, 0, len(ex))
for testcase, status := range ex {
statuses = append(statuses, testcaseStatus{testcase, status})
}
sort.Slice(statuses, func(i, j int) bool { return statuses[i].Testcase < statuses[j].Testcase })
e := json.NewEncoder(f)
e.SetIndent("", " ")
if err := e.Encode(&statuses); err != nil {
return fmt.Errorf("failed to save expectations file: %w", err)
}
return nil
}
// defaultNodePath looks for the node binary, first in dawn's third_party
// directory, falling back to PATH. This is used as the default for the --node
// command line flag.
func defaultNodePath() string {
if dir := thisDir(); dir != "" {
if dawnRoot := getDawnRoot(); dawnRoot != "" {
node := filepath.Join(dawnRoot, "third_party/node")
if info, err := os.Stat(node); err == nil && info.IsDir() {
path := ""
switch fmt.Sprintf("%v/%v", runtime.GOOS, runtime.GOARCH) { // See `go tool dist list`
case "darwin/amd64":
path = filepath.Join(node, "node-darwin-x64/bin/node")
case "darwin/arm64":
path = filepath.Join(node, "node-darwin-arm64/bin/node")
case "linux/amd64":
path = filepath.Join(node, "node-linux-x64/bin/node")
case "windows/amd64":
path = filepath.Join(node, "node.exe")
}
if _, err := os.Stat(path); err == nil {
return path
}
}
}
}
if path, err := exec.LookPath("node"); err == nil {
return path
}
return ""
}
// defaultCtsPath looks for the webgpu-cts directory in dawn's third_party
// directory. This is used as the default for the --cts command line flag.
func defaultCtsPath() string {
if dir := thisDir(); dir != "" {
if dawnRoot := getDawnRoot(); dawnRoot != "" {
cts := filepath.Join(dawnRoot, "third_party/webgpu-cts")
if info, err := os.Stat(cts); err == nil && info.IsDir() {
return cts
}
}
}
return ""
}
// getDawnRoot returns the path to the dawn project's root directory or empty
// string if not found.
func getDawnRoot() string {
return getPathOfFileInParentDirs(thisDir(), "DEPS")
}
// getPathOfFileInParentDirs looks for file with `name` in paths starting from
// `path`, and up into parent directories, returning the clean path in which the
// file is found, or empty string if not found.
func getPathOfFileInParentDirs(path string, name string) string {
sep := string(filepath.Separator)
path, _ = filepath.Abs(path)
numDirs := strings.Count(path, sep) + 1
for i := 0; i < numDirs; i++ {
test := filepath.Join(path, name)
if _, err := os.Stat(test); err == nil {
return filepath.Clean(path)
}
path = path + sep + ".."
}
return ""
}
// thisDir returns the path to the directory that holds the .go file of the
// caller function
func thisDir() string {
_, file, _, ok := runtime.Caller(1)
if !ok {
return ""
}
return filepath.Dir(file)
}