| // Copyright 2021 The Dawn Authors |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| // run-cts is a tool used to run the WebGPU CTS using the Dawn module for NodeJS |
| package main |
| |
| import ( |
| "bytes" |
| "context" |
| "encoding/json" |
| "errors" |
| "flag" |
| "fmt" |
| "io" |
| "io/ioutil" |
| "math" |
| "net/http" |
| "os" |
| "os/exec" |
| "os/signal" |
| "path/filepath" |
| "regexp" |
| "runtime" |
| "sort" |
| "strconv" |
| "strings" |
| "sync" |
| "syscall" |
| "time" |
| "unicode/utf8" |
| |
| "github.com/mattn/go-colorable" |
| "github.com/mattn/go-isatty" |
| ) |
| |
| const ( |
| testTimeout = time.Minute |
| ) |
| |
| func main() { |
| if err := run(); err != nil { |
| fmt.Println(err) |
| os.Exit(1) |
| } |
| } |
| |
| func showUsage() { |
| fmt.Println(` |
| run-cts is a tool used to run the WebGPU CTS using the Dawn module for NodeJS |
| |
| Usage: |
| run-cts --dawn-node=<path to dawn.node> --cts=<path to WebGPU CTS> [test-query]`) |
| os.Exit(1) |
| } |
| |
| var ( |
| colors bool |
| stdout io.Writer |
| mainCtx context.Context |
| ) |
| |
| // ANSI escape sequences |
| const ( |
| escape = "\u001B[" |
| positionLeft = escape + "0G" |
| ansiReset = escape + "0m" |
| |
| bold = escape + "1m" |
| |
| red = escape + "31m" |
| green = escape + "32m" |
| yellow = escape + "33m" |
| blue = escape + "34m" |
| magenta = escape + "35m" |
| cyan = escape + "36m" |
| white = escape + "37m" |
| ) |
| |
| type dawnNodeFlags []string |
| |
| func (f *dawnNodeFlags) String() string { |
| return fmt.Sprint(strings.Join(*f, "")) |
| } |
| |
| func (f *dawnNodeFlags) Set(value string) error { |
| // Multiple flags must be passed in individually: |
| // -flag=a=b -dawn_node_flag=c=d |
| *f = append(*f, value) |
| return nil |
| } |
| |
| func makeMainCtx() context.Context { |
| ctx, cancel := context.WithCancel(context.Background()) |
| sigs := make(chan os.Signal, 1) |
| signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) |
| go func() { |
| sig := <-sigs |
| fmt.Printf("Signal received: %v\n", sig) |
| cancel() |
| }() |
| return ctx |
| } |
| |
| func run() error { |
| mainCtx = makeMainCtx() |
| |
| colors = os.Getenv("TERM") != "dumb" || |
| isatty.IsTerminal(os.Stdout.Fd()) || |
| isatty.IsCygwinTerminal(os.Stdout.Fd()) |
| if colors { |
| if _, disable := os.LookupEnv("NO_COLOR"); disable { |
| colors = false |
| } |
| } |
| |
| backendDefault := "default" |
| if vkIcdFilenames := os.Getenv("VK_ICD_FILENAMES"); vkIcdFilenames != "" { |
| backendDefault = "vulkan" |
| } |
| |
| var dawnNode, cts, node, npx, resultsPath, expectationsPath, logFilename, backend string |
| var verbose, isolated, build bool |
| var numRunners int |
| var flags dawnNodeFlags |
| flag.StringVar(&dawnNode, "dawn-node", "", "path to dawn.node module") |
| flag.StringVar(&cts, "cts", defaultCtsPath(), "root directory of WebGPU CTS") |
| flag.StringVar(&node, "node", defaultNodePath(), "path to node executable") |
| flag.StringVar(&npx, "npx", "", "path to npx executable") |
| flag.StringVar(&resultsPath, "output", "", "path to write test results file") |
| flag.StringVar(&expectationsPath, "expect", "", "path to expectations file") |
| flag.BoolVar(&verbose, "verbose", false, "print extra information while testing") |
| flag.BoolVar(&build, "build", true, "attempt to build the CTS before running") |
| flag.BoolVar(&isolated, "isolate", false, "run each test in an isolated process") |
| flag.BoolVar(&colors, "colors", colors, "enable / disable colors") |
| flag.IntVar(&numRunners, "j", runtime.NumCPU()/2, "number of concurrent runners. 0 runs serially") |
| flag.StringVar(&logFilename, "log", "", "path to log file of tests run and result") |
| flag.Var(&flags, "flag", "flag to pass to dawn-node as flag=value. multiple flags must be passed in individually") |
| flag.StringVar(&backend, "backend", backendDefault, "backend to use: default|null|webgpu|d3d11|d3d12|metal|vulkan|opengl|opengles."+ |
| " set to 'vulkan' if VK_ICD_FILENAMES environment variable is set, 'default' otherwise") |
| flag.Parse() |
| |
| if colors { |
| stdout = colorable.NewColorableStdout() |
| } else { |
| stdout = colorable.NewNonColorable(os.Stdout) |
| } |
| |
| // Check mandatory arguments |
| if dawnNode == "" || cts == "" { |
| showUsage() |
| } |
| if !isFile(dawnNode) { |
| return fmt.Errorf("'%v' is not a file", dawnNode) |
| } |
| if !isDir(cts) { |
| return fmt.Errorf("'%v' is not a directory", cts) |
| } |
| |
| // Make paths absolute |
| for _, path := range []*string{&dawnNode, &cts} { |
| abs, err := filepath.Abs(*path) |
| if err != nil { |
| return fmt.Errorf("unable to get absolute path for '%v'", *path) |
| } |
| *path = abs |
| } |
| |
| // The test query is the optional unnamed argument |
| query := "webgpu:*" |
| switch len(flag.Args()) { |
| case 0: |
| case 1: |
| query = flag.Args()[0] |
| default: |
| return fmt.Errorf("only a single query can be provided") |
| } |
| |
| // Find node |
| if node == "" { |
| return fmt.Errorf("cannot find path to node. Specify with --node") |
| } |
| // Find npx |
| if npx == "" { |
| var err error |
| npx, err = exec.LookPath("npx") |
| if err != nil { |
| npx = "" |
| } |
| } |
| |
| // Forward the backend to use, if specified. |
| if backend != "default" { |
| fmt.Println("Forcing backend to", backend) |
| flags = append(flags, fmt.Sprint("dawn-backend=", backend)) |
| } |
| |
| // While running the CTS, always allow unsafe APIs so they can be tested. |
| disableDawnFeaturesFound := false |
| for i, flag := range flags { |
| if strings.HasPrefix(flag, "disable-dawn-features=") { |
| flags[i] = flag + ",disallow_unsafe_apis" |
| disableDawnFeaturesFound = true |
| } |
| } |
| if !disableDawnFeaturesFound { |
| flags = append(flags, "disable-dawn-features=disallow_unsafe_apis") |
| } |
| |
| r := runner{ |
| numRunners: numRunners, |
| verbose: verbose, |
| node: node, |
| npx: npx, |
| dawnNode: dawnNode, |
| cts: cts, |
| flags: flags, |
| results: testcaseStatuses{}, |
| evalScript: func(main string) string { |
| return fmt.Sprintf(`require('./src/common/tools/setup-ts-in-node.js');require('./src/common/runtime/%v.ts');`, main) |
| }, |
| } |
| |
| if logFilename != "" { |
| writer, err := os.Create(logFilename) |
| if err != nil { |
| return fmt.Errorf("failed to open log '%v': %w", logFilename, err) |
| } |
| defer writer.Close() |
| r.log = newLogger(writer) |
| } |
| |
| cache := cache{} |
| cachePath := dawnNode + ".runcts.cache" |
| if err := cache.load(cachePath); err != nil && verbose { |
| fmt.Println("failed to load cache from", cachePath, err) |
| } |
| defer cache.save(cachePath) |
| |
| // Scan the CTS source to determine the most recent change to the CTS source |
| mostRecentSourceChange, err := r.scanSourceTimestamps(verbose) |
| if err != nil { |
| return fmt.Errorf("failed to scan source files for modified timestamps: %w", err) |
| } |
| |
| ctsNeedsRebuild := mostRecentSourceChange.After(cache.BuildTimestamp) || |
| !isDir(filepath.Join(r.cts, "out-node")) |
| if build { |
| if verbose { |
| fmt.Println("CTS needs rebuild:", ctsNeedsRebuild) |
| } |
| |
| if npx != "" { |
| if ctsNeedsRebuild { |
| if err := r.buildCTS(verbose); err != nil { |
| return fmt.Errorf("failed to build CTS: %w", err) |
| } |
| cache.BuildTimestamp = mostRecentSourceChange |
| } |
| // Use the prebuilt CTS (instead of using the `setup-ts-in-node` transpiler) |
| r.evalScript = func(main string) string { |
| return fmt.Sprintf(`require('./out-node/common/runtime/%v.js');`, main) |
| } |
| } else { |
| fmt.Println("npx not found on PATH. Using runtime TypeScript transpilation (slow)") |
| } |
| } |
| |
| // If an expectations file was specified, load it. |
| if expectationsPath != "" { |
| if ex, err := loadExpectations(expectationsPath); err == nil { |
| r.expectations = ex |
| } else { |
| return err |
| } |
| } |
| |
| if numRunners > 0 { |
| // Find all the test cases that match the given queries. |
| if err := r.gatherTestCases(query, verbose); err != nil { |
| return fmt.Errorf("failed to gather test cases: %w", err) |
| } |
| |
| if isolated { |
| fmt.Println("Running in parallel isolated...") |
| fmt.Printf("Testing %d test cases...\n", len(r.testcases)) |
| if err := r.runParallelIsolated(); err != nil { |
| return err |
| } |
| } else { |
| fmt.Println("Running in parallel with server...") |
| fmt.Printf("Testing %d test cases...\n", len(r.testcases)) |
| if err := r.runParallelWithServer(); err != nil { |
| return err |
| } |
| } |
| } else { |
| fmt.Println("Running serially...") |
| if err := r.runSerially(query); err != nil { |
| return err |
| } |
| } |
| |
| if resultsPath != "" { |
| if err := saveExpectations(resultsPath, r.results); err != nil { |
| return err |
| } |
| } |
| |
| return nil |
| } |
| |
| type logger struct { |
| writer io.Writer |
| idx int |
| resultByIndex map[int]result |
| } |
| |
| // newLogger creates a new logger instance. |
| func newLogger(writer io.Writer) logger { |
| return logger{writer, 0, map[int]result{}} |
| } |
| |
| // logResult writes the test results to the log file in sequential order. |
| // logResult should be called whenever a new test result becomes available. |
| func (l *logger) logResults(res result) { |
| if l.writer == nil { |
| return |
| } |
| l.resultByIndex[res.index] = res |
| for { |
| logRes, ok := l.resultByIndex[l.idx] |
| if !ok { |
| break |
| } |
| fmt.Fprintf(l.writer, "%v [%v]\n", logRes.testcase, logRes.status) |
| l.idx++ |
| } |
| } |
| |
| // Cache holds cached information between runs to optimize runs |
| type cache struct { |
| BuildTimestamp time.Time |
| } |
| |
| // load loads the cache information from the JSON file at path |
| func (c *cache) load(path string) error { |
| f, err := os.Open(path) |
| if err != nil { |
| return err |
| } |
| defer f.Close() |
| return json.NewDecoder(f).Decode(c) |
| } |
| |
| // save saves the cache information to the JSON file at path |
| func (c *cache) save(path string) error { |
| f, err := os.Create(path) |
| if err != nil { |
| return err |
| } |
| defer f.Close() |
| return json.NewEncoder(f).Encode(c) |
| } |
| |
| type runner struct { |
| numRunners int |
| verbose bool |
| node, npx, dawnNode, cts string |
| flags dawnNodeFlags |
| evalScript func(string) string |
| testcases []string |
| expectations testcaseStatuses |
| results testcaseStatuses |
| log logger |
| } |
| |
| // scanSourceTimestamps scans all the .js and .ts files in all subdirectories of |
| // r.cts, and returns the file with the most recent timestamp. |
| func (r *runner) scanSourceTimestamps(verbose bool) (time.Time, error) { |
| if verbose { |
| start := time.Now() |
| fmt.Println("Scanning .js / .ts files for changes...") |
| defer func() { |
| fmt.Println("completed in", time.Since(start)) |
| }() |
| } |
| |
| dir := filepath.Join(r.cts, "src") |
| |
| mostRecentChange := time.Time{} |
| err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { |
| switch filepath.Ext(path) { |
| case ".ts", ".js": |
| if info.ModTime().After(mostRecentChange) { |
| mostRecentChange = info.ModTime() |
| } |
| } |
| return nil |
| }) |
| if err != nil { |
| return time.Time{}, err |
| } |
| return mostRecentChange, nil |
| } |
| |
| // buildCTS calls `npx grunt run:build-out-node` in the CTS directory to compile |
| // the TypeScript files down to JavaScript. Doing this once ahead of time can be |
| // much faster than dynamically transpiling when there are many tests to run. |
| func (r *runner) buildCTS(verbose bool) error { |
| if verbose { |
| start := time.Now() |
| fmt.Println("Building CTS...") |
| defer func() { |
| fmt.Println("completed in", time.Since(start)) |
| }() |
| } |
| |
| cmd := exec.Command(r.npx, "grunt", "run:build-out-node") |
| cmd.Dir = r.cts |
| out, err := cmd.CombinedOutput() |
| if err != nil { |
| return fmt.Errorf("%w: %v", err, string(out)) |
| } |
| return nil |
| } |
| |
| // gatherTestCases() queries the CTS for all test cases that match the given |
| // query. On success, gatherTestCases() populates r.testcases. |
| func (r *runner) gatherTestCases(query string, verbose bool) error { |
| if verbose { |
| start := time.Now() |
| fmt.Println("Gathering test cases...") |
| defer func() { |
| fmt.Println("completed in", time.Since(start)) |
| }() |
| } |
| |
| args := append([]string{ |
| "-e", r.evalScript("cmdline"), |
| "--", // Start of arguments |
| // src/common/runtime/helper/sys.ts expects 'node file.js <args>' |
| // and slices away the first two arguments. When running with '-e', args |
| // start at 1, so just inject a dummy argument. |
| "dummy-arg", |
| "--list", |
| }, query) |
| |
| cmd := exec.Command(r.node, args...) |
| cmd.Dir = r.cts |
| out, err := cmd.CombinedOutput() |
| if err != nil { |
| return fmt.Errorf("%w\n%v", err, string(out)) |
| } |
| |
| tests := filterTestcases(strings.Split(string(out), "\n")) |
| r.testcases = tests |
| return nil |
| } |
| |
| type portListener struct { |
| buffer strings.Builder |
| port chan int |
| } |
| |
| func newPortListener() portListener { |
| return portListener{strings.Builder{}, make(chan int)} |
| } |
| |
| var portRE = regexp.MustCompile(`\[\[(\d+)\]\]`) |
| |
| func (p *portListener) Write(data []byte) (n int, err error) { |
| if p.port != nil { |
| p.buffer.Write(data) |
| match := portRE.FindStringSubmatch(p.buffer.String()) |
| if len(match) == 2 { |
| port, err := strconv.Atoi(match[1]) |
| if err != nil { |
| return 0, err |
| } |
| p.port <- port |
| close(p.port) |
| p.port = nil |
| } |
| } |
| return len(data), nil |
| } |
| |
| // runParallelWithServer() starts r.numRunners instances of the CTS server test |
| // runner, and issues test run requests to those servers, concurrently. |
| func (r *runner) runParallelWithServer() error { |
| // Create a chan of test indices. |
| // This will be read by the test runner goroutines. |
| caseIndices := make(chan int, len(r.testcases)) |
| for i := range r.testcases { |
| caseIndices <- i |
| } |
| close(caseIndices) |
| |
| // Create a chan for the test results. |
| // This will be written to by the test runner goroutines. |
| results := make(chan result, len(r.testcases)) |
| |
| // Spin up the test runner goroutines |
| wg := &sync.WaitGroup{} |
| for i := 0; i < r.numRunners; i++ { |
| wg.Add(1) |
| go func() { |
| defer wg.Done() |
| if err := r.runServer(caseIndices, results); err != nil { |
| results <- result{ |
| status: fail, |
| error: fmt.Errorf("Test server error: %w", err), |
| } |
| } |
| }() |
| } |
| |
| r.streamResults(wg, results) |
| return nil |
| } |
| |
| type redirectingWriter struct { |
| io.Writer |
| } |
| |
| // runServer starts a test runner server instance, takes case indices from |
| // caseIndices, and requests the server run the test with the given index. |
| // The result of the test run is written to the results chan. |
| // Once the caseIndices chan has been closed, the server is stopped and |
| // runServer returns. |
| func (r *runner) runServer(caseIndices <-chan int, results chan<- result) error { |
| var port int |
| var rw redirectingWriter |
| |
| stopServer := func() {} |
| startServer := func() error { |
| args := []string{ |
| "-e", r.evalScript("server"), // Evaluate 'eval'. |
| "--", |
| // src/common/runtime/helper/sys.ts expects 'node file.js <args>' |
| // and slices away the first two arguments. When running with '-e', args |
| // start at 1, so just inject a dummy argument. |
| "dummy-arg", |
| // Actual arguments begin here |
| "--gpu-provider", r.dawnNode, |
| } |
| for _, f := range r.flags { |
| args = append(args, "--gpu-provider-flag", f) |
| } |
| |
| ctx := mainCtx |
| cmd := exec.CommandContext(ctx, r.node, args...) |
| |
| serverLog := &bytes.Buffer{} |
| |
| pl := newPortListener() |
| |
| cmd.Dir = r.cts |
| cmd.Stdout = io.MultiWriter(&rw, serverLog, &pl) |
| cmd.Stderr = io.MultiWriter(&rw, serverLog) |
| |
| err := cmd.Start() |
| if err != nil { |
| return fmt.Errorf("failed to start test runner server: %v", err) |
| } |
| |
| select { |
| case port = <-pl.port: |
| case <-time.After(time.Second * 10): |
| return fmt.Errorf("timeout waiting for server port:\n%v", serverLog.String()) |
| case <-ctx.Done(): |
| return ctx.Err() |
| } |
| |
| return nil |
| } |
| stopServer = func() { |
| if port > 0 { |
| go http.Post(fmt.Sprintf("http://localhost:%v/terminate", port), "", &bytes.Buffer{}) |
| time.Sleep(time.Millisecond * 100) |
| port = 0 |
| } |
| } |
| |
| for idx := range caseIndices { |
| // Redirect the server log per test case |
| caseServerLog := &bytes.Buffer{} |
| rw.Writer = caseServerLog |
| |
| if port == 0 { |
| if err := startServer(); err != nil { |
| return err |
| } |
| } |
| |
| res := result{index: idx, testcase: r.testcases[idx]} |
| |
| type Response struct { |
| Status string |
| Message string |
| } |
| postResp, err := http.Post(fmt.Sprintf("http://localhost:%v/run?%v", port, r.testcases[idx]), "", &bytes.Buffer{}) |
| if err != nil { |
| res.error = fmt.Errorf("server POST failure. Restarting server... This can happen when there is a crash. Try running with --isolate.") |
| res.status = fail |
| results <- res |
| stopServer() |
| continue |
| } |
| |
| if postResp.StatusCode == http.StatusOK { |
| var resp Response |
| if err := json.NewDecoder(postResp.Body).Decode(&resp); err != nil { |
| res.error = fmt.Errorf("server response decode failure") |
| res.status = fail |
| results <- res |
| continue |
| } |
| |
| switch resp.Status { |
| case "pass": |
| res.status = pass |
| res.message = resp.Message + caseServerLog.String() |
| case "warn": |
| res.status = warn |
| res.message = resp.Message + caseServerLog.String() |
| case "fail": |
| res.status = fail |
| res.message = resp.Message + caseServerLog.String() |
| case "skip": |
| res.status = skip |
| res.message = resp.Message + caseServerLog.String() |
| default: |
| res.status = fail |
| res.error = fmt.Errorf("unknown status: '%v'", resp.Status) |
| } |
| } else { |
| msg, err := ioutil.ReadAll(postResp.Body) |
| if err != nil { |
| msg = []byte(err.Error()) |
| } |
| res.status = fail |
| res.error = fmt.Errorf("server error: %v", string(msg)) |
| } |
| results <- res |
| } |
| |
| stopServer() |
| return nil |
| } |
| |
| // runParallelIsolated() calls the CTS command-line test runner to run each |
| // testcase in a separate process. This reduces possibility of state leakage |
| // between tests. |
| // Up to r.numRunners tests will be run concurrently. |
| func (r *runner) runParallelIsolated() error { |
| // Create a chan of test indices. |
| // This will be read by the test runner goroutines. |
| caseIndices := make(chan int, len(r.testcases)) |
| for i := range r.testcases { |
| caseIndices <- i |
| } |
| close(caseIndices) |
| |
| // Create a chan for the test results. |
| // This will be written to by the test runner goroutines. |
| results := make(chan result, len(r.testcases)) |
| |
| // Spin up the test runner goroutines |
| wg := &sync.WaitGroup{} |
| for i := 0; i < r.numRunners; i++ { |
| wg.Add(1) |
| go func() { |
| defer wg.Done() |
| for idx := range caseIndices { |
| res := r.runTestcase(r.testcases[idx]) |
| res.index = idx |
| results <- res |
| } |
| }() |
| } |
| |
| r.streamResults(wg, results) |
| return nil |
| } |
| |
| // streamResults reads from the chan 'results', printing the results in test-id |
| // sequential order. Once the WaitGroup 'wg' is complete, streamResults() will |
| // automatically close the 'results' chan. |
| // Once all the results have been printed, a summary will be printed and the |
| // function will return. |
| func (r *runner) streamResults(wg *sync.WaitGroup, results chan result) { |
| // Create another goroutine to close the results chan when all the runner |
| // goroutines have finished. |
| start := time.Now() |
| var timeTaken time.Duration |
| go func() { |
| wg.Wait() |
| timeTaken = time.Since(start) |
| close(results) |
| }() |
| |
| // Total number of tests, test counts binned by status |
| numTests, numByExpectedStatus := len(r.testcases), map[expectedStatus]int{} |
| |
| // Helper function for printing a progress bar. |
| lastStatusUpdate, animFrame := time.Now(), 0 |
| updateProgress := func() { |
| printANSIProgressBar(animFrame, numTests, numByExpectedStatus) |
| animFrame++ |
| lastStatusUpdate = time.Now() |
| } |
| |
| // Pull test results as they become available. |
| // Update the status counts, and print any failures (or all test results if --verbose) |
| progressUpdateRate := time.Millisecond * 10 |
| if !colors { |
| // No colors == no cursor control. Reduce progress updates so that |
| // we're not printing endless progress bars. |
| progressUpdateRate = time.Second |
| } |
| |
| for res := range results { |
| r.log.logResults(res) |
| r.results[res.testcase] = res.status |
| expected := r.expectations[res.testcase] |
| exStatus := expectedStatus{ |
| status: res.status, |
| expected: expected == res.status, |
| } |
| numByExpectedStatus[exStatus] = numByExpectedStatus[exStatus] + 1 |
| name := res.testcase |
| if r.verbose || |
| res.error != nil || |
| (exStatus.status != pass && exStatus.status != skip && !exStatus.expected) { |
| fmt.Printf("%v - %v: %v", name, res.status, res.message) |
| if expected != "" { |
| fmt.Printf(" [%v -> %v]", expected, res.status) |
| } |
| fmt.Println() |
| if res.error != nil { |
| fmt.Println(res.error) |
| } |
| updateProgress() |
| } |
| if time.Since(lastStatusUpdate) > progressUpdateRate { |
| updateProgress() |
| } |
| } |
| printANSIProgressBar(animFrame, numTests, numByExpectedStatus) |
| |
| // All done. Print final stats. |
| fmt.Printf("\nCompleted in %v\n", timeTaken) |
| |
| var numExpectedByStatus map[status]int |
| if r.expectations != nil { |
| // The status of each testcase that was run |
| numExpectedByStatus = map[status]int{} |
| for t, s := range r.expectations { |
| if _, wasTested := r.results[t]; wasTested { |
| numExpectedByStatus[s] = numExpectedByStatus[s] + 1 |
| } |
| } |
| } |
| |
| for _, s := range statuses { |
| // number of tests, just run, that resulted in the given status |
| numByStatus := numByExpectedStatus[expectedStatus{s, true}] + |
| numByExpectedStatus[expectedStatus{s, false}] |
| // difference in number of tests that had the given status from the |
| // expected number (taken from the expectations file) |
| diffFromExpected := 0 |
| if numExpectedByStatus != nil { |
| diffFromExpected = numByStatus - numExpectedByStatus[s] |
| } |
| if numByStatus == 0 && diffFromExpected == 0 { |
| continue |
| } |
| |
| fmt.Print(bold, statusColor[s]) |
| fmt.Print(alignRight(strings.ToUpper(string(s))+": ", 10)) |
| fmt.Print(ansiReset) |
| if numByStatus > 0 { |
| fmt.Print(bold) |
| } |
| fmt.Print(alignLeft(numByStatus, 10)) |
| fmt.Print(ansiReset) |
| fmt.Print(alignRight("("+percentage(numByStatus, numTests)+")", 6)) |
| |
| if diffFromExpected != 0 { |
| fmt.Print(bold, " [") |
| fmt.Printf("%+d", diffFromExpected) |
| fmt.Print(ansiReset, "]") |
| } |
| fmt.Println() |
| } |
| |
| } |
| |
| // runSerially() calls the CTS test runner to run the test query in a single |
| // process. |
| // TODO(bclayton): Support comparing against r.expectations |
| func (r *runner) runSerially(query string) error { |
| start := time.Now() |
| result := r.runTestcase(query) |
| timeTaken := time.Since(start) |
| |
| if r.verbose { |
| fmt.Println(result) |
| } |
| fmt.Println("Status:", result.status) |
| fmt.Println("Completed in", timeTaken) |
| return nil |
| } |
| |
| // status is an enumerator of test result status |
| type status string |
| |
| const ( |
| pass status = "pass" |
| warn status = "warn" |
| fail status = "fail" |
| skip status = "skip" |
| timeout status = "timeout" |
| ) |
| |
| // All the status types |
| var statuses = []status{pass, warn, fail, skip, timeout} |
| |
| var statusColor = map[status]string{ |
| pass: green, |
| warn: yellow, |
| skip: blue, |
| timeout: yellow, |
| fail: red, |
| } |
| |
| // expectedStatus is a test status, along with a boolean to indicate whether the |
| // status matches the test expectations |
| type expectedStatus struct { |
| status status |
| expected bool |
| } |
| |
| // result holds the information about a completed test |
| type result struct { |
| index int |
| testcase string |
| status status |
| message string |
| error error |
| } |
| |
| // runTestcase() runs the CTS testcase with the given query, returning the test |
| // result. |
| func (r *runner) runTestcase(query string) result { |
| ctx, cancel := context.WithTimeout(mainCtx, testTimeout) |
| defer cancel() |
| |
| args := []string{ |
| "-e", r.evalScript("cmdline"), // Evaluate 'eval'. |
| "--", |
| // src/common/runtime/helper/sys.ts expects 'node file.js <args>' |
| // and slices away the first two arguments. When running with '-e', args |
| // start at 1, so just inject a dummy argument. |
| "dummy-arg", |
| // Actual arguments begin here |
| "--gpu-provider", r.dawnNode, |
| "--verbose", |
| } |
| for _, f := range r.flags { |
| args = append(args, "--gpu-provider-flag", f) |
| } |
| args = append(args, query) |
| |
| cmd := exec.CommandContext(ctx, r.node, args...) |
| cmd.Dir = r.cts |
| |
| var buf bytes.Buffer |
| cmd.Stdout = &buf |
| cmd.Stderr = &buf |
| |
| err := cmd.Run() |
| msg := buf.String() |
| switch { |
| case errors.Is(err, context.DeadlineExceeded): |
| return result{testcase: query, status: timeout, message: msg} |
| case strings.Contains(msg, "[fail]"): |
| return result{testcase: query, status: fail, message: msg} |
| case strings.Contains(msg, "[warn]"): |
| return result{testcase: query, status: warn, message: msg} |
| case strings.Contains(msg, "[skip]"): |
| return result{testcase: query, status: skip, message: msg} |
| case strings.Contains(msg, "[pass]"), err == nil: |
| return result{testcase: query, status: pass, message: msg} |
| } |
| return result{testcase: query, status: fail, message: fmt.Sprint(msg, err), error: err} |
| } |
| |
| // filterTestcases returns in with empty strings removed |
| func filterTestcases(in []string) []string { |
| out := make([]string, 0, len(in)) |
| for _, c := range in { |
| if c != "" { |
| out = append(out, c) |
| } |
| } |
| return out |
| } |
| |
| // percentage returns the percentage of n out of total as a string |
| func percentage(n, total int) string { |
| if total == 0 { |
| return "-" |
| } |
| f := float64(n) / float64(total) |
| return fmt.Sprintf("%.1f%c", f*100.0, '%') |
| } |
| |
| // isDir returns true if the path resolves to a directory |
| func isDir(path string) bool { |
| s, err := os.Stat(path) |
| if err != nil { |
| return false |
| } |
| return s.IsDir() |
| } |
| |
| // isFile returns true if the path resolves to a file |
| func isFile(path string) bool { |
| s, err := os.Stat(path) |
| if err != nil { |
| return false |
| } |
| return !s.IsDir() |
| } |
| |
| // alignLeft returns the string of 'val' padded so that it is aligned left in |
| // a column of the given width |
| func alignLeft(val interface{}, width int) string { |
| s := fmt.Sprint(val) |
| padding := width - utf8.RuneCountInString(s) |
| if padding < 0 { |
| return s |
| } |
| return s + strings.Repeat(" ", padding) |
| } |
| |
| // alignRight returns the string of 'val' padded so that it is aligned right in |
| // a column of the given width |
| func alignRight(val interface{}, width int) string { |
| s := fmt.Sprint(val) |
| padding := width - utf8.RuneCountInString(s) |
| if padding < 0 { |
| return s |
| } |
| return strings.Repeat(" ", padding) + s |
| } |
| |
| // printANSIProgressBar prints a colored progress bar, providing realtime |
| // information about the status of the CTS run. |
| // Note: We'll want to skip this if !isatty or if we're running on windows. |
| func printANSIProgressBar(animFrame int, numTests int, numByExpectedStatus map[expectedStatus]int) { |
| const barWidth = 50 |
| |
| animSymbols := []rune{'⣾', '⣽', '⣻', '⢿', '⡿', '⣟', '⣯', '⣷'} |
| blockSymbols := []rune{'▏', '▎', '▍', '▌', '▋', '▊', '▉'} |
| |
| numBlocksPrinted := 0 |
| |
| fmt.Fprint(stdout, string(animSymbols[animFrame%len(animSymbols)]), " [") |
| animFrame++ |
| |
| numFinished := 0 |
| |
| for _, status := range statuses { |
| for _, expected := range []bool{true, false} { |
| color := statusColor[status] |
| if expected { |
| color += bold |
| } |
| |
| num := numByExpectedStatus[expectedStatus{status, expected}] |
| numFinished += num |
| statusFrac := float64(num) / float64(numTests) |
| fNumBlocks := barWidth * statusFrac |
| fmt.Fprint(stdout, color) |
| numBlocks := int(math.Ceil(fNumBlocks)) |
| if expected { |
| if numBlocks > 1 { |
| fmt.Print(strings.Repeat(string("░"), numBlocks)) |
| } |
| } else { |
| if numBlocks > 1 { |
| fmt.Print(strings.Repeat(string("▉"), numBlocks)) |
| } |
| if numBlocks > 0 { |
| frac := fNumBlocks - math.Floor(fNumBlocks) |
| symbol := blockSymbols[int(math.Round(frac*float64(len(blockSymbols)-1)))] |
| fmt.Print(string(symbol)) |
| } |
| } |
| numBlocksPrinted += numBlocks |
| } |
| } |
| |
| if barWidth > numBlocksPrinted { |
| fmt.Print(strings.Repeat(string(" "), barWidth-numBlocksPrinted)) |
| } |
| fmt.Fprint(stdout, ansiReset) |
| fmt.Print("] ", percentage(numFinished, numTests)) |
| |
| if colors { |
| // move cursor to start of line so the bar is overridden |
| fmt.Fprint(stdout, positionLeft) |
| } else { |
| // cannot move cursor, so newline |
| fmt.Println() |
| } |
| } |
| |
| // testcaseStatus is a pair of testcase name and result status |
| // Intended to be serialized for expectations files. |
| type testcaseStatus struct { |
| Testcase string |
| Status status |
| } |
| |
| // testcaseStatuses is a map of testcase to test status |
| type testcaseStatuses map[string]status |
| |
| // loadExpectations loads the test expectations from path |
| func loadExpectations(path string) (testcaseStatuses, error) { |
| f, err := os.Open(path) |
| if err != nil { |
| return nil, fmt.Errorf("failed to open expectations file: %w", err) |
| } |
| defer f.Close() |
| |
| statuses := []testcaseStatus{} |
| if err := json.NewDecoder(f).Decode(&statuses); err != nil { |
| return nil, fmt.Errorf("failed to read expectations file: %w", err) |
| } |
| |
| out := make(testcaseStatuses, len(statuses)) |
| for _, s := range statuses { |
| out[s.Testcase] = s.Status |
| } |
| return out, nil |
| } |
| |
| // saveExpectations saves the test results 'ex' as an expectations file to path |
| func saveExpectations(path string, ex testcaseStatuses) error { |
| f, err := os.Create(path) |
| if err != nil { |
| return fmt.Errorf("failed to create expectations file: %w", err) |
| } |
| defer f.Close() |
| |
| statuses := make([]testcaseStatus, 0, len(ex)) |
| for testcase, status := range ex { |
| statuses = append(statuses, testcaseStatus{testcase, status}) |
| } |
| sort.Slice(statuses, func(i, j int) bool { return statuses[i].Testcase < statuses[j].Testcase }) |
| |
| e := json.NewEncoder(f) |
| e.SetIndent("", " ") |
| if err := e.Encode(&statuses); err != nil { |
| return fmt.Errorf("failed to save expectations file: %w", err) |
| } |
| |
| return nil |
| } |
| |
| // defaultNodePath looks for the node binary, first in dawn's third_party |
| // directory, falling back to PATH. This is used as the default for the --node |
| // command line flag. |
| func defaultNodePath() string { |
| if dir := thisDir(); dir != "" { |
| if dawnRoot := getDawnRoot(); dawnRoot != "" { |
| node := filepath.Join(dawnRoot, "third_party/node") |
| if info, err := os.Stat(node); err == nil && info.IsDir() { |
| path := "" |
| switch fmt.Sprintf("%v/%v", runtime.GOOS, runtime.GOARCH) { // See `go tool dist list` |
| case "darwin/amd64": |
| path = filepath.Join(node, "node-darwin-x64/bin/node") |
| case "darwin/arm64": |
| path = filepath.Join(node, "node-darwin-arm64/bin/node") |
| case "linux/amd64": |
| path = filepath.Join(node, "node-linux-x64/bin/node") |
| case "windows/amd64": |
| path = filepath.Join(node, "node.exe") |
| } |
| if _, err := os.Stat(path); err == nil { |
| return path |
| } |
| } |
| } |
| } |
| |
| if path, err := exec.LookPath("node"); err == nil { |
| return path |
| } |
| |
| return "" |
| } |
| |
| // defaultCtsPath looks for the webgpu-cts directory in dawn's third_party |
| // directory. This is used as the default for the --cts command line flag. |
| func defaultCtsPath() string { |
| if dir := thisDir(); dir != "" { |
| if dawnRoot := getDawnRoot(); dawnRoot != "" { |
| cts := filepath.Join(dawnRoot, "third_party/webgpu-cts") |
| if info, err := os.Stat(cts); err == nil && info.IsDir() { |
| return cts |
| } |
| } |
| } |
| |
| return "" |
| } |
| |
| // getDawnRoot returns the path to the dawn project's root directory or empty |
| // string if not found. |
| func getDawnRoot() string { |
| return getPathOfFileInParentDirs(thisDir(), "DEPS") |
| } |
| |
| // getPathOfFileInParentDirs looks for file with `name` in paths starting from |
| // `path`, and up into parent directories, returning the clean path in which the |
| // file is found, or empty string if not found. |
| func getPathOfFileInParentDirs(path string, name string) string { |
| sep := string(filepath.Separator) |
| path, _ = filepath.Abs(path) |
| numDirs := strings.Count(path, sep) + 1 |
| for i := 0; i < numDirs; i++ { |
| test := filepath.Join(path, name) |
| if _, err := os.Stat(test); err == nil { |
| return filepath.Clean(path) |
| } |
| |
| path = path + sep + ".." |
| } |
| return "" |
| } |
| |
| // thisDir returns the path to the directory that holds the .go file of the |
| // caller function |
| func thisDir() string { |
| _, file, _, ok := runtime.Caller(1) |
| if !ok { |
| return "" |
| } |
| return filepath.Dir(file) |
| } |