blob: ce3ebb4111aec8353df6c7eae989fe7c77028ce9 [file] [log] [blame]
// Copyright 2023 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package common
import (
"bytes"
"context"
"fmt"
"io"
"os"
"strings"
"sync"
"time"
"unicode/utf8"
"dawn.googlesource.com/dawn/tools/src/browser"
"dawn.googlesource.com/dawn/tools/src/cov"
"dawn.googlesource.com/dawn/tools/src/fileutils"
"dawn.googlesource.com/dawn/tools/src/git"
"dawn.googlesource.com/dawn/tools/src/progressbar"
"dawn.googlesource.com/dawn/tools/src/term"
)
var statusColor = map[Status]string{
Pass: term.Green,
Warn: term.Yellow,
Skip: term.Cyan,
Timeout: term.Yellow,
Fail: term.Red,
}
var pbStatusColor = map[Status]progressbar.Color{
Pass: progressbar.Green,
Warn: progressbar.Yellow,
Skip: progressbar.Cyan,
Timeout: progressbar.Yellow,
Fail: progressbar.Red,
}
// Create another goroutine to close the results chan when all the runner
// goroutines have finished.
func CloseChanOnWaitGroupDone[T any](wg *sync.WaitGroup, c chan<- T) {
go func() {
wg.Wait()
close(c)
}()
}
// Coverage holds information about coverage gathering. Used by StreamResults.
type Coverage struct {
Env *cov.Env
OutputFile string
}
// StreamResults reads from the chan 'results', printing the results in test-id
// sequential order.
// Once all the results have been printed, a summary will be printed and the
// function will return.
func StreamResults(
ctx context.Context,
colors bool,
state State,
verbose bool,
coverage *Coverage, // Optional coverage generation info
numTestCases int, // Total number of test cases
stream <-chan Result) (Results, error) {
// If the context was already cancelled then just return
if err := ctx.Err(); err != nil {
return nil, err
}
stdout := state.Stdout
results := Results{}
// Total number of tests, test counts binned by status
numByExpectedStatus := map[expectedStatus]int{}
// Helper function for printing a progress bar.
lastStatusUpdate, animFrame := time.Now(), 0
updateProgress := func() {
drawProgressBar(stdout, animFrame, numTestCases, colors, numByExpectedStatus)
animFrame++
lastStatusUpdate = time.Now()
}
// Pull test results as they become available.
// Update the status counts, and print any failures (or all test results if --verbose)
progressUpdateRate := time.Millisecond * 10
if !colors {
// No colors == no cursor control. Reduce progress updates so that
// we're not printing endless progress bars.
progressUpdateRate = time.Second
}
var covTree *cov.Tree
if coverage != nil {
covTree = &cov.Tree{}
}
start := time.Now()
for res := range stream {
state.Log.logResults(res)
results[res.TestCase] = res.Status
expected := state.Expectations[res.TestCase]
exStatus := expectedStatus{
status: res.Status,
expected: expected == res.Status,
}
numByExpectedStatus[exStatus] = numByExpectedStatus[exStatus] + 1
name := res.TestCase
if verbose ||
res.Error != nil ||
(exStatus.status != Pass && exStatus.status != Skip && !exStatus.expected) {
buf := &bytes.Buffer{}
fmt.Fprint(buf, statusColor[res.Status])
if res.Message != "" {
fmt.Fprintf(buf, "%v - %v:\n", name, res.Status)
fmt.Fprintf(buf, term.Reset)
fmt.Fprintf(buf, "%v", res.Message)
} else {
fmt.Fprintf(buf, "%v - %v", name, res.Status)
}
if expected != "" {
fmt.Fprintf(buf, " [%v -> %v]", expected, res.Status)
}
fmt.Fprintln(buf)
if res.Error != nil {
fmt.Fprintln(buf, res.Error)
}
fmt.Fprint(buf, term.Reset)
fmt.Fprint(stdout, buf.String())
updateProgress()
}
if time.Since(lastStatusUpdate) > progressUpdateRate {
updateProgress()
}
if res.Coverage != nil {
covTree.Add(splitCTSQuery(res.TestCase), res.Coverage)
}
}
timeTaken := time.Since(start)
drawProgressBar(stdout, animFrame, numTestCases, colors, numByExpectedStatus)
// All done. Print final stats.
fmt.Fprintf(stdout, "\nCompleted in %v\n", timeTaken)
var numExpectedByStatus map[Status]int
if state.Expectations != nil {
// The status of each testcase that was run
numExpectedByStatus = map[Status]int{}
for t, s := range state.Expectations {
if _, wasTested := results[t]; wasTested {
numExpectedByStatus[s] = numExpectedByStatus[s] + 1
}
}
}
for _, s := range AllStatuses {
// number of tests, just run, that resulted in the given status
numByStatus := numByExpectedStatus[expectedStatus{s, true}] +
numByExpectedStatus[expectedStatus{s, false}]
// difference in number of tests that had the given status from the
// expected number (taken from the expectations file)
diffFromExpected := 0
if numExpectedByStatus != nil {
diffFromExpected = numByStatus - numExpectedByStatus[s]
}
if numByStatus == 0 && diffFromExpected == 0 {
continue
}
fmt.Fprint(stdout, term.Bold, statusColor[s])
fmt.Fprint(stdout, alignRight(strings.ToUpper(string(s))+": ", 10))
fmt.Fprint(stdout, term.Reset)
if numByStatus > 0 {
fmt.Fprint(stdout, term.Bold)
}
fmt.Fprint(stdout, alignLeft(numByStatus, 10))
fmt.Fprint(stdout, term.Reset)
fmt.Fprint(stdout, alignRight("("+percentage(numByStatus, numTestCases)+")", 6))
if diffFromExpected != 0 {
fmt.Fprint(stdout, term.Bold, " [")
fmt.Fprintf(stdout, "%+d", diffFromExpected)
fmt.Fprint(stdout, term.Reset, "]")
}
fmt.Fprintln(stdout)
}
if coverage != nil {
// Obtain the current git revision
revision := "HEAD"
if g, err := git.New(""); err == nil {
if r, err := g.Open(fileutils.DawnRoot()); err == nil {
if l, err := r.Log(&git.LogOptions{From: "HEAD", To: "HEAD"}); err == nil {
revision = l[0].Hash.String()
}
}
}
if coverage.OutputFile != "" {
file, err := os.Create(coverage.OutputFile)
if err != nil {
return nil, fmt.Errorf("failed to create the coverage file: %w", err)
}
defer file.Close()
if err := covTree.Encode(revision, file); err != nil {
return nil, fmt.Errorf("failed to encode coverage file: %w", err)
}
fmt.Fprintln(stdout)
fmt.Fprintln(stdout, "Coverage data written to "+coverage.OutputFile)
} else {
covData := &bytes.Buffer{}
if err := covTree.Encode(revision, covData); err != nil {
return nil, fmt.Errorf("failed to encode coverage file: %w", err)
}
const port = 9392
url := fmt.Sprintf("http://localhost:%v/index.html", port)
err := cov.StartServer(ctx, port, covData.Bytes(), func() error {
fmt.Fprintln(stdout)
fmt.Fprintln(stdout, term.Blue+"Serving coverage view at "+url+term.Reset)
return browser.Open(url)
})
if err != nil {
return nil, err
}
}
}
return results, nil
}
// expectedStatus is a test status, along with a boolean to indicate whether the
// status matches the test expectations
type expectedStatus struct {
status Status
expected bool
}
// percentage returns the percentage of n out of total as a string
func percentage(n, total int) string {
if total == 0 {
return "-"
}
f := float64(n) / float64(total)
return fmt.Sprintf("%.1f%c", f*100.0, '%')
}
// alignLeft returns the string of 'val' padded so that it is aligned left in
// a column of the given width
func alignLeft(val interface{}, width int) string {
s := fmt.Sprint(val)
padding := width - utf8.RuneCountInString(s)
if padding < 0 {
return s
}
return s + strings.Repeat(" ", padding)
}
// alignRight returns the string of 'val' padded so that it is aligned right in
// a column of the given width
func alignRight(val interface{}, width int) string {
s := fmt.Sprint(val)
padding := width - utf8.RuneCountInString(s)
if padding < 0 {
return s
}
return strings.Repeat(" ", padding) + s
}
// drawProgressBar draws an ANSI-colored progress bar, providing realtime
// information about the status of the CTS run.
// Note: We'll want to skip this if !isatty or if we're running on windows.
func drawProgressBar(stdout io.Writer, animFrame int, numTestCases int, colors bool, numByExpectedStatus map[expectedStatus]int) {
bar := progressbar.Status{Total: numTestCases}
for _, status := range AllStatuses {
for _, expected := range []bool{true, false} {
if num := numByExpectedStatus[expectedStatus{status, expected}]; num > 0 {
bar.Segments = append(bar.Segments,
progressbar.Segment{
Count: num,
Color: pbStatusColor[status],
Bold: expected,
Transparent: expected,
})
}
}
}
const width = 50
bar.Draw(stdout, width, colors, animFrame)
}
// splitCTSQuery splits a CTS query into a cov.Path
//
// Each WebGPU CTS test is uniquely identified by a test query.
// See https://github.com/gpuweb/cts/blob/main/docs/terms.md#queries about how a
// query is officially structured.
//
// A Path is a simplified form of a CTS Query, where all colons ':' and comma
// ',' denote a split point in the tree. These delimiters are included in the
// parent node's string.
//
// For example, the query string for the single test case:
//
// webgpu:shader,execution,expression,call,builtin,acos:f32:inputSource="storage_r";vectorize=4
//
// Is broken down into the following strings:
//
// 'webgpu:'
// 'shader,'
// 'execution,'
// 'expression,'
// 'call,'
// 'builtin,'
// 'acos:'
// 'f32:'
// 'inputSource="storage_r";vectorize=4'
func splitCTSQuery(testcase TestCase) cov.Path {
out := []string{}
s := 0
for e, r := range testcase {
switch r {
case ':', '.':
out = append(out, string(testcase[s:e+1]))
s = e + 1
}
}
if end := testcase[s:]; end != "" {
out = append(out, string(end))
}
return out
}