[tools] Replace test-all.sh with ./tools/run tests
• Simplify the usage by allowing globs, directories or single test file paths as unnamed arguments. Removes the `--filter` flag that nobody really understood.
• Make the table fit the terminal.
• Replace dawn-path with `<dawn>` in expectation files.
Change-Id: I45a0579e3589888664877c22edb575d1a4ed18c1
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/161701
Reviewed-by: dan sinclair <dsinclair@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
diff --git a/tools/src/cmd/test-runner/main.go b/tools/src/cmd/tests/main.go
similarity index 83%
rename from tools/src/cmd/test-runner/main.go
rename to tools/src/cmd/tests/main.go
index 791ea03..9866c9c 100644
--- a/tools/src/cmd/test-runner/main.go
+++ b/tools/src/cmd/tests/main.go
@@ -25,7 +25,7 @@
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// test-runner runs tint against a number of test shaders checking for expected behavior
+// tests runs tint against a number of test shaders checking for expected behavior
package main
import (
@@ -37,6 +37,7 @@
"io/ioutil"
"os"
"os/exec"
+ "path"
"path/filepath"
"regexp"
"runtime"
@@ -47,8 +48,11 @@
"dawn.googlesource.com/dawn/tools/src/fileutils"
"dawn.googlesource.com/dawn/tools/src/glob"
+ "dawn.googlesource.com/dawn/tools/src/match"
+ "dawn.googlesource.com/dawn/tools/src/transform"
"github.com/fatih/color"
"github.com/sergi/go-diff/diffmatchpatch"
+ "golang.org/x/term"
)
type outputFormat string
@@ -64,14 +68,27 @@
wgsl = outputFormat("wgsl")
)
+// The root directory of the dawn project
+var dawnRoot = fileutils.DawnRoot()
+
+// The default non-flag arguments to the command
+var defaultArgs = []string{"test/tint"}
+
+// The globs automatically appended if a glob argument is a directory
+var directoryGlobs = []string{
+ "**.wgsl",
+ "**.spvasm",
+ "**.spv",
+}
+
// Directories we don't generate expected PASS result files for.
// These directories contain large corpora of tests for which the generated code
// is uninteresting.
// These paths use unix-style slashes and do not contain the '/test/tint' prefix.
var dirsWithNoPassExpectations = []string{
- "benchmark/",
- "unittest/",
- "vk-gl-cts/",
+ dawnRoot + "/test/tint/benchmark/",
+ dawnRoot + "/test/tint/unittest/",
+ dawnRoot + "/test/tint/vk-gl-cts/",
}
func main() {
@@ -82,98 +99,116 @@
}
func showUsage() {
- fmt.Println(`
-test-runner runs tint against a number of test shaders checking for expected behavior
+ fmt.Printf(`
+tests runs tint against a number of test shaders checking for expected behavior
usage:
- test-runner [flags...] <executable> [<directory>]
+ tests [flags...] [globs...]
- <executable> the path to the tint executable
- <directory> the root directory of the test files
+ [globs] a list of project-root relative file globs, directory or file paths
+ of test cases.
+ A file path will be added to the test list.
+ A directory will automatically expand to the globs:
+ %v
+ Globs will include all test files that match the glob, but exclude
+ those that match the --ignore flag.
+ If omitted, defaults to: %v
-optional flags:`)
+optional flags:`,
+ transform.SliceNoErr(directoryGlobs, func(in string) string { return fmt.Sprintf("'<dir>/%v'", in) }),
+ transform.SliceNoErr(defaultArgs, func(in string) string { return fmt.Sprintf("'%v'", in) }))
flag.PrintDefaults()
fmt.Println(``)
os.Exit(1)
}
func run() error {
- var formatList, filter, dxcPath, fxcPath, xcrunPath string
- var maxFilenameColumnWidth int
+ terminalWidth, _, err := term.GetSize(int(os.Stdout.Fd()))
+ if err != nil {
+ terminalWidth = 0
+ }
+
+ var formatList, ignore, dxcPath, fxcPath, tintPath, xcrunPath string
+ var maxTableWidth int
numCPU := runtime.NumCPU()
verbose, useIr, generateExpected, generateSkip := false, false, false, false
flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl, hlsl-dxc, hlsl-fxc, glsl")
- flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
+ flag.StringVar(&ignore, "ignore", "**.expected.*", "files to ignore in globs")
flag.StringVar(&dxcPath, "dxc", "", "path to DXC executable for validating HLSL output")
flag.StringVar(&fxcPath, "fxc", "", "path to FXC DLL for validating HLSL output")
+ flag.StringVar(&tintPath, "tint", defaultTintPath(), "path to the tint executable")
flag.StringVar(&xcrunPath, "xcrun", "", "path to xcrun executable for validating MSL output")
flag.BoolVar(&verbose, "verbose", false, "print all run tests, including rows that all pass")
flag.BoolVar(&useIr, "use-ir", false, "generate with the IR enabled")
flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
flag.BoolVar(&generateSkip, "generate-skip", false, "create or update all expected outputs that fail with SKIP")
flag.IntVar(&numCPU, "j", numCPU, "maximum number of concurrent threads to run tests")
- flag.IntVar(&maxFilenameColumnWidth, "filename-column-width", 0, "maximum width of the filename column")
+ flag.IntVar(&maxTableWidth, "table-width", terminalWidth, "maximum width of the results table")
flag.Usage = showUsage
flag.Parse()
- args := flag.Args()
- if len(args) == 0 {
+ // Check the executable can be found and actually is executable
+ if !fileutils.IsExe(tintPath) {
+ fmt.Fprintln(os.Stderr, "tint executable not found, please specify with --tint")
showUsage()
}
- // executable path is the first argument
- exe, args := args[0], args[1:]
-
- // (optional) target directory is the second argument
- dir := "."
- if len(args) > 0 {
- dir, args = args[0], args[1:]
+ // Apply default args, if not provided
+ args := flag.Args()
+ if len(args) == 0 {
+ args = defaultArgs
}
- // Check the executable can be found and actually is executable
- if !fileutils.IsExe(exe) {
- return fmt.Errorf("'%s' not found or is not executable", exe)
- }
- exe, err := filepath.Abs(exe)
- if err != nil {
- return err
+ filePredicate := func(s string) bool { return true }
+ if m, err := match.New(ignore); err == nil {
+ filePredicate = func(s string) bool { return !m(s) }
+ } else {
+ return fmt.Errorf("failed to parse --ignore: %w", err)
}
- // Allow using '/' in the filter on Windows
- filter = strings.ReplaceAll(filter, "/", string(filepath.Separator))
+ // Transform args to globs, find the rootPath directory
+ absFiles := []string{}
+ rootPath := ""
+ globs := []string{}
+ for _, arg := range args {
+ // Make absolute
+ if !filepath.IsAbs(arg) {
+ arg = filepath.Join(dawnRoot, arg)
+ }
- // Split the --filter flag up by ',', trimming any whitespace at the start and end
- globIncludes := strings.Split(filter, ",")
- for i, s := range globIncludes {
- s = filepath.ToSlash(s) // Replace '\' with '/'
- globIncludes[i] = `"` + strings.TrimSpace(s) + `"`
- }
-
- // Glob the files to test
- files, err := glob.Scan(dir, glob.MustParseConfig(`{
- "paths": [
- {
- "include": [ `+strings.Join(globIncludes, ",")+` ]
- },
- {
- "exclude": [
- "**.expected.wgsl",
- "**.expected.spvasm",
- "**.expected.ir.spvasm",
- "**.expected.msl",
- "**.expected.fxc.hlsl",
- "**.expected.dxc.hlsl",
- "**.expected.glsl"
- ]
+ switch {
+ case fileutils.IsDir(arg):
+ // Argument is to a directory, expand out to N globs
+ for _, glob := range directoryGlobs {
+ globs = append(globs, path.Join(arg, glob))
}
- ]
- }`))
- if err != nil {
- return fmt.Errorf("Failed to glob files: %w", err)
+ case fileutils.IsFile(arg):
+ // Argument is a file, append to absFiles
+ absFiles = append(absFiles, arg)
+ default:
+ globs = append(globs, arg)
+ }
+
+ if rootPath == "" {
+ rootPath = filepath.Dir(arg)
+ } else {
+ rootPath = fileutils.CommonRootDir(rootPath, arg)
+ }
+ }
+
+ // Glob the absFiles to test
+ for _, g := range globs {
+ globFiles, err := glob.Glob(g)
+ if err != nil {
+ return fmt.Errorf("Failed to glob files: %w", err)
+ }
+ filtered := transform.Filter(globFiles, filePredicate)
+ normalized := transform.SliceNoErr(filtered, filepath.ToSlash)
+ absFiles = append(absFiles, normalized...)
}
// Ensure the files are sorted (globbing should do this, but why not)
- sort.Strings(files)
+ sort.Strings(absFiles)
// Parse --format into a list of outputFormat
formats := []outputFormat{}
@@ -255,8 +290,8 @@
// Build the list of results.
// These hold the chans used to report the job results.
- results := make([]map[outputFormat]chan status, len(files))
- for i := range files {
+ results := make([]map[outputFormat]chan status, len(absFiles))
+ for i := range absFiles {
fileResults := map[outputFormat]chan status{}
for _, format := range formats {
fileResults[format] = make(chan status, 1)
@@ -268,8 +303,8 @@
// Spawn numCPU job runners...
runCfg := runConfig{
- wd: dir,
- exe: exe,
+ rootPath: rootPath,
+ tintPath: tintPath,
dxcPath: dxcPath,
fxcPath: fxcPath,
xcrunPath: xcrunPath,
@@ -288,8 +323,7 @@
// Issue the jobs...
go func() {
- for i, file := range files { // For each test file...
- file := filepath.Join(dir, file)
+ for i, file := range absFiles { // For each test file...
flags := parseFlags(file)
for _, format := range formats { // For each output format...
pendingJobs <- job{
@@ -320,11 +354,31 @@
statsByFmt[format] = &stats{}
}
+ // Make file paths relative to rootPath, if possible
+ relFiles := transform.GoSliceNoErr(absFiles, func(path string) string {
+ path = filepath.ToSlash(path) // Normalize
+ if rel, err := filepath.Rel(rootPath, path); err == nil {
+ return rel
+ }
+ return path
+ })
+
// Print the table of file x format and gather per-format stats
failures := []failure{}
- filenameColumnWidth := maxStringLen(files)
- if maxFilenameColumnWidth > 0 {
- filenameColumnWidth = maxFilenameColumnWidth
+ filenameColumnWidth := maxStringLen(relFiles)
+
+ // Calculate the table width
+ tableWidth := filenameColumnWidth + 3
+ for _, format := range formats {
+ tableWidth += formatWidth(format) + 3
+ }
+
+ // Reduce filename column width if too big
+ if tableWidth > maxTableWidth {
+ filenameColumnWidth -= tableWidth - maxTableWidth
+ if filenameColumnWidth < 20 {
+ filenameColumnWidth = 20
+ }
}
red := color.New(color.FgRed)
@@ -358,7 +412,7 @@
newKnownGood := knownGoodHashes{}
- for i, file := range files {
+ for i, file := range relFiles {
results := results[i]
row := &strings.Builder{}
@@ -560,8 +614,8 @@
}
type runConfig struct {
- wd string
- exe string
+ rootPath string
+ tintPath string
dxcPath string
fxcPath string
xcrunPath string
@@ -591,7 +645,7 @@
// Is there an expected output file? If so, load it.
expected, expectedFileExists := "", false
- if content, err := ioutil.ReadFile(expectedFilePath); err == nil {
+ if content, err := os.ReadFile(expectedFilePath); err == nil {
expected = string(content)
expectedFileExists = true
}
@@ -603,17 +657,8 @@
expected = strings.ReplaceAll(expected, "\r\n", "\n")
- file, err := filepath.Rel(cfg.wd, j.file)
- if err != nil {
- file = j.file
- }
-
- // Make relative paths use forward slash separators (on Windows) so that paths in tint
- // output match expected output that contain errors
- file = strings.ReplaceAll(file, `\`, `/`)
-
args := []string{
- file,
+ j.file,
"--format", strings.Split(string(j.format), "-")[0], // 'hlsl-fxc' -> 'hlsl', etc.
"--print-hash",
}
@@ -624,7 +669,7 @@
// Append any skip-hashes, if they're found.
if j.format != "wgsl" { // Don't skip 'wgsl' as this 'toolchain' is ever changing.
- if skipHashes := cfg.validationCache.knownGood[fileAndFormat{file, j.format}]; len(skipHashes) > 0 {
+ if skipHashes := cfg.validationCache.knownGood[fileAndFormat{j.file, j.format}]; len(skipHashes) > 0 {
args = append(args, "--skip-hash", strings.Join(skipHashes, ","))
}
}
@@ -663,23 +708,24 @@
args = append(args, j.flags...)
start := time.Now()
- ok, out = invoke(cfg.wd, cfg.exe, args...)
+ ok, out = invoke(cfg.rootPath, cfg.tintPath, args...)
timeTaken := time.Since(start)
out = strings.ReplaceAll(out, "\r\n", "\n")
+ out = strings.ReplaceAll(out, filepath.ToSlash(dawnRoot), "<dawn>")
out, hashes := extractValidationHashes(out)
matched := expected == "" || expected == out
canEmitPassExpectationFile := true
for _, noPass := range dirsWithNoPassExpectations {
- if strings.HasPrefix(file, noPass) {
+ if strings.HasPrefix(j.file, noPass) {
canEmitPassExpectationFile = false
break
}
}
saveExpectedFile := func(path string, content string) error {
- return ioutil.WriteFile(path, []byte(content), 0666)
+ return os.WriteFile(path, []byte(content), 0666)
}
if ok && cfg.generateExpected && (validate || !skipped) {
@@ -998,3 +1044,13 @@
enc.SetIndent("", " ")
return enc.Encode(&out)
}
+
+// defaultRootPath returns the default path to the root of the test tree
+func defaultRootPath() string {
+ return filepath.Join(fileutils.DawnRoot(), "test/tint")
+}
+
+// defaultTintPath returns the default path to the tint executable
+func defaultTintPath() string {
+ return filepath.Join(fileutils.DawnRoot(), "out/active/tint")
+}
diff --git a/tools/src/fileutils/paths.go b/tools/src/fileutils/paths.go
index e6f03f8..a2edcfb 100644
--- a/tools/src/fileutils/paths.go
+++ b/tools/src/fileutils/paths.go
@@ -137,3 +137,29 @@
}
return !s.IsDir()
}
+
+// CommonRootDir returns the common directory for pathA and pathB
+func CommonRootDir(pathA, pathB string) string {
+ pathA, pathB = filepath.ToSlash(pathA), filepath.ToSlash(pathB) // Normalize to forward-slash
+ if !strings.HasSuffix(pathA, "/") {
+ pathA += "/"
+ }
+ if !strings.HasSuffix(pathB, "/") {
+ pathB += "/"
+ }
+ n := len(pathA)
+ if len(pathB) < n {
+ n = len(pathB)
+ }
+ common := ""
+ for i := 0; i < n; i++ {
+ a, b := pathA[i], pathB[i]
+ if a != b {
+ break
+ }
+ if a == '/' {
+ common = pathA[:i+1]
+ }
+ }
+ return common
+}
diff --git a/tools/src/fileutils/paths_test.go b/tools/src/fileutils/paths_test.go
index 46102b9..ec0f1d3 100644
--- a/tools/src/fileutils/paths_test.go
+++ b/tools/src/fileutils/paths_test.go
@@ -62,3 +62,26 @@
t.Errorf("DawnRoot() returned %v.\n%v", dr, diff)
}
}
+
+func TestCommonRootDir(t *testing.T) {
+ for _, test := range []struct {
+ a, b string
+ expect string
+ }{
+ {"", "", "/"},
+ {"a/b/c", "d/e/f", ""},
+ {"a/", "b", ""},
+ {"a/b/c", "a/b", "a/b/"},
+ {"a/b/c/", "a/b", "a/b/"},
+ {"a/b/c/", "a/b/", "a/b/"},
+ {"a/b/c", "a/b/d", "a/b/"},
+ {"a/b/c", "a/bc", "a/"},
+ } {
+ if got := fileutils.CommonRootDir(test.a, test.b); got != test.expect {
+ t.Errorf("CommonRootDir('%v', '%v') returned '%v'.\nExpected: '%v'", test.a, test.b, got, test.expect)
+ }
+ if got := fileutils.CommonRootDir(test.b, test.a); got != test.expect {
+ t.Errorf("CommonRootDir('%v', '%v') returned '%v'.\nExpected: '%v'", test.b, test.a, got, test.expect)
+ }
+ }
+}
diff --git a/tools/src/glob/glob.go b/tools/src/glob/glob.go
index 0e05306..94648d3 100644
--- a/tools/src/glob/glob.go
+++ b/tools/src/glob/glob.go
@@ -71,7 +71,7 @@
}
}
// No wildcard found. Does the file exist at 'str'?
- if s, err := os.Stat(str); err != nil && !s.IsDir() {
+ if s, err := os.Stat(str); err == nil && !s.IsDir() {
return []string{str}, nil
}
return []string{}, nil
diff --git a/tools/src/transform/slice.go b/tools/src/transform/slice.go
index de118bc..1f965a0 100644
--- a/tools/src/transform/slice.go
+++ b/tools/src/transform/slice.go
@@ -66,6 +66,15 @@
return out, nil
}
+// SliceNoErr returns a new slice by transforming each element with the function fn
+func SliceNoErr[IN any, OUT any](in []IN, fn func(in IN) OUT) []OUT {
+ out := make([]OUT, len(in))
+ for i, el := range in {
+ out[i] = fn(el)
+ }
+ return out
+}
+
// GoSlice returns a new slice by transforming each element with the function
// fn, called by multiple go-routines.
func GoSlice[IN any, OUT any](in []IN, fn func(in IN) (OUT, error)) ([]OUT, error) {
@@ -102,3 +111,35 @@
return out, nil
}
+
+// GoSliceNoErr returns a new slice by transforming each element with the function
+// fn, called by multiple go-routines.
+func GoSliceNoErr[IN any, OUT any](in []IN, fn func(in IN) OUT) []OUT {
+
+ // Create a channel of indices
+ indices := make(chan int, 256)
+ go func() {
+ for i := range in {
+ indices <- i
+ }
+ close(indices)
+ }()
+
+ out := make([]OUT, len(in))
+
+ // Kick a number of workers to process the elements
+ numWorkers := runtime.NumCPU()
+ wg := sync.WaitGroup{}
+ wg.Add(numWorkers)
+ for worker := 0; worker < numWorkers; worker++ {
+ go func() {
+ defer wg.Done()
+ for idx := range indices {
+ out[idx] = fn(in[idx])
+ }
+ }()
+ }
+ wg.Wait()
+
+ return out
+}
