tools/test-all.sh: Reimplement in golang
Makes future development easier.
New features:
* A more compact and cleaner results view
* Concurrent testing, much quicker across multiple cores
* Supports comparing output against an expected file, including a text diff of differences. Also has a flag for updating the expected outputs
* Advanced file-globbing support, including scanning for files in subdirectories
* Skip lists are now no longer hidden away in the tool, but defined as a SKIP header in the *.expected.* file
Change-Id: I4fac80bb084a720ec9a307b4acf9f73792973a1d
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/50903
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: David Neto <dneto@google.com>
diff --git a/test/bug_tint_749.spvasm.expected.hlsl b/test/bug_tint_749.spvasm.expected.hlsl
new file mode 100644
index 0000000..1b2d3db
--- /dev/null
+++ b/test/bug_tint_749.spvasm.expected.hlsl
@@ -0,0 +1 @@
+SKIP: TINT_UNIMPLEMENTED crbug.com/tint/726: module-scope private and workgroup variables not yet implemented
diff --git a/test/bug_tint_749.spvasm.expected.msl b/test/bug_tint_749.spvasm.expected.msl
new file mode 100644
index 0000000..1b2d3db
--- /dev/null
+++ b/test/bug_tint_749.spvasm.expected.msl
@@ -0,0 +1 @@
+SKIP: TINT_UNIMPLEMENTED crbug.com/tint/726: module-scope private and workgroup variables not yet implemented
diff --git a/test/bug_tint_749.spvasm.expected.spvasm b/test/bug_tint_749.spvasm.expected.spvasm
new file mode 100644
index 0000000..d7f3593
--- /dev/null
+++ b/test/bug_tint_749.spvasm.expected.spvasm
@@ -0,0 +1,5 @@
+SKIP:
+
+Validation Failure:
+ 1:1: OpLoad Pointer <id> '51[%51]' is not a logical pointer.
+ %52 = OpLoad %int %51
diff --git a/test/test-all.sh b/test/test-all.sh
index ac79421..efc0091 100755
--- a/test/test-all.sh
+++ b/test/test-all.sh
@@ -16,33 +16,18 @@
set -e # Fail on any error.
-TEXT_YELLOW="\033[0;33m"
-TEXT_GREEN="\033[0;32m"
-TEXT_RED="\033[0;31m"
-TEXT_DEFAULT="\033[0m"
-
-CHECK_WGSL=1
-CHECK_SPV=1
-CHECK_MSL=1
-CHECK_HLSL=1
-
-TINT="$1"
-ONLY_FORMAT="$2"
-TARGETDIR="$3"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
function usage() {
- echo "test-all.sh uses tint to compile .wgsl and .spvasm files, reporting errors as test failures."
+ echo "test-all.sh is a simple wrapper around <tint>/tools/test-runner that"
+ echo "injects the <tint>/tools directory as the second command line argument"
echo
- echo "Usage: test-all.sh <path-to-tint-executable> [<only-format> [directory]]"
- echo
- echo "<only-format> specifies which output format is tested."
- echo " Possible values are: all, wgsl, spv, msl, hlsl."
- echo " The default is 'all'."
- echo
- echo "[directory] specifies which directory holds the source files"
- echo " The default is to use the script directory."
+ echo "Usage of <tint>/tools/test-runner:"
+ SCRIPT_DIR/../tools/test-runner --help
}
+TINT="$1"
+
if [ -z "$TINT" ]; then
echo "error: missing argument: location of the 'tint' executable"
echo
@@ -56,132 +41,4 @@
exit 1
fi
-if [ -n "$ONLY_FORMAT" ]; then
- case "${ONLY_FORMAT}" in
- all)
- ;;
- wgsl)
- CHECK_WGSL=1
- CHECK_SPV=0
- CHECK_MSL=0
- CHECK_HLSL=0
- ;;
- spv)
- CHECK_WGSL=0
- CHECK_SPV=1
- CHECK_MSL=0
- CHECK_HLSL=0
- ;;
- msl)
- CHECK_WGSL=0
- CHECK_SPV=0
- CHECK_MSL=1
- CHECK_HLSL=0
- ;;
- hlsl)
- CHECK_WGSL=0
- CHECK_SPV=0
- CHECK_MSL=0
- CHECK_HLSL=1
- ;;
- *)
- echo "error: invalid format argument: $ONLY_FORMAT"
- echo
- usage
- exit 1
- esac
-fi
-
-if [ -n "$4" ]; then
- echo "error: Too many arguments"
- echo
- usage
- exit 1
-fi
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
-
-# If no subdirectory was specified, look in the script directory.
-if [ -z "${TARGETDIR}" ]; then
- TARGETDIR="${SCRIPT_DIR}"
-fi
-
-if [ ! -d "${TARGETDIR}" ]; then
- echo "error: ${TARGETDIR} is not a directory"
- exit 1
-fi
-
-NUM_PASS=0
-NUM_SKIP=0
-NUM_FAIL=0
-
-SKIPPED=""
-SKIPPED+="msl:bug_tint_749.spvasm" # TINT_UNIMPLEMENTED crbug.com/tint/726: module-scope private and workgroup variables not yet implemented
-SKIPPED+="hlsl:bug_tint_749.spvasm" # Failed to generate: error: pointers not supported in HLSL
-
-# should_skip(TEST_FILE, FORMAT)
-function should_skip() {
- local TEST="$1-$2"
- if [[ "$TEST" == "bug_tint_749.spvasm-msl" ]]; then
- echo 1
- return
- fi
- echo 0
- return
-}
-
-# check(TEST_FILE, FORMAT)
-function check() {
- local TEST_FILE="$1"
- local FORMAT=$2
- SKIP=
-
- TEST_FILE_WITHOUT_DIR=$(basename ${TEST_FILE})
- if [[ $SKIPPED == *"${FORMAT}:${TEST_FILE_WITHOUT_DIR}"* ]]; then
- SKIP=1
- fi
-
- printf "%7s: " "${FORMAT}"
- if [[ -n "$SKIP" ]]; then
- echo -e "${TEXT_YELLOW}SKIPPED${TEXT_DEFAULT}"
- NUM_SKIP=$((${NUM_SKIP}+1))
- return
- fi
- set +e
- "${TINT}" ${TEST_FILE} --format ${FORMAT} -o /dev/null
- if [ $? -eq 0 ]; then
- echo -e "${TEXT_GREEN}PASS${TEXT_DEFAULT}"
- NUM_PASS=$((${NUM_PASS}+1))
- else
- echo -e "${TEXT_RED}FAIL${TEXT_DEFAULT}"
- NUM_FAIL=$((${NUM_FAIL}+1))
- fi
- set -e
-}
-
-# check_formats(TEST_FILE)
-function check_formats() {
- local TEST_FILE=$1
- echo
- echo "Testing ${TEST_FILE}..."
- [ ${CHECK_WGSL} -eq 0 ] || check "${TEST_FILE}" wgsl
- [ ${CHECK_SPV} -eq 0 ] || check "${TEST_FILE}" spirv
- [ ${CHECK_MSL} -eq 0 ] || check "${TEST_FILE}" msl
- [ ${CHECK_HLSL} -eq 0 ] || check "${TEST_FILE}" hlsl
-}
-
-for F in "${TARGETDIR}"/*.spvasm "${TARGETDIR}"/*.wgsl
-do
- check_formats "$F"
-done
-
-if [ ${NUM_FAIL} -ne 0 ]; then
- echo
- echo -e "${TEXT_RED}${NUM_FAIL} tests failed. ${TEXT_DEFAULT}${NUM_SKIP} skipped. ${NUM_PASS} passed.${TEXT_DEFAULT}"
- echo
- exit 1
-else
- echo
- echo -e "${NUM_SKIP} tests skipped. ${TEXT_GREEN}${NUM_PASS} passed.${TEXT_DEFAULT}"
- echo
-fi
+"${SCRIPT_DIR}/../tools/test-runner" ${@:2} "${TINT}" "${SCRIPT_DIR}"
diff --git a/tools/src/cmd/test-runner/main.go b/tools/src/cmd/test-runner/main.go
new file mode 100644
index 0000000..60b11d8
--- /dev/null
+++ b/tools/src/cmd/test-runner/main.go
@@ -0,0 +1,401 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// test-runner runs tint against a number of test shaders checking for expected behavior
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "unicode/utf8"
+
+ "dawn.googlesource.com/tint/tools/src/fileutils"
+ "dawn.googlesource.com/tint/tools/src/glob"
+ "github.com/fatih/color"
+ "github.com/sergi/go-diff/diffmatchpatch"
+)
+
+type outputFormat string
+
+const (
+ wgsl = outputFormat("wgsl")
+ spvasm = outputFormat("spvasm")
+ msl = outputFormat("msl")
+ hlsl = outputFormat("hlsl")
+)
+
+func main() {
+ if err := run(); err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+}
+
+func showUsage() {
+ fmt.Println(`
+test-runner runs tint against a number of test shaders checking for expected behavior
+
+usage:
+ test-runner [flags...] <executable> [<directory>]
+
+ <executable> the path to the tint executable
+ <directory> the root directory of the test files
+
+optional flags:`)
+ flag.PrintDefaults()
+ fmt.Println(``)
+ os.Exit(1)
+}
+
+func run() error {
+ var formatList, filter string
+ generateExpected := false
+ flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl")
+ flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
+ flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
+ flag.Usage = showUsage
+ flag.Parse()
+
+ args := flag.Args()
+ if len(args) == 0 {
+ showUsage()
+ }
+
+ // executable path is the first argument
+ exe, args := args[0], args[1:]
+
+ // (optional) target directory is the second argument
+ dir := "."
+ if len(args) > 0 {
+ dir, args = args[0], args[1:]
+ }
+
+ // Check the executable can be found and actually is executable
+ if !fileutils.IsExe(exe) {
+ return fmt.Errorf("'%s' not found or is not executable", exe)
+ }
+
+ // Split the --filter flag up by ',', trimming any whitespace at the start and end
+ globIncludes := strings.Split(filter, ",")
+ for i, s := range globIncludes {
+ globIncludes[i] = `"` + strings.TrimSpace(s) + `"`
+ }
+
+ // Glob the files to test
+ files, err := glob.Scan(dir, glob.MustParseConfig(`{
+ "paths": [
+ {
+ "include": [ `+strings.Join(globIncludes, ",")+` ]
+ },
+ {
+ "exclude": [
+ "**.expected.wgsl",
+ "**.expected.spvasm",
+ "**.expected.msl",
+ "**.expected.hlsl"
+ ]
+ }
+ ]
+ }`))
+ if err != nil {
+ return fmt.Errorf("Failed to glob files: %w", err)
+ }
+
+ // Ensure the files are sorted (globbing should do this, but why not)
+ sort.Strings(files)
+
+ // Parse --format into a list of outputFormat
+ formats := []outputFormat{}
+ if formatList == "all" {
+ formats = []outputFormat{wgsl, spvasm, msl, hlsl}
+ } else {
+ for _, f := range strings.Split(formatList, ",") {
+ switch strings.TrimSpace(f) {
+ case "wgsl":
+ formats = append(formats, wgsl)
+ case "spvasm":
+ formats = append(formats, spvasm)
+ case "msl":
+ formats = append(formats, msl)
+ case "hlsl":
+ formats = append(formats, hlsl)
+ default:
+ return fmt.Errorf("unknown format '%s'", f)
+ }
+ }
+ }
+
+ // Structures to hold the results of the tests
+ type statusCode string
+ const (
+ fail statusCode = "FAIL"
+ pass statusCode = "PASS"
+ skip statusCode = "SKIP"
+ )
+ type status struct {
+ code statusCode
+ err error
+ }
+ type result map[outputFormat]status
+ results := make([]result, len(files))
+
+ // In parallel...
+ wg := sync.WaitGroup{}
+ wg.Add(len(files))
+ for i, file := range files { // For each test file...
+ i, file := i, filepath.Join(dir, file)
+ go func() {
+ defer wg.Done()
+ r := result{}
+ for _, format := range formats { // For each output format...
+
+ // Is there an expected output?
+ expected := loadExpectedFile(file, format)
+ if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
+ r[format] = status{code: skip}
+ continue
+ }
+
+ // Invoke the compiler...
+ var err error
+ if ok, out := invoke(exe, file, "--format", string(format), "--dawn-validation"); ok {
+ if generateExpected {
+ // If --generate-expected was passed, write out the output
+ err = saveExpectedFile(file, format, out)
+ } else if expected != "" && expected != out {
+ // Expected output did not match
+ dmp := diffmatchpatch.New()
+ diff := dmp.DiffPrettyText(dmp.DiffMain(expected, out, true))
+ err = fmt.Errorf(`Output was not as expected
+
+--------------------------------------------------------------------------------
+-- Expected: --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Got: --
+--------------------------------------------------------------------------------
+%s
+
+--------------------------------------------------------------------------------
+-- Diff: --
+--------------------------------------------------------------------------------
+%s`,
+ expected, out, diff)
+ }
+ } else {
+ // Compiler returned a non-zero exit code
+ err = fmt.Errorf("%s", out)
+ }
+
+ if err != nil {
+ r[format] = status{code: fail, err: err}
+ } else {
+ r[format] = status{code: pass}
+ }
+ }
+ results[i] = r
+ }()
+ }
+ wg.Wait()
+
+ // At this point all the tests have been run
+ // Time to print the outputs
+
+ // Start by printing the error message for any file x format combinations
+ // that failed...
+ for i, file := range files {
+ results := results[i]
+ for _, format := range formats {
+ if err := results[format].err; err != nil {
+ color.Set(color.FgBlue)
+ fmt.Printf("%s ", file)
+ color.Set(color.FgCyan)
+ fmt.Printf("%s ", format)
+ color.Set(color.FgRed)
+ fmt.Println("FAIL")
+ color.Unset()
+ fmt.Println(indent(err.Error(), 4))
+ }
+ }
+ }
+
+ // Now print the table of file x format
+ numTests, numPass, numSkip, numFail := 0, 0, 0, 0
+ filenameFmt := columnFormat(maxStringLen(files), false)
+
+ fmt.Println()
+ fmt.Printf(filenameFmt, "")
+ fmt.Printf(" ┃ ")
+ for _, format := range formats {
+ color.Set(color.FgCyan)
+ fmt.Printf(columnFormat(formatWidth(format), false), format)
+ color.Unset()
+ fmt.Printf(" │ ")
+ }
+ fmt.Println()
+ fmt.Printf(strings.Repeat("━", maxStringLen(files)))
+ fmt.Printf("━╋━")
+ for _, format := range formats {
+ fmt.Printf(strings.Repeat("━", formatWidth(format)))
+ fmt.Printf("━│━")
+ }
+ fmt.Println()
+
+ for i, file := range files {
+ results := results[i]
+
+ color.Set(color.FgBlue)
+ fmt.Printf(filenameFmt, file)
+ color.Unset()
+ fmt.Printf(" ┃ ")
+ for _, format := range formats {
+ formatFmt := columnFormat(formatWidth(format), true)
+ result := results[format]
+ numTests++
+ switch result.code {
+ case pass:
+ color.Set(color.FgGreen)
+ fmt.Printf(formatFmt, "PASS")
+ numPass++
+ case fail:
+ color.Set(color.FgRed)
+ fmt.Printf(formatFmt, "FAIL")
+ numFail++
+ case skip:
+ color.Set(color.FgYellow)
+ fmt.Printf(formatFmt, "SKIP")
+ numSkip++
+ default:
+ fmt.Printf(formatFmt, result.code)
+ }
+ color.Unset()
+ fmt.Printf(" │ ")
+ }
+ fmt.Println()
+ }
+ fmt.Println()
+
+ fmt.Printf("%d tests run", numTests)
+ if numPass > 0 {
+ fmt.Printf(", ")
+ color.Set(color.FgGreen)
+ fmt.Printf("%d tests pass", numPass)
+ color.Unset()
+ } else {
+ fmt.Printf(", %d tests pass", numPass)
+ }
+ if numSkip > 0 {
+ fmt.Printf(", ")
+ color.Set(color.FgYellow)
+ fmt.Printf("%d tests skipped", numSkip)
+ color.Unset()
+ } else {
+ fmt.Printf(", %d tests skipped", numSkip)
+ }
+ if numFail > 0 {
+ fmt.Printf(", ")
+ color.Set(color.FgRed)
+ fmt.Printf("%d tests failed", numFail)
+ color.Unset()
+ } else {
+ fmt.Printf(", %d tests failed", numFail)
+ }
+ fmt.Println()
+ fmt.Println()
+
+ return nil
+}
+
+// loadExpectedFile loads the expected output file for the test file at 'path'
+// and the output format 'format'. If the file does not exist, or cannot be
+// read, then an empty string is returned.
+func loadExpectedFile(path string, format outputFormat) string {
+ content, err := ioutil.ReadFile(expectedFilePath(path, format))
+ if err != nil {
+ return ""
+ }
+ return string(content)
+}
+
+// saveExpectedFile writes the expected output file for the test file at 'path'
+// and the output format 'format', with the content 'content'.
+func saveExpectedFile(path string, format outputFormat, content string) error {
+ return ioutil.WriteFile(expectedFilePath(path, format), []byte(content), 0666)
+}
+
+// expectedFilePath returns the expected output file path for the test file at
+// 'path' and the output format 'format'.
+func expectedFilePath(path string, format outputFormat) string {
+ return path + ".expected." + string(format)
+}
+
+// indent returns the string 's' indented with 'n' whitespace characters
+func indent(s string, n int) string {
+ tab := strings.Repeat(" ", n)
+ return tab + strings.ReplaceAll(s, "\n", "\n"+tab)
+}
+
+// columnFormat returns the printf format string to sprint a string with the
+// width of 'i' runes.
+func columnFormat(i int, alignLeft bool) string {
+ if alignLeft {
+ return "%-" + strconv.Itoa(i) + "s"
+ }
+ return "%" + strconv.Itoa(i) + "s"
+}
+
+// maxStringLen returns the maximum number of runes found in all the strings in
+// 'l'
+func maxStringLen(l []string) int {
+ max := 0
+ for _, s := range l {
+ if c := utf8.RuneCountInString(s); c > max {
+ max = c
+ }
+ }
+ return max
+}
+
+// formatWidth returns the width in runes for the outputFormat column 'b'
+func formatWidth(b outputFormat) int {
+ c := utf8.RuneCountInString(string(b))
+ if c > 4 {
+ return c
+ }
+ return 4
+}
+
+// invoke runs the executable 'exe' with the provided arguments.
+func invoke(exe string, args ...string) (ok bool, output string) {
+ cmd := exec.Command(exe, args...)
+ out, err := cmd.CombinedOutput()
+ str := string(out)
+ if err != nil {
+ if str != "" {
+ return false, str
+ }
+ return false, err.Error()
+ }
+ return true, str
+}
diff --git a/tools/src/fileutils/fileutils.go b/tools/src/fileutils/fileutils.go
new file mode 100644
index 0000000..7410086
--- /dev/null
+++ b/tools/src/fileutils/fileutils.go
@@ -0,0 +1,29 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fileutils contains utility functions for files
+package fileutils
+
+import (
+ "os"
+)
+
+// IsExe returns true if the file at path is an executable
+func IsExe(path string) bool {
+ s, err := os.Stat(path)
+ if err != nil {
+ return false
+ }
+ return s.Mode()&0100 != 0
+}
diff --git a/tools/src/glob/glob.go b/tools/src/glob/glob.go
index 0f109d4..e4c8327 100644
--- a/tools/src/glob/glob.go
+++ b/tools/src/glob/glob.go
@@ -22,6 +22,7 @@
"io/ioutil"
"os"
"path/filepath"
+ "strings"
"dawn.googlesource.com/tint/tools/src/match"
)
@@ -92,7 +93,12 @@
if err != nil {
return Config{}, err
}
- d := json.NewDecoder(bytes.NewReader(cfgBody))
+ return ParseConfig(string(cfgBody))
+}
+
+// ParseConfig parses the config from a JSON string.
+func ParseConfig(config string) (Config, error) {
+ d := json.NewDecoder(strings.NewReader(config))
cfg := Config{}
if err := d.Decode(&cfg); err != nil {
return Config{}, err
@@ -100,6 +106,17 @@
return cfg, nil
}
+// MustParseConfig parses the config from a JSON string, panicing if the config
+// does not parse
+func MustParseConfig(config string) Config {
+ d := json.NewDecoder(strings.NewReader(config))
+ cfg := Config{}
+ if err := d.Decode(&cfg); err != nil {
+ panic(fmt.Errorf("Failed to parse config: %w\nConfig:\n%v", err, config))
+ }
+ return cfg
+}
+
// rule is a search path predicate.
// root is the project relative path.
// cond is the value to return if the rule doesn't either include or exclude.
diff --git a/tools/src/go.mod b/tools/src/go.mod
index a23bdbc..cf82c77 100644
--- a/tools/src/go.mod
+++ b/tools/src/go.mod
@@ -2,4 +2,7 @@
go 1.16
-require github.com/sergi/go-diff v1.2.0
+require (
+ github.com/fatih/color v1.10.0
+ github.com/sergi/go-diff v1.2.0
+)
diff --git a/tools/src/go.sum b/tools/src/go.sum
index 930b8b0..5d71093 100644
--- a/tools/src/go.sum
+++ b/tools/src/go.sum
@@ -1,9 +1,15 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
@@ -11,6 +17,9 @@
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/tools/test-runner b/tools/test-runner
new file mode 100755
index 0000000..8e6ef15
--- /dev/null
+++ b/tools/test-runner
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2021 The Tint Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e # Fail on any error.
+
+if [ ! -x "$(which go)" ] ; then
+ echo "error: go needs to be on \$PATH to use $0"
+ exit 1
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
+BINARY="${SCRIPT_DIR}/bin/test-runner"
+
+# Rebuild the binary.
+# Note, go caches build artifacts, so this is quick for repeat calls
+pushd "${SCRIPT_DIR}/src/cmd/test-runner" > /dev/null
+ go build -o "${BINARY}" main.go
+popd > /dev/null
+
+"${BINARY}" "$@"
