test-runner: add 'fxc-and-dxc' arg to run both FXC and DXC for HLSL
When enabled, for each test, FXC is run first, and if it succeeds, DXC
is run. If both succeed, the test passes, otherwise it fails. This
option allows us to get meaningful feedback from running this script
against HLSL files, as well as taking advantage of the feature to delete
skips for tests that now pass (both FXC and DXC in this case).
Change-Id: Iae2ebfda7bd92f1b94893e648e2d1fb1f6979b39
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/84680
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Antonio Maiorano <amaiorano@google.com>
diff --git a/tools/src/cmd/test-runner/main.go b/tools/src/cmd/test-runner/main.go
index b47c51d..f1eeedc 100644
--- a/tools/src/cmd/test-runner/main.go
+++ b/tools/src/cmd/test-runner/main.go
@@ -84,12 +84,13 @@
var formatList, filter, dxcPath, xcrunPath string
var maxFilenameColumnWidth int
numCPU := runtime.NumCPU()
- fxc, verbose, generateExpected, generateSkip := false, false, false, false
+ fxc, fxcAndDxc, verbose, generateExpected, generateSkip := false, false, false, false, false
flag.StringVar(&formatList, "format", "all", "comma separated list of formats to emit. Possible values are: all, wgsl, spvasm, msl, hlsl, glsl")
flag.StringVar(&filter, "filter", "**.wgsl, **.spvasm, **.spv", "comma separated list of glob patterns for test files")
flag.StringVar(&dxcPath, "dxc", "", "path to DXC executable for validating HLSL output")
flag.StringVar(&xcrunPath, "xcrun", "", "path to xcrun executable for validating MSL output")
flag.BoolVar(&fxc, "fxc", false, "validate with FXC instead of DXC")
+ flag.BoolVar(&fxcAndDxc, "fxc-and-dxc", false, "validate with both FXC and DXC")
flag.BoolVar(&verbose, "verbose", false, "print all run tests, including rows that all pass")
flag.BoolVar(&generateExpected, "generate-expected", false, "create or update all expected outputs")
flag.BoolVar(&generateSkip, "generate-skip", false, "create or update all expected outputs that fail with SKIP")
@@ -103,6 +104,10 @@
showUsage()
}
+ if fxcAndDxc {
+ fxc = true
+ }
+
// executable path is the first argument
exe, args := args[0], args[1:]
@@ -210,7 +215,11 @@
color.Set(color.FgGreen)
tool_path := *tool.path
if fxc && tool.lang == "hlsl" {
- tool_path = "Tint will use FXC dll in PATH"
+ if fxcAndDxc {
+ tool_path += " AND Tint will use FXC dll in PATH"
+ } else {
+ tool_path = "Tint will use FXC dll in PATH"
+ }
}
fmt.Printf("ENABLED (" + tool_path + ")")
} else {
@@ -239,7 +248,7 @@
for cpu := 0; cpu < numCPU; cpu++ {
go func() {
for job := range pendingJobs {
- job.run(dir, exe, fxc, dxcPath, xcrunPath, generateExpected, generateSkip)
+ job.run(dir, exe, fxc, fxcAndDxc, dxcPath, xcrunPath, generateExpected, generateSkip)
}
}()
}
@@ -497,7 +506,7 @@
result chan status
}
-func (j job) run(wd, exe string, fxc bool, dxcPath, xcrunPath string, generateExpected, generateSkip bool) {
+func (j job) run(wd, exe string, fxc, fxcAndDxc bool, dxcPath, xcrunPath string, generateExpected, generateSkip bool) {
j.result <- func() status {
// expectedFilePath is the path to the expected output file for the given test
expectedFilePath := j.file + ".expected." + string(j.format)
@@ -539,13 +548,7 @@
args = append(args, "--validate") // spirv-val and glslang are statically linked, always available
validate = true
case hlsl:
- if fxc {
- args = append(args, "--fxc")
- validate = true
- } else if dxcPath != "" {
- args = append(args, "--dxc", dxcPath)
- validate = true
- }
+ // Handled below
case msl:
if xcrunPath != "" {
args = append(args, "--xcrun", xcrunPath)
@@ -553,13 +556,37 @@
}
}
- args = append(args, j.flags...)
-
// Invoke the compiler...
+ ok := false
+ var out string
start := time.Now()
- ok, out := invoke(wd, exe, args...)
- timeTaken := time.Since(start)
+ if j.format == hlsl {
+ // If fxcAndDxc is set, run FXC first as it's more likely to fail, then DXC iff FXC succeeded.
+ if fxc || fxcAndDxc {
+ validate = true
+ args_fxc := append(args, "--fxc")
+ args_fxc = append(args_fxc, j.flags...)
+ ok, out = invoke(wd, exe, args_fxc...)
+ }
+ if dxcPath != "" && (!fxc || (fxcAndDxc && ok)) {
+ validate = true
+ args_dxc := append(args, "--dxc", dxcPath)
+ args_dxc = append(args_dxc, j.flags...)
+ ok, out = invoke(wd, exe, args_dxc...)
+ }
+
+ // If we didn't run either fxc or dxc validation, run as usual
+ if !validate {
+ args = append(args, j.flags...)
+ ok, out = invoke(wd, exe, args...)
+ }
+
+ } else {
+ args = append(args, j.flags...)
+ ok, out = invoke(wd, exe, args...)
+ }
+ timeTaken := time.Since(start)
out = strings.ReplaceAll(out, "\r\n", "\n")
matched := expected == "" || expected == out
