[tint] Test expectations respect TIMEOUT
Some tint e2e tests timeout on the bots but pass locally. Here we
introduce a new SKIP param called TIMEOUT. Tests that skip and have been
labeled as TIMEOUT will not get expectations updated if the timeout or
pass when running locally. They will only get updated on failure.
Tests that now timeout locally that have not been label as such will not
get expecations updated as skip with TIMEOUT.
Bug:372645434
Change-Id: I6da844976347f9418c2e0e1b89ed8b748f4cd439
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/211277
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Commit-Queue: Peter McNeeley <petermcneeley@google.com>
diff --git a/test/tint/benchmark/skinned-shadowed-pbr-fragment.wgsl.expected.glsl b/test/tint/benchmark/skinned-shadowed-pbr-fragment.wgsl.expected.glsl
index 708a412..42538d2 100644
--- a/test/tint/benchmark/skinned-shadowed-pbr-fragment.wgsl.expected.glsl
+++ b/test/tint/benchmark/skinned-shadowed-pbr-fragment.wgsl.expected.glsl
@@ -1,3 +1,3 @@
-SKIP: FAILED
+SKIP: TIMEOUT
Times out on bots
diff --git a/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.spvasm.expected.ir.fxc.hlsl b/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.spvasm.expected.ir.fxc.hlsl
index 7371b80..6491cc8 100644
--- a/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.spvasm.expected.ir.fxc.hlsl
+++ b/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.spvasm.expected.ir.fxc.hlsl
@@ -1,4 +1,4 @@
-SKIP: FAILED
+SKIP: TIMEOUT
struct main_out {
float4 x_GLF_color_1;
diff --git a/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.wgsl.expected.ir.fxc.hlsl b/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.wgsl.expected.ir.fxc.hlsl
index 26cee9e..6fc67c6 100644
--- a/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.wgsl.expected.ir.fxc.hlsl
+++ b/test/tint/vk-gl-cts/graphicsfuzz/cov-nested-loops-global-loop-counter-do-while-accumulate-float/0-opt.wgsl.expected.ir.fxc.hlsl
@@ -1,4 +1,4 @@
-SKIP: FAILED
+SKIP: TIMEOUT
struct main_out {
float4 x_GLF_color_1;
diff --git a/tools/src/cmd/tests/main.go b/tools/src/cmd/tests/main.go
index 09236f8..89544de 100644
--- a/tools/src/cmd/tests/main.go
+++ b/tools/src/cmd/tests/main.go
@@ -697,13 +697,18 @@
expectedFileExists = true
}
- skipped := false
+ isSkipTest := false
if strings.HasPrefix(expected, "SKIP") { // Special SKIP token
- skipped = true
+ isSkipTest = true
}
- invalid_test := false
+ isSkipInvalidTest := false
if strings.HasPrefix(expected, "SKIP: INVALID") { // Special invalid test case token
- invalid_test = true
+ isSkipInvalidTest = true
+ }
+
+ isSkipTimeoutTest := false
+ if strings.HasPrefix(expected, "SKIP: TIMEOUT") { // Special timeout test case token
+ isSkipTimeoutTest = true
}
expected = strings.ReplaceAll(expected, "\r\n", "\n")
@@ -764,8 +769,9 @@
var out string
args = append(args, j.flags...)
+ timedOut := false
start := time.Now()
- ok, out = invoke(cfg.tintPath, args...)
+ ok, out, timedOut = invoke(cfg.tintPath, args...)
timeTaken := time.Since(start)
out = strings.ReplaceAll(out, "\r\n", "\n")
@@ -785,7 +791,8 @@
return os.WriteFile(path, []byte(content), 0666)
}
- if ok && cfg.generateExpected && (validate || !skipped) {
+ // Do not update expected if test is marked as SKIP: TIMEOUT
+ if ok && cfg.generateExpected && !isSkipTimeoutTest && (validate || !isSkipTest) {
// User requested to update PASS expectations, and test passed.
if canEmitPassExpectationFile {
saveExpectedFile(expectedFilePath, out)
@@ -799,11 +806,15 @@
}
var skip_str string = "FAILED"
- if invalid_test {
+ if isSkipInvalidTest {
skip_str = "INVALID"
}
- passed := ok && matched
+ if timedOut {
+ skip_str = "TIMEOUT"
+ }
+
+ passed := ok && (matched || isSkipTimeoutTest)
if !passed {
if j.format == hlslFXC || j.format == hlslFXCIR {
out = reFXCErrorStringHash.ReplaceAllString(out, `<scrubbed_path>${1}`)
@@ -811,16 +822,17 @@
}
switch {
- case ok && matched:
+ case passed:
// Test passed
return status{code: pass, timeTaken: timeTaken, passHashes: hashes}
// --- Below this point the test has failed ---
- case skipped:
- if cfg.generateSkip {
+ case isSkipTest:
+ // Do not update expected if timeout test actually timed out.
+ if cfg.generateSkip && !(isSkipTimeoutTest && timedOut) {
saveExpectedFile(expectedFilePath, "SKIP: "+skip_str+"\n\n"+out)
}
- if invalid_test {
+ if isSkipInvalidTest {
return status{code: invalid, timeTaken: timeTaken}
} else {
return status{code: skip, timeTaken: timeTaken}
@@ -954,7 +966,7 @@
}
// invoke runs the executable 'exe' with the provided arguments.
-func invoke(exe string, args ...string) (ok bool, output string) {
+func invoke(exe string, args ...string) (ok bool, output string, timedOut bool) {
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
@@ -963,15 +975,15 @@
str := string(out)
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
- return false, fmt.Sprintf("test timed out after %v", testTimeout)
+ return false, fmt.Sprintf("test timed out after %v", testTimeout), true
}
if str != "" {
str += fmt.Sprintf("\ntint executable returned error: %v\n", err.Error())
- return false, str
+ return false, str, false
}
- return false, err.Error()
+ return false, err.Error(), false
}
- return true, str
+ return true, str, false
}
var reFlags = regexp.MustCompile(`^\s*(?:\/\/|;)\s*(\[[\w-]+\])?\s*flags:(.*)`)
