tools/cts: Fix expectations update failure
appendConsumedResultsForSkippedTests() adds fake results for expectations that are 'Skip'ed, so that these sub-trees aren't collapsed as all passing.
However, this code was not handling the fact that there might actually be results for the cases.
This happened because there was an expectation collision in the expectations.txt file, and tests were not being skipped even though there was an expectation with a Skip (a collision expectation was used instead).
Change-Id: I2a1543d231db44dc8aa6683d051f884f4cb96853
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/113520
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Ben Clayton <bclayton@google.com>
Reviewed-by: Austin Eng <enga@chromium.org>
diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go
index 5f53824..155e142 100644
--- a/tools/src/cts/expectations/update.go
+++ b/tools/src/cts/expectations/update.go
@@ -103,15 +103,32 @@
for _, q := range testlist {
tree.Add(q, struct{}{})
}
- for _, c := range c.Chunks {
- for _, ex := range c.Expectations {
- if container.NewSet(ex.Status...).Contains(string(result.Skip)) {
- for _, variant := range variants {
- if !variant.ContainsAll(ex.Tags) {
- continue
- }
- glob, _ := tree.Glob(query.Parse(ex.Query))
- for _, qd := range glob {
+ // For each variant...
+ for _, variant := range variants {
+ resultsForVariant := container.NewSet[string]()
+ for _, result := range results.FilterByVariant(variant) {
+ resultsForVariant.Add(result.Query.String())
+ }
+
+ // For each expectation...
+ for _, c := range c.Chunks {
+ for _, ex := range c.Expectations {
+ // Does this expectation apply for variant?
+ if !variant.ContainsAll(ex.Tags) {
+ continue // Nope.
+ }
+
+ // Does the expectation contain a Skip status?
+ if !container.NewSet(ex.Status...).Contains(string(result.Skip)) {
+ continue // Nope.
+ }
+
+ // Gather all the tests that apply to the expectation
+ glob, _ := tree.Glob(query.Parse(ex.Query))
+ for _, qd := range glob {
+ // If we don't have a result for the test, then append a
+ // synthetic 'consumed' result.
+ if !resultsForVariant.Contains(qd.Query.String()) {
results = append(results, result.Result{
Query: qd.Query,
Tags: variant,
