tools/src/cts/result: Add more helpers Add result.List.StatusTree() for building a query.Tree[Status]. Add helpers for serializing results. Add helpers for merging and de-duplicating results. Change the interface of result.List.ReplaceDuplicates() so that the merging function takes a status set instead of a list of results. Bug: dawn:1342 Change-Id: I77580ec5fd4c8f12109fb6e9e83afea8b740260c Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/87240 Reviewed-by: Corentin Wallez <cwallez@chromium.org> Kokoro: Kokoro <noreply+kokoro@google.com>
diff --git a/tools/src/cts/result/result.go b/tools/src/cts/result/result.go index 2d47094..99d14d2 100644 --- a/tools/src/cts/result/result.go +++ b/tools/src/cts/result/result.go
@@ -16,7 +16,11 @@ package result import ( + "bufio" "fmt" + "io" + "os" + "path/filepath" "sort" "strings" @@ -126,29 +130,41 @@ // ReplaceDuplicates returns a new list with duplicate test results replaced. // When a duplicate is found, the function f is called with the duplicate // results. The returned status will be used as the replaced result. -func (l List) ReplaceDuplicates(f func(List) Status) List { +func (l List) ReplaceDuplicates(f func(Statuses) Status) List { type key struct { query query.Query tags string } - m := map[key]List{} + // Collect all duplicates + duplicates := map[key]Statuses{} for _, r := range l { k := key{r.Query, TagsToString(r.Tags)} - m[k] = append(m[k], r) - } - for key, results := range m { - if len(results) > 1 { - result := results[0] - result.Status = f(results) - m[key] = List{result} + if s, ok := duplicates[k]; ok { + s.Add(r.Status) + } else { + duplicates[k] = NewStatuses(r.Status) } } - out := make(List, 0, len(m)) + // Resolve duplicates + merged := map[key]Status{} + for key, statuses := range duplicates { + if len(statuses) > 1 { + merged[key] = f(statuses) + } else { + merged[key] = statuses.One() // Only one status + } + } + // Rebuild list + out := make(List, 0, len(duplicates)) for _, r := range l { k := key{r.Query, TagsToString(r.Tags)} - if unique, ok := m[k]; ok { - out = append(out, unique[0]) - delete(m, k) + if status, ok := merged[k]; ok { + out = append(out, Result{ + Query: r.Query, + Tags: r.Tags, + Status: status, + }) + delete(merged, k) // Remove from map to prevent duplicates } } return out @@ -201,11 +217,125 @@ }) } +// Statuses is a set of Status +type Statuses = container.Set[Status] + +// NewStatuses returns a new status set with the provided statuses +func NewStatuses(s ...Status) Statuses { return container.NewSet(s...) } + // Statuses returns a set of all the statuses in the list -func (l List) Statuses() container.Set[Status] { - set := container.NewSet[Status]() +func (l List) Statuses() Statuses { + set := NewStatuses() for _, r := range l { set.Add(r.Status) } return set } + +// StatusTree is a query tree of statuses +type StatusTree = query.Tree[Status] + +// StatusTree returns a query.Tree from the List, with the Status as the tree +// node data. +func (l List) StatusTree() (StatusTree, error) { + tree := StatusTree{} + for _, r := range l { + if err := tree.Add(r.Query, r.Status); err != nil { + return StatusTree{}, err + } + } + return tree, nil +} + +// Load loads the result list from the file with the given path +func Load(path string) (List, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + results, err := Read(file) + if err != nil { + return nil, fmt.Errorf("while reading '%v': %w", path, err) + } + return results, nil +} + +// Save saves the result list to the file with the given path +func Save(path string, results List) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + return Write(file, results) +} + +// Read reads a result list from the given reader +func Read(r io.Reader) (List, error) { + scanner := bufio.NewScanner(r) + l := List{} + for scanner.Scan() { + r, err := Parse(scanner.Text()) + if err != nil { + return nil, err + } + l = append(l, r) + } + return l, nil +} + +// Write writes a result list to the given writer +func Write(w io.Writer, l List) error { + for _, r := range l { + if _, err := fmt.Fprintln(w, r); err != nil { + return err + } + } + return nil +} + +// Merge merges and sorts two results lists. +// Duplicates are removed using the Deduplicate() function. +func Merge(a, b List) List { + merged := make(List, 0, len(a)+len(b)) + merged = append(merged, a...) + merged = append(merged, b...) + out := merged.ReplaceDuplicates(Deduplicate) + out.Sort() + return out +} + +// Deduplicate is the standard algorithm used to de-duplicating mixed results. +// This function is expected to be handed to List.ReplaceDuplicates(). +func Deduplicate(s Statuses) Status { + // If all results have the same status, then use that + if len(s) == 1 { + return s.One() + } + + // Mixed statuses. Replace with something appropriate. + switch { + // Crash + * = Crash + case s.Contains(Crash): + return Crash + // Abort + * = Abort + case s.Contains(Abort): + return Abort + // Unknown + * = Unknown + case s.Contains(Unknown): + return Unknown + // RetryOnFailure + ~(Crash | Abort | Unknown) = RetryOnFailure + case s.Contains(RetryOnFailure): + return RetryOnFailure + // Pass + ~(Crash | Abort | Unknown | RetryOnFailure | Slow) = RetryOnFailure + case s.Contains(Pass): + return RetryOnFailure + } + return Unknown +}
diff --git a/tools/src/cts/result/result_test.go b/tools/src/cts/result/result_test.go index fd76ccc..bc87bc6 100644 --- a/tools/src/cts/result/result_test.go +++ b/tools/src/cts/result/result_test.go
@@ -15,12 +15,14 @@ package result_test import ( + "bytes" "fmt" "testing" "dawn.googlesource.com/dawn/tools/src/container" "dawn.googlesource.com/dawn/tools/src/cts/query" "dawn.googlesource.com/dawn/tools/src/cts/result" + "dawn.googlesource.com/dawn/tools/src/utils" "github.com/google/go-cmp/cmp" ) @@ -304,16 +306,18 @@ func TestReplaceDuplicates(t *testing.T) { type Test struct { - results result.List - fn func(result.List) result.Status - expect result.List + location string + results result.List + fn func(result.Statuses) result.Status + expect result.List } for _, test := range []Test{ { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), results: result.List{ result.Result{Query: Q(`a`), Status: result.Pass}, }, - fn: func(l result.List) result.Status { + fn: func(result.Statuses) result.Status { return result.Abort }, expect: result.List{ @@ -321,23 +325,25 @@ }, }, { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), results: result.List{ result.Result{Query: Q(`a`), Status: result.Pass}, result.Result{Query: Q(`a`), Status: result.Pass}, }, - fn: func(l result.List) result.Status { + fn: func(result.Statuses) result.Status { return result.Abort }, expect: result.List{ - result.Result{Query: Q(`a`), Status: result.Abort}, + result.Result{Query: Q(`a`), Status: result.Pass}, }, }, { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), results: result.List{ result.Result{Query: Q(`a`), Status: result.Pass}, result.Result{Query: Q(`b`), Status: result.Pass}, }, - fn: func(l result.List) result.Status { + fn: func(result.Statuses) result.Status { return result.Abort }, expect: result.List{ @@ -346,16 +352,14 @@ }, }, { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), results: result.List{ result.Result{Query: Q(`a`), Status: result.Pass}, result.Result{Query: Q(`b`), Status: result.Pass}, result.Result{Query: Q(`a`), Status: result.Skip}, }, - fn: func(got result.List) result.Status { - expect := result.List{ - result.Result{Query: Q(`a`), Status: result.Pass}, - result.Result{Query: Q(`a`), Status: result.Skip}, - } + fn: func(got result.Statuses) result.Status { + expect := result.NewStatuses(result.Pass, result.Skip) if diff := cmp.Diff(got, expect); diff != "" { t.Errorf("function's parameter was not as expected:\n%v", diff) } @@ -369,7 +373,7 @@ } { got := test.results.ReplaceDuplicates(test.fn) if diff := cmp.Diff(got, test.expect); diff != "" { - t.Errorf("Results:\n%v\nReplaceDuplicates() was not as expected:\n%v", test.results, diff) + t.Errorf("\n%v ReplaceDuplicates() was not as expected:\n%v", test.location, diff) } } } @@ -847,3 +851,322 @@ } } } + +func TestStatusTree(t *testing.T) { + type Node = query.TreeNode[result.Status] + type Children = query.TreeNodeChildren[result.Status] + type ChildKey = query.TreeNodeChildKey + + pass := result.Pass + + type Test struct { + results result.List + expectErr error + expect result.StatusTree + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + results: result.List{}, + expect: result.StatusTree{}, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + {Query: Q(`suite:a:*`), Status: result.Pass}, + }, + expect: result.StatusTree{ + TreeNode: Node{ + Children: Children{ + ChildKey{Name: `suite`, Target: query.Suite}: &Node{ + Query: Q(`suite`), + Children: Children{ + ChildKey{Name: `a`, Target: query.Files}: &Node{ + Query: Q(`suite:a`), + Children: Children{ + ChildKey{Name: `*`, Target: query.Tests}: &Node{ + Query: Q(`suite:a:*`), + Data: &pass, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { ////////////////////////////////////////////////////////////////////// + results: result.List{ + {Query: Q(`suite:a:*`), Status: result.Pass}, + {Query: Q(`suite:a:*`), Status: result.Failure}, + }, + expectErr: query.ErrDuplicateData{Query: Q(`suite:a:*`)}, + }, + } { + got, err := test.results.StatusTree() + if diff := cmp.Diff(err, test.expectErr); diff != "" { + t.Errorf("Results:\n%v\nStatusTree() error was not as expected:\n%v", test.results, diff) + continue + } + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("Results:\n%v\nStatusTree() was not as expected:\n%v", test.results, diff) + } + } +} + +func TestReadWrite(t *testing.T) { + in := result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:b,*`), Tags: T(`y`), Status: result.Failure}, + {Query: Q(`suite:a:b:*`), Tags: T(`x`, `y`), Status: result.Skip}, + {Query: Q(`suite:a:c,*`), Tags: T(`y`, `x`), Status: result.Failure}, + {Query: Q(`suite:a,b:c,*`), Tags: T(`y`, `x`), Status: result.Crash}, + {Query: Q(`suite:a,b:c:*`), Status: result.Slow}, + } + buf := &bytes.Buffer{} + if err := result.Write(buf, in); err != nil { + t.Fatalf("Write(): %v", err) + } + got, err := result.Read(buf) + if err != nil { + t.Fatalf("Read(): %v", err) + } + if diff := cmp.Diff(got, in); diff != "" { + t.Errorf("Read() was not as expected:\n%v", diff) + } +} + +func TestMerge(t *testing.T) { + type Test struct { + location string + a, b result.List + expect result.List + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{}, + b: result.List{}, + expect: result.List{}, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + b: result.List{}, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{}, + b: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + b: result.List{ + {Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass}, + }, + b: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + b: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`y`), Status: result.Pass}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:a:*`), Tags: T(`y`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:a:*`), Status: result.Pass}, + }, + b: result.List{ + {Query: Q(`suite:a:*`), Status: result.Pass}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + b: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Crash}, + }, + b: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Crash}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Crash}, + }, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + a: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:c:*`), Tags: T(`x`), Status: result.Failure}, + {Query: Q(`suite:d:*`), Tags: T(`x`), Status: result.Failure}, + {Query: Q(`suite:e:*`), Tags: T(`x`), Status: result.Crash}, + }, + b: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.Failure}, + {Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:c:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:d:*`), Tags: T(`y`), Status: result.Pass}, + {Query: Q(`suite:e:*`), Tags: T(`x`), Status: result.Pass}, + }, + expect: result.List{ + {Query: Q(`suite:a:*`), Tags: T(`x`), Status: result.RetryOnFailure}, + {Query: Q(`suite:b:*`), Tags: T(`x`), Status: result.Pass}, + {Query: Q(`suite:c:*`), Tags: T(`x`), Status: result.RetryOnFailure}, + {Query: Q(`suite:d:*`), Tags: T(`x`), Status: result.Failure}, + {Query: Q(`suite:d:*`), Tags: T(`y`), Status: result.Pass}, + {Query: Q(`suite:e:*`), Tags: T(`x`), Status: result.Crash}, + }, + }, + } { + got := result.Merge(test.a, test.b) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("%v\nStatusTree() was not as expected:\n%v", test.location, diff) + } + } +} + +func TestDeduplicate(t *testing.T) { + type Test struct { + location string + statuses result.Statuses + expect result.Status + } + for _, test := range []Test{ + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass), + expect: result.Pass, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Abort), + expect: result.Abort, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Failure), + expect: result.Failure, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Skip), + expect: result.Skip, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Crash), + expect: result.Crash, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Slow), + expect: result.Slow, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Unknown), + expect: result.Unknown, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.RetryOnFailure), + expect: result.RetryOnFailure, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass, result.Failure), + expect: result.RetryOnFailure, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass, result.Abort), + expect: result.Abort, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass, result.Skip), + expect: result.RetryOnFailure, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass, result.Crash), + expect: result.Crash, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass, result.Slow), + expect: result.RetryOnFailure, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass, result.Unknown), + expect: result.Unknown, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Pass, result.RetryOnFailure), + expect: result.RetryOnFailure, + }, + { ////////////////////////////////////////////////////////////////////// + location: utils.ThisLine(), + statuses: result.NewStatuses(result.Status("??"), result.Status("?!")), + expect: result.Unknown, + }, + } { + got := result.Deduplicate(test.statuses) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("\n%v Deduplicate() was not as expected:\n%v", test.location, diff) + } + } +}
diff --git a/tools/src/cts/result/status.go b/tools/src/cts/result/status.go index 5e5fd15..50daae4 100644 --- a/tools/src/cts/result/status.go +++ b/tools/src/cts/result/status.go
@@ -14,6 +14,8 @@ package result +import "dawn.googlesource.com/dawn/tools/src/container" + // Status is an enumerator of test results type Status string @@ -28,3 +30,12 @@ Slow = Status("Slow") Unknown = Status("Unknown") ) + +// CommonStatus is a function that can be used by StatusTree.Reduce() to reduce +// tree nodes with the same status +func CommonStatus(statuses []Status) *Status { + if set := container.NewSet(statuses...); len(set) == 1 { + return &statuses[0] + } + return nil +}
diff --git a/tools/src/cts/result/status_test.go b/tools/src/cts/result/status_test.go new file mode 100644 index 0000000..e683831 --- /dev/null +++ b/tools/src/cts/result/status_test.go
@@ -0,0 +1,54 @@ +// Copyright 2022 The Dawn Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package result_test + +import ( + "testing" + + "dawn.googlesource.com/dawn/tools/src/cts/result" + "github.com/google/go-cmp/cmp" +) + +func TestCommonStatus(t *testing.T) { + pass := result.Pass + + type Test struct { + in []result.Status + expect *result.Status + } + for _, test := range []Test{ + { + in: nil, + expect: nil, + }, { + in: []result.Status{}, + expect: nil, + }, { + in: []result.Status{result.Pass}, + expect: &pass, + }, { + in: []result.Status{result.Pass, result.Pass, result.Pass}, + expect: &pass, + }, { + in: []result.Status{result.Pass, result.Failure, result.Pass}, + expect: nil, + }, + } { + got := result.CommonStatus(test.in) + if diff := cmp.Diff(got, test.expect); diff != "" { + t.Errorf("%v.CommonStatus('%v') was not as expected:\n%v", test.in, test.expect, diff) + } + } +}