| // Copyright 2022 The Dawn & Tint Authors |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are met: |
| // |
| // 1. Redistributions of source code must retain the above copyright notice, this |
| // list of conditions and the following disclaimer. |
| // |
| // 2. Redistributions in binary form must reproduce the above copyright notice, |
| // this list of conditions and the following disclaimer in the documentation |
| // and/or other materials provided with the distribution. |
| // |
| // 3. Neither the name of the copyright holder nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
| // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| // Package result holds types that describe CTS test results. |
| package result |
| |
| import ( |
| "bufio" |
| "fmt" |
| "io" |
| "os" |
| "path/filepath" |
| "sort" |
| "strconv" |
| "strings" |
| "time" |
| |
| "dawn.googlesource.com/dawn/tools/src/container" |
| "dawn.googlesource.com/dawn/tools/src/cts/query" |
| ) |
| |
| // Result holds the result of a CTS test |
| type Result struct { |
| Query query.Query |
| Tags Tags |
| Status Status |
| Duration time.Duration |
| // If true, this result may be exonerated if there are other |
| // results with the same query and tags that have MayExonerate: false |
| MayExonerate bool |
| } |
| |
| // Format writes the Result to the fmt.State |
| // The Result is printed as a single line, in the form: |
| // |
| // <query> <tags> <status> |
| // |
| // This matches the order in which results are sorted. |
| func (r Result) Format(f fmt.State, verb rune) { |
| if len(r.Tags) > 0 { |
| fmt.Fprintf(f, "%v %v %v %v %v", r.Query, TagsToString(r.Tags), r.Status, r.Duration, r.MayExonerate) |
| } else { |
| fmt.Fprintf(f, "%v %v %v %v", r.Query, r.Status, r.Duration, r.MayExonerate) |
| } |
| } |
| |
| // String returns the result as a string |
| func (r Result) String() string { |
| sb := strings.Builder{} |
| fmt.Fprint(&sb, r) |
| return sb.String() |
| } |
| |
| // Compare compares the relative order of r and o, returning: |
| // |
| // -1 if r should come before o |
| // 1 if r should come after o |
| // 0 if r and o are identical |
| // |
| // Note: Result.Duration is not considered in comparison. |
| func (r Result) Compare(o Result) int { |
| a, b := r, o |
| switch a.Query.Compare(b.Query) { |
| case -1: |
| return -1 |
| case 1: |
| return 1 |
| } |
| ta := strings.Join(a.Tags.List(), TagDelimiter) |
| tb := strings.Join(b.Tags.List(), TagDelimiter) |
| switch { |
| case ta < tb: |
| return -1 |
| case ta > tb: |
| return 1 |
| case a.Status < b.Status: |
| return -1 |
| case a.Status > b.Status: |
| return 1 |
| } |
| return 0 |
| } |
| |
| // Parse parses the result from a string of the form: |
| // |
| // <query> <tags> <status> |
| // |
| // <tags> may be omitted if there were no tags. |
| // |
| // Tests are separated into sections where the section name |
| // appears at the end of the list as just a line with |
| // |
| // <section-name> |
| func Parse(in string) (ExecutionMode, Result, error) { |
| line := in |
| token := func() string { |
| for i, c := range line { |
| if c != ' ' { |
| line = line[i:] |
| break |
| } |
| } |
| for i, c := range line { |
| if c == ' ' { |
| tok := line[:i] |
| line = line[i:] |
| return tok |
| } |
| } |
| tok := line |
| line = "" |
| return tok |
| } |
| |
| a := token() |
| b := token() |
| c := token() |
| d := token() |
| e := token() |
| |
| if a != "" && b == "" && token() == "" { |
| return ExecutionMode(a), Result{}, nil |
| } |
| if a == "" || b == "" || c == "" || d == "" || token() != "" { |
| return "", Result{}, fmt.Errorf("unable to parse result '%v'", in) |
| } |
| |
| query := query.Parse(a) |
| |
| if e == "" { |
| status := Status(b) |
| duration, err := time.ParseDuration(c) |
| if err != nil { |
| return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err) |
| } |
| mayExonerate, err := strconv.ParseBool(d) |
| if err != nil { |
| return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err) |
| } |
| return "", Result{query, nil, status, duration, mayExonerate}, nil |
| } else { |
| tags := StringToTags(b) |
| status := Status(c) |
| duration, err := time.ParseDuration(d) |
| if err != nil { |
| return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err) |
| } |
| mayExonerate, err := strconv.ParseBool(e) |
| if err != nil { |
| return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err) |
| } |
| return "", Result{query, tags, status, duration, mayExonerate}, nil |
| } |
| } |
| |
| // List is a list of results |
| type List []Result |
| |
| // The mode the tests were run in, "core" or" "compat" |
| type ExecutionMode string |
| |
| // Lists of test results by execution mode. |
| type ResultsByExecutionMode map[ExecutionMode]List |
| |
| // Variant is a collection of tags that uniquely identify a test |
| // configuration (e.g the combination of OS, GPU, validation-modes, etc). |
| type Variant = Tags |
| |
| // Variants returns the list of unique tags (variants) across all results. |
| func (l List) Variants() []Variant { |
| tags := container.NewMap[string, Variant]() |
| for _, r := range l { |
| tags.Add(TagsToString(r.Tags), r.Tags) |
| } |
| return tags.Values() |
| } |
| |
| // TransformTags returns the list of results with the tags transformed using f. |
| // TransformTags assumes that f will return the same output for the same input. |
| func (l List) TransformTags(f func(Tags) Tags) List { |
| cache := map[string]Tags{} |
| out := List{} |
| for _, r := range l { |
| key := TagsToString(r.Tags) |
| tags, cached := cache[key] |
| if !cached { |
| tags = f(r.Tags.Clone()) |
| cache[key] = tags |
| } |
| out = append(out, Result{ |
| Query: r.Query, |
| Tags: tags, |
| Status: r.Status, |
| Duration: r.Duration, |
| MayExonerate: r.MayExonerate, |
| }) |
| } |
| return out |
| } |
| |
| // ReplaceDuplicates returns a new list with duplicate test results replaced. |
| // When a duplicate is found, the function f is called with the duplicate |
| // results. The returned status will be used as the replaced result. |
| // Merged results will use the average (mean) duration of the duplicates. |
| func (l List) ReplaceDuplicates(f func(Statuses) Status) List { |
| type key struct { |
| query query.Query |
| tags string |
| } |
| // Collect all duplicates |
| keyToIndices := map[key][]int{} // key to index |
| for i, r := range l { |
| k := key{r.Query, TagsToString(r.Tags)} |
| keyToIndices[k] = append(keyToIndices[k], i) |
| } |
| // Filter out exonerated results |
| for key, indices := range keyToIndices { |
| keptIndices := []int{} |
| for _, i := range indices { |
| // Copy all indices which are not exonerated into keptIndices. |
| if !l[i].MayExonerate { |
| keptIndices = append(keptIndices, i) |
| } |
| } |
| |
| // Change indices to only the kept ones. If keptIndices is empty, |
| // then all results were marked with may_exonerate, and we keep all |
| // of them. |
| if len(keptIndices) > 0 { |
| keyToIndices[key] = keptIndices |
| } |
| } |
| // Resolve duplicates |
| type StatusAndDuration struct { |
| Status Status |
| Duration time.Duration |
| } |
| merged := map[key]StatusAndDuration{} |
| for key, indices := range keyToIndices { |
| statuses := NewStatuses() |
| duration := time.Duration(0) |
| for _, i := range indices { |
| r := l[i] |
| statuses.Add(r.Status) |
| duration += r.Duration |
| } |
| status := func() Status { |
| if len(statuses) > 1 { |
| return f(statuses) |
| } |
| return statuses.One() |
| }() |
| duration = duration / time.Duration(len(indices)) |
| merged[key] = StatusAndDuration{ |
| Status: status, |
| Duration: duration, |
| } |
| } |
| // Rebuild list |
| out := make(List, 0, len(keyToIndices)) |
| for _, r := range l { |
| k := key{r.Query, TagsToString(r.Tags)} |
| if sd, ok := merged[k]; ok { |
| out = append(out, Result{ |
| Query: r.Query, |
| Tags: r.Tags, |
| Status: sd.Status, |
| Duration: sd.Duration, |
| MayExonerate: l[keyToIndices[k][0]].MayExonerate, |
| }) |
| delete(merged, k) // Remove from map to prevent duplicates |
| } |
| } |
| return out |
| } |
| |
| // Sort sorts the list |
| func (l List) Sort() { |
| sort.Slice(l, func(i, j int) bool { return l[i].Compare(l[j]) < 0 }) |
| } |
| |
| // Filter returns the results that match the given predicate |
| func (l List) Filter(f func(Result) bool) List { |
| out := make(List, 0, len(l)) |
| for _, r := range l { |
| if f(r) { |
| out = append(out, r) |
| } |
| } |
| return out |
| } |
| |
| // FilterByStatus returns the results that the given status |
| func (l List) FilterByStatus(status Status) List { |
| return l.Filter(func(r Result) bool { |
| return r.Status == status |
| }) |
| } |
| |
| // FilterByTags returns the results that have all the given tags |
| func (l List) FilterByTags(tags Tags) List { |
| return l.Filter(func(r Result) bool { |
| return r.Tags.ContainsAll(tags) |
| }) |
| } |
| |
| // FilterByVariant returns the results that exactly match the given tags |
| func (l List) FilterByVariant(tags Tags) List { |
| str := TagsToString(tags) |
| return l.Filter(func(r Result) bool { |
| return len(r.Tags) == len(tags) && TagsToString(r.Tags) == str |
| }) |
| } |
| |
| // / FilterByQuery returns the results that match the given query |
| func (l List) FilterByQuery(q query.Query) List { |
| return l.Filter(func(r Result) bool { |
| return q.Contains(r.Query) |
| }) |
| } |
| |
| // Statuses is a set of Status |
| type Statuses = container.Set[Status] |
| |
| // NewStatuses returns a new status set with the provided statuses |
| func NewStatuses(s ...Status) Statuses { return container.NewSet(s...) } |
| |
| // Statuses returns a set of all the statuses in the list |
| func (l List) Statuses() Statuses { |
| set := NewStatuses() |
| for _, r := range l { |
| set.Add(r.Status) |
| } |
| return set |
| } |
| |
| // StatusTree is a query tree of statuses |
| type StatusTree = query.Tree[Status] |
| |
| // StatusTree returns a query.Tree from the List, with the Status as the tree |
| // node data. |
| func (l List) StatusTree() (StatusTree, error) { |
| tree := StatusTree{} |
| for _, r := range l { |
| if err := tree.Add(r.Query, r.Status); err != nil { |
| return StatusTree{}, err |
| } |
| } |
| return tree, nil |
| } |
| |
| // Load loads the result list from the file with the given path |
| func Load(path string) (ResultsByExecutionMode, error) { |
| file, err := os.Open(path) |
| if err != nil { |
| return nil, err |
| } |
| defer file.Close() |
| |
| results, err := Read(file) |
| if err != nil { |
| return nil, fmt.Errorf("while reading '%v': %w", path, err) |
| } |
| return results, nil |
| } |
| |
| // Save saves the result list to the file with the given path |
| func Save(path string, results ResultsByExecutionMode) error { |
| dir := filepath.Dir(path) |
| if err := os.MkdirAll(dir, 0777); err != nil { |
| return err |
| } |
| file, err := os.Create(path) |
| if err != nil { |
| return err |
| } |
| defer file.Close() |
| return Write(file, results) |
| } |
| |
| // Read reads a result list from the given reader |
| func Read(r io.Reader) (ResultsByExecutionMode, error) { |
| scanner := bufio.NewScanner(r) |
| results := ResultsByExecutionMode{} |
| l := List{} |
| for scanner.Scan() { |
| section, r, err := Parse(scanner.Text()) |
| if err != nil { |
| return nil, err |
| } |
| if section != "" { |
| results[section] = l |
| l = List{} |
| } else { |
| l = append(l, r) |
| } |
| } |
| return results, nil |
| } |
| |
| // Write writes a result list to the given writer |
| func Write(w io.Writer, r ResultsByExecutionMode) error { |
| for name, l := range r { |
| for _, r := range l { |
| if _, err := fmt.Fprintln(w, r); err != nil { |
| return err |
| } |
| } |
| if _, err := fmt.Fprintln(w, name); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| // Merge merges and sorts multiple results lists. |
| // Duplicates are removed using the Deduplicate() function. |
| func Merge(lists ...List) List { |
| n := 0 |
| for _, l := range lists { |
| n += len(l) |
| } |
| merged := make(List, 0, n) |
| for _, l := range lists { |
| merged = append(merged, l...) |
| } |
| out := merged.ReplaceDuplicates(Deduplicate) |
| out.Sort() |
| return out |
| } |
| |
| // Deduplicate is the standard algorithm used to de-duplicating mixed results. |
| // This function is expected to be handed to List.ReplaceDuplicates(). |
| func Deduplicate(s Statuses) Status { |
| // If all results have the same status, then use that |
| if len(s) == 1 { |
| return s.One() |
| } |
| |
| // Mixed statuses. Replace with something appropriate. |
| switch { |
| // Crash + * = Crash |
| case s.Contains(Crash): |
| return Crash |
| // Abort + * = Abort |
| case s.Contains(Abort): |
| return Abort |
| // Unknown + * = Unknown |
| case s.Contains(Unknown): |
| return Unknown |
| // RetryOnFailure + ~(Crash | Abort | Unknown) = RetryOnFailure |
| case s.Contains(RetryOnFailure): |
| return RetryOnFailure |
| // Pass + ~(Crash | Abort | Unknown | RetryOnFailure | Slow) = RetryOnFailure |
| case s.Contains(Pass): |
| return RetryOnFailure |
| } |
| return Unknown |
| } |