blob: 9f24b9b9ece54712707a3d6986ba1c723bd279e9 [file] [log] [blame]
// Copyright 2022 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Package result holds types that describe CTS test results.
package result
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/query"
)
// Result holds the result of a CTS test
type Result struct {
Query query.Query
Tags Tags
Status Status
Duration time.Duration
// If true, this result may be exonerated if there are other
// results with the same query and tags that have MayExonerate: false
MayExonerate bool
}
// Format writes the Result to the fmt.State
// The Result is printed as a single line, in the form:
//
// <query> <tags> <status>
//
// This matches the order in which results are sorted.
func (r Result) Format(f fmt.State, verb rune) {
if len(r.Tags) > 0 {
fmt.Fprintf(f, "%v %v %v %v %v", r.Query, TagsToString(r.Tags), r.Status, r.Duration, r.MayExonerate)
} else {
fmt.Fprintf(f, "%v %v %v %v", r.Query, r.Status, r.Duration, r.MayExonerate)
}
}
// String returns the result as a string
func (r Result) String() string {
sb := strings.Builder{}
fmt.Fprint(&sb, r)
return sb.String()
}
// Compare compares the relative order of r and o, returning:
//
// -1 if r should come before o
// 1 if r should come after o
// 0 if r and o are identical
//
// Note: Result.Duration is not considered in comparison.
func (r Result) Compare(o Result) int {
a, b := r, o
switch a.Query.Compare(b.Query) {
case -1:
return -1
case 1:
return 1
}
ta := strings.Join(a.Tags.List(), TagDelimiter)
tb := strings.Join(b.Tags.List(), TagDelimiter)
switch {
case ta < tb:
return -1
case ta > tb:
return 1
case a.Status < b.Status:
return -1
case a.Status > b.Status:
return 1
}
return 0
}
// Parse parses the result from a string of the form:
//
// <query> <tags> <status>
//
// <tags> may be omitted if there were no tags.
//
// Tests are separated into sections where the section name
// appears at the end of the list as just a line with
//
// <section-name>
func Parse(in string) (ExecutionMode, Result, error) {
line := in
token := func() string {
for i, c := range line {
if c != ' ' {
line = line[i:]
break
}
}
for i, c := range line {
if c == ' ' {
tok := line[:i]
line = line[i:]
return tok
}
}
tok := line
line = ""
return tok
}
a := token()
b := token()
c := token()
d := token()
e := token()
if a != "" && b == "" && token() == "" {
return ExecutionMode(a), Result{}, nil
}
if a == "" || b == "" || c == "" || d == "" || token() != "" {
return "", Result{}, fmt.Errorf("unable to parse result '%v'", in)
}
query := query.Parse(a)
if e == "" {
status := Status(b)
duration, err := time.ParseDuration(c)
if err != nil {
return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err)
}
mayExonerate, err := strconv.ParseBool(d)
if err != nil {
return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err)
}
return "", Result{query, nil, status, duration, mayExonerate}, nil
} else {
tags := StringToTags(b)
status := Status(c)
duration, err := time.ParseDuration(d)
if err != nil {
return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err)
}
mayExonerate, err := strconv.ParseBool(e)
if err != nil {
return "", Result{}, fmt.Errorf("unable to parse result '%v': %w", in, err)
}
return "", Result{query, tags, status, duration, mayExonerate}, nil
}
}
// List is a list of results
type List []Result
// The mode the tests were run in, "core" or" "compat"
type ExecutionMode string
// Lists of test results by execution mode.
type ResultsByExecutionMode map[ExecutionMode]List
// Variant is a collection of tags that uniquely identify a test
// configuration (e.g the combination of OS, GPU, validation-modes, etc).
type Variant = Tags
// Variants returns the list of unique tags (variants) across all results.
func (l List) Variants() []Variant {
tags := container.NewMap[string, Variant]()
for _, r := range l {
tags.Add(TagsToString(r.Tags), r.Tags)
}
return tags.Values()
}
// TransformTags returns the list of results with the tags transformed using f.
// TransformTags assumes that f will return the same output for the same input.
func (l List) TransformTags(f func(Tags) Tags) List {
cache := map[string]Tags{}
out := List{}
for _, r := range l {
key := TagsToString(r.Tags)
tags, cached := cache[key]
if !cached {
tags = f(r.Tags.Clone())
cache[key] = tags
}
out = append(out, Result{
Query: r.Query,
Tags: tags,
Status: r.Status,
Duration: r.Duration,
MayExonerate: r.MayExonerate,
})
}
return out
}
// ReplaceDuplicates returns a new list with duplicate test results replaced.
// When a duplicate is found, the function f is called with the duplicate
// results. The returned status will be used as the replaced result.
// Merged results will use the average (mean) duration of the duplicates.
func (l List) ReplaceDuplicates(f func(Statuses) Status) List {
type key struct {
query query.Query
tags string
}
// Collect all duplicates
keyToIndices := map[key][]int{} // key to index
for i, r := range l {
k := key{r.Query, TagsToString(r.Tags)}
keyToIndices[k] = append(keyToIndices[k], i)
}
// Filter out exonerated results
for key, indices := range keyToIndices {
keptIndices := []int{}
for _, i := range indices {
// Copy all indices which are not exonerated into keptIndices.
if !l[i].MayExonerate {
keptIndices = append(keptIndices, i)
}
}
// Change indices to only the kept ones. If keptIndices is empty,
// then all results were marked with may_exonerate, and we keep all
// of them.
if len(keptIndices) > 0 {
keyToIndices[key] = keptIndices
}
}
// Resolve duplicates
type StatusAndDuration struct {
Status Status
Duration time.Duration
}
merged := map[key]StatusAndDuration{}
for key, indices := range keyToIndices {
statuses := NewStatuses()
duration := time.Duration(0)
for _, i := range indices {
r := l[i]
statuses.Add(r.Status)
duration += r.Duration
}
status := func() Status {
if len(statuses) > 1 {
return f(statuses)
}
return statuses.One()
}()
duration = duration / time.Duration(len(indices))
merged[key] = StatusAndDuration{
Status: status,
Duration: duration,
}
}
// Rebuild list
out := make(List, 0, len(keyToIndices))
for _, r := range l {
k := key{r.Query, TagsToString(r.Tags)}
if sd, ok := merged[k]; ok {
out = append(out, Result{
Query: r.Query,
Tags: r.Tags,
Status: sd.Status,
Duration: sd.Duration,
MayExonerate: l[keyToIndices[k][0]].MayExonerate,
})
delete(merged, k) // Remove from map to prevent duplicates
}
}
return out
}
// Sort sorts the list
func (l List) Sort() {
sort.Slice(l, func(i, j int) bool { return l[i].Compare(l[j]) < 0 })
}
// Filter returns the results that match the given predicate
func (l List) Filter(f func(Result) bool) List {
out := make(List, 0, len(l))
for _, r := range l {
if f(r) {
out = append(out, r)
}
}
return out
}
// FilterByStatus returns the results that the given status
func (l List) FilterByStatus(status Status) List {
return l.Filter(func(r Result) bool {
return r.Status == status
})
}
// FilterByTags returns the results that have all the given tags
func (l List) FilterByTags(tags Tags) List {
return l.Filter(func(r Result) bool {
return r.Tags.ContainsAll(tags)
})
}
// FilterByVariant returns the results that exactly match the given tags
func (l List) FilterByVariant(tags Tags) List {
str := TagsToString(tags)
return l.Filter(func(r Result) bool {
return len(r.Tags) == len(tags) && TagsToString(r.Tags) == str
})
}
// FilterByQuery returns the results that match the given query
func (l List) FilterByQuery(q query.Query) List {
return l.Filter(func(r Result) bool {
return q.Contains(r.Query)
})
}
// Statuses is a set of Status
type Statuses = container.Set[Status]
// NewStatuses returns a new status set with the provided statuses
func NewStatuses(s ...Status) Statuses { return container.NewSet(s...) }
// Statuses returns a set of all the statuses in the list
func (l List) Statuses() Statuses {
set := NewStatuses()
for _, r := range l {
set.Add(r.Status)
}
return set
}
// StatusTree is a query tree of statuses
type StatusTree = query.Tree[Status]
// StatusTree returns a query.Tree from the List, with the Status as the tree
// node data.
func (l List) StatusTree() (StatusTree, error) {
tree := StatusTree{}
for _, r := range l {
if err := tree.Add(r.Query, r.Status); err != nil {
return StatusTree{}, err
}
}
return tree, nil
}
// Load loads the result list from the file with the given path
func Load(path string) (ResultsByExecutionMode, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
results, err := Read(file)
if err != nil {
return nil, fmt.Errorf("while reading '%v': %w", path, err)
}
return results, nil
}
// Save saves the result list to the file with the given path
func Save(path string, results ResultsByExecutionMode) error {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0777); err != nil {
return err
}
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
return Write(file, results)
}
// Read reads a result list from the given reader
func Read(r io.Reader) (ResultsByExecutionMode, error) {
scanner := bufio.NewScanner(r)
results := ResultsByExecutionMode{}
l := List{}
for scanner.Scan() {
section, r, err := Parse(scanner.Text())
if err != nil {
return nil, err
}
if section != "" {
results[section] = l
l = List{}
} else {
l = append(l, r)
}
}
return results, nil
}
// Write writes a result list to the given writer
func Write(w io.Writer, r ResultsByExecutionMode) error {
for name, l := range r {
for _, r := range l {
if _, err := fmt.Fprintln(w, r); err != nil {
return err
}
}
if _, err := fmt.Fprintln(w, name); err != nil {
return err
}
}
return nil
}
// Merge merges and sorts multiple results lists.
// Duplicates are removed using the Deduplicate() function.
func Merge(lists ...List) List {
n := 0
for _, l := range lists {
n += len(l)
}
merged := make(List, 0, n)
for _, l := range lists {
merged = append(merged, l...)
}
out := merged.ReplaceDuplicates(Deduplicate)
out.Sort()
return out
}
// Deduplicate is the standard algorithm used to de-duplicating mixed results.
// This function is expected to be handed to List.ReplaceDuplicates().
func Deduplicate(s Statuses) Status {
// If all results have the same status, then use that
if len(s) == 1 {
return s.One()
}
// Mixed statuses. Replace with something appropriate.
switch {
// Crash + * = Crash
case s.Contains(Crash):
return Crash
// Abort + * = Abort
case s.Contains(Abort):
return Abort
// Unknown + * = Unknown
case s.Contains(Unknown):
return Unknown
// RetryOnFailure + ~(Crash | Abort | Unknown) = RetryOnFailure
case s.Contains(RetryOnFailure):
return RetryOnFailure
// Pass + ~(Crash | Abort | Unknown | RetryOnFailure | Slow) = RetryOnFailure
case s.Contains(Pass):
return RetryOnFailure
}
return Unknown
}