[tools][cts] Remove old CTS roller codepath
Removes the old CTS roller codepath and all related code that is now
unused. There may be a bit of dead code remaining, but this should
remove the vast majority of it.
Bug: 372730248
Change-Id: I1f87e8139b9104f90680ef7f043e4819bf037f12
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/219374
Commit-Queue: Brian Sheedy <bsheedy@google.com>
Commit-Queue: Antonio Maiorano <amaiorano@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Auto-Submit: Brian Sheedy <bsheedy@google.com>
diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go
index 8e4e66f..16e1b1a 100644
--- a/tools/src/cmd/cts/roll/roll.go
+++ b/tools/src/cmd/cts/roll/roll.go
@@ -121,6 +121,8 @@
flag.BoolVar(&c.flags.preserve, "preserve", false, "do not abandon existing rolls")
flag.BoolVar(&c.flags.sendToGardener, "send-to-gardener", false, "send the CL to the WebGPU gardener for review")
flag.BoolVar(&c.flags.verbose, "verbose", false, "emit additional logging")
+ // Deprecated.
+ // TODO(crbug.com/372730248): Remove this flag once the roller stops passing it in.
flag.BoolVar(&c.flags.useSimplifiedCodepath, "use-simplified-codepath", false, "use the simplified codepath that only looks at unexpected failures")
flag.StringVar(&c.flags.parentSwarmingRunID, "parent-swarming-run-id", "", "parent swarming run id. All triggered tasks will be children of this task and will be canceled if the parent is canceled.")
flag.IntVar(&c.flags.maxAttempts, "max-attempts", 3, "number of update attempts before giving up")
@@ -414,11 +416,7 @@
// Gather the build results
log.Println("gathering results...")
- if r.flags.useSimplifiedCodepath {
- psResultsByExecutionMode, err = common.CacheUnsuppressedFailingResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds)
- } else {
- psResultsByExecutionMode, err = common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds)
- }
+ psResultsByExecutionMode, err = common.CacheUnsuppressedFailingResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds)
if err != nil {
return err
}
@@ -431,26 +429,14 @@
// Rebuild the expectations with the accumulated results
log.Println("building new expectations...")
for _, exInfo := range exInfos {
- if r.flags.useSimplifiedCodepath {
- // TODO(crbug.com/372730248): Modify exInfo.expectations in place once
- // the old code path is removed.
- exInfo.newExpectations = exInfo.expectations.Clone()
- err := exInfo.newExpectations.AddExpectationsForFailingResults(psResultsByExecutionMode[exInfo.executionMode], testlist, r.flags.verbose)
- if err != nil {
- return err
- }
- exInfo.expectations = exInfo.newExpectations
- } else {
- // Merge the new results into the accumulated results
- log.Printf("merging results for %s ...\n", exInfo.executionMode)
- exInfo.results = result.Merge(exInfo.results, psResultsByExecutionMode[exInfo.executionMode])
-
- exInfo.newExpectations = exInfo.expectations.Clone()
- _, err := exInfo.newExpectations.Update(exInfo.results, testlist, r.flags.verbose)
- if err != nil {
- return err
- }
+ // TODO(crbug.com/372730248): Modify exInfo.expectations in place once
+ // the old code path is removed.
+ exInfo.newExpectations = exInfo.expectations.Clone()
+ err := exInfo.newExpectations.AddExpectationsForFailingResults(psResultsByExecutionMode[exInfo.executionMode], testlist, r.flags.verbose)
+ if err != nil {
+ return err
}
+ exInfo.expectations = exInfo.newExpectations
}
// Otherwise, push the updated expectations, and try again
diff --git a/tools/src/cmd/cts/update/expectations/expectations.go b/tools/src/cmd/cts/update/expectations/expectations.go
index 8d49094..b32a059 100644
--- a/tools/src/cmd/cts/update/expectations/expectations.go
+++ b/tools/src/cmd/cts/update/expectations/expectations.go
@@ -80,6 +80,8 @@
c.flags.results.RegisterFlags(cfg)
c.flags.auth.Register(flag.CommandLine, auth.DefaultAuthOptions())
flag.BoolVar(&c.flags.verbose, "verbose", false, "emit additional logging")
+ // Deprecated.
+ // TODO(crbug.com/372730248): Remove this flag once the roller stops passing it in.
flag.BoolVar(&c.flags.useSimplifiedCodepath, "use-simplified-codepath", false, "use the simplified codepath that only looks at unexpected failures")
flag.Var(&c.flags.expectations, "expectations", "path to CTS expectations file(s) to update")
return nil, nil
@@ -111,12 +113,7 @@
// Fetch the results
log.Println("fetching results...")
- var resultsByExecutionMode result.ResultsByExecutionMode
- if c.flags.useSimplifiedCodepath {
- resultsByExecutionMode, err = c.flags.results.GetUnsuppressedFailingResults(ctx, cfg, auth)
- } else {
- resultsByExecutionMode, err = c.flags.results.GetResults(ctx, cfg, auth)
- }
+ resultsByExecutionMode, err := c.flags.results.GetUnsuppressedFailingResults(ctx, cfg, auth)
if err != nil {
return err
}
@@ -154,14 +151,9 @@
name = "compat"
}
- var diag expectations.Diagnostics
- if c.flags.useSimplifiedCodepath {
- err = ex.AddExpectationsForFailingResults(resultsByExecutionMode[name], testlist, c.flags.verbose)
- // TODO(crbug.com/372730248): Report actual diagnostics.
- diag = expectations.Diagnostics{}
- } else {
- diag, err = ex.Update(resultsByExecutionMode[name], testlist, c.flags.verbose)
- }
+ err = ex.AddExpectationsForFailingResults(resultsByExecutionMode[name], testlist, c.flags.verbose)
+ // TODO(crbug.com/372730248): Report actual diagnostics.
+ diag := expectations.Diagnostics{}
if err != nil {
return err
}
diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go
index fba9912..5a107d7 100644
--- a/tools/src/cts/expectations/update.go
+++ b/tools/src/cts/expectations/update.go
@@ -28,18 +28,18 @@
package expectations
import (
- "errors"
"fmt"
- "log"
- "os"
"strings"
"time"
"dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
- "dawn.googlesource.com/dawn/tools/src/progressbar"
- "github.com/mattn/go-isatty"
+)
+
+const (
+ // Chunk comment for the AddExpectationsForFailingResults path.
+ ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##"
)
// AddExpectationsForFailingResults adds new expectations for the provided
@@ -246,777 +246,3 @@
chunkToModify.Expectations.SortPrioritizeQuery()
return nil
}
-
-// Update performs an incremental update on the expectations using the provided
-// results.
-//
-// Update will:
-// - Remove any expectation lines that have a query where no results match.
-// - Remove expectations lines that are in a chunk which is not annotated with
-// 'KEEP', and all test results have the status 'Pass'.
-// - Remove chunks that have had all expectation lines removed.
-// - Appends new chunks for flaky and failing tests which are not covered by
-// existing expectation lines.
-//
-// Update returns a list of diagnostics for things that should be addressed.
-//
-// Note: Validate() should be called before attempting to update the
-// expectations. If Validate() returns errors, then Update() behaviour is
-// undefined.
-// TODO(crbug.com/371501714): Remove the Diagnostics return value since it is
-// no longer used with the removal of commenting on CLs.
-func (c *Content) Update(results result.List, testlist []query.Query, verbose bool) (Diagnostics, error) {
- // Make a copy of the results. This code mutates the list.
- results = append(result.List{}, results...)
-
- // Replace statuses that the CTS runner doesn't recognize with 'Failure'
- simplifyStatuses(results)
-
- // Produce a list of tag sets.
- // We reverse the declared order, as webgpu-cts/expectations.txt lists the
- // most important first (OS, GPU, etc), and result.MinimalVariantTags will
- // prioritize folding away the earlier tag-sets.
- tagSets := make([]result.Tags, len(c.Tags.Sets))
- for i, s := range c.Tags.Sets {
- tagSets[len(tagSets)-i-1] = s.Tags
- }
-
- // Scan the full result list to obtain all the test variants
- // (unique tag combinations).
- variants := results.Variants()
-
- if verbose {
- fmt.Println("result variants:")
- for i, tags := range variants {
- fmt.Printf(" (%.2d) %v\n", i, tags.List())
- }
- }
-
- // Add 'consumed' results for tests that were skipped.
- // This ensures that skipped results are not included in reduced trees.
- results = c.appendConsumedResultsForSkippedTests(results, testlist, variants)
-
- var pb *progressbar.ProgressBar
- if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd()) {
- pb = progressbar.New(os.Stdout, nil)
- defer pb.Stop()
- }
-
- testQueryTree, _ := query.NewTree[struct{}]()
- for _, query := range testlist {
- testQueryTree.Add(query, struct{}{})
- }
-
- u := updater{
- in: *c,
- out: Content{},
- resultQueryTree: buildResultQueryTree(results),
- testQueryTree: testQueryTree,
- variants: variants,
- tagSets: tagSets,
- pb: pb,
- }
-
- if err := u.preserveRetryOnFailures(); err != nil {
- return nil, err
- }
-
- // Update those expectations!
- if err := u.build(); err != nil {
- return nil, fmt.Errorf("while updating expectations: %w", err)
- }
-
- *c = u.out
- return u.diags, nil
-}
-
-// updater holds the state used for updating the expectations
-type updater struct {
- in Content // the original expectations Content
- out Content // newly built expectations Content
- resultQueryTree resultQueryTree // the results query tree
- testQueryTree query.Tree[struct{}]
- variants []container.Set[string]
- diags []Diagnostic // diagnostics raised during update
- tagSets []result.Tags // reverse-ordered tag-sets of 'in'
- pb *progressbar.ProgressBar // Progress bar, may be nil
-}
-
-// Returns 'results' with additional 'consumed' results for tests that have
-// 'Skip' expectations. This fills in gaps for results, preventing tree
-// reductions from marking skipped results as failure, which could result in
-// expectation collisions.
-func (c *Content) appendConsumedResultsForSkippedTests(results result.List,
- testlist []query.Query,
- variants []container.Set[string]) result.List {
- tree := query.Tree[struct{}]{}
- for _, q := range testlist {
- tree.Add(q, struct{}{})
- }
- // For each variant...
- for _, variant := range variants {
- resultsForVariant := container.NewSet[string]()
- for _, result := range results.FilterByVariant(variant) {
- resultsForVariant.Add(result.Query.String())
- }
-
- // For each expectation...
- for _, c := range c.Chunks {
- for _, ex := range c.Expectations {
- // Does this expectation apply for variant?
- if !variant.ContainsAll(ex.Tags) {
- continue // Nope.
- }
-
- // Does the expectation contain a Skip status?
- if !container.NewSet(ex.Status...).Contains(string(result.Skip)) {
- continue // Nope.
- }
-
- // Gather all the tests that apply to the expectation
- glob, _ := tree.Glob(query.Parse(ex.Query))
- for _, qd := range glob {
- // If we don't have a result for the test, then append a
- // synthetic 'consumed' result.
- if query := qd.Query.String(); !resultsForVariant.Contains(query) {
- resultsForVariant.Add(query)
- results = append(results, result.Result{
- Query: qd.Query,
- Tags: variant,
- Status: consumed,
- })
- }
- }
- }
- }
- }
- return results
-}
-
-// simplifyStatuses replaces all result statuses that are not one of
-// 'Pass', 'RetryOnFailure', 'Slow', 'Skip' with 'Failure', and also replaces
-// 'Skip' results with 'Pass'.
-func simplifyStatuses(results result.List) {
- for i, r := range results {
- switch r.Status {
- case result.Pass, result.RetryOnFailure, result.Slow:
- // keep
- case result.Skip:
- // Typically represents a .unimplemented() test
- results[i].Status = result.Pass
- default:
- results[i].Status = result.Failure
- }
- }
-}
-
-const (
- // Status used to mark results that have been already handled by an
- // expectation.
- consumed result.Status = "<<consumed>>"
- // Chunk comment for new flakes
- newFlakesComment = "# New flakes. Please triage - will be discarded/regenerated by the next roll:"
- // Chunk comment for new failures
- newFailuresComment = "# New failures. Please triage - will be discarded/regenerated by the next roll:"
- // Chunk comment for expectations the roller is allowed to mutate
- ROLLER_MUTABLE = "# ##ROLLER_MUTABLE##"
- // Chunk comment for expectations the roller should discard and rewrite
- ROLLER_DISCARD_AND_REWRITE = "# ##ROLLER_DISCARD_AND_REWRITE##"
- // Chunk comment for the AddExpectationsForFailingResults path.
- ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##"
-)
-
-// resultQueryTree holds tree of queries to all results (no filtering by tag or
-// status). The resultQueryTree is used to glob all the results that match a
-// particular query.
-type resultQueryTree struct {
- // All the results.
- results result.List
- // consumedAt is a list of line numbers for the i'th result in 'results'
- // Initially all line numbers are 0. When a result is consumed the line
- // number is set.
- consumedAt []int
- // Each tree node holds a list of indices to results.
- tree query.Tree[[]int]
-}
-
-// buildResultQueryTree builds the queryTree from the list of results.
-func buildResultQueryTree(results result.List) resultQueryTree {
- log.Println("building query tree...")
-
- // Build a map of query to result indices
- queryToIndices := map[query.Query][]int{}
- for i, r := range results {
- l := queryToIndices[r.Query]
- l = append(l, i)
- queryToIndices[r.Query] = l
- }
-
- // Construct the query tree to result indices
- tree := query.Tree[[]int]{}
- for query, indices := range queryToIndices {
- if err := tree.Add(query, indices); err != nil {
- // Unreachable: The only error we could get is duplicate data for
- // the same query, which should be impossible.
- panic(err)
- }
- }
-
- consumedAt := make([]int, len(results))
- return resultQueryTree{results, consumedAt, tree}
-}
-
-// glob returns the list of results matching the given tags under (or with) the
-// given query.
-func (qt *resultQueryTree) glob(q query.Query) (result.List, error) {
- glob, err := qt.tree.Glob(q)
- if err != nil {
- return nil, fmt.Errorf("while gathering results for query '%v': %w", q, err)
- }
-
- out := result.List{}
- for _, indices := range glob {
- for _, idx := range indices.Data {
- out = append(out, qt.results[idx])
- }
- }
-
- return out, nil
-}
-
-// globTags returns the list of results matching the given tags under (or with)
-// the given query.
-func (qt *resultQueryTree) globTags(q query.Query, t result.Tags) (result.List, error) {
- glob, err := qt.tree.Glob(q)
- if err != nil {
- return nil, err
- }
-
- out := result.List{}
- for _, indices := range glob {
- for _, idx := range indices.Data {
- if r := qt.results[idx]; r.Tags.ContainsAll(t) {
- out = append(out, r)
- }
- }
- }
- return out, nil
-}
-
-// markAsConsumed marks all the results matching the given tags
-// under (or with) the given query, as consumed.
-// line is used to record the line at which the results were consumed. If the
-// results were consumed as part of generating new expectations then line should
-// be 0.
-func (qt *resultQueryTree) markAsConsumed(q query.Query, t result.Tags, line int) {
- if glob, err := qt.tree.Glob(q); err == nil {
- for _, indices := range glob {
- for _, idx := range indices.Data {
- r := &qt.results[idx]
- if r.Tags.ContainsAll(t) {
- r.Status = consumed
- qt.consumedAt[idx] = line
- }
- }
- }
- }
-}
-
-// preserveRetryOnFailures changes any results matching expectations with a
-// RetryOnFailure expectation to RetryOnFailure.
-func (u *updater) preserveRetryOnFailures() error {
- // For each expectation...
- for _, c := range u.in.Chunks {
- for _, ex := range c.Expectations {
- // Does the expectation contain a RetryOnFailure status?
- if !container.NewSet(ex.Status...).Contains(string(result.RetryOnFailure)) {
- continue // Nope.
- }
-
- q := query.Parse(ex.Query)
-
- glob, err := u.resultQueryTree.tree.Glob(q)
- if err != nil {
- if errors.As(err, &query.ErrNoDataForQuery{}) {
- // No results for this RetryOnFailure expectation.
- // Flaky tests might have been removed from the CTS.
- // These expectations will be automatically removed by updater.expectation()
- continue
- }
- return err
- }
- for _, indices := range glob {
- for _, idx := range indices.Data {
- if u.resultQueryTree.results[idx].Tags.ContainsAll(ex.Tags) {
- u.resultQueryTree.results[idx].Status = result.RetryOnFailure
- }
- }
- }
- }
- }
- return nil
-}
-
-type Progress struct {
- totalExpectations int
- currentExpectation int
-}
-
-// build is the updater top-level function.
-// build first appends to u.out all chunks from 'u.in' with expectations updated
-// using the new results, and then appends any new expectations to u.out.
-func (u *updater) build() error {
- progress := Progress{}
-
- // Chunks are considered immutable by default, unless annotated as
- // ROLLER_MUTABLE or ROLLER_DISCARD_AND_REWRITE.
- mutableTokens := []string{
- ROLLER_MUTABLE,
- ROLLER_DISCARD_AND_REWRITE,
- }
-
- // Bin the chunks into those that contain any of the strings in
- // mutableTokens in the comments and those that do not have these strings.
- immutableChunks, mutableChunks := []Chunk{}, []Chunk{}
- for _, chunk := range u.in.Chunks {
- keep := true
-
- comments:
- for _, line := range chunk.Comments {
- for _, token := range mutableTokens {
- if strings.Contains(line, token) {
- keep = false
- break comments
- }
- }
- }
-
- if keep {
- immutableChunks = append(immutableChunks, chunk)
- } else {
- mutableChunks = append(mutableChunks, chunk)
- }
-
- progress.totalExpectations += len(chunk.Expectations)
- }
-
- log.Println("updating expectation chunks...")
-
- // Update all the existing chunks in two passes - those that are immutable
- // then those that are mutable. We do this because the former can't be
- // altered and may declare expectations that may collide with later
- // expectations.
- for _, group := range []struct {
- chunks []Chunk
- isImmutable bool
- }{
- {immutableChunks, true},
- {mutableChunks, false},
- } {
- for _, in := range group.chunks {
- out := u.chunk(in, group.isImmutable, &progress)
-
- // If all chunk had expectations, but now they've gone, remove the chunk
- if len(in.Expectations) > 0 && len(out.Expectations) == 0 {
- continue
- }
-
- u.out.Chunks = append(u.out.Chunks, out)
- }
- }
-
- // Emit new expectations (flaky, failing)
- if err := u.addNewExpectations(); err != nil {
- return fmt.Errorf("failed to add new expectations: %w", err)
- }
-
- return nil
-}
-
-// chunk returns a new Chunk, based on 'in', with the expectations updated.
-// isImmutable is true if the chunk is labelled with 'KEEP' and can't be changed.
-func (u *updater) chunk(in Chunk, isImmutable bool, progress *Progress) Chunk {
- if len(in.Expectations) == 0 {
- return in // Just a comment / blank line
- }
-
- // Skip over ROLLER_DISCARD_AND_REWRITE chunks (untriaged failures/flakes).
- // We'll just rebuild them at the end.
- for _, line := range in.Comments {
- if strings.HasPrefix(line, ROLLER_DISCARD_AND_REWRITE) {
- return Chunk{}
- }
- }
-
- // Build the new chunk's expectations
- newExpectations := container.NewMap[string, Expectation]()
- for _, exIn := range in.Expectations {
- if u.pb != nil {
- u.pb.Update(progressbar.Status{Total: progress.totalExpectations, Segments: []progressbar.Segment{
- {Count: 1 + progress.currentExpectation},
- }})
- progress.currentExpectation++
- }
-
- u.addExpectations(newExpectations, exIn, isImmutable)
- }
-
- // Sort the expectations to keep things clean and tidy.
- out := Chunk{Comments: in.Comments, Expectations: newExpectations.Values()}
- out.Expectations.Sort()
- return out
-}
-
-// expectation returns a new list of Expectations, based on the Expectation 'in',
-// using the new result data.
-func (u *updater) addExpectations(out container.Map[string, Expectation], in Expectation, isImmutable bool) {
- q := query.Parse(in.Query)
-
- // keyOf returns the map key for out
- keyOf := func(e Expectation) string { return fmt.Sprint(e.Tags, e.Query, e.Status) }
-
- // noResults is a helper for returning when the expectation has no test results.
- noResults := func() {
- if glob, err := u.testQueryTree.Glob(q); err == nil && len(glob) > 0 {
- // At least one test is found with the query in the test list - likely a variant that is not being run.
- if len(in.Tags) > 0 {
- u.diag(Note, in.Line, "no results found for query '%v' with tags %v", in.Query, in.Tags)
- } else {
- u.diag(Note, in.Line, "no results found for query '%v'", in.Query)
- }
- // Preserve.
- out.Add(keyOf(in), in)
- } else {
- // Remove the no-results expectation (do not add to out)
- u.diag(Warning, in.Line, "no tests exist with query '%v' - removing", in.Query)
- }
- }
-
- // Glob the results for the expectation's query + tag combination.
- // Ensure that none of these are already consumed.
- results, err := u.resultQueryTree.globTags(q, in.Tags)
- // If we can't find any results for this query + tag combination, then bail.
- switch {
- case errors.As(err, &query.ErrNoDataForQuery{}):
- noResults()
- return
- case err != nil:
- u.diag(Error, in.Line, "%v", err)
- return
- case len(results) == 0:
- noResults()
- return
- }
-
- // Before returning, mark all the results as consumed.
- // Note: this has to happen *after* we've generated the new expectations, as
- // marking the results as 'consumed' will impact the logic of
- // expectationsForRoot()
- defer u.resultQueryTree.markAsConsumed(q, in.Tags, in.Line)
-
- if isImmutable { // Expectation chunk was marked with 'KEEP'
- // Add a diagnostic if all tests of the expectation were 'Pass'
- if s := results.Statuses(); len(s) == 1 && s.One() == result.Pass {
- u.diagAllPass(in.Line, results)
- }
- out.Add(keyOf(in), in)
- return
- }
-
- // Rebuild the expectations for this query.
- expectations, somePass, someConsumed := u.expectationsForRoot(q, in.Line, in.Bug, in.Comment)
-
- // Add the new expectations to out
- for _, expectation := range expectations {
- out.Add(keyOf(expectation), expectation)
- }
-
- // Add a diagnostic if the expectation is filtered away
- if !out.Contains(keyOf(in)) && len(expectations) == 0 {
- switch {
- case somePass && someConsumed:
- u.diag(Note, in.Line, "expectation is partly covered by previous expectations and the remaining tests all pass")
- case someConsumed:
- u.diag(Note, in.Line, "expectation is fully covered by previous expectations")
- case somePass:
- u.diagAllPass(in.Line, results)
- }
- }
-
-}
-
-// addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and
-// failing tests.
-func (u *updater) addNewExpectations() error {
- // For each variant:
- // • Build a query tree using the results filtered to the variant, and then
- // reduce the tree.
- // • Take all the reduced-tree leaf nodes, and add these to 'roots'.
- // Once we've collected all the roots, we'll use these to build the
- // expectations across the reduced set of tags.
- log.Println("determining new expectation roots...")
- roots := query.Tree[bool]{}
- for i, variant := range u.variants {
- if u.pb != nil {
- u.pb.Update(progressbar.Status{Total: len(u.variants), Segments: []progressbar.Segment{
- {Count: 1 + i},
- }})
- }
-
- // Build a tree from the results matching the given variant.
- filtered := u.resultQueryTree.results.FilterByVariant(variant)
- tree, err := filtered.StatusTree()
- if err != nil {
- return fmt.Errorf("while building tree for tags '%v': %w", variant, err)
- }
- // Reduce the tree.
- tree.Reduce(treeReducer)
- // Add all the reduced leaf nodes to 'roots'.
- for _, qd := range tree.List() {
- if qd.Data != result.Pass {
- roots.Add(qd.Query, true)
- }
- }
- }
-
- // Build all the expectations for each of the roots.
- log.Println("building new expectations...")
- rootsList := roots.List()
- expectations := []Expectation{}
- for i, root := range rootsList {
- if u.pb != nil {
- u.pb.Update(progressbar.Status{Total: len(rootsList), Segments: []progressbar.Segment{
- {Count: 1 + i},
- }})
- }
- rootExpectations, _, _ := u.expectationsForRoot(
- root.Query, // Root query
- 0, // Line number
- "crbug.com/dawn/0000", // Bug
- "", // Comment
- )
- expectations = append(expectations, rootExpectations...)
- }
-
- // Bin the expectations by failure or flake.
- flakes, failures := []Expectation{}, []Expectation{}
- for _, r := range expectations {
- if container.NewSet(r.Status...).Contains(string(result.RetryOnFailure)) {
- flakes = append(flakes, r)
- } else {
- failures = append(failures, r)
- }
- }
-
- // Create chunks for any flakes and failures, in that order.
- for _, group := range []struct {
- results Expectations
- comment string
- }{
- {flakes, newFlakesComment},
- {failures, newFailuresComment},
- } {
- if len(group.results) > 0 {
- group.results.Sort()
- u.out.Chunks = append(u.out.Chunks, Chunk{
- Comments: []string{
- "################################################################################",
- group.comment,
- ROLLER_DISCARD_AND_REWRITE,
- "################################################################################",
- },
- Expectations: group.results,
- })
- }
- }
-
- return nil
-}
-
-// expectationsForRoot builds a list of expectations that cover the failing
-// tests for the results under root.
-// The returned list of expectations is optimized by reducing queries to the
-// most common root, and reducing tags to the smallest required set.
-func (u *updater) expectationsForRoot(
- root query.Query, // The sub-tree query root
- line int, // The originating line, when producing diagnostics
- bug string, // The bug to apply to all returned expectations
- comment string, // The comment to apply to all returned expectations
-) (
- expectations []Expectation, // The output expectations
- somePass bool, // Some of the results for the query had a Pass status
- someConsumed bool, // The query was at least partly consumed by previous expectations
-) {
- results, err := u.resultQueryTree.glob(root)
- if err != nil {
- u.diag(Error, line, "%v", err)
- return nil, false, false
- }
-
- // Using the full list of unfiltered tests, generate the minimal set of
- // variants (tags) that uniquely classify the results with differing status.
- minimalVariants := u.
- removeUnknownTags(results).
- MinimalVariantTags(u.tagSets)
-
- // For each minimized variant...
- reduced := result.List{}
- for _, variant := range minimalVariants {
- // Build a query tree from this variant...
- tree := result.StatusTree{}
- filtered := results.FilterByTags(variant)
- for _, r := range filtered {
- // Note: variants may overlap, but overlaped queries will have
- // identical statuses, so we can just ignore the error for Add().
- tree.Add(r.Query, r.Status)
- }
-
- // ... and reduce the tree by collapsing sub-trees that have common
- // statuses.
- tree.ReduceUnder(root, treeReducer)
-
- // Append the reduced tree nodes to the results list
- for _, qs := range tree.List() {
- reduced = append(reduced, result.Result{
- Query: qs.Query,
- Tags: variant,
- Status: qs.Data,
- })
- }
- }
-
- // Filter out any results that passed or have already been consumed
- filtered := result.List{}
- for _, r := range reduced {
- switch r.Status {
- case result.Pass:
- somePass = true
- case consumed:
- someConsumed = true
- default:
- filtered = append(filtered, r)
- }
- }
-
- // Mark all the new expectation results as consumed.
- for _, r := range filtered {
- u.resultQueryTree.markAsConsumed(r.Query, r.Tags, 0)
- }
-
- // Transform the results to expectations.
- expectations = u.resultsToExpectations(filtered, bug, comment)
- return expectations, somePass, someConsumed
-}
-
-// resultsToExpectations returns a list of expectations from the given results.
-// Each expectation will have the same query, tags and status as the input
-// result, along with the specified bug and comment.
-//
-// If the result query target is a test without a wildcard, then the expectation
-// will have a wildcard automatically appended. This is to satisfy a requirement
-// of the expectation validator.
-func (u *updater) resultsToExpectations(results result.List, bug, comment string) []Expectation {
- results.Sort()
-
- out := make([]Expectation, 0, len(results))
- addedExpectations := container.NewSet[string]()
- for _, r := range results {
- q := r.Query.String()
- if r.Query.Target() == query.Tests && !r.Query.IsWildcard() {
- // The expectation validator wants a trailing ':' for test queries
- q += query.TargetDelimiter
- }
- e := Expectation{
- Bug: bug,
- Tags: u.in.Tags.RemoveLowerPriorityTags(r.Tags),
- Query: q,
- Status: []string{string(r.Status)},
- Comment: comment,
- }
- key := e.AsExpectationFileString()
- // We keep track of all expectations we've added so far to avoid cases where
- // two distinct results create the same expectation due to
- // RemoveLowerPriorityTags removing the distinguishing tags.
- if !addedExpectations.Contains(key) {
- out = append(out, e)
- addedExpectations.Add(key)
- }
- }
-
- return out
-}
-
-// removeUnknownTags returns a copy of the provided results with all tags not
-// found in the expectations list removed
-func (u *updater) removeUnknownTags(results result.List) result.List {
- return results.TransformTags(func(t result.Tags) result.Tags {
- filtered := result.NewTags()
- for tag := range t {
- if _, ok := u.in.Tags.ByName[tag]; ok {
- filtered.Add(tag)
- }
- }
- return filtered
- })
-}
-
-// treeReducer is a function that can be used by StatusTree.Reduce() to reduce
-// tree nodes with the same status.
-// treeReducer will collapse trees nodes if any of the following are true:
-// - All child nodes have the same status
-// - More than 50% of the child nodes have a non-pass status, and none of the
-// children are consumed.
-// - There are more than 10 child nodes with a non-pass status, and none of the
-// children are consumed.
-func treeReducer(statuses []result.Status) *result.Status {
- counts := map[result.Status]int{}
- for _, s := range statuses {
- counts[s] = counts[s] + 1
- }
- if len(counts) == 1 {
- return &statuses[0] // All the same status
- }
- if counts[consumed] > 0 {
- return nil // Partially consumed trees cannot be merged
- }
- highestNonPassCount := 0
- highestNonPassStatus := result.Failure
- for s, n := range counts {
- if s != result.Pass {
- if percent := (100 * n) / len(statuses); percent > 50 {
- // Over 50% of all the children are of non-pass status s.
- return &s
- }
- if n > highestNonPassCount {
- highestNonPassCount = n
- highestNonPassStatus = s
- }
- }
- }
-
- if highestNonPassCount > 10 {
- // Over 10 child node failed.
- return &highestNonPassStatus
- }
-
- return nil
-}
-
-// diag appends a new diagnostic to u.diags with the given severity, line and
-// message.
-func (u *updater) diag(severity Severity, line int, msg string, args ...interface{}) {
- u.diags = append(u.diags, Diagnostic{
- Severity: severity,
- Line: line,
- Message: fmt.Sprintf(msg, args...),
- })
-}
-
-// diagAllPass appends a new note diagnostic that all the tests now pass
-func (u *updater) diagAllPass(line int, results result.List) {
- if c := len(results); c > 1 {
- u.diag(Note, line, "all %d tests now pass", len(results))
- } else {
- u.diag(Note, line, "test now passes")
- }
-}
diff --git a/tools/src/cts/expectations/update_test.go b/tools/src/cts/expectations/update_test.go
index a4c1273..ce172dc 100644
--- a/tools/src/cts/expectations/update_test.go
+++ b/tools/src/cts/expectations/update_test.go
@@ -28,933 +28,15 @@
package expectations
import (
- "strings"
"testing"
- "dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
- "github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
)
var Q = query.Parse
-func TestUpdate(t *testing.T) {
- header := `# BEGIN TAG HEADER
-# OS
-# tags: [ os-a os-b os-c ]
-# GPU
-# tags: [ gpu-a gpu-b gpu-c ]
-# END TAG HEADER
-`
- headerLines := strings.Count(header, "\n")
-
- type Test struct {
- name string
- expectations string
- results result.List
- updated string
- diagnostics Diagnostics
- err string
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- name: "empty results",
- expectations: ``,
- results: result.List{},
- },
- { //////////////////////////////////////////////////////////////////////
- name: "no results found",
- expectations: `
-# ##ROLLER_MUTABLE##
-crbug.com/a/123 a:missing,test,result:* [ Failure ]
-crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ]
-some:other,test:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Failure,
- },
- },
- updated: `
-# ##ROLLER_MUTABLE##
-some:other,test:* [ Failure ]
-crbug.com/a/123 a:missing,test,result:* [ Failure ]
-crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Note,
- Line: headerLines + 3,
- Message: "no results found for query 'a:missing,test,result:*'",
- },
- {
- Severity: Note,
- Line: headerLines + 4,
- Message: "no results found for query 'another:missing,test,result:*' with tags [tag]",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "no results found immutable",
- expectations: `
-crbug.com/a/123 a:missing,test,result:* [ Failure ]
-
-some:other,test:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Failure,
- },
- },
- updated: `
-crbug.com/a/123 a:missing,test,result:* [ Failure ]
-
-some:other,test:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Note,
- Line: headerLines + 2,
- Message: "no results found for query 'a:missing,test,result:*'",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "unknown test",
- expectations: `
-# ##ROLLER_MUTABLE##
-crbug.com/a/123 an:unknown,test:* [ Failure ]
-crbug.com/a/123 [ tag ] another:unknown:test [ Failure ]
-
-some:other,test:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Failure,
- },
- },
- updated: `
-some:other,test:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Warning,
- Line: headerLines + 3,
- Message: "no tests exist with query 'an:unknown,test:*' - removing",
- },
- {
- Severity: Warning,
- Line: headerLines + 4,
- Message: "no tests exist with query 'another:unknown:test' - removing",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "unknown test found in immutable chunk",
- expectations: `
-crbug.com/a/123 an:unknown,test:* [ Failure ]
-
-some:other,test:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("some:other,test:*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Failure,
- },
- },
- updated: `
-some:other,test:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Warning,
- Line: headerLines + 2,
- Message: "no tests exist with query 'an:unknown,test:*' - removing",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "simple expectation with tags",
- expectations: `
-# ##ROLLER_MUTABLE##
-[ os-a ] a:b,c:* [ Failure ]
-[ gpu-b ] a:b,c:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("os-a", "os-c", "gpu-b"),
- Status: result.Failure,
- },
- },
- updated: `
-# ##ROLLER_MUTABLE##
-a:b,c:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Note,
- Line: headerLines + 4,
- Message: "expectation is fully covered by previous expectations",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "simple expectation with tags new flakes implicitly mutable",
- expectations: `
-################################################################################
-# New flakes. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-[ os-a ] a:b,c:* [ RetryOnFailure ]
-[ gpu-b ] a:b,c:* [ RetryOnFailure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("os-a", "os-c", "gpu-b"),
- Status: result.RetryOnFailure,
- },
- },
- updated: `
-################################################################################
-# New flakes. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 a:* [ RetryOnFailure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "simple expectation with tags new failures implicitly mutable",
- expectations: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-[ os-a ] a:b,c:* [ Failure ]
-[ gpu-b ] a:b,c:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("os-a", "os-c", "gpu-b"),
- Status: result.Failure,
- },
- },
- updated: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 a:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "simple expectation with tags immutable",
- expectations: `
-[ os-a ] a:b,c:* [ Failure ]
-[ gpu-b ] a:b,c:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("os-a", "os-c", "gpu-b"),
- Status: result.Failure,
- },
- },
- updated: `
-[ gpu-b ] a:b,c:* [ Failure ]
-[ os-a ] a:b,c:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "expectation test now passes",
- expectations: `
-# ##ROLLER_MUTABLE##
-crbug.com/a/123 [ gpu-a os-a ] a:b,c:* [ Failure ]
-crbug.com/a/123 [ gpu-b os-b ] a:b,c:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Abort,
- },
- },
- updated: `
-# ##ROLLER_MUTABLE##
-crbug.com/a/123 [ os-b ] a:b,c:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Note,
- Line: headerLines + 4,
- Message: "expectation is fully covered by previous expectations",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "expectation case now passes",
- expectations: `
-# ##ROLLER_MUTABLE##
-crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ]
-crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:d:*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("a:b,c:d:*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Abort,
- },
- },
- updated: `
-# ##ROLLER_MUTABLE##
-crbug.com/a/123 [ os-b ] a:b,c:d:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Note,
- Line: headerLines + 4,
- Message: "expectation is fully covered by previous expectations",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "first expectation expands to cover later expectations - no diagnostics",
- expectations: `
-crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ]
-crbug.com/a/123 [ gpu-c os-a ] a:b,c:d:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("gpu-a", "os-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("gpu-a", "os-b"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("gpu-b", "os-a"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("gpu-b", "os-b"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("gpu-c", "os-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("gpu-c", "os-b"),
- Status: result.Pass,
- },
- },
- updated: `
-crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ]
-crbug.com/a/123 [ gpu-c os-a ] a:b,c:d:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "expectation case now passes immutable - single",
- expectations: `
-crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ]
-crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ]
-`,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("a:b,c:d:e"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Abort,
- },
- },
- updated: `
-crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ]
-crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Note,
- Line: headerLines + 2,
- Message: "test now passes",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "expectation case now passes immutable - multiple",
- expectations: `
-crbug.com/a/123 a:b,c:d:* [ Failure ]
-`,
- results: result.List{
- result.Result{Query: Q("a:b,c:d:a"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:d:b"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:d:c"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:d:d"), Status: result.Pass},
- },
- updated: `
-crbug.com/a/123 a:b,c:d:* [ Failure ]
-`,
- diagnostics: Diagnostics{
- {
- Severity: Note,
- Line: headerLines + 2,
- Message: "all 4 tests now pass",
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "new test results",
- expectations: `# A comment`,
- results: result.List{
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_a:*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Abort,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_a:*"),
- Tags: result.NewTags("os-a", "gpu-b"),
- Status: result.Abort,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_c:case=4;*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Crash,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_c:case=4;*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Crash,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_c:case=5;*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.RetryOnFailure,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_b:case=5;*"),
- Tags: result.NewTags("os-b", "gpu-b"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_b:case=6;*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.Slow,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_b:case=6;*"),
- Tags: result.NewTags("os-b", "gpu-a"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("suite:dir_a,dir_b:test_c:case=6;*"),
- Tags: result.NewTags("os-a", "gpu-a"),
- Status: result.RetryOnFailure,
- },
- },
- updated: `# A comment
-
-################################################################################
-# New flakes. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:case=6;* [ RetryOnFailure ]
-crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
-
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ]
-crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Slow ]
-crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
-crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ]
-crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ]
-`,
- },
-
- { //////////////////////////////////////////////////////////////////////
- name: "root node overlap",
- expectations: `# A comment`,
- results: result.List{
- // For variant ['os-a'], we have a root node 'a:b,c:d:*'.
- result.Result{
- Query: Q("a:b,c:d:x,*"),
- Tags: result.NewTags("os-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:d:y,*"),
- Tags: result.NewTags("os-a"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:e:*"),
- Tags: result.NewTags("os-a"),
- Status: result.Pass,
- },
- // For variant ['os-b'], we have a root node 'a:b,c:d:x,*'.
- result.Result{
- Query: Q("a:b,c:d:x,*"),
- Tags: result.NewTags("os-b"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:d:y,*"),
- Tags: result.NewTags("os-b"),
- Status: result.Pass,
- },
- result.Result{
- Query: Q("a:b,c:e:*"),
- Tags: result.NewTags("os-b"),
- Status: result.Pass,
- },
- },
- updated: `# A comment
-
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 [ os-a ] a:b,c:d:* [ Failure ]
-crbug.com/dawn/0000 [ os-b ] a:b,c:d:x,* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "filter unknown tags",
- expectations: ``,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-a", "gpu-x"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-b", "gpu-x"),
- Status: result.Crash,
- },
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-x", "gpu-b"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-x", "gpu-a"),
- Status: result.Crash,
- },
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-c", "gpu-c"),
- Status: result.Pass,
- },
- },
- updated: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 [ gpu-a ] a:* [ Failure ]
-crbug.com/dawn/0000 [ gpu-b ] a:* [ Failure ]
-crbug.com/dawn/0000 [ os-a ] a:* [ Failure ]
-crbug.com/dawn/0000 [ os-b ] a:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "prioritize tag sets",
- expectations: ``,
- results: result.List{
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-a", "os-c", "gpu-b"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("gpu-a", "os-b", "gpu-c"),
- Status: result.Failure,
- },
- result.Result{
- Query: Q("a:b,c:*"),
- Tags: result.NewTags("os-c", "gpu-c"),
- Status: result.Pass,
- },
- },
- updated: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 [ gpu-b os-c ] a:* [ Failure ]
-crbug.com/dawn/0000 [ gpu-c os-b ] a:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "merge when 50% or more children fail",
- expectations: ``,
- results: result.List{ // 4 pass, 6 fail (50%)
- result.Result{Query: Q("a:b,c:0:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:1:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:2:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:3:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:4:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:5:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:6:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:7:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:8:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:9:*"), Status: result.Pass},
- },
- updated: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 a:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "don't merge when 50% or fewer children fail",
- expectations: ``,
- results: result.List{ // 5 pass, 5 fail (50%)
- result.Result{Query: Q("a:b,c:0:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:1:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:2:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:3:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:4:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:5:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:6:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:7:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:8:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:9:*"), Status: result.Pass},
- },
- updated: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 a:b,c:0:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:2:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:5:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:6:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:8:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "merge when more than 10 children fail",
- expectations: ``,
- results: result.List{ // 19 pass, 11 fail (37%)
- result.Result{Query: Q("a:b,c:00:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:01:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:02:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:03:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:04:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:05:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:06:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:07:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:08:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:09:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:10:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:11:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:12:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:13:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:14:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:15:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:16:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:17:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:18:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:19:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:20:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:21:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:22:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:23:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:24:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:25:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:26:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:27:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:28:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:29:*"), Status: result.Failure},
- },
- updated: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 a:* [ Failure ]
-`,
- },
- { //////////////////////////////////////////////////////////////////////
- name: "don't merge when 10 or fewer children fail",
- expectations: ``,
- results: result.List{ // 20 pass, 10 fail (33%)
- result.Result{Query: Q("a:b,c:00:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:01:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:02:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:03:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:04:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:05:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:06:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:07:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:08:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:09:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:10:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:11:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:12:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:13:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:14:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:15:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:16:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:17:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:18:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:19:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:20:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:21:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:22:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:23:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:24:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:25:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:26:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:27:*"), Status: result.Pass},
- result.Result{Query: Q("a:b,c:28:*"), Status: result.Failure},
- result.Result{Query: Q("a:b,c:29:*"), Status: result.Failure},
- },
- updated: `
-################################################################################
-# New failures. Please triage - will be discarded/regenerated by the next roll:
-# ##ROLLER_DISCARD_AND_REWRITE##
-################################################################################
-crbug.com/dawn/0000 a:b,c:00:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:05:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:08:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:13:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:15:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:20:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:23:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:26:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:28:* [ Failure ]
-crbug.com/dawn/0000 a:b,c:29:* [ Failure ]
-`,
- },
- } {
- ex, err := Parse("expectations.txt", header+test.expectations)
- if err != nil {
- t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err)
- }
-
- testList := container.NewMap[string, query.Query]()
- for _, r := range test.results {
- testList.Add(r.Query.String(), r.Query)
- }
- for _, s := range []string{
- "a:missing,test,result:a=1,b=2",
- "another:missing,test,result:cat=meow,dog=woof",
- } {
- testList.Add(s, query.Parse(s))
- }
-
- errMsg := ""
- diagnostics, err := ex.Update(test.results, testList.Values() /* verbose */, false)
- if err != nil {
- errMsg = err.Error()
- }
- if diff := cmp.Diff(errMsg, test.err); diff != "" {
- t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff)
- }
-
- if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" {
- t.Errorf("'%v': diagnostics were not as expected:\n%v", test.name, diff)
- }
-
- if diff := cmp.Diff(
- strings.Split(ex.String(), "\n"),
- strings.Split(header+test.updated, "\n")); diff != "" {
- t.Errorf("'%v': updated was not as expected:\n%v", test.name, diff)
- }
- }
-}
-
-func createGenericUpdater(t *testing.T) updater {
- header := `
-# BEGIN TAG HEADER
-# OS
-# tags: [ linux win10 ]
-# GPU
-# tags: [ intel
-# nvidia nvidia-0x2184 ]
-# Driver
-# tags: [ nvidia_ge_31.0.15.4601 nvidia_lt_31.0.15.4601
-# nvidia_ge_535.183.01 nvidia_lt_535.183.01 ]
-# END TAG HEADER
-`
- inContent, err := Parse("expectations.txt", header)
- if err != nil {
- t.Fatalf("Failed to parse expectations: %v", err)
- }
-
- u := updater{
- in: inContent,
- }
- return u
-}
-
-// Tests basic result -> expectation conversion.
-func TestResultsToExpectationsBasic(t *testing.T) {
- results := result.List{
- {
- Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout:"),
- Tags: result.NewTags("linux", "nvidia"),
- Status: result.Failure,
- },
- {
- Query: query.Parse("webgpu:shader,execution,memory_model,barrier:"),
- Tags: result.NewTags("win10", "intel"),
- Status: result.Failure,
- },
- }
-
- expectedOutput := []Expectation{
- {
- Bug: "crbug.com/1234",
- Tags: result.NewTags("linux", "nvidia"),
- Query: "webgpu:shader,execution,memory_layout:read_layout:",
- Status: []string{"Failure"},
- Comment: "comment",
- },
- {
- Bug: "crbug.com/1234",
- Tags: result.NewTags("win10", "intel"),
- Query: "webgpu:shader,execution,memory_model,barrier",
- Status: []string{"Failure"},
- Comment: "comment",
- },
- }
-
- u := createGenericUpdater(t)
- output := u.resultsToExpectations(results, "crbug.com/1234", "comment")
- assert.Equal(t, output, expectedOutput)
-}
-
-// Tests behavior when two unique results end up creating the same expectation
-// due to lower priority tags being removed.
-func TestResultsToExpectationsOverlappingExpectations(t *testing.T) {
- results := result.List{
- {
- Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout:"),
- Tags: result.NewTags("nvidia", "nvidia-0x2184", "nvidia_ge_31.0.15.4601", "nvidia_lt_535.183.01"),
- Status: result.Failure,
- },
- {
- Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout:"),
- Tags: result.NewTags("nvidia", "nvidia-0x2184", "nvidia_lt_31.0.15.4601", "nvidia_lt_535.183.01"),
- Status: result.Failure,
- },
- }
-
- expectedOutput := []Expectation{
- {
- Bug: "crbug.com/1234",
- Tags: result.NewTags("nvidia-0x2184", "nvidia_lt_535.183.01"),
- Query: "webgpu:shader,execution,memory_layout:read_layout:",
- Status: []string{"Failure"},
- Comment: "comment",
- },
- }
-
- u := createGenericUpdater(t)
- output := u.resultsToExpectations(results, "crbug.com/1234", "comment")
- assert.Equal(t, output, expectedOutput)
-}
-
-// Tests behavior related to automatic inclusion of a trailing :.
-func TestResultsToExpectationsTrailingColon(t *testing.T) {
- results := result.List{
- // Should automatically have a : added since it's a test query.
- {
- Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout"),
- Tags: result.NewTags("linux", "nvidia"),
- Status: result.Failure,
- },
- // Should not have a : added since it is a wildcard query.
- {
- Query: query.Parse("webgpu:shader,execution,*"),
- Tags: result.NewTags("win10", "intel"),
- Status: result.Failure,
- },
- }
-
- expectedOutput := []Expectation{
- {
- Bug: "crbug.com/1234",
- Tags: result.NewTags("win10", "intel"),
- Query: "webgpu:shader,execution,*",
- Status: []string{"Failure"},
- Comment: "comment",
- },
- {
- Bug: "crbug.com/1234",
- Tags: result.NewTags("linux", "nvidia"),
- Query: "webgpu:shader,execution,memory_layout:read_layout:",
- Status: []string{"Failure"},
- Comment: "comment",
- },
- }
-
- u := createGenericUpdater(t)
- output := u.resultsToExpectations(results, "crbug.com/1234", "comment")
- assert.Equal(t, output, expectedOutput)
-}
-
/*******************************************************************************
* removeExpectationsForUnknownTests tests
******************************************************************************/
diff --git a/tools/src/cts/query/tree.go b/tools/src/cts/query/tree.go
index 91ac323..4288b6f 100644
--- a/tools/src/cts/query/tree.go
+++ b/tools/src/cts/query/tree.go
@@ -112,105 +112,6 @@
return nil
}
-// Merger is a function used to merge the children nodes of a tree.
-// Merger is called with the Data of each child node. If the function returns a
-// non-nil Data pointer, then this is used as the merged result. If the function
-// returns nil, then the node will not be merged.
-type Merger[Data any] func([]Data) *Data
-
-// merge collapses tree nodes based on child node data, using the function f.
-// merge operates on the leaf nodes first, working its way towards the root of
-// the tree.
-// Returns the merged target data for this node, or nil if the node is not a
-// leaf and its children has non-uniform data.
-func (n *TreeNode[Data]) merge(f Merger[Data]) *Data {
- // If the node is a leaf, then simply return the node's data.
- if len(n.Children) == 0 {
- return n.Data
- }
-
- // Build a map of child target to merged child data.
- // A nil for the value indicates that one or more children could not merge.
- mergedChildren := map[Target][]Data{}
- for key, child := range n.Children {
- // Call merge() on the child. Even if we cannot merge this node, we want
- // to do this for all children so they can merge their sub-graphs.
- childData := child.merge(f)
-
- if childData == nil {
- // If merge() returned nil, then the data could not be merged.
- // Mark the entire target as unmergeable.
- mergedChildren[key.Target] = nil
- continue
- }
-
- // Fetch the merge list for this child's target.
- list, found := mergedChildren[key.Target]
- if !found {
- // First child with the given target?
- mergedChildren[key.Target] = []Data{*childData}
- continue
- }
- if list != nil {
- mergedChildren[key.Target] = append(list, *childData)
- }
- }
-
- merge := func(in []Data) *Data {
- switch len(in) {
- case 0:
- return nil // nothing to merge.
- case 1:
- return &in[0] // merge of a single item results in that item
- default:
- return f(in)
- }
- }
-
- // Might it possible to merge this node?
- maybeMergeable := true
-
- // The merged data, per target
- mergedTargets := map[Target]Data{}
-
- // Attempt to merge each of the target's data
- for target, list := range mergedChildren {
- if list != nil { // nil == unmergeable target
- if data := merge(list); data != nil {
- // Merge success!
- mergedTargets[target] = *data
- continue
- }
- }
- maybeMergeable = false // Merge of this node is not possible
- }
-
- // Remove all children that have been merged
- for key := range n.Children {
- if _, merged := mergedTargets[key.Target]; merged {
- delete(n.Children, key)
- }
- }
-
- // Add wildcards for merged targets
- for target, data := range mergedTargets {
- data := data // Don't take address of iterator
- n.getOrCreateChild(TreeNodeChildKey{"*", target}).Data = &data
- }
-
- // If any of the targets are unmergeable, then we cannot merge the node itself.
- if !maybeMergeable {
- return nil
- }
-
- // All targets were merged. Attempt to merge each of the targets.
- data := make([]Data, 0, len(mergedTargets))
- for _, d := range mergedTargets {
- data = append(data, d)
- }
- return merge(data)
-}
-
// print writes a textual representation of this node and its children to w.
// prefix is used as the line prefix for each node, which is appended with
// whitespace for each child node.
@@ -280,23 +181,6 @@
return nil
}
-// Split adds a new data to the tree, clearing any ancestor node's data.
-// Returns ErrDuplicateData if the tree already contains a data for the given node at query
-func (t *Tree[Data]) Split(q Query, d Data) error {
- node := &t.TreeNode
- q.Walk(func(q Query, t Target, n string) error {
- delete(node.Children, TreeNodeChildKey{Name: "*", Target: t})
- node.Data = nil
- node = node.getOrCreateChild(TreeNodeChildKey{n, t})
- return nil
- })
- if node.Data != nil {
- return ErrDuplicateData{node.Query}
- }
- node.Data = &d
- return nil
-}
-
// GetOrCreate returns existing, or adds a new data to the tree.
func (t *Tree[Data]) GetOrCreate(q Query, create func() Data) *Data {
node := &t.TreeNode
@@ -327,40 +211,6 @@
return node
}
-// Reduce reduces the tree using the Merger function f.
-// If the Merger function returns a non-nil Data value, then this will be used
-// to replace the non-leaf node with a new leaf node holding the returned Data.
-// This process recurses up to the tree root.
-func (t *Tree[Data]) Reduce(f Merger[Data]) {
- for _, root := range t.TreeNode.Children {
- root.merge(f)
- }
-}
-
-// ReduceUnder reduces the sub-tree under the given query using the Merger
-// function f.
-// If the Merger function returns a non-nil Data value, then this will be used
-// to replace the non-leaf node with a new leaf node holding the returned Data.
-// This process recurses up to the node pointed at by the query to.
-func (t *Tree[Data]) ReduceUnder(to Query, f Merger[Data]) error {
- node := &t.TreeNode
- return to.Walk(func(q Query, t Target, n string) error {
- if n == "*" {
- node.merge(f)
- return nil
- }
- child, ok := node.Children[TreeNodeChildKey{n, t}]
- if !ok {
- return ErrNoDataForQuery{q}
- }
- node = child
- if q == to {
- node.merge(f)
- }
- return nil
- })
-}
-
// glob calls f for every node under the given query.
func (t *Tree[Data]) glob(fq Query, f func(f *TreeNode[Data]) error) error {
node := &t.TreeNode
@@ -403,44 +253,6 @@
})
}
-// Replace replaces the sub-tree matching the query 'what' with the Data 'with'
-func (t *Tree[Data]) Replace(what Query, with Data) error {
- node := &t.TreeNode
- return what.Walk(func(q Query, t Target, n string) error {
- childKey := TreeNodeChildKey{n, t}
- if q == what {
- for key, child := range node.Children {
- // Use Query.Contains() to handle matching of Cases
- // (which are not split into tree nodes)
- if q.Contains(child.Query) {
- delete(node.Children, key)
- }
- }
- node = node.getOrCreateChild(childKey)
- node.Data = &with
- } else {
- child, ok := node.Children[childKey]
- if !ok {
- return ErrNoDataForQuery{q}
- }
- node = child
- }
- return nil
- })
-}
-
-// List returns the tree nodes flattened as a list of QueryData
-func (t *Tree[Data]) List() []QueryData[Data] {
- out := []QueryData[Data]{}
- t.traverse(func(n *TreeNode[Data]) error {
- if n.Data != nil {
- out = append(out, QueryData[Data]{n.Query, *n.Data})
- }
- return nil
- })
- return out
-}
-
// Glob returns a list of QueryData's for every node that is under the given
// query, which holds data.
// Glob handles wildcards as well as non-wildcard queries:
diff --git a/tools/src/cts/query/tree_test.go b/tools/src/cts/query/tree_test.go
index c2e9ce7..9ed14f5 100644
--- a/tools/src/cts/query/tree_test.go
+++ b/tools/src/cts/query/tree_test.go
@@ -4,9 +4,7 @@
"fmt"
"testing"
- "dawn.googlesource.com/dawn/tools/src/container"
"dawn.googlesource.com/dawn/tools/src/cts/query"
- "dawn.googlesource.com/dawn/tools/src/fileutils"
"github.com/google/go-cmp/cmp"
)
@@ -350,635 +348,6 @@
}
}
-func TestSplit(t *testing.T) {
- type Tree = query.Tree[string]
- type Node = query.TreeNode[string]
- type QueryData = query.QueryData[string]
- type Children = query.TreeNodeChildren[string]
-
- type Test struct {
- in QueryData
- pre Tree
- post Tree
- }
- for _, test := range []Test{
- { /////////////////////////////////////////////////////////////////////
- in: QueryData{
- Query: Q(`suite:*`),
- Data: pass,
- },
- post: Tree{
- TreeNode: Node{
- Children: Children{
- query.TreeNodeChildKey{`suite`, query.Suite}: {
- Query: Q(`suite`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Files}: {
- Query: Q(`suite:*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- { /////////////////////////////////////////////////////////////////////
- in: QueryData{
- Query: Q(`suite:a,b:*`),
- Data: pass,
- },
- pre: Tree{
- TreeNode: Node{
- Children: Children{
- query.TreeNodeChildKey{`suite`, query.Suite}: {
- Query: Q(`suite`),
- Children: Children{
- query.TreeNodeChildKey{`a`, query.Files}: {
- Query: Q(`suite:a`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Files}: {
- Query: Q(`suite:a,*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- },
- post: Tree{
- TreeNode: Node{
- Children: Children{
- query.TreeNodeChildKey{`suite`, query.Suite}: {
- Query: Q(`suite`),
- Children: Children{
- query.TreeNodeChildKey{`a`, query.Files}: {
- Query: Q(`suite:a`),
- Children: Children{
- query.TreeNodeChildKey{`b`, query.Files}: {
- Query: Q(`suite:a,b`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Tests}: {
- Query: Q(`suite:a,b:*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- { /////////////////////////////////////////////////////////////////////
- in: QueryData{
- Query: Q(`suite:a:*`),
- Data: pass,
- },
- pre: Tree{
- TreeNode: Node{
- Children: Children{
- query.TreeNodeChildKey{`suite`, query.Suite}: {
- Query: Q(`suite`),
- Children: Children{
- query.TreeNodeChildKey{`a`, query.Files}: {
- Query: Q(`suite:a`),
- Children: Children{
- query.TreeNodeChildKey{`b`, query.Files}: {
- Query: Q(`suite:a,b`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Files}: {
- Query: Q(`suite:a,b,*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- post: Tree{
- TreeNode: Node{
- Children: Children{
- query.TreeNodeChildKey{`suite`, query.Suite}: {
- Query: Q(`suite`),
- Children: Children{
- query.TreeNodeChildKey{`a`, query.Files}: {
- Query: Q(`suite:a`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Tests}: {
- Query: Q(`suite:a:*`),
- Data: &pass,
- },
- query.TreeNodeChildKey{`b`, query.Files}: {
- Query: Q(`suite:a,b`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Files}: {
- Query: Q(`suite:a,b,*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- { /////////////////////////////////////////////////////////////////////
- in: QueryData{
- Query: Q(`suite:a,b:c:*`),
- Data: pass,
- },
- pre: Tree{
- TreeNode: Node{
- Children: Children{
- query.TreeNodeChildKey{`suite`, query.Suite}: {
- Query: Q(`suite`),
- Children: Children{
- query.TreeNodeChildKey{`a`, query.Files}: {
- Query: Q(`suite:a`),
- Children: Children{
- query.TreeNodeChildKey{`b`, query.Files}: {
- Query: Q(`suite:a,b`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Tests}: {
- Query: Q(`suite:a,b:*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- post: Tree{
- TreeNode: Node{
- Children: Children{
- query.TreeNodeChildKey{`suite`, query.Suite}: {
- Query: Q(`suite`),
- Children: Children{
- query.TreeNodeChildKey{`a`, query.Files}: {
- Query: Q(`suite:a`),
- Children: Children{
- query.TreeNodeChildKey{`b`, query.Files}: {
- Query: Q(`suite:a,b`),
- Children: Children{
- query.TreeNodeChildKey{`c`, query.Tests}: {
- Query: Q(`suite:a,b:c`),
- Children: Children{
- query.TreeNodeChildKey{`*`, query.Cases}: {
- Query: Q(`suite:a,b:c:*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- } {
- tree := test.pre
- if err := tree.Split(test.in.Query, test.in.Data); err != nil {
- t.Errorf("NewTree(%v): %v", test.in, err)
- continue
- }
- if diff := cmp.Diff(tree, test.post); diff != "" {
- t.Errorf("Split(%v) tree was not as expected:\n%v", test.in, diff)
- }
- }
-}
-
-func TestList(t *testing.T) {
- type QueryData = query.QueryData[string]
-
- tree, err := NewTree(t,
- QueryData{Query: Q(`suite:*`), Data: skip},
- QueryData{Query: Q(`suite:a,*`), Data: failure},
- QueryData{Query: Q(`suite:a,b,*`), Data: failure},
- QueryData{Query: Q(`suite:a,b:c:*`), Data: failure},
- QueryData{Query: Q(`suite:a,b:c:d;*`), Data: failure},
- QueryData{Query: Q(`suite:a,b:c:d="e";*`), Data: failure},
- QueryData{Query: Q(`suite:h,b:c:f="g";*`), Data: abort},
- QueryData{Query: Q(`suite:a,b:c:f="g";*`), Data: skip},
- )
- if err != nil {
- t.Fatalf("NewTree() returned %v", err)
- }
-
- got := tree.List()
- expect := []QueryData{
- {Query: Q(`suite:*`), Data: skip},
- {Query: Q(`suite:a,*`), Data: failure},
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,b:c:*`), Data: failure},
- {Query: Q(`suite:a,b:c:d;*`), Data: failure},
- {Query: Q(`suite:a,b:c:d="e";*`), Data: failure},
- {Query: Q(`suite:a,b:c:f="g";*`), Data: skip},
- {Query: Q(`suite:h,b:c:f="g";*`), Data: abort},
- }
- if diff := cmp.Diff(got, expect); diff != "" {
- t.Errorf("List() was not as expected:\n%v", diff)
- }
-}
-
-// reducer is used by Reduce() and ReduceUnder() tests for reducing the tree.
-// reducer returns a pointer to the common string if all strings in data are
-// equal, otherwise returns nil
-func reducer(data []string) *string {
- if s := container.NewSet(data...); len(s) == 1 {
- item := s.One()
- return &item
- }
- return nil
-}
-
-func TestReduce(t *testing.T) {
- type QueryData = query.QueryData[string]
-
- type Test struct {
- name string
- in []QueryData
- expect []QueryData
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- name: "Different file results - A",
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Different file results - B",
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: pass},
- {Query: Q(`suite:a,d,*`), Data: skip},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: pass},
- {Query: Q(`suite:a,d,*`), Data: skip},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Different test results",
- in: []QueryData{
- {Query: Q(`suite:a,b:*`), Data: failure},
- {Query: Q(`suite:a,c:*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,b:*`), Data: failure},
- {Query: Q(`suite:a,c:*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Same file results",
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: failure},
- },
- expect: []QueryData{
- {Query: Q(`suite:*`), Data: failure},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Same test results",
- in: []QueryData{
- {Query: Q(`suite:a,b:*`), Data: failure},
- {Query: Q(`suite:a,c:*`), Data: failure},
- },
- expect: []QueryData{
- {Query: Q(`suite:*`), Data: failure},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "File vs test",
- in: []QueryData{
- {Query: Q(`suite:a:b,c*`), Data: failure},
- {Query: Q(`suite:a,b,c*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,*`), Data: pass},
- {Query: Q(`suite:a:*`), Data: failure},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Sibling cases, no reduce",
- in: []QueryData{
- {Query: Q(`suite:a:b:c;d=e;f=g;*`), Data: failure},
- {Query: Q(`suite:a:b:c;d=e;f=h;*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a:b:c;d=e;f=g;*`), Data: failure},
- {Query: Q(`suite:a:b:c;d=e;f=h;*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Sibling cases, reduce to test",
- in: []QueryData{
- {Query: Q(`suite:a:b:c=1;d="x";*`), Data: failure},
- {Query: Q(`suite:a:b:c=1;d="y";*`), Data: failure},
- {Query: Q(`suite:a:z:*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a:b:*`), Data: failure},
- {Query: Q(`suite:a:z:*`), Data: pass},
- },
- },
- } {
- tree, err := NewTree(t, test.in...)
- if err != nil {
- t.Errorf("Test '%v':\nNewTree() returned %v", test.name, err)
- continue
- }
- tree.Reduce(reducer)
- results := tree.List()
- if diff := cmp.Diff(results, test.expect); diff != "" {
- t.Errorf("Test '%v':\n%v", test.name, diff)
- }
- }
-}
-
-func TestReduceUnder(t *testing.T) {
- type QueryData = query.QueryData[string]
-
- type Test struct {
- location string
- to query.Query
- in []QueryData
- expect []QueryData
- expectErr error
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:a,b,*`),
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:a,*`),
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,*`), Data: failure},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:*`),
- in: []QueryData{
- {Query: Q(`suite:a,b:*`), Data: failure},
- },
- expect: []QueryData{
- {Query: Q(`suite:*`), Data: failure},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:a,*`),
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:a,*`),
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: pass},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:a`),
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: pass},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:x`),
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: pass},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: pass},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- expectErr: query.ErrNoDataForQuery{
- Query: Q(`suite:x`),
- },
- },
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- to: Q(`suite:a,b,c,*`),
- in: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: pass},
- },
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: pass},
- },
- expectErr: query.ErrNoDataForQuery{
- Query: Q(`suite:a,b,c`),
- },
- },
- } {
- tree, err := NewTree(t, test.in...)
- if err != nil {
- t.Errorf("\n%v NewTree(): %v", test.location, err)
- continue
- }
- err = tree.ReduceUnder(test.to, reducer)
- if diff := cmp.Diff(err, test.expectErr); diff != "" {
- t.Errorf("\n%v ReduceUnder(): %v", test.location, err)
- }
- results := tree.List()
- if diff := cmp.Diff(results, test.expect); diff != "" {
- t.Errorf("\n%v List(): %v", test.location, diff)
- }
- }
-}
-
-func TestReplace(t *testing.T) {
- type QueryData = query.QueryData[string]
-
- type Test struct {
- name string
- base []QueryData
- replacement QueryData
- expect []QueryData
- expectErr error
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- name: "Replace file. Direct",
- base: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: failure},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- replacement: QueryData{Q(`suite:a,b,*`), skip},
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: skip},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Replace file. Indirect",
- base: []QueryData{
- {Query: Q(`suite:a,b,c,*`), Data: failure},
- {Query: Q(`suite:a,b,d,*`), Data: pass},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- replacement: QueryData{Q(`suite:a,b,*`), skip},
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: skip},
- {Query: Q(`suite:a,c,*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "File vs Test",
- base: []QueryData{
- {Query: Q(`suite:a,b:c,*`), Data: crash},
- {Query: Q(`suite:a,b:d,*`), Data: abort},
- {Query: Q(`suite:a,b,c,*`), Data: failure},
- {Query: Q(`suite:a,b,d,*`), Data: pass},
- },
- replacement: QueryData{Q(`suite:a,b,*`), skip},
- expect: []QueryData{
- {Query: Q(`suite:a,b,*`), Data: skip},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Cases. * with *",
- base: []QueryData{
- {Query: Q(`suite:file:test:*`), Data: failure},
- },
- replacement: QueryData{Q(`suite:file:test:*`), pass},
- expect: []QueryData{
- {Query: Q(`suite:file:test:*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Cases. Mixed with *",
- base: []QueryData{
- {Query: Q(`suite:file:test:a=1,*`), Data: failure},
- {Query: Q(`suite:file:test:a=2,*`), Data: skip},
- {Query: Q(`suite:file:test:a=3,*`), Data: crash},
- },
- replacement: QueryData{Q(`suite:file:test:*`), pass},
- expect: []QueryData{
- {Query: Q(`suite:file:test:*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Cases. Replace partial - (a=1)",
- base: []QueryData{
- {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure},
- {Query: Q(`suite:file:test:a=1;b=y;*`), Data: failure},
- {Query: Q(`suite:file:test:a=2;b=y;*`), Data: failure},
- },
- replacement: QueryData{Q(`suite:file:test:a=1;*`), pass},
- expect: []QueryData{
- {Query: Q(`suite:file:test:a=1;*`), Data: pass},
- {Query: Q(`suite:file:test:a=2;b=y;*`), Data: failure},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Cases. Replace partial - (b=y)",
- base: []QueryData{
- {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure},
- {Query: Q(`suite:file:test:a=1;b=y;*`), Data: failure},
- {Query: Q(`suite:file:test:a=2;b=y;*`), Data: failure},
- },
- replacement: QueryData{Q(`suite:file:test:b=y;*`), pass},
- expect: []QueryData{
- {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure},
- {Query: Q(`suite:file:test:b=y;*`), Data: pass},
- },
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Error. No data for query - short",
- base: []QueryData{
- {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure},
- },
- replacement: QueryData{Q(`suite:missing:*`), pass},
- expect: []QueryData{
- {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure},
- },
- expectErr: query.ErrNoDataForQuery{Q(`suite:missing`)},
- },
- { //////////////////////////////////////////////////////////////////////
- name: "Error. No data for query - long",
- base: []QueryData{
- {Query: Q(`suite:file:test:*`), Data: failure},
- },
- replacement: QueryData{Q(`suite:file:test,missing,*`), pass},
- expect: []QueryData{
- {Query: Q(`suite:file:test:*`), Data: failure},
- },
- expectErr: query.ErrNoDataForQuery{Q(`suite:file:test,missing`)},
- },
- } {
- tree, err := NewTree(t, test.base...)
- if err != nil {
- t.Errorf("Test '%v':\nNewTree(): %v", test.name, err)
- continue
- }
- err = tree.Replace(test.replacement.Query, test.replacement.Data)
- if diff := cmp.Diff(err, test.expectErr); diff != "" {
- t.Errorf("Test '%v':\nReplace() error: %v", test.name, err)
- continue
- }
- if diff := cmp.Diff(tree.List(), test.expect); diff != "" {
- t.Errorf("Test '%v':\n%v", test.name, diff)
- }
- }
-}
-
func TestGlob(t *testing.T) {
type QueryData = query.QueryData[string]
diff --git a/tools/src/cts/result/mvt.go b/tools/src/cts/result/mvt.go
deleted file mode 100644
index 32059a6..0000000
--- a/tools/src/cts/result/mvt.go
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright 2022 The Dawn & Tint Authors
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-//
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution.
-//
-// 3. Neither the name of the copyright holder nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-package result
-
-import (
- "sort"
-
- "dawn.googlesource.com/dawn/tools/src/cts/query"
-)
-
-// MinimalVariantTags accepts a list of tag-sets (e.g GPU tags, OS tags, etc),
-// and returns an optimized list of variants, folding together variants that
-// have identical result query-to-status mappings, and removing redundant tags.
-//
-// MinimalVariantTags will attempt to remove variant tags starting with the
-// first set of tags in tagSets, then second, and so on. If a tag-set cannot
-// be removed, then the tags of the set are left alone, and the algorithm will
-// progress to the next tag-set.
-//
-// MinimalVariantTags assumes that there are no duplicate results (same query,
-// same tags) in l.
-func (l List) MinimalVariantTags(tagSets []Tags) []Variant {
- type VariantData struct {
- // The variant tags
- tags Variant
- // The query -> status for all results in l that have this variant's
- // tags.
- queryToStatus map[query.Query]Status
- }
-
- variants := []VariantData{}
-
- // Build the initial list of variants from l.
- // Bin result [query -> status] to the variant.
- {
- variantIndices := map[string]int{}
- for _, r := range l {
- key := TagsToString(r.Tags)
- if idx, found := variantIndices[key]; !found {
- variantIndices[key] = len(variants)
- variants = append(variants, VariantData{
- tags: Variant(r.Tags.Clone()),
- queryToStatus: map[query.Query]Status{
- r.Query: r.Status,
- },
- })
- } else {
- variants[idx].queryToStatus[r.Query] = r.Status
- }
- }
- }
-
- // canReduce checks that the variant would match the same results if the
- // tags were reduced to 'tags'. Returns true if the variant's tags could
- // be reduced, otherwise false.
- canReduce := func(variant VariantData, tags Tags) bool {
- for _, r := range l.FilterByTags(tags) {
- existing, found := variant.queryToStatus[r.Query]
- if !found {
- // Removing the tag has expanded the set of queries.
- return false
- }
- if existing != r.Status {
- // Removing the tag has resulted in two queries with different
- // results.
- return false
- }
- }
- return true
- }
-
- // tryToRemoveTags will remove all the tags in 'tags' from all variants
- // iff doing so does not affect the set of results filtered by each variant.
- // If it was possible to remove the tags, then variants that now have the
- // same tags may be folded together, reducing the total number of variants.
- tryToRemoveTags := func(tags Tags) {
- newVariants := make([]VariantData, 0, len(variants))
-
- for _, v := range variants {
- // Does the variant even contain these tags?
- if !v.tags.ContainsAny(tags) {
- // Nope. Skip the canReduce() call, and keep the variant.
- newVariants = append(newVariants, v)
- continue
- }
-
- // Build the new set of tags with 'tags' removed.
- newTags := v.tags.Clone()
- newTags.RemoveAll(tags)
-
- // Check wether removal of these tags affected the outcome.
- if !canReduce(v, newTags) {
- // Removing these tags resulted in differences.
- return // Abort
- }
- newVariants = append(newVariants, VariantData{newTags, v.queryToStatus})
- }
-
- // Remove variants that are now subsets of others.
- // Start by sorting the variants by number of tags.
- // This ensures that the variants with fewer tags (fewer constraints)
- // come first.
- sort.Slice(newVariants, func(i, j int) bool {
- return len(newVariants[i].tags) < len(newVariants[j].tags)
- })
-
- // Now check each variant's tags against the previous variant tags.
- // As we've sorted, we know that supersets (fewer-tags) come before
- // subsets (more-tags).
- variants = []VariantData{}
-
- nextVariant:
- for i, v1 := range newVariants { // for variants 0..N
- for _, v2 := range newVariants[:i] { // for variants 0..i
- if v1.tags.ContainsAll(v2.tags) {
- continue nextVariant // v1 is a subset of v2. Omit.
- }
- }
- variants = append(variants, v1)
- }
- }
-
- // Attempt to remove the tag sets from the variants, one by one.
- for _, tags := range tagSets {
- tryToRemoveTags(tags)
- }
-
- // Return the final set of unique variants
- out := make([]Variant, len(variants))
- for i, v := range variants {
- out[i] = v.tags
- }
- return out
-}
diff --git a/tools/src/cts/result/mvt_test.go b/tools/src/cts/result/mvt_test.go
deleted file mode 100644
index d1df908..0000000
--- a/tools/src/cts/result/mvt_test.go
+++ /dev/null
@@ -1,130 +0,0 @@
-// Copyright 2022 The Dawn & Tint Authors
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-//
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution.
-//
-// 3. Neither the name of the copyright holder nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-package result_test
-
-import (
- "fmt"
- "testing"
-
- "dawn.googlesource.com/dawn/tools/src/cts/result"
- "dawn.googlesource.com/dawn/tools/src/fileutils"
- "github.com/google/go-cmp/cmp"
-)
-
-func TestMinimalVariantTags(t *testing.T) {
- type Test struct {
- location string
- results result.List
- expect []result.Variant
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- location: fileutils.ThisLine(),
- results: result.List{},
- expect: []result.Variant{},
- }, { ///////////////////////////////////////////////////////////////////
- // Single variant, that can be entirely optimized away
- location: fileutils.ThisLine(),
- results: result.List{
- {Query: Q("a:b,c:d,*"), Tags: T("a0", "b1", "c2"), Status: result.Pass},
- },
- expect: []result.Variant{T()},
- }, { ///////////////////////////////////////////////////////////////////
- // Multiple variants on the same query.
- // Can also be entirely optimized away.
- location: fileutils.ThisLine(),
- results: result.List{
- {Query: Q("a:b,c:d,*"), Tags: T("a0", "b1", "c2"), Status: result.Pass},
- {Query: Q("a:b,c:d,*"), Tags: T("a1", "b2", "c0"), Status: result.Pass},
- {Query: Q("a:b,c:d,*"), Tags: T("a2", "b1", "c0"), Status: result.Pass},
- },
- expect: []result.Variant{T()},
- }, { ///////////////////////////////////////////////////////////////////
- // Two variants where the 1st and 2nd tag-sets are redundant.
- location: fileutils.ThisLine(),
- results: result.List{
- {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass},
- {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c1"), Status: result.Failure},
- },
- expect: []result.Variant{T("c0"), T("c1")},
- }, { ///////////////////////////////////////////////////////////////////
- // Two variants where the 1st and 3rd tag-sets are redundant.
- location: fileutils.ThisLine(),
- results: result.List{
- {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass},
- {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c1"), Status: result.Failure},
- {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c1"), Status: result.Pass},
- {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c0"), Status: result.Failure},
- },
- expect: []result.Variant{T("b0"), T("b1")},
- }, { ///////////////////////////////////////////////////////////////////
- // Two variants where the 2nd and 3rd tag-sets are redundant.
- location: fileutils.ThisLine(),
- results: result.List{
- {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass},
- {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c1"), Status: result.Failure},
- {Query: Q("a:b,c:d,*"), Tags: T("a0", "b1", "c1"), Status: result.Pass},
- {Query: Q("a:b,c:d,*"), Tags: T("a1", "b0", "c0"), Status: result.Failure},
- },
- expect: []result.Variant{T("a0"), T("a1")},
- }, { ///////////////////////////////////////////////////////////////////
- // Check that variants aren't optimized to expand the set of results
- // they target, even if results are uniform
- location: fileutils.ThisLine(),
- results: result.List{
- {Query: Q("a:b,c:d0,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass},
- {Query: Q("a:b,c:d1,*"), Tags: T("a1", "b1", "c1"), Status: result.Pass},
- },
- expect: []result.Variant{T("c0"), T("c1")},
- }, { ///////////////////////////////////////////////////////////////////
- // Exercise the optimizations to skip checks on tag removals that
- // aren't found in all variants
- location: fileutils.ThisLine(),
- results: result.List{
- {Query: Q("a:b,c:d0,*"), Tags: T("a0"), Status: result.Pass},
- {Query: Q("a:b,c:d1,*"), Tags: T("b0"), Status: result.Pass},
- {Query: Q("a:b,c:d2,*"), Tags: T("c0"), Status: result.Pass},
- },
- expect: []result.Variant{T("a0"), T("b0"), T("c0")},
- },
- } {
- preReduce := fmt.Sprint(test.results)
- got := test.results.MinimalVariantTags([]result.Tags{
- T("a0", "a1", "a2"),
- T("b0", "b1", "b2"),
- T("c0", "c1", "c2"),
- })
- postReduce := fmt.Sprint(test.results)
- if diff := cmp.Diff(got, test.expect); diff != "" {
- t.Errorf("%v MinimalVariantTags() diff:\n%v", test.location, diff)
- }
- if diff := cmp.Diff(preReduce, postReduce); diff != "" {
- t.Errorf("%v MinimalVariantTags() modified original list:\n%v", test.location, diff)
- }
- }
-}
diff --git a/tools/src/cts/result/result.go b/tools/src/cts/result/result.go
index 9f24b9b..a3a9c7d 100644
--- a/tools/src/cts/result/result.go
+++ b/tools/src/cts/result/result.go
@@ -186,19 +186,6 @@
// Lists of test results by execution mode.
type ResultsByExecutionMode map[ExecutionMode]List
-// Variant is a collection of tags that uniquely identify a test
-// configuration (e.g the combination of OS, GPU, validation-modes, etc).
-type Variant = Tags
-
-// Variants returns the list of unique tags (variants) across all results.
-func (l List) Variants() []Variant {
- tags := container.NewMap[string, Variant]()
- for _, r := range l {
- tags.Add(TagsToString(r.Tags), r.Tags)
- }
- return tags.Values()
-}
-
// TransformTags returns the list of results with the tags transformed using f.
// TransformTags assumes that f will return the same output for the same input.
func (l List) TransformTags(f func(Tags) Tags) List {
@@ -328,14 +315,6 @@
})
}
-// FilterByVariant returns the results that exactly match the given tags
-func (l List) FilterByVariant(tags Tags) List {
- str := TagsToString(tags)
- return l.Filter(func(r Result) bool {
- return len(r.Tags) == len(tags) && TagsToString(r.Tags) == str
- })
-}
-
// FilterByQuery returns the results that match the given query
func (l List) FilterByQuery(q query.Query) List {
return l.Filter(func(r Result) bool {
@@ -358,21 +337,6 @@
return set
}
-// StatusTree is a query tree of statuses
-type StatusTree = query.Tree[Status]
-
-// StatusTree returns a query.Tree from the List, with the Status as the tree
-// node data.
-func (l List) StatusTree() (StatusTree, error) {
- tree := StatusTree{}
- for _, r := range l {
- if err := tree.Add(r.Query, r.Status); err != nil {
- return StatusTree{}, err
- }
- }
- return tree, nil
-}
-
// Load loads the result list from the file with the given path
func Load(path string) (ResultsByExecutionMode, error) {
file, err := os.Open(path)
diff --git a/tools/src/cts/result/result_test.go b/tools/src/cts/result/result_test.go
index 28b7007..456348f 100644
--- a/tools/src/cts/result/result_test.go
+++ b/tools/src/cts/result/result_test.go
@@ -116,120 +116,6 @@
}
}
-func TestVariants(t *testing.T) {
- type Test struct {
- results result.List
- expect []result.Tags
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags(),
- },
- },
- expect: []result.Tags{
- result.NewTags(),
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x"),
- },
- },
- expect: []result.Tags{
- result.NewTags("x"),
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- },
- expect: []result.Tags{
- result.NewTags("x", "y"),
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- result.Result{
- Query: Q(`b`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- },
- expect: []result.Tags{
- result.NewTags("x", "y"),
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- result.Result{
- Query: Q(`b`),
- Status: result.Pass,
- Tags: result.NewTags("y", "z"),
- },
- },
- expect: []result.Tags{
- result.NewTags("x", "y"),
- result.NewTags("y", "z"),
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- result.Result{
- Query: Q(`b`),
- Status: result.Pass,
- Tags: result.NewTags("y", "z"),
- },
- result.Result{
- Query: Q(`c`),
- Status: result.Pass,
- Tags: result.NewTags("z", "x"),
- },
- result.Result{
- Query: Q(`d`),
- Status: result.Pass,
- Tags: result.NewTags("y", "z"),
- },
- },
- expect: []result.Tags{
- result.NewTags("x", "y"),
- result.NewTags("x", "z"),
- result.NewTags("y", "z"),
- },
- },
- } {
- got := test.results.Variants()
- if diff := cmp.Diff(got, test.expect); diff != "" {
- t.Errorf("Results:\n%v\nUniqueTags() was not as expected:\n%v", test.results, diff)
- }
- }
-}
-
func TestTransformTags(t *testing.T) {
type Test struct {
results result.List
@@ -832,96 +718,6 @@
}
}
-func TestFilterByVariant(t *testing.T) {
- type Test struct {
- results result.List
- tags result.Tags
- expect result.List
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x"),
- },
- result.Result{
- Query: Q(`b`),
- Status: result.Failure,
- Tags: result.NewTags("y"),
- },
- result.Result{
- Query: Q(`c`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- },
- tags: result.NewTags("x", "y"),
- expect: result.List{
- result.Result{
- Query: Q(`c`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x"),
- },
- result.Result{
- Query: Q(`b`),
- Status: result.Failure,
- Tags: result.NewTags("y"),
- },
- result.Result{
- Query: Q(`c`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- },
- tags: result.NewTags("x"),
- expect: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x"),
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- result.Result{
- Query: Q(`a`),
- Status: result.Pass,
- Tags: result.NewTags("x"),
- },
- result.Result{
- Query: Q(`b`),
- Status: result.Failure,
- Tags: result.NewTags("y"),
- },
- result.Result{
- Query: Q(`c`),
- Status: result.Pass,
- Tags: result.NewTags("x", "y"),
- },
- },
- tags: result.NewTags("q"),
- expect: result.List{},
- },
- } {
- got := test.results.FilterByVariant(test.tags)
- if diff := cmp.Diff(got, test.expect); diff != "" {
- t.Errorf("Results:\n%v\nFilterByVariant(%v) was not as expected:\n%v", test.results, test.tags, diff)
- }
- }
-}
-
func TestStatuses(t *testing.T) {
type Test struct {
results result.List
@@ -992,67 +788,6 @@
}
}
-func TestStatusTree(t *testing.T) {
- type Node = query.TreeNode[result.Status]
- type Children = query.TreeNodeChildren[result.Status]
- type ChildKey = query.TreeNodeChildKey
-
- pass := result.Pass
-
- type Test struct {
- results result.List
- expectErr error
- expect result.StatusTree
- }
- for _, test := range []Test{
- { //////////////////////////////////////////////////////////////////////
- results: result.List{},
- expect: result.StatusTree{},
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- {Query: Q(`suite:a:*`), Status: result.Pass},
- },
- expect: result.StatusTree{
- TreeNode: Node{
- Children: Children{
- ChildKey{Name: `suite`, Target: query.Suite}: &Node{
- Query: Q(`suite`),
- Children: Children{
- ChildKey{Name: `a`, Target: query.Files}: &Node{
- Query: Q(`suite:a`),
- Children: Children{
- ChildKey{Name: `*`, Target: query.Tests}: &Node{
- Query: Q(`suite:a:*`),
- Data: &pass,
- },
- },
- },
- },
- },
- },
- },
- },
- },
- { //////////////////////////////////////////////////////////////////////
- results: result.List{
- {Query: Q(`suite:a:*`), Status: result.Pass},
- {Query: Q(`suite:a:*`), Status: result.Failure},
- },
- expectErr: query.ErrDuplicateData{Query: Q(`suite:a:*`)},
- },
- } {
- got, err := test.results.StatusTree()
- if diff := cmp.Diff(err, test.expectErr); diff != "" {
- t.Errorf("Results:\n%v\nStatusTree() error was not as expected:\n%v", test.results, diff)
- continue
- }
- if diff := cmp.Diff(got, test.expect); diff != "" {
- t.Errorf("Results:\n%v\nStatusTree() was not as expected:\n%v", test.results, diff)
- }
- }
-}
-
func TestReadWrite(t *testing.T) {
in := result.ResultsByExecutionMode{
"bar": result.List{
