tools/src/cts/expectations/update.go

// Copyright 2022 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
//    contributors may be used to endorse or promote products derived from
//    this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package expectations

import (
	"fmt"
	"strings"
	"time"

	"dawn.googlesource.com/dawn/tools/src/cts/result"
)

const (
	// Chunk comment for the AddExpectationsForFailingResults path.
	ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##"
)

// AddExpectationsForFailingResults adds new expectations for the provided
// failing results, with the following assumptions:
//   - The provided results do not have existing expectations
//   - Any expectations for invalid/non-existent tests have already been
//     removed
//
// This will:
//   - Reduce result tags down to either:
//     --- Only the most explicit tag from each set
//     --- Only a few very broad tags
//   - Merge identical results together
//   - Add new expectations to the one mutable chunk that is expected to
//     be present in the file.
//   - Sort the mutable chunk's expectations by test name, then tags.
//
// TODO(crbug.com/372730248): Return diagnostics.
func (c *Content) AddExpectationsForFailingResults(results result.List,
	useExplicitTags bool, verbose bool) error {
	// Make a copy of the results. This code mutates the list.
	results = append(result.List{}, results...)

	startTime := time.Now()
	if err := c.removeUnknownTags(&results); err != nil {
		return err
	}
	if verbose {
		fmt.Printf("Removing unknown tags took %s\n", time.Since(startTime).String())
	}

	startTime = time.Now()
	if useExplicitTags {
		if err := c.reduceTagsToMostExplicitOnly(&results); err != nil {
			return err
		}
	} else {
		if err := c.reduceTagsToBroadGroups(&results); err != nil {
			return err
		}
	}
	if verbose {
		fmt.Printf("Reducing tags took %s\n", time.Since(startTime).String())
	}

	// Merge identical results.
	startTime = time.Now()
	results = result.Merge(results)
	if verbose {
		fmt.Printf("Merging identical results took %s\n", time.Since(startTime).String())
	}

	startTime = time.Now()
	if err := c.addExpectationsToMutableChunk(&results); err != nil {
		return err
	}
	if verbose {
		fmt.Printf("Adding expectations took %s\n", time.Since(startTime).String())
	}
	return nil
}

// removeUnknownTags modifies |results| in place so that the Results contained
// within it only use tags that the Content is aware of.
func (c *Content) removeUnknownTags(results *result.List) error {
	*results = results.TransformTags(func(t result.Tags) result.Tags {
		filtered := result.NewTags()
		for tag := range t {
			if _, ok := c.Tags.ByName[tag]; ok {
				filtered.Add(tag)
			}
		}
		return filtered
	})
	return nil
}

// reduceTagsToMostExplicitOnly modifies the given results argument in place
// so that all contained results' tag sets only contain the most explicit tags
// based on the known tag sets contained within the Content.
func (c *Content) reduceTagsToMostExplicitOnly(results *result.List) error {
	for i, res := range *results {
		res.Tags = c.Tags.RemoveLowerPriorityTags(res.Tags)
		(*results)[i] = res
	}
	return nil
}

// reduceTagsToBroadGroups modifies the given results argument in place so that
// all contained results' tag sets only contain several broad but important
// tags.
func (c *Content) reduceTagsToBroadGroups(results *result.List) error {

	osTagSet, gpuTagSet, deviceTagSet, err := c.getBroadTagSets()
	if err != nil {
		return err
	}

	for i, res := range *results {
		// Do an initial culling so only the broad tags are left.
		lowPriorityTags := c.Tags.RemoveHigherPriorityTags(res.Tags)
		chosenTags := result.NewTags()

		// If this is a remote platform, only use the device tag since OS type +
		// GPU vendor can be inferred.
		chosenDeviceTag, err := getAndAssertOneOverlappingTag(deviceTagSet, lowPriorityTags)
		if err == nil {
			chosenTags.Add(chosenDeviceTag)
		} else {
			chosenOsTag, err := getAndAssertOneOverlappingTag(osTagSet, lowPriorityTags)
			if err != nil {
				return err
			}
			chosenTags.Add(chosenOsTag)

			chosenGpuTag, err := getAndAssertOneOverlappingTag(gpuTagSet, lowPriorityTags)
			if err != nil {
				return err
			}
			chosenTags.Add(chosenGpuTag)
		}

		// Modify the result's tags in place.
		res.Tags = chosenTags
		(*results)[i] = res
	}

	return nil
}

// getAndAssertOneOverlappingTag asserts that there is a single tag that is
// shared between the given tags and returns it.
func getAndAssertOneOverlappingTag(validTags, allTags result.Tags) (string, error) {
	overlappingTags := validTags.Intersection(allTags)
	if len(overlappingTags) != 1 {
		return "", fmt.Errorf("Tags %v did not have exactly one overlap with %v", validTags, allTags)
	}
	return overlappingTags.One(), nil
}

// getBroadTagSets retrieves the Content's tag sets which correspond to the three
// primary broad tags: OS, GPU, and device type.
func (c *Content) getBroadTagSets() (result.Tags, result.Tags, result.Tags, error) {
	// Determine which tag sets are the ones we care about. We do this by looking
	// for the tag set that contains one of the well-known tags of that type, e.g.
	// "android", "linux", "mac", or "win" for OS.
	var osTagSet, gpuTagSet, deviceTagSet result.Tags
	// TODO(crbug.com/344014313): Move these known tags into a JSON configuration
	// file for easy modification once dependency injection is available to make
	// testing possible.
	knownOsTags := result.NewTags("android", "linux", "mac", "win")
	knownGpuTags := result.NewTags("amd", "intel", "nvidia", "qualcomm")
	knownDeviceTags := result.NewTags("android-pixel-4", "android-pixel-6",
		"chromeos-board-amd64-generic", "fuchsia-board-qemu-x64")
	for _, tagSet := range c.Tags.Sets {
		if tagSet.Tags.ContainsAny(knownOsTags) {
			osTagSet = tagSet.Tags
		} else if tagSet.Tags.ContainsAny(knownGpuTags) {
			gpuTagSet = tagSet.Tags
		} else if tagSet.Tags.ContainsAny(knownDeviceTags) {
			deviceTagSet = tagSet.Tags
		}
	}

	if osTagSet == nil || gpuTagSet == nil || deviceTagSet == nil {
		return nil, nil, nil, fmt.Errorf("Did not find expected tag sets. Known tags may be out of date. osTagSet: %v, gpuTagSet: %v, deviceTagSet: %v",
			osTagSet, gpuTagSet, deviceTagSet)
	}

	return osTagSet, gpuTagSet, deviceTagSet, nil
}

// addExpectationsToMutableChunk adds expectations for the results contained
// within |results| to the one mutable chunk that  should be in the Content.
// If not found, a new one will be created at the end of the Content.
func (c *Content) addExpectationsToMutableChunk(results *result.List) error {
	// Find the mutable chunk.
	// Chunks are considered immutable by default, unless annotated as
	// ROLLER_AUTOGENERATED_FAILURES.
	mutableTokens := []string{
		ROLLER_AUTOGENERATED_FAILURES,
	}

	// Filter out chunks that do not contain any of the strings in mutableTokens in the comments
	mutableChunkIndices := []int{}
	for i, chunk := range c.Chunks {
		immutable := true

	comments:
		for _, line := range chunk.Comments {
			for _, token := range mutableTokens {
				if strings.Contains(line, token) {
					immutable = false
					break comments
				}
			}
		}

		if !immutable {
			mutableChunkIndices = append(mutableChunkIndices, i)
		}
	}

	var chunkToModify *Chunk
	if len(mutableChunkIndices) > 1 {
		return fmt.Errorf("Expected 1 mutable chunk, found %d", len(mutableChunkIndices))
	} else if len(mutableChunkIndices) == 0 {
		newChunk := Chunk{}
		newChunk.Comments = []string{
			"################################################################################",
			"# Autogenerated Failure expectations. Please triage.",
			ROLLER_AUTOGENERATED_FAILURES,
			"################################################################################",
		}
		c.Chunks = append(c.Chunks, newChunk)
		chunkToModify = &(c.Chunks[len(c.Chunks)-1])
	} else {
		chunkToModify = &(c.Chunks[mutableChunkIndices[0]])
	}

	// Add the new expectations to the mutable chunk.
	for _, res := range *results {
		expectation := Expectation{
			Bug:   "crbug.com/0000",
			Tags:  res.Tags,
			Query: res.Query.ExpectationFileString(),
			Status: []string{
				"Failure",
			},
		}
		chunkToModify.Expectations = append(chunkToModify.Expectations, expectation)
	}

	// Sort the mutable chunk's expectations.
	chunkToModify.Expectations.SortPrioritizeQuery()
	return nil
}