tools/src/cts/expectations/update.go - dawn - Git at Google

 // Copyright 2022 The Dawn & Tint Authors
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
 // 1. Redistributions of source code must retain the above copyright notice, this
 //    list of conditions and the following disclaimer.
 //
 // 2. Redistributions in binary form must reproduce the above copyright notice,
 //    this list of conditions and the following disclaimer in the documentation
 //    and/or other materials provided with the distribution.
 //
 // 3. Neither the name of the copyright holder nor the names of its
 //    contributors may be used to endorse or promote products derived from
 //    this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 package expectations

 import (
 	"fmt"
 	"strings"
 	"time"

 	"dawn.googlesource.com/dawn/tools/src/cts/result"
 )

 const (
 	// Chunk comment for the AddExpectationsForFailingResults path.
 	ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##"
 )

 // AddExpectationsForFailingResults adds new expectations for the provided
 // failing results, with the following assumptions:
 //   - The provided results do not have existing expectations
 //   - Any expectations for invalid/non-existent tests have already been
 //     removed
 //
 // This will:
 //   - Reduce result tags down to either:
 //     --- Only the most explicit tag from each set
 //     --- Only a few very broad tags
 //   - Merge identical results together
 //   - Add new expectations to the one mutable chunk that is expected to
 //     be present in the file.
 //   - Sort the mutable chunk's expectations by test name, then tags.
 //
 // TODO(crbug.com/372730248): Return diagnostics.
 func (c *Content) AddExpectationsForFailingResults(results result.List,
 	use_explicit_tags bool, verbose bool) error {
 	// Make a copy of the results. This code mutates the list.
 	results = append(result.List{}, results...)

 	startTime := time.Now()
 	if err := c.removeUnknownTags(&results); err != nil {
 		return err
 	}
 	if verbose {
 		fmt.Printf("Removing unknown tags took %s\n", time.Now().Sub(startTime).String())
 	}

 	startTime = time.Now()
 	if use_explicit_tags {
 		if err := c.reduceTagsToMostExplicitOnly(&results); err != nil {
 			return err
 		}
 	} else {
 		if err := c.reduceTagsToBroadGroups(&results); err != nil {
 			return err
 		}
 	}
 	if verbose {
 		fmt.Printf("Reducing tags took %s\n", time.Now().Sub(startTime).String())
 	}

 	// Merge identical results.
 	startTime = time.Now()
 	results = result.Merge(results)
 	if verbose {
 		fmt.Printf("Merging identical results took %s\n", time.Now().Sub(startTime).String())
 	}

 	startTime = time.Now()
 	if err := c.addExpectationsToMutableChunk(&results); err != nil {
 		return err
 	}
 	if verbose {
 		fmt.Printf("Adding expectations took %s\n", time.Now().Sub(startTime).String())
 	}
 	return nil
 }

 // removeUnknownTags modifies |results| in place so that the Results contained
 // within it only use tags that the Content is aware of.
 func (c *Content) removeUnknownTags(results *result.List) error {
 	*results = results.TransformTags(func(t result.Tags) result.Tags {
 		filtered := result.NewTags()
 		for tag := range t {
 			if _, ok := c.Tags.ByName[tag]; ok {
 				filtered.Add(tag)
 			}
 		}
 		return filtered
 	})
 	return nil
 }

 // reduceTagsToMostExplicitOnly modifies the given results argument in place
 // so that all contained results' tag sets only contain the most explicit tags
 // based on the known tag sets contained within the Content.
 func (c *Content) reduceTagsToMostExplicitOnly(results *result.List) error {
 	for i, res := range *results {
 		res.Tags = c.Tags.RemoveLowerPriorityTags(res.Tags)
 		(*results)[i] = res
 	}
 	return nil
 }

 // reduceTagsToBroadGroups modifies the given results argument in place so that
 // all contained results' tag sets only contain several broad but important
 // tags.
 func (c *Content) reduceTagsToBroadGroups(results *result.List) error {

 	osTagSet, gpuTagSet, deviceTagSet, err := c.getBroadTagSets()
 	if err != nil {
 		return err
 	}

 	for i, res := range *results {
 		// Do an initial culling so only the broad tags are left.
 		lowPriorityTags := c.Tags.RemoveHigherPriorityTags(res.Tags)
 		chosenTags := result.NewTags()

 		// If this is a remote platform, only use the device tag since OS type +
 		// GPU vendor can be inferred.
 		chosenDeviceTag, err := getAndAssertOneOverlappingTag(deviceTagSet, lowPriorityTags)
 		if err == nil {
 			chosenTags.Add(chosenDeviceTag)
 		} else {
 			chosenOsTag, err := getAndAssertOneOverlappingTag(osTagSet, lowPriorityTags)
 			if err != nil {
 				return err
 			}
 			chosenTags.Add(chosenOsTag)

 			chosenGpuTag, err := getAndAssertOneOverlappingTag(gpuTagSet, lowPriorityTags)
 			if err != nil {
 				return err
 			}
 			chosenTags.Add(chosenGpuTag)
 		}

 		// Modify the result's tags in place.
 		res.Tags = chosenTags
 		(*results)[i] = res
 	}

 	return nil
 }

 // getAndAssertOneOverlappingTag asserts that there is a single tag that is
 // shared between the given tags and returns it.
 func getAndAssertOneOverlappingTag(validTags, allTags result.Tags) (string, error) {
 	overlappingTags := validTags.Intersection(allTags)
 	if len(overlappingTags) != 1 {
 		return "", fmt.Errorf("Tags %v did not have exactly one overlap with %v", validTags, allTags)
 	}
 	return overlappingTags.One(), nil
 }

 // getBroadTagSets retrieves the Content's tag sets which correspond to the three
 // primary broad tags: OS, GPU, and device type.
 func (c *Content) getBroadTagSets() (result.Tags, result.Tags, result.Tags, error) {
 	// Determine which tag sets are the ones we care about. We do this by looking
 	// for the tag set that contains one of the well-known tags of that type, e.g.
 	// "android", "linux", "mac", or "win" for OS.
 	var osTagSet, gpuTagSet, deviceTagSet result.Tags
 	// TODO(crbug.com/344014313): Move these known tags into a JSON configuration
 	// file for easy modification once dependency injection is available to make
 	// testing possible.
 	knownOsTags := result.NewTags("android", "linux", "mac", "win")
 	knownGpuTags := result.NewTags("amd", "intel", "nvidia", "qualcomm")
 	knownDeviceTags := result.NewTags("android-pixel-4", "android-pixel-6",
 		"chromeos-board-amd64-generic", "fuchsia-board-qemu-x64")
 	for _, tagSet := range c.Tags.Sets {
 		if tagSet.Tags.ContainsAny(knownOsTags) {
 			osTagSet = tagSet.Tags
 		} else if tagSet.Tags.ContainsAny(knownGpuTags) {
 			gpuTagSet = tagSet.Tags
 		} else if tagSet.Tags.ContainsAny(knownDeviceTags) {
 			deviceTagSet = tagSet.Tags
 		}
 	}

 	if osTagSet == nil || gpuTagSet == nil || deviceTagSet == nil {
 		return nil, nil, nil, fmt.Errorf("Did not find expected tag sets. Known tags may be out of date. osTagSet: %v, gpuTagSet: %v, deviceTagSet: %v",
 			osTagSet, gpuTagSet, deviceTagSet)
 	}

 	return osTagSet, gpuTagSet, deviceTagSet, nil
 }

 // addExpectationsToMutableChunk adds expectations for the results contained
 // within |results| to the one mutable chunk that  should be in the Content.
 // If not found, a new one will be created at the end of the Content.
 func (c *Content) addExpectationsToMutableChunk(results *result.List) error {
 	// Find the mutable chunk.
 	// Chunks are considered immutable by default, unless annotated as
 	// ROLLER_AUTOGENERATED_FAILURES.
 	mutableTokens := []string{
 		ROLLER_AUTOGENERATED_FAILURES,
 	}

 	// Bin the chunks into those that contain any of the strings in
 	// mutableTokens in the comments and those that do not have these strings.
 	immutableChunkIndicies, mutableChunkIndices := []int{}, []int{}
 	for i, chunk := range c.Chunks {
 		immutable := true

 	comments:
 		for _, line := range chunk.Comments {
 			for _, token := range mutableTokens {
 				if strings.Contains(line, token) {
 					immutable = false
 					break comments
 				}
 			}
 		}

 		if immutable {
 			immutableChunkIndicies = append(immutableChunkIndicies, i)
 		} else {
 			mutableChunkIndices = append(mutableChunkIndices, i)
 		}
 	}

 	var chunkToModify *Chunk
 	if len(mutableChunkIndices) > 1 {
 		return fmt.Errorf("Expected 1 mutable chunk, found %d", len(mutableChunkIndices))
 	} else if len(mutableChunkIndices) == 0 {
 		newChunk := Chunk{}
 		newChunk.Comments = []string{
 			"################################################################################",
 			"# Autogenerated Failure expectations. Please triage.",
 			ROLLER_AUTOGENERATED_FAILURES,
 			"################################################################################",
 		}
 		c.Chunks = append(c.Chunks, newChunk)
 		chunkToModify = &(c.Chunks[len(c.Chunks)-1])
 	} else {
 		chunkToModify = &(c.Chunks[mutableChunkIndices[0]])
 	}

 	// Add the new expectations to the mutable chunk.
 	for _, res := range *results {
 		expectation := Expectation{
 			Bug:   "crbug.com/0000",
 			Tags:  res.Tags,
 			Query: res.Query.ExpectationFileString(),
 			Status: []string{
 				"Failure",
 			},
 		}
 		chunkToModify.Expectations = append(chunkToModify.Expectations, expectation)
 	}

 	// Sort the mutable chunk's expectations.
 	chunkToModify.Expectations.SortPrioritizeQuery()
 	return nil
 }
	// Copyright 2022 The Dawn & Tint Authors
	//
	// Redistribution and use in source and binary forms, with or without
	// modification, are permitted provided that the following conditions are met:
	//
	// 1. Redistributions of source code must retain the above copyright notice, this
	// list of conditions and the following disclaimer.
	//
	// 2. Redistributions in binary form must reproduce the above copyright notice,
	// this list of conditions and the following disclaimer in the documentation
	// and/or other materials provided with the distribution.
	//
	// 3. Neither the name of the copyright holder nor the names of its
	// contributors may be used to endorse or promote products derived from
	// this software without specific prior written permission.
	//
	// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
	// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	package expectations

	import (
	"fmt"
	"strings"
	"time"

	"dawn.googlesource.com/dawn/tools/src/cts/result"
	)

	const (
	// Chunk comment for the AddExpectationsForFailingResults path.
	ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##"
	)

	// AddExpectationsForFailingResults adds new expectations for the provided
	// failing results, with the following assumptions:
	// - The provided results do not have existing expectations
	// - Any expectations for invalid/non-existent tests have already been
	// removed
	//
	// This will:
	// - Reduce result tags down to either:
	// --- Only the most explicit tag from each set
	// --- Only a few very broad tags
	// - Merge identical results together
	// - Add new expectations to the one mutable chunk that is expected to
	// be present in the file.
	// - Sort the mutable chunk's expectations by test name, then tags.
	//
	// TODO(crbug.com/372730248): Return diagnostics.
	func (c *Content) AddExpectationsForFailingResults(results result.List,
	use_explicit_tags bool, verbose bool) error {
	// Make a copy of the results. This code mutates the list.
	results = append(result.List{}, results...)

	startTime := time.Now()
	if err := c.removeUnknownTags(&results); err != nil {
	return err
	}
	if verbose {
	fmt.Printf("Removing unknown tags took %s\n", time.Now().Sub(startTime).String())
	}

	startTime = time.Now()
	if use_explicit_tags {
	if err := c.reduceTagsToMostExplicitOnly(&results); err != nil {
	return err
	}
	} else {
	if err := c.reduceTagsToBroadGroups(&results); err != nil {
	return err
	}
	}
	if verbose {
	fmt.Printf("Reducing tags took %s\n", time.Now().Sub(startTime).String())
	}

	// Merge identical results.
	startTime = time.Now()
	results = result.Merge(results)
	if verbose {
	fmt.Printf("Merging identical results took %s\n", time.Now().Sub(startTime).String())
	}

	startTime = time.Now()
	if err := c.addExpectationsToMutableChunk(&results); err != nil {
	return err
	}
	if verbose {
	fmt.Printf("Adding expectations took %s\n", time.Now().Sub(startTime).String())
	}
	return nil
	}

	// removeUnknownTags modifies \|results\| in place so that the Results contained
	// within it only use tags that the Content is aware of.
	func (c Content) removeUnknownTags(results result.List) error {
	*results = results.TransformTags(func(t result.Tags) result.Tags {
	filtered := result.NewTags()
	for tag := range t {
	if _, ok := c.Tags.ByName[tag]; ok {
	filtered.Add(tag)
	}
	}
	return filtered
	})
	return nil
	}

	// reduceTagsToMostExplicitOnly modifies the given results argument in place
	// so that all contained results' tag sets only contain the most explicit tags
	// based on the known tag sets contained within the Content.
	func (c Content) reduceTagsToMostExplicitOnly(results result.List) error {
	for i, res := range *results {
	res.Tags = c.Tags.RemoveLowerPriorityTags(res.Tags)
	(*results)[i] = res
	}
	return nil
	}

	// reduceTagsToBroadGroups modifies the given results argument in place so that
	// all contained results' tag sets only contain several broad but important
	// tags.
	func (c Content) reduceTagsToBroadGroups(results result.List) error {

	osTagSet, gpuTagSet, deviceTagSet, err := c.getBroadTagSets()
	if err != nil {
	return err
	}

	for i, res := range *results {
	// Do an initial culling so only the broad tags are left.
	lowPriorityTags := c.Tags.RemoveHigherPriorityTags(res.Tags)
	chosenTags := result.NewTags()

	// If this is a remote platform, only use the device tag since OS type +
	// GPU vendor can be inferred.
	chosenDeviceTag, err := getAndAssertOneOverlappingTag(deviceTagSet, lowPriorityTags)
	if err == nil {
	chosenTags.Add(chosenDeviceTag)
	} else {
	chosenOsTag, err := getAndAssertOneOverlappingTag(osTagSet, lowPriorityTags)
	if err != nil {
	return err
	}
	chosenTags.Add(chosenOsTag)

	chosenGpuTag, err := getAndAssertOneOverlappingTag(gpuTagSet, lowPriorityTags)
	if err != nil {
	return err
	}
	chosenTags.Add(chosenGpuTag)
	}

	// Modify the result's tags in place.
	res.Tags = chosenTags
	(*results)[i] = res
	}

	return nil
	}

	// getAndAssertOneOverlappingTag asserts that there is a single tag that is
	// shared between the given tags and returns it.
	func getAndAssertOneOverlappingTag(validTags, allTags result.Tags) (string, error) {
	overlappingTags := validTags.Intersection(allTags)
	if len(overlappingTags) != 1 {
	return "", fmt.Errorf("Tags %v did not have exactly one overlap with %v", validTags, allTags)
	}
	return overlappingTags.One(), nil
	}

	// getBroadTagSets retrieves the Content's tag sets which correspond to the three
	// primary broad tags: OS, GPU, and device type.
	func (c *Content) getBroadTagSets() (result.Tags, result.Tags, result.Tags, error) {
	// Determine which tag sets are the ones we care about. We do this by looking
	// for the tag set that contains one of the well-known tags of that type, e.g.
	// "android", "linux", "mac", or "win" for OS.
	var osTagSet, gpuTagSet, deviceTagSet result.Tags
	// TODO(crbug.com/344014313): Move these known tags into a JSON configuration
	// file for easy modification once dependency injection is available to make
	// testing possible.
	knownOsTags := result.NewTags("android", "linux", "mac", "win")
	knownGpuTags := result.NewTags("amd", "intel", "nvidia", "qualcomm")
	knownDeviceTags := result.NewTags("android-pixel-4", "android-pixel-6",
	"chromeos-board-amd64-generic", "fuchsia-board-qemu-x64")
	for _, tagSet := range c.Tags.Sets {
	if tagSet.Tags.ContainsAny(knownOsTags) {
	osTagSet = tagSet.Tags
	} else if tagSet.Tags.ContainsAny(knownGpuTags) {
	gpuTagSet = tagSet.Tags
	} else if tagSet.Tags.ContainsAny(knownDeviceTags) {
	deviceTagSet = tagSet.Tags
	}
	}

	if osTagSet == nil \|\| gpuTagSet == nil \|\| deviceTagSet == nil {
	return nil, nil, nil, fmt.Errorf("Did not find expected tag sets. Known tags may be out of date. osTagSet: %v, gpuTagSet: %v, deviceTagSet: %v",
	osTagSet, gpuTagSet, deviceTagSet)
	}

	return osTagSet, gpuTagSet, deviceTagSet, nil
	}

	// addExpectationsToMutableChunk adds expectations for the results contained
	// within \|results\| to the one mutable chunk that should be in the Content.
	// If not found, a new one will be created at the end of the Content.
	func (c Content) addExpectationsToMutableChunk(results result.List) error {
	// Find the mutable chunk.
	// Chunks are considered immutable by default, unless annotated as
	// ROLLER_AUTOGENERATED_FAILURES.
	mutableTokens := []string{
	ROLLER_AUTOGENERATED_FAILURES,
	}

	// Bin the chunks into those that contain any of the strings in
	// mutableTokens in the comments and those that do not have these strings.
	immutableChunkIndicies, mutableChunkIndices := []int{}, []int{}
	for i, chunk := range c.Chunks {
	immutable := true

	comments:
	for _, line := range chunk.Comments {
	for _, token := range mutableTokens {
	if strings.Contains(line, token) {
	immutable = false
	break comments
	}
	}
	}

	if immutable {
	immutableChunkIndicies = append(immutableChunkIndicies, i)
	} else {
	mutableChunkIndices = append(mutableChunkIndices, i)
	}
	}

	var chunkToModify *Chunk
	if len(mutableChunkIndices) > 1 {
	return fmt.Errorf("Expected 1 mutable chunk, found %d", len(mutableChunkIndices))
	} else if len(mutableChunkIndices) == 0 {
	newChunk := Chunk{}
	newChunk.Comments = []string{
	"################################################################################",
	"# Autogenerated Failure expectations. Please triage.",
	ROLLER_AUTOGENERATED_FAILURES,
	"################################################################################",
	}
	c.Chunks = append(c.Chunks, newChunk)
	chunkToModify = &(c.Chunks[len(c.Chunks)-1])
	} else {
	chunkToModify = &(c.Chunks[mutableChunkIndices[0]])
	}

	// Add the new expectations to the mutable chunk.
	for _, res := range *results {
	expectation := Expectation{
	Bug: "crbug.com/0000",
	Tags: res.Tags,
	Query: res.Query.ExpectationFileString(),
	Status: []string{
	"Failure",
	},
	}
	chunkToModify.Expectations = append(chunkToModify.Expectations, expectation)
	}

	// Sort the mutable chunk's expectations.
	chunkToModify.Expectations.SortPrioritizeQuery()
	return nil
	}