[tools][cts] Add simplified roller codepath
Adds the simplified code path for modifying expectation file content
using only unexpected failure results. This new path is not called
anywhere yet, so it should be a no-op for roller functionality.
Bug: 372730248
Change-Id: I63c4d2b94f282369a689b802fb04c53af73ef18a
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/211874
Auto-Submit: Brian Sheedy <bsheedy@google.com>
Commit-Queue: Brian Sheedy <bsheedy@google.com>
Reviewed-by: dan sinclair <dsinclair@chromium.org>
diff --git a/tools/src/cts/expectations/expectations.go b/tools/src/cts/expectations/expectations.go
index 9b8ac45..acf5aa2 100644
--- a/tools/src/cts/expectations/expectations.go
+++ b/tools/src/cts/expectations/expectations.go
@@ -224,7 +224,37 @@
return 0
}
+// ComparePrioritizeQuery is the same as Compare, but compares in the following
+// order: query, tags, bug.
+func (e Expectation) ComparePrioritizeQuery(other Expectation) int {
+ switch strings.Compare(e.Query, other.Query) {
+ case -1:
+ return -1
+ case 1:
+ return 1
+ }
+ switch strings.Compare(result.TagsToString(e.Tags), result.TagsToString(other.Tags)) {
+ case -1:
+ return -1
+ case 1:
+ return 1
+ }
+ switch strings.Compare(e.Bug, other.Bug) {
+ case -1:
+ return -1
+ case 1:
+ return 1
+ }
+ return 0
+}
+
// Sort sorts the expectations in-place
func (e Expectations) Sort() {
sort.Slice(e, func(i, j int) bool { return e[i].Compare(e[j]) < 0 })
}
+
+// SortPrioritizeQuery sorts the expectations in-place, prioritizing the query for
+// sorting order.
+func (e Expectations) SortPrioritizeQuery() {
+ sort.Slice(e, func(i, j int) bool { return e[i].ComparePrioritizeQuery(e[j]) < 0 })
+}
diff --git a/tools/src/cts/expectations/expectations_test.go b/tools/src/cts/expectations/expectations_test.go
index 5ee40cd..e49f974 100644
--- a/tools/src/cts/expectations/expectations_test.go
+++ b/tools/src/cts/expectations/expectations_test.go
@@ -173,3 +173,171 @@
}
assert.Equal(t, e.AsExpectationFileString(), "query [ Failure Slow ]")
}
+
+func TestSort(t *testing.T) {
+ firstAndroidOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("android"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ firstAndroidTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("android"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ firstLinuxOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("linux"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ firstLinuxTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("linux"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ secondAndroidOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("android"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ secondAndroidTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("android"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ secondLinuxOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("linux"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ secondLinuxTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("linux"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ expectationsList := Expectations{
+ firstAndroidOne,
+ firstAndroidTwo,
+ firstLinuxOne,
+ firstLinuxTwo,
+ secondAndroidOne,
+ secondAndroidTwo,
+ secondLinuxOne,
+ secondLinuxTwo,
+ }
+
+ expectationsList.Sort()
+
+ expectedList := Expectations{
+ firstAndroidOne,
+ secondAndroidOne,
+ firstLinuxOne,
+ secondLinuxOne,
+ firstAndroidTwo,
+ secondAndroidTwo,
+ firstLinuxTwo,
+ secondLinuxTwo,
+ }
+
+ assert.Equal(t, expectationsList, expectedList)
+}
+
+func TestSortPrioritizeQuery(t *testing.T) {
+ firstAndroidOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("android"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ firstAndroidTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("android"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ firstLinuxOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("linux"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ firstLinuxTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("linux"),
+ Query: "first_query",
+ Status: []string{"Failure"},
+ }
+
+ secondAndroidOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("android"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ secondAndroidTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("android"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ secondLinuxOne := Expectation{
+ Bug: "crbug.com/1",
+ Tags: result.NewTags("linux"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ secondLinuxTwo := Expectation{
+ Bug: "crbug.com/2",
+ Tags: result.NewTags("linux"),
+ Query: "second_query",
+ Status: []string{"Failure"},
+ }
+
+ expectationsList := Expectations{
+ firstAndroidOne,
+ secondAndroidOne,
+ firstLinuxOne,
+ secondLinuxOne,
+ firstAndroidTwo,
+ secondAndroidTwo,
+ firstLinuxTwo,
+ secondLinuxTwo,
+ }
+
+ expectationsList.SortPrioritizeQuery()
+
+ expectedList := Expectations{
+ firstAndroidOne,
+ firstAndroidTwo,
+ firstLinuxOne,
+ firstLinuxTwo,
+ secondAndroidOne,
+ secondAndroidTwo,
+ secondLinuxOne,
+ secondLinuxTwo,
+ }
+
+ assert.Equal(t, expectationsList, expectedList)
+}
diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go
index d161fb8..84f3e75 100644
--- a/tools/src/cts/expectations/update.go
+++ b/tools/src/cts/expectations/update.go
@@ -41,6 +41,152 @@
"github.com/mattn/go-isatty"
)
+// AddExpectationsForFailingResults adds new expectations for the provided
+// failing results, with the assumption that the provided results do not
+// have existing expectations.
+//
+// This will:
+// - Remove expectations for non-existent tests.
+// - Reduce result tags down to only the most explicit tag from each set
+// - Merge identical results together
+// - Add new expectations to the one mutable chunk that is expected to
+// be present in the file.
+// - Sort the mutable chunk's expectations by test name, then tags.
+func (c *Content) AddExpectationsForFailingResults(results result.List,
+ testlist []query.Query, verbose bool) error {
+ // Make a copy of the results. This code mutates the list.
+ results = append(result.List{}, results...)
+
+ if err := c.removeExpectationsForUnknownTests(&testlist); err != nil {
+ return err
+ }
+
+ if err := c.reduceTagsToMostExplicitOnly(&results); err != nil {
+ return err
+ }
+
+ // Merge identical results.
+ results = result.Merge(results)
+
+ if err := c.addExpectationsToMutableChunk(&results); err != nil {
+ return err
+ }
+ return nil
+}
+
+// removeExpectationsForUnknownTests modifies the Content in place so that all
+// contained Expectations apply to tests in the given testlist.
+func (c *Content) removeExpectationsForUnknownTests(testlist *[]query.Query) error {
+ prunedChunkSlice := make([]Chunk, 0)
+ for _, chunk := range c.Chunks {
+ prunedChunk := chunk.Clone()
+ // If we don't have any expectations already, just add the chunk back
+ // immediately to avoid removing comments, especially the header.
+ if len(prunedChunk.Expectations) == 0 {
+ prunedChunkSlice = append(prunedChunkSlice, prunedChunk)
+ continue
+ }
+
+ prunedChunk.Expectations = make(Expectations, 0)
+ for _, expectation := range chunk.Expectations {
+ for _, testQuery := range *testlist {
+ expectationQuery := query.Parse(expectation.Query)
+ if expectationQuery.Contains(testQuery) {
+ prunedChunk.Expectations = append(prunedChunk.Expectations, expectation)
+ break
+ }
+ }
+ }
+
+ if len(prunedChunk.Expectations) > 0 {
+ prunedChunkSlice = append(prunedChunkSlice, prunedChunk)
+ }
+ }
+
+ c.Chunks = prunedChunkSlice
+ return nil
+}
+
+// reduceTagsToMostExplicitOnly modifies the given results argument in place
+// so that all contained results' tag sets only contain the most explicit tags
+// based on the known tag sets contained within the Content.
+func (c *Content) reduceTagsToMostExplicitOnly(results *result.List) error {
+ for i, res := range *results {
+ res.Tags = c.Tags.RemoveLowerPriorityTags(res.Tags)
+ (*results)[i] = res
+ }
+ return nil
+}
+
+// addExpectationsToMutableChunk adds expectations for the results contained
+// within |results| to the one mutable chunk that should be in the Content.
+// If not found, a new one will be created at the end of the Content.
+func (c *Content) addExpectationsToMutableChunk(results *result.List) error {
+ // Find the mutable chunk.
+ // Chunks are considered immutable by default, unless annotated as
+ // ROLLER_AUTOGENERATED_FAILURES.
+ mutableTokens := []string{
+ ROLLER_AUTOGENERATED_FAILURES,
+ }
+
+ // Bin the chunks into those that contain any of the strings in
+ // mutableTokens in the comments and those that do not have these strings.
+ immutableChunkIndicies, mutableChunkIndices := []int{}, []int{}
+ for i, chunk := range c.Chunks {
+ immutable := true
+
+ comments:
+ for _, line := range chunk.Comments {
+ for _, token := range mutableTokens {
+ if strings.Contains(line, token) {
+ immutable = false
+ break comments
+ }
+ }
+ }
+
+ if immutable {
+ immutableChunkIndicies = append(immutableChunkIndicies, i)
+ } else {
+ mutableChunkIndices = append(mutableChunkIndices, i)
+ }
+ }
+
+ var chunkToModify *Chunk
+ if len(mutableChunkIndices) > 1 {
+ return fmt.Errorf("Expected 1 mutable chunk, found %d", len(mutableChunkIndices))
+ } else if len(mutableChunkIndices) == 0 {
+ newChunk := Chunk{}
+ newChunk.Comments = []string{
+ "################################################################################",
+ "# Autogenerated Failure expectations. Please triage.",
+ ROLLER_AUTOGENERATED_FAILURES,
+ "################################################################################",
+ }
+ c.Chunks = append(c.Chunks, newChunk)
+ chunkToModify = &(c.Chunks[len(c.Chunks)-1])
+ } else {
+ chunkToModify = &(c.Chunks[mutableChunkIndices[0]])
+ }
+
+ // Add the new expectations to the mutable chunk.
+ for _, res := range *results {
+ expectation := Expectation{
+ Bug: "crbug.com/0000",
+ Tags: res.Tags,
+ Query: res.Query.String(),
+ Status: []string{
+ "Failure",
+ },
+ }
+ chunkToModify.Expectations = append(chunkToModify.Expectations, expectation)
+ }
+
+ // Sort the mutable chunk's expectations.
+ chunkToModify.Expectations.SortPrioritizeQuery()
+ return nil
+}
+
// Update performs an incremental update on the expectations using the provided
// results.
//
@@ -216,6 +362,8 @@
ROLLER_MUTABLE = "# ##ROLLER_MUTABLE##"
// Chunk comment for expectations the roller should discard and rewrite
ROLLER_DISCARD_AND_REWRITE = "# ##ROLLER_DISCARD_AND_REWRITE##"
+ // Chunk comment for the AddExpectationsForFailingResults path.
+ ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##"
)
// resultQueryTree holds tree of queries to all results (no filtering by tag or
diff --git a/tools/src/cts/expectations/update_test.go b/tools/src/cts/expectations/update_test.go
index 3075b90..7b92beed 100644
--- a/tools/src/cts/expectations/update_test.go
+++ b/tools/src/cts/expectations/update_test.go
@@ -954,3 +954,582 @@
output := u.resultsToExpectations(results, "crbug.com/1234", "comment")
assert.Equal(t, output, expectedOutput)
}
+
+/*******************************************************************************
+ * removeExpectationsForUnknownTests tests
+ ******************************************************************************/
+
+// Tests that expectations for unknown tests are properly removed, even if they
+// are in an immutable chunk.
+func TestRemoveExpectationsForUnknownTests(t *testing.T) {
+ startingContent := `
+# BEGIN TAG HEADER
+# OS
+# tags: [ android linux win10 ]
+# END TAG HEADER
+
+# Partially removed, immutable.
+[ linux ] valid_test1 [ Failure ]
+[ linux ] invalid_test [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+
+# Fully removed, immutable.
+[ android ] invalid_test [ Failure ]
+
+# Partially removed, mutable.
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] invalid_test [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+`
+
+ content, err := Parse("expectations.txt", startingContent)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ knownTests := []query.Query{
+ query.Parse("valid_test1"),
+ query.Parse("valid_test2"),
+ }
+
+ content.removeExpectationsForUnknownTests(&knownTests)
+
+ expectedContent := `# BEGIN TAG HEADER
+# OS
+# tags: [ android linux win10 ]
+# END TAG HEADER
+
+# Partially removed, immutable.
+[ linux ] valid_test1 [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+
+# Partially removed, mutable.
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+`
+
+ assert.Equal(t, expectedContent, content.String())
+}
+
+/*******************************************************************************
+ * reduceTagsToMostExplicitOnly tests
+ ******************************************************************************/
+
+// Tests that result tags are properly filtered to only include the most
+// explicit tags based on the tag sets in the header.
+func TestReduceTagsToMostExplicitOnly(t *testing.T) {
+ header := `
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-oreo android-pie android-r android-s android-t
+# android-14
+# chromeos
+# fuchsia
+# linux ubuntu
+# mac highsierra mojave catalina bigsur monterey ventura sonoma sequoia
+# win win8 win10 win11 ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x67ef amd-0x6821 amd-0x7340
+# apple apple-apple-m1 apple-apple-m2
+# apple-angle-metal-renderer:-apple-m1
+# apple-angle-metal-renderer:-apple-m2
+# arm
+# google google-0xffff google-0xc0de
+# imagination
+# intel intel-gen-9 intel-gen-12 intel-0xa2e intel-0xd26 intel-0xa011
+# intel-0x3e92 intel-0x3e9b intel-0x4680 intel-0x5912 intel-0x9bc5
+# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184 nvidia-0x2783
+# qualcomm qualcomm-0x41333430 qualcomm-0x36333630 qualcomm-0x36334330 ]
+# END TAG HEADER
+`
+
+ content, err := Parse("expectations.txt", header)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ resultList := result.List{
+ result.Result{
+ Query: query.Parse("test"),
+ Tags: result.NewTags("android", "android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("test"),
+ Tags: result.NewTags("win", "win10", "intel", "intel-gen-9", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ }
+
+ content.reduceTagsToMostExplicitOnly(&resultList)
+
+ expectedList := result.List{
+ result.Result{
+ Query: query.Parse("test"),
+ Tags: result.NewTags("android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("test"),
+ Tags: result.NewTags("win10", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ }
+
+ assert.Equal(t, expectedList, resultList)
+}
+
+/*******************************************************************************
+ * addExpectationsToMutableChunk tests
+ ******************************************************************************/
+
+// Tests that multiple mutable chunks result in an error.
+func TestAddExpectationsToMutableChunkMultipleMutableChunks(t *testing.T) {
+ fileContent := `
+# BEGIN TAG HEADER
+# OS
+# tags: [ android linux win10 ]
+# END TAG HEADER
+
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ linux ] valid_test1 [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+`
+
+ content, err := Parse("expectations.txt", fileContent)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ resultList := result.List{
+ result.Result{
+ Query: query.Parse("test"),
+ Tags: result.NewTags("android-14", "arm"),
+ Status: result.Failure,
+ },
+ }
+
+ err = content.addExpectationsToMutableChunk(&resultList)
+ assert.EqualError(t, err, "Expected 1 mutable chunk, found 2")
+}
+
+// Tests that a single mutable chunk gets expectations added to it.
+func TestAddExpectationsToMutableChunkSingleMutableChunk(t *testing.T) {
+ fileContent := `
+# BEGIN TAG HEADER
+# OS
+# tags: [ android linux win10 ]
+# END TAG HEADER
+
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+`
+
+ content, err := Parse("expectations.txt", fileContent)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ resultList := result.List{
+ result.Result{
+ Query: query.Parse("test"),
+ Tags: result.NewTags("android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("z_test"),
+ Tags: result.NewTags("android-14", "arm"),
+ Status: result.Failure,
+ },
+ }
+
+ err = content.addExpectationsToMutableChunk(&resultList)
+ assert.NoErrorf(t, err, "Failed to add expectations: %v", err)
+
+ expectedContent := `# BEGIN TAG HEADER
+# OS
+# tags: [ android linux win10 ]
+# END TAG HEADER
+
+# ##ROLLER_AUTOGENERATED_FAILURES##
+crbug.com/0000 [ android-14 arm ] test [ Failure ]
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+crbug.com/0000 [ android-14 arm ] z_test [ Failure ]
+`
+
+ assert.Equal(t, expectedContent, content.String())
+}
+
+// Tests that a lack of a mutable chunk causes a new mutable chunk to be added
+// which has the new expectations.
+func TestAddExpectationsToMutableChunkNoMutableChunk(t *testing.T) {
+ fileContent := `
+# BEGIN TAG HEADER
+# OS
+# tags: [ android linux win10 ]
+# END TAG HEADER
+
+# Immutable
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+`
+
+ content, err := Parse("expectations.txt", fileContent)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ resultList := result.List{
+ result.Result{
+ Query: query.Parse("test"),
+ Tags: result.NewTags("android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("z_test"),
+ Tags: result.NewTags("android-14", "arm"),
+ Status: result.Failure,
+ },
+ }
+
+ err = content.addExpectationsToMutableChunk(&resultList)
+ assert.NoErrorf(t, err, "Failed to add expectations: %v", err)
+
+ expectedContent := `# BEGIN TAG HEADER
+# OS
+# tags: [ android linux win10 ]
+# END TAG HEADER
+
+# Immutable
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+
+################################################################################
+# Autogenerated Failure expectations. Please triage.
+# ##ROLLER_AUTOGENERATED_FAILURES##
+################################################################################
+crbug.com/0000 [ android-14 arm ] test [ Failure ]
+crbug.com/0000 [ android-14 arm ] z_test [ Failure ]
+`
+
+ assert.Equal(t, expectedContent, content.String())
+}
+
+/*******************************************************************************
+ * AddExpectationsForFailingResults tests
+ ******************************************************************************/
+
+// Tests overall behavior with an existing mutable chunk.
+func TestAddExpectationsForFailingResultsExistingChunk(t *testing.T) {
+ fileContent := `
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-oreo android-pie android-r android-s android-t
+# android-14
+# chromeos
+# fuchsia
+# linux ubuntu
+# mac highsierra mojave catalina bigsur monterey ventura sonoma sequoia
+# win win8 win10 win11 ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x67ef amd-0x6821 amd-0x7340
+# apple apple-apple-m1 apple-apple-m2
+# apple-angle-metal-renderer:-apple-m1
+# apple-angle-metal-renderer:-apple-m2
+# arm
+# google google-0xffff google-0xc0de
+# imagination
+# intel intel-gen-9 intel-gen-12 intel-0xa2e intel-0xd26 intel-0xa011
+# intel-0x3e92 intel-0x3e9b intel-0x4680 intel-0x5912 intel-0x9bc5
+# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184 nvidia-0x2783
+# qualcomm qualcomm-0x41333430 qualcomm-0x36333630 qualcomm-0x36334330 ]
+# END TAG HEADER
+
+# Partially removed, immutable.
+[ linux ] valid_test1 [ Failure ]
+[ linux ] invalid_test [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+
+# Fully removed, immutable.
+[ android ] invalid_test [ Failure ]
+
+# Partially removed, mutable.
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] invalid_test [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+`
+
+ content, err := Parse("expectations.txt", fileContent)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ knownTests := []query.Query{
+ query.Parse("valid_test1"),
+ query.Parse("valid_test2"),
+ query.Parse("new_test"),
+ query.Parse("z_new_test"),
+ }
+
+ resultList := result.List{
+ result.Result{
+ Query: query.Parse("new_test"),
+ Tags: result.NewTags("android", "android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("z_new_test"),
+ Tags: result.NewTags("android", "android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("valid_test1"),
+ Tags: result.NewTags("linux", "intel", "intel-gen-9", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ // This should be merged with the one above after only the most explicit
+ // tags remain. Multiple generations mapping to the same actual GPU
+ // shouldn't happen in real life, but serves to test merging.
+ result.Result{
+ Query: query.Parse("valid_test1"),
+ Tags: result.NewTags("linux", "intel", "intel-gen-12", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ }
+
+ err = content.AddExpectationsForFailingResults(resultList, knownTests, false)
+ assert.NoErrorf(t, err, "Failed to add expectations: %v", err)
+
+ expectedContent := `# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-oreo android-pie android-r android-s android-t
+# android-14
+# chromeos
+# fuchsia
+# linux ubuntu
+# mac highsierra mojave catalina bigsur monterey ventura sonoma sequoia
+# win win8 win10 win11 ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x67ef amd-0x6821 amd-0x7340
+# apple apple-apple-m1 apple-apple-m2
+# apple-angle-metal-renderer:-apple-m1
+# apple-angle-metal-renderer:-apple-m2
+# arm
+# google google-0xffff google-0xc0de
+# imagination
+# intel intel-gen-9 intel-gen-12 intel-0xa2e intel-0xd26 intel-0xa011
+# intel-0x3e92 intel-0x3e9b intel-0x4680 intel-0x5912 intel-0x9bc5
+# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184 nvidia-0x2783
+# qualcomm qualcomm-0x41333430 qualcomm-0x36333630 qualcomm-0x36334330 ]
+# END TAG HEADER
+
+# Partially removed, immutable.
+[ linux ] valid_test1 [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+
+# Partially removed, mutable.
+# ##ROLLER_AUTOGENERATED_FAILURES##
+crbug.com/0000 [ android-14 arm ] new_test [ Failure ]
+crbug.com/0000 [ intel-0x3e9b linux ] valid_test1 [ Failure ]
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+crbug.com/0000 [ android-14 arm ] z_new_test [ Failure ]
+`
+
+ assert.Equal(t, expectedContent, content.String())
+}
+
+// Tests overall behavior without an existing mutable chunk.
+func TestAddExpectationsForFailingResultsWithoutExistingChunk(t *testing.T) {
+ fileContent := `
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-oreo android-pie android-r android-s android-t
+# android-14
+# chromeos
+# fuchsia
+# linux ubuntu
+# mac highsierra mojave catalina bigsur monterey ventura sonoma sequoia
+# win win8 win10 win11 ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x67ef amd-0x6821 amd-0x7340
+# apple apple-apple-m1 apple-apple-m2
+# apple-angle-metal-renderer:-apple-m1
+# apple-angle-metal-renderer:-apple-m2
+# arm
+# google google-0xffff google-0xc0de
+# imagination
+# intel intel-gen-9 intel-gen-12 intel-0xa2e intel-0xd26 intel-0xa011
+# intel-0x3e92 intel-0x3e9b intel-0x4680 intel-0x5912 intel-0x9bc5
+# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184 nvidia-0x2783
+# qualcomm qualcomm-0x41333430 qualcomm-0x36333630 qualcomm-0x36334330 ]
+# END TAG HEADER
+
+# Partially removed, immutable.
+[ linux ] valid_test1 [ Failure ]
+[ linux ] invalid_test [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+
+# Fully removed, immutable.
+[ android ] invalid_test [ Failure ]
+`
+
+ content, err := Parse("expectations.txt", fileContent)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ knownTests := []query.Query{
+ query.Parse("valid_test1"),
+ query.Parse("valid_test2"),
+ query.Parse("new_test"),
+ query.Parse("z_new_test"),
+ }
+
+ resultList := result.List{
+ result.Result{
+ Query: query.Parse("new_test"),
+ Tags: result.NewTags("android", "android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("z_new_test"),
+ Tags: result.NewTags("android", "android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("valid_test1"),
+ Tags: result.NewTags("linux", "intel", "intel-gen-9", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ // This should be merged with the one above after only the most explicit
+ // tags remain. Multiple generations mapping to the same actual GPU
+ // shouldn't happen in real life, but serves to test merging.
+ result.Result{
+ Query: query.Parse("valid_test1"),
+ Tags: result.NewTags("linux", "intel", "intel-gen-12", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ }
+
+ err = content.AddExpectationsForFailingResults(resultList, knownTests, false)
+ assert.NoErrorf(t, err, "Failed to add expectations: %v", err)
+
+ expectedContent := `# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-oreo android-pie android-r android-s android-t
+# android-14
+# chromeos
+# fuchsia
+# linux ubuntu
+# mac highsierra mojave catalina bigsur monterey ventura sonoma sequoia
+# win win8 win10 win11 ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x67ef amd-0x6821 amd-0x7340
+# apple apple-apple-m1 apple-apple-m2
+# apple-angle-metal-renderer:-apple-m1
+# apple-angle-metal-renderer:-apple-m2
+# arm
+# google google-0xffff google-0xc0de
+# imagination
+# intel intel-gen-9 intel-gen-12 intel-0xa2e intel-0xd26 intel-0xa011
+# intel-0x3e92 intel-0x3e9b intel-0x4680 intel-0x5912 intel-0x9bc5
+# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184 nvidia-0x2783
+# qualcomm qualcomm-0x41333430 qualcomm-0x36333630 qualcomm-0x36334330 ]
+# END TAG HEADER
+
+# Partially removed, immutable.
+[ linux ] valid_test1 [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+
+################################################################################
+# Autogenerated Failure expectations. Please triage.
+# ##ROLLER_AUTOGENERATED_FAILURES##
+################################################################################
+crbug.com/0000 [ android-14 arm ] new_test [ Failure ]
+crbug.com/0000 [ intel-0x3e9b linux ] valid_test1 [ Failure ]
+crbug.com/0000 [ android-14 arm ] z_new_test [ Failure ]
+`
+
+ assert.Equal(t, expectedContent, content.String())
+}
+
+// Tests overall behavior with multiple mutable chunks.
+func TestAddExpectationsForFailingResultsMultipleExistingChunk(t *testing.T) {
+ fileContent := `
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-oreo android-pie android-r android-s android-t
+# android-14
+# chromeos
+# fuchsia
+# linux ubuntu
+# mac highsierra mojave catalina bigsur monterey ventura sonoma sequoia
+# win win8 win10 win11 ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x67ef amd-0x6821 amd-0x7340
+# apple apple-apple-m1 apple-apple-m2
+# apple-angle-metal-renderer:-apple-m1
+# apple-angle-metal-renderer:-apple-m2
+# arm
+# google google-0xffff google-0xc0de
+# imagination
+# intel intel-gen-9 intel-gen-12 intel-0xa2e intel-0xd26 intel-0xa011
+# intel-0x3e92 intel-0x3e9b intel-0x4680 intel-0x5912 intel-0x9bc5
+# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184 nvidia-0x2783
+# qualcomm qualcomm-0x41333430 qualcomm-0x36333630 qualcomm-0x36334330 ]
+# END TAG HEADER
+
+# Partially removed, immutable.
+[ linux ] valid_test1 [ Failure ]
+[ linux ] invalid_test [ Failure ]
+[ linux ] valid_test2 [ Failure ]
+
+# Fully removed, immutable.
+[ android ] invalid_test [ Failure ]
+
+# Partially removed, mutable.
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ win10 ] valid_test1 [ Failure ]
+[ win10 ] invalid_test [ Failure ]
+[ win10 ] valid_test2 [ Failure ]
+
+# ##ROLLER_AUTOGENERATED_FAILURES##
+[ android ] valid_test1 [ Failure ]
+[ android ] valid_test2 [ Failure ]
+`
+
+ content, err := Parse("expectations.txt", fileContent)
+ assert.NoErrorf(t, err, "Failed to parse expectations: %v", err)
+
+ knownTests := []query.Query{
+ query.Parse("valid_test1"),
+ query.Parse("valid_test2"),
+ query.Parse("new_test"),
+ query.Parse("z_new_test"),
+ }
+
+ resultList := result.List{
+ result.Result{
+ Query: query.Parse("new_test"),
+ Tags: result.NewTags("android", "android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("z_new_test"),
+ Tags: result.NewTags("android", "android-14", "arm"),
+ Status: result.Failure,
+ },
+ result.Result{
+ Query: query.Parse("valid_test1"),
+ Tags: result.NewTags("linux", "intel", "intel-gen-9", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ // This should be merged with the one above after only the most explicit
+ // tags remain. Multiple generations mapping to the same actual GPU
+ // shouldn't happen in real life, but serves to test merging.
+ result.Result{
+ Query: query.Parse("valid_test1"),
+ Tags: result.NewTags("linux", "intel", "intel-gen-12", "intel-0x3e9b"),
+ Status: result.Failure,
+ },
+ }
+
+ err = content.AddExpectationsForFailingResults(resultList, knownTests, false)
+ assert.EqualError(t, err, "Expected 1 mutable chunk, found 2")
+}
