Use more distinctive annotations for expectations files
This makes the file a bit less magic and should be easier for gardeners
to understand.
"##ROLLER_MUTABLE##" instead of "MUTABLE"
"##ROLLER_DISCARD_AND_REWRITE##" instead of "New xxx. Please triage:"
- ROLLER_ prefix makes it clear that this applies to the roller and not
the Unexpected Pass Finder
- "##" wrapper makes it easier to talk about the annotations in
comments without accidentally triggering them
- Distinctive names make them greppable/codesearchable
- Descriptive names make them easier to understand
- Avoids using natural-looking "New xxx. Please triage:" as magic tokens
Bug: None
Change-Id: Ibfbbccc54441ed25d98dcff24069def2ddb7b0ce
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/199536
Reviewed-by: Brian Sheedy <bsheedy@google.com>
Commit-Queue: Kai Ninomiya <kainino@chromium.org>
diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go
index 0d24fda..43fe72f 100644
--- a/tools/src/cts/expectations/update.go
+++ b/tools/src/cts/expectations/update.go
@@ -207,9 +207,13 @@
// expectation.
consumed result.Status = "<<consumed>>"
// Chunk comment for new flakes
- newFlakesComment = "# New flakes. Please triage:"
+ newFlakesComment = "# New flakes. Please triage - will be discarded/regenerated by the next roll:"
// Chunk comment for new failures
- newFailuresComment = "# New failures. Please triage:"
+ newFailuresComment = "# New failures. Please triage - will be discarded/regenerated by the next roll:"
+ // Chunk comment for expectations the roller is allowed to mutate
+ ROLLER_MUTABLE = "# ##ROLLER_MUTABLE##"
+ // Chunk comment for expectations the roller should discard and rewrite
+ ROLLER_DISCARD_AND_REWRITE = "# ##ROLLER_DISCARD_AND_REWRITE##"
)
// resultQueryTree holds tree of queries to all results (no filtering by tag or
@@ -354,13 +358,11 @@
func (u *updater) build() error {
progress := Progress{}
- // All chunks other than the ones for untriaged flakes/failures produced by
- // the CTS roller are considered immutable unless they are explicitly
- // declared as mutable.
+ // Chunks are considered immutable by default, unless annotated as
+ // ROLLER_MUTABLE or ROLLER_DISCARD_AND_REWRITE.
mutableTokens := []string{
- newFlakesComment,
- newFailuresComment,
- "MUTABLE",
+ ROLLER_MUTABLE,
+ ROLLER_DISCARD_AND_REWRITE,
}
// Bin the chunks into those that contain any of the strings in
@@ -428,11 +430,10 @@
return in // Just a comment / blank line
}
- // Skip over any untriaged failures / flake chunks.
+ // Skip over ROLLER_DISCARD_AND_REWRITE chunks (untriaged failures/flakes).
// We'll just rebuild them at the end.
for _, line := range in.Comments {
- if strings.HasPrefix(line, newFailuresComment) ||
- strings.HasPrefix(line, newFlakesComment) {
+ if strings.HasPrefix(line, ROLLER_DISCARD_AND_REWRITE) {
return Chunk{}
}
}
@@ -611,6 +612,7 @@
Comments: []string{
"################################################################################",
group.comment,
+ ROLLER_DISCARD_AND_REWRITE,
"################################################################################",
},
Expectations: group.results,
diff --git a/tools/src/cts/expectations/update_test.go b/tools/src/cts/expectations/update_test.go
index c138c3a..aba4925 100644
--- a/tools/src/cts/expectations/update_test.go
+++ b/tools/src/cts/expectations/update_test.go
@@ -67,7 +67,7 @@
{ //////////////////////////////////////////////////////////////////////
name: "no results found",
expectations: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
crbug.com/a/123 a:missing,test,result:* [ Failure ]
crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ]
some:other,test:* [ Failure ]
@@ -85,7 +85,7 @@
},
},
updated: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
some:other,test:* [ Failure ]
crbug.com/a/123 a:missing,test,result:* [ Failure ]
crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ]
@@ -138,7 +138,7 @@
{ //////////////////////////////////////////////////////////////////////
name: "unknown test",
expectations: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
crbug.com/a/123 an:unknown,test:* [ Failure ]
crbug.com/a/123 [ tag ] another:unknown:test [ Failure ]
@@ -205,7 +205,7 @@
{ //////////////////////////////////////////////////////////////////////
name: "simple expectation with tags",
expectations: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
[ os-a ] a:b,c:* [ Failure ]
[ gpu-b ] a:b,c:* [ Failure ]
`,
@@ -217,7 +217,7 @@
},
},
updated: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
a:b,c:* [ Failure ]
`,
diagnostics: expectations.Diagnostics{
@@ -232,7 +232,8 @@
name: "simple expectation with tags new flakes implicitly mutable",
expectations: `
################################################################################
-# New flakes. Please triage:
+# New flakes. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
[ os-a ] a:b,c:* [ RetryOnFailure ]
[ gpu-b ] a:b,c:* [ RetryOnFailure ]
@@ -246,7 +247,8 @@
},
updated: `
################################################################################
-# New flakes. Please triage:
+# New flakes. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 a:* [ RetryOnFailure ]
`,
@@ -255,7 +257,8 @@
name: "simple expectation with tags new failures implicitly mutable",
expectations: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
[ os-a ] a:b,c:* [ Failure ]
[ gpu-b ] a:b,c:* [ Failure ]
@@ -269,7 +272,8 @@
},
updated: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 a:* [ Failure ]
`,
@@ -295,7 +299,7 @@
{ //////////////////////////////////////////////////////////////////////
name: "expectation test now passes",
expectations: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
crbug.com/a/123 [ gpu-a os-a ] a:b,c:* [ Failure ]
crbug.com/a/123 [ gpu-b os-b ] a:b,c:* [ Failure ]
`,
@@ -312,7 +316,7 @@
},
},
updated: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
crbug.com/a/123 [ os-b ] a:b,c:* [ Failure ]
`,
diagnostics: expectations.Diagnostics{
@@ -326,7 +330,7 @@
{ //////////////////////////////////////////////////////////////////////
name: "expectation case now passes",
expectations: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ]
crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ]
`,
@@ -343,7 +347,7 @@
},
},
updated: `
-# MUTABLE
+# ##ROLLER_MUTABLE##
crbug.com/a/123 [ os-b ] a:b,c:d:* [ Failure ]
`,
diagnostics: expectations.Diagnostics{
@@ -502,13 +506,15 @@
updated: `# A comment
################################################################################
-# New flakes. Please triage:
+# New flakes. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:case=6;* [ RetryOnFailure ]
crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ]
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ]
crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Slow ]
@@ -558,7 +564,8 @@
updated: `# A comment
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 [ os-a ] a:b,c:d:* [ Failure ]
crbug.com/dawn/0000 [ os-b ] a:b,c:d:x,* [ Failure ]
@@ -596,7 +603,8 @@
},
updated: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 [ gpu-a ] a:* [ Failure ]
crbug.com/dawn/0000 [ gpu-b ] a:* [ Failure ]
@@ -626,7 +634,8 @@
},
updated: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 [ gpu-b os-c ] a:* [ Failure ]
crbug.com/dawn/0000 [ gpu-c os-b ] a:* [ Failure ]
@@ -649,7 +658,8 @@
},
updated: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 a:* [ Failure ]
`,
@@ -671,7 +681,8 @@
},
updated: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 a:b,c:0:* [ Failure ]
crbug.com/dawn/0000 a:b,c:2:* [ Failure ]
@@ -717,7 +728,8 @@
},
updated: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 a:* [ Failure ]
`,
@@ -759,7 +771,8 @@
},
updated: `
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 a:b,c:00:* [ Failure ]
crbug.com/dawn/0000 a:b,c:05:* [ Failure ]
diff --git a/webgpu-cts/compat-expectations.txt b/webgpu-cts/compat-expectations.txt
index eaa7700..a17c322 100644
--- a/webgpu-cts/compat-expectations.txt
+++ b/webgpu-cts/compat-expectations.txt
@@ -485,7 +485,8 @@
[ intel-0x9bc5 ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="3d";readMethod="CopyToTexture";format="rgba8snorm" [ Failure ]
################################################################################
-# New flakes. Please triage:
+# New flakes. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 [ intel-0x9bc5 ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="1d";readMethod="CopyToTexture";format="r8snorm" [ RetryOnFailure ]
crbug.com/dawn/0000 [ intel-0x9bc5 ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="1d";readMethod="CopyToTexture";format="rgba8snorm" [ RetryOnFailure ]
@@ -499,7 +500,8 @@
crbug.com/dawn/0000 [ nvidia-0x2184 ] webgpu:shader,execution,expression,call,builtin,cross:f32:inputSource="storage_rw" [ RetryOnFailure ]
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 webgpu:api,operation,adapter,requestDevice:always_returns_device:compatibilityMode=false [ Failure ]
crbug.com/dawn/0000 webgpu:api,operation,uncapturederror:iff_uncaptured:errorType="out-of-memory" [ Failure ]
diff --git a/webgpu-cts/expectations.txt b/webgpu-cts/expectations.txt
index e23e91c..190f5d6 100644
--- a/webgpu-cts/expectations.txt
+++ b/webgpu-cts/expectations.txt
@@ -1935,7 +1935,8 @@
crbug.com/dawn/0000 [ nvidia-0x2184 ] webgpu:shader,execution,expression,call,builtin,fwidthFine:f32:vectorize=4;non_uniform_discard=true [ Failure ]
################################################################################
-# New flakes. Please triage:
+# New flakes. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 [ amd-0x67ef dawn-backend-validation ] webgpu:shader,validation,expression,call,builtin,sin:values:stage="override";type="vec2%3Cf32%3E" [ RetryOnFailure ]
crbug.com/dawn/0000 [ amd-0x67ef dawn-backend-validation mac webgpu-adapter-default webgpu-no-worker ] webgpu:api,operation,rendering,3d_texture_slices:multiple_color_attachments,same_mip_level:sameTexture=true [ RetryOnFailure ]
@@ -1979,7 +1980,8 @@
crbug.com/dawn/0000 [ nvidia-0x2184 webgpu-adapter-default webgpu-dedicated-worker win10 ] webgpu:web_platform,copyToTexture,canvas:copy_contents_from_gpu_context_canvas:canvasType="offscreen";srcAndDstInSameGPUDevice=false;dstColorFormat="bgra8unorm-srgb";srcAlphaMode="premultiplied";dstAlphaMode="premultiplied";srcDoFlipYDuringCopy=false [ RetryOnFailure ]
################################################################################
-# New failures. Please triage:
+# New failures. Please triage - will be discarded/regenerated by the next roll:
+# ##ROLLER_DISCARD_AND_REWRITE##
################################################################################
crbug.com/dawn/0000 webgpu:shader,validation,decl,var:address_space_access_mode:address_space="storage";access_mode="read";trailing_comma=true [ Failure ]
crbug.com/dawn/0000 webgpu:shader,validation,decl,var:address_space_access_mode:address_space="storage";access_mode="read_write";trailing_comma=true [ Failure ]
