Use more distinctive annotations for expectations files This makes the file a bit less magic and should be easier for gardeners to understand. "##ROLLER_MUTABLE##" instead of "MUTABLE" "##ROLLER_DISCARD_AND_REWRITE##" instead of "New xxx. Please triage:" - ROLLER_ prefix makes it clear that this applies to the roller and not the Unexpected Pass Finder - "##" wrapper makes it easier to talk about the annotations in comments without accidentally triggering them - Distinctive names make them greppable/codesearchable - Descriptive names make them easier to understand - Avoids using natural-looking "New xxx. Please triage:" as magic tokens Bug: None Change-Id: Ibfbbccc54441ed25d98dcff24069def2ddb7b0ce Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/199536 Reviewed-by: Brian Sheedy <bsheedy@google.com> Commit-Queue: Kai Ninomiya <kainino@chromium.org>
diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go index 0d24fda..43fe72f 100644 --- a/tools/src/cts/expectations/update.go +++ b/tools/src/cts/expectations/update.go
@@ -207,9 +207,13 @@ // expectation. consumed result.Status = "<<consumed>>" // Chunk comment for new flakes - newFlakesComment = "# New flakes. Please triage:" + newFlakesComment = "# New flakes. Please triage - will be discarded/regenerated by the next roll:" // Chunk comment for new failures - newFailuresComment = "# New failures. Please triage:" + newFailuresComment = "# New failures. Please triage - will be discarded/regenerated by the next roll:" + // Chunk comment for expectations the roller is allowed to mutate + ROLLER_MUTABLE = "# ##ROLLER_MUTABLE##" + // Chunk comment for expectations the roller should discard and rewrite + ROLLER_DISCARD_AND_REWRITE = "# ##ROLLER_DISCARD_AND_REWRITE##" ) // resultQueryTree holds tree of queries to all results (no filtering by tag or @@ -354,13 +358,11 @@ func (u *updater) build() error { progress := Progress{} - // All chunks other than the ones for untriaged flakes/failures produced by - // the CTS roller are considered immutable unless they are explicitly - // declared as mutable. + // Chunks are considered immutable by default, unless annotated as + // ROLLER_MUTABLE or ROLLER_DISCARD_AND_REWRITE. mutableTokens := []string{ - newFlakesComment, - newFailuresComment, - "MUTABLE", + ROLLER_MUTABLE, + ROLLER_DISCARD_AND_REWRITE, } // Bin the chunks into those that contain any of the strings in @@ -428,11 +430,10 @@ return in // Just a comment / blank line } - // Skip over any untriaged failures / flake chunks. + // Skip over ROLLER_DISCARD_AND_REWRITE chunks (untriaged failures/flakes). // We'll just rebuild them at the end. for _, line := range in.Comments { - if strings.HasPrefix(line, newFailuresComment) || - strings.HasPrefix(line, newFlakesComment) { + if strings.HasPrefix(line, ROLLER_DISCARD_AND_REWRITE) { return Chunk{} } } @@ -611,6 +612,7 @@ Comments: []string{ "################################################################################", group.comment, + ROLLER_DISCARD_AND_REWRITE, "################################################################################", }, Expectations: group.results,
diff --git a/tools/src/cts/expectations/update_test.go b/tools/src/cts/expectations/update_test.go index c138c3a..aba4925 100644 --- a/tools/src/cts/expectations/update_test.go +++ b/tools/src/cts/expectations/update_test.go
@@ -67,7 +67,7 @@ { ////////////////////////////////////////////////////////////////////// name: "no results found", expectations: ` -# MUTABLE +# ##ROLLER_MUTABLE## crbug.com/a/123 a:missing,test,result:* [ Failure ] crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ] some:other,test:* [ Failure ] @@ -85,7 +85,7 @@ }, }, updated: ` -# MUTABLE +# ##ROLLER_MUTABLE## some:other,test:* [ Failure ] crbug.com/a/123 a:missing,test,result:* [ Failure ] crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ] @@ -138,7 +138,7 @@ { ////////////////////////////////////////////////////////////////////// name: "unknown test", expectations: ` -# MUTABLE +# ##ROLLER_MUTABLE## crbug.com/a/123 an:unknown,test:* [ Failure ] crbug.com/a/123 [ tag ] another:unknown:test [ Failure ] @@ -205,7 +205,7 @@ { ////////////////////////////////////////////////////////////////////// name: "simple expectation with tags", expectations: ` -# MUTABLE +# ##ROLLER_MUTABLE## [ os-a ] a:b,c:* [ Failure ] [ gpu-b ] a:b,c:* [ Failure ] `, @@ -217,7 +217,7 @@ }, }, updated: ` -# MUTABLE +# ##ROLLER_MUTABLE## a:b,c:* [ Failure ] `, diagnostics: expectations.Diagnostics{ @@ -232,7 +232,8 @@ name: "simple expectation with tags new flakes implicitly mutable", expectations: ` ################################################################################ -# New flakes. Please triage: +# New flakes. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ [ os-a ] a:b,c:* [ RetryOnFailure ] [ gpu-b ] a:b,c:* [ RetryOnFailure ] @@ -246,7 +247,8 @@ }, updated: ` ################################################################################ -# New flakes. Please triage: +# New flakes. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 a:* [ RetryOnFailure ] `, @@ -255,7 +257,8 @@ name: "simple expectation with tags new failures implicitly mutable", expectations: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ [ os-a ] a:b,c:* [ Failure ] [ gpu-b ] a:b,c:* [ Failure ] @@ -269,7 +272,8 @@ }, updated: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 a:* [ Failure ] `, @@ -295,7 +299,7 @@ { ////////////////////////////////////////////////////////////////////// name: "expectation test now passes", expectations: ` -# MUTABLE +# ##ROLLER_MUTABLE## crbug.com/a/123 [ gpu-a os-a ] a:b,c:* [ Failure ] crbug.com/a/123 [ gpu-b os-b ] a:b,c:* [ Failure ] `, @@ -312,7 +316,7 @@ }, }, updated: ` -# MUTABLE +# ##ROLLER_MUTABLE## crbug.com/a/123 [ os-b ] a:b,c:* [ Failure ] `, diagnostics: expectations.Diagnostics{ @@ -326,7 +330,7 @@ { ////////////////////////////////////////////////////////////////////// name: "expectation case now passes", expectations: ` -# MUTABLE +# ##ROLLER_MUTABLE## crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ] crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ] `, @@ -343,7 +347,7 @@ }, }, updated: ` -# MUTABLE +# ##ROLLER_MUTABLE## crbug.com/a/123 [ os-b ] a:b,c:d:* [ Failure ] `, diagnostics: expectations.Diagnostics{ @@ -502,13 +506,15 @@ updated: `# A comment ################################################################################ -# New flakes. Please triage: +# New flakes. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:case=6;* [ RetryOnFailure ] crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ] ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ] crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Slow ] @@ -558,7 +564,8 @@ updated: `# A comment ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 [ os-a ] a:b,c:d:* [ Failure ] crbug.com/dawn/0000 [ os-b ] a:b,c:d:x,* [ Failure ] @@ -596,7 +603,8 @@ }, updated: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 [ gpu-a ] a:* [ Failure ] crbug.com/dawn/0000 [ gpu-b ] a:* [ Failure ] @@ -626,7 +634,8 @@ }, updated: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 [ gpu-b os-c ] a:* [ Failure ] crbug.com/dawn/0000 [ gpu-c os-b ] a:* [ Failure ] @@ -649,7 +658,8 @@ }, updated: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 a:* [ Failure ] `, @@ -671,7 +681,8 @@ }, updated: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 a:b,c:0:* [ Failure ] crbug.com/dawn/0000 a:b,c:2:* [ Failure ] @@ -717,7 +728,8 @@ }, updated: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 a:* [ Failure ] `, @@ -759,7 +771,8 @@ }, updated: ` ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 a:b,c:00:* [ Failure ] crbug.com/dawn/0000 a:b,c:05:* [ Failure ]
diff --git a/webgpu-cts/compat-expectations.txt b/webgpu-cts/compat-expectations.txt index eaa7700..a17c322 100644 --- a/webgpu-cts/compat-expectations.txt +++ b/webgpu-cts/compat-expectations.txt
@@ -485,7 +485,8 @@ [ intel-0x9bc5 ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="3d";readMethod="CopyToTexture";format="rgba8snorm" [ Failure ] ################################################################################ -# New flakes. Please triage: +# New flakes. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 [ intel-0x9bc5 ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="1d";readMethod="CopyToTexture";format="r8snorm" [ RetryOnFailure ] crbug.com/dawn/0000 [ intel-0x9bc5 ] webgpu:api,operation,resource_init,texture_zero:uninitialized_texture_is_zero:dimension="1d";readMethod="CopyToTexture";format="rgba8snorm" [ RetryOnFailure ] @@ -499,7 +500,8 @@ crbug.com/dawn/0000 [ nvidia-0x2184 ] webgpu:shader,execution,expression,call,builtin,cross:f32:inputSource="storage_rw" [ RetryOnFailure ] ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 webgpu:api,operation,adapter,requestDevice:always_returns_device:compatibilityMode=false [ Failure ] crbug.com/dawn/0000 webgpu:api,operation,uncapturederror:iff_uncaptured:errorType="out-of-memory" [ Failure ]
diff --git a/webgpu-cts/expectations.txt b/webgpu-cts/expectations.txt index e23e91c..190f5d6 100644 --- a/webgpu-cts/expectations.txt +++ b/webgpu-cts/expectations.txt
@@ -1935,7 +1935,8 @@ crbug.com/dawn/0000 [ nvidia-0x2184 ] webgpu:shader,execution,expression,call,builtin,fwidthFine:f32:vectorize=4;non_uniform_discard=true [ Failure ] ################################################################################ -# New flakes. Please triage: +# New flakes. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 [ amd-0x67ef dawn-backend-validation ] webgpu:shader,validation,expression,call,builtin,sin:values:stage="override";type="vec2%3Cf32%3E" [ RetryOnFailure ] crbug.com/dawn/0000 [ amd-0x67ef dawn-backend-validation mac webgpu-adapter-default webgpu-no-worker ] webgpu:api,operation,rendering,3d_texture_slices:multiple_color_attachments,same_mip_level:sameTexture=true [ RetryOnFailure ] @@ -1979,7 +1980,8 @@ crbug.com/dawn/0000 [ nvidia-0x2184 webgpu-adapter-default webgpu-dedicated-worker win10 ] webgpu:web_platform,copyToTexture,canvas:copy_contents_from_gpu_context_canvas:canvasType="offscreen";srcAndDstInSameGPUDevice=false;dstColorFormat="bgra8unorm-srgb";srcAlphaMode="premultiplied";dstAlphaMode="premultiplied";srcDoFlipYDuringCopy=false [ RetryOnFailure ] ################################################################################ -# New failures. Please triage: +# New failures. Please triage - will be discarded/regenerated by the next roll: +# ##ROLLER_DISCARD_AND_REWRITE## ################################################################################ crbug.com/dawn/0000 webgpu:shader,validation,decl,var:address_space_access_mode:address_space="storage";access_mode="read";trailing_comma=true [ Failure ] crbug.com/dawn/0000 webgpu:shader,validation,decl,var:address_space_access_mode:address_space="storage";access_mode="read_write";trailing_comma=true [ Failure ]