Add expectations for CTS failures/flakes on CI
Last attempt did not clean up as much as I hoped. Second attempt.
- A bunch of blocks are marked as KEEP to try to prevent them from being
removed or mangled by the autoroller.
- I upgraded several expectations from RetryOnFailure to Failure because
they were still very prevalent on CI. But it didn't seem like they
were actually failing due to retries so I'm not sure what's going on
here (are the expectations not taking effect?)
Bug: chromium:1515477, chromium:1516044, chromium:1496410
Bug: dawn:1879, dawn:1956, dawn:2324, dawn:2325, dawn:2326
Bug: dawn:2331, dawn:2332, tint:2129, tint:2130
Change-Id: I5c00f58a7389f94fd4573364c948d564de1a5b73
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/168303
Auto-Submit: Kai Ninomiya <kainino@chromium.org>
Kokoro: Kai Ninomiya <kainino@chromium.org>
Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Kai Ninomiya <kainino@chromium.org>
Commit-Queue: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
diff --git a/webgpu-cts/expectations.txt b/webgpu-cts/expectations.txt
index 29edb64..754bcc0 100644
--- a/webgpu-cts/expectations.txt
+++ b/webgpu-cts/expectations.txt
@@ -112,12 +112,6 @@
crbug.com/1421045 [ nvidia-0x2184 ubuntu ] webgpu:api,validation,capability_checks,limits,* [ Skip ]
################################################################################
-# Very flaky multisampled sample_mask tests on Mac AMD
-# KEEP
-################################################################################
-crbug.com/1496410 [ amd-0x67ef mac ] webgpu:api,operation,render_pipeline,sample_mask:fragment_output_mask:interpolated=true;sampleCount=4;* [ Failure ]
-
-################################################################################
# Causing bot instability on Mac AMD
# KEEP
################################################################################
@@ -623,9 +617,12 @@
################################################################################
# Some copyTextureToTexture tests fail on Qualcomm Android R (Pixel 4), don't appear to repro on Android T
+# KEEP
################################################################################
crbug.com/dawn/1956 [ android-r dawn-backend-validation ] webgpu:api,validation,encoding,cmds,copyTextureToTexture:copy_aspects:format="depth32float" [ Failure ]
+crbug.com/dawn/1956 [ android-r dawn-backend-validation ] webgpu:api,validation,encoding,cmds,copyTextureToTexture:copy_aspects:format="depth32float-stencil8" [ Failure ]
crbug.com/dawn/1956 [ android-r dawn-no-backend-validation ] webgpu:api,validation,encoding,cmds,copyTextureToTexture:copy_aspects:format="depth32float" [ RetryOnFailure ]
+crbug.com/dawn/1956 [ android-r dawn-no-backend-validation ] webgpu:api,validation,encoding,cmds,copyTextureToTexture:copy_aspects:format="depth32float-stencil8" [ RetryOnFailure ]
crbug.com/dawn/1956 [ android-r ] webgpu:api,validation,encoding,cmds,copyTextureToTexture:copy_ranges:dimension="1d";copyBoxOffsets={"x":0,"y":0,"z":0,"width":0,"height":0,"depthOrArrayLayers":0};srcCopyLevel=0;dstCopyLevel=0 [ Failure ]
crbug.com/dawn/1956 [ android-r dawn-backend-validation ] webgpu:api,validation,encoding,cmds,copyTextureToTexture:copy_ranges:dimension="2d";copyBoxOffsets={"x":0,"y":0,"z":0,"width":0,"height":1,"depthOrArrayLayers":-2};srcCopyLevel=1;dstCopyLevel=3 [ Failure ]
crbug.com/dawn/1956 [ android-r dawn-no-backend-validation ] webgpu:api,validation,encoding,cmds,copyTextureToTexture:copy_ranges:dimension="2d";copyBoxOffsets={"x":0,"y":0,"z":0,"width":0,"height":1,"depthOrArrayLayers":-2};srcCopyLevel=1;dstCopyLevel=3 [ RetryOnFailure ]
@@ -683,6 +680,7 @@
################################################################################
# DXC failures
################################################################################
+crbug.com/tint/2130 [ intel-0x9bc5 webgpu-dxc-enabled win10 ] webgpu:shader,execution,robust_access:linear_memory:* [ RetryOnFailure ]
crbug.com/dawn/2141 [ nvidia-0x2184 webgpu-dxc-enabled win10 ] webgpu:shader,execution,robust_access:linear_memory:addressSpace="workgroup";access="read";containerType="vector";shadowingMode="function-scope";isAtomic=false [ Failure ]
crbug.com/dawn/2141 [ nvidia-0x2184 webgpu-dxc-enabled win10 ] webgpu:shader,execution,robust_access:linear_memory:addressSpace="workgroup";access="read";containerType="vector";shadowingMode="module-scope";isAtomic=false [ Failure ]
crbug.com/dawn/2141 [ nvidia-0x2184 webgpu-dxc-enabled win10 ] webgpu:shader,execution,robust_access:linear_memory:addressSpace="workgroup";access="read";containerType="vector";shadowingMode="none";isAtomic=false [ Failure ]
@@ -698,59 +696,89 @@
################################################################################
# writeTimestamp is exposed to GPUCommandEncoder if webgpu-developer-features
# is enabled
+# KEEP
################################################################################
crbug.com/dawn/2269 webgpu:api,validation,capability_checks,features,query_types:timestamp:featureContainsTimestampQuery=true [ Failure ]
################################################################################
# Workgroup zero_init flaky on Mac/Intel
+# KEEP
################################################################################
crbug.com/dawn/2327 [ ventura intel-0x3e9b ] webgpu:shader,execution,zero_init:compute,zero_init:addressSpace="workgroup";* [ RetryOnFailure ]
################################################################################
# WebGPUTextureAlphaClearer invalid texture from IOSurface
+# KEEP
################################################################################
crbug.com/1515476 [ ventura intel-0x3e9b ] webgpu:web_platform,canvas,configure:viewFormats:canvasType="onscreen";format="bgra8unorm";viewFormatFeature="_undef_" [ RetryOnFailure ]
################################################################################
-# GPUExternalTexture expiry flaky on Android
+# D3D12 create command queue failed with DXGI_ERROR_DEVICE_REMOVED. Skipped because these tests likely cause downstream flakes.
+# KEEP
################################################################################
-crbug.com/1515477 [ android-r qualcomm ] webgpu:api,validation,gpu_external_texture_expiration:use_import_to_refresh: [ RetryOnFailure ]
-
-################################################################################
-# Unexpected out-of-memory error occurred
-################################################################################
-crbug.com/dawn/2326 [ android-r qualcomm ] webgpu:shader,execution,memory_model,atomicity:atomicity:memType="atomic_workgroup";testType="intra_workgroup" [ RetryOnFailure ]
-
-################################################################################
-# D3D12 create command queue failed with DXGI_ERROR_DEVICE_REMOVED
-################################################################################
-crbug.com/dawn/2325 [ win10 nvidia-0x2184 ] webgpu:api,validation,capability_checks,limits,* [ RetryOnFailure ]
-crbug.com/dawn/2325 [ win10 nvidia-0x2184 ] webgpu:api,validation,state,device_lost,destroy:createTexture,* [ RetryOnFailure ]
-crbug.com/dawn/2325 [ win10 nvidia-0x2184 ] webgpu:api,validation,state,device_lost,destroy:createView,* [ RetryOnFailure ]
+crbug.com/dawn/2325 [ nvidia-0x2184 win10 ] webgpu:api,validation,capability_checks,* [ Skip ]
+crbug.com/dawn/2325 [ nvidia-0x2184 win10 ] webgpu:api,validation,state,device_lost,destroy:* [ Skip ]
################################################################################
# Hazard READ_AFTER_WRITE on ASTC/ETC2/BC
+# KEEP
################################################################################
crbug.com/dawn/2324 [ ubuntu intel-0x9bc5 dawn-backend-validation webgpu-adapter-default ] webgpu:api,operation,command_buffer,image_copy:mip_levels:* [ Failure ]
crbug.com/dawn/1879 [ ubuntu intel-0x9bc5 dawn-backend-validation webgpu-adapter-default ] webgpu:api,operation,command_buffer,copyTextureToTexture:color_textures,compressed,non_array:* [ Failure ]
################################################################################
-# CPU out of memory on Win/Intel
+# CPU out of memory on Win/Intel. Possibly other problems too
+# KEEP
################################################################################
-crbug.com/tint/2129 [ win10 intel-0x9bc5 ] webgpu:shader,execution,expression,call,builtin,min:abstract_float:inputSource="const";* [ RetryOnFailure ]
-crbug.com/tint/2129 [ win10 intel-0x9bc5 ] webgpu:shader,execution,expression,call,builtin,mix:abstract_float_matching:inputSource="const";* [ RetryOnFailure ]
-crbug.com/tint/2129 [ win10 intel-0x9bc5 ] webgpu:shader,execution,expression,call,builtin,saturate:abstract_float:inputSource="const";* [ RetryOnFailure ]
-crbug.com/tint/2129 [ win10 intel-0x9bc5 ] webgpu:shader,execution,expression,call,builtin,transpose:abstract_float:inputSource="const";* [ RetryOnFailure ]
-crbug.com/tint/2129 [ win10 intel-0x9bc5 ] webgpu:shader,execution,zero_init:compute,zero_init:addressSpace="function";workgroupSize=[1,1,1];* [ RetryOnFailure ]
+crbug.com/tint/2129 [ intel-0x9bc5 win10 ] webgpu:shader,execution,expression,call,builtin,min:abstract_float:inputSource="const";* [ RetryOnFailure ]
+crbug.com/tint/2129 [ intel-0x9bc5 win10 ] webgpu:shader,execution,expression,call,builtin,mix:abstract_float_matching:inputSource="const";* [ RetryOnFailure ]
+crbug.com/tint/2129 [ intel-0x9bc5 win10 ] webgpu:shader,execution,expression,call,builtin,saturate:abstract_float:inputSource="const";* [ RetryOnFailure ]
+crbug.com/tint/2129 [ intel-0x9bc5 win10 ] webgpu:shader,execution,expression,call,builtin,transpose:abstract_float:inputSource="const";* [ RetryOnFailure ]
+crbug.com/tint/2129 [ intel-0x9bc5 win10 ] webgpu:shader,execution,zero_init:compute,zero_init:addressSpace="function";workgroupSize=[1,1,1];* [ RetryOnFailure ]
+crbug.com/tint/2129 [ intel-0x9bc5 win10 ] webgpu:shader,validation,shader_io,group_and_binding:* [ RetryOnFailure ]
+
+################################################################################
+# Flaky multisampled sample_mask tests on Mac AMD (crbug.com/1496410 plus other flakes)
+# KEEP
+################################################################################
+[ amd-0x67ef mac ] webgpu:api,operation,render_pipeline,sample_mask:fragment_output_mask:* [ RetryOnFailure ]
+
+################################################################################
+# Miscellaneous flakes
+# KEEP
+################################################################################
+crbug.com/dawn/2332 [ ubuntu intel-0x9bc5 dawn-backend-validation ] webgpu:api,operation,command_buffer,image_copy:origins_and_extents:initMethod="WriteTexture";checkMethod="FullCopyT2B";* [ RetryOnFailure ]
+crbug.com/dawn/2331 [ ubuntu intel-0x9bc5 dawn-backend-validation ] webgpu:api,validation,render_pipeline,depth_stencil_state:* [ RetryOnFailure ]
+crbug.com/dawn/2331 [ ubuntu intel-0x9bc5 dawn-backend-validation ] webgpu:api,validation,render_pipeline,fragment_state:* [ RetryOnFailure ]
+crbug.com/dawn/2331 [ ubuntu intel-0x9bc5 dawn-backend-validation ] webgpu:api,validation,render_pipeline,inter_stage:* [ RetryOnFailure ]
+crbug.com/dawn/2331 [ ubuntu intel-0x9bc5 dawn-backend-validation ] webgpu:api,validation,render_pipeline,overrides:* [ RetryOnFailure ]
+crbug.com/dawn/2331 [ ubuntu intel-0x9bc5 dawn-backend-validation ] webgpu:api,validation,render_pipeline,primitive_state:strip_index_format:* [ RetryOnFailure ]
+crbug.com/1516044 [ ubuntu intel-0x9bc5 dawn-backend-validation ] webgpu:web_platform,canvas,configure:viewFormats:* [ RetryOnFailure ]
+[ apple-angle-metal-renderer:-apple-m2 dawn-backend-validation ventura ] webgpu:api,operation,compute_pipeline,overrides:workgroup_size:* [ RetryOnFailure ]
+[ intel-0x9bc5 dawn-no-backend-validation webgpu-dxc-disabled win10 ] webgpu:shader,validation,uniformity,uniformity:binary_expressions:* [ RetryOnFailure ]
+[ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:shader,validation,functions,alias_analysis:one_pointer_one_module_scope:* [ RetryOnFailure ]
+[ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:web_platform,external_texture,video:importExternalTexture,compute:* [ RetryOnFailure ]
+[ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:web_platform,external_texture,video:importExternalTexture,sample:* [ RetryOnFailure ]
+[ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_ImageData:* [ RetryOnFailure ]
+[ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:web_platform,copyToTexture,ImageBitmap:from_canvas:* [ RetryOnFailure ]
+[ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:shader,execution,robust_access:linear_memory:* [ RetryOnFailure ]
+[ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:shader,execution,robust_access_vertex:vertex_buffer_access:* [ RetryOnFailure ]
+
+################################################################################
+# Miscellaneous flakes that are too flaky for retry
+# KEEP
+################################################################################
+crbug.com/dawn/0000 [ android-r qualcomm ] webgpu:api,operation,render_pipeline,overrides:precision:isAsync=true [ Failure ]
+crbug.com/dawn/0000 [ android-r dawn-no-backend-validation qualcomm ] webgpu:api,operation,render_pipeline,overrides:shared_shader_module:isAsync=true [ Failure ]
+crbug.com/dawn/0000 [ android-r qualcomm ] webgpu:shader,validation,expression,call,builtin,length:vec4:stage="constant";* [ Failure ]
+crbug.com/dawn/0000 [ android-r qualcomm ] webgpu:shader,validation,functions,restrictions:call_arg_types_match_params:* [ Failure ]
+crbug.com/dawn/2326 [ android-r qualcomm ] webgpu:shader,execution,memory_model,atomicity:atomicity:memType="atomic_workgroup";testType="intra_workgroup" [ Failure ]
+crbug.com/1515477 [ android-r qualcomm ] webgpu:api,validation,gpu_external_texture_expiration:use_import_to_refresh: [ Failure ]
################################################################################
# New flakes. Please triage:
################################################################################
-crbug.com/dawn/0000 [ apple-angle-metal-renderer:-apple-m2 dawn-backend-validation target-cpu-64 ventura webgpu-adapter-default ] webgpu:api,operation,compute_pipeline,overrides:workgroup_size:isAsync=false;type="i32";size=3;v="z" [ RetryOnFailure ]
-crbug.com/dawn/0000 [ apple-angle-metal-renderer:-apple-m2 dawn-backend-validation target-cpu-64 ventura webgpu-adapter-default ] webgpu:api,operation,compute_pipeline,overrides:workgroup_size:isAsync=true;type="i32";size=3;v="z" [ RetryOnFailure ]
crbug.com/dawn/0000 [ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:api,operation,render_pipeline,overrides:multi_entry_points:isAsync=true [ RetryOnFailure ]
-crbug.com/dawn/0000 [ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:api,operation,render_pipeline,overrides:precision:isAsync=true [ RetryOnFailure ]
-crbug.com/dawn/0000 [ android-r dawn-no-backend-validation qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:api,operation,render_pipeline,overrides:shared_shader_module:isAsync=true [ RetryOnFailure ]
crbug.com/dawn/0000 [ dawn-no-backend-validation nvidia-0x2184 target-cpu-64 webgpu-adapter-default webgpu-dxc-enabled win10 ] webgpu:api,validation,capability_checks,limits,maxVertexAttributes:createRenderPipeline,at_over:limitTest="betweenDefaultAndMaximum";testValueName="atLimit";async=true [ RetryOnFailure ]
crbug.com/dawn/0000 [ dawn-no-backend-validation nvidia-0x2184 target-cpu-64 webgpu-adapter-default webgpu-dxc-enabled win10 ] webgpu:shader,execution,expression,binary,f16_addition:vector:inputSource="const";vectorize=3 [ RetryOnFailure ]
crbug.com/dawn/0000 [ android-t arm dawn-no-backend-validation target-cpu-64 webgpu-adapter-default ] webgpu:shader,execution,memory_model,coherence:corr:memType="atomic_storage";testType="intra_workgroup" [ RetryOnFailure ]
@@ -758,7 +786,6 @@
crbug.com/dawn/0000 [ android-r dawn-backend-validation qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:shader,validation,expression,call,builtin,atan2:values:stage="override";type="vec3%3Cf32%3E" [ RetryOnFailure ]
crbug.com/dawn/0000 [ android-r dawn-no-backend-validation qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:shader,validation,expression,call,builtin,atan2:values:stage="override";type="vec3%3Cf32%3E" [ RetryOnFailure ]
crbug.com/dawn/0000 [ dawn-no-backend-validation intel-0x9bc5 target-cpu-32 webgpu-adapter-default webgpu-dxc-disabled win10 ] webgpu:shader,validation,expression,call,builtin,length:vec4:stage="constant";type="vec4%3Cabstract-float%3E" [ RetryOnFailure ]
-crbug.com/dawn/0000 [ android-r qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:shader,validation,expression,call,builtin,length:vec4:stage="constant";type="vec4%3Cf32%3E" [ RetryOnFailure ]
crbug.com/dawn/0000 [ android-r dawn-no-backend-validation qualcomm target-cpu-32 webgpu-adapter-default ] webgpu:shader,validation,expression,call,builtin,radians:values:stage="constant";type="vec2%3Cf32%3E" [ RetryOnFailure ]
crbug.com/dawn/0000 [ win10 nvidia-0x2184 webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="depth24plus-stencil8" [ RetryOnFailure ]
crbug.com/dawn/0000 [ win10 nvidia-0x2184 webgpu-adapter-default ] webgpu:web_platform,canvas,configure:format:canvasType="offscreen";format="r16sint" [ RetryOnFailure ]
