QueueMTL: Handle device loss in the command buffer completed handler.

The frontend could increase the completed serial on device lost, which
the ASSERT in the completed handler didn't expect. Remove the ASSERT in
favor of doing an atomic_max instead.

Bug: dawn:2330
Change-Id: Ibd58c5f51a6c74a1668a2edfd140bd7fd4fb22de
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/168860
Reviewed-by: Loko Kung <lokokung@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
diff --git a/src/dawn/native/metal/QueueMTL.mm b/src/dawn/native/metal/QueueMTL.mm
index 356bc0d..0a7ded6 100644
--- a/src/dawn/native/metal/QueueMTL.mm
+++ b/src/dawn/native/metal/QueueMTL.mm
@@ -165,8 +165,14 @@
     [*pendingCommands addCompletedHandler:^(id<MTLCommandBuffer>) {
         TRACE_EVENT_ASYNC_END0(platform, GPUWork, "DeviceMTL::SubmitPendingCommandBuffer",
                                uint64_t(pendingSerial));
-        DAWN_ASSERT(uint64_t(pendingSerial) > mCompletedSerial.load());
-        this->mCompletedSerial.store(uint64_t(pendingSerial), std::memory_order_release);
+
+        // Do an atomic_max on mCompletedSerial since it might have been increased outside the
+        // CommandBufferMTL completed handlers if the device has been lost, or if they handlers fire
+        // in an unordered way.
+        uint64_t currentCompleted = mCompletedSerial.load();
+        while (uint64_t(pendingSerial) > currentCompleted &&
+               !mCompletedSerial.compare_exchange_weak(currentCompleted, uint64_t(pendingSerial))) {
+        }
 
         this->UpdateWaitingEvents(pendingSerial);
     }];