DawnTest: Allow checking for device errors.

The previous assumption is that all validation tests would be unittests
and that the end2end tests would always produce valid sequences of
commands. This not true because we can't test the validation of
backend-specific entrypoints in the unittests.

BUG=dawn:112

Change-Id: I89e2fe017bf3ecf6a83c9e8cdf4324c33f95a721
Reviewed-on: https://dawn-review.googlesource.com/c/5100
Reviewed-by: Kai Ninomiya <kainino@chromium.org>
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>

src/tests/DawnTest.cpp

diff --git a/src/tests/DawnTest.cpp b/src/tests/DawnTest.cpp
index cc667e4..f512c8a 100644
--- a/src/tests/DawnTest.cpp
+++ b/src/tests/DawnTest.cpp
@@ -52,12 +52,6 @@
         }
     }
 
-    // End2end tests should test valid commands produce the expected result so no error
-    // should happen. Failure cases should be tested in the validation tests.
-    void DeviceErrorCauseTestFailure(const char* message, dawnCallbackUserdata) {
-        FAIL() << "Device level failure: " << message;
-    }
-
     struct MapReadUserdata {
         DawnTest* test;
         size_t slot;
@@ -315,8 +309,8 @@
         static_cast<dawn::TextureFormat>(mBinding->GetPreferredSwapChainTextureFormat()),
         dawn::TextureUsageBit::OutputAttachment, 400, 400);
 
-    // The end2end tests should never cause validation errors. These should be tested in unittests.
-    device.SetErrorCallback(DeviceErrorCauseTestFailure, 0);
+    device.SetErrorCallback(OnDeviceError,
+                            static_cast<dawnCallbackUserdata>(reinterpret_cast<uintptr_t>(this)));
 }
 
 void DawnTest::TearDown() {
@@ -330,6 +324,24 @@
     }
 }
 
+void DawnTest::StartExpectDeviceError() {
+    mExpectError = true;
+    mError = false;
+}
+bool DawnTest::EndExpectDeviceError() {
+    mExpectError = false;
+    return mError;
+}
+
+// static
+void DawnTest::OnDeviceError(const char* message, dawnCallbackUserdata userdata) {
+    DawnTest* self = reinterpret_cast<DawnTest*>(static_cast<uintptr_t>(userdata));
+
+    ASSERT_TRUE(self->mExpectError) << "Got unexpected device error: " << message;
+    ASSERT_FALSE(self->mError) << "Got two errors in expect block";
+    self->mError = true;
+}
+
 std::ostringstream& DawnTest::AddBufferExpectation(const char* file,
                                                    int line,
                                                    const dawn::Buffer& buffer,

src/tests/DawnTest.h

diff --git a/src/tests/DawnTest.h b/src/tests/DawnTest.h
index d2daa4b..3d2b879 100644
--- a/src/tests/DawnTest.h
+++ b/src/tests/DawnTest.h
@@ -41,6 +41,13 @@
                           sizeof(RGBA8),                                                  \
                           new detail::ExpectEq<RGBA8>(expected, (width) * (height)))
 
+// Should only be used to test validation of function that can't be tested by regular validation
+// tests;
+#define ASSERT_DEVICE_ERROR(statement) \
+    StartExpectDeviceError();          \
+    statement;                         \
+    ASSERT_TRUE(EndExpectDeviceError());
+
 struct RGBA8 {
     constexpr RGBA8() : RGBA8(0, 0, 0, 0) {
     }
@@ -124,6 +131,9 @@
     bool IsLinux() const;
     bool IsMacOS() const;
 
+    void StartExpectDeviceError();
+    bool EndExpectDeviceError();
+
   protected:
     dawn::Device device;
     dawn::Queue queue;
@@ -160,6 +170,11 @@
     std::unique_ptr<utils::TerribleCommandBuffer> mS2cBuf;
     void FlushWire();
 
+    // Tracking for validation errors
+    static void OnDeviceError(const char* message, dawnCallbackUserdata userdata);
+    bool mExpectError = false;
+    bool mError = false;
+
     // MapRead buffers used to get data for the expectations
     struct ReadbackSlot {
         dawn::Buffer buffer;

src/tests/end2end/BasicTests.cpp

diff --git a/src/tests/end2end/BasicTests.cpp b/src/tests/end2end/BasicTests.cpp
index 0400e06..cd8c373 100644
--- a/src/tests/end2end/BasicTests.cpp
+++ b/src/tests/end2end/BasicTests.cpp
@@ -33,4 +33,16 @@
     EXPECT_BUFFER_U32_EQ(value, buffer, 0);
 }
 
+// Test a validation error for buffer setSubData, but really this is the most basic test possible
+// for ASSERT_DEVICE_ERROR
+TEST_P(BasicTests, BufferSetSubDataError) {
+    dawn::BufferDescriptor descriptor;
+    descriptor.size = 4;
+    descriptor.usage = dawn::BufferUsageBit::TransferSrc | dawn::BufferUsageBit::TransferDst;
+    dawn::Buffer buffer = device.CreateBuffer(&descriptor);
+
+    uint8_t value = 187;
+    ASSERT_DEVICE_ERROR(buffer.SetSubData(1000, sizeof(value), &value));
+}
+
 DAWN_INSTANTIATE_TEST(BasicTests, D3D12Backend, MetalBackend, OpenGLBackend, VulkanBackend);

src/tests/unittests/validation/ValidationTest.cpp

diff --git a/src/tests/unittests/validation/ValidationTest.cpp b/src/tests/unittests/validation/ValidationTest.cpp
index e14a014..db344ef 100644
--- a/src/tests/unittests/validation/ValidationTest.cpp
+++ b/src/tests/unittests/validation/ValidationTest.cpp
@@ -84,6 +84,7 @@
     return mDeviceErrorMessage;
 }
 
+// static
 void ValidationTest::OnDeviceError(const char* message, dawnCallbackUserdata userdata) {
     auto self = reinterpret_cast<ValidationTest*>(static_cast<uintptr_t>(userdata));
     self->mDeviceErrorMessage = message;
@@ -100,6 +101,7 @@
     self->mError = true;
 }
 
+// static
 void ValidationTest::OnBuilderErrorStatus(dawnBuilderErrorStatus status, const char* message, dawn::CallbackUserdata userdata1, dawn::CallbackUserdata userdata2) {
     auto* self = reinterpret_cast<ValidationTest*>(static_cast<uintptr_t>(userdata1));
     size_t index = static_cast<size_t>(userdata2);