Add basic Dawn microbenchmarks using Google benchmark
The initial test tests bind group layout creation of
different sizes, cache / no-cache hit, with and without
multiple threads.
Change-Id: Ic9ed6c6f1c298d35cd1358c7ff492027c83649a7
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/127346
Commit-Queue: Austin Eng <enga@chromium.org>
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 346bfe2..3eb8136 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -133,6 +133,7 @@
option_if_not_defined(DAWN_BUILD_SAMPLES "Enables building Dawn's samples" ${BUILD_SAMPLES})
option_if_not_defined(DAWN_BUILD_NODE_BINDINGS "Enables building Dawn's NodeJS bindings" OFF)
option_if_not_defined(DAWN_ENABLE_SWIFTSHADER "Enables building Swiftshader as part of the build and Vulkan adapter discovery" OFF)
+option_if_not_defined(DAWN_BUILD_BENCHMARKS "Build Dawn benchmarks" OFF)
option_if_not_defined(DAWN_ENABLE_PIC "Build with Position-Independent-Code enabled" OFF)
@@ -164,7 +165,7 @@
option_if_not_defined(TINT_BUILD_SPIRV_TOOLS_FUZZER "Build SPIRV-Tools fuzzer" OFF)
option_if_not_defined(TINT_BUILD_AST_FUZZER "Build AST fuzzer" OFF)
option_if_not_defined(TINT_BUILD_REGEX_FUZZER "Build regex fuzzer" OFF)
-option_if_not_defined(TINT_BUILD_BENCHMARKS "Build benchmarks" OFF)
+option_if_not_defined(TINT_BUILD_BENCHMARKS "Build Tint benchmarks" OFF)
option_if_not_defined(TINT_BUILD_TESTS "Build tests" ON)
option_if_not_defined(TINT_BUILD_AS_OTHER_OS "Override OS detection to force building of *_other.cc files" OFF)
option_if_not_defined(TINT_BUILD_REMOTE_COMPILE "Build the remote-compile tool for validating shaders on a remote machine" OFF)
@@ -277,6 +278,7 @@
message(STATUS "Dawn build samples: ${DAWN_BUILD_SAMPLES}")
message(STATUS "Dawn build Node bindings: ${DAWN_BUILD_NODE_BINDINGS}")
message(STATUS "Dawn build Swiftshader: ${DAWN_ENABLE_SWIFTSHADER}")
+message(STATUS "Dawn build benchmarks: ${DAWN_BUILD_BENCHMARKS}")
message(STATUS "Dawn build PIC: ${DAWN_ENABLE_PIC}")
diff --git a/DEPS b/DEPS
index 363c82e..03b9b4e 100644
--- a/DEPS
+++ b/DEPS
@@ -110,6 +110,10 @@
'url': '{chromium_git}/catapult.git@c1e70d412ce01fb194f73f7abfdac710aae87dae',
'condition': 'dawn_standalone',
},
+ 'third_party/google_benchmark/src': {
+ 'url': '{chromium_git}/external/github.com/google/benchmark.git' + '@' + 'efc89f0b524780b1994d5dddd83a92718e5be492',
+ 'condition': 'dawn_standalone',
+ },
# Jinja2 and MarkupSafe for the code generator
'third_party/jinja2': {
@@ -204,10 +208,6 @@
},
# Misc dependencies inherited from Tint
- 'third_party/benchmark': {
- 'url': '{chromium_git}/external/github.com/google/benchmark.git@e991355c02b93fe17713efe04cbc2e278e00fdbd',
- 'condition': 'dawn_standalone',
- },
'third_party/protobuf': {
'url': '{chromium_git}/external/github.com/protocolbuffers/protobuf.git@fde7cf7358ec7cd69e8db9be4f1fa6a5c431386a',
'condition': 'dawn_standalone',
diff --git a/src/dawn/CMakeLists.txt b/src/dawn/CMakeLists.txt
index 8fb3302..6a32109 100644
--- a/src/dawn/CMakeLists.txt
+++ b/src/dawn/CMakeLists.txt
@@ -23,6 +23,7 @@
# TODO(dawn:269): Remove once the implementation-based swapchains are removed.
add_subdirectory(utils)
add_subdirectory(glfw)
+add_subdirectory(tests/benchmarks)
if (DAWN_BUILD_SAMPLES)
#TODO(dawn:269): Add this once implementation-based swapchains are removed.
diff --git a/src/dawn/tests/BUILD.gn b/src/dawn/tests/BUILD.gn
index f0a2932..284e26e 100644
--- a/src/dawn/tests/BUILD.gn
+++ b/src/dawn/tests/BUILD.gn
@@ -24,6 +24,7 @@
":dawn_end2end_tests",
":dawn_perf_tests",
":dawn_unittests",
+ "${dawn_root}/src/dawn/tests/benchmarks:dawn_benchmarks",
]
}
diff --git a/src/dawn/tests/benchmarks/BGLCreation.cpp b/src/dawn/tests/benchmarks/BGLCreation.cpp
new file mode 100644
index 0000000..36ff003
--- /dev/null
+++ b/src/dawn/tests/benchmarks/BGLCreation.cpp
@@ -0,0 +1,115 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <benchmark/benchmark.h>
+#include <dawn/webgpu_cpp.h>
+#include <array>
+#include <vector>
+
+#include "dawn/common/Log.h"
+#include "dawn/tests/benchmarks/NullDeviceSetup.h"
+
+static void RedundantBGLCreation(benchmark::State& state) {
+ static wgpu::Device device = nullptr;
+
+ if (state.thread_index() == 0) {
+ std::vector<wgpu::FeatureName> requiredFeatures;
+ if (state.threads() > 1) {
+ requiredFeatures.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization);
+ }
+
+ wgpu::DeviceDescriptor deviceDesc = {};
+ deviceDesc.requiredFeatures = requiredFeatures.data();
+ deviceDesc.requiredFeaturesCount = requiredFeatures.size();
+ device = CreateNullDevice(deviceDesc);
+ }
+
+ std::vector<wgpu::BindGroupLayoutEntry> entries(state.range(0));
+ for (uint32_t i = 0; i < entries.size(); ++i) {
+ entries[i].binding = i;
+ entries[i].visibility = wgpu::ShaderStage::Vertex | wgpu::ShaderStage::Fragment;
+ entries[i].buffer.type = wgpu::BufferBindingType::Uniform;
+ }
+
+ wgpu::BindGroupLayoutDescriptor bglDesc = {};
+ bglDesc.entryCount = entries.size();
+ bglDesc.entries = entries.data();
+
+ thread_local std::vector<wgpu::BindGroupLayout> bgls;
+ bgls.reserve(100000);
+ for (auto _ : state) {
+ bgls.push_back(device.CreateBindGroupLayout(&bglDesc));
+ }
+ bgls.clear();
+
+ if (state.thread_index() == 0) {
+ device = nullptr;
+ }
+}
+
+static void UniqueBGLCreation(benchmark::State& state) {
+ static wgpu::Device device = nullptr;
+
+ if (state.thread_index() == 0) {
+ std::vector<wgpu::FeatureName> requiredFeatures;
+ if (state.threads() > 1) {
+ requiredFeatures.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization);
+ }
+
+ wgpu::DeviceDescriptor deviceDesc = {};
+ deviceDesc.requiredFeatures = requiredFeatures.data();
+ deviceDesc.requiredFeaturesCount = requiredFeatures.size();
+ device = CreateNullDevice(deviceDesc);
+ }
+
+ std::vector<wgpu::BindGroupLayoutEntry> entries(state.range(0));
+ for (uint32_t i = 0; i < entries.size(); ++i) {
+ entries[i].binding = i;
+ entries[i].visibility = wgpu::ShaderStage::Vertex | wgpu::ShaderStage::Fragment;
+ entries[i].buffer.type = wgpu::BufferBindingType::Uniform;
+ }
+ entries[0].buffer.minBindingSize = 4u;
+
+ wgpu::BindGroupLayoutDescriptor bglDesc = {};
+ bglDesc.entryCount = entries.size();
+ bglDesc.entries = entries.data();
+
+ thread_local std::vector<wgpu::BindGroupLayout> bgls;
+ bgls.reserve(100000);
+ for (auto _ : state) {
+ entries[0].buffer.minBindingSize += 4;
+ bgls.push_back(device.CreateBindGroupLayout(&bglDesc));
+ }
+ bgls.clear();
+
+ if (state.thread_index() == 0) {
+ device = nullptr;
+ }
+}
+
+BENCHMARK(RedundantBGLCreation)
+ ->Setup(SetupNullBackend)
+ ->Arg(1)
+ ->Arg(12)
+ ->Threads(1)
+ ->Threads(4)
+ ->Threads(16);
+
+BENCHMARK(UniqueBGLCreation)
+ ->Setup(SetupNullBackend)
+ ->Arg(1)
+ ->Arg(12)
+ ->Threads(1)
+ ->Threads(4)
+ ->Threads(16);
diff --git a/src/dawn/tests/benchmarks/BUILD.gn b/src/dawn/tests/benchmarks/BUILD.gn
new file mode 100644
index 0000000..d8ea27b
--- /dev/null
+++ b/src/dawn/tests/benchmarks/BUILD.gn
@@ -0,0 +1,35 @@
+# Copyright 2023 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("../../../../scripts/dawn_overrides_with_defaults.gni")
+
+import("//testing/test.gni")
+
+test("dawn_benchmarks") {
+ deps = [
+ "${dawn_root}/src/dawn:cpp",
+ "${dawn_root}/src/dawn:proc",
+ "${dawn_root}/src/dawn/common",
+ "${dawn_root}/src/dawn/native:sources",
+ "${dawn_root}/src/dawn/native:static",
+ "//third_party/google_benchmark",
+ "//third_party/google_benchmark:benchmark_main",
+ ]
+ sources = [
+ "BGLCreation.cpp",
+ "NullDeviceSetup.cpp",
+ "NullDeviceSetup.h",
+ ]
+ configs += [ "${dawn_root}/include/dawn:public" ]
+}
diff --git a/src/dawn/tests/benchmarks/CMakeLists.txt b/src/dawn/tests/benchmarks/CMakeLists.txt
new file mode 100644
index 0000000..f575866
--- /dev/null
+++ b/src/dawn/tests/benchmarks/CMakeLists.txt
@@ -0,0 +1,34 @@
+# Copyright 2023 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if (${DAWN_BUILD_BENCHMARKS})
+ add_executable(dawn_benchmarks
+ "BGLCreation.cpp"
+ "NullDeviceSetup.cpp"
+ "NullDeviceSetup.h"
+ )
+ set_target_properties(dawn_benchmarks PROPERTIES FOLDER "Benchmarks")
+
+ target_include_directories(dawn_benchmarks PUBLIC "${PROJECT_SOURCE_DIR}/include")
+ target_include_directories(dawn_benchmarks PUBLIC "${PROJECT_SOURCE_DIR}/src")
+
+ target_link_libraries(dawn_benchmarks PRIVATE
+ benchmark::benchmark
+ benchmark::benchmark_main
+ dawn_common
+ dawn_native
+ dawncpp_headers
+ dawncpp
+ dawn_proc)
+endif()
diff --git a/src/dawn/tests/benchmarks/NullDeviceSetup.cpp b/src/dawn/tests/benchmarks/NullDeviceSetup.cpp
new file mode 100644
index 0000000..a4b76bb
--- /dev/null
+++ b/src/dawn/tests/benchmarks/NullDeviceSetup.cpp
@@ -0,0 +1,84 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dawn/tests/benchmarks/NullDeviceSetup.h"
+
+#include <benchmark/benchmark.h>
+#include <dawn/webgpu_cpp.h>
+#include <memory>
+
+#include "dawn/common/Assert.h"
+#include "dawn/common/Log.h"
+#include "dawn/dawn_proc.h"
+#include "dawn/native/DawnNative.h"
+
+namespace {
+std::unique_ptr<dawn::native::Instance> nativeInstance = nullptr;
+WGPUAdapter nullBackendAdapter = nullptr;
+} // namespace
+
+void SetupNullBackend(const benchmark::State& state) {
+ if (!nativeInstance) {
+ nativeInstance = std::make_unique<dawn::native::Instance>();
+ nativeInstance->DiscoverDefaultAdapters();
+ }
+
+ if (!nullBackendAdapter) {
+ for (auto& a : nativeInstance->GetAdapters()) {
+ wgpu::AdapterProperties properties;
+ a.GetProperties(&properties);
+ if (properties.backendType == wgpu::BackendType::Null) {
+ nullBackendAdapter = a.Get();
+ }
+ }
+ }
+ ASSERT(nullBackendAdapter != nullptr);
+}
+
+wgpu::Device CreateNullDevice(const wgpu::DeviceDescriptor& desc) {
+ dawnProcSetProcs(&dawn::native::GetProcs());
+
+ wgpu::Device device;
+
+ wgpu::Adapter(nullBackendAdapter)
+ .RequestDevice(
+ &desc,
+ [](WGPURequestDeviceStatus status, WGPUDevice cDevice, char const* message,
+ void* userdata) {
+ ASSERT(status == WGPURequestDeviceStatus_Success);
+ *reinterpret_cast<wgpu::Device*>(userdata) = wgpu::Device::Acquire(cDevice);
+ },
+ &device);
+ while (!device) {
+ wgpuInstanceProcessEvents(nativeInstance->Get());
+ }
+
+ device.SetUncapturedErrorCallback(
+ [](WGPUErrorType, char const* message, void* userdata) {
+ dawn::ErrorLog() << message;
+ UNREACHABLE();
+ },
+ nullptr);
+
+ device.SetDeviceLostCallback(
+ [](WGPUDeviceLostReason reason, char const* message, void* userdata) {
+ if (reason == WGPUDeviceLostReason_Undefined) {
+ dawn::ErrorLog() << message;
+ UNREACHABLE();
+ }
+ },
+ nullptr);
+
+ return device;
+}
diff --git a/src/dawn/tests/benchmarks/NullDeviceSetup.h b/src/dawn/tests/benchmarks/NullDeviceSetup.h
new file mode 100644
index 0000000..d8c7cad
--- /dev/null
+++ b/src/dawn/tests/benchmarks/NullDeviceSetup.h
@@ -0,0 +1,31 @@
+// Copyright 2023 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP
+#define DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP
+
+#include <dawn/webgpu_cpp.h>
+
+namespace benchmark {
+class State;
+}
+
+namespace wgpu {
+struct DeviceDescriptor;
+}
+
+void SetupNullBackend(const benchmark::State& state);
+wgpu::Device CreateNullDevice(const wgpu::DeviceDescriptor& desc);
+
+#endif // DAWN_TESTS_BENCHMARKS_NULLDEVICESETUP
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 6ca33d4..47ad676 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -106,9 +106,9 @@
add_subdirectory(${DAWN_SWIFTSHADER_DIR} "${CMAKE_CURRENT_BINARY_DIR}/swiftshader")
endif()
-if (${TINT_BUILD_BENCHMARKS})
+if (${TINT_BUILD_BENCHMARKS} OR ${DAWN_BUILD_BENCHMARKS})
set(BENCHMARK_ENABLE_TESTING FALSE CACHE BOOL FALSE FORCE)
- add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/benchmark EXCLUDE_FROM_ALL)
+ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/google_benchmark/src EXCLUDE_FROM_ALL)
endif()
if (NOT TARGET gmock AND ${TINT_BUILD_TESTS})
diff --git a/third_party/google_benchmark/BUILD.gn b/third_party/google_benchmark/BUILD.gn
new file mode 100644
index 0000000..779cf2a
--- /dev/null
+++ b/third_party/google_benchmark/BUILD.gn
@@ -0,0 +1,91 @@
+# Copyright 2019 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build_overrides/build.gni")
+
+config("benchmark_config") {
+ include_dirs = [ "src/include" ]
+
+ if (!is_component_build) {
+ defines = [ "BENCHMARK_STATIC_DEFINE" ]
+ }
+}
+
+component("google_benchmark") {
+ testonly = true
+
+ public = [
+ "src/include/benchmark/benchmark.h",
+ "src/include/benchmark/export.h",
+ ]
+
+ sources = [
+ "src/src/arraysize.h",
+ "src/src/benchmark.cc",
+ "src/src/benchmark_api_internal.cc",
+ "src/src/benchmark_api_internal.h",
+ "src/src/benchmark_name.cc",
+ "src/src/benchmark_register.cc",
+ "src/src/benchmark_register.h",
+ "src/src/benchmark_runner.cc",
+ "src/src/benchmark_runner.h",
+ "src/src/check.cc",
+ "src/src/check.h",
+ "src/src/colorprint.cc",
+ "src/src/colorprint.h",
+ "src/src/commandlineflags.cc",
+ "src/src/commandlineflags.h",
+ "src/src/complexity.cc",
+ "src/src/complexity.h",
+ "src/src/console_reporter.cc",
+ "src/src/counter.cc",
+ "src/src/counter.h",
+ "src/src/csv_reporter.cc",
+ "src/src/cycleclock.h",
+ "src/src/internal_macros.h",
+ "src/src/json_reporter.cc",
+ "src/src/log.h",
+ "src/src/mutex.h",
+ "src/src/perf_counters.cc",
+ "src/src/perf_counters.h",
+ "src/src/re.h",
+ "src/src/reporter.cc",
+ "src/src/statistics.cc",
+ "src/src/statistics.h",
+ "src/src/string_util.cc",
+ "src/src/string_util.h",
+ "src/src/sysinfo.cc",
+ "src/src/thread_manager.h",
+ "src/src/thread_timer.h",
+ "src/src/timers.cc",
+ "src/src/timers.h",
+ ]
+
+ all_dependent_configs = [ ":benchmark_config" ]
+
+ if (build_with_chromium) {
+ configs -= [ "//build/config/compiler:chromium_code" ]
+ configs += [ "//build/config/compiler:no_chromium_code" ]
+ }
+
+ if (is_win) {
+ configs -= [ "//build/config/win:nominmax" ]
+ }
+
+ defines = [
+ "benchmark_EXPORTS=1",
+
+ # Tell gtest to always use standard regular expressions.
+ "HAVE_GNU_POSIX_REGEX=0",
+ "HAVE_POSIX_REGEX=0",
+ "HAVE_STD_REGEX=1",
+ ]
+}
+
+component("benchmark_main") {
+ testonly = true
+ sources = [ "src/src/benchmark_main.cc" ]
+ defines = [ "benchmark_EXPORTS=1" ]
+ deps = [ ":google_benchmark" ]
+}
diff --git a/third_party/google_benchmark/README.chromium b/third_party/google_benchmark/README.chromium
new file mode 100644
index 0000000..34ce0eb
--- /dev/null
+++ b/third_party/google_benchmark/README.chromium
@@ -0,0 +1,13 @@
+Name: Google Benchmark
+Short Name: benchmark
+URL: https://github.com/google/benchmark
+Version: efc89f0b524780b1994d5dddd83a92718e5be492
+License: Apache 2.0
+License File: NOT_SHIPPED
+Security Critical: no
+
+Description:
+A microbenchmark support library.
+
+Local Additions:
+* gn file for building in chromium
