Add CTS telemetry harness support
Migrates harness support files added in
https://chromium-review.googlesource.com/c/chromium/src/+/3541414
into Dawn's repo.
Tested in
https://chromium-review.googlesource.com/c/chromium/src/+/3537743
Bug: chromium:1306640
Change-Id: I3000b1223219a1da293af910bf442570b70b7c92
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/84043
Reviewed-by: Loko Kung <lokokung@google.com>
Commit-Queue: Austin Eng <enga@chromium.org>
diff --git a/third_party/gn/webgpu-cts/BUILD.gn b/third_party/gn/webgpu-cts/BUILD.gn
index 4455250..7023941 100644
--- a/third_party/gn/webgpu-cts/BUILD.gn
+++ b/third_party/gn/webgpu-cts/BUILD.gn
@@ -1,11 +1,23 @@
-# Copyright 2022 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
# Note: This file is intentionally not used by any other BUILD.gn in Dawn.
# Instead, Chromium depends directly on this file to build the WebGPU CTS.
# Scripts called from this file assume Dawn is checked out inside Chromium.
+import("../../../scripts/dawn_overrides_with_defaults.gni")
+
group("webgpu-cts") {
public_deps = [
":compile_src",
@@ -38,12 +50,12 @@
}
action("compile_src") {
- script = "scripts/compile_src.py"
+ script = "${dawn_root}/webgpu-cts/scripts/compile_src.py"
inputs = [
"//third_party/node/node_modules/typescript/lib/tsc.js",
"//third_party/node/node.py",
- "scripts/tsc_ignore_errors.py",
+ "${dawn_root}/webgpu-cts/scripts/tsc_ignore_errors.py",
] + ts_source_inputs
outputs = js_outputs
@@ -74,7 +86,7 @@
}
action("verify_gen_ts_dep_list") {
- script = "scripts/gen_ts_dep_lists.py"
+ script = "${dawn_root}/webgpu-cts/scripts/gen_ts_dep_lists.py"
inputs = [
# TODO(kainino): Make sure this gets retriggered when the CTS dep changes.
"resource_files.txt",
diff --git a/third_party/gn/webgpu-cts/scripts/dir_paths.py b/third_party/gn/webgpu-cts/scripts/dir_paths.py
deleted file mode 100644
index 4b9d310..0000000
--- a/third_party/gn/webgpu-cts/scripts/dir_paths.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2022 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import os
-
-gn_webgpu_cts_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-dawn_third_party_dir = os.path.dirname(os.path.dirname(gn_webgpu_cts_dir))
-webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts')
-
-_possible_chromium_third_party_dir = os.path.dirname(
- os.path.dirname(dawn_third_party_dir))
-_possible_node_dir = os.path.join(_possible_chromium_third_party_dir, 'node')
-if os.path.exists(_possible_node_dir):
- chromium_third_party_dir = _possible_chromium_third_party_dir
- node_dir = _possible_node_dir
diff --git a/webgpu-cts/BUILD.gn b/webgpu-cts/BUILD.gn
new file mode 100644
index 0000000..a0ecb67
--- /dev/null
+++ b/webgpu-cts/BUILD.gn
@@ -0,0 +1,44 @@
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Note: This file is intentionally not used by any other BUILD.gn in Dawn.
+# Instead, Chromium depends directly on this file to build the WebGPU CTS.
+# Scripts called from this file assume Dawn is checked out inside Chromium.
+
+# This needs to be copied to the output directory since the CTS tests also
+# serve resources that are copied into it.
+copy("webgpu_cts_resources") {
+ testonly = true
+ sources = [
+ "test_page.html",
+ "test_runner.js",
+ ]
+ outputs = [ "$target_gen_dir/{{source_file_part}}" ]
+}
+
+group("webgpu-cts") {
+ testonly = true
+ data_deps = [
+ ":webgpu_cts_resources",
+ "../third_party/gn/webgpu-cts",
+ ]
+ data = [
+ "//third_party/node/",
+ "./scripts",
+ "./expectations.txt",
+ "../third_party/webgpu-cts/src",
+ "../third_party/webgpu-cts/node.tsconfig.json",
+ "../third_party/webgpu-cts/tsconfig.json",
+ ]
+}
diff --git a/webgpu-cts/README.md b/webgpu-cts/README.md
new file mode 100644
index 0000000..cc4d86d
--- /dev/null
+++ b/webgpu-cts/README.md
@@ -0,0 +1,20 @@
+# Running the WebGPU CTS Locally with Chrome
+
+Running the WebGPU CTS locally with Chrome requires a Chromium checkout.
+
+Follow [these instructions](https://www.chromium.org/developers/how-tos/get-the-code/) for checking out
+and building Chrome. You'll also need to build the `telemetry_gpu_integration_test` target.
+
+At the root of a Chromium checkout, run:
+`./content/test/gpu/run_gpu_integration_test.py webgpu_cts --browser=exact --browser-executable=path/to/your/chrome-executable`
+
+If you don't want to build Chrome, you can still run the CTS, by passing the path to an existing Chrome executable to the `--browser-executable` argument. You should still build the `telemetry_gpu_integration_test` target to support all harness
+functionality.
+
+Useful command-line arguments:
+ - `-l`: List all tests that would be run.
+ - `--test-filter`: Filter tests. Run `--help` for more information.
+ - `--help`: See more options.
+ - `--passthrough --show-stdout`: Show browser output. See also `--browser-logging-verbosity`.
+ - `--extra-browser-args`: Pass extra args to the browser executable.
+ - `--is-backend-validation`: Enable backend validation. TODO: rename this to `--backend-validation`.
diff --git a/webgpu-cts/expectations.txt b/webgpu-cts/expectations.txt
new file mode 100644
index 0000000..07d81d1
--- /dev/null
+++ b/webgpu-cts/expectations.txt
@@ -0,0 +1,69 @@
+# THIS FILE IS AUTOGENERATED. DO NOT MANUALLY EDIT.
+# SEE //content/test/gpu/process_generated_webgpu_expectations.py
+# BEGIN TAG HEADER (autogenerated, see validate_tag_consistency.py)
+# OS
+# tags: [ android android-lollipop android-marshmallow android-nougat
+# android-pie android-r android-s
+# chromeos
+# fuchsia
+# linux ubuntu
+# mac bigsur catalina lion highsierra mac-10.12 mojave monterey
+# mountainlion sierra
+# win win7 win8 win10 ]
+# Devices
+# tags: [ android-nexus-5 android-nexus-5x android-pixel-2 android-pixel-4
+# android-pixel-6 android-shield-android-tv
+# chromeos-board-amd64-generic chromeos-board-kevin chromeos-board-eve
+# fuchsia-board-astro fuchsia-board-sherlock fuchsia-board-qemu-x64 ]
+# Platform
+# tags: [ desktop
+# mobile ]
+# Browser
+# tags: [ android-chromium android-webview-instrumentation
+# debug debug-x64
+# release release-x64
+# fuchsia-chrome web-engine-shell ]
+# GPU
+# tags: [ amd amd-0x6613 amd-0x679e amd-0x6821 amd-0x7340
+# apple apple-apple-m1 apple-angle-metal-renderer:-apple-m1
+# arm
+# google google-0xffff
+# intel intel-0xa2e intel-0xd26 intel-0xa011 intel-0x3e92 intel-0x3e9b
+# intel-0x5912
+# nvidia nvidia-0xfe9 nvidia-0x1cb3 nvidia-0x2184
+# qualcomm ]
+# Decoder
+# tags: [ passthrough no-passthrough ]
+# ANGLE Backend
+# tags: [ angle-disabled
+# angle-d3d9 angle-d3d11
+# angle-metal
+# angle-opengl angle-opengles
+# angle-swiftshader
+# angle-vulkan ]
+# Skia Renderer
+# tags: [ skia-renderer-dawn
+# skia-renderer-disabled
+# skia-renderer-gl
+# skia-renderer-vulkan ]
+# SwiftShader
+# tags: [ swiftshader-gl no-swiftshader-gl ]
+# Driver
+# tags: [ mesa_lt_19.1 ]
+# ASan
+# tags: [ asan no-asan ]
+# Display Server
+# tags: [ display-server-wayland display-server-x ]
+# OOP-Canvas
+# tags: [ oop-c no-oop-c ]
+# WebGPU Backend Validation
+# tags: [ dawn-backend-validation dawn-no-backend-validation ]
+# results: [ Failure RetryOnFailure Skip Slow ]
+# END TAG HEADER
+
+# This will be slowly removed over time as more tests are moved from the web
+# test harness to the Telemetry harness.
+* [ Skip ]
+
+# BEGIN AUTOGENERATED EXPECTATIONS
+
diff --git a/third_party/gn/webgpu-cts/scripts/compile_src.py b/webgpu-cts/scripts/compile_src.py
similarity index 78%
rename from third_party/gn/webgpu-cts/scripts/compile_src.py
rename to webgpu-cts/scripts/compile_src.py
index 46016aa..b08c0a7 100755
--- a/third_party/gn/webgpu-cts/scripts/compile_src.py
+++ b/webgpu-cts/scripts/compile_src.py
@@ -1,7 +1,18 @@
#!/usr/bin/env python
-# Copyright 2021 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
import os
import shutil
diff --git a/webgpu-cts/scripts/dir_paths.py b/webgpu-cts/scripts/dir_paths.py
new file mode 100644
index 0000000..cc45758
--- /dev/null
+++ b/webgpu-cts/scripts/dir_paths.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+webgpu_cts_scripts_dir = os.path.dirname(os.path.abspath(__file__))
+dawn_third_party_dir = os.path.join(
+ os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir)), 'third_party')
+gn_webgpu_cts_dir = os.path.join(dawn_third_party_dir, 'gn', 'webgpu-cts')
+webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts')
+chromium_third_party_dir = None
+node_dir = None
+
+_possible_chromium_third_party_dir = os.path.dirname(
+ os.path.dirname(dawn_third_party_dir))
+_possible_node_dir = os.path.join(_possible_chromium_third_party_dir, 'node')
+if os.path.exists(_possible_node_dir):
+ chromium_third_party_dir = _possible_chromium_third_party_dir
+ node_dir = _possible_node_dir
diff --git a/third_party/gn/webgpu-cts/scripts/gen_ts_dep_lists.py b/webgpu-cts/scripts/gen_ts_dep_lists.py
similarity index 80%
rename from third_party/gn/webgpu-cts/scripts/gen_ts_dep_lists.py
rename to webgpu-cts/scripts/gen_ts_dep_lists.py
index 4b9498f..5862ab2 100755
--- a/third_party/gn/webgpu-cts/scripts/gen_ts_dep_lists.py
+++ b/webgpu-cts/scripts/gen_ts_dep_lists.py
@@ -1,7 +1,18 @@
#!/usr/bin/env python
-# Copyright 2021 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
import argparse
import os
diff --git a/webgpu-cts/scripts/list.py b/webgpu-cts/scripts/list.py
new file mode 100755
index 0000000..d63beb4
--- /dev/null
+++ b/webgpu-cts/scripts/list.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+import tempfile
+
+from dir_paths import node_dir
+
+from compile_src import compile_src_for_node
+
+
+def list_testcases(query, js_out_dir=None):
+ if js_out_dir is None:
+ js_out_dir = tempfile.mkdtemp()
+ delete_js_out_dir = True
+ else:
+ delete_js_out_dir = False
+
+ try:
+ logging.info('WebGPU CTS: Transpiling tools...')
+ compile_src_for_node(js_out_dir, [
+ '--incremental', '--tsBuildInfoFile',
+ os.path.join(js_out_dir, 'build.tsbuildinfo')
+ ],
+ clean=False)
+
+ old_sys_path = sys.path
+ try:
+ sys.path = old_sys_path + [node_dir]
+ from node import RunNode
+ finally:
+ sys.path = old_sys_path
+
+ return RunNode([
+ os.path.join(js_out_dir, 'common', 'runtime', 'cmdline.js'), query,
+ '--list'
+ ])
+ finally:
+ if delete_js_out_dir:
+ shutil.rmtree(js_out_dir)
+
+
+# List all testcases matching a test query.
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--query', default='webgpu:*', help='WebGPU CTS Query')
+ parser.add_argument(
+ '--js-out-dir',
+ default=None,
+ help='Output directory for intermediate compiled JS sources')
+ args = parser.parse_args()
+
+ print(list_testcases(args.query, args.js_out_dir))
diff --git a/third_party/gn/webgpu-cts/scripts/tsc_ignore_errors.py b/webgpu-cts/scripts/tsc_ignore_errors.py
similarity index 61%
rename from third_party/gn/webgpu-cts/scripts/tsc_ignore_errors.py
rename to webgpu-cts/scripts/tsc_ignore_errors.py
index 8fcbb5f..14b8455 100755
--- a/third_party/gn/webgpu-cts/scripts/tsc_ignore_errors.py
+++ b/webgpu-cts/scripts/tsc_ignore_errors.py
@@ -1,7 +1,18 @@
#!/usr/bin/env python
-# Copyright 2021 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
import subprocess
import sys
diff --git a/webgpu-cts/test_page.html b/webgpu-cts/test_page.html
new file mode 100644
index 0000000..89ef8086
--- /dev/null
+++ b/webgpu-cts/test_page.html
@@ -0,0 +1,24 @@
+<!doctype html>
+<!-- Copyright 2022 The Dawn Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. -->
+
+<title>WebGPU CTS</title>
+<meta charset=utf-8>
+<link rel=help href='https://gpuweb.github.io/gpuweb/'>
+
+<script type=module>
+ import { setBaseResourcePath } from '/gen/third_party/dawn/third_party/webgpu-cts/src/common/framework/resources.js';
+ setBaseResourcePath('/gen/third_party/dawn/third_party/webgpu-cts/resources');
+</script>
+<script type=module src=./test_runner.js></script>
diff --git a/webgpu-cts/test_runner.js b/webgpu-cts/test_runner.js
new file mode 100644
index 0000000..3f4eea2
--- /dev/null
+++ b/webgpu-cts/test_runner.js
@@ -0,0 +1,65 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { DefaultTestFileLoader } from '../third_party/webgpu-cts/src/common/internal/file_loader.js';
+import { prettyPrintLog } from '../third_party/webgpu-cts/src/common/internal/logging/log_message.js';
+import { Logger } from '../third_party/webgpu-cts/src/common/internal/logging/logger.js';
+import { parseQuery } from '../third_party/webgpu-cts/src/common/internal/query/parseQuery.js';
+
+import { TestWorker } from '../third_party/webgpu-cts/src/common/runtime/helper/test_worker.js';
+
+var socket;
+
+async function setupWebsocket(port) {
+ socket = new WebSocket('ws://127.0.0.1:' + port)
+ socket.addEventListener('message', runCtsTestViaSocket);
+}
+
+async function runCtsTestViaSocket(event) {
+ let input = JSON.parse(event.data);
+ runCtsTest(input['q'], input['w']);
+}
+
+async function runCtsTest(query, use_worker) {
+ const workerEnabled = use_worker;
+ const worker = workerEnabled ? new TestWorker(false) : undefined;
+
+ const loader = new DefaultTestFileLoader();
+ const filterQuery = parseQuery(query);
+ const testcases = await loader.loadCases(filterQuery);
+
+ const expectations = [];
+
+ const log = new Logger();
+
+ for (const testcase of testcases) {
+ const name = testcase.query.toString();
+
+ const wpt_fn = async () => {
+ const [rec, res] = log.record(name);
+ if (worker) {
+ await worker.run(rec, name, expectations);
+ } else {
+ await testcase.run(rec, expectations);
+ }
+
+ socket.send(JSON.stringify({'s': res.status,
+ 'l': (res.logs ?? []).map(prettyPrintLog)}));
+ };
+ await wpt_fn();
+ }
+}
+
+window.runCtsTest = runCtsTest;
+window.setupWebsocket = setupWebsocket
