Add CTS expectation adder script
Adds a script to take a CTS queries, expand them to individual test case
names, and append them to the end of expectations.txt as valid
expectations.
Bug: chromium:1306640
Change-Id: I55067c8c49aeb4360d488841da12c06d88346b8a
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/84740
Commit-Queue: Brian Sheedy <bsheedy@google.com>
Auto-Submit: Brian Sheedy <bsheedy@google.com>
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Austin Eng <enga@chromium.org>
diff --git a/.gitignore b/.gitignore
index fa211f1..b22f66c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,3 +99,6 @@
### Dawn node tools binaries
src/dawn/node/tools/bin/
+
+### Cached node transpiled tools
+/.node_transpile_work_dir
diff --git a/webgpu-cts/scripts/add_query_to_expectation_file.py b/webgpu-cts/scripts/add_query_to_expectation_file.py
new file mode 100755
index 0000000..3597369
--- /dev/null
+++ b/webgpu-cts/scripts/add_query_to_expectation_file.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+#
+# Copyright 2022 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Script for easily adding expectations to expectations.txt
+
+Converts one or more WebGPU CTS queries into one or more individual expectations
+and appends them to the end of the file.
+"""
+
+import argparse
+import logging
+import os
+import subprocess
+import sys
+
+import dir_paths
+
+LIST_SCRIPT_PATH = os.path.join(dir_paths.webgpu_cts_scripts_dir, 'list.py')
+TRANSPILE_DIR = os.path.join(dir_paths.dawn_dir, '.node_transpile_work_dir')
+EXPECTATION_FILE_PATH = os.path.join(dir_paths.dawn_dir, 'webgpu-cts',
+ 'expectations.txt')
+
+
+def expand_query(query):
+ cmd = [
+ sys.executable,
+ LIST_SCRIPT_PATH,
+ '--js-out-dir',
+ TRANSPILE_DIR,
+ '--query',
+ query,
+ ]
+ p = subprocess.run(cmd, stdout=subprocess.PIPE, check=True)
+ return p.stdout.decode('utf-8').splitlines()
+
+
+def generate_expectations(queries, tags, results, bug):
+ tags = '[ %s ] ' % ' '.join(tags) if tags else ''
+ results = ' [ %s ]' % ' '.join(results)
+ bug = bug + ' ' if bug else ''
+ content = ''
+ for q in queries:
+ test_names = expand_query(q)
+ if not test_names:
+ logging.warning('Did not get any test names for query %s', q)
+ for tn in test_names:
+ content += '{bug}{tags}{test}{results}\n'.format(bug=bug,
+ tags=tags,
+ test=tn,
+ results=results)
+ with open(EXPECTATION_FILE_PATH, 'a') as outfile:
+ outfile.write(content)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(
+ description=('Converts one or more WebGPU CTS queries into one or '
+ 'more individual expectations and appends them to the '
+ 'end of expectations.txt'))
+ parser.add_argument('-b',
+ '--bug',
+ help='The bug link to associate with the expectations')
+ parser.add_argument('-t',
+ '--tag',
+ action='append',
+ default=[],
+ dest='tags',
+ help=('A tag to restrict the expectation to. Can be '
+ 'specified multiple times.'))
+ parser.add_argument('-r',
+ '--result',
+ action='append',
+ default=[],
+ dest='results',
+ required=True,
+ help=('An expected result for the expectation. Can be '
+ 'specified multiple times, although a single '
+ 'result is the most common usage.'))
+ parser.add_argument('-q',
+ '--query',
+ action='append',
+ default=[],
+ dest='queries',
+ help=('A CTS query to expand into expectations. Can '
+ 'be specified multiple times.'))
+ args = parser.parse_args()
+ generate_expectations(args.queries, args.tags, args.results, args.bug)
diff --git a/webgpu-cts/scripts/dir_paths.py b/webgpu-cts/scripts/dir_paths.py
index cc45758..8e2b37e 100644
--- a/webgpu-cts/scripts/dir_paths.py
+++ b/webgpu-cts/scripts/dir_paths.py
@@ -17,8 +17,8 @@
import os
webgpu_cts_scripts_dir = os.path.dirname(os.path.abspath(__file__))
-dawn_third_party_dir = os.path.join(
- os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir)), 'third_party')
+dawn_dir = os.path.dirname(os.path.dirname(webgpu_cts_scripts_dir))
+dawn_third_party_dir = os.path.join(dawn_dir, 'third_party')
gn_webgpu_cts_dir = os.path.join(dawn_third_party_dir, 'gn', 'webgpu-cts')
webgpu_cts_root_dir = os.path.join(dawn_third_party_dir, 'webgpu-cts')
chromium_third_party_dir = None
