Add simple test runner.
This CL adds a script to run tint over the shaders in a given folder and
attempt to generate the WGSL, HLSL, MSL and SPIRV-ASM shaders. The
GPUWeb CTS is added to third_party and the validation folder set as the
default folder to execute.
Change-Id: I63a0af056416e2f99ed8e3f92f9e2ca31c2b3e49
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/25561
Reviewed-by: Sarah Mashayekhi <sarahmashay@google.com>
Commit-Queue: Sarah Mashayekhi <sarahmashay@google.com>
diff --git a/.gitignore b/.gitignore
index b21a226..5fcdc87 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@
third_party/cpplint
third_party/binutils
third_party/googletest
+third_party/gpuweb-cts
third_party/llvm-build
third_party/spirv-headers
third_party/spirv-tools
diff --git a/DEPS b/DEPS
index d75e3b2..e00a393 100644
--- a/DEPS
+++ b/DEPS
@@ -20,6 +20,7 @@
'clang_revision': '6412135b3979b680c20cf007ab242d968025fc3e',
'cpplint_revision': '305ac8725a166ed42e3f5dd3f80d6de2cf840ef1',
'googletest_revision': 'a781fe29bcf73003559a3583167fe3d647518464',
+ 'gpuweb_cts_revision': '40e337a38784ad72fa5c7b9afd1b9c358a9e0f1a',
'spirv_headers_revision': '308bd07424350a6000f35a77b5f85cd4f3da319e',
'spirv_tools_revision': '717e7877cac15d393fd3bb1bd872679de8b59add',
'testing_revision': 'cadd4e1eb3a45f562cc7eb5cd646c9b6f91c87dc',
@@ -29,6 +30,9 @@
'third_party/cpplint': Var('chromium_git') + Var('github') +
'/google/styleguide.git@' + Var('cpplint_revision'),
+ 'third_party/gpuweb-cts': Var('chromium_git') + Var('github') +
+ '/gpuweb/cts.git@' + Var('gpuweb_cts_revision'),
+
'third_party/spirv-headers': Var('chromium_git') + Var('github') +
'/KhronosGroup/SPIRV-Headers.git@' + Var('spirv_headers_revision'),
diff --git a/tools/run_tests.py b/tools/run_tests.py
new file mode 100755
index 0000000..6da4129
--- /dev/null
+++ b/tools/run_tests.py
@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# Copyright 2020 The Tint Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import difflib
+import optparse
+import os
+import platform
+import re
+import subprocess
+import sys
+import tempfile
+
+
+class TestCase:
+ def __init__(self, input_path, parse_only):
+ self.input_path = input_path
+ self.parse_only = parse_only
+ self.results = {}
+
+ def IsExpectedFail(self):
+ fail_re = re.compile('^.+[\.]fail[\.]wgsl')
+ return fail_re.match(self.GetInputPath())
+
+ def IsParseOnly(self):
+ return self.parse_only
+
+ def GetInputPath(self):
+ return self.input_path
+
+ def GetResult(self, fmt):
+ return self.results[fmt]
+
+
+class TestRunner:
+ def RunTest(self, tc):
+ print("Testing {}".format(tc.GetInputPath()))
+
+ cmd = [self.options.test_prog_path]
+ if tc.IsParseOnly():
+ cmd += ['--parse-only']
+
+ languages = ["wgsl", "spvasm", "msl", "hlsl"]
+ try:
+ for lang in languages:
+ lang_cmd = cmd.copy()
+ lang_cmd += ['--format', lang]
+ lang_cmd += [tc.GetInputPath()]
+ err = subprocess.check_output(lang_cmd,
+ stderr=subprocess.STDOUT)
+
+ except Exception as e:
+ if not tc.IsExpectedFail():
+ print("{}".format("".join(map(chr, bytearray(e.output)))))
+ print(e)
+ return False
+
+ return True
+
+ def RunTests(self):
+ for tc in self.test_cases:
+ result = self.RunTest(tc)
+
+ if not tc.IsExpectedFail() and not result:
+ self.failures.append(tc.GetInputPath())
+ elif tc.IsExpectedFail() and result:
+ print("Expected: " + tc.GetInputPath() +
+ " to fail but passed.")
+ self.failures.append(tc.GetInputPath())
+
+ def SummarizeResults(self):
+ if len(self.failures) > 0:
+ self.failures.sort()
+
+ print('\nSummary of Failures:')
+ for failure in self.failures:
+ print(failure)
+
+ print('')
+ print('Test cases executed: {}'.format(len(self.test_cases)))
+ print(' Successes: {}'.format(
+ (len(self.test_cases) - len(self.failures))))
+ print(' Failures: {}'.format(len(self.failures)))
+ print('')
+
+ def Run(self):
+ base_path = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), '..'))
+
+ usage = 'usage: %prog [options] (file)'
+ parser = optparse.OptionParser(usage=usage)
+ parser.add_option('--build-dir',
+ default=os.path.join(base_path, 'out', 'Debug'),
+ help='path to build directory')
+ parser.add_option('--test-dir',
+ default=os.path.join(os.path.dirname(__file__), '..',
+ 'third_party', 'gpuweb-cts',
+ 'src', 'webgpu', 'shader',
+ 'validation', 'wgsl'),
+ help='path to directory containing test files')
+ parser.add_option('--test-prog-path',
+ default=None,
+ help='path to program to test')
+ parser.add_option('--parse-only',
+ action="store_true",
+ default=False,
+ help='only parse test cases; do not compile')
+
+ self.options, self.args = parser.parse_args()
+
+ if self.options.test_prog_path == None:
+ test_prog = os.path.abspath(
+ os.path.join(self.options.build_dir, 'tint'))
+ if not os.path.isfile(test_prog):
+ print("Cannot find test program {}".format(test_prog))
+ return 1
+
+ self.options.test_prog_path = test_prog
+
+ if not os.path.isfile(self.options.test_prog_path):
+ print("--test-prog-path must point to an executable")
+ return 1
+
+ input_file_re = re.compile('^.+[\.]wgsl')
+ self.test_cases = []
+
+ if self.args:
+ for filename in self.args:
+ input_path = os.path.join(self.options.test_dir, filename)
+ if not os.path.isfile(input_path):
+ print("Cannot find test file '{}'".format(filename))
+ return 1
+
+ self.test_cases.append(
+ TestCase(input_path, self.options.parse_only))
+
+ else:
+ for file_dir, _, filename_list in os.walk(self.options.test_dir):
+ for input_filename in filename_list:
+ if input_file_re.match(input_filename):
+ input_path = os.path.join(file_dir, input_filename)
+ if os.path.isfile(input_path):
+ self.test_cases.append(
+ TestCase(input_path, self.options.parse_only))
+
+ self.failures = []
+
+ self.RunTests()
+ self.SummarizeResults()
+
+ return len(self.failures) != 0
+
+
+def main():
+ runner = TestRunner()
+ return runner.Run()
+
+
+if __name__ == '__main__':
+ sys.exit(main())
