| #!/usr/bin/env python |
| # Copyright 2020 The Tint Authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # Test runner for executing a test of tests with Tint. The runner will |
| # find all .wgsl files in the given folder and attempt to convert them |
| # to each of the backend formats. If the file contains a '.fail.' in the |
| # name then the runner will expect the file to fail conversion. |
| |
| import base64 |
| import copy |
| import difflib |
| import optparse |
| import os |
| import platform |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| |
| |
| """ |
| A single test case to be executed. Stores the path to the test file |
| and the result of executing the test. |
| """ |
| |
| |
| class TestCase: |
| def __init__(self, input_path, parse_only): |
| self.input_path = input_path |
| self.parse_only = parse_only |
| self.results = {} |
| |
| def IsExpectedFail(self): |
| fail_re = re.compile('^.+[\.]fail[\.]wgsl') |
| return fail_re.match(self.GetInputPath()) |
| |
| def IsParseOnly(self): |
| return self.parse_only |
| |
| def GetInputPath(self): |
| return self.input_path |
| |
| def GetResult(self, fmt): |
| return self.results[fmt] |
| |
| def GetReason(self): |
| with open(self.GetInputPath()) as test: |
| first_line = test.readline() |
| if (first_line.startswith("# v-")): |
| reason = first_line[2:8] |
| else: |
| reason = '' |
| return reason |
| |
| |
| """ |
| The test runner, will execute a series of test cases and record the |
| results. |
| """ |
| |
| |
| class TestRunner: |
| def RunTest(self, tc): |
| """Runs a single test.""" |
| print("Testing {}".format(tc.GetInputPath())) |
| |
| cmd = [self.options.test_prog_path] |
| if tc.IsParseOnly(): |
| cmd += ['--parse-only'] |
| |
| languages = ["wgsl", "spvasm", "msl", "hlsl"] |
| try: |
| for lang in languages: |
| lang_cmd = copy.copy(cmd) |
| lang_cmd += ['--format', lang] |
| lang_cmd += [tc.GetInputPath()] |
| err = subprocess.check_output(lang_cmd, |
| stderr=subprocess.STDOUT) |
| |
| except Exception as e: |
| failure_reason = "{}".format("".join(map(chr, |
| bytearray(e.output)))) |
| if tc.IsExpectedFail(): |
| right_reason = tc.GetReason() |
| if (right_reason in failure_reason): |
| return False, "" |
| else: |
| return False, right_reason |
| |
| if not tc.IsExpectedFail(): |
| print(failure_reason) |
| print(e) |
| return False, "" |
| |
| return True, "" |
| |
| def RunTests(self): |
| """Runs a set of test cases""" |
| for tc in self.test_cases: |
| result, reason = self.RunTest(tc) |
| """evaluate final result based on result, tc.IsExpectedFail() and reason""" |
| if not result: |
| # result == false, expected true, reason:don't care |
| if not tc.IsExpectedFail(): |
| print("Expected: " + tc.GetInputPath() + |
| " to pass but failed.") |
| self.failures.append(tc.GetInputPath()) |
| # result == false, expected false, reason: wrong |
| else: |
| if reason.startswith("v-"): |
| print("Failed for a wrong reason: " + |
| tc.GetInputPath() + |
| " expected with error code: " + reason) |
| self.failures_wrong_reason.append(tc.GetInputPath()) |
| # result == true, expected false, reason:don't care |
| elif tc.IsExpectedFail(): |
| print("Expected: " + tc.GetInputPath() + |
| " to fail but passed.") |
| self.failures.append(tc.GetInputPath()) |
| |
| def GetUnexpectedFailures(self): |
| for failure in self.failures + self.failures_wrong_reason: |
| if failure not in self.known_failures: |
| self.unexpected_failures.append(failure) |
| return |
| |
| def SummarizeResults(self): |
| """Prints a summarization of the test results to STDOUT""" |
| if len(self.unexpected_failures): |
| self.unexpected_failures.sort() |
| print('\nSummary of unexpected failures:') |
| for unexpected_fail in self.unexpected_failures: |
| print(unexpected_fail) |
| |
| for f in self.known_failures: |
| if f not in self.failures_wrong_reason + self.failures: |
| self.unexpected_successes.append(f) |
| |
| if len(self.unexpected_successes): |
| print('\nSummary of unexpected successes:') |
| for s in self.unexpected_successes: |
| print(s) |
| |
| print('') |
| print('Test cases executed: {}'.format(len(self.test_cases))) |
| print(' Successes: {}'.format( |
| (len(self.test_cases) - len(self.failures) - |
| len(self.failures_wrong_reason)))) |
| print(' Failures: {}'.format( |
| len(self.failures) + len(self.failures_wrong_reason))) |
| print(' Unexpected Failures: {}'.format(len( |
| self.unexpected_failures))) |
| print(' Unexpected Successes: {}'.format( |
| len(self.unexpected_successes))) |
| print('') |
| |
| def Run(self): |
| """Executes the test runner.""" |
| base_path = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), '..')) |
| |
| usage = 'usage: %prog [options] (file)' |
| parser = optparse.OptionParser(usage=usage) |
| parser.add_option('--build-dir', |
| default=os.path.join(base_path, 'out', 'Debug'), |
| help='path to build directory') |
| parser.add_option('--test-dir', |
| default=os.path.join(os.path.dirname(__file__), '..', |
| 'third_party', 'gpuweb-cts', |
| 'src', 'webgpu', 'shader', |
| 'validation', 'wgsl'), |
| help='path to directory containing test files') |
| parser.add_option( |
| '--known-failures-file', |
| default=os.path.join(base_path, 'tools', 'known_tint_failures'), |
| help='path to directory containing the known failures file') |
| parser.add_option( |
| '--test-prog-path', |
| default=None, |
| help='path to program to test (default build-dir/tint)') |
| parser.add_option('--parse-only', |
| action="store_true", |
| default=False, |
| help='only parse test cases; do not compile') |
| |
| self.options, self.args = parser.parse_args() |
| |
| if self.options.test_prog_path == None: |
| test_prog = os.path.abspath( |
| os.path.join(self.options.build_dir, 'tint')) |
| if not os.path.isfile(test_prog): |
| print("Cannot find test program {}".format(test_prog)) |
| return 1 |
| |
| self.options.test_prog_path = test_prog |
| |
| if not os.path.isfile(self.options.test_prog_path): |
| print("Cannot find test program '{}'".format( |
| self.options.test_prog_path)) |
| return 1 |
| |
| input_file_re = re.compile('^.+[\.]wgsl') |
| self.test_cases = [] |
| |
| if self.args: |
| for filename in self.args: |
| input_path = os.path.join(self.options.test_dir, filename) |
| if not os.path.isfile(input_path): |
| print("Cannot find test file '{}'".format(filename)) |
| return 1 |
| |
| self.test_cases.append( |
| TestCase(input_path, self.options.parse_only)) |
| |
| else: |
| for file_dir, _, filename_list in os.walk(self.options.test_dir): |
| for input_filename in filename_list: |
| if input_file_re.match(input_filename): |
| input_path = os.path.join(file_dir, input_filename) |
| if os.path.isfile(input_path): |
| self.test_cases.append( |
| TestCase(input_path, self.options.parse_only)) |
| known_failure_file = self.options.known_failures_file |
| self.known_failures = [] |
| with open(known_failure_file, 'r') as f: |
| for failure_filename in f.read().splitlines(): |
| self.known_failures.append( |
| os.path.join(self.options.test_dir, failure_filename)) |
| |
| self.failures = [] |
| self.failures_wrong_reason = [] |
| self.unexpected_failures = [] |
| self.unexpected_successes = [] |
| |
| self.RunTests() |
| self.GetUnexpectedFailures() |
| self.SummarizeResults() |
| |
| return not len(self.unexpected_failures + self.unexpected_successes) |
| |
| |
| def main(): |
| runner = TestRunner() |
| return runner.Run() |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
