Corentin Wallez | 59382b7 | 2020-04-17 20:43:07 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 2 | # |
| 3 | # Copyright 2019 The Dawn Authors |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | # Based on Angle's perf_test_runner.py |
| 18 | |
| 19 | import glob |
| 20 | import subprocess |
| 21 | import sys |
| 22 | import os |
| 23 | import re |
| 24 | |
Kai Ninomiya | 01aeca2 | 2020-07-15 19:51:17 +0000 | [diff] [blame] | 25 | base_path = os.path.abspath( |
| 26 | os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 27 | |
| 28 | # Look for a [Rr]elease build. |
| 29 | perftests_paths = glob.glob('out/*elease*') |
| 30 | metric = 'wall_time' |
| 31 | max_experiments = 10 |
| 32 | |
| 33 | binary_name = 'dawn_perf_tests' |
| 34 | if sys.platform == 'win32': |
| 35 | binary_name += '.exe' |
| 36 | |
| 37 | scores = [] |
| 38 | |
Kai Ninomiya | 01aeca2 | 2020-07-15 19:51:17 +0000 | [diff] [blame] | 39 | |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 40 | def mean(data): |
| 41 | """Return the sample arithmetic mean of data.""" |
| 42 | n = len(data) |
| 43 | if n < 1: |
| 44 | raise ValueError('mean requires at least one data point') |
| 45 | return float(sum(data)) / float(n) # in Python 2 use sum(data)/float(n) |
| 46 | |
| 47 | |
| 48 | def sum_of_square_deviations(data, c): |
| 49 | """Return sum of square deviations of sequence data.""" |
| 50 | ss = sum((float(x) - c)**2 for x in data) |
| 51 | return ss |
| 52 | |
| 53 | |
| 54 | def coefficient_of_variation(data): |
| 55 | """Calculates the population coefficient of variation.""" |
| 56 | n = len(data) |
| 57 | if n < 2: |
| 58 | raise ValueError('variance requires at least two data points') |
| 59 | c = mean(data) |
| 60 | ss = sum_of_square_deviations(data, c) |
| 61 | pvar = ss / n # the population variance |
| 62 | stddev = (pvar**0.5) # population standard deviation |
| 63 | return stddev / c |
| 64 | |
| 65 | |
| 66 | def truncated_list(data, n): |
| 67 | """Compute a truncated list, n is truncation size""" |
| 68 | if len(data) < n * 2: |
| 69 | raise ValueError('list not large enough to truncate') |
| 70 | return sorted(data)[n:-n] |
| 71 | |
| 72 | |
| 73 | def truncated_mean(data, n): |
| 74 | """Compute a truncated mean, n is truncation size""" |
| 75 | return mean(truncated_list(data, n)) |
| 76 | |
| 77 | |
| 78 | def truncated_cov(data, n): |
| 79 | """Compute a truncated coefficient of variation, n is truncation size""" |
| 80 | return coefficient_of_variation(truncated_list(data, n)) |
| 81 | |
| 82 | |
| 83 | # Find most recent binary |
| 84 | newest_binary = None |
| 85 | newest_mtime = None |
| 86 | |
| 87 | for path in perftests_paths: |
| 88 | binary_path = os.path.join(base_path, path, binary_name) |
| 89 | if os.path.exists(binary_path): |
| 90 | binary_mtime = os.path.getmtime(binary_path) |
| 91 | if (newest_binary is None) or (binary_mtime > newest_mtime): |
| 92 | newest_binary = binary_path |
| 93 | newest_mtime = binary_mtime |
| 94 | |
| 95 | perftests_path = newest_binary |
| 96 | |
| 97 | if perftests_path == None or not os.path.exists(perftests_path): |
| 98 | print('Cannot find Release %s!' % binary_name) |
| 99 | sys.exit(1) |
| 100 | |
| 101 | if len(sys.argv) >= 2: |
| 102 | test_name = sys.argv[1] |
| 103 | |
| 104 | print('Using test executable: ' + perftests_path) |
| 105 | print('Test name: ' + test_name) |
| 106 | |
Kai Ninomiya | 01aeca2 | 2020-07-15 19:51:17 +0000 | [diff] [blame] | 107 | |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 108 | def get_results(metric, extra_args=[]): |
| 109 | process = subprocess.Popen( |
| 110 | [perftests_path, '--gtest_filter=' + test_name] + extra_args, |
| 111 | stdout=subprocess.PIPE, |
| 112 | stderr=subprocess.PIPE) |
| 113 | output, err = process.communicate() |
| 114 | |
| 115 | m = re.search(r'Running (\d+) tests', output) |
| 116 | if m and int(m.group(1)) > 1: |
| 117 | print("Found more than one test result in output:") |
| 118 | print(output) |
| 119 | sys.exit(3) |
| 120 | |
Bryan Bernhart | 4b1be08 | 2020-03-26 17:46:25 +0000 | [diff] [blame] | 121 | pattern = metric + r'.*= ([0-9.]+)' |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 122 | m = re.findall(pattern, output) |
Bryan Bernhart | 4b1be08 | 2020-03-26 17:46:25 +0000 | [diff] [blame] | 123 | if not m: |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 124 | print("Did not find the metric '%s' in the test output:" % metric) |
| 125 | print(output) |
| 126 | sys.exit(1) |
| 127 | |
| 128 | return [float(value) for value in m] |
| 129 | |
| 130 | |
| 131 | # Calibrate the number of steps |
| 132 | steps = get_results("steps", ["--calibration"])[0] |
| 133 | print("running with %d steps." % steps) |
| 134 | |
| 135 | # Loop 'max_experiments' times, running the tests. |
| 136 | for experiment in range(max_experiments): |
| 137 | experiment_scores = get_results(metric, ["--override-steps", str(steps)]) |
| 138 | |
| 139 | for score in experiment_scores: |
| 140 | sys.stdout.write("%s: %.2f" % (metric, score)) |
| 141 | scores.append(score) |
| 142 | |
| 143 | if (len(scores) > 1): |
| 144 | sys.stdout.write(", mean: %.2f" % mean(scores)) |
Kai Ninomiya | 01aeca2 | 2020-07-15 19:51:17 +0000 | [diff] [blame] | 145 | sys.stdout.write(", variation: %.2f%%" % |
| 146 | (coefficient_of_variation(scores) * 100.0)) |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 147 | |
| 148 | if (len(scores) > 7): |
| 149 | truncation_n = len(scores) >> 3 |
Kai Ninomiya | 01aeca2 | 2020-07-15 19:51:17 +0000 | [diff] [blame] | 150 | sys.stdout.write(", truncated mean: %.2f" % |
| 151 | truncated_mean(scores, truncation_n)) |
| 152 | sys.stdout.write(", variation: %.2f%%" % |
| 153 | (truncated_cov(scores, truncation_n) * 100.0)) |
Austin Eng | ca0eac3 | 2019-08-28 23:18:10 +0000 | [diff] [blame] | 154 | |
| 155 | print("") |