blob: a2836381fba7ae21e0bad445bb5c7fc18922e959 [file] [log] [blame]
Ben Claytonbe2362b2022-01-18 18:58:16 +00001// Copyright 2022 The Tint Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Package bench provides types and methods for parsing Google benchmark results.
16package bench
17
18import (
19 "encoding/json"
20 "errors"
21 "fmt"
22 "regexp"
Ben Claytonc126bc92022-01-27 14:51:06 +000023 "sort"
Ben Claytonbe2362b2022-01-18 18:58:16 +000024 "strconv"
25 "strings"
26 "time"
Ben Claytonc126bc92022-01-27 14:51:06 +000027 "unicode/utf8"
Ben Claytonbe2362b2022-01-18 18:58:16 +000028)
29
Ben Claytonc126bc92022-01-27 14:51:06 +000030// Run holds all the benchmark results for a run, along with the context
31// information for the run.
32type Run struct {
33 Benchmarks []Benchmark
34 Context *Context
Ben Claytonbe2362b2022-01-18 18:58:16 +000035}
36
Ben Claytonc126bc92022-01-27 14:51:06 +000037// Context provides information about the environment used to perform the
38// benchmark.
39type Context struct {
40 Date time.Time
41 HostName string
42 Executable string
43 NumCPUs int
44 MhzPerCPU int
45 CPUScalingEnabled bool
46 Caches []ContextCache
47 LoadAvg []float32
48 LibraryBuildType string
Ben Claytonbe2362b2022-01-18 18:58:16 +000049}
50
Ben Claytonc126bc92022-01-27 14:51:06 +000051// ContextCache holds information about one of the system caches.
52type ContextCache struct {
53 Type string
54 Level int
55 Size int
56 NumSharing int
57}
58
59// Benchmark holds the results of a single benchmark test.
Ben Claytonbe2362b2022-01-18 18:58:16 +000060type Benchmark struct {
Ben Claytonc126bc92022-01-27 14:51:06 +000061 Name string
62 Duration time.Duration
63 AggregateType AggregateType
Ben Claytonbe2362b2022-01-18 18:58:16 +000064}
65
Ben Claytonc126bc92022-01-27 14:51:06 +000066// AggregateType is an enumerator of benchmark aggregate types.
67type AggregateType string
68
69// Enumerator values of AggregateType
70const (
71 NonAggregate AggregateType = "NonAggregate"
72 Mean AggregateType = "mean"
73 Median AggregateType = "median"
74 Stddev AggregateType = "stddev"
75)
76
Ben Claytonbe2362b2022-01-18 18:58:16 +000077// Parse parses the benchmark results from the string s.
78// Parse will handle the json and 'console' formats.
Ben Claytonc126bc92022-01-27 14:51:06 +000079func Parse(s string) (Run, error) {
80 type Parser = func(s string) (Run, error)
Ben Claytonbe2362b2022-01-18 18:58:16 +000081 for _, parser := range []Parser{parseConsole, parseJSON} {
Ben Claytonc126bc92022-01-27 14:51:06 +000082 r, err := parser(s)
Ben Claytonbe2362b2022-01-18 18:58:16 +000083 switch err {
84 case nil:
Ben Claytonc126bc92022-01-27 14:51:06 +000085 return r, nil
Ben Claytonbe2362b2022-01-18 18:58:16 +000086 case errWrongFormat:
87 default:
Ben Claytonc126bc92022-01-27 14:51:06 +000088 return Run{}, err
Ben Claytonbe2362b2022-01-18 18:58:16 +000089 }
90 }
91
Ben Claytonc126bc92022-01-27 14:51:06 +000092 return Run{}, errors.New("Unrecognised file format")
Ben Claytonbe2362b2022-01-18 18:58:16 +000093}
94
95var errWrongFormat = errors.New("Wrong format")
96var consoleLineRE = regexp.MustCompile(`([\w/:]+)\s+([0-9]+(?:.[0-9]+)?) ns\s+[0-9]+(?:.[0-9]+) ns\s+([0-9]+)`)
97
Ben Claytonc126bc92022-01-27 14:51:06 +000098func parseConsole(s string) (Run, error) {
Ben Claytonbe2362b2022-01-18 18:58:16 +000099 blocks := strings.Split(s, "------------------------------------------------------------------------------------------")
100 if len(blocks) != 3 {
Ben Claytonc126bc92022-01-27 14:51:06 +0000101 return Run{}, errWrongFormat
Ben Claytonbe2362b2022-01-18 18:58:16 +0000102 }
103
104 lines := strings.Split(blocks[2], "\n")
Ben Claytonc126bc92022-01-27 14:51:06 +0000105 b := make([]Benchmark, 0, len(lines))
106
Ben Claytonbe2362b2022-01-18 18:58:16 +0000107 for _, line := range lines {
108 if len(line) == 0 {
109 continue
110 }
111 matches := consoleLineRE.FindStringSubmatch(line)
112 if len(matches) != 4 {
Ben Claytonc126bc92022-01-27 14:51:06 +0000113 return Run{}, fmt.Errorf("Unable to parse the line:\n" + line)
Ben Claytonbe2362b2022-01-18 18:58:16 +0000114 }
115 ns, err := strconv.ParseFloat(matches[2], 64)
116 if err != nil {
Ben Claytonc126bc92022-01-27 14:51:06 +0000117 return Run{}, fmt.Errorf("Unable to parse the duration: " + matches[2])
Ben Claytonbe2362b2022-01-18 18:58:16 +0000118 }
119
Ben Claytonc126bc92022-01-27 14:51:06 +0000120 b = append(b, Benchmark{
121 Name: trimAggregateSuffix(matches[1]),
122 Duration: time.Nanosecond * time.Duration(ns),
123 })
Ben Claytonbe2362b2022-01-18 18:58:16 +0000124 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000125 return Run{Benchmarks: b}, nil
Ben Claytonbe2362b2022-01-18 18:58:16 +0000126}
127
Ben Claytonc126bc92022-01-27 14:51:06 +0000128func parseJSON(s string) (Run, error) {
129 type Data struct {
130 Context struct {
131 Date time.Time `json:"date"`
132 HostName string `json:"host_name"`
133 Executable string `json:"executable"`
134 NumCPUs int `json:"num_cpus"`
135 MhzPerCPU int `json:"mhz_per_cpu"`
136 CPUScalingEnabled bool `json:"cpu_scaling_enabled"`
137 LoadAvg []float32 `json:"load_avg"`
138 LibraryBuildType string `json:"library_build_type"`
139 Caches []struct {
140 Type string `json:"type"`
141 Level int `json:"level"`
142 Size int `json:"size"`
143 NumSharing int `json:"num_sharing"`
144 } `json:"caches"`
145 } `json:"context"`
146 Benchmarks []struct {
147 Name string `json:"name"`
148 Time float64 `json:"real_time"`
149 AggregateType AggregateType `json:"aggregate_name"`
150 } `json:"benchmarks"`
Ben Claytonbe2362b2022-01-18 18:58:16 +0000151 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000152 data := Data{}
Ben Claytonbe2362b2022-01-18 18:58:16 +0000153 d := json.NewDecoder(strings.NewReader(s))
Ben Claytonc126bc92022-01-27 14:51:06 +0000154 if err := d.Decode(&data); err != nil {
155 return Run{}, err
Ben Claytonbe2362b2022-01-18 18:58:16 +0000156 }
157
Ben Claytonc126bc92022-01-27 14:51:06 +0000158 out := Run{
159 Benchmarks: make([]Benchmark, len(data.Benchmarks)),
160 Context: &Context{
161 Date: data.Context.Date,
162 HostName: data.Context.HostName,
163 Executable: data.Context.Executable,
164 NumCPUs: data.Context.NumCPUs,
165 MhzPerCPU: data.Context.MhzPerCPU,
166 CPUScalingEnabled: data.Context.CPUScalingEnabled,
167 LoadAvg: data.Context.LoadAvg,
168 LibraryBuildType: data.Context.LibraryBuildType,
169 Caches: make([]ContextCache, len(data.Context.Caches)),
170 },
Ben Claytonbe2362b2022-01-18 18:58:16 +0000171 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000172 for i, c := range data.Context.Caches {
173 out.Context.Caches[i] = ContextCache{
174 Type: c.Type,
175 Level: c.Level,
176 Size: c.Size,
177 NumSharing: c.NumSharing,
Ben Claytonbe2362b2022-01-18 18:58:16 +0000178 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000179 }
180 for i, b := range data.Benchmarks {
181 out.Benchmarks[i] = Benchmark{
182 Name: trimAggregateSuffix(b.Name),
183 Duration: time.Nanosecond * time.Duration(int64(b.Time)),
184 AggregateType: b.AggregateType,
185 }
Ben Claytonbe2362b2022-01-18 18:58:16 +0000186 }
187
188 return out, nil
189}
Ben Claytonc126bc92022-01-27 14:51:06 +0000190
191// Diff describes the difference between two benchmarks
192type Diff struct {
193 TestName string
194 Delta time.Duration // Δ (A → B)
195 PercentChangeAB float64 // % (A → B)
196 PercentChangeBA float64 // % (A → B)
197 MultiplierChangeAB float64 // × (A → B)
198 MultiplierChangeBA float64 // × (A → B)
199 TimeA time.Duration // A
200 TimeB time.Duration // B
201}
202
203// Diffs is a list of Diff
204type Diffs []Diff
205
206// DiffFormat describes how a list of diffs should be formatted
207type DiffFormat struct {
208 TestName bool
209 Delta bool
210 PercentChangeAB bool
211 PercentChangeBA bool
212 MultiplierChangeAB bool
213 MultiplierChangeBA bool
214 TimeA bool
215 TimeB bool
216}
217
218func (diffs Diffs) Format(f DiffFormat) string {
219 if len(diffs) == 0 {
220 return "<no changes>"
221 }
222
223 type row []string
224
225 header := row{}
226 if f.TestName {
227 header = append(header, "Test name")
228 }
229 if f.Delta {
230 header = append(header, "Δ (A → B)")
231 }
232 if f.PercentChangeAB {
233 header = append(header, "% (A → B)")
234 }
235 if f.PercentChangeBA {
236 header = append(header, "% (B → A)")
237 }
238 if f.MultiplierChangeAB {
239 header = append(header, "× (A → B)")
240 }
241 if f.MultiplierChangeBA {
242 header = append(header, "× (B → A)")
243 }
244 if f.TimeA {
245 header = append(header, "A")
246 }
247 if f.TimeB {
248 header = append(header, "B")
249 }
250 if len(header) == 0 {
251 return ""
252 }
253
254 columns := []row{}
255 for _, d := range diffs {
256 r := make(row, 0, len(header))
257 if f.TestName {
258 r = append(r, d.TestName)
259 }
260 if f.Delta {
261 r = append(r, fmt.Sprintf("%v", d.Delta))
262 }
263 if f.PercentChangeAB {
264 r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeAB))
265 }
266 if f.PercentChangeBA {
267 r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeBA))
268 }
269 if f.MultiplierChangeAB {
270 r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeAB))
271 }
272 if f.MultiplierChangeBA {
273 r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeBA))
274 }
275 if f.TimeA {
276 r = append(r, fmt.Sprintf("%v", d.TimeA))
277 }
278 if f.TimeB {
279 r = append(r, fmt.Sprintf("%v", d.TimeB))
280 }
281 columns = append(columns, r)
282 }
283
284 // measure
285 widths := make([]int, len(header))
286 for i, h := range header {
287 widths[i] = utf8.RuneCountInString(h)
288 }
289 for _, row := range columns {
290 for i, cell := range row {
291 l := utf8.RuneCountInString(cell)
292 if widths[i] < l {
293 widths[i] = l
294 }
295 }
296 }
297
298 pad := func(s string, i int) string {
299 if n := i - utf8.RuneCountInString(s); n > 0 {
300 return s + strings.Repeat(" ", n)
301 }
302 return s
303 }
304
305 // Draw table
306 b := &strings.Builder{}
307
308 horizontal_bar := func() {
309 for i := range header {
310 fmt.Fprintf(b, "+%v", strings.Repeat("-", 2+widths[i]))
311 }
312 fmt.Fprintln(b, "+")
313 }
314
315 horizontal_bar()
316
317 for i, h := range header {
318 fmt.Fprintf(b, "| %v ", pad(h, widths[i]))
319 }
320 fmt.Fprintln(b, "|")
321
322 horizontal_bar()
323
324 for _, row := range columns {
325 for i, cell := range row {
326 fmt.Fprintf(b, "| %v ", pad(cell, widths[i]))
327 }
328 fmt.Fprintln(b, "|")
329 }
330
331 horizontal_bar()
332
333 return b.String()
334}
335
336// Compare returns a string describing differences in the two benchmarks
337// Absolute benchmark differences less than minDiff are omitted
338// Absolute relative differences between [1, 1+x] are omitted
339func Compare(a, b []Benchmark, minDiff time.Duration, minRelDiff float64) Diffs {
340 type times struct {
341 a time.Duration
342 b time.Duration
343 }
344 byName := map[string]times{}
345 for _, test := range a {
346 byName[test.Name] = times{a: test.Duration}
347 }
348 for _, test := range b {
349 t := byName[test.Name]
350 t.b = test.Duration
351 byName[test.Name] = t
352 }
353
354 type delta struct {
355 name string
356 times times
357 relDiff float64
358 absRelDiff float64
359 }
360 deltas := []delta{}
361 for name, times := range byName {
362 if times.a == 0 || times.b == 0 {
363 continue // Assuming test was missing from a or b
364 }
365 diff := times.b - times.a
366 absDiff := diff
367 if absDiff < 0 {
368 absDiff = -absDiff
369 }
370 if absDiff < minDiff {
371 continue
372 }
373
374 relDiff := float64(times.b) / float64(times.a)
375 absRelDiff := relDiff
376 if absRelDiff < 1 {
377 absRelDiff = 1.0 / absRelDiff
378 }
379 if absRelDiff < (1.0 + minRelDiff) {
380 continue
381 }
382
383 d := delta{
384 name: name,
385 times: times,
386 relDiff: relDiff,
387 absRelDiff: absRelDiff,
388 }
389 deltas = append(deltas, d)
390 }
391
392 sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
393
394 out := make(Diffs, len(deltas))
395
396 for i, delta := range deltas {
397 a2b := delta.times.b - delta.times.a
398 out[i] = Diff{
399 TestName: delta.name,
400 Delta: a2b,
401 PercentChangeAB: 100 * float64(a2b) / float64(delta.times.a),
402 PercentChangeBA: 100 * float64(-a2b) / float64(delta.times.b),
403 MultiplierChangeAB: float64(delta.times.b) / float64(delta.times.a),
404 MultiplierChangeBA: float64(delta.times.a) / float64(delta.times.b),
405 TimeA: delta.times.a,
406 TimeB: delta.times.b,
407 }
408 }
409 return out
410}
411
412func trimAggregateSuffix(name string) string {
413 name = strings.TrimSuffix(name, "_stddev")
414 name = strings.TrimSuffix(name, "_mean")
415 name = strings.TrimSuffix(name, "_median")
416 return name
417}