blob: 47776c54e74376382fdfa2306b04731a76ed4b84 [file] [log] [blame]
Austin Engcc2516a2023-10-17 20:57:54 +00001// Copyright 2022 The Dawn & Tint Authors
Ben Claytonbe2362b2022-01-18 18:58:16 +00002//
Austin Engcc2516a2023-10-17 20:57:54 +00003// Redistribution and use in source and binary forms, with or without
4// modification, are permitted provided that the following conditions are met:
Ben Claytonbe2362b2022-01-18 18:58:16 +00005//
Austin Engcc2516a2023-10-17 20:57:54 +00006// 1. Redistributions of source code must retain the above copyright notice, this
7// list of conditions and the following disclaimer.
Ben Claytonbe2362b2022-01-18 18:58:16 +00008//
Austin Engcc2516a2023-10-17 20:57:54 +00009// 2. Redistributions in binary form must reproduce the above copyright notice,
10// this list of conditions and the following disclaimer in the documentation
11// and/or other materials provided with the distribution.
12//
13// 3. Neither the name of the copyright holder nor the names of its
14// contributors may be used to endorse or promote products derived from
15// this software without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
21// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
24// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Ben Claytonbe2362b2022-01-18 18:58:16 +000027
28// Package bench provides types and methods for parsing Google benchmark results.
29package bench
30
31import (
32 "encoding/json"
33 "errors"
34 "fmt"
35 "regexp"
Ben Claytonc126bc92022-01-27 14:51:06 +000036 "sort"
Ben Claytonbe2362b2022-01-18 18:58:16 +000037 "strconv"
38 "strings"
39 "time"
Ben Claytonc126bc92022-01-27 14:51:06 +000040 "unicode/utf8"
Ben Claytonbe2362b2022-01-18 18:58:16 +000041)
42
Ben Claytonc126bc92022-01-27 14:51:06 +000043// Run holds all the benchmark results for a run, along with the context
44// information for the run.
45type Run struct {
46 Benchmarks []Benchmark
47 Context *Context
Ben Claytonbe2362b2022-01-18 18:58:16 +000048}
49
Ben Claytonc126bc92022-01-27 14:51:06 +000050// Context provides information about the environment used to perform the
51// benchmark.
52type Context struct {
53 Date time.Time
54 HostName string
55 Executable string
56 NumCPUs int
57 MhzPerCPU int
58 CPUScalingEnabled bool
59 Caches []ContextCache
60 LoadAvg []float32
61 LibraryBuildType string
Ben Claytonbe2362b2022-01-18 18:58:16 +000062}
63
Ben Claytonc126bc92022-01-27 14:51:06 +000064// ContextCache holds information about one of the system caches.
65type ContextCache struct {
66 Type string
67 Level int
68 Size int
69 NumSharing int
70}
71
72// Benchmark holds the results of a single benchmark test.
Ben Claytonbe2362b2022-01-18 18:58:16 +000073type Benchmark struct {
Ben Claytonc126bc92022-01-27 14:51:06 +000074 Name string
75 Duration time.Duration
76 AggregateType AggregateType
Ben Claytonbe2362b2022-01-18 18:58:16 +000077}
78
Ben Claytonc126bc92022-01-27 14:51:06 +000079// AggregateType is an enumerator of benchmark aggregate types.
80type AggregateType string
81
82// Enumerator values of AggregateType
83const (
84 NonAggregate AggregateType = "NonAggregate"
85 Mean AggregateType = "mean"
86 Median AggregateType = "median"
87 Stddev AggregateType = "stddev"
88)
89
Ben Claytonbe2362b2022-01-18 18:58:16 +000090// Parse parses the benchmark results from the string s.
91// Parse will handle the json and 'console' formats.
Ben Claytonc126bc92022-01-27 14:51:06 +000092func Parse(s string) (Run, error) {
93 type Parser = func(s string) (Run, error)
Ben Claytonbe2362b2022-01-18 18:58:16 +000094 for _, parser := range []Parser{parseConsole, parseJSON} {
Ben Claytonc126bc92022-01-27 14:51:06 +000095 r, err := parser(s)
Ben Claytonbe2362b2022-01-18 18:58:16 +000096 switch err {
97 case nil:
Ben Claytonc126bc92022-01-27 14:51:06 +000098 return r, nil
Ben Claytonbe2362b2022-01-18 18:58:16 +000099 case errWrongFormat:
100 default:
Ben Claytonc126bc92022-01-27 14:51:06 +0000101 return Run{}, err
Ben Claytonbe2362b2022-01-18 18:58:16 +0000102 }
103 }
104
Ben Claytonc126bc92022-01-27 14:51:06 +0000105 return Run{}, errors.New("Unrecognised file format")
Ben Claytonbe2362b2022-01-18 18:58:16 +0000106}
107
108var errWrongFormat = errors.New("Wrong format")
109var consoleLineRE = regexp.MustCompile(`([\w/:]+)\s+([0-9]+(?:.[0-9]+)?) ns\s+[0-9]+(?:.[0-9]+) ns\s+([0-9]+)`)
110
Ben Claytonc126bc92022-01-27 14:51:06 +0000111func parseConsole(s string) (Run, error) {
Ben Claytonbe2362b2022-01-18 18:58:16 +0000112 blocks := strings.Split(s, "------------------------------------------------------------------------------------------")
113 if len(blocks) != 3 {
Ben Claytonc126bc92022-01-27 14:51:06 +0000114 return Run{}, errWrongFormat
Ben Claytonbe2362b2022-01-18 18:58:16 +0000115 }
116
117 lines := strings.Split(blocks[2], "\n")
Ben Claytonc126bc92022-01-27 14:51:06 +0000118 b := make([]Benchmark, 0, len(lines))
119
Ben Claytonbe2362b2022-01-18 18:58:16 +0000120 for _, line := range lines {
121 if len(line) == 0 {
122 continue
123 }
124 matches := consoleLineRE.FindStringSubmatch(line)
125 if len(matches) != 4 {
Ben Claytonc126bc92022-01-27 14:51:06 +0000126 return Run{}, fmt.Errorf("Unable to parse the line:\n" + line)
Ben Claytonbe2362b2022-01-18 18:58:16 +0000127 }
128 ns, err := strconv.ParseFloat(matches[2], 64)
129 if err != nil {
Ben Claytonc126bc92022-01-27 14:51:06 +0000130 return Run{}, fmt.Errorf("Unable to parse the duration: " + matches[2])
Ben Claytonbe2362b2022-01-18 18:58:16 +0000131 }
132
Ben Claytonc126bc92022-01-27 14:51:06 +0000133 b = append(b, Benchmark{
134 Name: trimAggregateSuffix(matches[1]),
135 Duration: time.Nanosecond * time.Duration(ns),
136 })
Ben Claytonbe2362b2022-01-18 18:58:16 +0000137 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000138 return Run{Benchmarks: b}, nil
Ben Claytonbe2362b2022-01-18 18:58:16 +0000139}
140
Ben Claytonc126bc92022-01-27 14:51:06 +0000141func parseJSON(s string) (Run, error) {
142 type Data struct {
143 Context struct {
144 Date time.Time `json:"date"`
145 HostName string `json:"host_name"`
146 Executable string `json:"executable"`
147 NumCPUs int `json:"num_cpus"`
148 MhzPerCPU int `json:"mhz_per_cpu"`
149 CPUScalingEnabled bool `json:"cpu_scaling_enabled"`
150 LoadAvg []float32 `json:"load_avg"`
151 LibraryBuildType string `json:"library_build_type"`
152 Caches []struct {
153 Type string `json:"type"`
154 Level int `json:"level"`
155 Size int `json:"size"`
156 NumSharing int `json:"num_sharing"`
157 } `json:"caches"`
158 } `json:"context"`
159 Benchmarks []struct {
160 Name string `json:"name"`
Ben Clayton9dede342023-04-28 09:25:44 +0000161 Time float64 `json:"cpu_time"`
Ben Claytonc126bc92022-01-27 14:51:06 +0000162 AggregateType AggregateType `json:"aggregate_name"`
163 } `json:"benchmarks"`
Ben Claytonbe2362b2022-01-18 18:58:16 +0000164 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000165 data := Data{}
Ben Claytonbe2362b2022-01-18 18:58:16 +0000166 d := json.NewDecoder(strings.NewReader(s))
Ben Claytonc126bc92022-01-27 14:51:06 +0000167 if err := d.Decode(&data); err != nil {
168 return Run{}, err
Ben Claytonbe2362b2022-01-18 18:58:16 +0000169 }
170
Ben Claytonc126bc92022-01-27 14:51:06 +0000171 out := Run{
172 Benchmarks: make([]Benchmark, len(data.Benchmarks)),
173 Context: &Context{
174 Date: data.Context.Date,
175 HostName: data.Context.HostName,
176 Executable: data.Context.Executable,
177 NumCPUs: data.Context.NumCPUs,
178 MhzPerCPU: data.Context.MhzPerCPU,
179 CPUScalingEnabled: data.Context.CPUScalingEnabled,
180 LoadAvg: data.Context.LoadAvg,
181 LibraryBuildType: data.Context.LibraryBuildType,
182 Caches: make([]ContextCache, len(data.Context.Caches)),
183 },
Ben Claytonbe2362b2022-01-18 18:58:16 +0000184 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000185 for i, c := range data.Context.Caches {
186 out.Context.Caches[i] = ContextCache{
187 Type: c.Type,
188 Level: c.Level,
189 Size: c.Size,
190 NumSharing: c.NumSharing,
Ben Claytonbe2362b2022-01-18 18:58:16 +0000191 }
Ben Claytonc126bc92022-01-27 14:51:06 +0000192 }
193 for i, b := range data.Benchmarks {
194 out.Benchmarks[i] = Benchmark{
195 Name: trimAggregateSuffix(b.Name),
196 Duration: time.Nanosecond * time.Duration(int64(b.Time)),
197 AggregateType: b.AggregateType,
198 }
Ben Claytonbe2362b2022-01-18 18:58:16 +0000199 }
200
201 return out, nil
202}
Ben Claytonc126bc92022-01-27 14:51:06 +0000203
204// Diff describes the difference between two benchmarks
205type Diff struct {
206 TestName string
207 Delta time.Duration // Δ (A → B)
208 PercentChangeAB float64 // % (A → B)
David Neto79a1c202024-02-07 12:16:31 +0000209 PercentChangeBA float64 // % (B → A)
Ben Claytonc126bc92022-01-27 14:51:06 +0000210 MultiplierChangeAB float64 // × (A → B)
David Neto79a1c202024-02-07 12:16:31 +0000211 MultiplierChangeBA float64 // × (B → A)
Ben Claytonc126bc92022-01-27 14:51:06 +0000212 TimeA time.Duration // A
213 TimeB time.Duration // B
214}
215
216// Diffs is a list of Diff
217type Diffs []Diff
218
219// DiffFormat describes how a list of diffs should be formatted
220type DiffFormat struct {
221 TestName bool
222 Delta bool
223 PercentChangeAB bool
224 PercentChangeBA bool
225 MultiplierChangeAB bool
226 MultiplierChangeBA bool
227 TimeA bool
228 TimeB bool
229}
230
231func (diffs Diffs) Format(f DiffFormat) string {
232 if len(diffs) == 0 {
233 return "<no changes>"
234 }
235
236 type row []string
237
238 header := row{}
239 if f.TestName {
240 header = append(header, "Test name")
241 }
242 if f.Delta {
243 header = append(header, "Δ (A → B)")
244 }
245 if f.PercentChangeAB {
246 header = append(header, "% (A → B)")
247 }
248 if f.PercentChangeBA {
249 header = append(header, "% (B → A)")
250 }
251 if f.MultiplierChangeAB {
252 header = append(header, "× (A → B)")
253 }
254 if f.MultiplierChangeBA {
255 header = append(header, "× (B → A)")
256 }
257 if f.TimeA {
258 header = append(header, "A")
259 }
260 if f.TimeB {
261 header = append(header, "B")
262 }
263 if len(header) == 0 {
264 return ""
265 }
266
267 columns := []row{}
268 for _, d := range diffs {
269 r := make(row, 0, len(header))
270 if f.TestName {
271 r = append(r, d.TestName)
272 }
273 if f.Delta {
274 r = append(r, fmt.Sprintf("%v", d.Delta))
275 }
276 if f.PercentChangeAB {
277 r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeAB))
278 }
279 if f.PercentChangeBA {
280 r = append(r, fmt.Sprintf("%+2.1f%%", d.PercentChangeBA))
281 }
282 if f.MultiplierChangeAB {
283 r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeAB))
284 }
285 if f.MultiplierChangeBA {
286 r = append(r, fmt.Sprintf("%+.4f", d.MultiplierChangeBA))
287 }
288 if f.TimeA {
289 r = append(r, fmt.Sprintf("%v", d.TimeA))
290 }
291 if f.TimeB {
292 r = append(r, fmt.Sprintf("%v", d.TimeB))
293 }
294 columns = append(columns, r)
295 }
296
297 // measure
298 widths := make([]int, len(header))
299 for i, h := range header {
300 widths[i] = utf8.RuneCountInString(h)
301 }
302 for _, row := range columns {
303 for i, cell := range row {
304 l := utf8.RuneCountInString(cell)
305 if widths[i] < l {
306 widths[i] = l
307 }
308 }
309 }
310
311 pad := func(s string, i int) string {
312 if n := i - utf8.RuneCountInString(s); n > 0 {
313 return s + strings.Repeat(" ", n)
314 }
315 return s
316 }
317
318 // Draw table
319 b := &strings.Builder{}
320
321 horizontal_bar := func() {
322 for i := range header {
323 fmt.Fprintf(b, "+%v", strings.Repeat("-", 2+widths[i]))
324 }
325 fmt.Fprintln(b, "+")
326 }
327
328 horizontal_bar()
329
330 for i, h := range header {
331 fmt.Fprintf(b, "| %v ", pad(h, widths[i]))
332 }
333 fmt.Fprintln(b, "|")
334
335 horizontal_bar()
336
337 for _, row := range columns {
338 for i, cell := range row {
339 fmt.Fprintf(b, "| %v ", pad(cell, widths[i]))
340 }
341 fmt.Fprintln(b, "|")
342 }
343
344 horizontal_bar()
345
346 return b.String()
347}
348
349// Compare returns a string describing differences in the two benchmarks
350// Absolute benchmark differences less than minDiff are omitted
351// Absolute relative differences between [1, 1+x] are omitted
352func Compare(a, b []Benchmark, minDiff time.Duration, minRelDiff float64) Diffs {
353 type times struct {
354 a time.Duration
355 b time.Duration
356 }
357 byName := map[string]times{}
358 for _, test := range a {
359 byName[test.Name] = times{a: test.Duration}
360 }
361 for _, test := range b {
362 t := byName[test.Name]
363 t.b = test.Duration
364 byName[test.Name] = t
365 }
366
367 type delta struct {
368 name string
369 times times
370 relDiff float64
371 absRelDiff float64
372 }
373 deltas := []delta{}
374 for name, times := range byName {
375 if times.a == 0 || times.b == 0 {
376 continue // Assuming test was missing from a or b
377 }
378 diff := times.b - times.a
379 absDiff := diff
380 if absDiff < 0 {
381 absDiff = -absDiff
382 }
383 if absDiff < minDiff {
384 continue
385 }
386
387 relDiff := float64(times.b) / float64(times.a)
388 absRelDiff := relDiff
389 if absRelDiff < 1 {
390 absRelDiff = 1.0 / absRelDiff
391 }
392 if absRelDiff < (1.0 + minRelDiff) {
393 continue
394 }
395
396 d := delta{
397 name: name,
398 times: times,
399 relDiff: relDiff,
400 absRelDiff: absRelDiff,
401 }
402 deltas = append(deltas, d)
403 }
404
405 sort.Slice(deltas, func(i, j int) bool { return deltas[j].relDiff < deltas[i].relDiff })
406
407 out := make(Diffs, len(deltas))
408
409 for i, delta := range deltas {
410 a2b := delta.times.b - delta.times.a
411 out[i] = Diff{
412 TestName: delta.name,
413 Delta: a2b,
414 PercentChangeAB: 100 * float64(a2b) / float64(delta.times.a),
415 PercentChangeBA: 100 * float64(-a2b) / float64(delta.times.b),
416 MultiplierChangeAB: float64(delta.times.b) / float64(delta.times.a),
417 MultiplierChangeBA: float64(delta.times.a) / float64(delta.times.b),
418 TimeA: delta.times.a,
419 TimeB: delta.times.b,
420 }
421 }
422 return out
423}
424
425func trimAggregateSuffix(name string) string {
426 name = strings.TrimSuffix(name, "_stddev")
427 name = strings.TrimSuffix(name, "_mean")
428 name = strings.TrimSuffix(name, "_median")
429 return name
430}