blob: e2cec396be7dc2ec590b96676bc165d476af94d5 [file] [log] [blame] [edit]
// Copyright 2022 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package bench_test
import (
"reflect"
"testing"
"time"
"dawn.googlesource.com/dawn/tools/src/bench"
)
func TestParseJson(t *testing.T) {
json := `
{
"context": {
"date": "2022-01-24T10:28:13+00:00",
"host_name": "hostname",
"executable": "./myexe",
"num_cpus": 16,
"mhz_per_cpu": 2400,
"cpu_scaling_enabled": false,
"caches": [
{
"type": "Data",
"level": 1,
"size": 32768,
"num_sharing": 2
},
{
"type": "Instruction",
"level": 1,
"size": 32768,
"num_sharing": 2
},
{
"type": "Unified",
"level": 2,
"size": 262144,
"num_sharing": 2
},
{
"type": "Unified",
"level": 3,
"size": 16777216,
"num_sharing": 16
}
],
"load_avg": [2.60938,2.59863,2.55566],
"library_build_type": "release"
},
"benchmarks": [
{
"name": "MyBenchmark",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "iteration",
"repetitions": 2,
"repetition_index": 0,
"threads": 1,
"iterations": 402,
"real_time": 1.6392272438353568e+06,
"cpu_time": 1.6387412935323382e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "iteration",
"repetitions": 2,
"repetition_index": 1,
"threads": 1,
"iterations": 402,
"real_time": 1.7143936117703272e+06,
"cpu_time": 1.7124004975124374e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_mean",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "mean",
"iterations": 2,
"real_time": 1.6768104278028419e+06,
"cpu_time": 1.6755708955223879e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_median",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "median",
"iterations": 2,
"real_time": 1.6768104278028419e+06,
"cpu_time": 1.6755708955223879e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_stddev",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "stddev",
"iterations": 2,
"real_time": 5.3150648483981553e+04,
"cpu_time": 5.2084922631119407e+04,
"time_unit": "ns"
}
]
}
`
got, err := bench.Parse(json)
if err != nil {
t.Errorf("bench.Parse() returned %v", err)
return
}
expectedDate, err := time.Parse(time.RFC3339, "2022-01-24T10:28:13+00:00")
if err != nil {
t.Errorf("time.Parse() returned %v", err)
return
}
expect := bench.Run{
Benchmarks: []bench.Benchmark{
{Name: "MyBenchmark", Duration: time.Nanosecond * 1638741, AggregateType: ""},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1712400, AggregateType: ""},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1675570, AggregateType: "mean"},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1675570, AggregateType: "median"},
{Name: "MyBenchmark", Duration: time.Nanosecond * 52084, AggregateType: "stddev"},
},
Context: &bench.Context{
Date: expectedDate,
HostName: "hostname",
Executable: "./myexe",
NumCPUs: 16,
MhzPerCPU: 2400, CPUScalingEnabled: false,
Caches: []bench.ContextCache{
{Type: "Data", Level: 1, Size: 32768, NumSharing: 2},
{Type: "Instruction", Level: 1, Size: 32768, NumSharing: 2},
{Type: "Unified", Level: 2, Size: 262144, NumSharing: 2},
{Type: "Unified", Level: 3, Size: 16777216, NumSharing: 16},
},
LoadAvg: []float32{2.60938, 2.59863, 2.55566}, LibraryBuildType: "release"},
}
expectEqual(t, "bench.Parse().Benchmarks", got.Benchmarks, expect.Benchmarks)
expectEqual(t, "bench.Parse().Context", got.Context, expect.Context)
}
func TestCompare(t *testing.T) {
a := []bench.Benchmark{
{Name: "MyBenchmark1", Duration: time.Nanosecond * 1714393},
{Name: "MyBenchmark0", Duration: time.Nanosecond * 1639227},
{Name: "MyBenchmark3", Duration: time.Nanosecond * 1676810},
{Name: "MyBenchmark4", Duration: time.Nanosecond * 53150},
{Name: "MyBenchmark2", Duration: time.Nanosecond * 1676810},
}
b := []bench.Benchmark{
{Name: "MyBenchmark1", Duration: time.Nanosecond * 56747654},
{Name: "MyBenchmark0", Duration: time.Nanosecond * 236246},
{Name: "MyBenchmark2", Duration: time.Nanosecond * 675865},
{Name: "MyBenchmark4", Duration: time.Nanosecond * 2352336},
{Name: "MyBenchmark3", Duration: time.Nanosecond * 87657868},
}
minDiff := time.Millisecond * 2
minRelDiff := 35.0
cmp := bench.Compare(a, b, minDiff, minRelDiff)
expectEqual(t, "bench.Compare().Format", cmp.Format(bench.DiffFormat{}), "")
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true}), `
+-----------+
| A |
+-----------+
| 1.67681ms |
| 53.15µs |
+-----------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true, TimeB: true}), `
+-----------+-------------+
| A | B |
+-----------+-------------+
| 1.67681ms | 87.657868ms |
| 53.15µs | 2.352336ms |
+-----------+-------------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
TimeA: true,
TimeB: true,
}), `
+--------------+-------------+-----------+-----------+-------------+
| Test name | Δ (A → B) | % (A → B) | A | B |
+--------------+-------------+-----------+-----------+-------------+
| MyBenchmark3 | 85.981058ms | +5127.7% | 1.67681ms | 87.657868ms |
| MyBenchmark4 | 2.299186ms | +4325.8% | 53.15µs | 2.352336ms |
+--------------+-------------+-----------+-----------+-------------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
PercentChangeBA: true,
MultiplierChangeAB: true,
MultiplierChangeBA: true,
TimeA: true,
TimeB: true,
}), `
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
| Test name | Δ (A → B) | % (A → B) | % (B → A) | × (A → B) | × (B → A) | A | B |
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
| MyBenchmark3 | 85.981058ms | +5127.7% | -98.1% | +52.2766 | +0.0191 | 1.67681ms | 87.657868ms |
| MyBenchmark4 | 2.299186ms | +4325.8% | -97.7% | +44.2584 | +0.0226 | 53.15µs | 2.352336ms |
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
`)
}
func expectEqual(t *testing.T, desc string, got, expect interface{}) {
if !reflect.DeepEqual(got, expect) {
t.Errorf("%v was not expected:\nGot:\n%+v\nExpected:\n%+v", desc, got, expect)
}
}