blob: 3f9f6c08d1ab48b554af46cf16fe8d7e890be47f [file] [log] [blame]
// Copyright 2022 The Tint Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bench_test
import (
"reflect"
"testing"
"time"
"dawn.googlesource.com/tint/tools/src/bench"
)
func TestParseJson(t *testing.T) {
json := `
{
"context": {
"date": "2022-01-24T10:28:13+00:00",
"host_name": "hostname",
"executable": "./myexe",
"num_cpus": 16,
"mhz_per_cpu": 2400,
"cpu_scaling_enabled": false,
"caches": [
{
"type": "Data",
"level": 1,
"size": 32768,
"num_sharing": 2
},
{
"type": "Instruction",
"level": 1,
"size": 32768,
"num_sharing": 2
},
{
"type": "Unified",
"level": 2,
"size": 262144,
"num_sharing": 2
},
{
"type": "Unified",
"level": 3,
"size": 16777216,
"num_sharing": 16
}
],
"load_avg": [2.60938,2.59863,2.55566],
"library_build_type": "release"
},
"benchmarks": [
{
"name": "MyBenchmark",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "iteration",
"repetitions": 2,
"repetition_index": 0,
"threads": 1,
"iterations": 402,
"real_time": 1.6392272438353568e+06,
"cpu_time": 1.6387412935323382e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "iteration",
"repetitions": 2,
"repetition_index": 1,
"threads": 1,
"iterations": 402,
"real_time": 1.7143936117703272e+06,
"cpu_time": 1.7124004975124374e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_mean",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "mean",
"iterations": 2,
"real_time": 1.6768104278028419e+06,
"cpu_time": 1.6755708955223879e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_median",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "median",
"iterations": 2,
"real_time": 1.6768104278028419e+06,
"cpu_time": 1.6755708955223879e+06,
"time_unit": "ns"
},
{
"name": "MyBenchmark_stddev",
"family_index": 0,
"per_family_instance_index": 0,
"run_name": "MyBenchmark",
"run_type": "aggregate",
"repetitions": 2,
"threads": 1,
"aggregate_name": "stddev",
"iterations": 2,
"real_time": 5.3150648483981553e+04,
"cpu_time": 5.2084922631119407e+04,
"time_unit": "ns"
}
]
}
`
got, err := bench.Parse(json)
if err != nil {
t.Errorf("bench.Parse() returned %v", err)
return
}
expectedDate, err := time.Parse(time.RFC1123, "Mon, 24 Jan 2022 10:28:13 GMT")
if err != nil {
t.Errorf("time.Parse() returned %v", err)
return
}
expect := bench.Run{
Benchmarks: []bench.Benchmark{
{Name: "MyBenchmark", Duration: time.Nanosecond * 1639227, AggregateType: ""},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1714393, AggregateType: ""},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "mean"},
{Name: "MyBenchmark", Duration: time.Nanosecond * 1676810, AggregateType: "median"},
{Name: "MyBenchmark", Duration: time.Nanosecond * 53150, AggregateType: "stddev"},
},
Context: &bench.Context{
Date: expectedDate,
HostName: "hostname",
Executable: "./myexe",
NumCPUs: 16,
MhzPerCPU: 2400, CPUScalingEnabled: false,
Caches: []bench.ContextCache{
{Type: "Data", Level: 1, Size: 32768, NumSharing: 2},
{Type: "Instruction", Level: 1, Size: 32768, NumSharing: 2},
{Type: "Unified", Level: 2, Size: 262144, NumSharing: 2},
{Type: "Unified", Level: 3, Size: 16777216, NumSharing: 16},
},
LoadAvg: []float32{2.60938, 2.59863, 2.55566}, LibraryBuildType: "release"},
}
expectEqual(t, "bench.Parse().Benchmarks", got.Benchmarks, expect.Benchmarks)
expectEqual(t, "bench.Parse().Context", got.Benchmarks, expect.Benchmarks)
}
func TestCompare(t *testing.T) {
a := []bench.Benchmark{
{Name: "MyBenchmark1", Duration: time.Nanosecond * 1714393},
{Name: "MyBenchmark0", Duration: time.Nanosecond * 1639227},
{Name: "MyBenchmark3", Duration: time.Nanosecond * 1676810},
{Name: "MyBenchmark4", Duration: time.Nanosecond * 53150},
{Name: "MyBenchmark2", Duration: time.Nanosecond * 1676810},
}
b := []bench.Benchmark{
{Name: "MyBenchmark1", Duration: time.Nanosecond * 56747654},
{Name: "MyBenchmark0", Duration: time.Nanosecond * 236246},
{Name: "MyBenchmark2", Duration: time.Nanosecond * 675865},
{Name: "MyBenchmark4", Duration: time.Nanosecond * 2352336},
{Name: "MyBenchmark3", Duration: time.Nanosecond * 87657868},
}
minDiff := time.Millisecond * 2
minRelDiff := 35.0
cmp := bench.Compare(a, b, minDiff, minRelDiff)
expectEqual(t, "bench.Compare().Format", cmp.Format(bench.DiffFormat{}), "")
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true}), `
+-----------+
| A |
+-----------+
| 1.67681ms |
| 53.15µs |
+-----------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true, TimeB: true}), `
+-----------+-------------+
| A | B |
+-----------+-------------+
| 1.67681ms | 87.657868ms |
| 53.15µs | 2.352336ms |
+-----------+-------------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
TimeA: true,
TimeB: true,
}), `
+--------------+-------------+-----------+-----------+-------------+
| Test name | Δ (A → B) | % (A → B) | A | B |
+--------------+-------------+-----------+-----------+-------------+
| MyBenchmark3 | 85.981058ms | +5127.7% | 1.67681ms | 87.657868ms |
| MyBenchmark4 | 2.299186ms | +4325.8% | 53.15µs | 2.352336ms |
+--------------+-------------+-----------+-----------+-------------+
`)
expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{
TestName: true,
Delta: true,
PercentChangeAB: true,
PercentChangeBA: true,
MultiplierChangeAB: true,
MultiplierChangeBA: true,
TimeA: true,
TimeB: true,
}), `
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
| Test name | Δ (A → B) | % (A → B) | % (B → A) | × (A → B) | × (B → A) | A | B |
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
| MyBenchmark3 | 85.981058ms | +5127.7% | -98.1% | +52.2766 | +0.0191 | 1.67681ms | 87.657868ms |
| MyBenchmark4 | 2.299186ms | +4325.8% | -97.7% | +44.2584 | +0.0226 | 53.15µs | 2.352336ms |
+--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+
`)
}
func expectEqual(t *testing.T, desc string, got, expect interface{}) {
if !reflect.DeepEqual(got, expect) {
t.Errorf("%v was not expected:\nGot:\n%+v\nExpected:\n%+v", desc, got, expect)
}
}