| // Copyright 2022 The Dawn & Tint Authors |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are met: |
| // |
| // 1. Redistributions of source code must retain the above copyright notice, this |
| // list of conditions and the following disclaimer. |
| // |
| // 2. Redistributions in binary form must reproduce the above copyright notice, |
| // this list of conditions and the following disclaimer in the documentation |
| // and/or other materials provided with the distribution. |
| // |
| // 3. Neither the name of the copyright holder nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
| // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| package bench_test |
| |
| import ( |
| "reflect" |
| "testing" |
| "time" |
| |
| "dawn.googlesource.com/dawn/tools/src/bench" |
| ) |
| |
| func TestParseJson(t *testing.T) { |
| json := ` |
| { |
| "context": { |
| "date": "2022-01-24T10:28:13+00:00", |
| "host_name": "hostname", |
| "executable": "./myexe", |
| "num_cpus": 16, |
| "mhz_per_cpu": 2400, |
| "cpu_scaling_enabled": false, |
| "caches": [ |
| { |
| "type": "Data", |
| "level": 1, |
| "size": 32768, |
| "num_sharing": 2 |
| }, |
| { |
| "type": "Instruction", |
| "level": 1, |
| "size": 32768, |
| "num_sharing": 2 |
| }, |
| { |
| "type": "Unified", |
| "level": 2, |
| "size": 262144, |
| "num_sharing": 2 |
| }, |
| { |
| "type": "Unified", |
| "level": 3, |
| "size": 16777216, |
| "num_sharing": 16 |
| } |
| ], |
| "load_avg": [2.60938,2.59863,2.55566], |
| "library_build_type": "release" |
| }, |
| "benchmarks": [ |
| { |
| "name": "MyBenchmark", |
| "family_index": 0, |
| "per_family_instance_index": 0, |
| "run_name": "MyBenchmark", |
| "run_type": "iteration", |
| "repetitions": 2, |
| "repetition_index": 0, |
| "threads": 1, |
| "iterations": 402, |
| "real_time": 1.6392272438353568e+06, |
| "cpu_time": 1.6387412935323382e+06, |
| "time_unit": "ns" |
| }, |
| { |
| "name": "MyBenchmark", |
| "family_index": 0, |
| "per_family_instance_index": 0, |
| "run_name": "MyBenchmark", |
| "run_type": "iteration", |
| "repetitions": 2, |
| "repetition_index": 1, |
| "threads": 1, |
| "iterations": 402, |
| "real_time": 1.7143936117703272e+06, |
| "cpu_time": 1.7124004975124374e+06, |
| "time_unit": "ns" |
| }, |
| { |
| "name": "MyBenchmark_mean", |
| "family_index": 0, |
| "per_family_instance_index": 0, |
| "run_name": "MyBenchmark", |
| "run_type": "aggregate", |
| "repetitions": 2, |
| "threads": 1, |
| "aggregate_name": "mean", |
| "iterations": 2, |
| "real_time": 1.6768104278028419e+06, |
| "cpu_time": 1.6755708955223879e+06, |
| "time_unit": "ns" |
| }, |
| { |
| "name": "MyBenchmark_median", |
| "family_index": 0, |
| "per_family_instance_index": 0, |
| "run_name": "MyBenchmark", |
| "run_type": "aggregate", |
| "repetitions": 2, |
| "threads": 1, |
| "aggregate_name": "median", |
| "iterations": 2, |
| "real_time": 1.6768104278028419e+06, |
| "cpu_time": 1.6755708955223879e+06, |
| "time_unit": "ns" |
| }, |
| { |
| "name": "MyBenchmark_stddev", |
| "family_index": 0, |
| "per_family_instance_index": 0, |
| "run_name": "MyBenchmark", |
| "run_type": "aggregate", |
| "repetitions": 2, |
| "threads": 1, |
| "aggregate_name": "stddev", |
| "iterations": 2, |
| "real_time": 5.3150648483981553e+04, |
| "cpu_time": 5.2084922631119407e+04, |
| "time_unit": "ns" |
| } |
| ] |
| } |
| ` |
| got, err := bench.Parse(json) |
| if err != nil { |
| t.Errorf("bench.Parse() returned %v", err) |
| return |
| } |
| |
| expectedDate, err := time.Parse(time.RFC3339, "2022-01-24T10:28:13+00:00") |
| if err != nil { |
| t.Errorf("time.Parse() returned %v", err) |
| return |
| } |
| |
| expect := bench.Run{ |
| Benchmarks: []bench.Benchmark{ |
| {Name: "MyBenchmark", Duration: time.Nanosecond * 1638741, AggregateType: ""}, |
| {Name: "MyBenchmark", Duration: time.Nanosecond * 1712400, AggregateType: ""}, |
| {Name: "MyBenchmark", Duration: time.Nanosecond * 1675570, AggregateType: "mean"}, |
| {Name: "MyBenchmark", Duration: time.Nanosecond * 1675570, AggregateType: "median"}, |
| {Name: "MyBenchmark", Duration: time.Nanosecond * 52084, AggregateType: "stddev"}, |
| }, |
| Context: &bench.Context{ |
| Date: expectedDate, |
| HostName: "hostname", |
| Executable: "./myexe", |
| NumCPUs: 16, |
| MhzPerCPU: 2400, CPUScalingEnabled: false, |
| Caches: []bench.ContextCache{ |
| {Type: "Data", Level: 1, Size: 32768, NumSharing: 2}, |
| {Type: "Instruction", Level: 1, Size: 32768, NumSharing: 2}, |
| {Type: "Unified", Level: 2, Size: 262144, NumSharing: 2}, |
| {Type: "Unified", Level: 3, Size: 16777216, NumSharing: 16}, |
| }, |
| LoadAvg: []float32{2.60938, 2.59863, 2.55566}, LibraryBuildType: "release"}, |
| } |
| |
| expectEqual(t, "bench.Parse().Benchmarks", got.Benchmarks, expect.Benchmarks) |
| expectEqual(t, "bench.Parse().Context", got.Context, expect.Context) |
| } |
| |
| func TestCompare(t *testing.T) { |
| a := []bench.Benchmark{ |
| {Name: "MyBenchmark1", Duration: time.Nanosecond * 1714393}, |
| {Name: "MyBenchmark0", Duration: time.Nanosecond * 1639227}, |
| {Name: "MyBenchmark3", Duration: time.Nanosecond * 1676810}, |
| {Name: "MyBenchmark4", Duration: time.Nanosecond * 53150}, |
| {Name: "MyBenchmark2", Duration: time.Nanosecond * 1676810}, |
| } |
| b := []bench.Benchmark{ |
| {Name: "MyBenchmark1", Duration: time.Nanosecond * 56747654}, |
| {Name: "MyBenchmark0", Duration: time.Nanosecond * 236246}, |
| {Name: "MyBenchmark2", Duration: time.Nanosecond * 675865}, |
| {Name: "MyBenchmark4", Duration: time.Nanosecond * 2352336}, |
| {Name: "MyBenchmark3", Duration: time.Nanosecond * 87657868}, |
| } |
| |
| minDiff := time.Millisecond * 2 |
| minRelDiff := 35.0 |
| |
| cmp := bench.Compare(a, b, minDiff, minRelDiff) |
| |
| expectEqual(t, "bench.Compare().Format", cmp.Format(bench.DiffFormat{}), "") |
| expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true}), ` |
| +-----------+ |
| | A | |
| +-----------+ |
| | 1.67681ms | |
| | 53.15µs | |
| +-----------+ |
| `) |
| expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{TimeA: true, TimeB: true}), ` |
| +-----------+-------------+ |
| | A | B | |
| +-----------+-------------+ |
| | 1.67681ms | 87.657868ms | |
| | 53.15µs | 2.352336ms | |
| +-----------+-------------+ |
| `) |
| expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{ |
| TestName: true, |
| Delta: true, |
| PercentChangeAB: true, |
| TimeA: true, |
| TimeB: true, |
| }), ` |
| +--------------+-------------+-----------+-----------+-------------+ |
| | Test name | Δ (A → B) | % (A → B) | A | B | |
| +--------------+-------------+-----------+-----------+-------------+ |
| | MyBenchmark3 | 85.981058ms | +5127.7% | 1.67681ms | 87.657868ms | |
| | MyBenchmark4 | 2.299186ms | +4325.8% | 53.15µs | 2.352336ms | |
| +--------------+-------------+-----------+-----------+-------------+ |
| `) |
| expectEqual(t, "bench.Compare().Format", "\n"+cmp.Format(bench.DiffFormat{ |
| TestName: true, |
| Delta: true, |
| PercentChangeAB: true, |
| PercentChangeBA: true, |
| MultiplierChangeAB: true, |
| MultiplierChangeBA: true, |
| TimeA: true, |
| TimeB: true, |
| }), ` |
| +--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+ |
| | Test name | Δ (A → B) | % (A → B) | % (B → A) | × (A → B) | × (B → A) | A | B | |
| +--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+ |
| | MyBenchmark3 | 85.981058ms | +5127.7% | -98.1% | +52.2766 | +0.0191 | 1.67681ms | 87.657868ms | |
| | MyBenchmark4 | 2.299186ms | +4325.8% | -97.7% | +44.2584 | +0.0226 | 53.15µs | 2.352336ms | |
| +--------------+-------------+-----------+-----------+-----------+-----------+-----------+-------------+ |
| `) |
| } |
| |
| func expectEqual(t *testing.T, desc string, got, expect interface{}) { |
| if !reflect.DeepEqual(got, expect) { |
| t.Errorf("%v was not expected:\nGot:\n%+v\nExpected:\n%+v", desc, got, expect) |
| } |
| } |
