blob: e696cfd260fd03f687ddb00383adf0296b6b4bd4 [file] [log] [blame]
// Copyright 2024 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package common
import (
"context"
"testing"
"time"
"dawn.googlesource.com/dawn/tools/src/buildbucket"
"dawn.googlesource.com/dawn/tools/src/cts/query"
"dawn.googlesource.com/dawn/tools/src/cts/result"
"dawn.googlesource.com/dawn/tools/src/resultsdb"
"github.com/stretchr/testify/assert"
)
// TODO(crbug.com/342554800): Add test coverage for:
// ResultSource.GetResults (requires breaking out into helper functions)
// ResultSource.GetUnsuppressedFailingResults (ditto)
// CacheResults (requires os abstraction crbug.com/344014313)
// CacheUnsuppressedFailingResults (ditto)
// LatestCTSRoll
// LatestPatchset
// MostRecentResultsForChange (requires os abstraction crbug.com/344014313)
// MostRecentUnsuppressedFailingResultsForChange (ditto)
/*******************************************************************************
* Fake implementations
******************************************************************************/
// A fake version of dawn/tools/src/resultsdb's BigQueryClient.
type mockedBigQueryClient struct {
returnValues []resultsdb.QueryResult
unsuppressedFailureReturnValues []resultsdb.QueryResult
}
func (bq mockedBigQueryClient) QueryTestResults(
ctx context.Context, builds []buildbucket.BuildID, testPrefix string, f func(*resultsdb.QueryResult) error) error {
for _, result := range bq.returnValues {
if err := f(&result); err != nil {
return err
}
}
return nil
}
func (bq mockedBigQueryClient) QueryUnsuppressedFailingTestResults(
ctx context.Context, builds []buildbucket.BuildID, testPrefix string, f func(*resultsdb.QueryResult) error) error {
for _, result := range bq.unsuppressedFailureReturnValues {
if err := f(&result); err != nil {
return err
}
}
return nil
}
/*******************************************************************************
* GetResults tests
******************************************************************************/
func generateGoodGetResultsInputs() (context.Context, Config, *mockedBigQueryClient, BuildsByName) {
ctx := context.Background()
cfg := Config{
Tests: []TestConfig{
TestConfig{
ExecutionMode: result.ExecutionMode("execution_mode"),
Prefixes: []string{"prefix"},
},
},
}
client := &mockedBigQueryClient{
returnValues: []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test",
Status: "PASS",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
},
},
}
builds := make(BuildsByName)
return ctx, cfg, client, builds
}
// Tests that valid results are properly cleaned, sorted, and returned.
func TestGetResultsHappyPath(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetResultsInputs()
cfg.Tag.Remove = []string{
"remove_me",
}
client.returnValues = []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_2",
Status: "PASS",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "typ_tag",
Value: "remove_me",
},
resultsdb.TagPair{
Key: "typ_tag",
Value: "win",
},
},
Duration: 2.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_1",
Status: "PASS",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "typ_tag",
Value: "linux",
},
},
Duration: 1.0,
},
}
expectedResultsList := result.List{
result.Result{
Query: query.Parse("_test_1"),
Status: result.Pass,
Tags: result.NewTags("linux"),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("_test_2"),
Status: result.Pass,
Tags: result.NewTags("win"),
Duration: 2 * time.Second,
MayExonerate: false,
},
}
expectedResults := make(result.ResultsByExecutionMode)
expectedResults["execution_mode"] = expectedResultsList
results, err := GetResults(ctx, cfg, client, builds)
assert.Nil(t, err)
assert.Equal(t, results, expectedResults)
}
// Tests that errors from GetRawResults are properly surfaced.
func TestGetResultsGetRawResultsErrorSurfaced(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetResultsInputs()
client.returnValues[0].TestId = "bad_test"
results, err := GetResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
}
/*******************************************************************************
* GetUnsuppressedFailingResults tests
******************************************************************************/
func generateGoodGetUnsuppressedFailingResultsInputs() (context.Context, Config, *mockedBigQueryClient, BuildsByName) {
ctx := context.Background()
cfg := Config{
Tests: []TestConfig{
TestConfig{
ExecutionMode: result.ExecutionMode("execution_mode"),
Prefixes: []string{"prefix"},
},
},
}
client := &mockedBigQueryClient{
unsuppressedFailureReturnValues: []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test",
Status: "FAIL",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
},
},
}
builds := make(BuildsByName)
return ctx, cfg, client, builds
}
// Tests that valid results are properly cleaned, sorted, and returned.
func TestGetUnsuppressedFailingResultsHappyPath(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
cfg.Tag.Remove = []string{
"remove_me",
}
client.unsuppressedFailureReturnValues = []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "typ_tag",
Value: "remove_me",
},
resultsdb.TagPair{
Key: "typ_tag",
Value: "win",
},
},
Duration: 2.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "typ_tag",
Value: "linux",
},
},
Duration: 1.0,
},
}
expectedResultsList := result.List{
result.Result{
Query: query.Parse("_test_1"),
Status: result.Failure,
Tags: result.NewTags("linux"),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("_test_2"),
Status: result.Failure,
Tags: result.NewTags("win"),
Duration: 2 * time.Second,
MayExonerate: false,
},
}
expectedResults := make(result.ResultsByExecutionMode)
expectedResults["execution_mode"] = expectedResultsList
results, err := GetUnsuppressedFailingResults(ctx, cfg, client, builds)
assert.Nil(t, err)
assert.Equal(t, results, expectedResults)
}
// Tests that errors from GetRawResults are properly surfaced.
func TestGetUnsuppressedFailingResultsGetRawResultsErrorSurfaced(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
client.unsuppressedFailureReturnValues[0].TestId = "bad_test"
results, err := GetUnsuppressedFailingResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
}
/*******************************************************************************
* GetRawResults tests
******************************************************************************/
// Tests that valid results are properly parsed and returned.
func TestGetRawResultsHappyPath(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetResultsInputs()
client.returnValues = []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_1",
Status: "PASS",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "javascript_duration",
Value: "0.5s",
},
},
Duration: 2.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_3",
Status: "SKIP",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "may_exonerate",
Value: "true",
},
},
Duration: 3.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_4",
Status: "SomeStatus",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "typ_tag",
Value: "linux",
},
resultsdb.TagPair{
Key: "typ_tag",
Value: "intel",
},
},
Duration: 4.0,
},
}
expectedResultsList := result.List{
result.Result{
Query: query.Parse("_test_1"),
Status: result.Pass,
Tags: result.NewTags(),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("_test_2"),
Status: result.Failure,
Tags: result.NewTags(),
Duration: 500 * time.Millisecond,
MayExonerate: false,
},
result.Result{
Query: query.Parse("_test_3"),
Status: result.Skip,
Tags: result.NewTags(),
Duration: 3 * time.Second,
MayExonerate: true,
},
result.Result{
Query: query.Parse("_test_4"),
Status: result.Unknown,
Tags: result.NewTags("linux", "intel"),
Duration: 4 * time.Second,
MayExonerate: false,
},
}
expectedResults := make(result.ResultsByExecutionMode)
expectedResults["execution_mode"] = expectedResultsList
results, err := GetRawResults(ctx, cfg, client, builds)
assert.Nil(t, err)
assert.Equal(t, results, expectedResults)
}
// Tests that a mismatched prefix results in an error.
func TestGetRawResultsPrefixMismatch(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetResultsInputs()
client.returnValues[0].TestId = "bad_test"
results, err := GetRawResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
}
// Tests that a JavaScript duration that cannot be parsed results in an error.
func TestGetRawResultsBadJavaScriptDuration(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetResultsInputs()
client.returnValues[0].Tags = []resultsdb.TagPair{
resultsdb.TagPair{
Key: "javascript_duration",
Value: "1000foo",
},
}
results, err := GetRawResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, `time: unknown unit "foo" in duration "1000foo"`)
}
// Tests that a non-boolean may_exonerate value results in an error.
func TestGetRawResultsBadMayExonerate(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetResultsInputs()
client.returnValues[0].Tags = []resultsdb.TagPair{
resultsdb.TagPair{
Key: "may_exonerate",
Value: "yesnt",
},
}
results, err := GetRawResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, `strconv.ParseBool: parsing "yesnt": invalid syntax`)
}
/*******************************************************************************
* GetRawUnsuppressedFailingResults tests
******************************************************************************/
// Tests that valid results are properly parsed and returned.
func TestGetRawUnsuppressedFailingResultsHappyPath(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
client.unsuppressedFailureReturnValues = []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "javascript_duration",
Value: "0.5s",
},
},
Duration: 2.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_3",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "may_exonerate",
Value: "true",
},
},
Duration: 3.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_4",
Status: "SomeStatus",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
Key: "typ_tag",
Value: "linux",
},
resultsdb.TagPair{
Key: "typ_tag",
Value: "intel",
},
},
Duration: 4.0,
},
}
expectedResultsList := result.List{
result.Result{
Query: query.Parse("_test_1"),
Status: result.Failure,
Tags: result.NewTags(),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("_test_2"),
Status: result.Failure,
Tags: result.NewTags(),
Duration: 500 * time.Millisecond,
MayExonerate: false,
},
result.Result{
Query: query.Parse("_test_3"),
Status: result.Failure,
Tags: result.NewTags(),
Duration: 3 * time.Second,
MayExonerate: true,
},
result.Result{
Query: query.Parse("_test_4"),
Status: result.Unknown,
Tags: result.NewTags("linux", "intel"),
Duration: 4 * time.Second,
MayExonerate: false,
},
}
expectedResults := make(result.ResultsByExecutionMode)
expectedResults["execution_mode"] = expectedResultsList
results, err := GetRawUnsuppressedFailingResults(ctx, cfg, client, builds)
assert.Nil(t, err)
assert.Equal(t, results, expectedResults)
}
// Tests that a mismatched prefix results in an error.
func TestGetRawUnsuppressedFailingResultsPrefixMismatch(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
client.unsuppressedFailureReturnValues[0].TestId = "bad_test"
results, err := GetRawUnsuppressedFailingResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
}
// Tests that a JavaScript duration that cannot be parsed results in an error.
func TestGetRawUnsuppressedFailingResultsBadJavaScriptDuration(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
client.unsuppressedFailureReturnValues[0].Tags = []resultsdb.TagPair{
resultsdb.TagPair{
Key: "javascript_duration",
Value: "1000foo",
},
}
results, err := GetRawUnsuppressedFailingResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, `time: unknown unit "foo" in duration "1000foo"`)
}
// Tests that a non-boolean may_exonerate value results in an error.
func TestGetRawUnsuppressedFailingResultsBadMayExonerate(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
client.unsuppressedFailureReturnValues[0].Tags = []resultsdb.TagPair{
resultsdb.TagPair{
Key: "may_exonerate",
Value: "yesnt",
},
}
results, err := GetRawUnsuppressedFailingResults(ctx, cfg, client, builds)
assert.Nil(t, results)
assert.ErrorContains(t, err, `strconv.ParseBool: parsing "yesnt": invalid syntax`)
}
/*******************************************************************************
* CleanResults tests
******************************************************************************/
// Tests that tags specified for removal are properly removed.
func TestCleanResultsTagRemoval(t *testing.T) {
cfg := Config{}
cfg.Tag.Remove = []string{
"remove_1",
"remove_2",
"remove_3",
"missing",
}
results := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: result.Pass,
Tags: result.NewTags("remove_1", "remove_2", "linux"),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("test_2"),
Status: result.Pass,
Tags: result.NewTags("remove_2", "remove_3"),
Duration: time.Second,
MayExonerate: false,
},
}
expectedResults := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: result.Pass,
Tags: result.NewTags("linux"),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("test_2"),
Status: result.Pass,
Tags: result.NewTags(),
Duration: time.Second,
MayExonerate: false,
},
}
CleanResults(cfg, &results)
assert.Equal(t, results, expectedResults)
}
// Tests that duplicate results with the same status always use that status.
func TestCleanResultsDuplicateResultSameStatus(t *testing.T) {
cfg := Config{}
for _, status := range []result.Status{result.Abort, result.Crash,
result.Failure, result.Pass, result.RetryOnFailure, result.Skip, result.Slow, result.Unknown} {
results := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: status,
Tags: result.NewTags(),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("test_1"),
Status: status,
Tags: result.NewTags(),
Duration: 3 * time.Second,
MayExonerate: false,
},
}
expectedResults := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: status,
Tags: result.NewTags(),
Duration: 2 * time.Second,
MayExonerate: false,
},
}
CleanResults(cfg, &results)
assert.Equal(t, results, expectedResults)
}
}
func runPriorityTest(t *testing.T, testedStatus result.Status, lowerPriorityStatuses *[]result.Status) {
cfg := Config{}
for _, status := range *lowerPriorityStatuses {
results := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: status,
Tags: result.NewTags(),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("test_1"),
Status: testedStatus,
Tags: result.NewTags(),
Duration: 3 * time.Second,
MayExonerate: false,
},
}
expectedResults := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: testedStatus,
Tags: result.NewTags(),
Duration: 2 * time.Second,
MayExonerate: false,
},
}
CleanResults(cfg, &results)
assert.Equal(t, results, expectedResults)
}
}
// Tests that Crash has the highest priority among statuses.
func TestCleanResultsReplaceResultCrash(t *testing.T) {
lowerPriorityStatuses := []result.Status{result.Abort, result.Failure, result.Slow, result.Pass, result.RetryOnFailure, result.Skip, result.Unknown}
runPriorityTest(t, result.Crash, &lowerPriorityStatuses)
}
// Tests that Abort has the second highest priority among statuses.
func TestCleanResultsReplaceResultAbort(t *testing.T) {
lowerPriorityStatuses := []result.Status{result.Failure, result.Slow, result.Pass, result.RetryOnFailure, result.Skip, result.Unknown}
runPriorityTest(t, result.Abort, &lowerPriorityStatuses)
}
// Tests that Failure has the third highest priority among statuses.
func TestCleanResultsReplaceResultFailure(t *testing.T) {
lowerPriorityStatuses := []result.Status{result.Slow, result.Pass, result.RetryOnFailure, result.Skip, result.Unknown}
runPriorityTest(t, result.Failure, &lowerPriorityStatuses)
}
// Tests that Slow has the fourth highest priority among statuses.
func TestCleanResultsReplaceResultSlow(t *testing.T) {
lowerPriorityStatuses := []result.Status{result.Pass, result.RetryOnFailure, result.Skip, result.Unknown}
runPriorityTest(t, result.Slow, &lowerPriorityStatuses)
}
// Tests that statuses without explicit priority default to Failure.
func TestCleanResultsReplaceResultDefault(t *testing.T) {
cfg := Config{}
statuses := []result.Status{result.Pass, result.RetryOnFailure, result.Skip, result.Unknown}
for i, leftStatus := range statuses {
for j, rightStatus := range statuses {
if i == j {
continue
}
results := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: leftStatus,
Tags: result.NewTags(),
Duration: time.Second,
MayExonerate: false,
},
result.Result{
Query: query.Parse("test_1"),
Status: rightStatus,
Tags: result.NewTags(),
Duration: 3 * time.Second,
MayExonerate: false,
},
}
expectedResults := result.List{
result.Result{
Query: query.Parse("test_1"),
Status: result.Failure,
Tags: result.NewTags(),
Duration: 2 * time.Second,
MayExonerate: false,
},
}
CleanResults(cfg, &results)
assert.Equal(t, results, expectedResults)
}
}
}