blob: 66a3b9b3f11b9c96df8cd73b111aa16abd07b283 [file] [log] [blame]
// Copyright 2022 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Package resultsdb provides helpers for interfacing with resultsdb
package resultsdb
import (
"context"
"fmt"
"strings"
"cloud.google.com/go/bigquery"
"dawn.googlesource.com/dawn/tools/src/buildbucket"
"google.golang.org/api/iterator"
)
// BigQueryClient is a wrapper around bigquery.Client so that we can define new
// methods.
type BigQueryClient struct {
client *bigquery.Client
}
// QueryResult contains all of the data for a single test result from a ResultDB
// BigQuery query.
type QueryResult struct {
TestId string
Status string
Tags []struct {
Key string
Value string
}
Duration float64
}
// DefaultQueryProject is the default BigQuery project to use when running
// queries.
const DefaultQueryProject string = "chrome-unexpected-pass-data"
// NewBigQueryClient creates a client for running BigQuery queries. The
// intention is for this to be used for querying ResultDB tables, but there is
// nothing ResultDB-specific about the resulting client.
func NewBigQueryClient(ctx context.Context, project string) (*BigQueryClient, error) {
client, err := bigquery.NewClient(ctx, project)
if err != nil {
return nil, err
}
// By default, results are retrieved in chunks as they're iterated over, but
// that results in slow performance. Enabling the Storage API allows us to get
// all results at once, resulting in a ~8-10x speed increase.
err = client.EnableStorageReadClient(ctx)
if err != nil {
return nil, err
}
return &BigQueryClient{client}, nil
}
// QueryTestResults fetches the test results for the given builds using
// BigQuery.
//
// f is called once per result and is expected to handle any processing or
// storage of results.
func (bq *BigQueryClient) QueryTestResults(
ctx context.Context, builds []buildbucket.BuildID, testPrefix string, f func(*QueryResult) error) error {
// test_id gets renamed since the column names need to match the struct names
// unless we want to get results in a generic bigquery.Value slice and
// manually copy data over.
base_query := `
SELECT
test_id AS testid,
status,
tags,
duration
FROM ` + "`chrome-luci-data.chromium.gpu_try_test_results`" + ` tr
WHERE
exported.id IN UNNEST([%s])
AND STARTS_WITH(tr.test_id, "%v")`
var buildIds []string
for _, id := range builds {
buildIds = append(buildIds, fmt.Sprintf(`"build-%v"`, id))
}
query := fmt.Sprintf(base_query, strings.Join(buildIds, ","), testPrefix)
q := bq.client.Query(query)
iter, err := q.Read(ctx)
if err != nil {
return err
}
var row QueryResult
for {
err := iter.Next(&row)
if err == iterator.Done {
break
}
if err != nil {
return err
}
if err := f(&row); err != nil {
return err
}
}
return nil
}