Migrate WebGPU Autoroller to support Chrome structured test IDs
The queries now include a 'test_metadata.name' field which contains the
actual test name. Updated tools to use the new field instead of parsing
the TestId. Checks for prefix mismatches are removed as the queries are
expected to filter correctly.
Bug: 448003842
Change-Id: I8923a9bdb6b66acc32531555e05f493b190f960f
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/271474
Reviewed-by: Brian Sheedy <bsheedy@google.com>
Reviewed-by: Ben Pastene <bpastene@google.com>
Commit-Queue: Daniel Saldaña <dansal@google.com>
diff --git a/tools/src/cmd/cts/common/results.go b/tools/src/cmd/cts/common/results.go
index 571246d..b03886e 100644
--- a/tools/src/cmd/cts/common/results.go
+++ b/tools/src/cmd/cts/common/results.go
@@ -35,7 +35,6 @@
"path/filepath"
"sort"
"strconv"
- "strings"
"time"
"dawn.googlesource.com/dawn/tools/src/buildbucket"
@@ -349,11 +348,7 @@
fmt.Printf(".")
}
- if !strings.HasPrefix(r.TestId, prefix) {
- return fmt.Errorf("Test ID %s did not start with %s even though query should have filtered.", r.TestId, prefix)
- }
-
- testName := r.TestId[len(prefix):]
+ testName := r.Name
status := convertRdbStatus(r.Status)
tags := result.NewTags()
duration := time.Duration(r.Duration * float64(time.Second))
@@ -653,18 +648,14 @@
for _, prefix := range test.Prefixes {
rowHandler := func(r *resultsdb.QueryResult) error {
- if !strings.HasPrefix(r.TestId, prefix) {
- return fmt.Errorf(
- "Test ID %s did not start with %s even though query should have filtered.",
- r.TestId, prefix)
- }
-
- testName := r.TestId[len(prefix):]
+ testName := r.Name
tags := result.NewTags()
for _, tagPair := range r.Tags {
- if tagPair.Key != "typ_tag" {
- return fmt.Errorf("Got tag key %v when only typ_tag should be present", tagPair.Key)
+ // We only care about typ_tag, but the new test id format adds a
+ // gpu_test_class tag that we need to ignore.
+ if tagPair.Key != "typ_tag" && tagPair.Key != "gpu_test_class" {
+ return fmt.Errorf("Got tag key %v when only typ_tag and gpu_test_class should be present", tagPair.Key)
}
tags.Add(tagPair.Value)
}
diff --git a/tools/src/cmd/cts/common/results_test.go b/tools/src/cmd/cts/common/results_test.go
index 1136077..f7773c2 100644
--- a/tools/src/cmd/cts/common/results_test.go
+++ b/tools/src/cmd/cts/common/results_test.go
@@ -109,6 +109,7 @@
"prefix": []resultsdb.QueryResult{
{
TestId: "bad_test",
+ Name: "bad_test",
Status: "FAIL",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
@@ -137,45 +138,6 @@
require.Equal(t, cachedResults, resultsByExecutionMode)
}
-func TestCacheResults_GetRawResultsError(t *testing.T) {
- testCacheResults_GetRawResultsError_Impl(t, false)
-}
-
-func TestCacheUnsuppressedFailingResults_GetRawResultsError(t *testing.T) {
- testCacheResults_GetRawResultsError_Impl(t, true)
-}
-
-func testCacheResults_GetRawResultsError_Impl(t *testing.T, unsuppressedOnly bool) {
- client := resultsdb.MockBigQueryClient{}
- var testedFunc cacheResultsFunc
- var clientDataField *resultsdb.PrefixGroupedQueryResults
- if unsuppressedOnly {
- testedFunc = CacheUnsuppressedFailingResults
- clientDataField = &client.UnsuppressedFailureReturnValues
- } else {
- testedFunc = CacheResults
- clientDataField = &client.ReturnValues
- }
-
- ctx, cfg, _, patchset, cacheDir := getCacheResultsSharedSetupData()
-
- *clientDataField = resultsdb.PrefixGroupedQueryResults{
- "prefix": []resultsdb.QueryResult{
- {
- TestId: "bad_test",
- Status: "FAIL",
- Tags: []resultsdb.TagPair{},
- Duration: 1.0,
- },
- },
- }
-
- resultsByExecutionMode, err := testedFunc(ctx, cfg, patchset, cacheDir, client, BuildsByName{})
- require.Nil(t, resultsByExecutionMode)
- require.ErrorContains(t, err,
- "Test ID bad_test did not start with prefix even though query should have filtered.")
-}
-
func TestCacheResults_Success(t *testing.T) {
testCacheResults_Success_Impl(t, false)
}
@@ -202,6 +164,7 @@
"prefix": []resultsdb.QueryResult{
{
TestId: "prefix_test_2",
+ Name: "_test_2",
Status: "CRASH",
Tags: []resultsdb.TagPair{
{
@@ -213,6 +176,7 @@
},
{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -230,6 +194,7 @@
// Should be merged into the above result by CleanResults()
{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "PASS",
Tags: []resultsdb.TagPair{
{
@@ -304,6 +269,7 @@
"prefix": []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test",
+ Name: "_test",
Status: "PASS",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
@@ -329,6 +295,7 @@
"prefix": []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_2",
+ Name: "_test_2",
Status: "PASS",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -344,6 +311,7 @@
},
resultsdb.QueryResult{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "PASS",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -381,16 +349,6 @@
require.Equal(t, expectedResults, results)
}
-// Tests that errors from GetRawResults are properly surfaced.
-func TestGetResultsGetRawResultsErrorSurfaced(t *testing.T) {
- ctx, cfg, client, builds := generateGoodGetResultsInputs()
- client.ReturnValues["prefix"][0].TestId = "bad_test"
-
- results, err := GetResults(ctx, cfg, client, builds)
- require.Nil(t, results)
- require.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
-}
-
/*******************************************************************************
* GetUnsuppressedFailingResults tests
******************************************************************************/
@@ -414,6 +372,7 @@
"prefix": []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test",
+ Name: "_test",
Status: "FAIL",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
@@ -439,6 +398,7 @@
"prefix": []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_2",
+ Name: "_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -454,6 +414,7 @@
},
resultsdb.QueryResult{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -491,16 +452,6 @@
require.Equal(t, expectedResults, results)
}
-// Tests that errors from GetRawResults are properly surfaced.
-func TestGetUnsuppressedFailingResultsGetRawResultsErrorSurfaced(t *testing.T) {
- ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
- client.UnsuppressedFailureReturnValues["prefix"][0].TestId = "bad_test"
-
- results, err := GetUnsuppressedFailingResults(ctx, cfg, client, builds)
- require.Nil(t, results)
- require.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
-}
-
/*******************************************************************************
* GetRawResults tests
******************************************************************************/
@@ -512,12 +463,14 @@
"prefix": []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "PASS",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_2",
+ Name: "_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -529,6 +482,7 @@
},
resultsdb.QueryResult{
TestId: "prefix_test_3",
+ Name: "_test_3",
Status: "SKIP",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -540,6 +494,7 @@
},
resultsdb.QueryResult{
TestId: "prefix_test_4",
+ Name: "_test_4",
Status: "SomeStatus",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -595,16 +550,6 @@
require.Equal(t, expectedResults, results)
}
-// Tests that a mismatched prefix results in an error.
-func TestGetRawResultsPrefixMismatch(t *testing.T) {
- ctx, cfg, client, builds := generateGoodGetResultsInputs()
- client.ReturnValues["prefix"][0].TestId = "bad_test"
-
- results, err := GetRawResults(ctx, cfg, client, builds)
- require.Nil(t, results)
- require.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
-}
-
// Tests that a JavaScript duration that cannot be parsed results in an error.
func TestGetRawResultsBadJavaScriptDuration(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetResultsInputs()
@@ -696,12 +641,14 @@
"prefix": []resultsdb.QueryResult{
resultsdb.QueryResult{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{},
Duration: 1.0,
},
resultsdb.QueryResult{
TestId: "prefix_test_2",
+ Name: "_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -713,6 +660,7 @@
},
resultsdb.QueryResult{
TestId: "prefix_test_3",
+ Name: "_test_3",
Status: "FAIL",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -724,6 +672,7 @@
},
resultsdb.QueryResult{
TestId: "prefix_test_4",
+ Name: "_test_4",
Status: "SomeStatus",
Tags: []resultsdb.TagPair{
resultsdb.TagPair{
@@ -779,16 +728,6 @@
require.Equal(t, expectedResults, results)
}
-// Tests that a mismatched prefix results in an error.
-func TestGetRawUnsuppressedFailingResultsPrefixMismatch(t *testing.T) {
- ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
- client.UnsuppressedFailureReturnValues["prefix"][0].TestId = "bad_test"
-
- results, err := GetRawUnsuppressedFailingResults(ctx, cfg, client, builds)
- require.Nil(t, results)
- require.ErrorContains(t, err, "Test ID bad_test did not start with prefix even though query should have filtered.")
-}
-
// Tests that a JavaScript duration that cannot be parsed results in an error.
func TestGetRawUnsuppressedFailingResultsBadJavaScriptDuration(t *testing.T) {
ctx, cfg, client, builds := generateGoodGetUnsuppressedFailingResultsInputs()
@@ -1034,6 +973,7 @@
"core_prefix": []resultsdb.QueryResult{
{
TestId: "core_prefix_test_1",
+ Name: "_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -1045,6 +985,7 @@
},
{
TestId: "core_prefix_test_2",
+ Name: "_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -1058,6 +999,7 @@
"compat_prefix": []resultsdb.QueryResult{
{
TestId: "compat_prefix_test_3",
+ Name: "_test_3",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -1069,6 +1011,7 @@
},
{
TestId: "compat_prefix_test_4",
+ Name: "_test_4",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -1119,24 +1062,6 @@
}
}
-func TestCacheRecentUniqueSuppressedCoreResults_ErrorSurfaced(t *testing.T) {
- ctx := context.Background()
- cfg := getMultiPrefixConfig()
- wrapper := oswrapper.CreateMemMapOSWrapper()
-
- results := getMultiPrefixQueryResults()
- results["core_prefix"][0].TestId = "bad_test"
- client := resultsdb.MockBigQueryClient{
- RecentUniqueSuppressedReturnValues: results,
- }
-
- resultsList, err := CacheRecentUniqueSuppressedCoreResults(
- ctx, cfg, fileutils.ThisDir(), client, wrapper)
- require.Nil(t, resultsList)
- require.ErrorContains(t, err,
- "Test ID bad_test did not start with core_prefix even though query should have filtered.")
-}
-
func TestCacheRecentUniqueSuppressedCoreResults_Success(t *testing.T) {
ctx := context.Background()
cfg := getMultiPrefixConfig()
@@ -1152,24 +1077,6 @@
require.Equal(t, resultsList, getExpectedMultiPrefixResults()["core"])
}
-func TestCAcheRecentUniqueSuppressedCompatResults_ErrorSurfaced(t *testing.T) {
- ctx := context.Background()
- cfg := getMultiPrefixConfig()
- wrapper := oswrapper.CreateMemMapOSWrapper()
-
- results := getMultiPrefixQueryResults()
- results["compat_prefix"][0].TestId = "bad_test"
- client := resultsdb.MockBigQueryClient{
- RecentUniqueSuppressedReturnValues: results,
- }
-
- resultsList, err := CacheRecentUniqueSuppressedCoreResults(
- ctx, cfg, fileutils.ThisDir(), client, wrapper)
- require.Nil(t, resultsList)
- require.ErrorContains(t, err,
- "Test ID bad_test did not start with compat_prefix even though query should have filtered.")
-}
-
func TestCacheRecentUniqueSuppressedCompatResults_Success(t *testing.T) {
ctx := context.Background()
cfg := getMultiPrefixConfig()
@@ -1257,7 +1164,7 @@
ctx, cfg, fileutils.ThisDir(), client, wrapper)
require.Nil(t, resultsByExecutionMode)
require.ErrorContains(t, err,
- "Got tag key non_typ_tag when only typ_tag should be present")
+ "Got tag key non_typ_tag when only typ_tag and gpu_test_class should be present")
}
func TestCacheRecentUniqueSuppressedResults_Success(t *testing.T) {
@@ -1278,37 +1185,6 @@
* getRecentUniqueSuppressedResults tests
******************************************************************************/
-func TestGetRecentUniqueSuppressedResults_PrefixMismatch(t *testing.T) {
- ctx := context.Background()
-
- cfg := Config{
- Tests: []TestConfig{
- TestConfig{
- ExecutionMode: result.ExecutionMode("execution_mode"),
- Prefixes: []string{"prefix"},
- },
- },
- }
-
- client := resultsdb.MockBigQueryClient{
- RecentUniqueSuppressedReturnValues: resultsdb.PrefixGroupedQueryResults{
- "prefix": []resultsdb.QueryResult{
- {
- TestId: "bad_test",
- Status: "FAIL",
- Tags: []resultsdb.TagPair{},
- Duration: 1.0,
- },
- },
- },
- }
-
- resultsByExecutionMode, err := getRecentUniqueSuppressedResults(ctx, cfg, client)
- require.Nil(t, resultsByExecutionMode)
- require.ErrorContains(t, err,
- "Test ID bad_test did not start with prefix even though query should have filtered.")
-}
-
func TestGetRecentUniqueSuppressedResults_NonTypTag(t *testing.T) {
ctx := context.Background()
@@ -1342,7 +1218,7 @@
resultsByExecutionMode, err := getRecentUniqueSuppressedResults(ctx, cfg, client)
require.Nil(t, resultsByExecutionMode)
require.ErrorContains(t, err,
- "Got tag key non_typ_tag when only typ_tag should be present")
+ "Got tag key non_typ_tag when only typ_tag and gpu_test_class should be present")
}
func TestGetRecentUniqueSuppressedResults_Success(t *testing.T) {
@@ -1362,6 +1238,7 @@
"prefix": []resultsdb.QueryResult{
{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -1373,6 +1250,7 @@
},
{
TestId: "prefix_test_2",
+ Name: "_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -1384,6 +1262,7 @@
},
{
TestId: "prefix_test_1",
+ Name: "_test_1",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
@@ -1395,6 +1274,7 @@
},
{
TestId: "prefix_test_2",
+ Name: "_test_2",
Status: "FAIL",
Tags: []resultsdb.TagPair{
{
diff --git a/tools/src/cmd/cts/config.json b/tools/src/cmd/cts/config.json
index 1de8367..ce66fb7 100644
--- a/tools/src/cmd/cts/config.json
+++ b/tools/src/cmd/cts/config.json
@@ -4,14 +4,22 @@
"ExecutionMode": "core",
"Prefixes": [
"ninja://chrome/test:telemetry_gpu_integration_test/gpu_tests.webgpu_cts_integration_test.WebGpuCtsIntegrationTest.",
- "ninja://chrome/test:telemetry_gpu_integration_test_android_chrome/gpu_tests.webgpu_cts_integration_test.WebGpuCtsIntegrationTest."
+ "ninja://chrome/test:telemetry_gpu_integration_test_android_chrome/gpu_tests.webgpu_cts_integration_test.WebGpuCtsIntegrationTest.",
+ "://chrome/test\\:telemetry_gpu_integration_test!webgpucts:",
+ "://chrome/test\\:telemetry_gpu_integration_test_android_chrome!webgpucts:",
+ "://chrome/test\\:telemetry_gpu_integration_test!flat:",
+ "://chrome/test\\:telemetry_gpu_integration_test_android_chrome!flat:"
]
},
{
"ExecutionMode": "compat",
"Prefixes": [
"ninja://chrome/test:telemetry_gpu_integration_test/gpu_tests.webgpu_compat_cts_integration_test.WebGpuCompatCtsIntegrationTest.",
- "ninja://chrome/test:telemetry_gpu_integration_test_android_chrome/gpu_tests.webgpu_compat_cts_integration_test.WebGpuCompatCtsIntegrationTest."
+ "ninja://chrome/test:telemetry_gpu_integration_test_android_chrome/gpu_tests.webgpu_compat_cts_integration_test.WebGpuCompatCtsIntegrationTest.",
+ "://chrome/test\\:telemetry_gpu_integration_test!webgpucts:",
+ "://chrome/test\\:telemetry_gpu_integration_test_android_chrome!webgpucts:",
+ "://chrome/test\\:telemetry_gpu_integration_test!flat:",
+ "://chrome/test\\:telemetry_gpu_integration_test_android_chrome!flat:"
]
}
],
diff --git a/tools/src/cmd/cts/expectationcoverage/expectationcoverage_test.go b/tools/src/cmd/cts/expectationcoverage/expectationcoverage_test.go
index 96c9cdb..b5c77c2 100644
--- a/tools/src/cmd/cts/expectationcoverage/expectationcoverage_test.go
+++ b/tools/src/cmd/cts/expectationcoverage/expectationcoverage_test.go
@@ -135,31 +135,6 @@
require.ErrorContains(t, err, "/webgpu-cts/expectations.txt:6:31 error: expected status")
}
-func TestRun_GetResultsFailure(t *testing.T) {
- wrapper, err := common.CreateMemMapOSWrapperWithFakeDefaultPaths()
- require.NoErrorf(t, err, "Error creating fake Dawn directories: %v", err)
- client := resultsdb.MockBigQueryClient{}
-
- expectationFileContent := getExpectationContentForTrimmedContentTest()
- wrapper.WriteFile(common.DefaultExpectationsPath(wrapper), []byte(expectationFileContent), 0o700)
-
- client.RecentUniqueSuppressedReturnValues = resultsdb.PrefixGroupedQueryResults{
- "core_prefix": []resultsdb.QueryResult{
- {
- TestId: "invalid_prefix_test",
- Status: "FAIL",
- },
- },
- }
-
- ctx := context.Background()
- cfg := createConfig(wrapper, client)
- c := cmd{}
- err = c.Run(ctx, cfg)
- require.ErrorContains(t, err,
- "Test ID invalid_prefix_test did not start with core_prefix even though query should have filtered.")
-}
-
func TestRun_SuccessCore(t *testing.T) {
wrapper, err := common.CreateMemMapOSWrapperWithFakeDefaultPaths()
require.NoErrorf(t, err, "Error creating fake Dawn directories: %v", err)
diff --git a/tools/src/resultsdb/resultsdb.go b/tools/src/resultsdb/resultsdb.go
index c410806..d671356 100644
--- a/tools/src/resultsdb/resultsdb.go
+++ b/tools/src/resultsdb/resultsdb.go
@@ -57,6 +57,7 @@
// BigQuery query.
type QueryResult struct {
TestId string
+ Name string
Status string
Tags []TagPair
Duration float64
@@ -116,6 +117,7 @@
baseQuery := `
SELECT
test_id AS testid,
+ test_metadata.name AS testname,
status,
tags,
duration
@@ -146,6 +148,7 @@
failing_results AS (
SELECT
test_id AS testid,
+ test_metadata.name AS testname,
status,
tags,
duration,
@@ -194,6 +197,7 @@
recent_results AS (
SELECT
test_id AS testid,
+ test_metadata.name AS testname,
ARRAY(
SELECT t
FROM tr.tags t
@@ -223,10 +227,13 @@
(
ARRAY_LENGTH(typ_expectations) > 1
)
- GROUP BY testid, tags
+ GROUP BY testid, testname, tags
`
- query := fmt.Sprintf(baseQuery, testPrefix)
+ // We need to escape backslashes since they are special characters in
+ // BigQuery strings.
+ escapedTestPrefix := strings.ReplaceAll(testPrefix, `\`, `\\`)
+ query := fmt.Sprintf(baseQuery, escapedTestPrefix)
return bq.runQuery(ctx, query, f)
}
@@ -239,7 +246,10 @@
for _, id := range builds {
buildIds = append(buildIds, fmt.Sprintf(`"build-%v"`, id))
}
- query := fmt.Sprintf(baseQuery, strings.Join(buildIds, ","), testPrefix)
+ // We need to escape backslashes since they are special characters in
+ // BigQuery strings.
+ escapedTestPrefix := strings.ReplaceAll(testPrefix, `\`, `\\`)
+ query := fmt.Sprintf(baseQuery, strings.Join(buildIds, ","), escapedTestPrefix)
return bq.runQuery(ctx, query, f)
}
