tools/src/cmd/cts/update/expectations/expectations_test.go

// Copyright 2025 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//  1. Redistributions of source code must retain the above copyright notice, this
//     list of conditions and the following disclaimer.
//
//  2. Redistributions in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//  3. Neither the name of the copyright holder nor the names of its
//     contributors may be used to endorse or promote products derived from
//     this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package expectations

import (
	"context"
	"path/filepath"
	"testing"

	"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
	"dawn.googlesource.com/dawn/tools/src/fileutils"
	"dawn.googlesource.com/dawn/tools/src/oswrapper"
	"github.com/stretchr/testify/require"
)

func TestUpdateExpectations(t *testing.T) {
	tests := []struct {
		name                 string
		resultsFile          string
		resultsContent       string
		testListPath         string
		testListContent      string
		expectationsPath     string
		expectationsContent  string
		expectationsFlag     []string
		wantErr              string
		wantLog              string
		verifyContent        func(t *testing.T, fs oswrapper.FilesystemReader, path string)
		generateExplicitTags bool
	}{
		{
			name:             "Success",
			resultsFile:      "results.txt",
			resultsContent:   "webgpu:test:one linux,nvidia Failure 0s false\ncore",
			testListPath:     "third_party/gn/webgpu-cts/test_list.txt",
			testListContent:  "webgpu:test:one\nwebgpu:test:two",
			expectationsPath: "expectations.txt",
			expectationsContent: `# BEGIN TAG HEADER
# OS
# tags: [ mac win linux android ]
# GPU
# tags: [ amd intel nvidia qualcomm ]
# Device
# tags: [ android-pixel-4 android-pixel-6 chromeos-board-amd64-generic fuchsia-board-qemu-x64 ]
# END TAG HEADER

# existing
[ mac ] webgpu:test:two: [ Failure ]
`,
			expectationsFlag: []string{"expectations.txt"},
			verifyContent: func(t *testing.T, fs oswrapper.FilesystemReader, path string) {
				content, err := fs.ReadFile(path)
				require.NoError(t, err)
				text := string(content)
				require.Contains(t, text, "[ linux nvidia ] webgpu:test:one: [ Failure ]")
				require.Contains(t, text, "[ mac ] webgpu:test:two: [ Failure ]")
			},
		},
		{
			name:             "Results Fetch Error",
			resultsFile:      "nonexistent.txt",
			testListPath:     "third_party/gn/webgpu-cts/test_list.txt",
			testListContent:  "webgpu:test:one",
			expectationsPath: "expectations.txt",
			expectationsFlag: []string{"expectations.txt"},
			wantErr:          "does not exist",
		},
		{
			name:             "Test List Load Error",
			resultsFile:      "results.txt",
			resultsContent:   "webgpu:test:one linux Failure 0s false\ncore",
			testListPath:     "nonexistent/test_list.txt",
			expectationsPath: "expectations.txt",
			expectationsFlag: []string{"expectations.txt"},
			wantErr:          "failed to load test list",
		},
		{
			name:             "Expectations Load Error",
			resultsFile:      "results.txt",
			resultsContent:   "webgpu:test:one linux Failure 0s false\ncore",
			testListPath:     "third_party/gn/webgpu-cts/test_list.txt",
			testListContent:  "webgpu:test:one\nwebgpu:test:two",
			expectationsPath: "expectations.txt",
			expectationsFlag: []string{"nonexistent.txt"},
			wantErr:          "does not exist",
		},
		{
			name:                "Expectations Validation Error",
			resultsFile:         "results.txt",
			resultsContent:      "webgpu:test:one linux Failure 0s false\ncore",
			testListPath:        "third_party/gn/webgpu-cts/test_list.txt",
			testListContent:     "webgpu:test:one",
			expectationsPath:    "expectations.txt",
			expectationsContent: "INVALID SYNTAX",
			expectationsFlag:    []string{"expectations.txt"},
			wantErr:             "expected status",
		},
		{
			name:             "Compat Mode",
			resultsFile:      "results.txt",
			resultsContent:   "webgpu:test:compat linux,nvidia Failure 0s false\ncompat",
			testListPath:     "third_party/gn/webgpu-cts/test_list.txt",
			testListContent:  "webgpu:test:compat",
			expectationsPath: "compat_expectations.txt",
			expectationsContent: `# BEGIN TAG HEADER
# OS
# tags: [ mac win linux android ]
# GPU
# tags: [ amd intel nvidia qualcomm ]
# Device
# tags: [ android-pixel-4 android-pixel-6 chromeos-board-amd64-generic fuchsia-board-qemu-x64 ]
# END TAG HEADER
`,
			expectationsFlag: []string{"compat_expectations.txt"},
			verifyContent: func(t *testing.T, fs oswrapper.FilesystemReader, path string) {
				content, err := fs.ReadFile(path)
				require.NoError(t, err)
				require.Contains(t, string(content), "[ linux nvidia ] webgpu:test:compat: [ Failure ]")
			},
		},
		{
			name:             "Unknown Tests Removal",
			resultsFile:      "results.txt",
			resultsContent:   "webgpu:test:known linux,nvidia Failure 0s false\ncore",
			testListPath:     "third_party/gn/webgpu-cts/test_list.txt",
			testListContent:  "webgpu:test:known",
			expectationsPath: "expectations.txt",
			expectationsContent: `# BEGIN TAG HEADER
# OS
# tags: [ mac win linux android ]
# GPU
# tags: [ amd intel nvidia qualcomm ]
# Device
# tags: [ android-pixel-4 android-pixel-6 chromeos-board-amd64-generic fuchsia-board-qemu-x64 ]
# END TAG HEADER

webgpu:test:unknown: [ Failure ]`,
			expectationsFlag: []string{"expectations.txt"},
			verifyContent: func(t *testing.T, fs oswrapper.FilesystemReader, path string) {
				content, err := fs.ReadFile(path)
				require.NoError(t, err)
				require.NotContains(t, string(content), "webgpu:test:unknown")
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			wrapper := oswrapper.CreateFSTestOSWrapper()
			realWrapper := oswrapper.GetRealOSWrapper()
			rootDir := fileutils.DawnRoot(realWrapper)

			// Helper to write file relative to root
			write := func(path, content string) {
				fullPath := filepath.Join(rootDir, path)
				require.NoError(t, wrapper.MkdirAll(filepath.Dir(fullPath), 0755))
				require.NoError(t, wrapper.WriteFile(fullPath, []byte(content), 0666))
			}

			if tt.resultsContent != "" {
				write(tt.resultsFile, tt.resultsContent)
			}
			if tt.testListContent != "" {
				write(tt.testListPath, tt.testListContent)
			}
			if tt.expectationsContent != "" {
				write(tt.expectationsPath, tt.expectationsContent)
			}

			c := &cmd{}

			// Resolve paths to absolute for flags
			absExpectations := make([]string, len(tt.expectationsFlag))
			for i, p := range tt.expectationsFlag {
				absExpectations[i] = filepath.Join(rootDir, p)
			}
			c.flags.expectations = absExpectations

			if tt.resultsFile != "" {
				c.flags.results.File = filepath.Join(rootDir, tt.resultsFile)
			}

			c.flags.verbose = true
			c.flags.generateExplicitTags = tt.generateExplicitTags

			ctx := context.Background()
			cfg := common.Config{OsWrapper: wrapper}

			write("DEPS", "") // Ensure DawnRoot detection works

			err := c.Run(ctx, cfg)
			if tt.wantErr != "" {
				require.Error(t, err)
				require.ErrorContains(t, err, tt.wantErr)
			} else {
				require.NoError(t, err)
			}

			if tt.verifyContent != nil {
				// Check content of the first expectations file
				path := filepath.Join(rootDir, tt.expectationsPath)
				tt.verifyContent(t, wrapper, path)
			}
		})
	}
}