blob: 990a9096a42bc39f8b4f36a7c01ea9a86e3119be [file] [log] [blame]
Ben Clayton85965352023-10-30 20:08:49 +00001// Copyright 2023 The Dawn & Tint Authors
2//
3// Redistribution and use in source and binary forms, with or without
4// modification, are permitted provided that the following conditions are met:
5//
6// 1. Redistributions of source code must retain the above copyright notice, this
7// list of conditions and the following disclaimer.
8//
9// 2. Redistributions in binary form must reproduce the above copyright notice,
10// this list of conditions and the following disclaimer in the documentation
11// and/or other materials provided with the distribution.
12//
13// 3. Neither the name of the copyright holder nor the names of its
14// contributors may be used to endorse or promote products derived from
15// this software without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
21// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
24// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28package node
29
30import (
31 "bytes"
32 "context"
33 "errors"
34 "fmt"
35 "os"
36 "os/exec"
37 "path/filepath"
38 "strings"
39 "sync"
Ben Claytond0154132024-02-29 16:51:45 +000040 "time"
Ben Clayton85965352023-10-30 20:08:49 +000041
42 "dawn.googlesource.com/dawn/tools/src/cmd/run-cts/common"
43)
44
45// runTestCasesWithCmdline() calls the CTS command-line test runner to run each
46// test case in a separate process. This reduces possibility of state leakage
47// between tests.
48// Up to c.flags.numRunners tests will be run concurrently.
49func (c *cmd) runTestCasesWithCmdline(ctx context.Context, testCases []common.TestCase, results chan<- common.Result) {
50 // Create a chan of test indices.
51 // This will be read by the test runner goroutines.
52 testCaseIndices := make(chan int, 256)
53 go func() {
54 for i := range testCases {
55 testCaseIndices <- i
56 }
57 close(testCaseIndices)
58 }()
59
60 // Spin up the test runner goroutines
61 wg := &sync.WaitGroup{}
62 for i := 0; i < c.flags.NumRunners; i++ {
63 wg.Add(1)
64 go func() {
65 defer wg.Done()
66 for idx := range testCaseIndices {
67 res := c.runTestCaseWithCmdline(ctx, testCases[idx])
68 res.Index = idx
69 results <- res
70 if err := ctx.Err(); err != nil {
71 return
72 }
73 }
74 }()
75 }
76 wg.Wait()
77}
78
79// runTestCaseWithCmdline() runs a single CTS test case with the command line tool,
80// returning the test result.
81func (c *cmd) runTestCaseWithCmdline(ctx context.Context, testCase common.TestCase) common.Result {
82 ctx, cancel := context.WithTimeout(ctx, common.TestCaseTimeout)
83 defer cancel()
84
85 args := []string{
Ben Clayton2880e5d2023-11-16 17:16:57 +000086 "-e", "require('./out-node/common/runtime/cmdline.js');",
Ben Clayton85965352023-10-30 20:08:49 +000087 "--",
88 // src/common/runtime/helper/sys.ts expects 'node file.js <args>'
89 // and slices away the first two arguments. When running with '-e', args
90 // start at 1, so just inject a placeholder argument.
91 "placeholder-arg",
92 // Actual arguments begin here
93 "--gpu-provider", filepath.Join(c.flags.bin, "cts.js"),
94 "--verbose", // always required to emit test pass results
95 "--quiet",
96 }
97 if c.flags.Verbose {
98 args = append(args, "--gpu-provider-flag", "verbose=1")
99 }
100 if c.coverage != nil {
101 args = append(args, "--coverage")
102 }
103 if c.flags.Colors {
104 args = append(args, "--colors")
105 }
106 if c.flags.unrollConstEvalLoops {
107 args = append(args, "--unroll-const-eval-loops")
108 }
109 if c.flags.compatibilityMode {
110 args = append(args, "--compat")
111 }
112 for _, f := range c.flags.dawn {
113 args = append(args, "--gpu-provider-flag", f)
114 }
115 args = append(args, string(testCase))
116
117 cmd := exec.CommandContext(ctx, c.flags.Node, args...)
118 cmd.Dir = c.flags.CTS
119
120 var buf bytes.Buffer
121 cmd.Stdout = &buf
122 cmd.Stderr = &buf
123
124 if c.flags.Verbose {
dan sinclair92d94fd2024-03-06 18:43:48 +0000125 PrintCommand(cmd, c.flags.skipVSCodeInfo)
Ben Clayton85965352023-10-30 20:08:49 +0000126 }
Ben Claytond0154132024-02-29 16:51:45 +0000127
128 start := time.Now()
Ben Clayton85965352023-10-30 20:08:49 +0000129 err := cmd.Run()
130
131 msg := buf.String()
132 res := common.Result{
133 TestCase: testCase,
134 Status: common.Pass,
135 Message: msg,
136 Error: err,
Ben Claytond0154132024-02-29 16:51:45 +0000137 Duration: time.Since(start),
Ben Clayton85965352023-10-30 20:08:49 +0000138 }
139
140 if err == nil && c.coverage != nil {
141 const header = "Code-coverage: [["
142 const footer = "]]"
143 if headerStart := strings.Index(msg, header); headerStart >= 0 {
144 if footerStart := strings.Index(msg[headerStart:], footer); footerStart >= 0 {
145 footerStart += headerStart
146 path := msg[headerStart+len(header) : footerStart]
147 res.Message = msg[:headerStart] + msg[footerStart+len(footer):] // Strip out the coverage from the message
148 coverage, covErr := c.coverage.Env.Import(path)
149 os.Remove(path)
150 if covErr == nil {
151 res.Coverage = coverage
152 } else {
153 err = fmt.Errorf("could not import coverage data from '%v': %v", path, covErr)
154 }
155 }
156 }
157 if err == nil && res.Coverage == nil {
158 err = fmt.Errorf("failed to parse code coverage from output")
159 }
160 }
161
162 switch {
163 case errors.Is(err, context.DeadlineExceeded):
164 res.Status = common.Timeout
165 case err != nil, strings.Contains(msg, "[fail]"):
166 res.Status = common.Fail
167 case strings.Contains(msg, "[warn]"):
168 res.Status = common.Warn
169 case strings.Contains(msg, "[skip]"):
170 res.Status = common.Skip
171 case strings.Contains(msg, "[pass]"):
172 break
173 default:
174 res.Status = common.Fail
175 msg += "\ncould not parse test output"
176 }
177
178 if res.Error != nil {
179 res.Message = fmt.Sprint(res.Message, res.Error)
180 }
181 return res
182}