tools/run-cts: Fix code coverage in `--isolate` mode
Use the new `cmdline.ts --coverage` flag in https://github.com/gpuweb/cts/pull/2206 to write out the per-test coverage to a unique file.
Change-Id: Iea273217eede2fa615b78cd6f69f036976dcfd35
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/117883
Kokoro: Ben Clayton <bclayton@google.com>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Commit-Queue: Ben Clayton <bclayton@chromium.org>
diff --git a/tools/src/cmd/run-cts/main.go b/tools/src/cmd/run-cts/main.go
index 7ca1a9a..d69a49e 100644
--- a/tools/src/cmd/run-cts/main.go
+++ b/tools/src/cmd/run-cts/main.go
@@ -850,16 +850,10 @@
for i := 0; i < r.numRunners; i++ {
wg.Add(1)
- profraw := ""
- if r.covEnv != nil {
- profraw = filepath.Join(r.tmpDir, fmt.Sprintf("cts-%v.profraw", i))
- defer os.Remove(profraw)
- }
-
go func() {
defer wg.Done()
for idx := range caseIndices {
- res := r.runTestcase(ctx, r.testcases[idx], profraw)
+ res := r.runTestcase(ctx, r.testcases[idx])
res.index = idx
results <- res
@@ -1053,17 +1047,11 @@
return nil
}
-// runSerially() calls the CTS test runner to run the test query in a single
-// process.
+// runSerially() calls the CTS test runner to run the test query in a single process.
// TODO(bclayton): Support comparing against r.expectations
func (r *runner) runSerially(ctx context.Context, query string) error {
- profraw := ""
- if r.covEnv != nil {
- profraw = filepath.Join(r.tmpDir, "cts.profraw")
- }
-
start := time.Now()
- result := r.runTestcase(ctx, query, profraw)
+ result := r.runTestcase(ctx, query)
timeTaken := time.Since(start)
if r.verbose {
@@ -1115,7 +1103,7 @@
// runTestcase() runs the CTS testcase with the given query, returning the test
// result.
-func (r *runner) runTestcase(ctx context.Context, query string, profraw string) result {
+func (r *runner) runTestcase(ctx context.Context, query string) result {
ctx, cancel := context.WithTimeout(ctx, testTimeout)
defer cancel()
@@ -1134,6 +1122,9 @@
if r.verbose {
args = append(args, "--gpu-provider-flag", "verbose=1")
}
+ if r.covEnv != nil {
+ args = append(args, "--coverage")
+ }
if r.colors {
args = append(args, "--colors")
}
@@ -1148,11 +1139,6 @@
cmd := exec.CommandContext(ctx, r.node, args...)
cmd.Dir = r.cts
- if profraw != "" {
- cmd.Env = os.Environ()
- cmd.Env = append(cmd.Env, cov.RuntimeEnv(cmd.Env, profraw))
- }
-
var buf bytes.Buffer
cmd.Stdout = &buf
cmd.Stderr = &buf
@@ -1160,18 +1146,33 @@
err := cmd.Run()
msg := buf.String()
- res := result{testcase: query,
- status: pass,
- message: msg,
- error: err,
+ res := result{
+ testcase: query,
+ status: pass,
+ message: msg,
+ error: err,
}
- if r.covEnv != nil {
- coverage, covErr := r.covEnv.Import(profraw)
- if covErr != nil {
- err = fmt.Errorf("could not import coverage data: %v", err)
+ if err == nil && r.covEnv != nil {
+ const header = "Code-coverage: [["
+ const footer = "]]"
+ if headerStart := strings.Index(msg, header); headerStart >= 0 {
+ if footerStart := strings.Index(msg[headerStart:], footer); footerStart >= 0 {
+ footerStart += headerStart
+ path := msg[headerStart+len(header) : footerStart]
+ res.message = msg[:headerStart] + msg[footerStart+len(footer):] // Strip out the coverage from the message
+ coverage, covErr := r.covEnv.Import(path)
+ os.Remove(path)
+ if covErr == nil {
+ res.coverage = coverage
+ } else {
+ err = fmt.Errorf("could not import coverage data from '%v': %v", path, covErr)
+ }
+ }
}
- res.coverage = coverage
+ if err == nil && res.coverage == nil {
+ err = fmt.Errorf("failed to parse code coverage from output")
+ }
}
switch {
diff --git a/tools/src/cov/import.go b/tools/src/cov/import.go
index 176c6bf..7d1ef21 100644
--- a/tools/src/cov/import.go
+++ b/tools/src/cov/import.go
@@ -48,12 +48,6 @@
TurboCov string // path to the turbo-cov tool (one of Cov or TurboCov must be supplied)
}
-// RuntimeEnv returns the environment variable key=value pair for setting
-// LLVM_PROFILE_FILE to coverageFile
-func RuntimeEnv(env []string, coverageFile string) string {
- return "LLVM_PROFILE_FILE=" + coverageFile
-}
-
// AllSourceFiles returns a *Coverage containing all the source files without
// coverage data. This populates the coverage view with files even if they
// didn't get compiled.
