Add tools/fix-tests
A simple regex based tool for fixing unit tests that fail due to unexpected output
Change-Id: I72c47abaff6d6f4ba8cd497240eadc171af0fec3
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/47629
Commit-Queue: Ben Clayton <bclayton@chromium.org>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
diff --git a/tools/fix-tests/fix-tests.go b/tools/fix-tests/fix-tests.go
new file mode 100644
index 0000000..6b8b860
--- /dev/null
+++ b/tools/fix-tests/fix-tests.go
@@ -0,0 +1,277 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// fix-tests is a tool to update tests with new expected output.
+package main
+
+import (
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+)
+
+func main() {
+ if err := run(); err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+}
+
+func showUsage() {
+ fmt.Println(`
+fix-tests is a tool to update tests with new expected output.
+
+Usage:
+ fix-tests <executable>
+
+ executable - the path to the test executable to run.`)
+ os.Exit(1)
+}
+
+func run() error {
+ flag.Parse()
+ args := flag.Args()
+ if len(args) < 1 {
+ showUsage()
+ }
+
+ exe := args[0] // The path to the test executable
+ wd := filepath.Dir(exe) // The directory holding the test exe
+
+ // Create a temporary directory to hold the 'test-results.json' file
+ tmpDir, err := ioutil.TempDir("", "fix-tests")
+ if err != nil {
+ return err
+ }
+ if err := os.MkdirAll(tmpDir, 0666); err != nil {
+ return fmt.Errorf("Failed to create temporary directory: %w", err)
+ }
+ defer os.RemoveAll(tmpDir)
+
+ // Full path to the 'test-results.json' in the temporary directory
+ testResultsPath := filepath.Join(tmpDir, "test-results.json")
+
+ // Run the tests
+ switch err := exec.Command(exe, "--gtest_output=json:"+testResultsPath).Run().(type) {
+ default:
+ return err
+ case nil:
+ fmt.Println("All tests passed")
+ case *exec.ExitError:
+ }
+
+ // Read the 'test-results.json' file
+ testResultsFile, err := os.Open(testResultsPath)
+ if err != nil {
+ return err
+ }
+
+ var testResults Results
+ if err := json.NewDecoder(testResultsFile).Decode(&testResults); err != nil {
+ return err
+ }
+
+ // For each failing test...
+ var errs []error
+ numFixes := 0
+ for _, group := range testResults.Groups {
+ for _, suite := range group.Testsuites {
+ for _, failure := range suite.Failures {
+ // .. attempt to fix the problem
+ test := group.Name + "." + suite.Name
+ if err := processFailure(test, wd, failure.Failure); err != nil {
+ errs = append(errs, fmt.Errorf("%v: %w", test, err))
+ } else {
+ numFixes++
+ }
+ }
+ }
+ }
+
+ if numFixes > 0 {
+ fmt.Printf("%v tests fixed\n", numFixes)
+ }
+ if n := len(errs); n > 0 {
+ fmt.Printf("%v tests could not be fixed:\n", n)
+ for _, err := range errs {
+ fmt.Println(err)
+ }
+ }
+ return nil
+}
+
+var (
+ // Regular expression to match a test declaration
+ reTests = regexp.MustCompile(`TEST(?:_[FP])?\((\w+),[ \n]*(\w+)\)`)
+ // Regular expression to match a EXPECT_EQ failure for strings
+ reExpectEq = regexp.MustCompile(`^([./\\a-z_-]*):(\d+).*\nExpected equality of these values:\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?)[^\\]"\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?)[^\\]"`)
+)
+
+func processFailure(test, wd, failure string) error {
+ // Start by un-escaping newlines in the failure message
+ failure = strings.ReplaceAll(failure, "\\n", "\n")
+
+ // Look for a EXPECT_EQ failure pattern
+ var file, a, b string
+ if parts := reExpectEq.FindStringSubmatch(failure); len(parts) == 5 {
+ file, a, b = parts[1], parts[3], parts[4]
+ } else {
+ return fmt.Errorf("Cannot fix this type of failure")
+ }
+
+ // Now un-escape any quotes (the regex is sensitive to these)
+ a = strings.ReplaceAll(a, `\"`, `"`)
+ b = strings.ReplaceAll(b, `\"`, `"`)
+
+ // Get the path to the source file containing the test failure
+ sourcePath := filepath.Join(wd, file)
+
+ // Parse the source file, split into tests
+ sourceFile, err := parseSourceFile(sourcePath)
+ if err != nil {
+ return fmt.Errorf("Couldn't parse tests from file '%v': %w", file, err)
+ }
+
+ // Find the test
+ testIdx, ok := sourceFile.tests[test]
+ if !ok {
+ return fmt.Errorf("Test '%v' not found in '%v'", test, file)
+ }
+
+ // Grab the source for the particular test
+ testSource := sourceFile.parts[testIdx]
+
+ // We don't know if a or b is the expected, so just try flipping the string
+ // to the other form.
+ switch {
+ case strings.Contains(testSource, a):
+ testSource = strings.Replace(testSource, a, b, -1)
+ case strings.Contains(testSource, b):
+ testSource = strings.Replace(testSource, b, a, -1)
+ default:
+ // Try escaping for R"(...)" strings
+ a = strings.ReplaceAll(a, "\n", `\n`)
+ b = strings.ReplaceAll(b, "\n", `\n`)
+ a = strings.ReplaceAll(a, "\"", `\"`)
+ b = strings.ReplaceAll(b, "\"", `\"`)
+ switch {
+ case strings.Contains(testSource, a):
+ testSource = strings.Replace(testSource, a, b, -1)
+ case strings.Contains(testSource, b):
+ testSource = strings.Replace(testSource, b, a, -1)
+ default:
+ return fmt.Errorf("Could not fix test '%v' in '%v'", test, file)
+ }
+ }
+
+ // Replace the part of the source file
+ sourceFile.parts[testIdx] = testSource
+
+ // Write out the source file
+ return writeSourceFile(sourcePath, sourceFile)
+}
+
+// parseSourceFile() reads the file at path, splitting the content into chunks
+// for each TEST.
+func parseSourceFile(path string) (sourceFile, error) {
+ fileBytes, err := ioutil.ReadFile(path)
+ if err != nil {
+ return sourceFile{}, err
+ }
+ fileContent := string(fileBytes)
+
+ out := sourceFile{
+ tests: map[string]int{},
+ }
+
+ pos := 0
+ for _, span := range reTests.FindAllStringIndex(fileContent, -1) {
+ out.parts = append(out.parts, fileContent[pos:span[0]])
+ pos = span[0]
+
+ match := reTests.FindStringSubmatch(fileContent[span[0]:span[1]])
+ group := match[1]
+ suite := match[2]
+ out.tests[group+"."+suite] = len(out.parts)
+ }
+ out.parts = append(out.parts, fileContent[pos:])
+
+ return out, nil
+}
+
+// writeSourceFile() joins the chunks of the file, and writes the content out to
+// path.
+func writeSourceFile(path string, file sourceFile) error {
+ body := strings.Join(file.parts, "")
+ return ioutil.WriteFile(path, []byte(body), 0666)
+}
+
+type sourceFile struct {
+ parts []string
+ tests map[string]int // "X.Y" -> part index
+}
+
+// Results is the root JSON structure of the JSON --gtest_output file .
+type Results struct {
+ Tests int `json:"tests"`
+ Failures int `json:"failures"`
+ Disabled int `json:"disabled"`
+ Errors int `json:"errors"`
+ Timestamp string `json:"timestamp"`
+ Time string `json:"time"`
+ Name string `json:"name"`
+ Groups []TestsuiteGroup `json:"testsuites"`
+}
+
+// TestsuiteGroup is a group of test suites in the JSON --gtest_output file .
+type TestsuiteGroup struct {
+ Name string `json:"name"`
+ Tests int `json:"tests"`
+ Failures int `json:"failures"`
+ Disabled int `json:"disabled"`
+ Errors int `json:"errors"`
+ Timestamp string `json:"timestamp"`
+ Time string `json:"time"`
+ Testsuites []Testsuite `json:"testsuite"`
+}
+
+// Testsuite is a suite of tests in the JSON --gtest_output file.
+type Testsuite struct {
+ Name string `json:"name"`
+ ValueParam string `json:"value_param,omitempty"`
+ Status Status `json:"status"`
+ Result Result `json:"result"`
+ Timestamp string `json:"timestamp"`
+ Time string `json:"time"`
+ Classname string `json:"classname"`
+ Failures []Failure `json:"failures,omitempty"`
+}
+
+// Failure is a reported test failure in the JSON --gtest_output file.
+type Failure struct {
+ Failure string `json:"failure"`
+ Type string `json:"type"`
+}
+
+// Status is a status code in the JSON --gtest_output file.
+type Status string
+
+// Result is a result code in the JSON --gtest_output file.
+type Result string
