Revert "[tools] Improvements to 'fuzz'"
This reverts commit 340b2d93d7364766e9287916935277752080629b.
Reason for revert: broke fuzzer check due to DXC failures
Original change's description:
> [tools] Improvements to 'fuzz'
>
> Add support for running multiple fuzzer targets (only 'wgsl' right now).
> Copy filtered files out of tint/test to the corpus directory, so that we don't fuzz uninteresting files.
> Pass --dxc flag to fuzzers
> Add a flag for ignoring non-security crashes.
>
> Change-Id: I1a49c61c642118880da71ea2eaa42b1734ec44df
> Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/190922
> Reviewed-by: Ryan Harrison <rharrison@chromium.org>
> Commit-Queue: Ryan Harrison <rharrison@chromium.org>
> Auto-Submit: Ben Clayton <bclayton@google.com>
TBR=rharrison@chromium.org,bclayton@google.com,dawn-scoped@luci-project-accounts.iam.gserviceaccount.com
Change-Id: Iefa757426e4f27f33ce4a232867b5924917bd99d
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/190941
Reviewed-by: James Price <jrprice@google.com>
Reviewed-by: Ryan Harrison <rharrison@chromium.org>
diff --git a/tools/src/cmd/fuzz/main.go b/tools/src/cmd/fuzz/main.go
index 2fd4736..7bddaee 100644
--- a/tools/src/cmd/fuzz/main.go
+++ b/tools/src/cmd/fuzz/main.go
@@ -38,7 +38,6 @@
"os"
"os/exec"
"path/filepath"
- "regexp"
"runtime"
"strings"
"sync/atomic"
@@ -51,6 +50,10 @@
"dawn.googlesource.com/dawn/tools/src/utils"
)
+const (
+ wgslDictionaryRelPath = "src/tint/cmd/fuzz/wgsl/dictionary.txt"
+)
+
func main() {
if err := run(); err != nil {
fmt.Println(err)
@@ -58,64 +61,34 @@
}
}
-type fuzzerInfo struct {
- name string // Short name of the fuzzer
- path string // Path to the fuzzer executable
- ext string // File extensions used by the fuzzer
- dict string // Optional path to a dictionary file for the fuzzer
-}
-
-func run() error {
- t := tool{}
-
- allFuzzers := []fuzzerInfo{
- {
- name: "wgsl",
- path: "tint_wgsl_fuzzer",
- ext: ".wgsl",
- dict: "src/tint/cmd/fuzz/wgsl/dictionary.txt",
- },
- }
- fuzzerByName := map[string]fuzzerInfo{}
- for _, fuzzer := range allFuzzers {
- fuzzerByName[fuzzer.name] = fuzzer
- }
- allFuzzerNames := transform.SliceNoErr(allFuzzers, func(f fuzzerInfo) string { return f.name })
-
- check := false
- build := ""
- flag.BoolVar(&t.verbose, "verbose", false, "print additional output")
- flag.BoolVar(&check, "check", false, "check that all the end-to-end test do not fail")
- flag.BoolVar(&t.securityOnly, "security-only", false, "ignore issues that are not considered security impacting")
- flag.StringVar(&t.filter, "filter", "", "filter the fuzzers run to those with this substring")
- flag.StringVar(&t.corpus, "corpus", defaultCorpusDir(), "the corpus directory")
- flag.StringVar(&build, "build", defaultBuildDir(), "the build directory")
- flag.StringVar(&t.out, "out", "<tmp>", "the directory to hold generated test files")
- flag.IntVar(&t.numProcesses, "j", runtime.NumCPU(), "number of concurrent fuzzers to run")
- flag.Usage = func() {
- fmt.Printf(`
+func showUsage() {
+ fmt.Println(`
fuzz is a helper for running the tint fuzzer executables
fuzz can check that the corpus does not trigger any issues with the fuzzers, and
simplify running of the fuzzers locally.
usage:
- fuzz [fuzzers] [flags...]
+ fuzz [flags...]`)
+ flag.PrintDefaults()
+ fmt.Println(``)
+ os.Exit(1)
+}
-fuzzers are the fuzzer types to run, defaults to all.
- Possible values: ` + strings.Join(allFuzzerNames, ", ") + `
-`)
- flag.PrintDefaults()
- fmt.Println(``)
- os.Exit(1)
- }
+func run() error {
+ t := tool{}
+
+ check := false
+ build := ""
+ flag.BoolVar(&t.verbose, "verbose", false, "print additional output")
+ flag.BoolVar(&check, "check", false, "check that all the end-to-end test do not fail")
+ flag.StringVar(&t.filter, "filter", "", "filter the fuzzers run to those with this substring")
+ flag.StringVar(&t.corpus, "corpus", defaultCorpusDir(), "the corpus directory")
+ flag.StringVar(&build, "build", defaultBuildDir(), "the build directory")
+ flag.StringVar(&t.out, "out", "<tmp>", "the directory to hold generated test files")
+ flag.IntVar(&t.numProcesses, "j", runtime.NumCPU(), "number of concurrent fuzzers to run")
flag.Parse()
- selectedFuzzers := flag.Args()
- if len(selectedFuzzers) == 0 {
- selectedFuzzers = allFuzzerNames
- }
-
if t.numProcesses < 1 {
t.numProcesses = 1
}
@@ -137,31 +110,17 @@
return fmt.Errorf("output directory '%v' does not exist", t.out)
}
- // Register all the fuzzers
- for _, name := range selectedFuzzers {
- fuzzer, ok := fuzzerByName[name]
- if !ok {
- return fmt.Errorf("unknown fuzzer '%v'. Possible values: %v", name, strings.Join(allFuzzerNames, ", "))
+ // Verify all of the fuzzer executables are present
+ for _, fuzzer := range []struct {
+ name string
+ path *string
+ }{
+ {"tint_wgsl_fuzzer", &t.wgslFuzzer},
+ } {
+ *fuzzer.path = filepath.Join(build, fuzzer.name)
+ if !fileutils.IsExe(*fuzzer.path) {
+ return fmt.Errorf("fuzzer not found at '%v'", *fuzzer.path)
}
-
- fuzzer.path = filepath.Join(build, fuzzer.path)
- if !fileutils.IsExe(fuzzer.path) {
- return fmt.Errorf("fuzzer not found at '%v'", fuzzer.path)
- }
-
- if fuzzer.dict != "" {
- dictPath, err := filepath.Abs(filepath.Join(fileutils.DawnRoot(), fuzzer.dict))
- if err != nil || !fileutils.IsFile(dictPath) {
- return fmt.Errorf("failed to obtain the dictionary.txt path: %w", err)
- }
- fuzzer.dict = dictPath
- }
-
- t.fuzzers = append(t.fuzzers, fuzzer)
- }
-
- if dxc := filepath.Join(build, dxcFileName()); fileutils.IsFile(dxc) {
- t.dxc = dxc
}
// If --check was passed, then just ensure that all the files in the corpus
@@ -176,39 +135,27 @@
type tool struct {
verbose bool
- filter string // filter fuzzers to those with this substring
- corpus string // directory of test files
- out string // where to emit new test files
- dxc string // path to the DXC DLL / so
- fuzzers []fuzzerInfo // the fuzzers to run
- numProcesses int // number of concurrent processes to spawn
- securityOnly bool // Ignore non-security crashes
+ filter string // filter fuzzers to those with this substring
+ corpus string // directory of test files
+ out string // where to emit new test files
+ wgslFuzzer string // path to tint_wgsl_fuzzer
+ numProcesses int // number of concurrent processes to spawn
}
// check() runs the fuzzers against all the .wgsl files under to the corpus directory,
// ensuring that the fuzzers do not error for the given file.
func (t tool) check() error {
- type job struct {
- file string
- exe string
+ wgslFiles, err := glob.Glob(filepath.Join(t.corpus, "**.wgsl"))
+ if err != nil {
+ return err
}
- jobs := []job{}
+ // Remove '*.expected.wgsl'
+ wgslFiles = transform.Filter(wgslFiles, func(s string) bool { return !strings.Contains(s, ".expected.") })
- for _, fuzzer := range t.fuzzers {
- files, err := t.fuzzerCorpusFiles(fuzzer)
- if err != nil {
- return err
- }
+ log.Printf("checking %v files...\n", len(wgslFiles))
- log.Printf("%v: checking %v files...\n", fuzzer.name, len(files))
-
- for _, file := range files {
- jobs = append(jobs, job{file: file, exe: fuzzer.path})
- }
- }
-
- remaining := transform.SliceToChan(jobs)
+ remaining := transform.SliceToChan(wgslFiles)
var pb *progressbar.ProgressBar
if term.CanUseAnsiEscapeSequences() {
@@ -218,32 +165,26 @@
var numDone uint32
routine := func() error {
- for job := range remaining {
+ for file := range remaining {
atomic.AddUint32(&numDone, 1)
if pb != nil {
pb.Update(progressbar.Status{
- Total: len(jobs),
+ Total: len(wgslFiles),
Segments: []progressbar.Segment{
{Count: int(atomic.LoadUint32(&numDone))},
},
})
}
- args := []string{}
- if t.dxc != "" {
- args = append(args, "--dxc="+t.dxc)
- }
- args = append(args, job.file)
-
- if out, err := exec.Command(job.exe, args...).CombinedOutput(); err != nil {
- _, fuzzer := filepath.Split(job.exe)
- return fmt.Errorf("%v run with %v failed with %v\n\n%v", fuzzer, job, err, string(out))
+ if out, err := exec.Command(t.wgslFuzzer, file).CombinedOutput(); err != nil {
+ _, fuzzer := filepath.Split(t.wgslFuzzer)
+ return fmt.Errorf("%v run with %v failed with %v\n\n%v", fuzzer, file, err, string(out))
}
}
return nil
}
- if err := utils.RunConcurrent(t.numProcesses, routine); err != nil {
+ if err = utils.RunConcurrent(t.numProcesses, routine); err != nil {
return err
}
@@ -256,105 +197,55 @@
// New cases are written to t.out.
// Blocks until a fuzzer errors, or the process is interrupted.
func (t tool) run() error {
- // Regular expression used to identify the crash file written by libfuzzer
- var reCrashFile = regexp.MustCompile("crash-[a-z0-9]{40}")
-
ctx := utils.CancelOnInterruptContext(context.Background())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
- routinesPerFuzzer := t.numProcesses / len(t.fuzzers)
- if routinesPerFuzzer == 0 {
- routinesPerFuzzer = 1
+ dictPath, err := filepath.Abs(filepath.Join(fileutils.DawnRoot(), wgslDictionaryRelPath))
+ if err != nil || !fileutils.IsFile(dictPath) {
+ return fmt.Errorf("failed to obtain the dictionary.txt path: %w", err)
}
- errs := make(chan error, 8)
+ args := []string{t.out, t.corpus,
+ "-dict=" + dictPath,
+ }
+ if t.verbose {
+ args = append(args, "--verbose")
+ }
+ if t.filter != "" {
+ args = append(args, "--filter="+t.filter)
+ }
- for _, fuzzer := range t.fuzzers {
- fuzzer := fuzzer
-
- corpusFiles, err := t.fuzzerCorpusFiles(fuzzer)
- if err != nil {
- return err
- }
-
- log.Println("copying", len(corpusFiles), fuzzer.ext, "files to", t.out+"...")
- for _, path := range corpusFiles {
- _, file := filepath.Split(path)
- if err := fileutils.CopyFile(filepath.Join(t.out, file), path); err != nil {
- return err
+ fmt.Println("running", t.numProcesses, "fuzzer instances")
+ errs := make(chan error, t.numProcesses)
+ for i := 0; i < t.numProcesses; i++ {
+ go func() {
+ cmd := exec.CommandContext(ctx, t.wgslFuzzer, args...)
+ out := bytes.Buffer{}
+ cmd.Stdout = &out
+ cmd.Stderr = &out
+ if t.verbose {
+ cmd.Stdout = io.MultiWriter(&out, os.Stdout)
+ cmd.Stderr = io.MultiWriter(&out, os.Stderr)
}
- }
-
- args := []string{t.out}
- if fuzzer.dict != "" {
- args = append(args, "-dict="+fuzzer.dict)
- }
- if t.verbose {
- args = append(args, "--verbose")
- }
- if t.filter != "" {
- args = append(args, "--filter="+t.filter)
- }
- if t.dxc != "" {
- args = append(args, "--dxc="+t.dxc)
- }
-
- log.Println("running", routinesPerFuzzer, fuzzer.name, "fuzzer instances...")
- for i := 0; i < routinesPerFuzzer; i++ {
- go func() {
- for {
- cmd := exec.CommandContext(ctx, fuzzer.path, args...)
- out := bytes.Buffer{}
- cmd.Stdout = &out
- cmd.Stderr = &out
- if t.verbose {
- cmd.Stdout = io.MultiWriter(&out, os.Stdout)
- cmd.Stderr = io.MultiWriter(&out, os.Stderr)
- }
- if err := cmd.Run(); err != nil {
- if ctxErr := ctx.Err(); ctxErr != nil {
- errs <- ctxErr
- } else {
- if t.securityOnly && isFailureNonSecurity(out.String()) {
- log.Println("non-security crash found. restarting...")
- if file := reCrashFile.FindString(out.String()); file != "" {
- os.Remove(file)
- }
- continue
- }
- _, fuzzer := filepath.Split(fuzzer.ext)
- errs <- fmt.Errorf("%v failed with %v\n\n%v", fuzzer, err, out.String())
- }
- } else {
- errs <- fmt.Errorf("fuzzer unexpectedly terminated without error:\n%v", out.String())
- }
- break
+ if err := cmd.Run(); err != nil {
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ errs <- ctxErr
+ } else {
+ _, fuzzer := filepath.Split(t.wgslFuzzer)
+ errs <- fmt.Errorf("%v failed with %v\n\n%v", fuzzer, err, out.String())
}
- }()
- }
+ } else {
+ errs <- fmt.Errorf("fuzzer unexpectedly terminated without error:\n%v", out.String())
+ }
+ }()
}
-
for err := range errs {
return err
}
return nil
}
-func (t tool) fuzzerCorpusFiles(f fuzzerInfo) ([]string, error) {
- files, err := glob.Glob(filepath.Join(t.corpus, "**"+f.ext))
- if err != nil {
- return nil, err
- }
-
- // Remove '*.expected.wgsl'
- if f.name == "wgsl" {
- files = transform.Filter(files, func(s string) bool { return !strings.Contains(s, ".expected.wgsl") })
- }
-
- return files, nil
-}
-
func defaultCorpusDir() string {
return filepath.Join(fileutils.DawnRoot(), "test/tint")
}
@@ -362,26 +253,3 @@
func defaultBuildDir() string {
return filepath.Join(fileutils.DawnRoot(), "out/active")
}
-
-func dxcFileName() string {
- switch runtime.GOOS {
- case "windows":
- return "dxcompiler.dll"
- case "darwin":
- return "libdxcompiler.dylib"
- default:
- return "libdxcompiler.so"
- }
-}
-
-func isFailureNonSecurity(out string) bool {
- for _, str := range []string{
- "AddressSanitizer: SEGV on unknown address 0x000000000000",
- "ICE while running fuzzer",
- } {
- if strings.Contains(out, str) {
- return true
- }
- }
- return false
-}
