Parallelize cpplint
On my machine this reduces the time taken from 23 seconds -> 2 seconds
Change-Id: I676b89251fc183171cc3d955873960b00cb48bc1
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/44164
Reviewed-by: David Neto <dneto@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
diff --git a/tools/lint b/tools/lint
index 0157fa6..1224c0e 100755
--- a/tools/lint
+++ b/tools/lint
@@ -13,8 +13,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
+
set -e # fail on error
FILTER="-runtime/references"
-./third_party/cpplint/cpplint/cpplint.py --root=. --filter="$FILTER" `find src -type f`
-./third_party/cpplint/cpplint/cpplint.py --root=. --filter="$FILTER" `find samples -type f`
+
+FILES="`find src -type f` `find samples -type f`"
+
+if command -v go &> /dev/null
+then
+ # Go is installed. Run cpplint in parallel for speed wins
+ go run $SCRIPT_DIR/run-parallel/main.go \
+ --only-print-failures \
+ ./third_party/cpplint/cpplint/cpplint.py \
+ --root=$ROOT_DIR \
+ --filter="$FILTER" \
+ $ -- $FILES
+else
+ ./third_party/cpplint/cpplint/cpplint.py --root=$ROOT_DIR --filter="$FILTER" $FILES
+fi
+
diff --git a/tools/run-parallel/main.go b/tools/run-parallel/main.go
new file mode 100644
index 0000000..4fd3e48
--- /dev/null
+++ b/tools/run-parallel/main.go
@@ -0,0 +1,122 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// run-parallel is a tool to run an executable with the provided templated
+// arguments across all the hardware threads.
+package main
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "os/exec"
+ "runtime"
+ "strings"
+ "sync"
+)
+
+func main() {
+ if err := run(); err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+}
+
+func showUsage() {
+ fmt.Println(`
+run-parallel is a tool to run an executable with the provided templated
+arguments across all the hardware threads.
+
+Usage:
+ run-parallel <executable> [arguments...] -- [per-instance-value...]
+
+ executable - the path to the executable to run.
+ arguments - a list of arguments to pass to the executable.
+ Any occurrance of $ will be substituted with the
+ per-instance-value for the given invocation.
+ per-instance-value - a list of values. The executable will be invoked for each
+ value in this list.`)
+ os.Exit(1)
+}
+
+func run() error {
+ onlyPrintFailures := flag.Bool("only-print-failures", false, "Omit output for processes that did not fail")
+ flag.Parse()
+
+ args := flag.Args()
+ if len(args) < 2 {
+ showUsage()
+ }
+ exe := args[0]
+ args = args[1:]
+
+ var perInstanceValues []string
+ for i, arg := range args {
+ if arg == "--" {
+ perInstanceValues = args[i+1:]
+ args = args[:i]
+ break
+ }
+ }
+ if perInstanceValues == nil {
+ showUsage()
+ }
+
+ taskIndices := make(chan int, 64)
+ results := make([]string, len(perInstanceValues))
+
+ numCPU := runtime.NumCPU()
+ wg := sync.WaitGroup{}
+ wg.Add(numCPU)
+ for i := 0; i < numCPU; i++ {
+ go func() {
+ defer wg.Done()
+ for idx := range taskIndices {
+ taskArgs := make([]string, len(args))
+ for i, arg := range args {
+ taskArgs[i] = strings.ReplaceAll(arg, "$", perInstanceValues[idx])
+ }
+ success, out := invoke(exe, taskArgs)
+ if !success || !*onlyPrintFailures {
+ results[idx] = out
+ }
+ }
+ }()
+ }
+
+ for i := range perInstanceValues {
+ taskIndices <- i
+ }
+ close(taskIndices)
+
+ wg.Wait()
+
+ for _, output := range results {
+ if output != "" {
+ fmt.Println(output)
+ }
+ }
+
+ return nil
+}
+
+func invoke(exe string, args []string) (ok bool, output string) {
+ cmd := exec.Command(exe, args...)
+ out, err := cmd.CombinedOutput()
+ str := string(out)
+ if err != nil {
+ return false, "\n" + err.Error()
+ }
+ return true, str
+}
