Parallel compiling: arduino-builder will leverage your multi core pc

If -jobs X param is specified and X is greater than 0, arduino-builder won't spawn more than X processes. Fixes arduino#17

- Rebased from Federico Fissore branch arduino@0a3d888#diff-8f074e2b3fc911dbe8255891fc1a9b13R96
- Add HumanTagsLogger compatibility
- Remove an if which suppress some GCC error messages here: arduino@0a3d888#diff-8f074e2b3fc911dbe8255891fc1a9b13R96

Author: copercini

Diff for: src/arduino.cc/arduino-builder/main.go

@@ -37,6 +37,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"runtime"
 	"strings"
 	"syscall"
 
@@ -79,6 +80,7 @@ const FLAG_LOGGER_HUMANTAGS = "humantags"
 const FLAG_LOGGER_MACHINE = "machine"
 const FLAG_VERSION = "version"
 const FLAG_VID_PID = "vid-pid"
+const FLAG_JOBS = "jobs"
 
 type foldersFlag []string
 
@@ -136,6 +138,7 @@ var warningsLevelFlag *string
 var loggerFlag *string
 var versionFlag *bool
 var vidPidFlag *string
+var jobsFlag *int
 
 func init() {
 	compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch")
@@ -159,6 +162,7 @@ func init() {
 	loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'")
 	versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits")
 	vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt")
+	jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine")
 }
 
 func main() {
@@ -173,6 +177,12 @@ func main() {
 		return
 	}
 
+	if *jobsFlag > 0 {
+		runtime.GOMAXPROCS(*jobsFlag)
+	} else {
+		runtime.GOMAXPROCS(runtime.NumCPU())
+	}
+
 	ctx := &types.Context{}
 
 	if *buildOptionsFileFlag != "" {

Diff for: src/arduino.cc/builder/builder_utils/utils.go

@@ -31,15 +31,16 @@ package builder_utils
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/i18n"
+	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
 	"arduino.cc/properties"
 )
@@ -146,15 +147,47 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) {
 }
 
 func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
+	if len(sources) == 0 {
+		return objectFiles, nil
+	}
+	objectFilesChan := make(chan string)
+	errorsChan := make(chan error)
+	doneChan := make(chan struct{})
+
+	var wg sync.WaitGroup
+	wg.Add(len(sources))
+
 	for _, source := range sources {
-		objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
-		if err != nil {
+		go func(source string) {
+			defer wg.Done()
+			objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
+			if err != nil {
+				errorsChan <- err
+			} else {
+				objectFilesChan <- objectFile
+			}
+		}(source)
+	}
+
+	go func() {
+		wg.Wait()
+		doneChan <- struct{}{}
+	}()
+
+	for {
+		select {
+		case objectFile := <-objectFilesChan:
+			objectFiles = append(objectFiles, objectFile)
+		case err := <-errorsChan:
 			return nil, i18n.WrapError(err)
+		case <-doneChan:
+			close(objectFilesChan)
+			for objectFile := range objectFilesChan {
+				objectFiles = append(objectFiles, objectFile)
+			}
+			return objectFiles, nil
 		}
-
-		objectFiles = append(objectFiles, objectFile)
 	}
-	return objectFiles, nil
 }
 
 func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {
@@ -361,10 +394,20 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties
 	}
 
 	if echoOutput {
-		command.Stdout = os.Stdout
+		printToStdOut := func(data []byte) {
+			logger.UnformattedWrite(os.Stdout, data)
+		}
+		stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}}
+		defer stdout.Flush()
+		command.Stdout = stdout
 	}
 
-	command.Stderr = os.Stderr
+	printToStdErr := func(data []byte) {
+		logger.UnformattedWrite(os.Stderr, data)
+	}
+	stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}}
+	defer stderr.Flush()
+	command.Stderr = stderr
 
 	if echoOutput {
 		err := command.Run()
@@ -396,7 +439,7 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo
 	}
 
 	if echoCommandLine {
-		fmt.Println(commandLine)
+		logger.UnformattedFprintln(os.Stdout, commandLine)
 	}
 
 	return command, nil

Diff for: src/arduino.cc/builder/ctags_runner.go

@@ -30,7 +30,7 @@
 package builder
 
 import (
-	"fmt"
+	"os"
 
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/ctags"
@@ -63,7 +63,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error {
 
 	verbose := ctx.Verbose
 	if verbose {
-		fmt.Println(commandLine)
+		logger.UnformattedFprintln(os.Stdout, commandLine)
 	}
 
 	sourceBytes, err := command.Output()

Diff for: src/arduino.cc/builder/i18n/i18n.go

@@ -38,12 +38,15 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 )
 
 var PLACEHOLDER = regexp.MustCompile("{(\\d)}")
 
 type Logger interface {
 	Fprintln(w io.Writer, level string, format string, a ...interface{})
+	UnformattedFprintln(w io.Writer, s string)
+	UnformattedWrite(w io.Writer, data []byte)
 	Println(level string, format string, a ...interface{})
 	Name() string
 }
@@ -52,6 +55,10 @@ type NoopLogger struct{}
 
 func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {}
 
+func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {}
+
+func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {}
+
 func (s NoopLogger) Println(level string, format string, a ...interface{}) {}
 
 func (s NoopLogger) Name() string {
@@ -62,55 +69,98 @@ type HumanTagsLogger struct{}
 
 func (s HumanTagsLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
 	format = "[" + level + "] " + format
-	fmt.Fprintln(w, Format(format, a...))
+	fprintln(w, Format(format, a...))
 }
 
 func (s HumanTagsLogger) Println(level string, format string, a ...interface{}) {
 	s.Fprintln(os.Stdout, level, format, a...)
 }
 
+func (s HumanTagsLogger) UnformattedFprintln(w io.Writer, str string) {
+	fprintln(w, str)
+}
+
+func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) {
+	write(w, data)
+}
+
 func (s HumanTagsLogger) Name() string {
 	return "humantags"
 }
 
 type HumanLogger struct{}
 
 func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
-	fmt.Fprintln(w, Format(format, a...))
+	fprintln(w, Format(format, a...))
 }
 
 func (s HumanLogger) Println(level string, format string, a ...interface{}) {
 	s.Fprintln(os.Stdout, level, format, a...)
 }
 
+func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) {
+	fprintln(w, str)
+}
+
+func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) {
+	write(w, data)
+}
+
 func (s HumanLogger) Name() string {
 	return "human"
 }
 
 type MachineLogger struct{}
 
-func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) {
+func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
+	printMachineFormattedLogLine(w, level, format, a)
+}
+
+func (s MachineLogger) Println(level string, format string, a ...interface{}) {
+	printMachineFormattedLogLine(os.Stdout, level, format, a)
+}
+
+func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) {
+	fprintln(w, str)
+}
+
+func (s MachineLogger) Name() string {
+	return "machine"
+}
+
+func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) {
+	write(w, data)
+}
+
+func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) {
 	a = append([]interface{}(nil), a...)
 	for idx, value := range a {
 		typeof := reflect.Indirect(reflect.ValueOf(value)).Kind()
 		if typeof == reflect.String {
 			a[idx] = url.QueryEscape(value.(string))
 		}
 	}
-	fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a)
-	fmt.Fprintln(w)
+	fprintf(w, "===%s ||| %s ||| %s\n", level, format, a)
 }
 
-func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
-	s.printWithoutFormatting(w, level, format, a)
+var lock sync.Mutex
+
+func fprintln(w io.Writer, s string) {
+	lock.Lock()
+	defer lock.Unlock()
+	fmt.Fprintln(w, s)
 }
 
-func (s MachineLogger) Println(level string, format string, a ...interface{}) {
-	s.printWithoutFormatting(os.Stdout, level, format, a)
+func write(w io.Writer, data []byte) {
+	lock.Lock()
+	defer lock.Unlock()
+	w.Write(data)
 }
 
-func (s MachineLogger) Name() string {
-	return "machine"
+func fprintf(w io.Writer, format string, a ...interface{}) {
+	lock.Lock()
+	defer lock.Unlock()
+	fmt.Fprintf(w, format, a...)
 }
 
 func FromJavaToGoSyntax(s string) string {

Diff for: src/arduino.cc/builder/types/accessories.go

@@ -29,6 +29,11 @@
 
 package types
 
+import (
+	"bytes"
+	"sync"
+)
+
 type UniqueStringQueue []string
 
 func (queue UniqueStringQueue) Len() int           { return len(queue) }
@@ -74,3 +79,32 @@ func (queue *UniqueSourceFileQueue) Pop() SourceFile {
 func (queue *UniqueSourceFileQueue) Empty() bool {
 	return queue.Len() == 0
 }
+
+type BufferedUntilNewLineWriter struct {
+	PrintFunc PrintFunc
+	Buffer    bytes.Buffer
+	lock      sync.Mutex
+}
+
+type PrintFunc func([]byte)
+
+func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	writtenToBuffer, err := w.Buffer.Write(p)
+	return writtenToBuffer, err
+}
+
+func (w *BufferedUntilNewLineWriter) Flush() {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	remainingBytes := w.Buffer.Bytes()
+	if len(remainingBytes) > 0 {
+		if remainingBytes[len(remainingBytes)-1] != '\n' {
+			remainingBytes = append(remainingBytes, '\n')
+		}
+		w.PrintFunc(remainingBytes)
+	}
+}