Parallel compiling: arduino-builder will leverage your multi core pc

If -jobs X param is specified and X is greater than 0, arduino-builder won't spawn more than X processes. Fixes arduino#17

Rebased from Federico Fissore branch arduino@0a3d888#diff-8f074e2b3fc911dbe8255891fc1a9b13R96
Remove and if which suppress some GCC error messages here: arduino@0a3d888#diff-8f074e2b3fc911dbe8255891fc1a9b13R96

Author: copercini

Diff for: src/arduino.cc/builder/builder_utils/utils.go

@@ -31,16 +31,17 @@ package builder_utils
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"	
 
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/utils"
+	"arduino.cc/builder/types"
 	"arduino.cc/properties"
 )
 
@@ -146,16 +147,49 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) {
 }
 
 func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
+	if len(sources) == 0 {
+		return objectFiles, nil
+	}
+	objectFilesChan := make(chan string)
+	errorsChan := make(chan error)
+	doneChan := make(chan struct{})
+
+	var wg sync.WaitGroup
+	wg.Add(len(sources))
+
 	for _, source := range sources {
-		objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
-		if err != nil {
-			return nil, i18n.WrapError(err)
-		}
+		go func(source string) {
+			defer wg.Done()
+			objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
+			if err != nil {
+				errorsChan <- err
+			} else {
+				objectFilesChan <- objectFile
+			}
+		}(source)
+	}
+
+	go func() {
+		wg.Wait()
+		doneChan <- struct{}{}
+	}()
 
-		objectFiles = append(objectFiles, objectFile)
+	for {
+		select {
+		case objectFile := <-objectFilesChan:
+			objectFiles = append(objectFiles, objectFile)
+		case err := <-errorsChan:
+ 			return nil, i18n.WrapError(err)
+		case <-doneChan:
+			close(objectFilesChan)
+			for objectFile := range objectFilesChan {
+				objectFiles = append(objectFiles, objectFile)
+			}
+			return objectFiles, nil
+ 		}
 	}
-	return objectFiles, nil
 }
+  
 
 func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {
 	properties := buildProperties.Clone()
@@ -361,11 +395,22 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties
 	}
 
 	if echoOutput {
-		command.Stdout = os.Stdout
+		printToStdOut := func(data []byte) {
+			logger.UnformattedWrite(os.Stdout, data)
+		}
+		stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}}
+		defer stdout.Flush()
+		command.Stdout = stdout
 	}
 
-	command.Stderr = os.Stderr
-
+	printToStdErr := func(data []byte) {
+		logger.UnformattedWrite(os.Stderr, data)
+	}
+	stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}}
+	defer stderr.Flush()
+	command.Stderr = stderr
+	
+	
 	if echoOutput {
 		err := command.Run()
 		return nil, i18n.WrapError(err)
@@ -396,7 +441,7 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo
 	}
 
 	if echoCommandLine {
-		fmt.Println(commandLine)
+		logger.UnformattedFprintln(os.Stdout, commandLine)
 	}
 
 	return command, nil

Diff for: src/arduino.cc/builder/ctags_runner.go

@@ -30,7 +30,7 @@
 package builder
 
 import (
-	"fmt"
+	"os"
 
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/ctags"
@@ -63,7 +63,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error {
 
 	verbose := ctx.Verbose
 	if verbose {
-		fmt.Println(commandLine)
+		logger.UnformattedFprintln(os.Stdout, commandLine)
 	}
 
 	sourceBytes, err := command.Output()

Diff for: src/arduino.cc/builder/i18n/i18n.go

@@ -38,12 +38,15 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 )
 
 var PLACEHOLDER = regexp.MustCompile("{(\\d)}")
 
 type Logger interface {
 	Fprintln(w io.Writer, level string, format string, a ...interface{})
+	UnformattedFprintln(w io.Writer, s string)
+	UnformattedWrite(w io.Writer, data []byte)	
 	Println(level string, format string, a ...interface{})
 	Name() string
 }
@@ -52,6 +55,10 @@ type NoopLogger struct{}
 
 func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {}
 
+func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {}
+
+func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {}
+
 func (s NoopLogger) Println(level string, format string, a ...interface{}) {}
 
 func (s NoopLogger) Name() string {
@@ -62,55 +69,101 @@ type HumanTagsLogger struct{}
 
 func (s HumanTagsLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
 	format = "[" + level + "] " + format
-	fmt.Fprintln(w, Format(format, a...))
+	fprintln(w, Format(format, a...)) 
 }
 
 func (s HumanTagsLogger) Println(level string, format string, a ...interface{}) {
 	s.Fprintln(os.Stdout, level, format, a...)
 }
 
+func (s HumanTagsLogger) UnformattedFprintln(w io.Writer, str string) {
+	fprintln(w, str)
+}
+
+func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) {
+	write(w, data)
+}
+	
 func (s HumanTagsLogger) Name() string {
 	return "humantags"
 }
 
+
 type HumanLogger struct{}
 
 func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
-	fmt.Fprintln(w, Format(format, a...))
+	fprintln(w, Format(format, a...))
 }
 
 func (s HumanLogger) Println(level string, format string, a ...interface{}) {
 	s.Fprintln(os.Stdout, level, format, a...)
 }
 
+func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) {
+	fprintln(w, str)
+}
+
+func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) {
+	write(w, data)
+}
+
 func (s HumanLogger) Name() string {
 	return "human"
 }
 
+
 type MachineLogger struct{}
 
-func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) {
+func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
+	printMachineFormattedLogLine(w, level, format, a)
+}
+
+func (s MachineLogger) Println(level string, format string, a ...interface{}) {
+	printMachineFormattedLogLine(os.Stdout, level, format, a)
+}
+
+func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) {
+	fprintln(w, str)
+}
+
+func (s MachineLogger) Name() string {
+	return "machine"
+}
+
+func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) {
+	write(w, data)
+}
+
+
+func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) {
 	a = append([]interface{}(nil), a...)
 	for idx, value := range a {
 		typeof := reflect.Indirect(reflect.ValueOf(value)).Kind()
 		if typeof == reflect.String {
 			a[idx] = url.QueryEscape(value.(string))
 		}
 	}
-	fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a)
-	fmt.Fprintln(w)
+	fprintf(w, "===%s ||| %s ||| %s\n", level, format, a)
 }
 
-func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {
-	s.printWithoutFormatting(w, level, format, a)
+var lock sync.Mutex
+
+func fprintln(w io.Writer, s string) {
+	lock.Lock()
+	defer lock.Unlock()
+	fmt.Fprintln(w, s)
 }
 
-func (s MachineLogger) Println(level string, format string, a ...interface{}) {
-	s.printWithoutFormatting(os.Stdout, level, format, a)
+func write(w io.Writer, data []byte) {
+	lock.Lock()
+	defer lock.Unlock()
+	w.Write(data)
 }
 
-func (s MachineLogger) Name() string {
-	return "machine"
+func fprintf(w io.Writer, format string, a ...interface{}) {
+	lock.Lock()
+	defer lock.Unlock()
+	fmt.Fprintf(w, format, a...)
 }
 
 func FromJavaToGoSyntax(s string) string {