Parallel compiling: arduino-builder will leverage your multi core pc. If

-jobs X param is specified and X is greater than 0, arduino-builder won't spawn
more than X processes. Fixes #17

Signed-off-by: Federico Fissore <[email protected]>

Author: Federico Fissore

Diff for: main.go

@@ -43,6 +43,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"runtime"
 	"strings"
 	"syscall"
 )
@@ -74,6 +75,7 @@ const FLAG_LOGGER_HUMAN = "human"
 const FLAG_LOGGER_MACHINE = "machine"
 const FLAG_VERSION = "version"
 const FLAG_VID_PID = "vid-pid"
+const FLAG_JOBS = "jobs"
 
 type slice []string
 
@@ -116,6 +118,7 @@ var warningsLevelFlag *string
 var loggerFlag *string
 var versionFlag *bool
 var vidPidFlag *string
+var jobsFlag *int
 
 func init() {
 	compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch")
@@ -137,6 +140,7 @@ func init() {
 	loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_MACHINE+"'")
 	versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits")
 	vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt")
+	jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine")
 }
 
 func main() {
@@ -152,6 +156,12 @@ func main() {
 		return
 	}
 
+	if *jobsFlag > 0 {
+		runtime.GOMAXPROCS(*jobsFlag)
+	} else {
+		runtime.GOMAXPROCS(runtime.NumCPU())
+	}
+
 	context := make(map[string]interface{})
 
 	buildOptions := make(map[string]string)

Diff for: src/arduino.cc/builder/builder_utils/utils.go

@@ -40,6 +40,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 )
 
 func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties map[string]string, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
@@ -116,15 +117,47 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str
 }
 
 func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties map[string]string, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) {
+	if len(sources) == 0 {
+		return objectFiles, nil
+	}
+	objectFilesChan := make(chan string)
+	errorsChan := make(chan error)
+	doneChan := make(chan struct{})
+
+	var wg sync.WaitGroup
+	wg.Add(len(sources))
+
 	for _, source := range sources {
-		objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
-		if err != nil {
+		go func(source string) {
+			defer wg.Done()
+			objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger)
+			if err != nil {
+				errorsChan <- err
+			} else {
+				objectFilesChan <- objectFile
+			}
+		}(source)
+	}
+
+	go func() {
+		wg.Wait()
+		doneChan <- struct{}{}
+	}()
+
+	for {
+		select {
+		case objectFile := <-objectFilesChan:
+			objectFiles = append(objectFiles, objectFile)
+		case err := <-errorsChan:
 			return nil, utils.WrapError(err)
+		case <-doneChan:
+			close(objectFilesChan)
+			for objectFile := range objectFilesChan {
+				objectFiles = append(objectFiles, objectFile)
+			}
+			return objectFiles, nil
 		}
-
-		objectFiles = append(objectFiles, objectFile)
 	}
-	return objectFiles, nil
 }
 
 func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties map[string]string, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) {

Diff for: src/arduino.cc/builder/test/builder_test.go

@@ -36,11 +36,13 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"testing"
 )
 
 func TestBuilderEmptySketch(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -75,6 +77,7 @@ func TestBuilderEmptySketch(t *testing.T) {
 
 func TestBuilderBridge(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -110,6 +113,7 @@ func TestBuilderBridge(t *testing.T) {
 
 func TestBuilderSketchWithConfig(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -144,6 +148,7 @@ func TestBuilderSketchWithConfig(t *testing.T) {
 
 func TestBuilderBridgeTwice(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -182,6 +187,7 @@ func TestBuilderBridgeTwice(t *testing.T) {
 
 func TestBuilderBridgeSAM(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -226,6 +232,7 @@ func TestBuilderBridgeSAM(t *testing.T) {
 
 func TestBuilderBridgeRedBearLab(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -260,6 +267,7 @@ func TestBuilderBridgeRedBearLab(t *testing.T) {
 
 func TestBuilderSketchNoFunctions(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -281,6 +289,7 @@ func TestBuilderSketchNoFunctions(t *testing.T) {
 
 func TestBuilderSketchWithBackup(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -302,6 +311,7 @@ func TestBuilderSketchWithBackup(t *testing.T) {
 
 func TestBuilderSketchWithOldLib(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -323,6 +333,7 @@ func TestBuilderSketchWithOldLib(t *testing.T) {
 
 func TestBuilderSketchWithSubfolders(t *testing.T) {
 	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	context := make(map[string]interface{})
 
@@ -341,3 +352,26 @@ func TestBuilderSketchWithSubfolders(t *testing.T) {
 	err := command.Run(context)
 	NoError(t, err)
 }
+
+func TestBuilderSketchWithSyntaxError(t *testing.T) {
+	DownloadCoresAndToolsAndLibraries(t)
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	context := make(map[string]interface{})
+
+	buildPath := SetupBuildPath(t, context)
+	defer os.RemoveAll(buildPath)
+
+	context[constants.CTX_HARDWARE_FOLDERS] = []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}
+	context[constants.CTX_TOOLS_FOLDERS] = []string{"downloaded_tools"}
+	context[constants.CTX_FQBN] = "arduino:avr:uno"
+	context[constants.CTX_SKETCH_LOCATION] = filepath.Join("sketch_with_syntax_error", "sketch.ino")
+	context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"}
+	context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"}
+	context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
+	context[constants.CTX_VERBOSE] = true
+
+	command := builder.Builder{}
+	err := command.Run(context)
+	require.Error(t, err)
+}

Diff for: src/arduino.cc/builder/test/sketch_with_syntax_error/sketch.ino

@@ -0,0 +1,3 @@
+void setup() {
+
+void loop() {}