Fix progressbar

Port of arduino/arduino-builder#289

Author: facchinm

Diff for: legacy/builder/builder.go

@@ -143,7 +143,7 @@ func (s *PreprocessSketch) Run(ctx *types.Context) error {
 	} else {
 		commands = append(commands, &ContainerAddPrototypes{})
 	}
-	return runCommands(ctx, commands, true)
+	return runCommands(ctx, commands, false)
 }
 
 type Preprocess struct{}
@@ -201,12 +201,12 @@ func (s *ParseHardwareAndDumpBuildProperties) Run(ctx *types.Context) error {
 func runCommands(ctx *types.Context, commands []types.Command, progressEnabled bool) error {
 
 	ctx.Progress.PrintEnabled = progressEnabled
-	ctx.Progress.Progress = 0
 
 	for _, command := range commands {
 		PrintRingNameIfDebug(ctx, command)
-		ctx.Progress.Steps = 100.0 / float64(len(commands))
-		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx)
+		stepSize := float64(100-(6*len(ctx.ImportedLibraries))) / float64(len(commands))
+		ctx.Progress.Steps = stepSize
+		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx, progressEnabled, stepSize)
 		err := command.Run(ctx)
 		if err != nil {
 			return i18n.WrapError(err)

Diff for: legacy/builder/builder_utils/utils.go

@@ -33,7 +33,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -46,16 +45,20 @@ import (
 	"github.com/arduino/go-properties-orderedmap"
 )
 
-func PrintProgressIfProgressEnabledAndMachineLogger(ctx *types.Context) {
+func PrintProgressIfProgressEnabledAndMachineLogger(ctx *types.Context, progressEnabled bool, stepSize float64) {
 
-	if !ctx.Progress.PrintEnabled {
+	var mut sync.Mutex
+	if !progressEnabled {
 		return
 	}
 
 	log := ctx.GetLogger()
 	if log.Name() == "machine" {
+		mut.Lock()
 		log.Println(constants.LOG_LEVEL_INFO, constants.MSG_PROGRESS, strconv.FormatFloat(ctx.Progress.Progress, 'f', 2, 32))
-		ctx.Progress.Progress += ctx.Progress.Steps
+		ctx.Progress.Progress += stepSize
+		log.Flush()
+		mut.Unlock()
 	}
 }
 
@@ -170,61 +173,57 @@ func compileFilesWithRecipe(ctx *types.Context, sourcePath *paths.Path, sources
 	if len(sources) == 0 {
 		return objectFiles, nil
 	}
-	var objectFilesMux sync.Mutex
-	var errors []error
-	var errorsMux sync.Mutex
+	objectFilesChan := make(chan *paths.Path)
+	errorsChan := make(chan error)
+	doneChan := make(chan struct{})
 
-	ctx.Progress.Steps = ctx.Progress.Steps / float64(len(sources))
+	stepSize := ctx.Progress.Steps / float64(len(sources))
+	var wg sync.WaitGroup
 
-	queue := make(chan *paths.Path)
-	job := func(source *paths.Path) {
-		PrintProgressIfProgressEnabledAndMachineLogger(ctx)
-		objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe)
-		if err != nil {
-			errorsMux.Lock()
-			errors = append(errors, err)
-			errorsMux.Unlock()
-		} else {
-			objectFilesMux.Lock()
-			objectFiles.Add(objectFile)
-			objectFilesMux.Unlock()
+	// Split jobs into batches of N jobs each; wait for the completion of a batch to start the next
+	par := ctx.Jobs
+
+	go func() {
+		for total := 0; total < len(sources); total += par {
+			for i := total; i < total+par && i < len(sources); i++ {
+				wg.Add(1)
+				go func(source *paths.Path) {
+					defer wg.Done()
+					go PrintProgressIfProgressEnabledAndMachineLogger(ctx, true, stepSize)
+					objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe)
+					if err != nil {
+						errorsChan <- err
+					} else {
+						objectFilesChan <- objectFile
+					}
+				}(sources[i])
+			}
+			wg.Wait()
 		}
-	}
 
-	// Spawn jobs runners
-	var wg sync.WaitGroup
-	jobs := ctx.Jobs
-	if jobs == 0 {
-		jobs = runtime.NumCPU()
-	}
-	for i := 0; i < jobs; i++ {
-		wg.Add(1)
-		go func() {
-			for source := range queue {
-				job(source)
-			}
-			wg.Done()
-		}()
-	}
+		doneChan <- struct{}{}
+	}()
+
+	go func() {
+		wg.Wait()
+		doneChan <- struct{}{}
+	}()
 
-	// Feed jobs until error or done
-	for _, source := range sources {
-		errorsMux.Lock()
-		gotError := len(errors) > 0
-		errorsMux.Unlock()
-		if gotError {
-			break
+	for {
+		select {
+		case objectFile := <-objectFilesChan:
+			objectFiles.Add(objectFile)
+		case err := <-errorsChan:
+			return nil, i18n.WrapError(err)
+		case <-doneChan:
+			close(objectFilesChan)
+			for objectFile := range objectFilesChan {
+				objectFiles.Add(objectFile)
+			}
+			objectFiles.Sort()
+			return objectFiles, nil
 		}
-		queue <- source
-	}
-	close(queue)
-	wg.Wait()
-	if len(errors) > 0 {
-		// output the first error
-		return nil, i18n.WrapError(errors[0])
 	}
-	objectFiles.Sort()
-	return objectFiles, nil
 }
 
 func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string, recipe string) (*paths.Path, error) {

Diff for: legacy/builder/container_find_includes.go

@@ -148,8 +148,11 @@ func (s *ContainerFindIncludes) Run(ctx *types.Context) error {
 		queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, srcSubfolderPath, true /* recurse */)
 	}
 
+	stepSize := ctx.Progress.Steps / 20.0
+	stepLimit := ctx.Progress.Progress + ctx.Progress.Steps
+
 	for !sourceFilePaths.Empty() {
-		err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop())
+		err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop(), stepSize, stepLimit)
 		if err != nil {
 			cachePath.Remove()
 			return i18n.WrapError(err)
@@ -290,7 +293,7 @@ func writeCache(cache *includeCache, path *paths.Path) error {
 	return nil
 }
 
-func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error {
+func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile, stepSize, stepLimit float64) error {
 	sourcePath := sourceFile.SourcePath(ctx)
 	targetFilePath := paths.NullPath()
 	depPath := sourceFile.DepfilePath(ctx)
@@ -318,6 +321,8 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t
 		var include string
 		cache.ExpectFile(sourcePath)
 
+		go builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx, ctx.Progress.Progress < stepLimit, stepSize)
+
 		includes := ctx.IncludeFolders
 		if library, ok := sourceFile.Origin.(*libraries.Library); ok && library.UtilityDir != nil {
 			includes = append(includes, library.UtilityDir)

Diff for: legacy/builder/container_setup.go

@@ -52,10 +52,11 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context)
 		&LibrariesLoader{},
 	}
 
-	ctx.Progress.Steps = ctx.Progress.Steps / float64(len(commands))
+	stepSize := ctx.Progress.Steps / float64(len(commands))
+	ctx.Progress.Steps = stepSize
 
 	for _, command := range commands {
-		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx)
+		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx, ctx.Progress.PrintEnabled, stepSize)
 		PrintRingNameIfDebug(ctx, command)
 		err := command.Run(ctx)
 		if err != nil {
@@ -84,10 +85,11 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context)
 		&AddMissingBuildPropertiesFromParentPlatformTxtFiles{},
 	}
 
-	ctx.Progress.Steps = ctx.Progress.Steps / float64(len(commands))
+	stepSize = ctx.Progress.Steps / float64(len(commands))
+	ctx.Progress.Steps = stepSize
 
 	for _, command := range commands {
-		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx)
+		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx, ctx.Progress.PrintEnabled, stepSize)
 		PrintRingNameIfDebug(ctx, command)
 		err := command.Run(ctx)
 		if err != nil {