Normalize preprocessed output

Solves autocomplete over grcp
SOlves arduino/Arduino#6816

Author: facchinm

Diff for: src/arduino.cc/builder/builder.go

@@ -230,6 +230,15 @@ func RunParseHardwareAndDumpBuildProperties(ctx *types.Context) error {
 }
 
 func RunPreprocess(ctx *types.Context) error {
+	oldlogger := ctx.GetLogger()
+	if ctx.CodeCompleteAt != "" {
+		logger := i18n.NoopLogger{}
+		ctx.SetLogger(logger)
+	}
 	command := Preprocess{}
-	return command.Run(ctx)
+	ret := command.Run(ctx)
+	if ctx.CodeCompleteAt != "" {
+		ctx.SetLogger(oldlogger)
+	}
+	return ret
 }

Diff for: src/arduino.cc/builder/grpc/rpc.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"arduino.cc/builder"
-	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
 	"github.com/fsnotify/fsnotify"
@@ -98,14 +97,10 @@ func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams)
 	s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0]
 
 	s.watch()
-	oldlogger := s.ctx.GetLogger()
-	logger := i18n.NoopLogger{}
-	s.ctx.SetLogger(logger)
 
 	err := builder.RunPreprocess(s.ctx)
 
 	response := pb.Response{Line: s.ctx.CodeCompletions}
-	s.ctx.SetLogger(oldlogger)
 
 	return &response, err
 }

Diff for: src/arduino.cc/builder/preprocess_sketch.go

@@ -30,7 +30,9 @@
 package builder
 
 import (
+	"errors"
 	"fmt"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -118,14 +120,16 @@ func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error {
 
 	buf, err := command.Output()
 	if err != nil {
-		return i18n.WrapError(err)
+		return errors.New(i18n.WrapError(err).Error() + string(err.(*exec.ExitError).Stderr))
 	}
-	output := string(buf)
+
+	result := utils.NormalizeUTF8(buf)
+
 	//fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output)
 	if ctx.CodeCompleteAt != "" {
-		ctx.CodeCompletions = output
+		ctx.CodeCompletions = string(result)
 	} else {
-		ctx.Source = output
+		ctx.Source = string(result)
 	}
 	return nil
 }