diff --git a/.travis.yml b/.travis.yml index 8ec2e23e..910c086e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: - go get github.com/arduino/go-timeutils script: + - go get github.com/arduino/arduino-builder/arduino-builder - go build -o $HOME/arduino-builder -v github.com/arduino/arduino-builder/arduino-builder - export TEST_PACKAGES=`go list github.com/arduino/arduino-builder/...` - RES=0; I=0; for PKG in $TEST_PACKAGES; do go test -v -timeout 30m -covermode=count -coverprofile=coverage.$I.out $PKG; ((RES=RES+$?)); ((I++)); done; ( exit $RES ) diff --git a/arduino-builder/main.go b/arduino-builder/main.go index d2ca98c4..89c31bdb 100644 --- a/arduino-builder/main.go +++ b/arduino-builder/main.go @@ -37,11 +37,16 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strings" "syscall" + "runtime/pprof" + "runtime/trace" + "github.com/arduino/arduino-builder" "github.com/arduino/arduino-builder/gohasissues" + "github.com/arduino/arduino-builder/grpc" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" @@ -54,6 +59,7 @@ const VERSION = "1.3.25" const FLAG_ACTION_COMPILE = "compile" const FLAG_ACTION_PREPROCESS = "preprocess" const FLAG_ACTION_DUMP_PREFS = "dump-prefs" +const FLAG_ACTION_CODE_COMPLETE_AT = "code-complete-at" const FLAG_BUILD_OPTIONS_FILE = "build-options-file" const FLAG_HARDWARE = "hardware" const FLAG_TOOLS = "tools" @@ -78,7 +84,11 @@ const FLAG_LOGGER_HUMAN = "human" const FLAG_LOGGER_HUMANTAGS = "humantags" const FLAG_LOGGER_MACHINE = "machine" const FLAG_VERSION = "version" +const FLAG_DAEMON = "daemon" const FLAG_VID_PID = "vid-pid" +const FLAG_JOBS = "jobs" +const FLAG_TRACE = "trace" +const FLAG_EXPERIMENTAL = "experimental" type foldersFlag []string @@ -118,6 +128,7 @@ func (h *propertiesFlag) Set(value string) error { var compileFlag *bool var preprocessFlag *bool var dumpPrefsFlag *bool +var codeCompleteAtFlag *string var buildOptionsFileFlag *string var hardwareFoldersFlag foldersFlag var toolsFoldersFlag foldersFlag @@ -135,12 +146,17 @@ var debugLevelFlag *int var warningsLevelFlag *string var loggerFlag *string var versionFlag *bool +var daemonFlag *bool var vidPidFlag *string +var jobsFlag *int +var traceFlag *bool +var experimentalFeatures *bool func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") preprocessFlag = flag.Bool(FLAG_ACTION_PREPROCESS, false, "preprocess the given sketch") dumpPrefsFlag = flag.Bool(FLAG_ACTION_DUMP_PREFS, false, "dumps build properties used when compiling") + codeCompleteAtFlag = flag.String(FLAG_ACTION_CODE_COMPLETE_AT, "", "output code completions for sketch at a specific location. Location format is \"file:line:col\"") buildOptionsFileFlag = flag.String(FLAG_BUILD_OPTIONS_FILE, "", "Instead of specifying --"+FLAG_HARDWARE+", --"+FLAG_TOOLS+" etc every time, you can load all such options from a file") flag.Var(&hardwareFoldersFlag, FLAG_HARDWARE, "Specify a 'hardware' folder. Can be added multiple times for specifying multiple 'hardware' folders") flag.Var(&toolsFoldersFlag, FLAG_TOOLS, "Specify a 'tools' folder. Can be added multiple times for specifying multiple 'tools' folders") @@ -158,12 +174,40 @@ func init() { warningsLevelFlag = flag.String(FLAG_WARNINGS, "", "Sets warnings level. Available values are '"+FLAG_WARNINGS_NONE+"', '"+FLAG_WARNINGS_DEFAULT+"', '"+FLAG_WARNINGS_MORE+"' and '"+FLAG_WARNINGS_ALL+"'") loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'") versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits") + daemonFlag = flag.Bool(FLAG_DAEMON, false, "daemonizes and serves its functions via rpc") vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt") + jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine") + traceFlag = flag.Bool(FLAG_TRACE, false, "traces the whole process lifecycle") + experimentalFeatures = flag.Bool(FLAG_EXPERIMENTAL, false, "enables experimental features") } func main() { + flag.Parse() + if *traceFlag { + f, err := os.Create("trace.out") + if err != nil { + panic(err) + } + defer f.Close() + + f2, err := os.Create("profile.out") + if err != nil { + panic(err) + } + defer f2.Close() + + pprof.StartCPUProfile(f2) + defer pprof.StopCPUProfile() + + err = trace.Start(f) + if err != nil { + panic(err) + } + defer trace.Stop() + } + if *versionFlag { fmt.Println("Arduino Builder " + VERSION) fmt.Println("Copyright (C) 2015 Arduino LLC and contributors") @@ -173,8 +217,28 @@ func main() { return } + if *jobsFlag > 0 { + runtime.GOMAXPROCS(*jobsFlag) + } else { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + ctx := &types.Context{} + // place here all experimental features that should live under this flag + if *experimentalFeatures { + ctx.UseArduinoPreprocessor = true + } + + if *daemonFlag { + var loggerBuffer []string + logger := i18n.AccumulatorLogger{} + logger.Buffer = &loggerBuffer + //logger := i18n.HumanLogger{} + ctx.SetLogger(logger) + jsonrpc.RegisterAndServeJsonRPC(ctx) + } + if *buildOptionsFileFlag != "" { buildOptions := make(properties.Map) if _, err := os.Stat(*buildOptionsFileFlag); err == nil { @@ -330,7 +394,8 @@ func main() { if *dumpPrefsFlag { err = builder.RunParseHardwareAndDumpBuildProperties(ctx) - } else if *preprocessFlag { + } else if *preprocessFlag || *codeCompleteAtFlag != "" { + ctx.CodeCompleteAt = *codeCompleteAtFlag err = builder.RunPreprocess(ctx) } else { if flag.NArg() == 0 { diff --git a/builder.go b/builder.go index d5b8eff7..6c143996 100644 --- a/builder.go +++ b/builder.go @@ -35,6 +35,7 @@ import ( "strconv" "time" + "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/phases" @@ -89,7 +90,7 @@ func (s *Builder) Run(ctx *types.Context) error { &WarnAboutArchIncompatibleLibraries{}, utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Generating function prototypes..."), - &ContainerAddPrototypes{}, + &PreprocessSketch{}, utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Compiling sketch..."), &RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_SKETCH_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX}, @@ -128,6 +129,8 @@ func (s *Builder) Run(ctx *types.Context) error { &PrintUsedLibrariesIfVerbose{}, + &ExportProjectCMake{SketchError: mainErr != nil}, + &phases.Sizer{SketchError: mainErr != nil}, } otherErr := runCommands(ctx, commands, false) @@ -139,6 +142,18 @@ func (s *Builder) Run(ctx *types.Context) error { return otherErr } +type PreprocessSketch struct{} + +func (s *PreprocessSketch) Run(ctx *types.Context) error { + var commands []types.Command + if ctx.UseArduinoPreprocessor { + commands = append(commands, &PreprocessSketchArduino{}) + } else { + commands = append(commands, &ContainerAddPrototypes{}) + } + return runCommands(ctx, commands, true) +} + type Preprocess struct{} func (s *Preprocess) Run(ctx *types.Context) error { @@ -158,7 +173,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { &WarnAboutArchIncompatibleLibraries{}, - &ContainerAddPrototypes{}, + &PreprocessSketch{}, &PrintPreprocessedSource{}, } @@ -181,36 +196,22 @@ func (s *ParseHardwareAndDumpBuildProperties) Run(ctx *types.Context) error { } func runCommands(ctx *types.Context, commands []types.Command, progressEnabled bool) error { - commandsLength := len(commands) - progressForEachCommand := float32(100) / float32(commandsLength) - progress := float32(0) + ctx.Progress.PrintEnabled = progressEnabled + ctx.Progress.Progress = 0 + for _, command := range commands { PrintRingNameIfDebug(ctx, command) - printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, progress) + ctx.Progress.Steps = 100.0 / float64(len(commands)) + builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx) err := command.Run(ctx) if err != nil { return i18n.WrapError(err) } - progress += progressForEachCommand } - - printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, 100) - return nil } -func printProgressIfProgressEnabledAndMachineLogger(progressEnabled bool, ctx *types.Context, progress float32) { - if !progressEnabled { - return - } - - log := ctx.GetLogger() - if log.Name() == "machine" { - log.Println(constants.LOG_LEVEL_INFO, constants.MSG_PROGRESS, strconv.FormatFloat(float64(progress), 'f', 2, 32)) - } -} - func PrintRingNameIfDebug(ctx *types.Context, command types.Command) { if ctx.DebugLevel >= 10 { ctx.GetLogger().Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_COMMAND, strconv.FormatInt(time.Now().Unix(), 10), reflect.Indirect(reflect.ValueOf(command)).Type().Name()) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 139ad770..ec76ad3a 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -30,22 +30,37 @@ package builder_utils import ( - "bytes" - "fmt" "io" "os" "os/exec" "path/filepath" + "sort" + "strconv" "strings" + "sync" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" ) -func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, logger) +func PrintProgressIfProgressEnabledAndMachineLogger(ctx *types.Context) { + + if !ctx.Progress.PrintEnabled { + return + } + + log := ctx.GetLogger() + if log.Name() == "machine" { + log.Println(constants.LOG_LEVEL_INFO, constants.MSG_PROGRESS, strconv.FormatFloat(ctx.Progress.Progress, 'f', 2, 32)) + ctx.Progress.Progress += ctx.Progress.Steps + } +} + +func CompileFilesRecursive(ctx *types.Context, objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + objectFiles, err := CompileFiles(ctx, objectFiles, sourcePath, false, buildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -56,7 +71,7 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st } for _, folder := range folders { - objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = CompileFilesRecursive(ctx, objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -65,28 +80,28 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st return objectFiles, nil } -func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, logger) +func CompileFiles(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + objectFiles, err := compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN) if err != nil { return nil, i18n.WrapError(err) } return objectFiles, nil } -func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithExtensionWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string) ([]string, error) { sources, err := findFilesInFolder(sourcePath, extension, recurse) if err != nil { return nil, i18n.WrapError(err) } - return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + return compileFilesWithRecipe(ctx, objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe) } func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]string, error) { @@ -145,21 +160,57 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { return sources, nil } -func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string) ([]string, error) { + if len(sources) == 0 { + return objectFiles, nil + } + objectFilesChan := make(chan string) + errorsChan := make(chan error) + doneChan := make(chan struct{}) + + ctx.Progress.Steps = ctx.Progress.Steps / float64(len(sources)) + var wg sync.WaitGroup + wg.Add(len(sources)) + for _, source := range sources { - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) - if err != nil { + go func(source string) { + defer wg.Done() + PrintProgressIfProgressEnabledAndMachineLogger(ctx) + objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe) + if err != nil { + errorsChan <- err + } else { + objectFilesChan <- objectFile + } + }(source) + } + + go func() { + wg.Wait() + doneChan <- struct{}{} + }() + + for { + select { + case objectFile := <-objectFilesChan: + objectFiles = append(objectFiles, objectFile) + case err := <-errorsChan: return nil, i18n.WrapError(err) + case <-doneChan: + close(objectFilesChan) + for objectFile := range objectFilesChan { + objectFiles = append(objectFiles, objectFile) + } + sort.Strings(objectFiles) + return objectFiles, nil } - - objectFiles = append(objectFiles, objectFile) } - return objectFiles, nil } -func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { +func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string) (string, error) { + logger := ctx.GetLogger() properties := buildProperties.Clone() - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = source relativeSource, err := filepath.Rel(sourcePath, source) @@ -173,27 +224,33 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b return "", i18n.WrapError(err) } - objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d")) + objIsUpToDate, err := ObjFileIsUpToDate(ctx, properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d")) if err != nil { return "", i18n.WrapError(err) } if !objIsUpToDate { - _, err = ExecRecipe(properties, recipe, false, verbose, verbose, logger) + _, _, err = ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return "", i18n.WrapError(err) } - } else if verbose { + } else if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, properties[constants.BUILD_PROPERTIES_OBJECT_FILE]) } return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, error) { +func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFile string) (bool, error) { sourceFile = filepath.Clean(sourceFile) objectFile = filepath.Clean(objectFile) dependencyFile = filepath.Clean(dependencyFile) + logger := ctx.GetLogger() + debugLevel := ctx.DebugLevel + + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking previous results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile) + } sourceFileStat, err := os.Stat(sourceFile) if err != nil { @@ -203,6 +260,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err objectFileStat, err := os.Stat(objectFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", objectFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -212,6 +272,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err dependencyFileStat, err := os.Stat(dependencyFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", dependencyFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -219,9 +282,15 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err } if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, objectFile) + } return false, nil } if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, dependencyFile) + } return false, nil } @@ -241,10 +310,16 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err firstRow := rows[0] if !strings.HasSuffix(firstRow, ":") { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "No colon in first line of depfile") + } return false, nil } objFileInDepFile := firstRow[:len(firstRow)-1] if objFileInDepFile != objectFile { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Depfile is about different file: {0}", objFileInDepFile) + } return false, nil } @@ -254,12 +329,22 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err if err != nil && !os.IsNotExist(err) { // There is probably a parsing error of the dep file // Ignore the error and trigger a full rebuild anyway + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Failed to read: {0}", row) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, i18n.WrapError(err).Error()) + } return false, nil } if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", row) + } return false, nil } if depStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, objectFile) + } return false, nil } } @@ -309,7 +394,34 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) return true } -func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) { +func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile string) bool { + + targetFileStat, err := os.Stat(targetFile) + if err == nil { + files, err := findAllFilesInFolder(corePath, true) + if err != nil { + return true + } + for _, file := range files { + // report changes only for .txt files + if filepath.Ext(file) != ".txt" { + continue + } + fileStat, err := os.Stat(file) + if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { + return true + } + } + if targetCorePath != constants.EMPTY_STRING && !strings.EqualFold(corePath, targetCorePath) { + return TXTBuildRulesHaveChanged(targetCorePath, constants.EMPTY_STRING, targetFile) + } + return false + } + return true +} + +func ArchiveCompiledFiles(ctx *types.Context, buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map) (string, error) { + logger := ctx.GetLogger() archiveFilePath := filepath.Join(buildPath, archiveFile) rebuildArchive := false @@ -332,7 +444,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st return "", i18n.WrapError(err) } } else { - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, archiveFilePath) } return archiveFilePath, nil @@ -345,7 +457,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = archiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objectFile - _, err := ExecRecipe(properties, constants.RECIPE_AR_PATTERN, false, verbose, verbose, logger) + _, _, err := ExecRecipe(ctx, properties, constants.RECIPE_AR_PATTERN, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return "", i18n.WrapError(err) } @@ -354,28 +466,20 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st return archiveFilePath, nil } -func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) { - command, err := PrepareCommandForRecipe(properties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger) +// See util.ExecCommand for stdout/stderr arguments +func ExecRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool, stdout int, stderr int) ([]byte, []byte, error) { + command, err := PrepareCommandForRecipe(ctx, buildProperties, recipe, removeUnsetProperties) if err != nil { - return nil, i18n.WrapError(err) + return nil, nil, i18n.WrapError(err) } - if echoOutput { - command.Stdout = os.Stdout - } - - command.Stderr = os.Stderr - - if echoOutput { - err := command.Run() - return nil, i18n.WrapError(err) - } - - bytes, err := command.Output() - return bytes, i18n.WrapError(err) + return utils.ExecCommand(ctx, command, stdout, stderr) } -func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (*exec.Cmd, error) { +const COMMANDLINE_LIMIT = 30000 + +func PrepareCommandForRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool) (*exec.Cmd, error) { + logger := ctx.GetLogger() pattern := buildProperties[recipe] if pattern == constants.EMPTY_STRING { return nil, i18n.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, recipe) @@ -387,32 +491,22 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) } - command, err := utils.PrepareCommand(commandLine, logger) - if err != nil { - return nil, i18n.WrapError(err) - } + relativePath := "" - if echoCommandLine { - fmt.Println(commandLine) + if len(commandLine) > COMMANDLINE_LIMIT { + relativePath = buildProperties[constants.BUILD_PROPERTIES_BUILD_PATH] } - return command, nil -} - -func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (string, error) { - command, err := PrepareCommandForRecipe(buildProperties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger) + command, err := utils.PrepareCommand(commandLine, logger, relativePath) if err != nil { - return "", i18n.WrapError(err) + return nil, i18n.WrapError(err) } - buffer := &bytes.Buffer{} - command.Stderr = buffer - command.Run() - return string(buffer.Bytes()), nil -} + if ctx.Verbose { + logger.UnformattedFprintln(os.Stdout, commandLine) + } -func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { - buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1) + return command, nil } // CopyFile copies the contents of the file named src to the file named diff --git a/client/client.go b/client/client.go new file mode 100644 index 00000000..29e2ce7c --- /dev/null +++ b/client/client.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a simple gRPC client that demonstrates how to use gRPC-Go libraries +// to perform unary, client streaming, server streaming and full duplex RPCs. +// +// It interacts with the route guide service whose definition can be found in routeguide/route_guide.proto. +package main + +import ( + "io" + "log" + + pb "github.com/arduino/arduino-builder/grpc/proto" + "golang.org/x/net/context" + "google.golang.org/grpc" +) + +// printFeature gets the feature for the given point. +func autocomplete(client pb.BuilderClient, in *pb.BuildParams) { + resp, err := client.Autocomplete(context.Background(), in) + if err != nil { + log.Fatalf("%v.GetFeatures(_) = _, %v: ", client, err) + } + log.Println(resp) +} + +// printFeatures lists all the features within the given bounding Rectangle. +func build(client pb.BuilderClient, in *pb.BuildParams) { + stream, err := client.Build(context.Background(), in) + if err != nil { + log.Fatalf("%v.ListFeatures(_) = _, %v", client, err) + } + for { + line, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("%v.ListFeatures(_) = _, %v", client, err) + } + log.Println(line) + } +} + +func main() { + conn, err := grpc.Dial("localhost:12345", grpc.WithInsecure()) + if err != nil { + log.Fatalf("fail to dial: %v", err) + } + defer conn.Close() + + client := pb.NewBuilderClient(conn) + + exampleParames := pb.BuildParams{ + BuiltInLibrariesFolders: "/ssd/Arduino-master/build/linux/work/libraries", + CustomBuildProperties: "build.warn_data_percentage=75", + FQBN: "arduino:avr:mega:cpu=atmega2560", + HardwareFolders: "/ssd/Arduino-master/build/linux/work/hardware,/home/martino/.arduino15/packages,/home/martino/eslov-sk/hardware", + OtherLibrariesFolders: "/home/martino/eslov-sk/libraries", + ArduinoAPIVersion: "10805", + SketchLocation: "/home/martino/eslov-sk/libraries/WiFi101/examples/ScanNetworks/ScanNetworks.ino", + ToolsFolders: "/ssd/Arduino-master/build/linux/work/tools-builder,/ssd/Arduino-master/build/linux/work/hardware/tools/avr,/home/martino/.arduino15/packages", + Verbose: true, + WarningsLevel: "all", + BuildCachePath: "/tmp/arduino_cache_761418/", + CodeCompleteAt: "/home/martino/eslov-sk/libraries/WiFi101/examples/ScanNetworks/ScanNetworks.ino:56:9", + } + + //build(client, &exampleParames) + autocomplete(client, &exampleParames) +} diff --git a/constants/constants.go b/constants/constants.go index d83ee273..7254c2df 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -48,10 +48,12 @@ const BUILD_PROPERTIES_BUILD_SYSTEM_PATH = "build.system.path" const BUILD_PROPERTIES_BUILD_VARIANT = "build.variant" const BUILD_PROPERTIES_BUILD_VARIANT_PATH = "build.variant.path" const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags" -const BUILD_PROPERTIES_COMPILER_C_ELF_EXTRAFLAGS = "compiler.c.elf.extra_flags" +const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags" +const BUILD_PROPERTIES_COMPILER_LIBRARIES_LDFLAGS = "compiler.libraries.ldflags" const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags" const BUILD_PROPERTIES_COMPILER_PATH = "compiler.path" const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags" +const BUILD_PROPERTIES_COMPILER_EXPORT_CMAKE_FLAGS = "compiler.export_cmake" const BUILD_PROPERTIES_EXTRA_TIME_DST = "extra.time.dst" const BUILD_PROPERTIES_EXTRA_TIME_LOCAL = "extra.time.local" const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc" @@ -64,6 +66,7 @@ const BUILD_PROPERTIES_PATTERN = "pattern" const BUILD_PROPERTIES_PID = "pid" const BUILD_PROPERTIES_PREPROCESSED_FILE_PATH = "preprocessed_file_path" const BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH = "runtime.hardware.path" +const BUILD_PROPERTIES_RUNTIME_IDE_PATH = "runtime.ide.path" const BUILD_PROPERTIES_RUNTIME_OS = "runtime.os" const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path" const BUILD_PROPERTIES_RUNTIME_TOOLS_PREFIX = "runtime.tools." diff --git a/container_add_prototypes.go b/container_add_prototypes.go index cd0d56f7..13b777d3 100644 --- a/container_add_prototypes.go +++ b/container_add_prototypes.go @@ -35,15 +35,28 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" ) type ContainerAddPrototypes struct{} func (s *ContainerAddPrototypes) Run(ctx *types.Context) error { sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") + + // Generate the full pathname for the preproc output file + err := utils.EnsureFolderExists(ctx.PreprocPath) + if err != nil { + return i18n.WrapError(err) + } + targetFilePath := filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E) + + // Run preprocessor + err = GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders) + if err != nil { + return i18n.WrapError(err) + } commands := []types.Command{ - &GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, - &ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE}, + &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE}, &FilterSketchSource{Source: &ctx.SourceGccMinusE}, &CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E}, &CTagsRunner{}, diff --git a/container_find_includes.go b/container_find_includes.go index dbd31c71..df0fca81 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -110,6 +110,7 @@ import ( "encoding/json" "io/ioutil" "os" + "os/exec" "path/filepath" "time" @@ -118,6 +119,8 @@ import ( "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" + + "github.com/go-errors/errors" ) type ContainerFindIncludes struct{} @@ -213,7 +216,7 @@ func (cache *includeCache) Next() includeCacheEntry { // not, or no entry is available, the cache is invalidated. Does not // advance the cache. func (cache *includeCache) ExpectFile(sourcefile string) { - if cache.valid && cache.next < len(cache.entries) && cache.Next().Sourcefile != sourcefile { + if cache.valid && (cache.next >= len(cache.entries) || cache.Next().Sourcefile != sourcefile) { cache.valid = false cache.entries = cache.entries[:cache.next] } @@ -289,6 +292,8 @@ func writeCache(cache *includeCache, path string) error { func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error { sourcePath := sourceFile.SourcePath(ctx) + depPath := sourceFile.DepfilePath(ctx) + objPath := sourceFile.ObjectPath(ctx) targetFilePath := utils.NULLFile() // TODO: This should perhaps also compare against the @@ -303,7 +308,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx)) + unchanged, err := builder_utils.ObjFileIsUpToDate(ctx, sourcePath, objPath, depPath) if err != nil { return i18n.WrapError(err) } @@ -317,23 +322,33 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t if library, ok := sourceFile.Origin.(*types.Library); ok && library.UtilityFolder != "" { includes = append(includes, library.UtilityFolder) } + var preproc_err error + var preproc_stderr []byte if unchanged && cache.valid { include = cache.Next().Include if first && ctx.Verbose { ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_CACHED_INCLUDES, sourcePath) } } else { - commands := []types.Command{ - &GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, TargetFilePath: targetFilePath, Includes: includes}, - &IncludesFinderWithRegExp{Source: &ctx.SourceGccMinusE}, - } - for _, command := range commands { - err := runCommand(ctx, command) - if err != nil { - return i18n.WrapError(err) + preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + // Unwrap error and see if it is an ExitError. + _, is_exit_error := i18n.UnwrapError(preproc_err).(*exec.ExitError) + if preproc_err == nil { + // Preprocessor successful, done + include = "" + } else if !is_exit_error || preproc_stderr == nil { + // Ignore ExitErrors (e.g. gcc returning + // non-zero status), but bail out on + // other errors + return i18n.WrapError(preproc_err) + } else { + include = IncludesFinderWithRegExp(ctx, string(preproc_stderr)) + if include == "" { + // No include found? Bail out. + os.Stderr.Write(preproc_stderr) + return i18n.WrapError(preproc_err) } } - include = ctx.IncludeJustFound } if include == "" { @@ -345,8 +360,19 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t library := ResolveLibrary(ctx, include) if library == nil { // Library could not be resolved, show error - err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes}) - return i18n.WrapError(err) + if preproc_err == nil || preproc_stderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + if preproc_err == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New("Internal error in cache") + } + } + os.Stderr.Write(preproc_stderr) + return i18n.WrapError(preproc_err) } // Add this library to the list of libraries, the diff --git a/container_setup.go b/container_setup.go index 93ee6d73..57749aa2 100644 --- a/container_setup.go +++ b/container_setup.go @@ -30,6 +30,7 @@ package builder import ( + "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" ) @@ -54,7 +55,10 @@ func (s *ContainerSetupHardwareToolsLibsSketchAndProps) Run(ctx *types.Context) &AddMissingBuildPropertiesFromParentPlatformTxtFiles{}, } + ctx.Progress.Steps = ctx.Progress.Steps / float64(len(commands)) + for _, command := range commands { + builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx) PrintRingNameIfDebug(ctx, command) err := command.Run(ctx) if err != nil { diff --git a/create_cmake_rule.go b/create_cmake_rule.go new file mode 100644 index 00000000..7038dd54 --- /dev/null +++ b/create_cmake_rule.go @@ -0,0 +1,250 @@ +/* + * This file is part of Arduino Builder. + * + * Arduino Builder is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + * + * Copyright 2015 Arduino LLC (http://www.arduino.cc/) + */ + +package builder + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/arduino/arduino-builder/builder_utils" + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" +) + +var VALID_EXPORT_EXTENSIONS = map[string]bool{".h": true, ".c": true, ".hpp": true, ".hh": true, ".cpp": true, ".s": true, ".a": true} +var DOTHEXTENSION = map[string]bool{".h": true, ".hh": true, ".hpp": true} +var DOTAEXTENSION = map[string]bool{".a": true} + +type ExportProjectCMake struct { + // Was there an error while compiling the sketch? + SketchError bool +} + +func (s *ExportProjectCMake) Run(ctx *types.Context) error { + //verbose := ctx.Verbose + logger := ctx.GetLogger() + + if s.SketchError || !canExportCmakeProject(ctx) { + return nil + } + + // Create new cmake subFolder - clean if the folder is already there + cmakeFolder := filepath.Join(ctx.BuildPath, "_cmake") + if _, err := os.Stat(cmakeFolder); err == nil { + os.RemoveAll(cmakeFolder) + } + os.Mkdir(cmakeFolder, 0777) + + // Create lib and build subfolders + libBaseFolder := filepath.Join(cmakeFolder, "lib") + os.Mkdir(libBaseFolder, 0777) + buildBaseFolder := filepath.Join(cmakeFolder, "build") + os.Mkdir(buildBaseFolder, 0777) + + // Create core subfolder path (don't create it yet) + coreFolder := filepath.Join(cmakeFolder, "core") + cmakeFile := filepath.Join(cmakeFolder, "CMakeLists.txt") + + // Copy used libraries in the correct folder + extensions := func(ext string) bool { return VALID_EXPORT_EXTENSIONS[ext] } + for _, library := range ctx.ImportedLibraries { + libFolder := filepath.Join(libBaseFolder, library.Name) + utils.CopyDir(library.Folder, libFolder, extensions) + // Remove examples folder + if _, err := os.Stat(filepath.Join(libFolder, "examples")); err == nil { + os.RemoveAll(filepath.Join(libFolder, "examples")) + } + // Remove stray folders contining incompatible libraries + staticLibsExtensions := func(ext string) bool { return DOTAEXTENSION[ext] } + mcu := ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_MCU] + var files []string + utils.FindFilesInFolder(&files, filepath.Join(libFolder, "src"), staticLibsExtensions, true) + for _, file := range files { + if !strings.Contains(filepath.Dir(file), mcu) { + os.RemoveAll(filepath.Dir(file)) + } + } + } + + // Copy core + variant in use + preprocessed sketch in the correct folders + err := utils.CopyDir(ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH], coreFolder, extensions) + if err != nil { + fmt.Println(err) + } + err = utils.CopyDir(ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH], filepath.Join(coreFolder, "variant"), extensions) + if err != nil { + fmt.Println(err) + } + + // Use old ctags method to generate export file + commands := []types.Command{ + &ContainerMergeCopySketchFiles{}, + &ContainerAddPrototypes{}, + &FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true}, + &SketchSaver{}, + } + + for _, command := range commands { + command.Run(ctx) + } + + err = utils.CopyDir(ctx.SketchBuildPath, filepath.Join(cmakeFolder, "sketch"), extensions) + if err != nil { + fmt.Println(err) + } + + // Extract CFLAGS, CPPFLAGS and LDFLAGS + var defines []string + var linkerflags []string + var libs []string + var linkDirectories []string + + extractCompileFlags(ctx, constants.RECIPE_C_COMBINE_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger) + extractCompileFlags(ctx, constants.RECIPE_C_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger) + extractCompileFlags(ctx, constants.RECIPE_CPP_PATTERN, &defines, &libs, &linkerflags, &linkDirectories, logger) + + // Extract folders with .h in them for adding in include list + var headerFiles []string + isHeader := func(ext string) bool { return DOTHEXTENSION[ext] } + utils.FindFilesInFolder(&headerFiles, cmakeFolder, isHeader, true) + foldersContainingDotH := findUniqueFoldersRelative(headerFiles, cmakeFolder) + + // Extract folders with .a in them for adding in static libs paths list + var staticLibsFiles []string + isStaticLib := func(ext string) bool { return DOTAEXTENSION[ext] } + utils.FindFilesInFolder(&staticLibsFiles, cmakeFolder, isStaticLib, true) + + // Generate the CMakeLists global file + + projectName := strings.TrimSuffix(filepath.Base(ctx.Sketch.MainFile.Name), filepath.Ext(ctx.Sketch.MainFile.Name)) + + cmakelist := "cmake_minimum_required(VERSION 3.5.0)\n" + cmakelist += "INCLUDE(FindPkgConfig)\n" + cmakelist += "project (" + projectName + " C CXX)\n" + cmakelist += "add_definitions (" + strings.Join(defines, " ") + " " + strings.Join(linkerflags, " ") + ")\n" + cmakelist += "include_directories (" + foldersContainingDotH + ")\n" + + // Make link directories relative + // We can totally discard them since they mostly are outside the core folder + // If they are inside the core they are not getting copied :) + var relLinkDirectories []string + for _, dir := range linkDirectories { + if strings.Contains(dir, cmakeFolder) { + relLinkDirectories = append(relLinkDirectories, strings.TrimPrefix(dir, cmakeFolder)) + } + } + + // Add SO_PATHS option for libraries not getting found by pkg_config + cmakelist += "set(EXTRA_LIBS_DIRS \"\" CACHE STRING \"Additional paths for dynamic libraries\")\n" + + for i, lib := range libs { + // Dynamic libraries should be discovered by pkg_config + lib = strings.TrimPrefix(lib, "-l") + libs[i] = lib + cmakelist += "pkg_search_module (" + strings.ToUpper(lib) + " " + lib + ")\n" + relLinkDirectories = append(relLinkDirectories, "${"+strings.ToUpper(lib)+"_LIBRARY_DIRS}") + } + cmakelist += "link_directories (" + strings.Join(relLinkDirectories, " ") + " ${EXTRA_LIBS_DIRS})\n" + for _, staticLibsFile := range staticLibsFiles { + // Static libraries are fully configured + lib := filepath.Base(staticLibsFile) + lib = strings.TrimPrefix(lib, "lib") + lib = strings.TrimSuffix(lib, ".a") + if !utils.SliceContains(libs, lib) { + libs = append(libs, lib) + cmakelist += "add_library (" + lib + " STATIC IMPORTED)\n" + location := strings.TrimPrefix(staticLibsFile, cmakeFolder) + cmakelist += "set_property(TARGET " + lib + " PROPERTY IMPORTED_LOCATION " + "${PROJECT_SOURCE_DIR}" + location + " )\n" + } + } + // Include source files + // TODO: remove .cpp and .h from libraries example folders + cmakelist += "file (GLOB_RECURSE SOURCES core/*.c* lib/*.c* sketch/*.c*)\n" + + // Compile and link project + cmakelist += "add_executable (" + projectName + " ${SOURCES} ${SOURCES_LIBS})\n" + cmakelist += "target_link_libraries( " + projectName + " -Wl,--as-needed -Wl,--start-group " + strings.Join(libs, " ") + " -Wl,--end-group)\n" + + utils.WriteFile(cmakeFile, cmakelist) + + return nil +} + +func canExportCmakeProject(ctx *types.Context) bool { + return ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_EXPORT_CMAKE_FLAGS] != "" +} + +func extractCompileFlags(ctx *types.Context, receipe string, defines, libs, linkerflags, linkDirectories *[]string, logger i18n.Logger) { + command, _ := builder_utils.PrepareCommandForRecipe(ctx, ctx.BuildProperties, receipe, true) + + for _, arg := range command.Args { + if strings.HasPrefix(arg, "-D") { + *defines = appendIfUnique(*defines, arg) + continue + } + if strings.HasPrefix(arg, "-l") { + *libs = appendIfUnique(*libs, arg) + continue + } + if strings.HasPrefix(arg, "-L") { + *linkDirectories = appendIfUnique(*linkDirectories, strings.TrimPrefix(arg, "-L")) + continue + } + if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "-I") && !strings.HasPrefix(arg, "-o") { + // HACK : from linkerflags remove MMD (no cache is produced) + if !strings.HasPrefix(arg, "-MMD") { + *linkerflags = appendIfUnique(*linkerflags, arg) + } + } + } +} + +func findUniqueFoldersRelative(slice []string, base string) string { + var out []string + for _, element := range slice { + path := filepath.Dir(element) + path = strings.TrimPrefix(path, base+"/") + if !utils.SliceContains(out, path) { + out = append(out, path) + } + } + return strings.Join(out, " ") +} + +func appendIfUnique(slice []string, element string) []string { + if !utils.SliceContains(slice, element) { + slice = append(slice, element) + } + return slice +} diff --git a/ctags_runner.go b/ctags_runner.go index 82e4165c..ef4dc64a 100644 --- a/ctags_runner.go +++ b/ctags_runner.go @@ -30,7 +30,7 @@ package builder import ( - "fmt" + "os" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/ctags" @@ -56,14 +56,14 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { } commandLine := properties.ExpandPropsInString(pattern) - command, err := utils.PrepareCommand(commandLine, logger) + command, err := utils.PrepareCommand(commandLine, logger, "") if err != nil { return i18n.WrapError(err) } verbose := ctx.Verbose if verbose { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } sourceBytes, err := command.Output() diff --git a/filter_sketch_source.go b/filter_sketch_source.go index 6de658cd..a6abd730 100644 --- a/filter_sketch_source.go +++ b/filter_sketch_source.go @@ -39,7 +39,8 @@ import ( ) type FilterSketchSource struct { - Source *string + Source *string + RemoveLineMarkers bool } func (s *FilterSketchSource) Run(ctx *types.Context) error { @@ -57,6 +58,9 @@ func (s *FilterSketchSource) Run(ctx *types.Context) error { filename := parseLineMarker(line) if filename != "" { inSketch = utils.SliceContains(fileNames, filename) + if inSketch && s.RemoveLineMarkers { + continue + } } if inSketch { @@ -79,7 +83,7 @@ func parseLineMarker(line string) string { // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 split := strings.SplitN(line, " ", 3) - if len(split) < 3 || split[0] != "#" { + if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { return "" } diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index b571cacc..3c801f89 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -30,7 +30,7 @@ package builder import ( - "path/filepath" + "os/exec" "strings" "github.com/arduino/arduino-builder/builder_utils" @@ -38,87 +38,59 @@ import ( "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "github.com/arduino/go-properties-map" ) -type GCCPreprocRunner struct { - SourceFilePath string - TargetFileName string - Includes []string -} - -func (s *GCCPreprocRunner) Run(ctx *types.Context) error { - properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFileName, s.Includes) +func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) error { + cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { return i18n.WrapError(err) } - if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { - //generate PREPROC_MACROS from RECIPE_CPP_PATTERN - properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) - } - - verbose := ctx.Verbose - logger := ctx.GetLogger() - _, err = builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger) + _, _, err = utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return i18n.WrapError(err) } - ctx.FileToRead = targetFilePath - return nil } -type GCCPreprocRunnerForDiscoveringIncludes struct { - SourceFilePath string - TargetFilePath string - Includes []string -} - -func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error { - properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFilePath, s.Includes) +func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) ([]byte, error) { + cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { - return i18n.WrapError(err) + return nil, i18n.WrapError(err) } - verbose := ctx.Verbose - logger := ctx.GetLogger() - - if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { - //generate PREPROC_MACROS from RECIPE_CPP_PATTERN - properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) - } - - stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger) + _, stderr, err := utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Capture) if err != nil { - return i18n.WrapError(err) + return stderr, i18n.WrapError(err) } - ctx.SourceGccMinusE = string(stderr) - - return nil + return stderr, nil } -func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) { - if targetFilePath != utils.NULLFile() { - preprocPath := ctx.PreprocPath - err := utils.EnsureFolderExists(preprocPath) - if err != nil { - return nil, "", i18n.WrapError(err) - } - targetFilePath = filepath.Join(preprocPath, targetFilePath) - } - +func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (*exec.Cmd, error) { properties := ctx.BuildProperties.Clone() properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath includes = utils.Map(includes, utils.WrapWithHyphenI) properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) - builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties) - return properties, targetFilePath, nil + if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { + //generate PREPROC_MACROS from RECIPE_CPP_PATTERN + properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) + } + + cmd, err := builder_utils.PrepareCommandForRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true) + if err != nil { + return nil, i18n.WrapError(err) + } + + // Remove -MMD argument if present. Leaving it will make gcc try + // to create a /dev/null.d dependency file, which won't work. + cmd.Args = utils.Filter(cmd.Args, func(a string) bool { return a != "-MMD" }) + + return cmd, nil } func GeneratePreprocPatternFromCompile(compilePattern string) string { diff --git a/gcc_preproc_source_saver.go b/gcc_preproc_source_saver.go deleted file mode 100644 index 556cd276..00000000 --- a/gcc_preproc_source_saver.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - * This file is part of Arduino Builder. - * - * Arduino Builder is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - * As a special exception, you may use this file as part of a free software - * library without restriction. Specifically, if other files instantiate - * templates or use macros or inline functions from this file, or you compile - * this file and link it with other files to produce an executable, this - * file does not by itself cause the resulting executable to be covered by - * the GNU General Public License. This exception does not however - * invalidate any other reasons why the executable file might be covered by - * the GNU General Public License. - * - * Copyright 2015 Arduino LLC (http://www.arduino.cc/) - */ - -package builder - -// XXX: OBSOLETE? - -import ( - "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/i18n" - "github.com/arduino/arduino-builder/types" - "github.com/arduino/arduino-builder/utils" - "path/filepath" -) - -type GCCPreprocSourceSaver struct{} - -func (s *GCCPreprocSourceSaver) Run(ctx *types.Context) error { - preprocPath := ctx.PreprocPath - err := utils.EnsureFolderExists(preprocPath) - if err != nil { - return i18n.WrapError(err) - } - - err = utils.WriteFile(filepath.Join(preprocPath, constants.FILE_GCC_PREPROC_TARGET), ctx.Source) - return i18n.WrapError(err) -} diff --git a/grpc/proto/builder.pb.go b/grpc/proto/builder.pb.go new file mode 100644 index 00000000..4690c81c --- /dev/null +++ b/grpc/proto/builder.pb.go @@ -0,0 +1,396 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: builder.proto + +/* +Package proto is a generated protocol buffer package. + +It is generated from these files: + builder.proto + +It has these top-level messages: + BuildParams + VerboseParams + Response +*/ +package proto + +import proto1 "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto1.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto1.ProtoPackageIsVersion2 // please upgrade the proto package + +type BuildParams struct { + HardwareFolders string `protobuf:"bytes,1,opt,name=hardwareFolders" json:"hardwareFolders,omitempty"` + ToolsFolders string `protobuf:"bytes,2,opt,name=toolsFolders" json:"toolsFolders,omitempty"` + BuiltInLibrariesFolders string `protobuf:"bytes,3,opt,name=builtInLibrariesFolders" json:"builtInLibrariesFolders,omitempty"` + OtherLibrariesFolders string `protobuf:"bytes,4,opt,name=otherLibrariesFolders" json:"otherLibrariesFolders,omitempty"` + SketchLocation string `protobuf:"bytes,5,opt,name=sketchLocation" json:"sketchLocation,omitempty"` + FQBN string `protobuf:"bytes,6,opt,name=fQBN" json:"fQBN,omitempty"` + ArduinoAPIVersion string `protobuf:"bytes,7,opt,name=arduinoAPIVersion" json:"arduinoAPIVersion,omitempty"` + CustomBuildProperties string `protobuf:"bytes,8,opt,name=customBuildProperties" json:"customBuildProperties,omitempty"` + BuildCachePath string `protobuf:"bytes,9,opt,name=buildCachePath" json:"buildCachePath,omitempty"` + BuildPath string `protobuf:"bytes,10,opt,name=buildPath" json:"buildPath,omitempty"` + WarningsLevel string `protobuf:"bytes,11,opt,name=warningsLevel" json:"warningsLevel,omitempty"` + CodeCompleteAt string `protobuf:"bytes,12,opt,name=codeCompleteAt" json:"codeCompleteAt,omitempty"` + Verbose bool `protobuf:"varint,13,opt,name=verbose" json:"verbose,omitempty"` +} + +func (m *BuildParams) Reset() { *m = BuildParams{} } +func (m *BuildParams) String() string { return proto1.CompactTextString(m) } +func (*BuildParams) ProtoMessage() {} +func (*BuildParams) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *BuildParams) GetHardwareFolders() string { + if m != nil { + return m.HardwareFolders + } + return "" +} + +func (m *BuildParams) GetToolsFolders() string { + if m != nil { + return m.ToolsFolders + } + return "" +} + +func (m *BuildParams) GetBuiltInLibrariesFolders() string { + if m != nil { + return m.BuiltInLibrariesFolders + } + return "" +} + +func (m *BuildParams) GetOtherLibrariesFolders() string { + if m != nil { + return m.OtherLibrariesFolders + } + return "" +} + +func (m *BuildParams) GetSketchLocation() string { + if m != nil { + return m.SketchLocation + } + return "" +} + +func (m *BuildParams) GetFQBN() string { + if m != nil { + return m.FQBN + } + return "" +} + +func (m *BuildParams) GetArduinoAPIVersion() string { + if m != nil { + return m.ArduinoAPIVersion + } + return "" +} + +func (m *BuildParams) GetCustomBuildProperties() string { + if m != nil { + return m.CustomBuildProperties + } + return "" +} + +func (m *BuildParams) GetBuildCachePath() string { + if m != nil { + return m.BuildCachePath + } + return "" +} + +func (m *BuildParams) GetBuildPath() string { + if m != nil { + return m.BuildPath + } + return "" +} + +func (m *BuildParams) GetWarningsLevel() string { + if m != nil { + return m.WarningsLevel + } + return "" +} + +func (m *BuildParams) GetCodeCompleteAt() string { + if m != nil { + return m.CodeCompleteAt + } + return "" +} + +func (m *BuildParams) GetVerbose() bool { + if m != nil { + return m.Verbose + } + return false +} + +type VerboseParams struct { + Verbose bool `protobuf:"varint,1,opt,name=verbose" json:"verbose,omitempty"` +} + +func (m *VerboseParams) Reset() { *m = VerboseParams{} } +func (m *VerboseParams) String() string { return proto1.CompactTextString(m) } +func (*VerboseParams) ProtoMessage() {} +func (*VerboseParams) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *VerboseParams) GetVerbose() bool { + if m != nil { + return m.Verbose + } + return false +} + +type Response struct { + Line string `protobuf:"bytes,1,opt,name=line" json:"line,omitempty"` +} + +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto1.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +func (m *Response) GetLine() string { + if m != nil { + return m.Line + } + return "" +} + +func init() { + proto1.RegisterType((*BuildParams)(nil), "proto.BuildParams") + proto1.RegisterType((*VerboseParams)(nil), "proto.VerboseParams") + proto1.RegisterType((*Response)(nil), "proto.Response") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Builder service + +type BuilderClient interface { + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + Build(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (Builder_BuildClient, error) + Autocomplete(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (*Response, error) + DropCache(ctx context.Context, in *VerboseParams, opts ...grpc.CallOption) (*Response, error) +} + +type builderClient struct { + cc *grpc.ClientConn +} + +func NewBuilderClient(cc *grpc.ClientConn) BuilderClient { + return &builderClient{cc} +} + +func (c *builderClient) Build(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (Builder_BuildClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Builder_serviceDesc.Streams[0], c.cc, "/proto.Builder/Build", opts...) + if err != nil { + return nil, err + } + x := &builderBuildClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Builder_BuildClient interface { + Recv() (*Response, error) + grpc.ClientStream +} + +type builderBuildClient struct { + grpc.ClientStream +} + +func (x *builderBuildClient) Recv() (*Response, error) { + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *builderClient) Autocomplete(ctx context.Context, in *BuildParams, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := grpc.Invoke(ctx, "/proto.Builder/Autocomplete", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *builderClient) DropCache(ctx context.Context, in *VerboseParams, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := grpc.Invoke(ctx, "/proto.Builder/DropCache", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Builder service + +type BuilderServer interface { + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + Build(*BuildParams, Builder_BuildServer) error + Autocomplete(context.Context, *BuildParams) (*Response, error) + DropCache(context.Context, *VerboseParams) (*Response, error) +} + +func RegisterBuilderServer(s *grpc.Server, srv BuilderServer) { + s.RegisterService(&_Builder_serviceDesc, srv) +} + +func _Builder_Build_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(BuildParams) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BuilderServer).Build(m, &builderBuildServer{stream}) +} + +type Builder_BuildServer interface { + Send(*Response) error + grpc.ServerStream +} + +type builderBuildServer struct { + grpc.ServerStream +} + +func (x *builderBuildServer) Send(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func _Builder_Autocomplete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BuildParams) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BuilderServer).Autocomplete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Builder/Autocomplete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BuilderServer).Autocomplete(ctx, req.(*BuildParams)) + } + return interceptor(ctx, in, info, handler) +} + +func _Builder_DropCache_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VerboseParams) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BuilderServer).DropCache(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Builder/DropCache", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BuilderServer).DropCache(ctx, req.(*VerboseParams)) + } + return interceptor(ctx, in, info, handler) +} + +var _Builder_serviceDesc = grpc.ServiceDesc{ + ServiceName: "proto.Builder", + HandlerType: (*BuilderServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Autocomplete", + Handler: _Builder_Autocomplete_Handler, + }, + { + MethodName: "DropCache", + Handler: _Builder_DropCache_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Build", + Handler: _Builder_Build_Handler, + ServerStreams: true, + }, + }, + Metadata: "builder.proto", +} + +func init() { proto1.RegisterFile("builder.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 420 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x93, 0x6f, 0x6b, 0x13, 0x41, + 0x10, 0xc6, 0x3d, 0x4d, 0x9a, 0x64, 0x9a, 0x58, 0x1c, 0x14, 0x17, 0x11, 0x29, 0xa1, 0x48, 0x04, + 0x09, 0x45, 0x2b, 0xf8, 0x36, 0x57, 0x11, 0x0a, 0x41, 0xce, 0xbc, 0xe8, 0xfb, 0xbd, 0xcd, 0xe8, + 0x2d, 0x5e, 0x6e, 0x8e, 0xdd, 0x4d, 0xfb, 0x59, 0xfc, 0x06, 0x7e, 0x4c, 0xd9, 0x3f, 0xd1, 0x5e, + 0x13, 0xc1, 0x57, 0x99, 0x3c, 0xcf, 0x6f, 0x6e, 0x9f, 0xdb, 0x99, 0x83, 0x49, 0xb9, 0xd5, 0xf5, + 0x9a, 0xcc, 0xbc, 0x35, 0xec, 0x18, 0xfb, 0xe1, 0x67, 0xfa, 0xb3, 0x07, 0xc7, 0xb9, 0x37, 0x0a, + 0x69, 0xe4, 0xc6, 0xe2, 0x0c, 0x4e, 0x2a, 0x69, 0xd6, 0xb7, 0xd2, 0xd0, 0x67, 0xf6, 0xb8, 0x15, + 0xd9, 0x69, 0x36, 0x1b, 0xad, 0xee, 0xcb, 0x38, 0x85, 0xb1, 0x63, 0xae, 0xed, 0x0e, 0x7b, 0x18, + 0xb0, 0x8e, 0x86, 0x1f, 0xe1, 0xb9, 0x3f, 0xd5, 0x5d, 0x35, 0x4b, 0x5d, 0x1a, 0x69, 0x34, 0xfd, + 0xc1, 0x1f, 0x05, 0xfc, 0x5f, 0x36, 0x5e, 0xc0, 0x33, 0x76, 0x15, 0x99, 0xbd, 0xbe, 0x5e, 0xe8, + 0x3b, 0x6c, 0xe2, 0x6b, 0x78, 0x6c, 0x7f, 0x90, 0x53, 0xd5, 0x92, 0x95, 0x74, 0x9a, 0x1b, 0xd1, + 0x0f, 0xf8, 0x3d, 0x15, 0x11, 0x7a, 0xdf, 0xbe, 0xe6, 0x5f, 0xc4, 0x51, 0x70, 0x43, 0x8d, 0x6f, + 0xe1, 0x89, 0x34, 0xeb, 0xad, 0x6e, 0x78, 0x51, 0x5c, 0x5d, 0x93, 0xb1, 0xbe, 0x7d, 0x10, 0x80, + 0x7d, 0xc3, 0xe7, 0x53, 0x5b, 0xeb, 0x78, 0x13, 0x2f, 0xcf, 0x70, 0x4b, 0xc6, 0x69, 0xb2, 0x62, + 0x18, 0xf3, 0x1d, 0x34, 0x7d, 0xbe, 0x30, 0x85, 0x4b, 0xa9, 0x2a, 0x2a, 0xa4, 0xab, 0xc4, 0x28, + 0xe6, 0xeb, 0xaa, 0xf8, 0x12, 0x46, 0x65, 0x1c, 0x8a, 0xab, 0x04, 0x04, 0xe4, 0xaf, 0x80, 0x67, + 0x30, 0xb9, 0x95, 0xa6, 0xd1, 0xcd, 0x77, 0xbb, 0xa4, 0x1b, 0xaa, 0xc5, 0x71, 0x20, 0xba, 0xa2, + 0x3f, 0x4b, 0xf1, 0x9a, 0x2e, 0x79, 0xd3, 0xd6, 0xe4, 0x68, 0xe1, 0xc4, 0x38, 0x9e, 0xd5, 0x55, + 0x51, 0xc0, 0xe0, 0x86, 0x4c, 0xc9, 0x96, 0xc4, 0xe4, 0x34, 0x9b, 0x0d, 0x57, 0xbb, 0xbf, 0xd3, + 0x37, 0x30, 0xb9, 0x8e, 0x65, 0x5a, 0x8e, 0x3b, 0x68, 0xd6, 0x45, 0x5f, 0xc1, 0x70, 0x45, 0xb6, + 0xe5, 0xc6, 0x92, 0xbf, 0xdc, 0x5a, 0x37, 0x94, 0xf6, 0x26, 0xd4, 0xef, 0x7e, 0x65, 0x30, 0xc8, + 0xe3, 0xfe, 0xe1, 0x39, 0xf4, 0x43, 0x89, 0x18, 0x57, 0x71, 0x7e, 0x67, 0xff, 0x5e, 0x9c, 0x24, + 0x6d, 0xf7, 0xb4, 0xe9, 0x83, 0xf3, 0x0c, 0x3f, 0xc0, 0x78, 0xb1, 0x75, 0xac, 0x52, 0xe8, 0xff, + 0x6c, 0xc4, 0x0b, 0x18, 0x7d, 0x32, 0xdc, 0x86, 0x6b, 0xc5, 0xa7, 0xc9, 0xef, 0xbc, 0xd1, 0x81, + 0xae, 0xfc, 0x0c, 0x50, 0xa9, 0x79, 0x9a, 0xf8, 0x3c, 0x7d, 0x34, 0xf9, 0x38, 0xa5, 0x2f, 0x3c, + 0x5e, 0x64, 0xe5, 0x51, 0xe8, 0x7b, 0xff, 0x3b, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x39, 0x66, 0x51, + 0x56, 0x03, 0x00, 0x00, +} diff --git a/grpc/proto/builder.proto b/grpc/proto/builder.proto new file mode 100644 index 00000000..9b310284 --- /dev/null +++ b/grpc/proto/builder.proto @@ -0,0 +1,63 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// compile me with: protoc -I proto/ proto/builder.proto --go_out=plugins=grpc:proto + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "cc.arduino.builder"; +option java_outer_classname = "BuilderProto"; + +package proto; + +// Interface exported by the server. +service Builder { + + // A server-to-client streaming RPC. + // + // Obtains the Features available within the given Rectangle. Results are + // streamed rather than returned at once (e.g. in a response message with a + // repeated field), as the rectangle may cover a large area and contain a + // huge number of features. + rpc Build(BuildParams) returns (stream Response) {} + + rpc Autocomplete(BuildParams) returns (Response) {} + + rpc DropCache(VerboseParams) returns (Response) {} +} + +message BuildParams { + string hardwareFolders = 1; + string toolsFolders = 2; + string builtInLibrariesFolders = 3; + string otherLibrariesFolders = 4; + string sketchLocation = 5; + string fQBN = 6; + string arduinoAPIVersion = 7; + string customBuildProperties = 8; + string buildCachePath = 9; + string buildPath = 10; + string warningsLevel = 11; + string codeCompleteAt = 12; + bool verbose = 13; +} + +message VerboseParams { + bool verbose = 1; +} + +message Response { + string line = 1; +} diff --git a/grpc/rpc.go b/grpc/rpc.go new file mode 100644 index 00000000..55cd78ff --- /dev/null +++ b/grpc/rpc.go @@ -0,0 +1,224 @@ +package jsonrpc + +import ( + "fmt" + "io" + "log" + "net" + "os" + "strings" + + builder "github.com/arduino/arduino-builder" + "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" + "github.com/fsnotify/fsnotify" + "golang.org/x/net/context" + "google.golang.org/grpc" + + pb "github.com/arduino/arduino-builder/grpc/proto" +) + +type StreamLogger struct { + stream pb.Builder_BuildServer +} + +func (s StreamLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + s.stream.Send(&pb.Response{Line: fmt.Sprintf(format, a...)}) +} + +func (s StreamLogger) UnformattedFprintln(w io.Writer, str string) { + s.stream.Send(&pb.Response{Line: str}) +} + +func (s StreamLogger) UnformattedWrite(w io.Writer, data []byte) { + s.stream.Send(&pb.Response{Line: string(data)}) +} + +func (s StreamLogger) Println(level string, format string, a ...interface{}) { + s.stream.Send(&pb.Response{Line: fmt.Sprintf(format, a...)}) +} + +func (s StreamLogger) Flush() string { + return "" +} + +func (s StreamLogger) Name() string { + return "streamlogger" +} + +type builderServer struct { + resp []*pb.Response + ctx *types.Context + watcher *fsnotify.Watcher +} + +func (s *builderServer) watch() { + folders := [][]string{s.ctx.HardwareFolders, s.ctx.ToolsFolders, s.ctx.BuiltInLibrariesFolders, s.ctx.OtherLibrariesFolders} + + for _, category := range folders { + for _, folder := range category { + if !utils.SliceContains(s.ctx.WatchedLocations, folder) { + var subfolders []string + utils.FindAllSubdirectories(folder, &subfolders) + subfolders = append(subfolders, folder) + for _, element := range subfolders { + s.watcher.Add(element) + s.ctx.WatchedLocations = utils.AppendIfNotPresent(s.ctx.WatchedLocations, element) + } + } + } + } +} + +func (s *builderServer) DropCache(ctx context.Context, args *pb.VerboseParams) (*pb.Response, error) { + s.ctx.CanUseCachedTools = false + response := pb.Response{Line: "Tools cache dropped"} + return &response, nil +} + +// GetFeature returns the feature at the given point. +func (s *builderServer) Autocomplete(ctx context.Context, args *pb.BuildParams) (*pb.Response, error) { + + s.ctx.HardwareFolders = strings.Split(args.HardwareFolders, ",") + s.ctx.ToolsFolders = strings.Split(args.ToolsFolders, ",") + s.ctx.BuiltInLibrariesFolders = strings.Split(args.BuiltInLibrariesFolders, ",") + s.ctx.OtherLibrariesFolders = strings.Split(args.OtherLibrariesFolders, ",") + s.ctx.SketchLocation = args.SketchLocation + s.ctx.CustomBuildProperties = strings.Split(args.CustomBuildProperties, ",") + s.ctx.ArduinoAPIVersion = args.ArduinoAPIVersion + s.ctx.FQBN = args.FQBN + s.ctx.Verbose = false //p.Verbose + s.ctx.BuildCachePath = args.BuildCachePath + s.ctx.BuildPath = args.BuildPath + s.ctx.WarningsLevel = args.WarningsLevel + s.ctx.PrototypesSection = "" + s.ctx.CodeCompleteAt = args.CodeCompleteAt + s.ctx.CodeCompletions = "" + + s.ctx.IncludeFolders = s.ctx.IncludeFolders[0:0] + s.ctx.LibrariesObjectFiles = s.ctx.LibrariesObjectFiles[0:0] + s.ctx.CoreObjectsFiles = s.ctx.CoreObjectsFiles[0:0] + s.ctx.SketchObjectFiles = s.ctx.SketchObjectFiles[0:0] + + s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0] + + //s.watch() + + oldlogger := s.ctx.GetLogger() + logger := i18n.NoopLogger{} + s.ctx.SetLogger(logger) + + err := builder.RunPreprocess(s.ctx) + + response := pb.Response{Line: s.ctx.CodeCompletions} + s.ctx.SetLogger(oldlogger) + + return &response, err +} + +// GetFeature returns the feature at the given point. +func (s *builderServer) Build(args *pb.BuildParams, stream pb.Builder_BuildServer) error { + + s.ctx.HardwareFolders = strings.Split(args.HardwareFolders, ",") + s.ctx.ToolsFolders = strings.Split(args.ToolsFolders, ",") + s.ctx.BuiltInLibrariesFolders = strings.Split(args.BuiltInLibrariesFolders, ",") + s.ctx.OtherLibrariesFolders = strings.Split(args.OtherLibrariesFolders, ",") + s.ctx.SketchLocation = args.SketchLocation + s.ctx.CustomBuildProperties = strings.Split(args.CustomBuildProperties, ",") + s.ctx.ArduinoAPIVersion = args.ArduinoAPIVersion + s.ctx.FQBN = args.FQBN + s.ctx.Verbose = args.Verbose + s.ctx.BuildCachePath = args.BuildCachePath + s.ctx.BuildPath = args.BuildPath + s.ctx.WarningsLevel = args.WarningsLevel + s.ctx.PrototypesSection = "" + s.ctx.CodeCompleteAt = "" + + s.ctx.IncludeFolders = s.ctx.IncludeFolders[0:0] + s.ctx.LibrariesObjectFiles = s.ctx.LibrariesObjectFiles[0:0] + s.ctx.CoreObjectsFiles = s.ctx.CoreObjectsFiles[0:0] + s.ctx.SketchObjectFiles = s.ctx.SketchObjectFiles[0:0] + + s.ctx.ImportedLibraries = s.ctx.ImportedLibraries[0:0] + + // setup logger to send via protobuf + oldlogger := s.ctx.GetLogger() + logger := StreamLogger{stream} + s.ctx.SetLogger(logger) + + //s.watch() + + err := builder.RunBuilder(s.ctx) + s.ctx.SetLogger(oldlogger) + if err != nil { + return err + } + + // No feature was found, return an unnamed feature + return nil +} + +/* +func (h *WatchHandler) ServeJSONRPC(c context.Context, params *json.RawMessage) (interface{}, *jsonrpc.Error) { + + var p WatchParams + if err := jsonrpc.Unmarshal(params, &p); err != nil { + return nil, err + } + + err := h.watcher.Add(p.Path) + if err != nil { + return nil, jsonrpc.ErrInvalidParams() + } + return BuildResult{ + Message: "OK " + p.Path, + }, nil +} +*/ + +func startWatching(ctx *types.Context) *fsnotify.Watcher { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + + go func() { + for { + select { + case event := <-watcher.Events: + ctx.CanUseCachedTools = false + log.Println("event:", event) + case err := <-watcher.Errors: + log.Println("error:", err) + } + } + }() + return watcher +} + +func newServerWithWatcher(ctx *types.Context, watcher *fsnotify.Watcher) *builderServer { + s := new(builderServer) + s.ctx = ctx + s.watcher = watcher + return s +} + +func newServer(ctx *types.Context) *builderServer { + s := new(builderServer) + s.ctx = ctx + return s +} + +func RegisterAndServeJsonRPC(ctx *types.Context) { + lis, err := net.Listen("tcp", "localhost:12345") + if err != nil { + //can't spawn two grpc servers on the same port + os.Exit(0) + } + //watcher := startWatching(ctx) + + grpcServer := grpc.NewServer() + pb.RegisterBuilderServer(grpcServer, newServer(ctx)) + grpcServer.Serve(lis) +} diff --git a/hardware/platform.txt b/hardware/platform.txt index ca8df1f7..144df2fb 100644 --- a/hardware/platform.txt +++ b/hardware/platform.txt @@ -1,3 +1,10 @@ +# arduino-preprocessor +# -------------------- + +tools.arduino-preprocessor.path={runtime.tools.arduino-preprocessor.path} +tools.arduino-preprocessor.cmd.path={path}/arduino-preprocessor +tools.arduino-preprocessor.pattern="{cmd.path}" "{source_file}" "{codecomplete}" -- -std=gnu++11 + # ctags # ------------------------------ tools.ctags.path={runtime.tools.ctags.path} diff --git a/i18n/errors.go b/i18n/errors.go index ba2a86ea..b3617800 100644 --- a/i18n/errors.go +++ b/i18n/errors.go @@ -18,3 +18,14 @@ func WrapError(err error) error { } return errors.Wrap(err, 0) } + +func UnwrapError(err error) error { + // Perhaps go-errors can do this already in later versions? + // See https://github.com/go-errors/errors/issues/14 + switch e := err.(type) { + case *errors.Error: + return e.Err + default: + return err + } +} diff --git a/i18n/i18n.go b/i18n/i18n.go index ce85d22c..202d51c7 100644 --- a/i18n/i18n.go +++ b/i18n/i18n.go @@ -38,37 +38,91 @@ import ( "regexp" "strconv" "strings" + "sync" ) var PLACEHOLDER = regexp.MustCompile("{(\\d)}") type Logger interface { Fprintln(w io.Writer, level string, format string, a ...interface{}) + UnformattedFprintln(w io.Writer, s string) + UnformattedWrite(w io.Writer, data []byte) Println(level string, format string, a ...interface{}) Name() string + Flush() string } type NoopLogger struct{} func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {} +func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {} + +func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {} + func (s NoopLogger) Println(level string, format string, a ...interface{}) {} +func (s NoopLogger) Flush() string { + return "" +} + func (s NoopLogger) Name() string { return "noop" } +type AccumulatorLogger struct { + Buffer *[]string +} + +func (s AccumulatorLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + *s.Buffer = append(*s.Buffer, Format(format, a...)) +} + +func (s AccumulatorLogger) UnformattedFprintln(w io.Writer, str string) { + *s.Buffer = append(*s.Buffer, str) +} + +func (s AccumulatorLogger) UnformattedWrite(w io.Writer, data []byte) { + *s.Buffer = append(*s.Buffer, string(data)) +} + +func (s AccumulatorLogger) Println(level string, format string, a ...interface{}) { + s.Fprintln(nil, level, format, a...) +} + +func (s AccumulatorLogger) Flush() string { + str := strings.Join(*s.Buffer, "\n") + *s.Buffer = (*s.Buffer)[0:0] + return str +} + +func (s AccumulatorLogger) Name() string { + return "accumulator" +} + type HumanTagsLogger struct{} func (s HumanTagsLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { format = "[" + level + "] " + format - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) } func (s HumanTagsLogger) Println(level string, format string, a ...interface{}) { s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanTagsLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + +func (s HumanTagsLogger) Flush() string { + return "" +} + func (s HumanTagsLogger) Name() string { return "humantags" } @@ -76,20 +130,56 @@ func (s HumanTagsLogger) Name() string { type HumanLogger struct{} func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) } func (s HumanLogger) Println(level string, format string, a ...interface{}) { s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + +func (s HumanLogger) Flush() string { + return "" +} + func (s HumanLogger) Name() string { return "human" } type MachineLogger struct{} -func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) { +func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + printMachineFormattedLogLine(w, level, format, a) +} + +func (s MachineLogger) Println(level string, format string, a ...interface{}) { + printMachineFormattedLogLine(os.Stdout, level, format, a) +} + +func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s MachineLogger) Flush() string { + return "" +} + +func (s MachineLogger) Name() string { + return "machine" +} + +func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + +func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) { a = append([]interface{}(nil), a...) for idx, value := range a { typeof := reflect.Indirect(reflect.ValueOf(value)).Kind() @@ -97,20 +187,27 @@ func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format a[idx] = url.QueryEscape(value.(string)) } } - fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a) - fmt.Fprintln(w) + fprintf(w, "===%s ||| %s ||| %s\n", level, format, a) } -func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - s.printWithoutFormatting(w, level, format, a) +var lock sync.Mutex + +func fprintln(w io.Writer, s string) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintln(w, s) } -func (s MachineLogger) Println(level string, format string, a ...interface{}) { - s.printWithoutFormatting(os.Stdout, level, format, a) +func write(w io.Writer, data []byte) { + lock.Lock() + defer lock.Unlock() + w.Write(data) } -func (s MachineLogger) Name() string { - return "machine" +func fprintf(w io.Writer, format string, a ...interface{}) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintf(w, format, a...) } func FromJavaToGoSyntax(s string) string { diff --git a/includes_finder_with_regexp.go b/includes_finder_with_regexp.go index 56ac955e..abfe7635 100644 --- a/includes_finder_with_regexp.go +++ b/includes_finder_with_regexp.go @@ -37,21 +37,13 @@ import ( var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") -type IncludesFinderWithRegExp struct { - Source *string -} - -func (s *IncludesFinderWithRegExp) Run(ctx *types.Context) error { - source := *s.Source - +func IncludesFinderWithRegExp(ctx *types.Context, source string) string { match := INCLUDE_REGEXP.FindStringSubmatch(source) if match != nil { - ctx.IncludeJustFound = strings.TrimSpace(match[1]) + return strings.TrimSpace(match[1]) } else { - ctx.IncludeJustFound = findIncludeForOldCompilers(source) + return findIncludeForOldCompilers(source) } - - return nil } func findIncludeForOldCompilers(source string) string { diff --git a/phases/core_builder.go b/phases/core_builder.go index 75b6535d..cecea3de 100644 --- a/phases/core_builder.go +++ b/phases/core_builder.go @@ -46,9 +46,6 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { coreBuildPath := ctx.CoreBuildPath coreBuildCachePath := ctx.CoreBuildCachePath buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(coreBuildPath) if err != nil { @@ -62,7 +59,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { } } - archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger) + archiveFile, objectFiles, err := compileCore(ctx, coreBuildPath, coreBuildCachePath, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -73,7 +70,8 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return nil } -func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { +func compileCore(ctx *types.Context, buildPath string, buildCachePath string, buildProperties properties.Map) (string, []string, error) { + logger := ctx.GetLogger() coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] @@ -90,7 +88,7 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper variantObjectFiles := []string{} if variantFolder != constants.EMPTY_STRING { - variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + variantObjectFiles, err = builder_utils.CompileFiles(ctx, variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes) if err != nil { return "", nil, i18n.WrapError(err) } @@ -107,26 +105,26 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper if canUseArchivedCore { // use archived core - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core: {0}", targetArchivedCore) } return targetArchivedCore, variantObjectFiles, nil } } - coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + coreObjectFiles, err := builder_utils.CompileFiles(ctx, []string{}, coreFolder, true, buildPath, buildProperties, includes) if err != nil { return "", nil, i18n.WrapError(err) } - archiveFile, err := builder_utils.ArchiveCompiledFiles(buildPath, "core.a", coreObjectFiles, buildProperties, verbose, logger) + archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, "core.a", coreObjectFiles, buildProperties) if err != nil { return "", nil, i18n.WrapError(err) } // archive core.a if targetArchivedCore != "" { - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_ARCHIVING_CORE_CACHE, targetArchivedCore) } builder_utils.CopyFile(archiveFile, targetArchivedCore) diff --git a/phases/libraries_builder.go b/phases/libraries_builder.go index fda56b80..dc0fdf12 100644 --- a/phases/libraries_builder.go +++ b/phases/libraries_builder.go @@ -52,16 +52,13 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { includes := ctx.IncludeFolders includes = utils.Map(includes, utils.WrapWithHyphenI) libraries := ctx.ImportedLibraries - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(librariesBuildPath) if err != nil { return i18n.WrapError(err) } - objectFiles, err := compileLibraries(libraries, librariesBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err := compileLibraries(ctx, libraries, librariesBuildPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } @@ -93,16 +90,16 @@ func fixLDFLAGforPrecompiledLibraries(ctx *types.Context, libraries []*types.Lib name = strings.Replace(name, "lib", "", 1) libs_cmd += "-l" + name + " " } - ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_EXTRAFLAGS] += "\"-L" + path + "\" " + libs_cmd + ctx.BuildProperties[constants.BUILD_PROPERTIES_COMPILER_LIBRARIES_LDFLAGS] += "\"-L" + path + "\" " + libs_cmd + " " } } return nil } -func compileLibraries(libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileLibraries(ctx *types.Context, libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { objectFiles := []string{} for _, library := range libraries { - libraryObjectFiles, err := compileLibrary(library, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + libraryObjectFiles, err := compileLibrary(ctx, library, buildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -113,8 +110,9 @@ func compileLibraries(libraries []*types.Library, buildPath string, buildPropert } -func compileLibrary(library *types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - if verbose { +func compileLibrary(ctx *types.Context, library *types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + logger := ctx.GetLogger() + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, "Compiling library \"{0}\"", library.Name) } libraryBuildPath := filepath.Join(buildPath, library.Name) @@ -144,12 +142,12 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr } if library.Layout == types.LIBRARY_RECURSIVE { - objectFiles, err = builder_utils.CompileFilesRecursive(objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFilesRecursive(ctx, objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } if library.DotALinkage { - archiveFile, err := builder_utils.ArchiveCompiledFiles(libraryBuildPath, library.Name+".a", objectFiles, buildProperties, verbose, logger) + archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, libraryBuildPath, library.Name+".a", objectFiles, buildProperties) if err != nil { return nil, i18n.WrapError(err) } @@ -159,14 +157,14 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr if library.UtilityFolder != "" { includes = append(includes, utils.WrapWithHyphenI(library.UtilityFolder)) } - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } if library.UtilityFolder != "" { utilityBuildPath := filepath.Join(libraryBuildPath, constants.LIBRARY_FOLDER_UTILITY) - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } diff --git a/phases/linker.go b/phases/linker.go index d35f4993..2514b406 100644 --- a/phases/linker.go +++ b/phases/linker.go @@ -61,11 +61,8 @@ func (s *Linker) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() - err = link(objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties, verbose, warningsLevel, logger) + err = link(ctx, objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -73,7 +70,7 @@ func (s *Linker) Run(ctx *types.Context) error { return nil } -func link(objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error { +func link(ctx *types.Context, objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map) error { optRelax := addRelaxTrickIfATMEGA2560(buildProperties) objectFiles = utils.Map(objectFiles, wrapWithDoubleQuotes) @@ -81,12 +78,12 @@ func link(objectFiles []string, coreDotARelPath string, coreArchiveFilePath stri properties := buildProperties.Clone() properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] + optRelax - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = coreDotARelPath properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = coreArchiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILES] = objectFileList - _, err := builder_utils.ExecRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, verbose, verbose, logger) + _, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_C_COMBINE_PATTERN, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) return err } diff --git a/phases/sizer.go b/phases/sizer.go index 7ea86997..fa719e18 100644 --- a/phases/sizer.go +++ b/phases/sizer.go @@ -38,6 +38,7 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" ) @@ -52,11 +53,8 @@ func (s *Sizer) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() - err := checkSize(buildProperties, verbose, warningsLevel, logger) + err := checkSize(ctx, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -64,10 +62,11 @@ func (s *Sizer) Run(ctx *types.Context) error { return nil } -func checkSize(buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error { +func checkSize(ctx *types.Context, buildProperties properties.Map) error { + logger := ctx.GetLogger() properties := buildProperties.Clone() - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] maxTextSizeString := properties[constants.PROPERTY_UPLOAD_MAX_SIZE] maxDataSizeString := properties[constants.PROPERTY_UPLOAD_MAX_DATA_SIZE] @@ -89,7 +88,7 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin } } - textSize, dataSize, _, err := execSizeReceipe(properties, logger) + textSize, dataSize, _, err := execSizeRecipe(ctx, properties) if err != nil { logger.Println(constants.LOG_LEVEL_WARN, constants.MSG_SIZER_ERROR_NO_RULE) return nil @@ -127,8 +126,8 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin return nil } -func execSizeReceipe(properties properties.Map, logger i18n.Logger) (textSize int, dataSize int, eepromSize int, resErr error) { - out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, false, false, logger) +func execSizeRecipe(ctx *types.Context, properties properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { + out, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_SIZE_PATTERN, false /* stdout */, utils.Capture /* stderr */, utils.Show) if err != nil { resErr = errors.New("Error while determining sketch size: " + err.Error()) return diff --git a/phases/sketch_builder.go b/phases/sketch_builder.go index 04c7ded7..8a3158d9 100644 --- a/phases/sketch_builder.go +++ b/phases/sketch_builder.go @@ -46,9 +46,6 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { buildProperties := ctx.BuildProperties includes := ctx.IncludeFolders includes = utils.Map(includes, utils.WrapWithHyphenI) - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(sketchBuildPath) if err != nil { @@ -56,7 +53,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { } var objectFiles []string - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } @@ -64,7 +61,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := filepath.Join(sketchBuildPath, constants.SKETCH_FOLDER_SRC) if info, err := os.Stat(sketchSrcPath); err == nil && info.IsDir() { - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } diff --git a/preprocess_sketch.go b/preprocess_sketch.go new file mode 100644 index 00000000..d36c04c5 --- /dev/null +++ b/preprocess_sketch.go @@ -0,0 +1,152 @@ +/* + * This file is part of Arduino Builder. + * + * Arduino Builder is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + * + * Copyright 2015 Arduino LLC (http://www.arduino.cc/) + */ + +package builder + +import ( + "errors" + "fmt" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" +) + +type PreprocessSketchArduino struct{} + +func (s *PreprocessSketchArduino) Run(ctx *types.Context) error { + sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") + commands := []types.Command{ + &ArduinoPreprocessorRunner{}, + } + + err := utils.EnsureFolderExists(ctx.PreprocPath) + if err != nil { + return i18n.WrapError(err) + } + + if ctx.CodeCompleteAt != "" { + commands = append(commands, &OutputCodeCompletions{}) + } else { + commands = append(commands, &SketchSaver{}) + } + + GCCPreprocRunner(ctx, sourceFile, filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), ctx.IncludeFolders) + + for _, command := range commands { + PrintRingNameIfDebug(ctx, command) + err := command.Run(ctx) + if err != nil { + return i18n.WrapError(err) + } + } + + return nil +} + +type ArduinoPreprocessorRunner struct{} + +func (s *ArduinoPreprocessorRunner) Run(ctx *types.Context) error { + buildProperties := ctx.BuildProperties + targetFilePath := filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E) + logger := ctx.GetLogger() + + properties := buildProperties.Clone() + toolProps := buildProperties.SubTree("tools").SubTree("arduino-preprocessor") + properties.Merge(toolProps) + properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = targetFilePath + if ctx.CodeCompleteAt != "" { + if runtime.GOOS == "windows" { + //use relative filepath to avoid ":" escaping + splt := strings.Split(ctx.CodeCompleteAt, ":") + if len(splt) == 3 { + //all right, do nothing + } else { + splt[1] = filepath.Base(splt[0] + ":" + splt[1]) + ctx.CodeCompleteAt = strings.Join(splt[1:], ":") + } + } + properties["codecomplete"] = "-output-code-completions=" + ctx.CodeCompleteAt + } else { + properties["codecomplete"] = "" + } + + pattern := properties[constants.BUILD_PROPERTIES_PATTERN] + if pattern == constants.EMPTY_STRING { + return i18n.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, "arduino-preprocessor") + } + + commandLine := properties.ExpandPropsInString(pattern) + command, err := utils.PrepareCommand(commandLine, logger, "") + if err != nil { + return i18n.WrapError(err) + } + + if runtime.GOOS == "windows" { + // chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2) + command.Dir = filepath.VolumeName(command.Args[0]) + "/" + //command.Args[0], _ = filepath.Rel(command.Dir, command.Args[0]) + } + + verbose := ctx.Verbose + if verbose { + fmt.Println(commandLine) + } + + buf, err := command.Output() + if err != nil { + return errors.New(i18n.WrapError(err).Error() + string(err.(*exec.ExitError).Stderr)) + } + + result := utils.NormalizeUTF8(buf) + + //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output) + if ctx.CodeCompleteAt != "" { + ctx.CodeCompletions = string(result) + } else { + ctx.Source = string(result) + } + return nil +} + +type OutputCodeCompletions struct{} + +func (s *OutputCodeCompletions) Run(ctx *types.Context) error { + if ctx.CodeCompletions == "" { + // we assume it is a json, let's make it compliant at least + ctx.CodeCompletions = "[]" + } + ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, ctx.CodeCompletions) + return nil +} diff --git a/print_preprocessed_source.go b/print_preprocessed_source.go index c6afae14..b320c127 100644 --- a/print_preprocessed_source.go +++ b/print_preprocessed_source.go @@ -30,13 +30,16 @@ package builder import ( - "github.com/arduino/arduino-builder/types" "fmt" + + "github.com/arduino/arduino-builder/types" ) type PrintPreprocessedSource struct{} func (s *PrintPreprocessedSource) Run(ctx *types.Context) error { - fmt.Println(ctx.SourceGccMinusE) + if ctx.SourceGccMinusE != "" { + fmt.Println(ctx.SourceGccMinusE) + } return nil } diff --git a/read_file_and_store_in_context.go b/read_file_and_store_in_context.go index a2533929..bd165617 100644 --- a/read_file_and_store_in_context.go +++ b/read_file_and_store_in_context.go @@ -36,11 +36,12 @@ import ( ) type ReadFileAndStoreInContext struct { + FileToRead string Target *string } func (s *ReadFileAndStoreInContext) Run(ctx *types.Context) error { - bytes, err := ioutil.ReadFile(ctx.FileToRead) + bytes, err := ioutil.ReadFile(s.FileToRead) if err != nil { return i18n.WrapError(err) } diff --git a/recipe_runner.go b/recipe_runner.go index bec23f90..bb1bb962 100644 --- a/recipe_runner.go +++ b/recipe_runner.go @@ -30,13 +30,15 @@ package builder import ( + "os" + "sort" + "strings" + "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" - "os" - "sort" - "strings" + "github.com/arduino/arduino-builder/utils" ) type RecipeByPrefixSuffixRunner struct { @@ -51,8 +53,6 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties.Clone() - verbose := ctx.Verbose - recipes := findRecipes(buildProperties, s.Prefix, s.Suffix) properties := buildProperties.Clone() @@ -60,7 +60,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { if ctx.DebugLevel >= 10 { logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_RECIPE, recipe) } - _, err := builder_utils.ExecRecipe(properties, recipe, false, verbose, verbose, logger) + _, _, err := builder_utils.ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return i18n.WrapError(err) } diff --git a/setup_build_properties.go b/setup_build_properties.go index b18d643a..c1983551 100644 --- a/setup_build_properties.go +++ b/setup_build_properties.go @@ -30,6 +30,7 @@ package builder import ( + "os" "path/filepath" "strconv" "strings" @@ -64,12 +65,20 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error { } buildProperties[constants.BUILD_PROPERTIES_BUILD_ARCH] = strings.ToUpper(targetPlatform.PlatformId) + // get base folder and use it to populate BUILD_PROPERTIES_RUNTIME_IDE_PATH (arduino and arduino-builder live in the same dir) + ex, err := os.Executable() + exPath := "" + if err == nil { + exPath = filepath.Dir(ex) + } + buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE] = ctx.BuildCore buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] = filepath.Join(actualPlatform.Folder, constants.FOLDER_CORES, buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE]) buildProperties[constants.BUILD_PROPERTIES_BUILD_SYSTEM_PATH] = filepath.Join(actualPlatform.Folder, constants.FOLDER_SYSTEM) buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] = targetPlatform.Folder buildProperties[constants.BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH] = filepath.Join(targetPlatform.Folder, "..") buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = ctx.ArduinoAPIVersion + buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_PATH] = exPath buildProperties[constants.BUILD_PROPERTIES_FQBN] = ctx.FQBN buildProperties[constants.IDE_VERSION] = ctx.ArduinoAPIVersion buildProperties[constants.BUILD_PROPERTIES_RUNTIME_OS] = utils.PrettyOSName() diff --git a/test/builder_utils_test.go b/test/builder_utils_test.go index c6fc941c..ef07ddf0 100644 --- a/test/builder_utils_test.go +++ b/test/builder_utils_test.go @@ -30,13 +30,15 @@ package test import ( - "github.com/arduino/arduino-builder/builder_utils" - "github.com/arduino/arduino-builder/utils" - "github.com/stretchr/testify/require" "io/ioutil" "os" "testing" "time" + + "github.com/arduino/arduino-builder/builder_utils" + "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" + "github.com/stretchr/testify/require" ) func sleep(t *testing.T) { @@ -52,27 +54,33 @@ func tempFile(t *testing.T, prefix string) string { } func TestObjFileIsUpToDateObjMissing(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, "", "") + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, "", "") NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateDepMissing(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) objFile := tempFile(t, "obj") defer os.RemoveAll(objFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, "") + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, "") NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateObjOlder(t *testing.T) { + ctx := &types.Context{} + objFile := tempFile(t, "obj") defer os.RemoveAll(objFile) depFile := tempFile(t, "dep") @@ -83,12 +91,14 @@ func TestObjFileIsUpToDateObjOlder(t *testing.T) { sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateObjNewer(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -99,12 +109,14 @@ func TestObjFileIsUpToDateObjNewer(t *testing.T) { depFile := tempFile(t, "dep") defer os.RemoveAll(depFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -122,12 +134,14 @@ func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -143,12 +157,14 @@ func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -166,7 +182,7 @@ func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { utils.WriteFile(depFile, sourceFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } diff --git a/test/create_build_options_map_test.go b/test/create_build_options_map_test.go index dce062e8..d3fde15e 100644 --- a/test/create_build_options_map_test.go +++ b/test/create_build_options_map_test.go @@ -54,6 +54,7 @@ func TestCreateBuildOptionsMap(t *testing.T) { NoError(t, err) require.Equal(t, "{\n"+ + " \"additionalFiles\": \"\",\n"+ " \"builtInLibrariesFolders\": \"\",\n"+ " \"customBuildProperties\": \"\",\n"+ " \"fqbn\": \"fqbn\",\n"+ diff --git a/test/helper_tools_downloader.go b/test/helper_tools_downloader.go index cfce881c..db6dc9dc 100644 --- a/test/helper_tools_downloader.go +++ b/test/helper_tools_downloader.go @@ -112,7 +112,17 @@ func DownloadCoresAndToolsAndLibraries(t *testing.T) { OsUrl{Os: "i686-mingw32", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-i686-mingw32.zip"}, OsUrl{Os: "x86_64-apple-darwin", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-x86_64-apple-darwin.zip"}, OsUrl{Os: "arm-linux-gnueabihf", Url: "http://downloads.arduino.cc/tools/ctags-5.8-arduino11-armv6-linux-gnueabihf.tar.bz2"}, - }}, + }, + }, + Tool{Name: "arduino-preprocessor", Version: "0.1.5", + OsUrls: []OsUrl{ + OsUrl{Os: "i686-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-i686-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "x86_64-pc-linux-gnu", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-x86_64-pc-linux-gnu.tar.bz2"}, + OsUrl{Os: "i686-mingw32", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-i686-w64-mingw32.tar.bz2"}, + OsUrl{Os: "x86_64-apple-darwin", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-x86_64-apple-darwin11.tar.bz2"}, + OsUrl{Os: "arm-linux-gnueabihf", Url: "https://github.com/arduino/arduino-preprocessor/releases/download/0.1.5/arduino-preprocessor-0.1.5-arm-linux-gnueabihf.tar.bz2"}, + }, + }, } boardsManagerTools := []Tool{ @@ -698,9 +708,9 @@ func translateGOOSGOARCHToPackageIndexValue() []string { case "linux-386": return []string{"i686-pc-linux-gnu", "i686-linux-gnu"} case "windows-amd64": - return []string{"i686-mingw32"} + return []string{"i686-mingw32", "i686-cygwin"} case "windows-386": - return []string{"i686-mingw32"} + return []string{"i686-mingw32", "i686-cygwin"} case "darwin-amd64": return []string{"i386-apple-darwin11", "x86_64-apple-darwin"} case "linux-arm": diff --git a/test/includes_finder_with_regexp_test.go b/test/includes_finder_with_regexp_test.go index d95153a5..c3c048cc 100644 --- a/test/includes_finder_with_regexp_test.go +++ b/test/includes_finder_with_regexp_test.go @@ -43,27 +43,17 @@ func TestIncludesFinderWithRegExp(t *testing.T) { "#include \n" + "^\n" + "compilation terminated." - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, output) - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "SPI.h", ctx.IncludeJustFound) + require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpEmptyOutput(t *testing.T) { ctx := &types.Context{} - output := "" - - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, "") - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "", ctx.IncludeJustFound) + require.Equal(t, "", include) } func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { @@ -73,13 +63,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { " # include \n" + " ^\n" + "compilation terminated.\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "Wire.h", ctx.IncludeJustFound) + require.Equal(t, "Wire.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { @@ -89,13 +75,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { " #\t\t\tinclude \n" + " ^\n" + "compilation terminated.\n" - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, output) - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "Wire.h", ctx.IncludeJustFound) + require.Equal(t, "Wire.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { @@ -104,13 +86,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { output := "/some/path/sketch.ino:1:33: fatal error: SPI.h: No such file or directory\n" + "compilation terminated.\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "SPI.h", ctx.IncludeJustFound) + require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { @@ -119,11 +97,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { output := "In file included from /tmp/arduino_modified_sketch_815412/binouts.ino:52:0:\n" + "/tmp/arduino_build_static/sketch/regtable.h:31:22: fatal error: register.h: No such file or directory\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "register.h", ctx.IncludeJustFound) + require.Equal(t, "register.h", include) } diff --git a/test/read_file_and_store_in_context_test.go b/test/read_file_and_store_in_context_test.go index a83cfa0c..b33ff47c 100644 --- a/test/read_file_and_store_in_context_test.go +++ b/test/read_file_and_store_in_context_test.go @@ -47,9 +47,8 @@ func TestReadFileAndStoreInContext(t *testing.T) { utils.WriteFile(file.Name(), "test test\nciao") ctx := &types.Context{} - ctx.FileToRead = file.Name() - command := &builder.ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE} + command := &builder.ReadFileAndStoreInContext{FileToRead: file.Name(), Target: &ctx.SourceGccMinusE} err = command.Run(ctx) NoError(t, err) diff --git a/test/store_build_options_map_test.go b/test/store_build_options_map_test.go index 31c20f1d..ebd593c2 100644 --- a/test/store_build_options_map_test.go +++ b/test/store_build_options_map_test.go @@ -74,6 +74,7 @@ func TestStoreBuildOptionsMap(t *testing.T) { NoError(t, err) require.Equal(t, "{\n"+ + " \"additionalFiles\": \"\",\n"+ " \"builtInLibrariesFolders\": \"built-in libraries\",\n"+ " \"customBuildProperties\": \"custom=prop\",\n"+ " \"fqbn\": \"fqbn\",\n"+ diff --git a/test/tools_loader_test.go b/test/tools_loader_test.go index 3507afe8..926bca37 100644 --- a/test/tools_loader_test.go +++ b/test/tools_loader_test.go @@ -30,11 +30,12 @@ package test import ( + "sort" + "testing" + "github.com/arduino/arduino-builder" "github.com/arduino/arduino-builder/types" "github.com/stretchr/testify/require" - "sort" - "testing" ) type ByToolIDAndVersion []*types.Tool @@ -64,11 +65,15 @@ func TestLoadTools(t *testing.T) { NoError(t, err) tools := ctx.Tools - require.Equal(t, 6, len(tools)) + require.Equal(t, 7, len(tools)) sort.Sort(ByToolIDAndVersion(tools)) idx := 0 + require.Equal(t, "arduino-preprocessor", tools[idx].Name) + require.Equal(t, "0.1.5", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.5"), tools[idx].Folder) + idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) require.Equal(t, Abs(t, "./downloaded_tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder) @@ -136,7 +141,7 @@ func TestLoadLotsOfTools(t *testing.T) { NoError(t, err) tools := ctx.Tools - require.Equal(t, 8, len(tools)) + require.Equal(t, 9, len(tools)) sort.Sort(ByToolIDAndVersion(tools)) @@ -145,6 +150,10 @@ func TestLoadLotsOfTools(t *testing.T) { require.Equal(t, "4.0.0-atmel", tools[idx].Version) require.Equal(t, Abs(t, "./downloaded_board_manager_stuff/arduino/tools/CMSIS/4.0.0-atmel"), tools[idx].Folder) idx++ + require.Equal(t, "arduino-preprocessor", tools[idx].Name) + require.Equal(t, "0.1.5", tools[idx].Version) + require.Equal(t, Abs(t, "./downloaded_tools/arduino-preprocessor/0.1.5"), tools[idx].Folder) + idx++ require.Equal(t, "arm-none-eabi-gcc", tools[idx].Name) require.Equal(t, "4.8.3-2014q1", tools[idx].Version) require.Equal(t, Abs(t, "./downloaded_tools/arm-none-eabi-gcc/4.8.3-2014q1"), tools[idx].Folder) diff --git a/test/utils_test.go b/test/utils_test.go index da8395fa..6ada0c29 100644 --- a/test/utils_test.go +++ b/test/utils_test.go @@ -70,6 +70,25 @@ func TestCommandLineParser(t *testing.T) { require.Equal(t, "/tmp/sketch321469072.cpp", parts[22]) } +func TestPrintableCommand(t *testing.T) { + parts := []string{ + "/path/to/dir with spaces/cmd", + "arg1", + "arg-\"with\"-quotes", + "specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument", + "arg with spaces", + "arg\twith\t\ttabs", + "lastarg", + } + correct := "\"/path/to/dir with spaces/cmd\"" + + " arg1 \"arg-\\\"with\\\"-quotes\"" + + " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + + " lastarg" + result := utils.PrintableCommand(parts) + require.Equal(t, correct, result) +} + func TestCommandLineParserError(t *testing.T) { command := "\"command missing quote" diff --git a/tools_loader.go b/tools_loader.go index 2d9ee00a..8fd3241d 100644 --- a/tools_loader.go +++ b/tools_loader.go @@ -30,14 +30,15 @@ package builder import ( + "os" + "path/filepath" + "strings" + "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/gohasissues" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "os" - "path/filepath" - "strings" ) type ToolsLoader struct{} @@ -47,6 +48,11 @@ func (s *ToolsLoader) Run(ctx *types.Context) error { tools := []*types.Tool{} + if ctx.CanUseCachedTools { + //fmt.Println("no fs modification, can use cached ctx") + return nil + } + for _, folder := range folders { builtinToolsVersionsFile, err := findBuiltinToolsVersionsFile(folder) if err != nil { @@ -82,6 +88,8 @@ func (s *ToolsLoader) Run(ctx *types.Context) error { } } + ctx.CanUseCachedTools = true + ctx.Tools = tools return nil diff --git a/types/accessories.go b/types/accessories.go index 6601bbaa..4c87c30c 100644 --- a/types/accessories.go +++ b/types/accessories.go @@ -29,6 +29,11 @@ package types +import ( + "bytes" + "sync" +) + type UniqueStringQueue []string func (queue UniqueStringQueue) Len() int { return len(queue) } @@ -74,3 +79,32 @@ func (queue *UniqueSourceFileQueue) Pop() SourceFile { func (queue *UniqueSourceFileQueue) Empty() bool { return queue.Len() == 0 } + +type BufferedUntilNewLineWriter struct { + PrintFunc PrintFunc + Buffer bytes.Buffer + lock sync.Mutex +} + +type PrintFunc func([]byte) + +func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) { + w.lock.Lock() + defer w.lock.Unlock() + + writtenToBuffer, err := w.Buffer.Write(p) + return writtenToBuffer, err +} + +func (w *BufferedUntilNewLineWriter) Flush() { + w.lock.Lock() + defer w.lock.Unlock() + + remainingBytes := w.Buffer.Bytes() + if len(remainingBytes) > 0 { + if remainingBytes[len(remainingBytes)-1] != '\n' { + remainingBytes = append(remainingBytes, '\n') + } + w.PrintFunc(remainingBytes) + } +} diff --git a/types/context.go b/types/context.go index bc9491dd..e9368c08 100644 --- a/types/context.go +++ b/types/context.go @@ -1,12 +1,19 @@ package types import ( + "path/filepath" "strings" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/go-properties-map" ) +type ProgressStruct struct { + PrintEnabled bool + Steps float64 + Progress float64 +} + // Context structure type Context struct { // Build options @@ -15,9 +22,11 @@ type Context struct { LibrariesFolders []string BuiltInLibrariesFolders []string OtherLibrariesFolders []string + WatchedLocations []string SketchLocation string ArduinoAPIVersion string FQBN string + CodeCompleteAt string // Build options are serialized here BuildOptionsJson string @@ -53,6 +62,7 @@ type Context struct { Sketch *Sketch Source string SourceGccMinusE string + CodeCompletions string WarningsLevel string @@ -61,7 +71,6 @@ type Context struct { HeaderToLibraries map[string][]*Library ImportedLibraries []*Library LibrariesResolutionResults map[string]LibraryResolutionResult - IncludeJustFound string IncludeFolders []string OutputGccMinusM string @@ -79,6 +88,9 @@ type Context struct { Verbose bool DebugPreprocessor bool + // Dry run, only create progress map + Progress ProgressStruct + // Contents of a custom build properties file (line by line) CustomBuildProperties []string @@ -86,12 +98,23 @@ type Context struct { logger i18n.Logger DebugLevel int - // ReadFileAndStoreInContext command - FileToRead string + // Reuse old tools since the backing storage didn't change + CanUseCachedTools bool + + // Experimental: use arduino-preprocessor to create prototypes + UseArduinoPreprocessor bool } func (ctx *Context) ExtractBuildOptions() properties.Map { opts := make(properties.Map) + var additionalFilesRelative []string + if ctx.Sketch != nil { + for _, sketch := range ctx.Sketch.AdditionalFiles { + absPath := filepath.Dir(ctx.SketchLocation) + relPath := strings.TrimPrefix(sketch.Name, absPath) + additionalFilesRelative = append(additionalFilesRelative, relPath) + } + } opts["hardwareFolders"] = strings.Join(ctx.HardwareFolders, ",") opts["toolsFolders"] = strings.Join(ctx.ToolsFolders, ",") opts["builtInLibrariesFolders"] = strings.Join(ctx.BuiltInLibrariesFolders, ",") @@ -100,6 +123,7 @@ func (ctx *Context) ExtractBuildOptions() properties.Map { opts["fqbn"] = ctx.FQBN opts["runtime.ide.version"] = ctx.ArduinoAPIVersion opts["customBuildProperties"] = strings.Join(ctx.CustomBuildProperties, ",") + opts["additionalFiles"] = strings.Join(additionalFilesRelative, ",") return opts } diff --git a/utils/utils.go b/utils/utils.go index 318b3984..b1002c65 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -30,20 +30,26 @@ package utils import ( + "bytes" "crypto/md5" "encoding/hex" + "fmt" + "io" "io/ioutil" "os" "os/exec" "path/filepath" "runtime" "strings" + "unicode" "unicode/utf8" "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/gohasissues" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" ) func KeysOfMapOfStringInterface(input map[string]interface{}) []string { @@ -229,7 +235,7 @@ func TrimSpace(value string) string { type argFilterFunc func(int, string, []string) bool -func PrepareCommandFilteredArgs(pattern string, filter argFilterFunc, logger i18n.Logger) (*exec.Cmd, error) { +func PrepareCommandFilteredArgs(pattern string, filter argFilterFunc, logger i18n.Logger, relativePath string) (*exec.Cmd, error) { parts, err := ParseCommandLine(pattern, logger) if err != nil { return nil, i18n.WrapError(err) @@ -239,19 +245,95 @@ func PrepareCommandFilteredArgs(pattern string, filter argFilterFunc, logger i18 var args []string for idx, part := range parts { if filter(idx, part, parts) { + // if relativePath is specified, the overall commandline is too long for the platform + // try reducing the length by making the filenames relative + // and changing working directory to build.path + if relativePath != "" { + if _, err := os.Stat(part); !os.IsNotExist(err) { + tmp, err := filepath.Rel(relativePath, part) + if err == nil { + part = tmp + } + } + } args = append(args, part) } } - return exec.Command(command, args...), nil + cmd := exec.Command(command, args...) + + if relativePath != "" { + cmd.Dir = relativePath + } + + return cmd, nil } func filterEmptyArg(_ int, arg string, _ []string) bool { return arg != constants.EMPTY_STRING } -func PrepareCommand(pattern string, logger i18n.Logger) (*exec.Cmd, error) { - return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger) +func PrepareCommand(pattern string, logger i18n.Logger, relativePath string) (*exec.Cmd, error) { + return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger, relativePath) +} + +func printableArgument(arg string) string { + if strings.ContainsAny(arg, "\"\\ \t") { + arg = strings.Replace(arg, "\\", "\\\\", -1) + arg = strings.Replace(arg, "\"", "\\\"", -1) + return "\"" + arg + "\"" + } else { + return arg + } +} + +// Convert a command and argument slice back to a printable string. +// This adds basic escaping which is sufficient for debug output, but +// probably not for shell interpretation. This essentially reverses +// ParseCommandLine. +func PrintableCommand(parts []string) string { + return strings.Join(Map(parts, printableArgument), " ") +} + +const ( + Ignore = 0 // Redirect to null + Show = 1 // Show on stdout/stderr as normal + ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise + Capture = 3 // Capture into buffer +) + +func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { + + if stdout == Capture { + buffer := &bytes.Buffer{} + command.Stdout = buffer + } else if stdout == Show || stdout == ShowIfVerbose && ctx.Verbose { + command.Stdout = os.Stdout + } + + if stderr == Capture { + buffer := &bytes.Buffer{} + command.Stderr = buffer + } else if stderr == Show || stderr == ShowIfVerbose && ctx.Verbose { + command.Stderr = os.Stderr + } + + err := command.Start() + if err != nil { + return nil, nil, i18n.WrapError(err) + } + + err = command.Wait() + + var outbytes, errbytes []byte + if buf, ok := command.Stdout.(*bytes.Buffer); ok { + outbytes = buf.Bytes() + } + if buf, ok := command.Stderr.(*bytes.Buffer); ok { + errbytes = buf.Bytes() + } + + return outbytes, errbytes, i18n.WrapError(err) } func MapHas(aMap map[string]interface{}, key string) bool { @@ -274,6 +356,9 @@ func SliceToMapStringBool(keys []string, value bool) map[string]bool { func AbsolutizePaths(files []string) ([]string, error) { for idx, file := range files { + if file == "" { + continue + } absFile, err := filepath.Abs(file) if err != nil { return nil, i18n.WrapError(err) @@ -323,6 +408,30 @@ func FilterOutFoldersByNames(folders []os.FileInfo, names ...string) []os.FileIn type CheckExtensionFunc func(ext string) bool +func FindAllSubdirectories(folder string, output *[]string) error { + walkFunc := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Skip source control and hidden files and directories + if IsSCCSOrHiddenFile(info) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Skip directories unless recurse is on, or this is the + // root directory + if info.IsDir() { + *output = AppendIfNotPresent(*output, path) + } + return nil + } + return gohasissues.Walk(folder, walkFunc) +} + func FindFilesInFolder(files *[]string, folder string, extensions CheckExtensionFunc, recurse bool) error { walkFunc := func(path string, info os.FileInfo, err error) error { if err != nil { @@ -488,3 +597,121 @@ func ParseCppString(line string) (string, string, bool) { i += width } } + +func isMn(r rune) bool { + return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks +} + +// Normalizes an UTF8 byte slice +// TODO: use it more often troughout all the project (maybe on logger interface?) +func NormalizeUTF8(buf []byte) []byte { + t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) + result, _, _ := transform.Bytes(t, buf) + return result +} + +// CopyFile copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. The file mode will be copied from the source and +// the copied data is synced/flushed to stable storage. +func CopyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return +} + +// CopyDir recursively copies a directory tree, attempting to preserve permissions. +// Source directory must exist, destination directory must *not* exist. +// Symlinks are ignored and skipped. +func CopyDir(src string, dst string, extensions CheckExtensionFunc) (err error) { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return fmt.Errorf("source is not a directory") + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return + } + if err == nil { + return fmt.Errorf("destination already exists") + } + + err = os.MkdirAll(dst, si.Mode()) + if err != nil { + return + } + + entries, err := ioutil.ReadDir(src) + if err != nil { + return + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + err = CopyDir(srcPath, dstPath, extensions) + if err != nil { + return + } + } else { + // Skip symlinks. + if entry.Mode()&os.ModeSymlink != 0 { + continue + } + + if extensions != nil && !extensions(strings.ToLower(filepath.Ext(srcPath))) { + continue + } + + err = CopyFile(srcPath, dstPath) + if err != nil { + return + } + } + } + + return +} diff --git a/wipeout_build_path_if_build_options_changed.go b/wipeout_build_path_if_build_options_changed.go index 0764345d..87048e77 100644 --- a/wipeout_build_path_if_build_options_changed.go +++ b/wipeout_build_path_if_build_options_changed.go @@ -33,6 +33,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "github.com/arduino/arduino-builder/builder_utils" "github.com/arduino/arduino-builder/constants" @@ -72,7 +73,7 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] realCoreFolder := utils.GetParentFolder(coreFolder, 2) jsonPath := filepath.Join(ctx.BuildPath, constants.BUILD_OPTIONS_FILE) - coreHasChanged := builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, jsonPath) + coreHasChanged := builder_utils.TXTBuildRulesHaveChanged(realCoreFolder, targetCoreFolder, jsonPath) if opts.Equals(prevOpts) && !coreHasChanged { return nil @@ -85,6 +86,10 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { if err != nil { return i18n.WrapError(err) } + // if build path is inside the sketch folder, also wipe ctx.AdditionalFiles + if strings.Contains(ctx.BuildPath, filepath.Dir(ctx.SketchLocation)) { + ctx.Sketch.AdditionalFiles = ctx.Sketch.AdditionalFiles[:0] + } for _, file := range files { os.RemoveAll(filepath.Join(buildPath, file.Name())) }