From ac23e586f0f2359d627a18adee0638c6e5652a0b Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 21:39:50 +0200 Subject: [PATCH 01/20] remove unused LibraryDir from legacy context --- commands/compile/compile.go | 4 ++-- legacy/builder/test/builder_test.go | 2 +- legacy/builder/test/libraries_loader_test.go | 8 ++++---- legacy/builder/types/context.go | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 1daa6f31714..503dcc24f7c 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -193,7 +193,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.BuiltInToolsDirs = configuration.BuiltinToolsDirectories(configuration.Settings) builderCtx.OtherLibrariesDirs = paths.NewPathList(req.GetLibraries()...) builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings)) - builderCtx.LibraryDirs = paths.NewPathList(req.Library...) builderCtx.CompilationDatabase = bldr.NewCompilationDatabase( builderCtx.BuildPath.Join("compile_commands.json"), @@ -255,9 +254,10 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream libsManager = lm } useCachedLibrariesResolution := req.GetSkipLibrariesDiscovery() + libraryDir := paths.NewPathList(req.Library...) libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader( useCachedLibrariesResolution, libsManager, - builderCtx.BuiltInLibrariesDirs, builderCtx.LibraryDirs, builderCtx.OtherLibrariesDirs, + builderCtx.BuiltInLibrariesDirs, libraryDir, builderCtx.OtherLibrariesDirs, builderCtx.ActualPlatform, builderCtx.TargetPlatform, ) if err != nil { diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index c4f61212cc6..6a1315076b9 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -125,7 +125,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat if !stepToSkip[skipLibraries] { lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) diff --git a/legacy/builder/test/libraries_loader_test.go b/legacy/builder/test/libraries_loader_test.go index b265ca848a2..5866fbcfa98 100644 --- a/legacy/builder/test/libraries_loader_test.go +++ b/legacy/builder/test/libraries_loader_test.go @@ -49,7 +49,7 @@ func TestLoadLibrariesAVR(t *testing.T) { lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) @@ -153,7 +153,7 @@ func TestLoadLibrariesSAM(t *testing.T) { lm, libsResolver, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) @@ -230,7 +230,7 @@ func TestLoadLibrariesAVRNoDuplicateLibrariesFolders(t *testing.T) { lm, _, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) @@ -253,7 +253,7 @@ func TestLoadLibrariesMyAVRPlatform(t *testing.T) { lm, _, _, err := detector.LibrariesLoader( false, nil, - ctx.BuiltInLibrariesDirs, ctx.LibraryDirs, ctx.OtherLibrariesDirs, + ctx.BuiltInLibrariesDirs, nil, ctx.OtherLibrariesDirs, ctx.ActualPlatform, ctx.TargetPlatform, ) require.NoError(t, err) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index a6570bf3268..9466c7cffc5 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -70,7 +70,6 @@ type Context struct { BuiltInToolsDirs paths.PathList BuiltInLibrariesDirs *paths.Path OtherLibrariesDirs paths.PathList - LibraryDirs paths.PathList // List of paths pointing to individual library root folders WatchedLocations paths.PathList FQBN *cores.FQBN Clean bool From 8c92b303fc702c3b86e20f526f6c77d8f627da57 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 21:44:12 +0200 Subject: [PATCH 02/20] remove unused WatchedLocation from legacy context --- legacy/builder/types/context.go | 1 - 1 file changed, 1 deletion(-) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 9466c7cffc5..394815cd131 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -70,7 +70,6 @@ type Context struct { BuiltInToolsDirs paths.PathList BuiltInLibrariesDirs *paths.Path OtherLibrariesDirs paths.PathList - WatchedLocations paths.PathList FQBN *cores.FQBN Clean bool From 284703435678d28975d2efdaeeb14751ad7e32cc Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 21:48:06 +0200 Subject: [PATCH 03/20] remove unused IgnoreSketchFolderNameErrors from legacy context --- legacy/builder/types/context.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 394815cd131..48686463a62 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -84,17 +84,16 @@ type Context struct { TargetPlatform *cores.PlatformRelease ActualPlatform *cores.PlatformRelease - BuildProperties *properties.Map - BuildPath *paths.Path - SketchBuildPath *paths.Path - CoreBuildPath *paths.Path - CoreBuildCachePath *paths.Path - CoreArchiveFilePath *paths.Path - CoreObjectsFiles paths.PathList - LibrariesBuildPath *paths.Path - LibrariesObjectFiles paths.PathList - SketchObjectFiles paths.PathList - IgnoreSketchFolderNameErrors bool + BuildProperties *properties.Map + BuildPath *paths.Path + SketchBuildPath *paths.Path + CoreBuildPath *paths.Path + CoreBuildCachePath *paths.Path + CoreArchiveFilePath *paths.Path + CoreObjectsFiles paths.PathList + LibrariesBuildPath *paths.Path + LibrariesObjectFiles paths.PathList + SketchObjectFiles paths.PathList Sketch *sketch.Sketch WarningsLevel string From 0b4aee25d289870624a94dfe24132a21cdc3d55c Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 21:50:52 +0200 Subject: [PATCH 04/20] remove CanUseCachedTools from legacy context --- legacy/builder/test/builder_test.go | 7 ++----- legacy/builder/types/context.go | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 6a1315076b9..1a0ec482567 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -84,11 +84,8 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat // NoError(t, err) fmt.Println(err) } - if !ctx.CanUseCachedTools { - if ctx.BuiltInToolsDirs != nil { - pmb.LoadToolsFromBundleDirectories(ctx.BuiltInToolsDirs) - } - ctx.CanUseCachedTools = true + if ctx.BuiltInToolsDirs != nil { + pmb.LoadToolsFromBundleDirectories(ctx.BuiltInToolsDirs) } pm := pmb.Build() pme, _ /* never release... */ := pm.NewExplorer() diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 48686463a62..b7b61ef52c3 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -112,9 +112,6 @@ type Context struct { // Custom build properties defined by user (line by line as "key=value" pairs) CustomBuildProperties []string - // Reuse old tools since the backing storage didn't change - CanUseCachedTools bool - // Experimental: use arduino-preprocessor to create prototypes UseArduinoPreprocessor bool From 402de11bfc0e7ae11216a50e2f2141a85ae93c71 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Tue, 5 Sep 2023 21:51:43 +0200 Subject: [PATCH 05/20] remove UseArduinoPreprocessor from legacy context --- legacy/builder/builder.go | 3 --- legacy/builder/types/context.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index ff517a5a2d4..41579d2bf1c 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -140,9 +140,6 @@ func (s *Builder) Run(ctx *types.Context) error { func PreprocessSketch(ctx *types.Context) error { preprocessorImpl := preprocessor.PreprocessSketchWithCtags - if ctx.UseArduinoPreprocessor { - preprocessorImpl = preprocessor.PreprocessSketchWithArduinoPreprocessor - } normalOutput, verboseOutput, err := preprocessorImpl( ctx.Sketch, ctx.BuildPath, ctx.SketchLibrariesDetector.IncludeFolders(), ctx.LineOffset, ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index b7b61ef52c3..d7a9d79d23e 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -112,9 +112,6 @@ type Context struct { // Custom build properties defined by user (line by line as "key=value" pairs) CustomBuildProperties []string - // Experimental: use arduino-preprocessor to create prototypes - UseArduinoPreprocessor bool - // Parallel processes Jobs int From e82e52e81d1dd892618deff813fcfa70f75d2b43 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 11:08:13 +0200 Subject: [PATCH 06/20] make the CoreBuilder command a function --- legacy/builder/builder.go | 24 ++++++- legacy/builder/phases/core_builder.go | 96 +++++++++++++++------------ 2 files changed, 78 insertions(+), 42 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 41579d2bf1c..c058c0df167 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -72,7 +72,29 @@ func (s *Builder) Run(ctx *types.Context) error { utils.LogIfVerbose(false, tr("Compiling core...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.prebuild", Suffix: ".pattern"}, - &phases.CoreBuilder{}, + + types.BareCommand(func(ctx *types.Context) error { + objectFiles, archiveFile, coreBuildCachePath, + normalOut, verboseOut, err := phases.CoreBuilder( + ctx, + ctx.BuildPath, ctx.CoreBuildPath, ctx.CoreBuildCachePath, + ctx.BuildProperties, + ctx.ActualPlatform, + ctx.Verbose, ctx.OnlyUpdateCompilationDatabase, ctx.Clean, + ) + + ctx.CoreObjectsFiles = objectFiles + ctx.CoreArchiveFilePath = archiveFile + ctx.CoreBuildCachePath = coreBuildCachePath + + ctx.Info(string(normalOut)) + if ctx.Verbose { + ctx.Info(string(verboseOut)) + } + + return err + }), + &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, utils.LogIfVerbose(false, tr("Linking everything together...")), diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 0b629365eba..e6e3f61c07e 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -16,6 +16,7 @@ package phases import ( + "bytes" "crypto/md5" "encoding/hex" "fmt" @@ -23,6 +24,7 @@ import ( "strings" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" @@ -34,61 +36,72 @@ import ( "github.com/pkg/errors" ) -type CoreBuilder struct{} +// Trasformo in un command +// provo a spostare builder_utils in arduino/Builder var tr = i18n.Tr -func (s *CoreBuilder) Run(ctx *types.Context) error { - coreBuildPath := ctx.CoreBuildPath - coreBuildCachePath := ctx.CoreBuildCachePath - buildProperties := ctx.BuildProperties - +func CoreBuilder( + ctx *types.Context, + buildPath, coreBuildPath, coreBuildCachePath *paths.Path, + buildProperties *properties.Map, + actualPlatform *cores.PlatformRelease, + verbose, onlyUpdateCompilationDatabase, clean bool, +) (paths.PathList, *paths.Path, *paths.Path, []byte, []byte, error) { + normalOut := &bytes.Buffer{} if err := coreBuildPath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, nil, coreBuildCachePath, nil, nil, errors.WithStack(err) } if coreBuildCachePath != nil { - if _, err := coreBuildCachePath.RelTo(ctx.BuildPath); err != nil { - ctx.Info(tr("Couldn't deeply cache core build: %[1]s", err)) - ctx.Info(tr("Running normal build of the core...")) + if _, err := coreBuildCachePath.RelTo(buildPath); err != nil { + normalOut.WriteString(tr("Couldn't deeply cache core build: %[1]s", err)) + normalOut.WriteString(tr("Running normal build of the core...")) coreBuildCachePath = nil - ctx.CoreBuildCachePath = nil } else if err := coreBuildCachePath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, nil, coreBuildCachePath, nil, nil, errors.WithStack(err) } } - archiveFile, objectFiles, err := compileCore(ctx, coreBuildPath, coreBuildCachePath, buildProperties) + archiveFile, objectFiles, verboseOut, err := compileCore( + ctx, + verbose, onlyUpdateCompilationDatabase, clean, + actualPlatform, + coreBuildPath, coreBuildCachePath, + buildProperties, + ) if err != nil { - return errors.WithStack(err) + return nil, nil, coreBuildCachePath, normalOut.Bytes(), verboseOut, errors.WithStack(err) } - ctx.CoreArchiveFilePath = archiveFile - ctx.CoreObjectsFiles = objectFiles - - return nil + return objectFiles, archiveFile, coreBuildCachePath, normalOut.Bytes(), verboseOut, nil } -func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *paths.Path, buildProperties *properties.Map) (*paths.Path, paths.PathList, error) { +func compileCore( + ctx *types.Context, + verbose, onlyUpdateCompilationDatabase, clean bool, + actualPlatform *cores.PlatformRelease, + buildPath, buildCachePath *paths.Path, + buildProperties *properties.Map, +) (*paths.Path, paths.PathList, []byte, error) { + verboseOut := &bytes.Buffer{} + coreFolder := buildProperties.GetPath("build.core.path") variantFolder := buildProperties.GetPath("build.variant.path") - targetCoreFolder := buildProperties.GetPath(constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH) - includes := []string{} - includes = append(includes, coreFolder.String()) + includes := []string{coreFolder.String()} if variantFolder != nil && variantFolder.IsDir() { includes = append(includes, variantFolder.String()) } includes = f.Map(includes, cpp.WrapWithHyphenI) var err error - variantObjectFiles := paths.NewPathList() if variantFolder != nil && variantFolder.IsDir() { variantObjectFiles, err = builder_utils.CompileFilesRecursive(ctx, variantFolder, buildPath, buildProperties, includes) if err != nil { - return nil, nil, errors.WithStack(err) + return nil, nil, verboseOut.Bytes(), errors.WithStack(err) } } @@ -98,15 +111,16 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path archivedCoreName := GetCachedCoreArchiveDirName( buildProperties.Get("build.fqbn"), buildProperties.Get("compiler.optimization_flags"), - realCoreFolder) + realCoreFolder, + ) targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") if _, err := buildcache.New(buildCachePath).GetOrCreate(archivedCoreName); errors.Is(err, buildcache.CreateDirErr) { - return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) + return nil, nil, verboseOut.Bytes(), fmt.Errorf(tr("creating core cache folder: %s", err)) } var canUseArchivedCore bool - if ctx.OnlyUpdateCompilationDatabase || ctx.Clean { + if onlyUpdateCompilationDatabase || clean { canUseArchivedCore = false } else if isOlder, err := builder_utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { // Recreate the archive if ANY of the core files (including platform.txt) has changed @@ -122,47 +136,47 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path if canUseArchivedCore { // use archived core - if ctx.Verbose { - ctx.Info(tr("Using precompiled core: %[1]s", targetArchivedCore)) + if verbose { + verboseOut.WriteString(tr("Using precompiled core: %[1]s", targetArchivedCore)) } - return targetArchivedCore, variantObjectFiles, nil + return targetArchivedCore, variantObjectFiles, verboseOut.Bytes(), nil } } coreObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, coreFolder, buildPath, buildProperties, includes) if err != nil { - return nil, nil, errors.WithStack(err) + return nil, nil, verboseOut.Bytes(), errors.WithStack(err) } archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, paths.New("core.a"), coreObjectFiles, buildProperties) if err != nil { - return nil, nil, errors.WithStack(err) + return nil, nil, verboseOut.Bytes(), errors.WithStack(err) } // archive core.a - if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase { + if targetArchivedCore != nil && !onlyUpdateCompilationDatabase { err := archiveFile.CopyTo(targetArchivedCore) - if ctx.Verbose { + if verbose { if err == nil { - ctx.Info(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore)) + verboseOut.WriteString(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore)) } else if os.IsNotExist(err) { - ctx.Info(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s", - ctx.ActualPlatform, + verboseOut.WriteString(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s", + actualPlatform, "https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file")) } else { - ctx.Info(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err)) + verboseOut.WriteString(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err)) } } } - return archiveFile, variantObjectFiles, nil + return archiveFile, variantObjectFiles, verboseOut.Bytes(), nil } // GetCachedCoreArchiveDirName returns the directory name to be used to store // the global cached core.a. func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string { - fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) - fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) + fqbnToUnderscore := strings.ReplaceAll(fqbn, ":", "_") + fqbnToUnderscore = strings.ReplaceAll(fqbnToUnderscore, "=", "_") if absCoreFolder, err := coreFolder.Abs(); err == nil { coreFolder = absCoreFolder } // silently continue if absolute path can't be detected From b41be5514ca6447546d0c9a4001ffde7a8af86cd Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 12:39:54 +0200 Subject: [PATCH 07/20] remove the use of context from builder_utils --- legacy/builder/builder.go | 17 +- legacy/builder/builder_utils/utils.go | 193 ++++++++++++++++----- legacy/builder/phases/core_builder.go | 101 +++++++---- legacy/builder/phases/libraries_builder.go | 51 +++++- legacy/builder/phases/linker.go | 12 +- legacy/builder/phases/sizer.go | 10 +- legacy/builder/phases/sketch_builder.go | 28 ++- legacy/builder/recipe_runner.go | 5 +- legacy/builder/utils/utils.go | 28 +-- 9 files changed, 343 insertions(+), 102 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index c058c0df167..cb7d18ce157 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -74,24 +74,25 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - objectFiles, archiveFile, coreBuildCachePath, - normalOut, verboseOut, err := phases.CoreBuilder( - ctx, + objectFiles, archiveFile, coreBuildCachePath, err := phases.CoreBuilder( ctx.BuildPath, ctx.CoreBuildPath, ctx.CoreBuildCachePath, ctx.BuildProperties, ctx.ActualPlatform, ctx.Verbose, ctx.OnlyUpdateCompilationDatabase, ctx.Clean, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, ) ctx.CoreObjectsFiles = objectFiles ctx.CoreArchiveFilePath = archiveFile ctx.CoreBuildCachePath = coreBuildCachePath - ctx.Info(string(normalOut)) - if ctx.Verbose { - ctx.Info(string(verboseOut)) - } - return err }), diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index 1d53860b163..8e0010815a0 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -16,13 +16,16 @@ package builder_utils import ( + "bytes" "fmt" + "io" "os" "path/filepath" "runtime" "strings" "sync" + "github.com/arduino/arduino-cli/arduino/builder" bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/executils" @@ -30,6 +33,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -63,15 +67,81 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st return true, nil } -func CompileFiles(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourceDir, false, buildPath, buildProperties, includes) +func CompileFiles( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + false, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) } -func CompileFilesRecursive(ctx *types.Context, sourceDir *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - return compileFiles(ctx, sourceDir, true, buildPath, buildProperties, includes) +func CompileFilesRecursive( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + true, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) } -func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { +func compileFiles( + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + sourceDir *paths.Path, + recurse bool, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *types.ProgressStruct, + progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { validExtensions := []string{} for ext := range globals.SourceFilesValidExtensions { validExtensions = append(validExtensions, ext) @@ -82,8 +152,8 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build return nil, err } - ctx.Progress.AddSubSteps(len(sources)) - defer ctx.Progress.RemoveSubSteps() + progress.AddSubSteps(len(sources)) + defer progress.RemoveSubSteps() objectFiles := paths.NewPathList() var objectFilesMux sync.Mutex @@ -99,7 +169,19 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build if !buildProperties.ContainsKey(recipe) { recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) } - objectFile, err := compileFileWithRecipe(ctx, sourceDir, source, buildPath, buildProperties, includes, recipe) + objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe( + stdoutWriter, stderrWriter, + warningsLevel, + compilationDatabase, + verbose, + onlyUpdateCompilationDatabase, + sourceDir, source, buildPath, buildProperties, includes, recipe, + ) + if verbose { + verboseStdoutFn(verboseStdout) + verboseInfoFn(string(verboseInfo)) + } + verboseStderrFn(stderr) if err != nil { errorsMux.Lock() errorsList = append(errorsList, err) @@ -113,7 +195,6 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build // Spawn jobs runners var wg sync.WaitGroup - jobs := ctx.Jobs if jobs == 0 { jobs = runtime.NumCPU() } @@ -137,8 +218,14 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build } queue <- source - ctx.Progress.CompleteStep() - ctx.PushProgress() + progress.CompleteStep() + // PushProgress + if progressCB != nil { + progressCB(&rpc.TaskProgress{ + Percent: progress.Progress, + Completed: progress.Progress >= 100.0, + }) + } } close(queue) wg.Wait() @@ -150,14 +237,27 @@ func compileFiles(ctx *types.Context, sourceDir *paths.Path, recurse bool, build return objectFiles, nil } -func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string, recipe string) (*paths.Path, error) { +func compileFileWithRecipe( + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, + compilationDatabase *builder.CompilationDatabase, + verbose, onlyUpdateCompilationDatabase bool, + sourcePath *paths.Path, + source *paths.Path, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + recipe string, +) (*paths.Path, []byte, []byte, []byte, error) { + verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + properties := buildProperties.Clone() - properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel)) + properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel)) properties.Set(constants.BUILD_PROPERTIES_INCLUDES, strings.Join(includes, constants.SPACE)) properties.SetPath("source_file", source) relativeSource, err := sourcePath.RelTo(source) if err != nil { - return nil, errors.WithStack(err) + return nil, nil, nil, nil, errors.WithStack(err) } depsFile := buildPath.Join(relativeSource.String() + ".d") objectFile := buildPath.Join(relativeSource.String() + ".o") @@ -165,53 +265,59 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) err = objectFile.Parent().MkdirAll() if err != nil { - return nil, errors.WithStack(err) + return nil, nil, nil, nil, errors.WithStack(err) } objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile) if err != nil { - return nil, errors.WithStack(err) + return nil, nil, nil, nil, errors.WithStack(err) } command, err := PrepareCommandForRecipe(properties, recipe, false) if err != nil { - return nil, errors.WithStack(err) + return nil, nil, nil, nil, errors.WithStack(err) } - if ctx.CompilationDatabase != nil { - ctx.CompilationDatabase.Add(source, command) + if compilationDatabase != nil { + compilationDatabase.Add(source, command) } - if !objIsUpToDate && !ctx.OnlyUpdateCompilationDatabase { + if !objIsUpToDate && !onlyUpdateCompilationDatabase { // Since this compile could be multithreaded, we first capture the command output - stdout, stderr, err := utils.ExecCommand(ctx, command, utils.Capture, utils.Capture) + info, stdout, stderr, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture, utils.Capture) // and transfer all at once at the end... - if ctx.Verbose { - ctx.WriteStdout(stdout) + if verbose { + verboseInfo.Write(info) + verboseStdout.Write(stdout) } - ctx.WriteStderr(stderr) + errOut.Write(stderr) // ...and then return the error if err != nil { - return nil, errors.WithStack(err) + return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err) } - } else if ctx.Verbose { + } else if verbose { if objIsUpToDate { - ctx.Info(tr("Using previously compiled file: %[1]s", objectFile)) + verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile)) } else { - ctx.Info(tr("Skipping compile of: %[1]s", objectFile)) + verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile)) } } - return objectFile, nil + return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil } -func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { +func ArchiveCompiledFiles( + buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map, + onlyUpdateCompilationDatabase, verbose bool, + stdoutWriter, stderrWriter io.Writer, +) (*paths.Path, []byte, error) { + verboseInfobuf := &bytes.Buffer{} archiveFilePath := buildPath.JoinPath(archiveFile) - if ctx.OnlyUpdateCompilationDatabase { - if ctx.Verbose { - ctx.Info(tr("Skipping archive creation of: %[1]s", archiveFilePath)) + if onlyUpdateCompilationDatabase { + if verbose { + verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath)) } - return archiveFilePath, nil + return archiveFilePath, verboseInfobuf.Bytes(), nil } if archiveFileStat, err := archiveFilePath.Stat(); err == nil { @@ -228,13 +334,13 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile // something changed, rebuild the core archive if rebuildArchive { if err := archiveFilePath.Remove(); err != nil { - return nil, errors.WithStack(err) + return nil, nil, errors.WithStack(err) } } else { - if ctx.Verbose { - ctx.Info(tr("Using previously compiled file: %[1]s", archiveFilePath)) + if verbose { + verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath)) } - return archiveFilePath, nil + return archiveFilePath, verboseInfobuf.Bytes(), nil } } @@ -246,16 +352,19 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) if err != nil { - return nil, errors.WithStack(err) + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) } - _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + if verbose { + verboseInfobuf.WriteString(string(verboseInfo)) + } if err != nil { - return nil, errors.WithStack(err) + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) } } - return archiveFilePath, nil + return archiveFilePath, verboseInfobuf.Bytes(), nil } const COMMANDLINE_LIMIT = 30000 diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index e6e3f61c07e..e7ae16d404a 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -16,13 +16,14 @@ package phases import ( - "bytes" "crypto/md5" "encoding/hex" "fmt" + "io" "os" "strings" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/buildcache" @@ -31,6 +32,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -42,50 +44,65 @@ import ( var tr = i18n.Tr func CoreBuilder( - ctx *types.Context, buildPath, coreBuildPath, coreBuildCachePath *paths.Path, buildProperties *properties.Map, actualPlatform *cores.PlatformRelease, verbose, onlyUpdateCompilationDatabase, clean bool, -) (paths.PathList, *paths.Path, *paths.Path, []byte, []byte, error) { - normalOut := &bytes.Buffer{} + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, +) (paths.PathList, *paths.Path, *paths.Path, error) { if err := coreBuildPath.MkdirAll(); err != nil { - return nil, nil, coreBuildCachePath, nil, nil, errors.WithStack(err) + return nil, nil, coreBuildCachePath, errors.WithStack(err) } if coreBuildCachePath != nil { if _, err := coreBuildCachePath.RelTo(buildPath); err != nil { - normalOut.WriteString(tr("Couldn't deeply cache core build: %[1]s", err)) - normalOut.WriteString(tr("Running normal build of the core...")) + verboseInfoFn(tr("Couldn't deeply cache core build: %[1]s", err)) + verboseInfoFn(tr("Running normal build of the core...")) coreBuildCachePath = nil } else if err := coreBuildCachePath.MkdirAll(); err != nil { - return nil, nil, coreBuildCachePath, nil, nil, errors.WithStack(err) + return nil, nil, coreBuildCachePath, errors.WithStack(err) } } - archiveFile, objectFiles, verboseOut, err := compileCore( - ctx, + archiveFile, objectFiles, err := compileCore( verbose, onlyUpdateCompilationDatabase, clean, actualPlatform, coreBuildPath, coreBuildCachePath, buildProperties, + compilationDatabase, + jobs, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, verboseStderrFn, + progress, progressCB, ) if err != nil { - return nil, nil, coreBuildCachePath, normalOut.Bytes(), verboseOut, errors.WithStack(err) + return nil, nil, coreBuildCachePath, errors.WithStack(err) } - return objectFiles, archiveFile, coreBuildCachePath, normalOut.Bytes(), verboseOut, nil + return objectFiles, archiveFile, coreBuildCachePath, nil } func compileCore( - ctx *types.Context, verbose, onlyUpdateCompilationDatabase, clean bool, actualPlatform *cores.PlatformRelease, buildPath, buildCachePath *paths.Path, buildProperties *properties.Map, -) (*paths.Path, paths.PathList, []byte, error) { - verboseOut := &bytes.Buffer{} - + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, +) (*paths.Path, paths.PathList, error) { coreFolder := buildProperties.GetPath("build.core.path") variantFolder := buildProperties.GetPath("build.variant.path") targetCoreFolder := buildProperties.GetPath(constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH) @@ -99,9 +116,19 @@ func compileCore( var err error variantObjectFiles := paths.NewPathList() if variantFolder != nil && variantFolder.IsDir() { - variantObjectFiles, err = builder_utils.CompileFilesRecursive(ctx, variantFolder, buildPath, buildProperties, includes) + variantObjectFiles, err = builder_utils.CompileFilesRecursive( + variantFolder, buildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { - return nil, nil, verboseOut.Bytes(), errors.WithStack(err) + return nil, nil, errors.WithStack(err) } } @@ -116,7 +143,7 @@ func compileCore( targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") if _, err := buildcache.New(buildCachePath).GetOrCreate(archivedCoreName); errors.Is(err, buildcache.CreateDirErr) { - return nil, nil, verboseOut.Bytes(), fmt.Errorf(tr("creating core cache folder: %s", err)) + return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) } var canUseArchivedCore bool @@ -137,20 +164,36 @@ func compileCore( if canUseArchivedCore { // use archived core if verbose { - verboseOut.WriteString(tr("Using precompiled core: %[1]s", targetArchivedCore)) + verboseInfoFn(tr("Using precompiled core: %[1]s", targetArchivedCore)) } - return targetArchivedCore, variantObjectFiles, verboseOut.Bytes(), nil + return targetArchivedCore, variantObjectFiles, nil } } - coreObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, coreFolder, buildPath, buildProperties, includes) + coreObjectFiles, err := builder_utils.CompileFilesRecursive( + coreFolder, buildPath, buildProperties, includes, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { - return nil, nil, verboseOut.Bytes(), errors.WithStack(err) + return nil, nil, errors.WithStack(err) } - archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, paths.New("core.a"), coreObjectFiles, buildProperties) + archiveFile, verboseInfo, err := builder_utils.ArchiveCompiledFiles( + buildPath, paths.New("core.a"), coreObjectFiles, buildProperties, + onlyUpdateCompilationDatabase, verbose, stdoutWriter, stderrWriter, + ) + if verbose { + verboseInfoFn(string(verboseInfo)) + } if err != nil { - return nil, nil, verboseOut.Bytes(), errors.WithStack(err) + return nil, nil, errors.WithStack(err) } // archive core.a @@ -158,18 +201,18 @@ func compileCore( err := archiveFile.CopyTo(targetArchivedCore) if verbose { if err == nil { - verboseOut.WriteString(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore)) + verboseInfoFn(tr("Archiving built core (caching) in: %[1]s", targetArchivedCore)) } else if os.IsNotExist(err) { - verboseOut.WriteString(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s", + verboseInfoFn(tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s", actualPlatform, "https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file")) } else { - verboseOut.WriteString(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err)) + verboseInfoFn(tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err)) } } } - return archiveFile, variantObjectFiles, verboseOut.Bytes(), nil + return archiveFile, variantObjectFiles, nil } // GetCachedCoreArchiveDirName returns the directory name to be used to store diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 832bc9b0f8f..bbd02bada1a 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -186,12 +186,31 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } if library.Layout == libraries.RecursiveLayout { - libObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, library.SourceDir, libraryBuildPath, buildProperties, includes) + libObjectFiles, err := builder_utils.CompileFilesRecursive( + library.SourceDir, libraryBuildPath, buildProperties, includes, + ctx.OnlyUpdateCompilationDatabase, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.Verbose, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) if err != nil { return nil, errors.WithStack(err) } if library.DotALinkage { - archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, buildProperties) + archiveFile, verboseInfo, err := builder_utils.ArchiveCompiledFiles( + libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, buildProperties, + ctx.OnlyUpdateCompilationDatabase, ctx.Verbose, + ctx.Stdout, ctx.Stderr, + ) + if ctx.Verbose { + ctx.Info(string(verboseInfo)) + } if err != nil { return nil, errors.WithStack(err) } @@ -203,7 +222,19 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } - libObjectFiles, err := builder_utils.CompileFiles(ctx, library.SourceDir, libraryBuildPath, buildProperties, includes) + libObjectFiles, err := builder_utils.CompileFiles( + library.SourceDir, libraryBuildPath, buildProperties, includes, + ctx.OnlyUpdateCompilationDatabase, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.Verbose, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) if err != nil { return nil, errors.WithStack(err) } @@ -211,7 +242,19 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { utilityBuildPath := libraryBuildPath.Join("utility") - utilityObjectFiles, err := builder_utils.CompileFiles(ctx, library.UtilityDir, utilityBuildPath, buildProperties, includes) + utilityObjectFiles, err := builder_utils.CompileFiles( + library.UtilityDir, utilityBuildPath, buildProperties, includes, + ctx.OnlyUpdateCompilationDatabase, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.Verbose, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) if err != nil { return nil, errors.WithStack(err) } diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index 1f324576586..5ba954714d4 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -98,7 +98,10 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths return errors.WithStack(err) } - if _, _, err := utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil { + if verboseInfo, _, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil { + if ctx.Verbose { + ctx.Info(string(verboseInfo)) + } return errors.WithStack(err) } } @@ -117,9 +120,12 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) if err != nil { return err -} + } - _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + verboseInfo, _, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + if ctx.Verbose { + ctx.Info(string(verboseInfo)) + } return err } diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index c064fbe717b..90ee5981658 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -55,7 +55,10 @@ func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { return errors.New(tr("Error while determining sketch size: %s", err)) } - out, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.Show /* stderr */) + verboseInfo, out, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.Capture /* stdout */, utils.Show /* stderr */) + if ctx.Verbose { + ctx.Info(string(verboseInfo)) + } if err != nil { return errors.New(tr("Error while determining sketch size: %s", err)) } @@ -185,7 +188,10 @@ func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize in return } - out, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.Show /* stderr */) + verboseInfo, out, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.Capture /* stdout */, utils.Show /* stderr */) + if ctx.Verbose { + ctx.Info(string(verboseInfo)) + } if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index 51057a21835..a6e4ad14314 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -35,7 +35,19 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { return errors.WithStack(err) } - objectFiles, err := builder_utils.CompileFiles(ctx, sketchBuildPath, sketchBuildPath, buildProperties, includes) + objectFiles, err := builder_utils.CompileFiles( + sketchBuildPath, sketchBuildPath, buildProperties, includes, + ctx.OnlyUpdateCompilationDatabase, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.Verbose, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) if err != nil { return errors.WithStack(err) } @@ -43,7 +55,19 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := sketchBuildPath.Join("src") if sketchSrcPath.IsDir() { - srcObjectFiles, err := builder_utils.CompileFilesRecursive(ctx, sketchSrcPath, sketchSrcPath, buildProperties, includes) + srcObjectFiles, err := builder_utils.CompileFilesRecursive( + sketchSrcPath, sketchSrcPath, buildProperties, includes, + ctx.OnlyUpdateCompilationDatabase, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.Verbose, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index 01257267287..497c5502ba3 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -56,7 +56,10 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { return nil } - _, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + verboseInfo, _, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + if ctx.Verbose { + ctx.Info(string(verboseInfo)) + } if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index ec34d1f2c3b..2dde7157990 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -17,6 +17,7 @@ package utils import ( "bytes" + "io" "os" "strings" @@ -51,17 +52,22 @@ const ( Capture = 3 // Capture into buffer ) -func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, stderr int) ([]byte, []byte, error) { - if ctx.Verbose { - ctx.Info(PrintableCommand(command.GetArgs())) +func ExecCommand( + verbose bool, + stdoutWriter, stderrWriter io.Writer, + command *executils.Process, stdout int, stderr int, +) ([]byte, []byte, []byte, error) { + verboseInfoBuf := &bytes.Buffer{} + if verbose { + verboseInfoBuf.WriteString(PrintableCommand(command.GetArgs())) } stdoutBuffer := &bytes.Buffer{} if stdout == Capture { command.RedirectStdoutTo(stdoutBuffer) - } else if stdout == Show || (stdout == ShowIfVerbose && ctx.Verbose) { - if ctx.Stdout != nil { - command.RedirectStdoutTo(ctx.Stdout) + } else if stdout == Show || (stdout == ShowIfVerbose && verbose) { + if stdoutWriter != nil { + command.RedirectStdoutTo(stdoutWriter) } else { command.RedirectStdoutTo(os.Stdout) } @@ -70,9 +76,9 @@ func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, std stderrBuffer := &bytes.Buffer{} if stderr == Capture { command.RedirectStderrTo(stderrBuffer) - } else if stderr == Show || (stderr == ShowIfVerbose && ctx.Verbose) { - if ctx.Stderr != nil { - command.RedirectStderrTo(ctx.Stderr) + } else if stderr == Show || (stderr == ShowIfVerbose && verbose) { + if stderrWriter != nil { + command.RedirectStderrTo(stderrWriter) } else { command.RedirectStderrTo(os.Stderr) } @@ -80,11 +86,11 @@ func ExecCommand(ctx *types.Context, command *executils.Process, stdout int, std err := command.Start() if err != nil { - return nil, nil, errors.WithStack(err) + return verboseInfoBuf.Bytes(), nil, nil, errors.WithStack(err) } err = command.Wait() - return stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) + return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) } type loggerAction struct { From 407bc42186cb71f3b88291dc7d68d66c1ce3836c Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 12:59:09 +0200 Subject: [PATCH 08/20] mvoe types.ProgressStruct in a dedicated pkg --- arduino/builder/progress/progress.go | 29 +++++++++++++++++ .../builder/progress}/progress_test.go | 4 +-- legacy/builder/builder_utils/utils.go | 8 ++--- legacy/builder/phases/core_builder.go | 6 ++-- legacy/builder/types/context.go | 31 ++----------------- 5 files changed, 40 insertions(+), 38 deletions(-) create mode 100644 arduino/builder/progress/progress.go rename {legacy/builder/types => arduino/builder/progress}/progress_test.go (97%) diff --git a/arduino/builder/progress/progress.go b/arduino/builder/progress/progress.go new file mode 100644 index 00000000000..278ce09c152 --- /dev/null +++ b/arduino/builder/progress/progress.go @@ -0,0 +1,29 @@ +package progress + +type Struct struct { + Progress float32 + StepAmount float32 + Parent *Struct +} + +func (p *Struct) AddSubSteps(steps int) { + p.Parent = &Struct{ + Progress: p.Progress, + StepAmount: p.StepAmount, + Parent: p.Parent, + } + if p.StepAmount == 0.0 { + p.StepAmount = 100.0 + } + p.StepAmount /= float32(steps) +} + +func (p *Struct) RemoveSubSteps() { + p.Progress = p.Parent.Progress + p.StepAmount = p.Parent.StepAmount + p.Parent = p.Parent.Parent +} + +func (p *Struct) CompleteStep() { + p.Progress += p.StepAmount +} diff --git a/legacy/builder/types/progress_test.go b/arduino/builder/progress/progress_test.go similarity index 97% rename from legacy/builder/types/progress_test.go rename to arduino/builder/progress/progress_test.go index 2d367acf898..fff54ca3415 100644 --- a/legacy/builder/types/progress_test.go +++ b/arduino/builder/progress/progress_test.go @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package types +package progress import ( "fmt" @@ -23,7 +23,7 @@ import ( ) func TestProgress(t *testing.T) { - p := &ProgressStruct{} + p := &Struct{} p.AddSubSteps(3) require.Equal(t, float32(0.0), p.Progress) require.InEpsilon(t, 33.33333, p.StepAmount, 0.00001) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index 8e0010815a0..c7f7216a4d1 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -26,12 +26,12 @@ import ( "sync" "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/progress" bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" @@ -79,7 +79,7 @@ func CompileFiles( stdoutWriter, stderrWriter io.Writer, verboseInfoFn func(msg string), verboseStdoutFn, verboseStderrFn func(data []byte), - progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, + progress *progress.Struct, progressCB rpc.TaskProgressCB, ) (paths.PathList, error) { return compileFiles( onlyUpdateCompilationDatabase, @@ -108,7 +108,7 @@ func CompileFilesRecursive( stdoutWriter, stderrWriter io.Writer, verboseInfoFn func(msg string), verboseStdoutFn, verboseStderrFn func(data []byte), - progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, + progress *progress.Struct, progressCB rpc.TaskProgressCB, ) (paths.PathList, error) { return compileFiles( onlyUpdateCompilationDatabase, @@ -139,7 +139,7 @@ func compileFiles( stdoutWriter, stderrWriter io.Writer, verboseInfoFn func(msg string), verboseStdoutFn, verboseStderrFn func(data []byte), - progress *types.ProgressStruct, + progress *progress.Struct, progressCB rpc.TaskProgressCB, ) (paths.PathList, error) { validExtensions := []string{} diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index e7ae16d404a..b6ced10ad86 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -25,13 +25,13 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/progress" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -54,7 +54,7 @@ func CoreBuilder( stdoutWriter, stderrWriter io.Writer, verboseInfoFn func(msg string), verboseStdoutFn, verboseStderrFn func(data []byte), - progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, + progress *progress.Struct, progressCB rpc.TaskProgressCB, ) (paths.PathList, *paths.Path, *paths.Path, error) { if err := coreBuildPath.MkdirAll(); err != nil { return nil, nil, coreBuildCachePath, errors.WithStack(err) @@ -101,7 +101,7 @@ func compileCore( stdoutWriter, stderrWriter io.Writer, verboseInfoFn func(msg string), verboseStdoutFn, verboseStderrFn func(data []byte), - progress *types.ProgressStruct, progressCB rpc.TaskProgressCB, + progress *progress.Struct, progressCB rpc.TaskProgressCB, ) (*paths.Path, paths.PathList, error) { coreFolder := buildProperties.GetPath("build.core.path") variantFolder := buildProperties.GetPath("build.variant.path") diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index d7a9d79d23e..bb29b33499e 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -24,6 +24,7 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/detector" + "github.com/arduino/arduino-cli/arduino/builder/progress" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" @@ -32,34 +33,6 @@ import ( properties "github.com/arduino/go-properties-orderedmap" ) -type ProgressStruct struct { - Progress float32 - StepAmount float32 - Parent *ProgressStruct -} - -func (p *ProgressStruct) AddSubSteps(steps int) { - p.Parent = &ProgressStruct{ - Progress: p.Progress, - StepAmount: p.StepAmount, - Parent: p.Parent, - } - if p.StepAmount == 0.0 { - p.StepAmount = 100.0 - } - p.StepAmount /= float32(steps) -} - -func (p *ProgressStruct) RemoveSubSteps() { - p.Progress = p.Parent.Progress - p.StepAmount = p.Parent.StepAmount - p.Parent = p.Parent.Parent -} - -func (p *ProgressStruct) CompleteStep() { - p.Progress += p.StepAmount -} - // Context structure type Context struct { Builder *builder.Builder @@ -105,7 +78,7 @@ type Context struct { Verbose bool // Dry run, only create progress map - Progress ProgressStruct + Progress progress.Struct // Send progress events to this callback ProgressCB rpc.TaskProgressCB From c3ffb4ff6a198f4831337c395b2842e9c9238602 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 13:45:07 +0200 Subject: [PATCH 09/20] move ExecCommand under arduino/utils --- arduino/builder/utils/utils.go | 69 +++++++++++++++++++++++++ arduino/builder/utils/utils_test.go | 27 ++++++++++ legacy/builder/builder_utils/utils.go | 9 ++-- legacy/builder/phases/linker.go | 2 +- legacy/builder/phases/sizer.go | 2 +- legacy/builder/recipe_runner.go | 2 +- legacy/builder/test/utils_test.go | 42 --------------- legacy/builder/utils/utils.go | 74 --------------------------- 8 files changed, 103 insertions(+), 124 deletions(-) create mode 100644 arduino/builder/utils/utils_test.go delete mode 100644 legacy/builder/test/utils_test.go diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index ae64dacb47c..32e5c84435b 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -1,10 +1,13 @@ package utils import ( + "bytes" + "io" "os" "strings" "unicode" + "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/go-paths-helper" "github.com/pkg/errors" @@ -179,3 +182,69 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat } return dir.ReadDir(fileFilter) } + +const ( + Ignore = 0 // Redirect to null + Show = 1 // Show on stdout/stderr as normal + ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise + Capture = 3 // Capture into buffer +) + +func printableArgument(arg string) string { + if strings.ContainsAny(arg, "\"\\ \t") { + arg = strings.ReplaceAll(arg, "\\", "\\\\") + arg = strings.ReplaceAll(arg, "\"", "\\\"") + return "\"" + arg + "\"" + } else { + return arg + } +} + +// Convert a command and argument slice back to a printable string. +// This adds basic escaping which is sufficient for debug output, but +// probably not for shell interpretation. This essentially reverses +// ParseCommandLine. +func PrintableCommand(parts []string) string { + return strings.Join(f.Map(parts, printableArgument), " ") +} + +func ExecCommand( + verbose bool, + stdoutWriter, stderrWriter io.Writer, + command *executils.Process, stdout int, stderr int, +) ([]byte, []byte, []byte, error) { + verboseInfoBuf := &bytes.Buffer{} + if verbose { + verboseInfoBuf.WriteString(PrintableCommand(command.GetArgs())) + } + + stdoutBuffer := &bytes.Buffer{} + if stdout == Capture { + command.RedirectStdoutTo(stdoutBuffer) + } else if stdout == Show || (stdout == ShowIfVerbose && verbose) { + if stdoutWriter != nil { + command.RedirectStdoutTo(stdoutWriter) + } else { + command.RedirectStdoutTo(os.Stdout) + } + } + + stderrBuffer := &bytes.Buffer{} + if stderr == Capture { + command.RedirectStderrTo(stderrBuffer) + } else if stderr == Show || (stderr == ShowIfVerbose && verbose) { + if stderrWriter != nil { + command.RedirectStderrTo(stderrWriter) + } else { + command.RedirectStderrTo(os.Stderr) + } + } + + err := command.Start() + if err != nil { + return verboseInfoBuf.Bytes(), nil, nil, errors.WithStack(err) + } + + err = command.Wait() + return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) +} diff --git a/arduino/builder/utils/utils_test.go b/arduino/builder/utils/utils_test.go new file mode 100644 index 00000000000..c7116bdcc21 --- /dev/null +++ b/arduino/builder/utils/utils_test.go @@ -0,0 +1,27 @@ +package utils_test + +import ( + "testing" + + "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/stretchr/testify/require" +) + +func TestPrintableCommand(t *testing.T) { + parts := []string{ + "/path/to/dir with spaces/cmd", + "arg1", + "arg-\"with\"-quotes", + "specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument", + "arg with spaces", + "arg\twith\t\ttabs", + "lastarg", + } + correct := "\"/path/to/dir with spaces/cmd\"" + + " arg1 \"arg-\\\"with\\\"-quotes\"" + + " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + + " lastarg" + result := utils.PrintableCommand(parts) + require.Equal(t, correct, result) +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index c7f7216a4d1..1a76ad9074e 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -27,12 +27,11 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/progress" - bUtils "github.com/arduino/arduino-cli/arduino/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/utils" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -51,7 +50,7 @@ func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...st } targetModTime := targetStat.ModTime() - files, err := bUtils.FindFilesInFolder(dir, true, extensions...) + files, err := utils.FindFilesInFolder(dir, true, extensions...) if err != nil { return false, err } @@ -147,7 +146,7 @@ func compileFiles( validExtensions = append(validExtensions, ext) } - sources, err := bUtils.FindFilesInFolder(sourceDir, recurse, validExtensions...) + sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) if err != nil { return nil, err } @@ -268,7 +267,7 @@ func compileFileWithRecipe( return nil, nil, nil, nil, errors.WithStack(err) } - objIsUpToDate, err := bUtils.ObjFileIsUpToDate(source, objectFile, depsFile) + objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile) if err != nil { return nil, nil, nil, nil, errors.WithStack(err) } diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index 5ba954714d4..68d1584e73a 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -18,11 +18,11 @@ package phases import ( "strings" + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index 90ee5981658..a4c89a4bc84 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -21,9 +21,9 @@ import ( "regexp" "strconv" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index 497c5502ba3..c5663705230 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -22,7 +22,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" properties "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/legacy/builder/test/utils_test.go b/legacy/builder/test/utils_test.go deleted file mode 100644 index a6a4dea8fb4..00000000000 --- a/legacy/builder/test/utils_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "testing" - - "github.com/arduino/arduino-cli/legacy/builder/utils" - "github.com/stretchr/testify/require" -) - -func TestPrintableCommand(t *testing.T) { - parts := []string{ - "/path/to/dir with spaces/cmd", - "arg1", - "arg-\"with\"-quotes", - "specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument", - "arg with spaces", - "arg\twith\t\ttabs", - "lastarg", - } - correct := "\"/path/to/dir with spaces/cmd\"" + - " arg1 \"arg-\\\"with\\\"-quotes\"" + - " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + - " \"arg with spaces\" \"arg\twith\t\ttabs\"" + - " lastarg" - result := utils.PrintableCommand(parts) - require.Equal(t, correct, result) -} diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 2dde7157990..954830c19c4 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -16,83 +16,9 @@ package utils import ( - "bytes" - "io" - "os" - "strings" - - "github.com/arduino/arduino-cli/executils" - f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" ) -func printableArgument(arg string) string { - if strings.ContainsAny(arg, "\"\\ \t") { - arg = strings.Replace(arg, "\\", "\\\\", -1) - arg = strings.Replace(arg, "\"", "\\\"", -1) - return "\"" + arg + "\"" - } else { - return arg - } -} - -// Convert a command and argument slice back to a printable string. -// This adds basic escaping which is sufficient for debug output, but -// probably not for shell interpretation. This essentially reverses -// ParseCommandLine. -func PrintableCommand(parts []string) string { - return strings.Join(f.Map(parts, printableArgument), " ") -} - -const ( - Ignore = 0 // Redirect to null - Show = 1 // Show on stdout/stderr as normal - ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise - Capture = 3 // Capture into buffer -) - -func ExecCommand( - verbose bool, - stdoutWriter, stderrWriter io.Writer, - command *executils.Process, stdout int, stderr int, -) ([]byte, []byte, []byte, error) { - verboseInfoBuf := &bytes.Buffer{} - if verbose { - verboseInfoBuf.WriteString(PrintableCommand(command.GetArgs())) - } - - stdoutBuffer := &bytes.Buffer{} - if stdout == Capture { - command.RedirectStdoutTo(stdoutBuffer) - } else if stdout == Show || (stdout == ShowIfVerbose && verbose) { - if stdoutWriter != nil { - command.RedirectStdoutTo(stdoutWriter) - } else { - command.RedirectStdoutTo(os.Stdout) - } - } - - stderrBuffer := &bytes.Buffer{} - if stderr == Capture { - command.RedirectStderrTo(stderrBuffer) - } else if stderr == Show || (stderr == ShowIfVerbose && verbose) { - if stderrWriter != nil { - command.RedirectStderrTo(stderrWriter) - } else { - command.RedirectStderrTo(os.Stderr) - } - } - - err := command.Start() - if err != nil { - return verboseInfoBuf.Bytes(), nil, nil, errors.WithStack(err) - } - - err = command.Wait() - return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) -} - type loggerAction struct { onlyIfVerbose bool warn bool From a2a9b99c55c67531aab47083b0d4f8ab6b367640 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 13:51:44 +0200 Subject: [PATCH 10/20] move LogIfVerbose from utils to legacy builder --- legacy/builder/builder.go | 27 +++++++++++++++++------ legacy/builder/utils/utils.go | 41 ----------------------------------- 2 files changed, 20 insertions(+), 48 deletions(-) delete mode 100644 legacy/builder/utils/utils.go diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index cb7d18ce157..ebc63291889 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -23,7 +23,6 @@ import ( "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/phases" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -51,26 +50,26 @@ func (s *Builder) Run(ctx *types.Context) error { return _err }), - utils.LogIfVerbose(false, tr("Detecting libraries used...")), + logIfVerbose(false, tr("Detecting libraries used...")), findIncludes(ctx), &WarnAboutArchIncompatibleLibraries{}, - utils.LogIfVerbose(false, tr("Generating function prototypes...")), + logIfVerbose(false, tr("Generating function prototypes...")), types.BareCommand(PreprocessSketch), - utils.LogIfVerbose(false, tr("Compiling sketch...")), + logIfVerbose(false, tr("Compiling sketch...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"}, &phases.SketchBuilder{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, - utils.LogIfVerbose(false, tr("Compiling libraries...")), + logIfVerbose(false, tr("Compiling libraries...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.prebuild", Suffix: ".pattern"}, &UnusedCompiledLibrariesRemover{}, &phases.LibrariesBuilder{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, - utils.LogIfVerbose(false, tr("Compiling core...")), + logIfVerbose(false, tr("Compiling core...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { @@ -98,7 +97,7 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, - utils.LogIfVerbose(false, tr("Linking everything together...")), + logIfVerbose(false, tr("Linking everything together...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.prelink", Suffix: ".pattern"}, &phases.Linker{}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.postlink", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, @@ -255,3 +254,17 @@ func findIncludes(ctx *types.Context) types.BareCommand { ) }) } + +func logIfVerbose(warn bool, msg string) types.BareCommand { + return types.BareCommand(func(ctx *types.Context) error { + if !ctx.Verbose { + return nil + } + if warn { + ctx.Warn(msg) + } else { + ctx.Info(msg) + } + return nil + }) +} diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go deleted file mode 100644 index 954830c19c4..00000000000 --- a/legacy/builder/utils/utils.go +++ /dev/null @@ -1,41 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package utils - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" -) - -type loggerAction struct { - onlyIfVerbose bool - warn bool - msg string -} - -func (l *loggerAction) Run(ctx *types.Context) error { - if !l.onlyIfVerbose || ctx.Verbose { - if l.warn { - ctx.Warn(l.msg) - } else { - ctx.Info(l.msg) - } - } - return nil -} - -func LogIfVerbose(warn bool, msg string) types.Command { - return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg} -} From 3ab6d54729976d67de7f41955dbebd853f165760 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 14:05:15 +0200 Subject: [PATCH 11/20] move some legacy constans in builder package --- arduino/builder/builder.go | 10 ++++++++++ legacy/builder/builder_utils/utils.go | 15 +++++++-------- legacy/builder/constants/constants.go | 7 ------- legacy/builder/phases/linker.go | 9 +++++---- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/arduino/builder/builder.go b/arduino/builder/builder.go index 7408ece5c3f..5238d74d2ce 100644 --- a/arduino/builder/builder.go +++ b/arduino/builder/builder.go @@ -17,6 +17,16 @@ package builder import "github.com/arduino/arduino-cli/arduino/sketch" +const ( + BuildPropertiesArchiveFile = "archive_file" + BuildPropertiesArchiveFilePath = "archive_file_path" + BuildPropertiesObjectFile = "object_file" + RecipeARPattern = "recipe.ar.pattern" + BuildPropertiesIncludes = "includes" + BuildPropertiesCompilerWarningFlags = "compiler.warning_flags" + Space = " " +) + // Builder is a Sketch builder. type Builder struct { sketch *sketch.Sketch diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index 1a76ad9074e..0b4c47d5d96 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -31,7 +31,6 @@ import ( "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" - "github.com/arduino/arduino-cli/legacy/builder/constants" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" @@ -251,8 +250,8 @@ func compileFileWithRecipe( verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} properties := buildProperties.Clone() - properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel)) - properties.Set(constants.BUILD_PROPERTIES_INCLUDES, strings.Join(includes, constants.SPACE)) + properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) + properties.Set(builder.BuildPropertiesIncludes, strings.Join(includes, builder.Space)) properties.SetPath("source_file", source) relativeSource, err := sourcePath.RelTo(source) if err != nil { @@ -261,7 +260,7 @@ func compileFileWithRecipe( depsFile := buildPath.Join(relativeSource.String() + ".d") objectFile := buildPath.Join(relativeSource.String() + ".o") - properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) err = objectFile.Parent().MkdirAll() if err != nil { return nil, nil, nil, nil, errors.WithStack(err) @@ -345,11 +344,11 @@ func ArchiveCompiledFiles( for _, objectFile := range objectFilesToArchive { properties := buildProperties.Clone() - properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE, archiveFilePath.Base()) - properties.SetPath(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, archiveFilePath) - properties.SetPath(constants.BUILD_PROPERTIES_OBJECT_FILE, objectFile) + properties.Set(builder.BuildPropertiesArchiveFile, archiveFilePath.Base()) + properties.SetPath(builder.BuildPropertiesArchiveFilePath, archiveFilePath) + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) - command, err := PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) + command, err := PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) if err != nil { return nil, verboseInfobuf.Bytes(), errors.WithStack(err) } diff --git a/legacy/builder/constants/constants.go b/legacy/builder/constants/constants.go index 19a828584e3..6834d1bc1b7 100644 --- a/legacy/builder/constants/constants.go +++ b/legacy/builder/constants/constants.go @@ -17,8 +17,6 @@ package constants const BUILD_OPTIONS_FILE = "build.options.json" -const BUILD_PROPERTIES_ARCHIVE_FILE = "archive_file" -const BUILD_PROPERTIES_ARCHIVE_FILE_PATH = "archive_file_path" const BUILD_PROPERTIES_ARCH_OVERRIDE_CHECK = "architecture.override_check" const BUILD_PROPERTIES_BOOTLOADER_FILE = "bootloader.file" const BUILD_PROPERTIES_BOOTLOADER_NOBLINK = "bootloader.noblink" @@ -27,9 +25,6 @@ const BUILD_PROPERTIES_BUILD_MCU = "build.mcu" const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags" const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags" const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags" -const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags" -const BUILD_PROPERTIES_INCLUDES = "includes" -const BUILD_PROPERTIES_OBJECT_FILE = "object_file" const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path" const EMPTY_STRING = "" const FOLDER_BOOTLOADERS = "bootloaders" @@ -50,9 +45,7 @@ const PACKAGE_TOOLS = "tools" const PLATFORM_ARCHITECTURE = "architecture" const PLATFORM_URL = "url" const PLATFORM_VERSION = "version" -const RECIPE_AR_PATTERN = "recipe.ar.pattern" const RECIPE_C_COMBINE_PATTERN = "recipe.c.combine.pattern" -const SPACE = " " const TOOL_NAME = "name" const TOOL_URL = "url" const TOOL_VERSION = "version" diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index 68d1584e73a..8d01d55b4b4 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -18,6 +18,7 @@ package phases import ( "strings" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" @@ -93,7 +94,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_AR_PATTERN, false) + command, err := builder_utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) if err != nil { return errors.WithStack(err) } @@ -112,9 +113,9 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties := buildProperties.Clone() properties.Set(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS)) - properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel)) - properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE, coreDotARelPath.String()) - properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, coreArchiveFilePath.String()) + properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+ctx.WarningsLevel)) + properties.Set(builder.BuildPropertiesArchiveFile, coreDotARelPath.String()) + properties.Set(builder.BuildPropertiesArchiveFilePath, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) From 5b6ed66fba290db8a1dab39a145d6d4c132769c6 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 14:16:13 +0200 Subject: [PATCH 12/20] move builder_utils under arduino/builder/utils pkg --- arduino/builder/utils/utils.go | 405 ++++++++++++++++- arduino/builder/utils/utils_test.go | 5 +- legacy/builder/builder_utils/utils.go | 412 ------------------ legacy/builder/create_cmake_rule.go | 3 +- legacy/builder/phases/core_builder.go | 15 +- legacy/builder/phases/libraries_builder.go | 12 +- legacy/builder/phases/linker.go | 5 +- legacy/builder/phases/sizer.go | 5 +- legacy/builder/phases/sketch_builder.go | 6 +- legacy/builder/recipe_runner.go | 3 +- ...out_build_path_if_build_options_changed.go | 6 +- 11 files changed, 422 insertions(+), 455 deletions(-) delete mode 100644 legacy/builder/builder_utils/utils.go diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index 32e5c84435b..d9455385d76 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -2,14 +2,24 @@ package utils import ( "bytes" + "fmt" "io" "os" + "path/filepath" + "runtime" "strings" + "sync" "unicode" + "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/executils" + "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/text/runes" @@ -17,6 +27,8 @@ import ( "golang.org/x/text/unicode/norm" ) +var tr = i18n.Tr + // ObjFileIsUpToDate fixdoc func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) @@ -140,13 +152,13 @@ func NormalizeUTF8(buf []byte) []byte { var sourceControlFolders = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} -// FilterOutSCCS is a ReadDirFilter that excludes known VSC or project files -func FilterOutSCCS(file *paths.Path) bool { +// filterOutSCCS is a ReadDirFilter that excludes known VSC or project files +func filterOutSCCS(file *paths.Path) bool { return !sourceControlFolders[file.Base()] } -// FilterReadableFiles is a ReadDirFilter that accepts only readable files -func FilterReadableFiles(file *paths.Path) bool { +// filterReadableFiles is a ReadDirFilter that accepts only readable files +func filterReadableFiles(file *paths.Path) bool { // See if the file is readable by opening it f, err := file.Open() if err != nil { @@ -163,9 +175,9 @@ var filterOutHiddenFiles = paths.FilterOutPrefixes(".") func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { fileFilter := paths.AndFilter( filterOutHiddenFiles, - FilterOutSCCS, + filterOutSCCS, paths.FilterOutDirectories(), - FilterReadableFiles, + filterReadableFiles, ) if len(extensions) > 0 { fileFilter = paths.AndFilter( @@ -176,7 +188,7 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat if recurse { dirFilter := paths.AndFilter( filterOutHiddenFiles, - FilterOutSCCS, + filterOutSCCS, ) return dir.ReadDirRecursiveFiltered(dirFilter, fileFilter) } @@ -204,10 +216,11 @@ func printableArgument(arg string) string { // This adds basic escaping which is sufficient for debug output, but // probably not for shell interpretation. This essentially reverses // ParseCommandLine. -func PrintableCommand(parts []string) string { +func printableCommand(parts []string) string { return strings.Join(f.Map(parts, printableArgument), " ") } +// ExecCommand fixdoc func ExecCommand( verbose bool, stdoutWriter, stderrWriter io.Writer, @@ -215,7 +228,7 @@ func ExecCommand( ) ([]byte, []byte, []byte, error) { verboseInfoBuf := &bytes.Buffer{} if verbose { - verboseInfoBuf.WriteString(PrintableCommand(command.GetArgs())) + verboseInfoBuf.WriteString(printableCommand(command.GetArgs())) } stdoutBuffer := &bytes.Buffer{} @@ -248,3 +261,377 @@ func ExecCommand( err = command.Wait() return verboseInfoBuf.Bytes(), stdoutBuffer.Bytes(), stderrBuffer.Bytes(), errors.WithStack(err) } + +// DirContentIsOlderThan DirContentIsOlderThan returns true if the content of the given directory is +// older than target file. If extensions are given, only the files with these +// extensions are tested. +func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { + targetStat, err := target.Stat() + if err != nil { + return false, err + } + targetModTime := targetStat.ModTime() + + files, err := FindFilesInFolder(dir, true, extensions...) + if err != nil { + return false, err + } + for _, file := range files { + file, err := file.Stat() + if err != nil { + return false, err + } + if file.ModTime().After(targetModTime) { + return false, nil + } + } + return true, nil +} + +// PrepareCommandForRecipe fixdoc +func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { + pattern := buildProperties.Get(recipe) + if pattern == "" { + return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) + } + + commandLine := buildProperties.ExpandPropsInString(pattern) + if removeUnsetProperties { + commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) + } + + parts, err := properties.SplitQuotedString(commandLine, `"'`, false) + if err != nil { + return nil, errors.WithStack(err) + } + + // if the overall commandline is too long for the platform + // try reducing the length by making the filenames relative + // and changing working directory to build.path + var relativePath string + if len(commandLine) > 30000 { + relativePath = buildProperties.Get("build.path") + for i, arg := range parts { + if _, err := os.Stat(arg); os.IsNotExist(err) { + continue + } + rel, err := filepath.Rel(relativePath, arg) + if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { + parts[i] = rel + } + } + } + + command, err := executils.NewProcess(nil, parts...) + if err != nil { + return nil, errors.WithStack(err) + } + if relativePath != "" { + command.SetDir(relativePath) + } + + return command, nil +} + +// CompileFiles fixdoc +func CompileFiles( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + false, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) +} + +// CompileFilesRecursive fixdoc +func CompileFilesRecursive( + sourceDir, buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + return compileFiles( + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + sourceDir, + true, + buildPath, buildProperties, includes, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) +} + +func compileFiles( + onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + sourceDir *paths.Path, + recurse bool, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + verbose bool, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, + progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + validExtensions := []string{} + for ext := range globals.SourceFilesValidExtensions { + validExtensions = append(validExtensions, ext) + } + + sources, err := FindFilesInFolder(sourceDir, recurse, validExtensions...) + if err != nil { + return nil, err + } + + progress.AddSubSteps(len(sources)) + defer progress.RemoveSubSteps() + + objectFiles := paths.NewPathList() + var objectFilesMux sync.Mutex + if len(sources) == 0 { + return objectFiles, nil + } + var errorsList []error + var errorsMux sync.Mutex + + queue := make(chan *paths.Path) + job := func(source *paths.Path) { + recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) + if !buildProperties.ContainsKey(recipe) { + recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) + } + objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe( + stdoutWriter, stderrWriter, + warningsLevel, + compilationDatabase, + verbose, + onlyUpdateCompilationDatabase, + sourceDir, source, buildPath, buildProperties, includes, recipe, + ) + if verbose { + verboseStdoutFn(verboseStdout) + verboseInfoFn(string(verboseInfo)) + } + verboseStderrFn(stderr) + if err != nil { + errorsMux.Lock() + errorsList = append(errorsList, err) + errorsMux.Unlock() + } else { + objectFilesMux.Lock() + objectFiles.Add(objectFile) + objectFilesMux.Unlock() + } + } + + // Spawn jobs runners + var wg sync.WaitGroup + if jobs == 0 { + jobs = runtime.NumCPU() + } + for i := 0; i < jobs; i++ { + wg.Add(1) + go func() { + for source := range queue { + job(source) + } + wg.Done() + }() + } + + // Feed jobs until error or done + for _, source := range sources { + errorsMux.Lock() + gotError := len(errorsList) > 0 + errorsMux.Unlock() + if gotError { + break + } + queue <- source + + progress.CompleteStep() + // PushProgress + if progressCB != nil { + progressCB(&rpc.TaskProgress{ + Percent: progress.Progress, + Completed: progress.Progress >= 100.0, + }) + } + } + close(queue) + wg.Wait() + if len(errorsList) > 0 { + // output the first error + return nil, errors.WithStack(errorsList[0]) + } + objectFiles.Sort() + return objectFiles, nil +} + +func compileFileWithRecipe( + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, + compilationDatabase *builder.CompilationDatabase, + verbose, onlyUpdateCompilationDatabase bool, + sourcePath *paths.Path, + source *paths.Path, + buildPath *paths.Path, + buildProperties *properties.Map, + includes []string, + recipe string, +) (*paths.Path, []byte, []byte, []byte, error) { + verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + + properties := buildProperties.Clone() + properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) + properties.Set(builder.BuildPropertiesIncludes, strings.Join(includes, builder.Space)) + properties.SetPath("source_file", source) + relativeSource, err := sourcePath.RelTo(source) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + depsFile := buildPath.Join(relativeSource.String() + ".d") + objectFile := buildPath.Join(relativeSource.String() + ".o") + + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) + err = objectFile.Parent().MkdirAll() + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + + objIsUpToDate, err := ObjFileIsUpToDate(source, objectFile, depsFile) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + + command, err := PrepareCommandForRecipe(properties, recipe, false) + if err != nil { + return nil, nil, nil, nil, errors.WithStack(err) + } + if compilationDatabase != nil { + compilationDatabase.Add(source, command) + } + if !objIsUpToDate && !onlyUpdateCompilationDatabase { + // Since this compile could be multithreaded, we first capture the command output + info, stdout, stderr, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, Capture, Capture) + // and transfer all at once at the end... + if verbose { + verboseInfo.Write(info) + verboseStdout.Write(stdout) + } + errOut.Write(stderr) + + // ...and then return the error + if err != nil { + return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err) + } + } else if verbose { + if objIsUpToDate { + verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile)) + } else { + verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile)) + } + } + + return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil +} + +// ArchiveCompiledFiles fixdoc +func ArchiveCompiledFiles( + buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map, + onlyUpdateCompilationDatabase, verbose bool, + stdoutWriter, stderrWriter io.Writer, +) (*paths.Path, []byte, error) { + verboseInfobuf := &bytes.Buffer{} + archiveFilePath := buildPath.JoinPath(archiveFile) + + if onlyUpdateCompilationDatabase { + if verbose { + verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath)) + } + return archiveFilePath, verboseInfobuf.Bytes(), nil + } + + if archiveFileStat, err := archiveFilePath.Stat(); err == nil { + rebuildArchive := false + for _, objectFile := range objectFilesToArchive { + objectFileStat, err := objectFile.Stat() + if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { + // need to rebuild the archive + rebuildArchive = true + break + } + } + + // something changed, rebuild the core archive + if rebuildArchive { + if err := archiveFilePath.Remove(); err != nil { + return nil, nil, errors.WithStack(err) + } + } else { + if verbose { + verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath)) + } + return archiveFilePath, verboseInfobuf.Bytes(), nil + } + } + + for _, objectFile := range objectFilesToArchive { + properties := buildProperties.Clone() + properties.Set(builder.BuildPropertiesArchiveFile, archiveFilePath.Base()) + properties.SetPath(builder.BuildPropertiesArchiveFilePath, archiveFilePath) + properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) + + command, err := PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) + if err != nil { + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) + } + + verboseInfo, _, _, err := ExecCommand(verbose, stdoutWriter, stderrWriter, command, ShowIfVerbose /* stdout */, Show /* stderr */) + if verbose { + verboseInfobuf.WriteString(string(verboseInfo)) + } + if err != nil { + return nil, verboseInfobuf.Bytes(), errors.WithStack(err) + } + } + + return archiveFilePath, verboseInfobuf.Bytes(), nil +} diff --git a/arduino/builder/utils/utils_test.go b/arduino/builder/utils/utils_test.go index c7116bdcc21..84cd9d699ec 100644 --- a/arduino/builder/utils/utils_test.go +++ b/arduino/builder/utils/utils_test.go @@ -1,9 +1,8 @@ -package utils_test +package utils import ( "testing" - "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/stretchr/testify/require" ) @@ -22,6 +21,6 @@ func TestPrintableCommand(t *testing.T) { " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + " lastarg" - result := utils.PrintableCommand(parts) + result := printableCommand(parts) require.Equal(t, correct, result) } diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go deleted file mode 100644 index 0b4c47d5d96..00000000000 --- a/legacy/builder/builder_utils/utils.go +++ /dev/null @@ -1,412 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder_utils - -import ( - "bytes" - "fmt" - "io" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - - "github.com/arduino/arduino-cli/arduino/builder" - "github.com/arduino/arduino-cli/arduino/builder/progress" - "github.com/arduino/arduino-cli/arduino/builder/utils" - "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/executils" - "github.com/arduino/arduino-cli/i18n" - rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" - "github.com/arduino/go-paths-helper" - "github.com/arduino/go-properties-orderedmap" - "github.com/pkg/errors" -) - -var tr = i18n.Tr - -// DirContentIsOlderThan returns true if the content of the given directory is -// older than target file. If extensions are given, only the files with these -// extensions are tested. -func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { - targetStat, err := target.Stat() - if err != nil { - return false, err - } - targetModTime := targetStat.ModTime() - - files, err := utils.FindFilesInFolder(dir, true, extensions...) - if err != nil { - return false, err - } - for _, file := range files { - file, err := file.Stat() - if err != nil { - return false, err - } - if file.ModTime().After(targetModTime) { - return false, nil - } - } - return true, nil -} - -func CompileFiles( - sourceDir, buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - onlyUpdateCompilationDatabase bool, - compilationDatabase *builder.CompilationDatabase, - jobs int, - verbose bool, - warningsLevel string, - stdoutWriter, stderrWriter io.Writer, - verboseInfoFn func(msg string), - verboseStdoutFn, verboseStderrFn func(data []byte), - progress *progress.Struct, progressCB rpc.TaskProgressCB, -) (paths.PathList, error) { - return compileFiles( - onlyUpdateCompilationDatabase, - compilationDatabase, - jobs, - sourceDir, - false, - buildPath, buildProperties, includes, - verbose, - warningsLevel, - stdoutWriter, stderrWriter, - verboseInfoFn, verboseStdoutFn, verboseStderrFn, - progress, progressCB, - ) -} - -func CompileFilesRecursive( - sourceDir, buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - onlyUpdateCompilationDatabase bool, - compilationDatabase *builder.CompilationDatabase, - jobs int, - verbose bool, - warningsLevel string, - stdoutWriter, stderrWriter io.Writer, - verboseInfoFn func(msg string), - verboseStdoutFn, verboseStderrFn func(data []byte), - progress *progress.Struct, progressCB rpc.TaskProgressCB, -) (paths.PathList, error) { - return compileFiles( - onlyUpdateCompilationDatabase, - compilationDatabase, - jobs, - sourceDir, - true, - buildPath, buildProperties, includes, - verbose, - warningsLevel, - stdoutWriter, stderrWriter, - verboseInfoFn, verboseStdoutFn, verboseStderrFn, - progress, progressCB, - ) -} - -func compileFiles( - onlyUpdateCompilationDatabase bool, - compilationDatabase *builder.CompilationDatabase, - jobs int, - sourceDir *paths.Path, - recurse bool, - buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - verbose bool, - warningsLevel string, - stdoutWriter, stderrWriter io.Writer, - verboseInfoFn func(msg string), - verboseStdoutFn, verboseStderrFn func(data []byte), - progress *progress.Struct, - progressCB rpc.TaskProgressCB, -) (paths.PathList, error) { - validExtensions := []string{} - for ext := range globals.SourceFilesValidExtensions { - validExtensions = append(validExtensions, ext) - } - - sources, err := utils.FindFilesInFolder(sourceDir, recurse, validExtensions...) - if err != nil { - return nil, err - } - - progress.AddSubSteps(len(sources)) - defer progress.RemoveSubSteps() - - objectFiles := paths.NewPathList() - var objectFilesMux sync.Mutex - if len(sources) == 0 { - return objectFiles, nil - } - var errorsList []error - var errorsMux sync.Mutex - - queue := make(chan *paths.Path) - job := func(source *paths.Path) { - recipe := fmt.Sprintf("recipe%s.o.pattern", source.Ext()) - if !buildProperties.ContainsKey(recipe) { - recipe = fmt.Sprintf("recipe%s.o.pattern", globals.SourceFilesValidExtensions[source.Ext()]) - } - objectFile, verboseInfo, verboseStdout, stderr, err := compileFileWithRecipe( - stdoutWriter, stderrWriter, - warningsLevel, - compilationDatabase, - verbose, - onlyUpdateCompilationDatabase, - sourceDir, source, buildPath, buildProperties, includes, recipe, - ) - if verbose { - verboseStdoutFn(verboseStdout) - verboseInfoFn(string(verboseInfo)) - } - verboseStderrFn(stderr) - if err != nil { - errorsMux.Lock() - errorsList = append(errorsList, err) - errorsMux.Unlock() - } else { - objectFilesMux.Lock() - objectFiles.Add(objectFile) - objectFilesMux.Unlock() - } - } - - // Spawn jobs runners - var wg sync.WaitGroup - if jobs == 0 { - jobs = runtime.NumCPU() - } - for i := 0; i < jobs; i++ { - wg.Add(1) - go func() { - for source := range queue { - job(source) - } - wg.Done() - }() - } - - // Feed jobs until error or done - for _, source := range sources { - errorsMux.Lock() - gotError := len(errorsList) > 0 - errorsMux.Unlock() - if gotError { - break - } - queue <- source - - progress.CompleteStep() - // PushProgress - if progressCB != nil { - progressCB(&rpc.TaskProgress{ - Percent: progress.Progress, - Completed: progress.Progress >= 100.0, - }) - } - } - close(queue) - wg.Wait() - if len(errorsList) > 0 { - // output the first error - return nil, errors.WithStack(errorsList[0]) - } - objectFiles.Sort() - return objectFiles, nil -} - -func compileFileWithRecipe( - stdoutWriter, stderrWriter io.Writer, - warningsLevel string, - compilationDatabase *builder.CompilationDatabase, - verbose, onlyUpdateCompilationDatabase bool, - sourcePath *paths.Path, - source *paths.Path, - buildPath *paths.Path, - buildProperties *properties.Map, - includes []string, - recipe string, -) (*paths.Path, []byte, []byte, []byte, error) { - verboseStdout, verboseInfo, errOut := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} - - properties := buildProperties.Clone() - properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) - properties.Set(builder.BuildPropertiesIncludes, strings.Join(includes, builder.Space)) - properties.SetPath("source_file", source) - relativeSource, err := sourcePath.RelTo(source) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - depsFile := buildPath.Join(relativeSource.String() + ".d") - objectFile := buildPath.Join(relativeSource.String() + ".o") - - properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) - err = objectFile.Parent().MkdirAll() - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - - objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - - command, err := PrepareCommandForRecipe(properties, recipe, false) - if err != nil { - return nil, nil, nil, nil, errors.WithStack(err) - } - if compilationDatabase != nil { - compilationDatabase.Add(source, command) - } - if !objIsUpToDate && !onlyUpdateCompilationDatabase { - // Since this compile could be multithreaded, we first capture the command output - info, stdout, stderr, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture, utils.Capture) - // and transfer all at once at the end... - if verbose { - verboseInfo.Write(info) - verboseStdout.Write(stdout) - } - errOut.Write(stderr) - - // ...and then return the error - if err != nil { - return nil, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), errors.WithStack(err) - } - } else if verbose { - if objIsUpToDate { - verboseInfo.WriteString(tr("Using previously compiled file: %[1]s", objectFile)) - } else { - verboseInfo.WriteString(tr("Skipping compile of: %[1]s", objectFile)) - } - } - - return objectFile, verboseInfo.Bytes(), verboseStdout.Bytes(), errOut.Bytes(), nil -} - -func ArchiveCompiledFiles( - buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map, - onlyUpdateCompilationDatabase, verbose bool, - stdoutWriter, stderrWriter io.Writer, -) (*paths.Path, []byte, error) { - verboseInfobuf := &bytes.Buffer{} - archiveFilePath := buildPath.JoinPath(archiveFile) - - if onlyUpdateCompilationDatabase { - if verbose { - verboseInfobuf.WriteString(tr("Skipping archive creation of: %[1]s", archiveFilePath)) - } - return archiveFilePath, verboseInfobuf.Bytes(), nil - } - - if archiveFileStat, err := archiveFilePath.Stat(); err == nil { - rebuildArchive := false - for _, objectFile := range objectFilesToArchive { - objectFileStat, err := objectFile.Stat() - if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) { - // need to rebuild the archive - rebuildArchive = true - break - } - } - - // something changed, rebuild the core archive - if rebuildArchive { - if err := archiveFilePath.Remove(); err != nil { - return nil, nil, errors.WithStack(err) - } - } else { - if verbose { - verboseInfobuf.WriteString(tr("Using previously compiled file: %[1]s", archiveFilePath)) - } - return archiveFilePath, verboseInfobuf.Bytes(), nil - } - } - - for _, objectFile := range objectFilesToArchive { - properties := buildProperties.Clone() - properties.Set(builder.BuildPropertiesArchiveFile, archiveFilePath.Base()) - properties.SetPath(builder.BuildPropertiesArchiveFilePath, archiveFilePath) - properties.SetPath(builder.BuildPropertiesObjectFile, objectFile) - - command, err := PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) - if err != nil { - return nil, verboseInfobuf.Bytes(), errors.WithStack(err) - } - - verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) - if verbose { - verboseInfobuf.WriteString(string(verboseInfo)) - } - if err != nil { - return nil, verboseInfobuf.Bytes(), errors.WithStack(err) - } - } - - return archiveFilePath, verboseInfobuf.Bytes(), nil -} - -const COMMANDLINE_LIMIT = 30000 - -func PrepareCommandForRecipe(buildProperties *properties.Map, recipe string, removeUnsetProperties bool) (*executils.Process, error) { - pattern := buildProperties.Get(recipe) - if pattern == "" { - return nil, errors.Errorf(tr("%[1]s pattern is missing"), recipe) - } - - commandLine := buildProperties.ExpandPropsInString(pattern) - if removeUnsetProperties { - commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) - } - - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return nil, errors.WithStack(err) - } - - // if the overall commandline is too long for the platform - // try reducing the length by making the filenames relative - // and changing working directory to build.path - var relativePath string - if len(commandLine) > COMMANDLINE_LIMIT { - relativePath = buildProperties.Get("build.path") - for i, arg := range parts { - if _, err := os.Stat(arg); os.IsNotExist(err) { - continue - } - rel, err := filepath.Rel(relativePath, arg) - if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - parts[i] = rel - } - } - } - - command, err := executils.NewProcess(nil, parts...) - if err != nil { - return nil, errors.WithStack(err) - } - if relativePath != "" { - command.SetDir(relativePath) - } - - return command, nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 6d6b67ae1d6..9ce4b068832 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -28,7 +28,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/globals" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" ) @@ -367,7 +366,7 @@ func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs return target } - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) + command, _ := utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true) for _, arg := range command.GetArgs() { if strings.HasPrefix(arg, "-D") { diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index b6ced10ad86..36a8335d1bf 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -26,11 +26,11 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/builder/progress" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" @@ -38,9 +38,6 @@ import ( "github.com/pkg/errors" ) -// Trasformo in un command -// provo a spostare builder_utils in arduino/Builder - var tr = i18n.Tr func CoreBuilder( @@ -116,7 +113,7 @@ func compileCore( var err error variantObjectFiles := paths.NewPathList() if variantFolder != nil && variantFolder.IsDir() { - variantObjectFiles, err = builder_utils.CompileFilesRecursive( + variantObjectFiles, err = utils.CompileFilesRecursive( variantFolder, buildPath, buildProperties, includes, onlyUpdateCompilationDatabase, compilationDatabase, @@ -149,12 +146,12 @@ func compileCore( var canUseArchivedCore bool if onlyUpdateCompilationDatabase || clean { canUseArchivedCore = false - } else if isOlder, err := builder_utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { + } else if isOlder, err := utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { // Recreate the archive if ANY of the core files (including platform.txt) has changed canUseArchivedCore = false } else if targetCoreFolder == nil || realCoreFolder.EquivalentTo(targetCoreFolder) { canUseArchivedCore = true - } else if isOlder, err := builder_utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { + } else if isOlder, err := utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { // Recreate the archive if ANY of the build core files (including platform.txt) has changed canUseArchivedCore = false } else { @@ -170,7 +167,7 @@ func compileCore( } } - coreObjectFiles, err := builder_utils.CompileFilesRecursive( + coreObjectFiles, err := utils.CompileFilesRecursive( coreFolder, buildPath, buildProperties, includes, onlyUpdateCompilationDatabase, compilationDatabase, @@ -185,7 +182,7 @@ func compileCore( return nil, nil, errors.WithStack(err) } - archiveFile, verboseInfo, err := builder_utils.ArchiveCompiledFiles( + archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( buildPath, paths.New("core.a"), coreObjectFiles, buildProperties, onlyUpdateCompilationDatabase, verbose, stdoutWriter, stderrWriter, ) diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index bbd02bada1a..15d34383989 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -19,9 +19,9 @@ import ( "strings" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -67,7 +67,7 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) + command, _ := utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" for _, el := range command.GetArgs() { if strings.Contains(el, FPU_CFLAG) { @@ -186,7 +186,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } if library.Layout == libraries.RecursiveLayout { - libObjectFiles, err := builder_utils.CompileFilesRecursive( + libObjectFiles, err := utils.CompileFilesRecursive( library.SourceDir, libraryBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, @@ -203,7 +203,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p return nil, errors.WithStack(err) } if library.DotALinkage { - archiveFile, verboseInfo, err := builder_utils.ArchiveCompiledFiles( + archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, buildProperties, ctx.OnlyUpdateCompilationDatabase, ctx.Verbose, ctx.Stdout, ctx.Stderr, @@ -222,7 +222,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { includes = append(includes, cpp.WrapWithHyphenI(library.UtilityDir.String())) } - libObjectFiles, err := builder_utils.CompileFiles( + libObjectFiles, err := utils.CompileFiles( library.SourceDir, libraryBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, @@ -242,7 +242,7 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.UtilityDir != nil { utilityBuildPath := libraryBuildPath.Join("utility") - utilityObjectFiles, err := builder_utils.CompileFiles( + utilityObjectFiles, err := utils.CompileFiles( library.UtilityDir, utilityBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index 8d01d55b4b4..ed1d8399a26 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -94,7 +93,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.SetPath("archive_file_path", archive) properties.SetPath("object_file", object) - command, err := builder_utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) + command, err := utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) if err != nil { return errors.WithStack(err) } @@ -118,7 +117,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths properties.Set(builder.BuildPropertiesArchiveFilePath, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := builder_utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) + command, err := utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) if err != nil { return err } diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index a4c89a4bc84..b0261daa105 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -22,7 +22,6 @@ import ( "strconv" "github.com/arduino/arduino-cli/arduino/builder/utils" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -50,7 +49,7 @@ func (s *Sizer) Run(ctx *types.Context) error { } func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) + command, err := utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) if err != nil { return errors.New(tr("Error while determining sketch size: %s", err)) } @@ -182,7 +181,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { } func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { - command, err := builder_utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) + command, err := utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index a6e4ad14314..bd71c06cd63 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -17,8 +17,8 @@ package phases import ( "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/pkg/errors" ) @@ -35,7 +35,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { return errors.WithStack(err) } - objectFiles, err := builder_utils.CompileFiles( + objectFiles, err := utils.CompileFiles( sketchBuildPath, sketchBuildPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, @@ -55,7 +55,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := sketchBuildPath.Join("src") if sketchSrcPath.IsDir() { - srcObjectFiles, err := builder_utils.CompileFilesRecursive( + srcObjectFiles, err := utils.CompileFilesRecursive( sketchSrcPath, sketchSrcPath, buildProperties, includes, ctx.OnlyUpdateCompilationDatabase, ctx.CompilationDatabase, diff --git a/legacy/builder/recipe_runner.go b/legacy/builder/recipe_runner.go index c5663705230..9e8bdcd2425 100644 --- a/legacy/builder/recipe_runner.go +++ b/legacy/builder/recipe_runner.go @@ -20,7 +20,6 @@ import ( "sort" "strings" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/arduino/builder/utils" properties "github.com/arduino/go-properties-orderedmap" @@ -44,7 +43,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { for _, recipe := range recipes { logrus.Debugf(fmt.Sprintf("Running recipe: %s", recipe)) - command, err := builder_utils.PrepareCommandForRecipe(properties, recipe, false) + command, err := utils.PrepareCommandForRecipe(properties, recipe, false) if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/wipeout_build_path_if_build_options_changed.go b/legacy/builder/wipeout_build_path_if_build_options_changed.go index 2a6873de97b..72b00b8671a 100644 --- a/legacy/builder/wipeout_build_path_if_build_options_changed.go +++ b/legacy/builder/wipeout_build_path_if_build_options_changed.go @@ -19,7 +19,7 @@ import ( "encoding/json" "path/filepath" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" + "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -66,9 +66,9 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { coreFolder := buildProperties.GetPath("build.core.path") realCoreFolder := coreFolder.Parent().Parent() jsonPath := ctx.BuildPath.Join(constants.BUILD_OPTIONS_FILE) - coreUnchanged, _ := builder_utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") + coreUnchanged, _ := utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") if coreUnchanged && targetCoreFolder != nil && !realCoreFolder.EqualsTo(targetCoreFolder) { - coreUnchanged, _ = builder_utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") + coreUnchanged, _ = utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") } if coreUnchanged { return nil From f4108043e972ea74900c51ec7c31cbba34ce1867 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 14:18:48 +0200 Subject: [PATCH 13/20] appease golint --- arduino/builder/builder.go | 1 + arduino/builder/progress/progress.go | 4 ++++ arduino/builder/utils/utils.go | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/arduino/builder/builder.go b/arduino/builder/builder.go index 5238d74d2ce..e8374b66e5a 100644 --- a/arduino/builder/builder.go +++ b/arduino/builder/builder.go @@ -17,6 +17,7 @@ package builder import "github.com/arduino/arduino-cli/arduino/sketch" +// nolint const ( BuildPropertiesArchiveFile = "archive_file" BuildPropertiesArchiveFilePath = "archive_file_path" diff --git a/arduino/builder/progress/progress.go b/arduino/builder/progress/progress.go index 278ce09c152..53a64e803bf 100644 --- a/arduino/builder/progress/progress.go +++ b/arduino/builder/progress/progress.go @@ -1,11 +1,13 @@ package progress +// Struct fixdoc type Struct struct { Progress float32 StepAmount float32 Parent *Struct } +// AddSubSteps fixdoc func (p *Struct) AddSubSteps(steps int) { p.Parent = &Struct{ Progress: p.Progress, @@ -18,12 +20,14 @@ func (p *Struct) AddSubSteps(steps int) { p.StepAmount /= float32(steps) } +// RemoveSubSteps fixdoc func (p *Struct) RemoveSubSteps() { p.Progress = p.Parent.Progress p.StepAmount = p.Parent.StepAmount p.Parent = p.Parent.Parent } +// CompleteStep fixdoc func (p *Struct) CompleteStep() { p.Progress += p.StepAmount } diff --git a/arduino/builder/utils/utils.go b/arduino/builder/utils/utils.go index d9455385d76..f5aad02c810 100644 --- a/arduino/builder/utils/utils.go +++ b/arduino/builder/utils/utils.go @@ -195,6 +195,7 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (pat return dir.ReadDir(fileFilter) } +// nolint const ( Ignore = 0 // Redirect to null Show = 1 // Show on stdout/stderr as normal @@ -207,9 +208,8 @@ func printableArgument(arg string) string { arg = strings.ReplaceAll(arg, "\\", "\\\\") arg = strings.ReplaceAll(arg, "\"", "\\\"") return "\"" + arg + "\"" - } else { - return arg } + return arg } // Convert a command and argument slice back to a printable string. From 3bd6c7fcf557c64948719d00554a377d0b90dd77 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Wed, 6 Sep 2023 17:34:25 +0200 Subject: [PATCH 14/20] move coreBuildCachePath in the arduino Builder --- arduino/builder/builder.go | 13 +++++++++--- arduino/builder/core.go | 8 ++++++++ arduino/builder/sketch_test.go | 6 +++--- commands/compile/compile.go | 29 ++++++++++++++------------- legacy/builder/builder.go | 5 ++--- legacy/builder/phases/core_builder.go | 10 ++++----- legacy/builder/test/builder_test.go | 2 +- legacy/builder/test/helper.go | 8 -------- legacy/builder/types/context.go | 1 - 9 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 arduino/builder/core.go diff --git a/arduino/builder/builder.go b/arduino/builder/builder.go index e8374b66e5a..70ea2ec4a07 100644 --- a/arduino/builder/builder.go +++ b/arduino/builder/builder.go @@ -15,7 +15,10 @@ package builder -import "github.com/arduino/arduino-cli/arduino/sketch" +import ( + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/go-paths-helper" +) // nolint const ( @@ -31,11 +34,15 @@ const ( // Builder is a Sketch builder. type Builder struct { sketch *sketch.Sketch + + // core related + coreBuildCachePath *paths.Path } // NewBuilder creates a sketch Builder. -func NewBuilder(sk *sketch.Sketch) *Builder { +func NewBuilder(sk *sketch.Sketch, coreBuildCachePath *paths.Path) *Builder { return &Builder{ - sketch: sk, + sketch: sk, + coreBuildCachePath: coreBuildCachePath, } } diff --git a/arduino/builder/core.go b/arduino/builder/core.go new file mode 100644 index 00000000000..c4d096f6f27 --- /dev/null +++ b/arduino/builder/core.go @@ -0,0 +1,8 @@ +package builder + +import "github.com/arduino/go-paths-helper" + +// CoreBuildCachePath fixdoc +func (b *Builder) CoreBuildCachePath() *paths.Path { + return b.coreBuildCachePath +} diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index b1fda4e6e8c..d8c1df2c4c0 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -48,7 +48,7 @@ func TestMergeSketchSources(t *testing.T) { } mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource) - b := NewBuilder(sk) + b := NewBuilder(sk, nil) offset, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 2, offset) @@ -61,7 +61,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) { require.NotNil(t, sk) // ensure not to include Arduino.h when it's already there - b := NewBuilder(sk) + b := NewBuilder(sk, nil) _, source, err := b.sketchMergeSources(nil) require.Nil(t, err) require.Equal(t, 1, strings.Count(source, "")) @@ -76,7 +76,7 @@ func TestCopyAdditionalFiles(t *testing.T) { sk1, err := sketch.New(paths.New("testdata", t.Name())) require.Nil(t, err) require.Equal(t, sk1.AdditionalFiles.Len(), 1) - b1 := NewBuilder(sk1) + b1 := NewBuilder(sk1, nil) // copy the sketch over, create a fake main file we don't care about it // but we need it for `SketchLoad` to succeed later diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 503dcc24f7c..aa81cbcbf75 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -155,7 +155,21 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream // cache is purged after compilation to not remove entries that might be required defer maybePurgeBuildCache() - sketchBuilder := bldr.NewBuilder(sk) + var coreBuildCachePath *paths.Path + if req.GetBuildCachePath() == "" { + coreBuildCachePath = paths.TempDir().Join("arduino", "cores") + } else { + buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs() + if err != nil { + return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} + } + if err := buildCachePath.MkdirAll(); err != nil { + return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} + } + coreBuildCachePath = buildCachePath.Join("core") + } + + sketchBuilder := bldr.NewBuilder(sk, coreBuildCachePath) // Add build properites related to sketch data buildProperties = sketchBuilder.SetupBuildProperties(buildProperties, buildPath, req.GetOptimizeForDebug()) @@ -206,19 +220,6 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.WarningsLevel = builder.DEFAULT_WARNINGS_LEVEL } - if req.GetBuildCachePath() == "" { - builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores") - } else { - buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs() - if err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} - } - if err := buildCachePath.MkdirAll(); err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build cache directory"), Cause: err} - } - builderCtx.CoreBuildCachePath = buildCachePath.Join("core") - } - builderCtx.BuiltInLibrariesDirs = configuration.IDEBuiltinLibrariesDir(configuration.Settings) builderCtx.Stdout = outStream diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index ebc63291889..12a35fccbca 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -73,8 +73,8 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.core.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - objectFiles, archiveFile, coreBuildCachePath, err := phases.CoreBuilder( - ctx.BuildPath, ctx.CoreBuildPath, ctx.CoreBuildCachePath, + objectFiles, archiveFile, err := phases.CoreBuilder( + ctx.BuildPath, ctx.CoreBuildPath, ctx.Builder.CoreBuildCachePath(), ctx.BuildProperties, ctx.ActualPlatform, ctx.Verbose, ctx.OnlyUpdateCompilationDatabase, ctx.Clean, @@ -90,7 +90,6 @@ func (s *Builder) Run(ctx *types.Context) error { ctx.CoreObjectsFiles = objectFiles ctx.CoreArchiveFilePath = archiveFile - ctx.CoreBuildCachePath = coreBuildCachePath return err }), diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 36a8335d1bf..5c5e0339e41 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -52,9 +52,9 @@ func CoreBuilder( verboseInfoFn func(msg string), verboseStdoutFn, verboseStderrFn func(data []byte), progress *progress.Struct, progressCB rpc.TaskProgressCB, -) (paths.PathList, *paths.Path, *paths.Path, error) { +) (paths.PathList, *paths.Path, error) { if err := coreBuildPath.MkdirAll(); err != nil { - return nil, nil, coreBuildCachePath, errors.WithStack(err) + return nil, nil, errors.WithStack(err) } if coreBuildCachePath != nil { @@ -63,7 +63,7 @@ func CoreBuilder( verboseInfoFn(tr("Running normal build of the core...")) coreBuildCachePath = nil } else if err := coreBuildCachePath.MkdirAll(); err != nil { - return nil, nil, coreBuildCachePath, errors.WithStack(err) + return nil, nil, errors.WithStack(err) } } @@ -81,10 +81,10 @@ func CoreBuilder( progress, progressCB, ) if err != nil { - return nil, nil, coreBuildCachePath, errors.WithStack(err) + return nil, nil, errors.WithStack(err) } - return objectFiles, archiveFile, coreBuildCachePath, nil + return objectFiles, archiveFile, nil } func compileCore( diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 1a0ec482567..deafc113154 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -97,7 +97,7 @@ func prepareBuilderTestContext(t *testing.T, ctx *types.Context, sketchPath *pat ctx.Sketch = sk } - ctx.Builder = bldr.NewBuilder(ctx.Sketch) + ctx.Builder = bldr.NewBuilder(ctx.Sketch, nil) if fqbn != "" { ctx.FQBN = parseFQBN(t, fqbn) targetPackage, targetPlatform, targetBoard, buildProperties, buildPlatform, err := pme.ResolveFQBN(ctx.FQBN) diff --git a/legacy/builder/test/helper.go b/legacy/builder/test/helper.go index 4ca6f318157..33b9aa1e2cf 100644 --- a/legacy/builder/test/helper.go +++ b/legacy/builder/test/helper.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" - "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -40,13 +39,6 @@ func SetupBuildPath(t *testing.T, ctx *types.Context) *paths.Path { return buildPath } -func SetupBuildCachePath(t *testing.T, ctx *types.Context) *paths.Path { - buildCachePath, err := paths.MkTempDir(constants.EMPTY_STRING, "test_build_cache") - require.NoError(t, err) - ctx.CoreBuildCachePath = buildCachePath - return buildCachePath -} - func parseFQBN(t *testing.T, fqbnIn string) *cores.FQBN { fqbn, err := cores.ParseFQBN(fqbnIn) require.NoError(t, err) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index bb29b33499e..14616a15fb8 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -61,7 +61,6 @@ type Context struct { BuildPath *paths.Path SketchBuildPath *paths.Path CoreBuildPath *paths.Path - CoreBuildCachePath *paths.Path CoreArchiveFilePath *paths.Path CoreObjectsFiles paths.PathList LibrariesBuildPath *paths.Path From 447419d1aa164507e7a1f1af21b7dc9614ab238c Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 7 Sep 2023 10:01:37 +0200 Subject: [PATCH 15/20] refactor Linker command in a function --- legacy/builder/builder.go | 22 ++++++- legacy/builder/constants/constants.go | 1 - legacy/builder/phases/linker.go | 85 +++++++++++++++------------ 3 files changed, 68 insertions(+), 40 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 12a35fccbca..9b4a8b0a01c 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -98,7 +98,27 @@ func (s *Builder) Run(ctx *types.Context) error { logIfVerbose(false, tr("Linking everything together...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.prelink", Suffix: ".pattern"}, - &phases.Linker{}, + + types.BareCommand(func(ctx *types.Context) error { + verboseInfoOut, err := phases.Linker( + ctx.OnlyUpdateCompilationDatabase, + ctx.Verbose, + ctx.SketchObjectFiles, + ctx.LibrariesObjectFiles, + ctx.CoreObjectsFiles, + ctx.CoreArchiveFilePath, + ctx.BuildPath, + ctx.BuildProperties, + ctx.Stdout, + ctx.Stderr, + ctx.WarningsLevel, + ) + if ctx.Verbose { + ctx.Info(string(verboseInfoOut)) + } + return err + }), + &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.linking.postlink", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.objcopy.preobjcopy", Suffix: ".pattern"}, diff --git a/legacy/builder/constants/constants.go b/legacy/builder/constants/constants.go index 6834d1bc1b7..d1a6e4887c6 100644 --- a/legacy/builder/constants/constants.go +++ b/legacy/builder/constants/constants.go @@ -22,7 +22,6 @@ const BUILD_PROPERTIES_BOOTLOADER_FILE = "bootloader.file" const BUILD_PROPERTIES_BOOTLOADER_NOBLINK = "bootloader.noblink" const BUILD_PROPERTIES_BUILD_BOARD = "build.board" const BUILD_PROPERTIES_BUILD_MCU = "build.mcu" -const BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS = "compiler.c.elf.flags" const BUILD_PROPERTIES_COMPILER_LDFLAGS = "compiler.ldflags" const BUILD_PROPERTIES_COMPILER_CPP_FLAGS = "compiler.cpp.flags" const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path" diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index ed1d8399a26..654c4e29bc4 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -16,55 +16,68 @@ package phases import ( + "bytes" + "io" "strings" "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type Linker struct{} - -func (s *Linker) Run(ctx *types.Context) error { - if ctx.OnlyUpdateCompilationDatabase { - if ctx.Verbose { - ctx.Info(tr("Skip linking of final executable.")) +func Linker( + onlyUpdateCompilationDatabase, verbose bool, + sketchObjectFiles, librariesObjectFiles, coreObjectsFiles paths.PathList, + coreArchiveFilePath, buildPath *paths.Path, + buildProperties *properties.Map, + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, +) ([]byte, error) { + verboseInfo := &bytes.Buffer{} + if onlyUpdateCompilationDatabase { + if verbose { + verboseInfo.WriteString(tr("Skip linking of final executable.")) } - return nil + return verboseInfo.Bytes(), nil } - objectFilesSketch := ctx.SketchObjectFiles - objectFilesLibraries := ctx.LibrariesObjectFiles - objectFilesCore := ctx.CoreObjectsFiles + objectFilesSketch := sketchObjectFiles + objectFilesLibraries := librariesObjectFiles + objectFilesCore := coreObjectsFiles objectFiles := paths.NewPathList() objectFiles.AddAll(objectFilesSketch) objectFiles.AddAll(objectFilesLibraries) objectFiles.AddAll(objectFilesCore) - coreArchiveFilePath := ctx.CoreArchiveFilePath - buildPath := ctx.BuildPath coreDotARelPath, err := buildPath.RelTo(coreArchiveFilePath) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - buildProperties := ctx.BuildProperties - - err = link(ctx, objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties) + verboseInfoOut, err := link( + objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties, + verbose, stdoutWriter, stderrWriter, warningsLevel, + ) + verboseInfo.Write(verboseInfoOut) if err != nil { - return errors.WithStack(err) + return verboseInfo.Bytes(), errors.WithStack(err) } - return nil + return verboseInfo.Bytes(), nil } -func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map) error { +func link( + objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + warningsLevel string, +) ([]byte, error) { + verboseBuffer := &bytes.Buffer{} + wrapWithDoubleQuotes := func(value string) string { return "\"" + value + "\"" } objectFileList := strings.Join(f.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ") // If command line length is too big (> 30000 chars), try to collect the object files into archives @@ -95,14 +108,14 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths command, err := utils.PrepareCommandForRecipe(properties, builder.RecipeARPattern, false) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - if verboseInfo, _, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil { - if ctx.Verbose { - ctx.Info(string(verboseInfo)) + if verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */); err != nil { + if verbose { + verboseBuffer.WriteString(string(verboseInfo)) } - return errors.WithStack(err) + return verboseBuffer.Bytes(), errors.WithStack(err) } } @@ -111,24 +124,20 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths } properties := buildProperties.Clone() - properties.Set(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS)) - properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+ctx.WarningsLevel)) + properties.Set("compiler.c.elf.flags", properties.Get("compiler.c.elf.flags")) + properties.Set(builder.BuildPropertiesCompilerWarningFlags, properties.Get(builder.BuildPropertiesCompilerWarningFlags+"."+warningsLevel)) properties.Set(builder.BuildPropertiesArchiveFile, coreDotARelPath.String()) properties.Set(builder.BuildPropertiesArchiveFilePath, coreArchiveFilePath.String()) properties.Set("object_files", objectFileList) - command, err := utils.PrepareCommandForRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false) + command, err := utils.PrepareCommandForRecipe(properties, "recipe.c.combine.pattern", false) if err != nil { - return err + return verboseBuffer.Bytes(), err } - verboseInfo, _, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) - if ctx.Verbose { - ctx.Info(string(verboseInfo)) + verboseInfo, _, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */) + if verbose { + verboseBuffer.WriteString(string(verboseInfo)) } - return err -} - -func wrapWithDoubleQuotes(value string) string { - return "\"" + value + "\"" + return verboseBuffer.Bytes(), err } From a688f452ab59515c2e34df1620ce8b2ac025df3c Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 7 Sep 2023 10:12:53 +0200 Subject: [PATCH 16/20] refactor SketchBuilder in a function --- legacy/builder/builder.go | 23 ++++++- legacy/builder/phases/sketch_builder.go | 81 ++++++++++++++----------- 2 files changed, 68 insertions(+), 36 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 9b4a8b0a01c..05fb52fa059 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -60,7 +60,28 @@ func (s *Builder) Run(ctx *types.Context) error { logIfVerbose(false, tr("Compiling sketch...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"}, - &phases.SketchBuilder{}, + types.BareCommand(func(ctx *types.Context) error { + sketchObjectFiles, err := phases.SketchBuilder( + ctx.SketchBuildPath, + ctx.BuildProperties, + ctx.SketchLibrariesDetector.IncludeFolders(), + ctx.OnlyUpdateCompilationDatabase, + ctx.Verbose, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.WarningsLevel, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) + if err != nil { + return err + } + ctx.SketchObjectFiles = sketchObjectFiles + return nil + }), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, logIfVerbose(false, tr("Compiling libraries...")), diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index bd71c06cd63..190604ddc0a 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,40 +16,53 @@ package phases import ( + "io" + + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/progress" "github.com/arduino/arduino-cli/arduino/builder/utils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/types" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + "github.com/arduino/go-paths-helper" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type SketchBuilder struct{} - -func (s *SketchBuilder) Run(ctx *types.Context) error { - sketchBuildPath := ctx.SketchBuildPath - buildProperties := ctx.BuildProperties - includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() +func SketchBuilder( + sketchBuildPath *paths.Path, + buildProperties *properties.Map, + includesFolders paths.PathList, + onlyUpdateCompilationDatabase, verbose bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectFiles, err := utils.CompileFiles( + sketchObjectFiles, err := utils.CompileFiles( sketchBuildPath, sketchBuildPath, buildProperties, includes, - ctx.OnlyUpdateCompilationDatabase, - ctx.CompilationDatabase, - ctx.Jobs, - ctx.Verbose, - ctx.WarningsLevel, - ctx.Stdout, ctx.Stderr, - func(msg string) { ctx.Info(msg) }, - func(data []byte) { ctx.WriteStdout(data) }, - func(data []byte) { ctx.WriteStderr(data) }, - &ctx.Progress, ctx.ProgressCB, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, + verboseStderrFn, + progress, progressCB, ) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } // The "src/" subdirectory of a sketch is compiled recursively @@ -57,24 +70,22 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { if sketchSrcPath.IsDir() { srcObjectFiles, err := utils.CompileFilesRecursive( sketchSrcPath, sketchSrcPath, buildProperties, includes, - ctx.OnlyUpdateCompilationDatabase, - ctx.CompilationDatabase, - ctx.Jobs, - ctx.Verbose, - ctx.WarningsLevel, - ctx.Stdout, ctx.Stderr, - func(msg string) { ctx.Info(msg) }, - func(data []byte) { ctx.WriteStdout(data) }, - func(data []byte) { ctx.WriteStderr(data) }, - &ctx.Progress, ctx.ProgressCB, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, + verboseStderrFn, + progress, progressCB, ) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectFiles.AddAll(srcObjectFiles) + sketchObjectFiles.AddAll(srcObjectFiles) } - ctx.SketchObjectFiles = objectFiles - - return nil + return sketchObjectFiles, nil } From a05fa1217fc1ea4e96cf30f09ea484cf81ba47bd Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 7 Sep 2023 10:38:26 +0200 Subject: [PATCH 17/20] refactor LibrariesBuilder in a function --- legacy/builder/builder.go | 26 ++- legacy/builder/phases/libraries_builder.go | 195 +++++++++++++-------- 2 files changed, 151 insertions(+), 70 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 05fb52fa059..278245fcf9f 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -87,7 +87,31 @@ func (s *Builder) Run(ctx *types.Context) error { logIfVerbose(false, tr("Compiling libraries...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.prebuild", Suffix: ".pattern"}, &UnusedCompiledLibrariesRemover{}, - &phases.LibrariesBuilder{}, + types.BareCommand(func(ctx *types.Context) error { + librariesObjectFiles, err := phases.LibrariesBuilder( + ctx.LibrariesBuildPath, + ctx.BuildProperties, + ctx.SketchLibrariesDetector.IncludeFolders(), + ctx.SketchLibrariesDetector.ImportedLibraries(), + ctx.Verbose, + ctx.OnlyUpdateCompilationDatabase, + ctx.CompilationDatabase, + ctx.Jobs, + ctx.WarningsLevel, + ctx.Stdout, + ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(data []byte) { ctx.WriteStdout(data) }, + func(data []byte) { ctx.WriteStderr(data) }, + &ctx.Progress, ctx.ProgressCB, + ) + if err != nil { + return err + } + ctx.LibrariesObjectFiles = librariesObjectFiles + + return nil + }), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.libraries.postbuild", Suffix: ".pattern", SkipIfOnlyUpdatingCompilationDatabase: true}, logIfVerbose(false, tr("Compiling core...")), diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 15d34383989..8608b292916 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -16,14 +16,17 @@ package phases import ( + "io" "strings" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/arduino/arduino-cli/arduino/builder/progress" "github.com/arduino/arduino-cli/arduino/builder/utils" "github.com/arduino/arduino-cli/arduino/libraries" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/types" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -32,26 +35,43 @@ import ( var FLOAT_ABI_CFLAG = "float-abi" var FPU_CFLAG = "fpu" -type LibrariesBuilder struct{} - -func (s *LibrariesBuilder) Run(ctx *types.Context) error { - librariesBuildPath := ctx.LibrariesBuildPath - buildProperties := ctx.BuildProperties - includesFolders := ctx.SketchLibrariesDetector.IncludeFolders() +func LibrariesBuilder( + librariesBuildPath *paths.Path, + buildProperties *properties.Map, + includesFolders paths.PathList, + importedLibraries libraries.List, + verbose, onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { includes := f.Map(includesFolders.AsStrings(), cpp.WrapWithHyphenI) - libs := ctx.SketchLibrariesDetector.ImportedLibraries() + libs := importedLibraries if err := librariesBuildPath.MkdirAll(); err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - objectFiles, err := compileLibraries(ctx, libs, librariesBuildPath, buildProperties, includes) + librariesObjectFiles, err := compileLibraries( + libs, librariesBuildPath, buildProperties, includes, + verbose, onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, + verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - ctx.LibrariesObjectFiles = objectFiles - return nil + return librariesObjectFiles, nil } func directoryContainsFile(folder *paths.Path) bool { @@ -62,12 +82,16 @@ func directoryContainsFile(folder *paths.Path) bool { return false } -func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Library) *paths.Path { - mcu := ctx.BuildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) +func findExpectedPrecompiledLibFolder( + library *libraries.Library, + buildProperties *properties.Map, + verboseInfoFn func(msg string), +) *paths.Path { + mcu := buildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) // Add fpu specifications if they exist // To do so, resolve recipe.cpp.o.pattern, // search for -mfpu=xxx -mfloat-abi=yyy and add to a subfolder - command, _ := utils.PrepareCommandForRecipe(ctx.BuildProperties, "recipe.cpp.o.pattern", true) + command, _ := utils.PrepareCommandForRecipe(buildProperties, "recipe.cpp.o.pattern", true) fpuSpecs := "" for _, el := range command.GetArgs() { if strings.Contains(el, FPU_CFLAG) { @@ -88,50 +112,85 @@ func findExpectedPrecompiledLibFolder(ctx *types.Context, library *libraries.Lib } } - ctx.Info(tr("Library %[1]s has been declared precompiled:", library.Name)) + verboseInfoFn(tr("Library %[1]s has been declared precompiled:", library.Name)) // Try directory with full fpuSpecs first, if available if len(fpuSpecs) > 0 { fpuSpecs = strings.TrimRight(fpuSpecs, "-") fullPrecompDir := library.SourceDir.Join(mcu).Join(fpuSpecs) if fullPrecompDir.Exist() && directoryContainsFile(fullPrecompDir) { - ctx.Info(tr("Using precompiled library in %[1]s", fullPrecompDir)) + verboseInfoFn(tr("Using precompiled library in %[1]s", fullPrecompDir)) return fullPrecompDir } - ctx.Info(tr(`Precompiled library in "%[1]s" not found`, fullPrecompDir)) + verboseInfoFn(tr(`Precompiled library in "%[1]s" not found`, fullPrecompDir)) } precompDir := library.SourceDir.Join(mcu) if precompDir.Exist() && directoryContainsFile(precompDir) { - ctx.Info(tr("Using precompiled library in %[1]s", precompDir)) + verboseInfoFn(tr("Using precompiled library in %[1]s", precompDir)) return precompDir } - ctx.Info(tr(`Precompiled library in "%[1]s" not found`, precompDir)) + verboseInfoFn(tr(`Precompiled library in "%[1]s" not found`, precompDir)) return nil } -func compileLibraries(ctx *types.Context, libraries libraries.List, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - ctx.Progress.AddSubSteps(len(libraries)) - defer ctx.Progress.RemoveSubSteps() +func compileLibraries( + libraries libraries.List, buildPath *paths.Path, buildProperties *properties.Map, includes []string, + verbose, onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + progress.AddSubSteps(len(libraries)) + defer progress.RemoveSubSteps() objectFiles := paths.NewPathList() for _, library := range libraries { - libraryObjectFiles, err := compileLibrary(ctx, library, buildPath, buildProperties, includes) + libraryObjectFiles, err := compileLibrary( + library, buildPath, buildProperties, includes, + verbose, onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, + ) if err != nil { return nil, errors.WithStack(err) } objectFiles.AddAll(libraryObjectFiles) - ctx.Progress.CompleteStep() - ctx.PushProgress() + progress.CompleteStep() + // PushProgress + if progressCB != nil { + progressCB(&rpc.TaskProgress{ + Percent: progress.Progress, + Completed: progress.Progress >= 100.0, + }) + } } return objectFiles, nil } -func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { - if ctx.Verbose { - ctx.Info(tr(`Compiling library "%[1]s"`, library.Name)) +func compileLibrary( + library *libraries.Library, buildPath *paths.Path, buildProperties *properties.Map, includes []string, + verbose, onlyUpdateCompilationDatabase bool, + compilationDatabase *builder.CompilationDatabase, + jobs int, + warningsLevel string, + stdoutWriter, stderrWriter io.Writer, + verboseInfoFn func(msg string), + verboseStdoutFn, verboseStderrFn func(data []byte), + progress *progress.Struct, progressCB rpc.TaskProgressCB, +) (paths.PathList, error) { + if verbose { + verboseInfoFn(tr(`Compiling library "%[1]s"`, library.Name)) } libraryBuildPath := buildPath.Join(library.DirName) @@ -142,11 +201,15 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p objectFiles := paths.NewPathList() if library.Precompiled { - coreSupportPrecompiled := ctx.BuildProperties.ContainsKey("compiler.libraries.ldflags") - precompiledPath := findExpectedPrecompiledLibFolder(ctx, library) + coreSupportPrecompiled := buildProperties.ContainsKey("compiler.libraries.ldflags") + precompiledPath := findExpectedPrecompiledLibFolder( + library, + buildProperties, + verboseInfoFn, + ) if !coreSupportPrecompiled { - ctx.Info(tr("The platform does not support '%[1]s' for precompiled libraries.", "compiler.libraries.ldflags")) + verboseInfoFn(tr("The platform does not support '%[1]s' for precompiled libraries.", "compiler.libraries.ldflags")) } else if precompiledPath != nil { // Find all libraries in precompiledPath libs, err := precompiledPath.ReadDir() @@ -165,8 +228,8 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } } - currLDFlags := ctx.BuildProperties.Get("compiler.libraries.ldflags") - ctx.BuildProperties.Set("compiler.libraries.ldflags", currLDFlags+" \"-L"+precompiledPath.String()+"\" "+libsCmd+" ") + currLDFlags := buildProperties.Get("compiler.libraries.ldflags") + buildProperties.Set("compiler.libraries.ldflags", currLDFlags+" \"-L"+precompiledPath.String()+"\" "+libsCmd+" ") // TODO: This codepath is just taken for .a with unusual names that would // be ignored by -L / -l methods. @@ -188,16 +251,14 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.Layout == libraries.RecursiveLayout { libObjectFiles, err := utils.CompileFilesRecursive( library.SourceDir, libraryBuildPath, buildProperties, includes, - ctx.OnlyUpdateCompilationDatabase, - ctx.CompilationDatabase, - ctx.Jobs, - ctx.Verbose, - ctx.WarningsLevel, - ctx.Stdout, ctx.Stderr, - func(msg string) { ctx.Info(msg) }, - func(data []byte) { ctx.WriteStdout(data) }, - func(data []byte) { ctx.WriteStderr(data) }, - &ctx.Progress, ctx.ProgressCB, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, ) if err != nil { return nil, errors.WithStack(err) @@ -205,11 +266,11 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p if library.DotALinkage { archiveFile, verboseInfo, err := utils.ArchiveCompiledFiles( libraryBuildPath, paths.New(library.DirName+".a"), libObjectFiles, buildProperties, - ctx.OnlyUpdateCompilationDatabase, ctx.Verbose, - ctx.Stdout, ctx.Stderr, + onlyUpdateCompilationDatabase, verbose, + stdoutWriter, stderrWriter, ) - if ctx.Verbose { - ctx.Info(string(verboseInfo)) + if verbose { + verboseInfoFn(string(verboseInfo)) } if err != nil { return nil, errors.WithStack(err) @@ -224,16 +285,14 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p } libObjectFiles, err := utils.CompileFiles( library.SourceDir, libraryBuildPath, buildProperties, includes, - ctx.OnlyUpdateCompilationDatabase, - ctx.CompilationDatabase, - ctx.Jobs, - ctx.Verbose, - ctx.WarningsLevel, - ctx.Stdout, ctx.Stderr, - func(msg string) { ctx.Info(msg) }, - func(data []byte) { ctx.WriteStdout(data) }, - func(data []byte) { ctx.WriteStderr(data) }, - &ctx.Progress, ctx.ProgressCB, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, ) if err != nil { return nil, errors.WithStack(err) @@ -244,16 +303,14 @@ func compileLibrary(ctx *types.Context, library *libraries.Library, buildPath *p utilityBuildPath := libraryBuildPath.Join("utility") utilityObjectFiles, err := utils.CompileFiles( library.UtilityDir, utilityBuildPath, buildProperties, includes, - ctx.OnlyUpdateCompilationDatabase, - ctx.CompilationDatabase, - ctx.Jobs, - ctx.Verbose, - ctx.WarningsLevel, - ctx.Stdout, ctx.Stderr, - func(msg string) { ctx.Info(msg) }, - func(data []byte) { ctx.WriteStdout(data) }, - func(data []byte) { ctx.WriteStderr(data) }, - &ctx.Progress, ctx.ProgressCB, + onlyUpdateCompilationDatabase, + compilationDatabase, + jobs, + verbose, + warningsLevel, + stdoutWriter, stderrWriter, + verboseInfoFn, verboseStdoutFn, verboseStderrFn, + progress, progressCB, ) if err != nil { return nil, errors.WithStack(err) From 70a018f20bed9f099285a435bea3253b0b10036c Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 7 Sep 2023 11:18:35 +0200 Subject: [PATCH 18/20] refactor Sizer in a function --- arduino/builder/sizer.go | 26 +++++++ legacy/builder/builder.go | 13 +++- legacy/builder/phases/sizer.go | 121 +++++++++++++++++--------------- legacy/builder/types/context.go | 25 +------ 4 files changed, 105 insertions(+), 80 deletions(-) create mode 100644 arduino/builder/sizer.go diff --git a/arduino/builder/sizer.go b/arduino/builder/sizer.go new file mode 100644 index 00000000000..911f261af67 --- /dev/null +++ b/arduino/builder/sizer.go @@ -0,0 +1,26 @@ +package builder + +import rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + +// ExecutableSectionSize represents a section of the executable output file +type ExecutableSectionSize struct { + Name string `json:"name"` + Size int `json:"size"` + MaxSize int `json:"max_size"` +} + +// ExecutablesFileSections is an array of ExecutablesFileSection +type ExecutablesFileSections []ExecutableSectionSize + +// ToRPCExecutableSectionSizeArray transforms this array into a []*rpc.ExecutableSectionSize +func (s ExecutablesFileSections) ToRPCExecutableSectionSizeArray() []*rpc.ExecutableSectionSize { + res := []*rpc.ExecutableSectionSize{} + for _, section := range s { + res = append(res, &rpc.ExecutableSectionSize{ + Name: section.Name, + Size: int64(section.Size), + MaxSize: int64(section.MaxSize), + }) + } + return res +} diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 278245fcf9f..2ad23985ce5 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -204,7 +204,18 @@ func (s *Builder) Run(ctx *types.Context) error { &ExportProjectCMake{SketchError: mainErr != nil}, - &phases.Sizer{SketchError: mainErr != nil}, + types.BareCommand(func(ctx *types.Context) error { + executableSectionsSize, err := phases.Sizer( + ctx.OnlyUpdateCompilationDatabase, mainErr != nil, ctx.Verbose, + ctx.BuildProperties, + ctx.Stdout, ctx.Stderr, + func(msg string) { ctx.Info(msg) }, + func(msg string) { ctx.Warn(msg) }, + ctx.WarningsLevel, + ) + ctx.ExecutableSectionsSize = executableSectionsSize + return err + }), } for _, command := range commands { PrintRingNameIfDebug(ctx, command) diff --git a/legacy/builder/phases/sizer.go b/legacy/builder/phases/sizer.go index b0261daa105..cfc28f987c1 100644 --- a/legacy/builder/phases/sizer.go +++ b/legacy/builder/phases/sizer.go @@ -18,48 +18,50 @@ package phases import ( "encoding/json" "fmt" + "io" "regexp" "strconv" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/builder/utils" - "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type Sizer struct { - SketchError bool -} - -func (s *Sizer) Run(ctx *types.Context) error { - if ctx.OnlyUpdateCompilationDatabase { - return nil +func Sizer( + onlyUpdateCompilationDatabase, sketchError, verbose bool, + buildProperties *properties.Map, + stdoutWriter, stderrWriter io.Writer, + printInfoFn, printWarnFn func(msg string), + warningsLevel string, +) (builder.ExecutablesFileSections, error) { + if onlyUpdateCompilationDatabase || sketchError { + return nil, nil } - if s.SketchError { - return nil - } - - buildProperties := ctx.BuildProperties if buildProperties.ContainsKey("recipe.advanced_size.pattern") { - return checkSizeAdvanced(ctx, buildProperties) + return checkSizeAdvanced(buildProperties, verbose, stdoutWriter, stderrWriter, printInfoFn, printWarnFn) } - return checkSize(ctx, buildProperties) + return checkSize(buildProperties, verbose, stdoutWriter, stderrWriter, printInfoFn, printWarnFn, warningsLevel) } -func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { - command, err := utils.PrepareCommandForRecipe(properties, "recipe.advanced_size.pattern", false) +func checkSizeAdvanced(buildProperties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + printInfoFn, printWarnFn func(msg string), +) (builder.ExecutablesFileSections, error) { + command, err := utils.PrepareCommandForRecipe(buildProperties, "recipe.advanced_size.pattern", false) if err != nil { - return errors.New(tr("Error while determining sketch size: %s", err)) + return nil, errors.New(tr("Error while determining sketch size: %s", err)) } - verboseInfo, out, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.Capture /* stdout */, utils.Show /* stderr */) - if ctx.Verbose { - ctx.Info(string(verboseInfo)) + verboseInfo, out, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture /* stdout */, utils.Show /* stderr */) + if verbose { + printInfoFn(string(verboseInfo)) } if err != nil { - return errors.New(tr("Error while determining sketch size: %s", err)) + return nil, errors.New(tr("Error while determining sketch size: %s", err)) } type AdvancedSizerResponse struct { @@ -69,7 +71,7 @@ func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { // likely be printed in red. Errors will stop build/upload. Severity string `json:"severity"` // Sections are the sections sizes for machine readable use - Sections []types.ExecutableSectionSize `json:"sections"` + Sections []builder.ExecutableSectionSize `json:"sections"` // ErrorMessage is a one line error message like: // "text section exceeds available space in board" // it must be set when Severity is "error" @@ -78,71 +80,76 @@ func checkSizeAdvanced(ctx *types.Context, properties *properties.Map) error { var resp AdvancedSizerResponse if err := json.Unmarshal(out, &resp); err != nil { - return errors.New(tr("Error while determining sketch size: %s", err)) + return nil, errors.New(tr("Error while determining sketch size: %s", err)) } - ctx.ExecutableSectionsSize = resp.Sections + executableSectionsSize := resp.Sections switch resp.Severity { case "error": - ctx.Warn(resp.Output) - return errors.New(resp.ErrorMessage) + printWarnFn(resp.Output) + return executableSectionsSize, errors.New(resp.ErrorMessage) case "warning": - ctx.Warn(resp.Output) + printWarnFn(resp.Output) case "info": - ctx.Info(resp.Output) + printInfoFn(resp.Output) default: - return fmt.Errorf("invalid '%s' severity from sketch sizer: it must be 'error', 'warning' or 'info'", resp.Severity) + return executableSectionsSize, fmt.Errorf("invalid '%s' severity from sketch sizer: it must be 'error', 'warning' or 'info'", resp.Severity) } - return nil + return executableSectionsSize, nil } -func checkSize(ctx *types.Context, buildProperties *properties.Map) error { +func checkSize(buildProperties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + printInfoFn, printWarnFn func(msg string), + warningsLevel string, +) (builder.ExecutablesFileSections, error) { properties := buildProperties.Clone() - properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+ctx.WarningsLevel)) + properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+warningsLevel)) maxTextSizeString := properties.Get("upload.maximum_size") maxDataSizeString := properties.Get("upload.maximum_data_size") if maxTextSizeString == "" { - return nil + return nil, nil } maxTextSize, err := strconv.Atoi(maxTextSizeString) if err != nil { - return err + return nil, err } maxDataSize := -1 if maxDataSizeString != "" { maxDataSize, err = strconv.Atoi(maxDataSizeString) if err != nil { - return err + return nil, err } } - textSize, dataSize, _, err := execSizeRecipe(ctx, properties) + textSize, dataSize, _, err := execSizeRecipe(properties, verbose, stdoutWriter, stderrWriter, printInfoFn) if err != nil { - ctx.Warn(tr("Couldn't determine program size")) - return nil + printWarnFn(tr("Couldn't determine program size")) + return nil, nil } - ctx.Info(tr("Sketch uses %[1]s bytes (%[3]s%%) of program storage space. Maximum is %[2]s bytes.", + printInfoFn(tr("Sketch uses %[1]s bytes (%[3]s%%) of program storage space. Maximum is %[2]s bytes.", strconv.Itoa(textSize), strconv.Itoa(maxTextSize), strconv.Itoa(textSize*100/maxTextSize))) if dataSize >= 0 { if maxDataSize > 0 { - ctx.Info(tr("Global variables use %[1]s bytes (%[3]s%%) of dynamic memory, leaving %[4]s bytes for local variables. Maximum is %[2]s bytes.", + printInfoFn(tr("Global variables use %[1]s bytes (%[3]s%%) of dynamic memory, leaving %[4]s bytes for local variables. Maximum is %[2]s bytes.", strconv.Itoa(dataSize), strconv.Itoa(maxDataSize), strconv.Itoa(dataSize*100/maxDataSize), strconv.Itoa(maxDataSize-dataSize))) } else { - ctx.Info(tr("Global variables use %[1]s bytes of dynamic memory.", strconv.Itoa(dataSize))) + printInfoFn(tr("Global variables use %[1]s bytes of dynamic memory.", strconv.Itoa(dataSize))) } } - ctx.ExecutableSectionsSize = []types.ExecutableSectionSize{ + executableSectionsSize := []builder.ExecutableSectionSize{ { Name: "text", Size: textSize, @@ -150,7 +157,7 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { }, } if maxDataSize > 0 { - ctx.ExecutableSectionsSize = append(ctx.ExecutableSectionsSize, types.ExecutableSectionSize{ + executableSectionsSize = append(executableSectionsSize, builder.ExecutableSectionSize{ Name: "data", Size: dataSize, MaxSize: maxDataSize, @@ -158,38 +165,42 @@ func checkSize(ctx *types.Context, buildProperties *properties.Map) error { } if textSize > maxTextSize { - ctx.Warn(tr("Sketch too big; see %[1]s for tips on reducing it.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) - return errors.New(tr("text section exceeds available space in board")) + printWarnFn(tr("Sketch too big; see %[1]s for tips on reducing it.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) + return executableSectionsSize, errors.New(tr("text section exceeds available space in board")) } if maxDataSize > 0 && dataSize > maxDataSize { - ctx.Warn(tr("Not enough memory; see %[1]s for tips on reducing your footprint.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) - return errors.New(tr("data section exceeds available space in board")) + printWarnFn(tr("Not enough memory; see %[1]s for tips on reducing your footprint.", "https://support.arduino.cc/hc/en-us/articles/360013825179")) + return executableSectionsSize, errors.New(tr("data section exceeds available space in board")) } if w := properties.Get("build.warn_data_percentage"); w != "" { warnDataPercentage, err := strconv.Atoi(w) if err != nil { - return err + return executableSectionsSize, err } if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 { - ctx.Warn(tr("Low memory available, stability problems may occur.")) + printWarnFn(tr("Low memory available, stability problems may occur.")) } } - return nil + return executableSectionsSize, nil } -func execSizeRecipe(ctx *types.Context, properties *properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { +func execSizeRecipe(properties *properties.Map, + verbose bool, + stdoutWriter, stderrWriter io.Writer, + printInfoFn func(msg string), +) (textSize int, dataSize int, eepromSize int, resErr error) { command, err := utils.PrepareCommandForRecipe(properties, "recipe.size.pattern", false) if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) return } - verboseInfo, out, _, err := utils.ExecCommand(ctx.Verbose, ctx.Stdout, ctx.Stderr, command, utils.Capture /* stdout */, utils.Show /* stderr */) - if ctx.Verbose { - ctx.Info(string(verboseInfo)) + verboseInfo, out, _, err := utils.ExecCommand(verbose, stdoutWriter, stderrWriter, command, utils.Capture /* stdout */, utils.Show /* stderr */) + if verbose { + printInfoFn(string(verboseInfo)) } if err != nil { resErr = fmt.Errorf(tr("Error while determining sketch size: %s"), err) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 14616a15fb8..0c9b1a18458 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -93,7 +93,7 @@ type Context struct { stdLock sync.Mutex // Sizer results - ExecutableSectionsSize ExecutablesFileSections + ExecutableSectionsSize builder.ExecutablesFileSections // Compilation Database to build/update CompilationDatabase *builder.CompilationDatabase @@ -106,29 +106,6 @@ type Context struct { SourceOverride map[string]string } -// ExecutableSectionSize represents a section of the executable output file -type ExecutableSectionSize struct { - Name string `json:"name"` - Size int `json:"size"` - MaxSize int `json:"max_size"` -} - -// ExecutablesFileSections is an array of ExecutablesFileSection -type ExecutablesFileSections []ExecutableSectionSize - -// ToRPCExecutableSectionSizeArray transforms this array into a []*rpc.ExecutableSectionSize -func (s ExecutablesFileSections) ToRPCExecutableSectionSizeArray() []*rpc.ExecutableSectionSize { - res := []*rpc.ExecutableSectionSize{} - for _, section := range s { - res = append(res, &rpc.ExecutableSectionSize{ - Name: section.Name, - Size: int64(section.Size), - MaxSize: int64(section.MaxSize), - }) - } - return res -} - func (ctx *Context) ExtractBuildOptions() *properties.Map { opts := properties.NewMap() opts.Set("hardwareFolders", strings.Join(ctx.HardwareDirs.AsStrings(), ",")) From d4454f5093f834f1d2563d8ba74d1346e74fd47e Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 7 Sep 2023 11:26:50 +0200 Subject: [PATCH 19/20] remove empty file --- legacy/builder/ctags_target_file_saver.go | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 legacy/builder/ctags_target_file_saver.go diff --git a/legacy/builder/ctags_target_file_saver.go b/legacy/builder/ctags_target_file_saver.go deleted file mode 100644 index 261bc7c759a..00000000000 --- a/legacy/builder/ctags_target_file_saver.go +++ /dev/null @@ -1,16 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder From f57fbeb53aeb6f0f36b48280f3dccedfb2fc5e24 Mon Sep 17 00:00:00 2001 From: Alessio Perugini Date: Thu, 7 Sep 2023 14:38:46 +0200 Subject: [PATCH 20/20] remove unused struct FailIfBuildPathEqualsSketchPath --- .../fail_if_buildpath_equals_sketchpath.go | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 legacy/builder/fail_if_buildpath_equals_sketchpath.go diff --git a/legacy/builder/fail_if_buildpath_equals_sketchpath.go b/legacy/builder/fail_if_buildpath_equals_sketchpath.go deleted file mode 100644 index 2ec5c0da662..00000000000 --- a/legacy/builder/fail_if_buildpath_equals_sketchpath.go +++ /dev/null @@ -1,33 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type FailIfBuildPathEqualsSketchPath struct{} - -func (s *FailIfBuildPathEqualsSketchPath) Run(ctx *types.Context) error { - buildPath := ctx.BuildPath.Canonical() - sketchPath := ctx.Sketch.FullPath.Canonical() - if buildPath.EqualsTo(sketchPath) { - return errors.New(tr("Sketch cannot be located in build path. Please specify a different build path")) - } - - return nil -}