builder: use ar-chives for linking big sketches (was: use the @ syntax to reduce command line length if needed) (#961)

* Small cosmetic changes

* Create an archive file if the number of object files is too big

This should fix the command line too big issue on Windows.

* fixed comments

* When exploiting ar-chives make a .a file for each soruce subfolder

This is required because gcc-ar checks if an object file is already
in the archive by looking ONLY at the filename WITHOUT the path, so
it may happens that, for example, an object file named "subdir/spi.o",
already inside the archive, may be overwritten by an object file in
"anotherdir/spi.o" only because they are both named spi.o and even if
they are compiled on different directories.

* using paths.PathList to keep objectFileList

Author: cmaglie

Diff for: legacy/builder/constants/constants.go

@@ -31,7 +31,6 @@ const BUILD_PROPERTIES_COMPILER_WARNING_FLAGS = "compiler.warning_flags"
 const BUILD_PROPERTIES_FQBN = "build.fqbn"
 const BUILD_PROPERTIES_INCLUDES = "includes"
 const BUILD_PROPERTIES_OBJECT_FILE = "object_file"
-const BUILD_PROPERTIES_OBJECT_FILES = "object_files"
 const BUILD_PROPERTIES_PATTERN = "pattern"
 const BUILD_PROPERTIES_PID = "pid"
 const BUILD_PROPERTIES_PREPROCESSED_FILE_PATH = "preprocessed_file_path"

Diff for: legacy/builder/phases/libraries_builder.go

@@ -112,7 +112,7 @@ func compileLibraries(ctx *types.Context, libraries libraries.List, buildPath *p
 		if err != nil {
 			return nil, errors.WithStack(err)
 		}
-		objectFiles = append(objectFiles, libraryObjectFiles...)
+		objectFiles.AddAll(libraryObjectFiles)
 
 		ctx.Progress.CompleteStep()
 		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx)

Diff for: legacy/builder/phases/linker.go

@@ -57,17 +57,47 @@ func (s *Linker) Run(ctx *types.Context) error {
 }
 
 func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map) error {
-	quotedObjectFiles := utils.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes)
-	objectFileList := strings.Join(quotedObjectFiles, constants.SPACE)
+	objectFileList := strings.Join(utils.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ")
+
+	// If command line length is too big (> 30000 chars), try to collect the object files into archives
+	// and use that archives to complete the build.
+	if len(objectFileList) > 30000 {
+
+		// We must create an object file for each visited directory: this is required becuase gcc-ar checks
+		// if an object file is already in the archive by looking ONLY at the filename WITHOUT the path, so
+		// it may happen that a subdir/spi.o inside the archive may be overwritten by a anotherdir/spi.o
+		// because thery are both named spi.o.
+
+		properties := buildProperties.Clone()
+		archives := paths.NewPathList()
+		for _, object := range objectFiles {
+			archive := object.Parent().Join("objs.a")
+			if !archives.Contains(archive) {
+				archives.Add(archive)
+				// Cleanup old archives
+				_ = archive.Remove()
+			}
+			properties.Set("archive_file", archive.Base())
+			properties.SetPath("archive_file_path", archive)
+			properties.SetPath("object_file", object)
+			_, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_AR_PATTERN, false, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
+			if err != nil {
+				return err
+			}
+		}
+
+		objectFileList = strings.Join(utils.Map(archives.AsStrings(), wrapWithDoubleQuotes), " ")
+		objectFileList = "-Wl,--whole-archive " + objectFileList + " -Wl,--no-whole-archive"
+	}
 
 	properties := buildProperties.Clone()
 	properties.Set(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS))
 	properties.Set(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS, properties.Get(constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel))
 	properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE, coreDotARelPath.String())
 	properties.Set(constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH, coreArchiveFilePath.String())
-	properties.Set(constants.BUILD_PROPERTIES_OBJECT_FILES, objectFileList)
+	properties.Set("object_files", objectFileList)
 
-	_, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_C_COMBINE_PATTERN, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show)
+	_, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_C_COMBINE_PATTERN, false, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
 	return err
 }