Limit recursive sketch compilation to the src directory

src/arduino.cc/builder/collect_all_source_files_from_folders_with_sources.go

@@ -30,31 +30,22 @@
 package builder
 
 import (
-	"arduino.cc/builder/gohasissues"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
 )
 
 type CollectAllSourceFilesFromFoldersWithSources struct{}
 
 func (s *CollectAllSourceFilesFromFoldersWithSources) Run(ctx *types.Context) error {
 	foldersWithSources := ctx.FoldersWithSourceFiles
 	sourceFiles := ctx.CollectedSourceFiles
+	extensions := func(ext string) bool { return ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext] }
 
 	filePaths := []string{}
 	for !foldersWithSources.Empty() {
 		sourceFolder := foldersWithSources.Pop().(types.SourceFolder)
-		var err error
-		if sourceFolder.Recurse {
-			err = collectByWalk(&filePaths, sourceFolder.Folder)
-		} else {
-			err = collectByReadDir(&filePaths, sourceFolder.Folder)
-		}
+		err := utils.FindFilesInFolder(&filePaths, sourceFolder.Folder, extensions, sourceFolder.Recurse)
 		if err != nil {
 			return i18n.WrapError(err)
 		}
@@ -66,32 +57,3 @@ func (s *CollectAllSourceFilesFromFoldersWithSources) Run(ctx *types.Context) er
 
 	return nil
 }
-
-func collectByWalk(filePaths *[]string, folder string) error {
-	checkExtensionFunc := func(filePath string) bool {
-		name := filepath.Base(filePath)
-		ext := strings.ToLower(filepath.Ext(filePath))
-		return !strings.HasPrefix(name, ".") && ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext]
-	}
-	walkFunc := utils.CollectAllReadableFiles(filePaths, checkExtensionFunc)
-	err := gohasissues.Walk(folder, walkFunc)
-	return i18n.WrapError(err)
-}
-
-func collectByReadDir(filePaths *[]string, folder string) error {
-	if _, err := os.Stat(folder); err != nil && os.IsNotExist(err) {
-		return nil
-	}
-
-	files, err := ioutil.ReadDir(folder)
-	if err != nil {
-		return i18n.WrapError(err)
-	}
-	for _, file := range files {
-		ext := strings.ToLower(filepath.Ext(file.Name()))
-		if ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext] {
-			*filePaths = append(*filePaths, filepath.Join(folder, file.Name()))
-		}
-	}
-	return nil
-}

src/arduino.cc/builder/constants/constants.go

@@ -204,6 +204,7 @@ const RECIPE_S_PATTERN = "recipe.S.o.pattern"
 const REWRITING_DISABLED = "disabled"
 const REWRITING = "rewriting"
 const SPACE = " "
+const SKETCH_FOLDER_SRC = "src"
 const TOOL_NAME = "name"
 const TOOL_URL = "url"
 const TOOL_VERSION = "version"

src/arduino.cc/builder/phases/sketch_builder.go

@@ -34,6 +34,9 @@ import (
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
+	"arduino.cc/builder/constants"
+	"path/filepath"
+	"os"
 )
 
 type SketchBuilder struct{}
@@ -53,11 +56,20 @@ func (s *SketchBuilder) Run(ctx *types.Context) error {
 	}
 
 	var objectFiles []string
-	objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, true, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, logger)
+	objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, logger)
 	if err != nil {
 		return i18n.WrapError(err)
 	}
 
+	// The "src/" subdirectory of a sketch is compiled recursively
+	sketchSrcPath := filepath.Join(sketchBuildPath, constants.SKETCH_FOLDER_SRC)
+	if info, err := os.Stat(sketchSrcPath); err == nil && info.IsDir() {
+		objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes, verbose, warningsLevel, logger)
+		if err != nil {
+			return i18n.WrapError(err)
+		}
+	}
+
 	ctx.SketchObjectFiles = objectFiles
 
 	return nil

src/arduino.cc/builder/sketch_loader.go

@@ -31,7 +31,6 @@ package builder
 
 import (
 	"arduino.cc/builder/constants"
-	"arduino.cc/builder/gohasissues"
 	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
@@ -89,13 +88,21 @@ func (s *SketchLoader) Run(ctx *types.Context) error {
 
 func collectAllSketchFiles(from string) ([]string, error) {
 	filePaths := []string{}
-	checkExtensionFunc := func(filePath string) bool {
-		name := filepath.Base(filePath)
-		ext := strings.ToLower(filepath.Ext(filePath))
-		return !strings.HasPrefix(name, ".") && MAIN_FILE_VALID_EXTENSIONS[ext] || ADDITIONAL_FILE_VALID_EXTENSIONS[ext]
+	// Source files in the root are compiled, non-recursively. This
+	// is the only place where .ino files can be present.
+	rootExtensions := func(ext string) bool { return MAIN_FILE_VALID_EXTENSIONS[ext] || ADDITIONAL_FILE_VALID_EXTENSIONS[ext] }
+	err := utils.FindFilesInFolder(&filePaths, from, rootExtensions, /* recurse */ false)
+	if err != nil {
+		return nil, i18n.WrapError(err)
+	}
+
+	// The "src/" subdirectory of a sketch is compiled recursively
+	// (but .ino files are *not* compiled)
+	srcPath := filepath.Join(from, constants.SKETCH_FOLDER_SRC)
+	if info, err := os.Stat(srcPath); err == nil && info.IsDir() {
+		srcExtensions := func(ext string) bool { return ADDITIONAL_FILE_VALID_EXTENSIONS[ext] }
+		err = utils.FindFilesInFolder(&filePaths, srcPath, srcExtensions, /* recurse */ true)
 	}
-	walkFunc := utils.CollectAllReadableFiles(&filePaths, checkExtensionFunc)
-	err := gohasissues.Walk(from, walkFunc)
 	return filePaths, i18n.WrapError(err)
 }
 

src/arduino.cc/builder/test/additional_sketch_files_copier_test.go

@@ -83,9 +83,9 @@ func TestCopyOtherFiles(t *testing.T) {
 	sort.Sort(ByFileInfoName(files))
 	require.Equal(t, "header.h", files[0].Name())
 	require.Equal(t, "s_file.S", files[1].Name())
-	require.Equal(t, "subfolder", files[2].Name())
+	require.Equal(t, "src", files[2].Name())
 
-	files, err1 = gohasissues.ReadDir(filepath.Join(buildPath, constants.FOLDER_SKETCH, "subfolder"))
+	files, err1 = gohasissues.ReadDir(filepath.Join(buildPath, constants.FOLDER_SKETCH, "src"))
 	NoError(t, err1)
 	require.Equal(t, 1, len(files))
 	require.Equal(t, "helper.h", files[0].Name())

src/arduino.cc/builder/test/builder_test.go

@@ -373,3 +373,32 @@ func TestBuilderSketchBuildPathContainsUnusedPreviouslyCompiledLibrary(t *testin
 	_, err = os.Stat(filepath.Join(buildPath, constants.FOLDER_LIBRARIES, "Bridge"))
 	NoError(t, err)
 }
+
+func TestBuilderWithBuildPathInSketchDir(t *testing.T) {
+	DownloadCoresAndToolsAndLibraries(t)
+
+	ctx := &types.Context{
+		HardwareFolders:         []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"},
+		ToolsFolders:            []string{"downloaded_tools"},
+		BuiltInLibrariesFolders: []string{"downloaded_libraries"},
+		OtherLibrariesFolders:   []string{"libraries"},
+		SketchLocation:          filepath.Join("sketch1", "sketch.ino"),
+		FQBN:                    "arduino:avr:uno",
+		ArduinoAPIVersion:       "10600",
+		Verbose:                 true,
+	}
+
+	var err error
+	ctx.BuildPath, err = filepath.Abs(filepath.Join("sketch1", "build"))
+	NoError(t, err)
+	defer os.RemoveAll(ctx.BuildPath)
+
+	command := builder.Builder{}
+	err = command.Run(ctx)
+	NoError(t, err)
+
+	// Run build twice, to verify the build still works when the
+	// build directory is present at the start
+	err = command.Run(ctx)
+	NoError(t, err)
+}

src/arduino.cc/builder/test/collect_all_source_files_from_folders_with_sources_test.go

@@ -60,7 +60,7 @@ func TestCollectAllSourceFilesFromFoldersWithSources(t *testing.T) {
 	require.Equal(t, 0, len(*foldersWithSources))
 	sort.Strings(*sourceFiles)
 
-	require.Equal(t, Abs(t, filepath.Join("sketch_with_config", "includes", "de bug.cpp")), sourceFiles.Pop())
+	require.Equal(t, Abs(t, filepath.Join("sketch_with_config", "src", "includes", "de bug.cpp")), sourceFiles.Pop())
 	require.Equal(t, 0, len(*sourceFiles))
 }
 
@@ -106,7 +106,6 @@ func TestCollectAllSourceFilesFromFoldersWithSourcesOfOldLibrary(t *testing.T) {
 	foldersWithSources := &types.UniqueSourceFolderQueue{}
 	foldersWithSources.Push(types.SourceFolder{Folder: Abs(t, filepath.Join("libraries", "ShouldNotRecurseWithOldLibs")), Recurse: false})
 	foldersWithSources.Push(types.SourceFolder{Folder: Abs(t, filepath.Join("libraries", "ShouldNotRecurseWithOldLibs", "utility")), Recurse: false})
-	foldersWithSources.Push(types.SourceFolder{Folder: Abs(t, "non existent folder"), Recurse: false})
 	ctx.FoldersWithSourceFiles = foldersWithSources
 
 	commands := []types.Command{

src/arduino.cc/builder/test/sketch_with_config/sketch_with_config.ino

@@ -1,7 +1,7 @@
 #include "config.h"
 
 #ifdef DEBUG
-#include "includes/de bug.h"
+#include "src/includes/de bug.h"
 #endif
 
 #ifdef UNDEF

src/arduino.cc/builder/test/sketch_with_config/sketch_with_config.preprocessed.txt

@@ -4,7 +4,7 @@
 #include "config.h"
 
 #ifdef DEBUG
-#include "includes/de bug.h"
+#include "src/includes/de bug.h"
 #endif
 
 #ifdef UNDEF

src/arduino.cc/builder/test/sketch_with_subfolders/sketch_with_subfolders.ino

@@ -1,4 +1,4 @@
-#include "subfolder/other.h"
+#include "src/subfolder/other.h"
 
 MyClass myClass;
 
@@ -7,4 +7,4 @@ void setup() {
 }
 
 void loop() {
-}
+}

src/arduino.cc/builder/test/sketch_with_subfolders/src/subfolder/dont_load_me.ino

@@ -0,0 +1 @@
+#error "Whattya looking at?"

src/arduino.cc/builder/test/sketch_with_subfolders/subfolder/dont_load_me.cpp

@@ -0,0 +1 @@
+#error "Whattya looking at?"

src/arduino.cc/builder/types/types.go

@@ -34,6 +34,7 @@ import (
 	"arduino.cc/builder/props"
 	"path/filepath"
 	"strconv"
+	"os"
 )
 
 type SketchFile struct {
@@ -201,7 +202,9 @@ func LibraryToSourceFolder(library *Library) []SourceFolder {
 	sourceFolders = append(sourceFolders, SourceFolder{Folder: library.SrcFolder, Recurse: recurse})
 	if library.Layout == LIBRARY_FLAT {
 		utility := filepath.Join(library.SrcFolder, constants.LIBRARY_FOLDER_UTILITY)
-		sourceFolders = append(sourceFolders, SourceFolder{Folder: utility, Recurse: false})
+		if info, err := os.Stat(utility); err == nil && info.IsDir() {
+			sourceFolders = append(sourceFolders, SourceFolder{Folder: utility, Recurse: false})
+		}
 	}
 	return sourceFolders
 }

src/arduino.cc/builder/utils/utils.go

@@ -304,30 +304,48 @@ func FilterOutFoldersByNames(folders []os.FileInfo, names ...string) []os.FileIn
 	return filtered
 }
 
-type CheckFilePathFunc func(filePath string) bool
+type CheckExtensionFunc func(ext string) bool
 
-func CollectAllReadableFiles(collector *[]string, test CheckFilePathFunc) filepath.WalkFunc {
-	walkFunc := func(currentPath string, info os.FileInfo, err error) error {
+func FindFilesInFolder(files *[]string, folder string, extensions CheckExtensionFunc, recurse bool) error {
+	walkFunc := func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		if info.IsDir() {
+		// Skip source control and hidden files and directories
+		if IsSCCSOrHiddenFile(info) {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
 			return nil
 		}
-		if !test(currentPath) {
+
+		// Skip directories unless recurse is on, or this is the
+		// root directory
+		if info.IsDir() {
+			if recurse || path == folder {
+				return nil
+			} else {
+				return filepath.SkipDir
+			}
+		}
+
+		// Check (lowercased) extension against list of extensions
+		if extensions != nil && !extensions(strings.ToLower(filepath.Ext(path))) {
 			return nil
 		}
-		currentFile, err := os.Open(currentPath)
+
+		// See if the file is readable by opening it
+		currentFile, err := os.Open(path)
 		if err != nil {
 			return nil
 		}
 		currentFile.Close()
 
-		*collector = append(*collector, currentPath)
+		*files = append(*files, path)
 		return nil
 	}
-	return walkFunc
+	return gohasissues.Walk(folder, walkFunc)
 }
 
 func AppendIfNotPresent(target []string, elements ...string) []string {