From 1ca6d372f11365d4d207bc1801d42ec0b089059d Mon Sep 17 00:00:00 2001 From: PaulStoffregen Date: Wed, 18 Mar 2015 20:03:55 -0700 Subject: [PATCH 1/3] Add library dependency detection --- .../src/processing/app/debug/Compiler.java | 170 ++++++++++++++++-- 1 file changed, 160 insertions(+), 10 deletions(-) diff --git a/arduino-core/src/processing/app/debug/Compiler.java b/arduino-core/src/processing/app/debug/Compiler.java index dd5d7128f65..74c70e224d5 100644 --- a/arduino-core/src/processing/app/debug/Compiler.java +++ b/arduino-core/src/processing/app/debug/Compiler.java @@ -30,6 +30,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; +import java.io.InputStreamReader; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; @@ -358,16 +359,38 @@ public boolean compile(boolean _verbose) throws RunnerException, PreferencesMapE // 0. include paths for core + all libraries progressListener.progress(20); List includeFolders = new ArrayList(); - includeFolders.add(prefs.getFile("build.core.path")); - if (prefs.getFile("build.variant.path") != null) - includeFolders.add(prefs.getFile("build.variant.path")); - for (Library lib : importedLibraries) { - if (verbose) - System.out.println(I18n - .format(_("Using library {0} in folder: {1} {2}"), lib.getName(), - lib.getFolder(), lib.isLegacy() ? "(legacy)" : "")); - includeFolders.add(lib.getSrcFolder()); - } + boolean addRequiredLibrary; + do { + includeFolders.clear(); + includeFolders.add(prefs.getFile("build.core.path")); + if (prefs.getFile("build.variant.path") != null) + includeFolders.add(prefs.getFile("build.variant.path")); + for (Library lib : importedLibraries) { + if (verbose) + System.out.println(I18n + .format(_("Using library {0} in folder: {1} {2}"), lib.getName(), + lib.getFolder(), lib.isLegacy() ? "(legacy)" : "")); + includeFolders.add(lib.getSrcFolder()); + } + Library required = null; + addRequiredLibrary = false; + for (Library lib : importedLibraries) { + String header = dependencyCheckLibrary(lib, includeFolders); + if (header != null) { + required = BaseNoGui.importToLibraryTable.get(header); + if (required != null && !importedLibraries.contains(required)) { + if (verbose) { + System.out.println("Adding library " + required.getName() + + " required by " + lib.getName()); + } + addRequiredLibrary = true; + break; + } + } + } + if (addRequiredLibrary) importedLibraries.add(required); + } while (addRequiredLibrary); + if (verbose) System.out.println(); @@ -881,6 +904,133 @@ static public List findFilesInFolder(File folder, String extension, return files; } + + // 0. dependency check library + // returns the name of any header file not found (on all includeFolders) + // or null if every file the library includes is found + private String dependencyCheckLibrary(Library lib, List includeFolders) + throws RunnerException, PreferencesMapException { + File libFolder = lib.getSrcFolder(); + String header = null; + if (lib.useRecursion()) { + header = recursiveDependencyCheckFolder(libFolder, includeFolders); + } else { + File utilityFolder = new File(libFolder, "utility"); + includeFolders.add(utilityFolder); + header = dependencyCheckFolder(libFolder, includeFolders); + if (header == null) { + header = dependencyCheckFolder(utilityFolder, includeFolders); + } + includeFolders.remove(utilityFolder); + } + return header; + } + + private String recursiveDependencyCheckFolder(File srcFolder, List includeFolders) + throws RunnerException, PreferencesMapException { + String header = dependencyCheckFolder(srcFolder, includeFolders); + if (header != null) return header; + for (File subFolder : srcFolder.listFiles(new OnlyDirs())) { + header = recursiveDependencyCheckFolder(subFolder, includeFolders); + if (header != null) return header; + } + return null; + } + + private String dependencyCheckFolder(File sourcePath, List includeFolders) + throws RunnerException, PreferencesMapException { + for (File file : findFilesInFolder(sourcePath, "S", false)) { + String header = execDependencyCommand(includeFolders, file, "recipe.S.o.pattern"); + if (header != null) return header; + } + for (File file : findFilesInFolder(sourcePath, "c", false)) { + String header = execDependencyCommand(includeFolders, file, "recipe.c.o.pattern"); + if (header != null) return header; + } + for (File file : findFilesInFolder(sourcePath, "cpp", false)) { + String header = execDependencyCommand(includeFolders, file, "recipe.cpp.o.pattern"); + if (header != null) return header; + } + return null; + } + + private String execDependencyCommand(List includeFolders, File sourceFile, String recipe) + throws PreferencesMapException, RunnerException { + // Generate the normal compiler command from its recipe + String[] cmd = getCommandCompilerByRecipe(includeFolders, + sourceFile, new File("dummy.o"), recipe); + // Remove options that interfere with preprocessor-only dependency analysis + int count = 0; + for (int i=1; i < cmd.length; i++) { + if (cmd[i].equals("-c") || cmd[i].equals("-S") || cmd[i].startsWith("-M")) { + cmd[i] = null; + } else if (cmd[i].equals("-o")) { + cmd[i++] = null; + if (i < cmd.length) cmd[i] = null; + } else { + count++; + } + } + // Create a modified compiler command to output the dependency list + String[] dep = new String[count + 4]; + dep[0] = cmd[0]; + dep[1] = "-E"; + dep[2] = "-M"; + dep[3] = "-MG"; + int j = 1; + for (int i=0; i < count; i++) { + while (cmd[j] == null) j++; + dep[i + 4] = cmd[j++]; + } + // Run the compiler to analyze dependencies + // Only the (hopefully fast) preprocessor is run. The slow + // compile and code generation steps are skipped. + Process process; + try { + process = ProcessUtils.exec(dep); + } catch (IOException e) { + RunnerException re = new RunnerException(e.getMessage()); + re.hideStackTrace(); + throw re; + } + //uncomment this to see the actual compiler commands + //for (String c : dep) System.out.print(c + " "); + //System.out.println(); + BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream())); + try { + process.waitFor(); + } catch (InterruptedException e) { + try { + in.close(); + } catch (IOException io) { } + return null; + } + // Read the dependency info and check if the files + try { + while (true) { + String line = in.readLine(); + if (line == null) break; + if (line.endsWith("\\")) line = line.substring(0, line.length() - 1); + line = line.trim(); + String[] filenames = line.split("(? includeFolders) throws RunnerException, PreferencesMapException { File buildPath = prefs.getFile("build.path"); From c1bf485205f8531f60682eb7203ec2a49f30e028 Mon Sep 17 00:00:00 2001 From: PaulStoffregen Date: Thu, 19 Mar 2015 16:49:43 -0700 Subject: [PATCH 2/3] Add comments & fix white space --- .../src/processing/app/debug/Compiler.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/arduino-core/src/processing/app/debug/Compiler.java b/arduino-core/src/processing/app/debug/Compiler.java index 74c70e224d5..545660321f6 100644 --- a/arduino-core/src/processing/app/debug/Compiler.java +++ b/arduino-core/src/processing/app/debug/Compiler.java @@ -384,6 +384,11 @@ public boolean compile(boolean _verbose) throws RunnerException, PreferencesMapE " required by " + lib.getName()); } addRequiredLibrary = true; + // When any library must be added to the importedLibraries list + // immediately drop out of this loop, so only a single library + // is added. Each new library can cause different #define + // behavior in other code, so building the includeFolders list + // must be restarted from scratch with each new library. break; } } @@ -1009,13 +1014,25 @@ private String execDependencyCommand(List includeFolders, File sourceFile, try { while (true) { String line = in.readLine(); + // gcc provides dependency analysis in Makefile rule format. Files + // that exist have full pathnames. Those which don't exist are given + // as the appeared in the #include. + // For example: + // Sd2Card.o: \ + // /home/paul/Arduino/libraries/SD/src/utility/Sd2Card.cpp \ + // /home/paul/Arduino/hardware/arduino/avr/cores/arduino/Arduino.h \ + // /home/paul/Arduino/hardware/tools/avr/avr/include/stdlib.h \ + // SPI.h if (line == null) break; if (line.endsWith("\\")) line = line.substring(0, line.length() - 1); line = line.trim(); - String[] filenames = line.split("(? Date: Thu, 19 Mar 2015 16:57:13 -0700 Subject: [PATCH 3/3] Avoid printing duplicate "Using library" lines --- .../src/processing/app/debug/Compiler.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arduino-core/src/processing/app/debug/Compiler.java b/arduino-core/src/processing/app/debug/Compiler.java index 545660321f6..ad07d61deae 100644 --- a/arduino-core/src/processing/app/debug/Compiler.java +++ b/arduino-core/src/processing/app/debug/Compiler.java @@ -366,10 +366,6 @@ public boolean compile(boolean _verbose) throws RunnerException, PreferencesMapE if (prefs.getFile("build.variant.path") != null) includeFolders.add(prefs.getFile("build.variant.path")); for (Library lib : importedLibraries) { - if (verbose) - System.out.println(I18n - .format(_("Using library {0} in folder: {1} {2}"), lib.getName(), - lib.getFolder(), lib.isLegacy() ? "(legacy)" : "")); includeFolders.add(lib.getSrcFolder()); } Library required = null; @@ -380,8 +376,8 @@ public boolean compile(boolean _verbose) throws RunnerException, PreferencesMapE required = BaseNoGui.importToLibraryTable.get(header); if (required != null && !importedLibraries.contains(required)) { if (verbose) { - System.out.println("Adding library " + required.getName() + - " required by " + lib.getName()); + System.out.println(I18n.format(_("Adding library {0} required by {1}"), + required.getName(), lib.getName())); } addRequiredLibrary = true; // When any library must be added to the importedLibraries list @@ -396,8 +392,13 @@ public boolean compile(boolean _verbose) throws RunnerException, PreferencesMapE if (addRequiredLibrary) importedLibraries.add(required); } while (addRequiredLibrary); - if (verbose) + if (verbose) { + for (Library lib : importedLibraries) { + System.out.println(I18n.format(_("Using library {0} in folder: {1} {2}"), + lib.getName(), lib.getFolder(), lib.isLegacy() ? "(legacy)" : "")); + } System.out.println(); + } List archs = new ArrayList(); archs.add(BaseNoGui.getTargetPlatform().getId());