Move include detection from PdeProcessor/Sketch to the new HeuristicResolver class

matthijskooijman · matthijskooijman · commit aa08d1393440 · 2014-07-10T09:54:25.000+02:00
This offers a more consistent interface to the library autodetection that can next be expanded to also detect inter-library dependencies. As a side effect, the resolver now looks through all sketch code, including any .c, .cpp and .h files. Previously, only the .ino and .pde files were inspected during preprocessing. This fixes arduino#636. This commit is based on code by Christian Maglie.
diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java
@@ -44,10 +44,11 @@
 import processing.app.helpers.FileUtils;
 import processing.app.helpers.PreferencesMap;
 import processing.app.helpers.filefilters.OnlyDirs;
-import processing.app.helpers.filefilters.OnlyFilesWithExtension;
 import processing.app.javax.swing.filechooser.FileNameExtensionFilter;
+import processing.app.packages.HeuristicResolver;
 import processing.app.packages.Library;
 import processing.app.packages.LibraryList;
+import processing.app.packages.LibraryResolver;
 import processing.app.tools.MenuScroller;
 import processing.app.tools.ZipDeflater;
 import processing.core.*;
@@ -110,7 +111,7 @@ public class Base {
   static private LibraryList libraries;
 
   // maps #included files to their library folder
-  static Map<String, Library> importToLibraryTable;
+  static private LibraryResolver libraryResolver;
 
   // classpath for all known libraries for p5
   // (both those in the p5/libs folder and those with lib subfolders
@@ -1346,22 +1347,8 @@ public void onBoardOrPortChange() {
       showWarning(_("Error"), _("Error loading libraries"), e);
     }
 
-    // Populate importToLibraryTable
-    importToLibraryTable = new HashMap<String, Library>();
-    for (Library lib : libraries) {
-      List<String> headers = lib.getPublicHeaders();
-      for (String header : headers) {
-        Library old = importToLibraryTable.get(header);
-        if (old != null) {
-          // If a library was already found with this header, keep
-          // it if the library's name matches the header name.
-          String name = header.substring(0, header.length() - 2);
-          if (old.getFolder().getPath().endsWith(name))
-            continue;
-        }
-        importToLibraryTable.put(header, lib);
-      }
-    }
+    // Create library resolver
+    libraryResolver = new HeuristicResolver(libraries);
 
     // Update editors status bar
     for (Editor editor : editors)
@@ -2996,4 +2983,8 @@ public void handleAddLibrary() {
   public static DiscoveryManager getDiscoveryManager() {
     return discoveryManager;
   }
+
+  public static LibraryResolver getLibraryResolver() {
+    return libraryResolver;
+  }
 }
diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java
@@ -34,13 +34,13 @@
 import processing.app.helpers.FileUtils;
 import processing.app.packages.Library;
 import processing.app.packages.LibraryList;
+import processing.app.packages.LibraryResolver;
 import processing.app.preproc.*;
 import processing.core.*;
 import static processing.app.I18n._;
 
 import java.io.*;
 import java.util.*;
-import java.util.List;
 
 import javax.swing.*;
 
@@ -91,11 +91,6 @@ public class Sketch {
   /** Class path determined during build. */
   private String classPath;
 
-  /**
-   * List of library folders.
-   */
-  private LibraryList importedLibraries;
-
   /**
    * File inside the build directory that contains the build options
    * used for the last build.
@@ -1348,18 +1343,8 @@ public void preprocess(String buildPath, PdePreprocessor preprocessor) throws Ru
       ex.printStackTrace();
       throw new RunnerException(ex.toString());
     }
-
-    // grab the imports from the code just preproc'd
-
-    importedLibraries = new LibraryList();
-    for (String item : preprocessor.getExtraImports()) {
-      Library lib = Base.importToLibraryTable.get(item);
-      if (lib != null && !importedLibraries.contains(lib)) {
-        importedLibraries.add(lib);
-      }
-    }
-
-    // 3. then loop over the code[] and save each .java file
+    
+    // 2. then loop over the code[] and save each .java file
 
     for (SketchCode sc : code) {
       if (sc.isExtension(Base.SOURCE_EXTENSIONS) || sc.isExtension(Base.HEADER_EXTENSIONS)) {
@@ -1383,12 +1368,6 @@ public void preprocess(String buildPath, PdePreprocessor preprocessor) throws Ru
     }
   }
 
-
-  public LibraryList getImportedLibraries() {
-    return importedLibraries;
-  }
-
-
   /**
    * Map an error from a set of processed .java files back to its location
    * in the actual sketch.
diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java
@@ -45,6 +45,7 @@
 import processing.app.helpers.StringReplacer;
 import processing.app.helpers.filefilters.OnlyDirs;
 import processing.app.packages.Library;
+import processing.app.packages.LibraryList;
 import processing.core.PApplet;
 
 public class Compiler implements MessageConsumer {
@@ -89,7 +90,8 @@ public boolean compile(boolean _verbose) throws RunnerException {
     includeFolders.add(prefs.getFile("build.core.path"));
     if (prefs.getFile("build.variant.path") != null)
       includeFolders.add(prefs.getFile("build.variant.path"));
-    for (Library lib : sketch.getImportedLibraries()) {
+    LibraryList libs = Base.getLibraryResolver().findDirectDependencies(sketch);
+    for (Library lib : libs) {
       if (verbose)
         System.out.println(I18n
             .format(_("Using library {0} in folder: {1} {2}"), lib.getName(),
@@ -105,7 +107,7 @@ public boolean compile(boolean _verbose) throws RunnerException {
       String[] overrides = prefs.get("architecture.override_check").split(",");
       archs.addAll(Arrays.asList(overrides));
     }
-    for (Library lib : sketch.getImportedLibraries()) {
+    for (Library lib : libs) {
       if (!lib.supportsArchitecture(archs)) {
         System.err.println(I18n
             .format(_("WARNING: library {0} claims to run on {1} "
@@ -124,7 +126,8 @@ public boolean compile(boolean _verbose) throws RunnerException {
     // 2. compile the libraries, outputting .o files to: <buildPath>/<library>/
     // Doesn't really use configPreferences
     sketch.setCompilingProgress(40);
-    compileLibraries(includeFolders);
+    for (Library lib : libs)
+      compileLibrary(lib, includeFolders);
 
     // 3. compile the core, outputting .o files to <buildPath> and then
     // collecting them into the core.a library file.
@@ -613,12 +616,6 @@ void compileSketch(List<File> includeFolders) throws RunnerException {
 
   // 2. compile the libraries, outputting .o files to:
   // <buildPath>/<library>/
-  void compileLibraries(List<File> includeFolders) throws RunnerException {
-    for (Library lib : sketch.getImportedLibraries()) {
-      compileLibrary(lib, includeFolders);
-    }
-  }
-
   private void compileLibrary(Library lib, List<File> includeFolders)
       throws RunnerException {
     File libFolder = lib.getSrcFolder();
diff --git a/app/src/processing/app/packages/HeuristicResolver.java b/app/src/processing/app/packages/HeuristicResolver.java
@@ -0,0 +1,142 @@
+/*
+ * This file is part of Arduino.
+ *
+ * Arduino is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As a special exception, you may use this file as part of a free software
+ * library without restriction.  Specifically, if other files instantiate
+ * templates or use macros or inline functions from this file, or you compile
+ * this file and link it with other files to produce an executable, this
+ * file does not by itself cause the resulting executable to be covered by
+ * the GNU General Public License.  This exception does not however
+ * invalidate any other reasons why the executable file might be covered by
+ * the GNU General Public License.
+ *
+ * Copyright 2013 Arduino LLC (http://www.arduino.cc/)
+ */
+
+package processing.app.packages;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Map;
+
+import processing.app.Sketch;
+import processing.app.SketchCode;
+import processing.app.helpers.FileUtils;
+import processing.app.helpers.filefilters.OnlyFilesWithExtension;
+import processing.app.preproc.PdePreprocessor;
+import processing.core.PApplet;
+
+/**
+ * This resolver uses an heuristic approach to resolve dependencies
+ * without looking into metadata.
+ */
+public class HeuristicResolver implements LibraryResolver {
+
+  private LibraryList libraries;
+  private Map<String, Library> importToLibrary;
+
+  public HeuristicResolver(LibraryList _libraries) {
+    libraries = _libraries;
+    importToLibrary = new HashMap<String, Library>();
+
+    // Populate importToLibrary table
+    for (Library library : libraries) {
+      for (String header : library.getPublicHeaders()) {
+        Library old = importToLibrary.get(header);
+        if (old != null) {
+          // If a library was already found with this header, keep
+          // it if the library's name matches the header name.
+          String name = header.substring(0, header.length() - 2);
+          if (old.getFolder().getPath().endsWith(name))
+            continue;
+        }
+        importToLibrary.put(header, library);
+      }
+    }
+  }
+
+  @Override
+  public LibraryList findDirectDependencies(Sketch sketch) {
+    SketchCode files[] = sketch.getCode();
+    LibraryList result = new LibraryList();
+    for (SketchCode code : files)
+      result.addOrReplaceAll(findSourceDependencies(code.getFile(), null, null));
+    return result;
+  }
+
+  /**
+   * Inspect headerFile and search for dependencies
+   *
+   * @param headerFile
+   * @param excluded
+   * @param library
+   */
+  private LibraryList findSourceDependencies(File headerFile,
+                                             List<String> excluded,
+                                             Library library) {
+    LibraryList res = new LibraryList();
+
+
+    // Extract #includes from header file
+    List<String> imports;
+    String contents;
+    try {
+      contents = FileUtils.readFileToString(headerFile);
+    } catch (IOException e) {
+      e.printStackTrace();
+      return res;
+    }
+
+    String importRegexp = "^\\s*#include\\s*[<\"](\\S+)[\">]";
+    String[][] pieces = PApplet.matchAll(contents, importRegexp);
+
+    // For every #include found...
+    if (pieces != null) {
+      for (int i = 0; i < pieces.length; i++) {
+        String libImport = pieces[i][1];
+
+        // ...check if the include is not in the exclusion list
+        if (excluded != null && excluded.contains(libImport))
+          continue;
+
+        // ...check if there is a matching library
+        Library depLib = importToLibrary.get(libImport);
+        if (depLib == null || depLib == library)
+          continue;
+
+        // ...that we didn't see before
+        if (res.contains(depLib))
+          continue;
+
+        // add the dependency
+        res.add(depLib);
+      }
+    }
+
+    return res;
+  }
+
+  @Override
+  public Library importToLibrary(String h) {
+    return importToLibrary.get(h);
+  }
+
+}
diff --git a/app/src/processing/app/packages/LibraryResolver.java b/app/src/processing/app/packages/LibraryResolver.java
@@ -0,0 +1,53 @@
+/*
+ * This file is part of Arduino.
+ *
+ * Arduino is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As a special exception, you may use this file as part of a free software
+ * library without restriction.  Specifically, if other files instantiate
+ * templates or use macros or inline functions from this file, or you compile
+ * this file and link it with other files to produce an executable, this
+ * file does not by itself cause the resulting executable to be covered by
+ * the GNU General Public License.  This exception does not however
+ * invalidate any other reasons why the executable file might be covered by
+ * the GNU General Public License.
+ *
+ * Copyright 2013 Arduino LLC (http://www.arduino.cc/)
+ */
+
+package processing.app.packages;
+
+import processing.app.Sketch;
+
+public interface LibraryResolver {
+
+  /**
+   * Resolve direct dependencies for a sketch
+   *
+   * @param sketch
+   * @return A LibraryList containing the dependencies
+   */
+  public abstract LibraryList findDirectDependencies(Sketch sketch);
+
+  /**
+   * Returns the Library referenced by the include file name
+   *
+   * @param header
+   *          The include file name, for example "SPI.h".
+   * @return The referenced library
+   */
+  public abstract Library importToLibrary(String header);
+
+}
diff --git a/app/src/processing/app/preproc/PdePreprocessor.java b/app/src/processing/app/preproc/PdePreprocessor.java
