Merge pull request diffblue#291 from diffblue/feature/include_libs

SEC-16: Preparation steps for load analysed web apps with libraries included.

Author: marek-trtik

benchmarks/LIBRARIES/models/model/mkrtm.py

@@ -0,0 +1,66 @@
+import os
+import shutil
+import argparse
+import tempfile
+import contextlib
+
+
+@contextlib.contextmanager
+def _working_dir(new_dir):
+    previous_dir = os.getcwd()
+    if not os.path.isdir(new_dir):
+        os.makedirs(new_dir)
+    os.chdir(new_dir)
+    yield
+    os.chdir(previous_dir)
+
+
+def _parse_cmd_line():
+    parser = argparse.ArgumentParser(
+        description="Creates a JAR file consisting of modelled class files and "
+                    "other class files in the passed 'rt.jar'.")
+    parser.add_argument("rt", type=str,
+                        help="A path-name of the 'rt.jar' file.")
+    parser.add_argument("--output-name", type=str, default="rtm.jar",
+                        help="A name of the final JAR file (without path).")
+    cmdline = parser.parse_args()
+    if not os.path.isfile(cmdline.rt):
+        print("ERROR: The passed file '" + cmdline.rt + "' does not exist.")
+        exit(1)
+    if os.path.splitext(cmdline.rt)[1].lower() != ".jar":
+        print("ERROR: The passed file '" + cmdline.rt + "' does not have '.jar' extension.")
+        exit(1)
+    cmdline.rt = os.path.abspath(cmdline.rt)
+    return cmdline
+
+
+def _mkrtm(cmdline):
+    model_classes_root_dir = os.path.abspath(os.path.join("target", "classes"))
+    temp_root_dir = tempfile.mkdtemp()
+    classes_root_dir = os.path.join(temp_root_dir, "classes")
+    rt_unpack_root_dir = os.path.join(temp_root_dir, "RT_UNPACK")
+    dst_jar_pathname = os.path.abspath(os.path.join("target", cmdline.output_name))
+
+    shutil.copytree(model_classes_root_dir, classes_root_dir)
+    with _working_dir(rt_unpack_root_dir):
+        os.system("jar -xvf  '" + cmdline.rt + "'")
+    for root, _, file_names in os.walk(rt_unpack_root_dir):
+        for filename in file_names:
+            if os.path.splitext(filename)[1].lower() == ".class":
+                src_pathname = os.path.abspath(os.path.join(root, filename))
+                rel_pathname = os.path.relpath(src_pathname, rt_unpack_root_dir)
+                dst_pathname = os.path.join(classes_root_dir, rel_pathname)
+                if not os.path.isfile(dst_pathname):
+                    if not os.path.isdir(os.path.dirname(dst_pathname)):
+                        os.makedirs(os.path.dirname(dst_pathname))
+                    shutil.copy(src_pathname, dst_pathname)
+    with _working_dir(classes_root_dir):
+        os.system("jar cvfm " + os.path.basename(dst_jar_pathname) + " " +
+                  os.path.join(rt_unpack_root_dir, "META-INF", "MANIFEST.MF") +
+                  " .")
+    shutil.copy(os.path.join(classes_root_dir, os.path.basename(dst_jar_pathname)),
+                dst_jar_pathname)
+
+
+if __name__ == '__main__':
+    _mkrtm(_parse_cmd_line())

driver/mkbench.py

@@ -70,7 +70,7 @@ def _read_info_of_class_files(class_files, temp_dir, verbosity):
     return classes_info, java_class_info_call_duration
 
 
-def collect_java_binaries(app_binary_dirs, temp_dir, output_json, verbosity):
+def collect_java_binaries(app_binary_dirs, list_of_classpaths, temp_dir, output_json, verbosity):
     prof_start_time = time.time()
     prof = dict()
 
@@ -164,7 +164,10 @@ def collect_java_binaries(app_binary_dirs, temp_dir, output_json, verbosity):
     if not os.path.isdir(os.path.dirname(output_json)):
         os.makedirs(os.path.dirname(output_json))
     with open(output_json, "w") as ofile:
-        ofile.write(json.dumps({"jar": java_binaries.jar_files}, sort_keys=True, indent=4))
+        ofile.write(json.dumps({
+            "jar": sorted(java_binaries.jar_files),
+            "classpath": sorted([p for p in list_of_classpaths if os.path.exists(p)])
+            }, sort_keys=True, indent=4))
 
     # Lastly, we complete and return statistics from this stage.
     prof["num_jars_final"] = len(java_binaries.jar_files)

driver/run.py

@@ -28,6 +28,9 @@ def __parse_cmd_line():
                          help="The install directory of the Java web application to be analysed. Under that "
                               "directory (directly or indirectly) there should be all binaries files "
                               "(WAR/JAR/CLASS files) of that wab application.")
+    parser.add_argument("-L", "--libraries", nargs='+', default=[],
+                        help="A list of disk paths to libraries you want to include into class path. A path "
+                             "can either be a path-name of a JAR file, or a directory.")
     parser.add_argument("-R", "--results-dir", type=str,
                          help="A directory into which all results from the analysis of the given Java web applicaton "
                               "will be written.")
@@ -104,6 +107,7 @@ def evaluate(cmdline):
     classes_jar_pathname = os.path.abspath(os.path.join(cmdline.results_dir, "program.json"))
     prof["collect_java_binaries"] = mkbench.collect_java_binaries(
         input_search_dirs,
+        cmdline.libraries,
         cmdline.temp_dir,
         classes_jar_pathname,
         cmdline.verbosity

src/taint-analysis/taint_config.cpp

@@ -191,6 +191,39 @@ bool taint_configt::load(message_handlert &handler)
   return false;
 }
 
+struct cmdline_updatert : public cmdlinet
+{
+  void add_to_classpaths(const std::vector<std::string> &classpaths)
+  {
+    PRECONDITION(!classpaths.empty());
+    int idx=getoptnr("classpath");
+    if(idx==-1)
+    {
+      options.push_back({});
+      idx=int(options.size()-1UL);
+    }
+    if(options.at(idx).isset==false)
+    {
+      options.at(idx).isset=true;
+      if(options.at(idx).values.empty())
+        options.at(idx).values.push_back("");
+    }
+    std::string &dst_classpaths = options.at(idx).values.front();
+    for(const auto &classpath : classpaths)
+    {
+      if(!dst_classpaths.empty())
+        dst_classpaths+=
+#ifdef _WIN32
+          ';'
+#else
+          ':'
+#endif
+          ;
+      dst_classpaths+=classpath;
+    }
+  }
+};
+
 void taint_build_cmdline_from_config(
   const std::string &cfg_file_path,
   messaget * const logger,
@@ -213,5 +246,12 @@ void taint_build_cmdline_from_config(
     return;
   for(auto const &jar : cfg["jar"].array)
     cmdline.args.push_back(jar.value);
+  if(!cfg["classpath"].array.empty())
+  {
+    std::vector<std::string> classpaths;
+    for(auto const &classpath : cfg["classpath"].array)
+      classpaths.push_back(classpath.value);
+    static_cast<cmdline_updatert*>(&cmdline)->add_to_classpaths(classpaths);
+  }
   cmdline.set("lazy-methods");
 }