Merge pull request diffblue#460 from diffblue/enhancement/configuration_generation

[SEC-424] Automatic Entry-point Detection

Author: JohnDumbell

driver/analyser.py

@@ -43,86 +43,45 @@ def get_symex_pathname():
     return os.path.abspath(os.path.join(_tools_binary_dir, "symex"))
 
 
-def run_security_analyser(
-        program_json_file,
-        transition_rules_file_pathname,
-        timeout,
-        dump_html_summaries,
-        dump_html_statistics,
-        dump_html_slice,
-        dump_html_program,
-        verbosity,
-        do_not_use_precise_access_paths,
-        data_flow_insensitive_instrumentation,
-        results_dir,
-        temp_root_dir,
-        lazy_methods_context_sensitive,
-        verify_csvsa_sparse_domains,
-        use_goto_binary
-        ):
+def run_security_analyser(config_json_file, cmdline):
     prof = {}
     prof_start_time = time.time()
 
-    if not os.path.exists(results_dir):
-        os.makedirs(results_dir)
-
-    # Building the root config file for the security analyser
-    root_config_json_fname = os.path.abspath(os.path.join(results_dir, "config.json"))
-    print("Building the root config JSON file for 'goto-analyser': " + root_config_json_fname)
-    root_config_json = {
-        "program": program_json_file,
-        "rules": transition_rules_file_pathname,
-        "timeout": timeout,
-        "verbosity": verbosity,
-        "dump_html_summaries": dump_html_summaries,
-        "dump_html_statistics": dump_html_statistics,
-        "dump_html_slice": dump_html_slice,
-        "dump_html_program": dump_html_program,
-        "use_goto_binary": use_goto_binary,
-        "do_not_use_precise_access_paths": do_not_use_precise_access_paths,
-        "output-dir": "./",
-        "temp-dir": temp_root_dir,
-        "data-flow-insensitive-instrumentation": data_flow_insensitive_instrumentation,
-        "lazy-methods-context-sensitive": lazy_methods_context_sensitive,
-        "verify-csvsa-sparse-domains": verify_csvsa_sparse_domains
-    }
-    with open(root_config_json_fname, "w") as root_config_json_file:
-        root_config_json_file.write(json.dumps(root_config_json, sort_keys=True, indent=4))
-
-    if not os.path.exists(results_dir):
-        os.makedirs(results_dir)
+    if not os.path.exists(cmdline.results_dir):
+        os.makedirs(cmdline.results_dir)
 
     old_cwd = os.getcwd()
-    os.chdir(results_dir)
+    os.chdir(cmdline.results_dir)
 
-    if use_goto_binary:
+    if cmdline.use_goto_binary:
+        program_json_file = os.path.abspath(os.path.join(cmdline.results_dir, "program.json"))
         with open(program_json_file, "r") as ifile:
             program_info = json.load(ifile)
         assert "gbf" in program_info
         if not os.path.isfile(program_info["gbf"]):
             command = (
                 get_security_analyser_pathname() + " --security-scanner '" +
-                root_config_json_fname + "' --output-goto-binary '" +
+                config_json_file + "' --output-goto-binary '" +
                 program_info["gbf"] + "' "
                 )
             prof["calling_security_analyser_for_goto_program_generation"] = {}
             prof_calling_security_analyser_start_time = time.time()
             print("Invoking 'security-analyser' to translate Java to GOTO program.")
-            if verbosity >= 9:
-                print("CWD: " + results_dir)
+            if cmdline.verbosity >= 9:
+                print("CWD: " + cmdline.results_dir)
                 print("CMD: " + command)
             os.system(command)
             prof["calling_security_analyser_for_goto_program_generation"]["duration"] = time.time() - prof_calling_security_analyser_start_time
 
     command = (
         get_security_analyser_pathname() + " "
-        "--security-scanner '" + root_config_json_fname + "' "
+        "--security-scanner '" + config_json_file + "' "
         )
     prof["calling_security_analyser"] = {}
     prof_calling_security_analyser_start_time = time.time()
     print("Invoking 'security-analyser' ...")
-    if verbosity >= 9:
-        print("CWD: " + results_dir)
+    if cmdline.verbosity >= 9:
+        print("CWD: " + cmdline.results_dir)
         print("CMD: " + command)
     os.system(command)
     prof["calling_security_analyser"]["duration"] = time.time() - prof_calling_security_analyser_start_time

driver/mkbench.py

@@ -35,20 +35,21 @@ def _find_java_binaries(start_path, java_binaries):
                 _add_java_binary(full_pathname, java_binaries)
 
 
-def _read_info_of_class_files(class_files, temp_dir, verbosity):
+def _read_info_of_class_files(class_files, configuration, cmdline):
+
     assert isinstance(class_files, list) and all(isinstance(s, str) for s in class_files)
-    class_list_json_file = os.path.join(temp_dir, "collected_classes.json")
+    class_list_json_file = configuration["collected_classes_path"]
     with open(class_list_json_file, "w") as ofile:
         ofile.write(json.dumps(class_files, sort_keys=True, indent=4))
-    class_info_json_file = os.path.join(temp_dir, "collected_classes_info.json")
+    class_info_json_file = configuration["collected_classes_info_path"]
+
     os_command = (
         analyser.get_java_class_info_tool_pathname() +
-        " --in-json \"" + class_list_json_file + "\" " +
-        " --out-json \"" + class_info_json_file + "\""
+        " --configuration-path \"" + cmdline.prepare_scan + "\""
     )
     prof_calling_java_class_info_start_time = time.time()
     print("Invoking 'java-class-info' ...")
-    if verbosity >= 9:
+    if cmdline.verbosity >= 9:
         print("CWD: " + os.getcwd())
         print("CMD: " + os_command)
     os.system(os_command)
@@ -71,29 +72,32 @@ def _read_info_of_class_files(class_files, temp_dir, verbosity):
     return classes_info, java_class_info_call_duration
 
 
-def collect_java_binaries(app_path, list_of_classpaths, entry_point, temp_dir, output_json, verbosity):
+def collect_java_binaries(cmdline):
+    with open(cmdline.prepare_scan) as config_file:
+        configuration = json.load(config_file)
+
     prof_start_time = time.time()
     prof = dict()
 
-    if os.path.splitext(app_path)[1].lower() == ".war":
+    if os.path.splitext(cmdline.input_path)[1].lower() == ".war":
         # Loading of WAR files is not supported yet in our front-end, see PR
         #   https://github.com/diffblue/cbmc/pull/608
         # Therefore, we have to unpack them and collect Java binaries.
-        unpack_dir = os.path.abspath(os.path.join(temp_dir, "WAR_UNPACK"))
+        unpack_dir = os.path.abspath(os.path.join(cmdline.common_dir, "WAR_UNPACK"))
         assert not os.path.exists(unpack_dir)
         os.makedirs(unpack_dir)
 
         with utility.PushCwd(unpack_dir):
-            os.system("jar xf " + app_path)
+            os.system("jar xf " + cmdline.input_path)
 
-        app_path = unpack_dir
+        cmdline.input_path = unpack_dir
 
     java_binaries = CollectedJavaBinaries()
-    _find_java_binaries(app_path, java_binaries)
+    _find_java_binaries(cmdline.input_path, java_binaries)
 
     library_directories = []
     java_libraries = CollectedJavaBinaries()
-    for path in list_of_classpaths:
+    for path in cmdline.libraries:
         if os.path.isdir(path):
             _find_java_binaries(path, java_libraries)
             library_directories.append(path)
@@ -106,19 +110,17 @@ def collect_java_binaries(app_path, list_of_classpaths, entry_point, temp_dir, o
 
     # First we read packages of all collected class files.
     classes_info, java_class_info_call_duration = _read_info_of_class_files(
-        java_binaries.class_files,
-        temp_dir,
-        verbosity
-        )
+        java_binaries.class_files, configuration, cmdline)
+
     prof["java-class-info"] = {"duration": java_class_info_call_duration}
 
     assert classes_info is not None and len(classes_info["results"]) != 0
 
     # We copy classes into correct locations for the package structure
     # inside the TEMP directory and pack them into a JAR file.
-    classes_temp_dir = os.path.join(temp_dir, "CLASSES")
-    assert not os.path.exists(classes_temp_dir)
-    os.makedirs(classes_temp_dir)
+    classes_temp_dir = os.path.join(cmdline.common_dir, "CLASSES")
+    if not os.path.exists(classes_temp_dir):
+        os.makedirs(classes_temp_dir)
 
     for class_src_pathname, class_file_info in classes_info["results"].items():
         class_dst_pathname = os.path.join(classes_temp_dir, class_file_info["name"].replace(".", "/") + ".class")
@@ -135,23 +137,96 @@ def collect_java_binaries(app_path, list_of_classpaths, entry_point, temp_dir, o
     # which causes multiple insertions of symbols from models library to the
     # symbols table (because 'core-models' library is automatically added to
     # each 'languaget' instance.
-    java_binaries.jar_file = os.path.join(temp_dir, "collected_classes.jar")
+    java_binaries.jar_file = os.path.join(cmdline.common_dir, "collected_classes.jar")
     with utility.PushCwd(classes_temp_dir):
         os.system("jar cf \"" + java_binaries.jar_file + "\" .")
 
-    # We write the lists of collected Java binaries into the output JSON file.
-    if not os.path.isdir(os.path.dirname(output_json)):
-        os.makedirs(os.path.dirname(output_json))
-    with open(output_json, "w") as ofile:
+    entry_points_file = configuration["detected_entry_points_path"]
+    if not os.path.exists(entry_points_file):
+        print("WARNING: Unable to find any detected entry points. No analysis will be run.")
+        return
+
+    with open(entry_points_file) as ep_config_file:
+        ep_config = json.load(ep_config_file)
+
+    # Copy the current commandline and transpose into a dictionary.
+    copied_command_line = {key.replace("_", "-"): val for key, val in vars(cmdline).items()}
+
+    # Loop over all our detected entry points and create a folder for each.
+    class_paths = [p for p in java_binaries.classpath_jar_files + java_libraries.classpath_jar_files + library_directories]
+    for ep_data in ep_config["entry_points"]:
+
+        method_data = ep_data["method"]
+
+        # We don't add the descriptor here as it's not recognized (and makes folder names too long).
+        fully_qualified_method = ep_data["class_name"] + "." + method_data["name"]
+
+        # Try to make sure the folder name isn't invalid.
+        folder_name = fully_qualified_method.replace(os.path.sep, '.')
+        output_folder = os.path.join(cmdline.results_dir, folder_name)
+
+        # If our folder exists then just add (#number) to it and try creation again.
+        incrementor = 1
+        while os.path.exists(output_folder):
+            folder_name = fully_qualified_method.replace(os.path.sep, '.') + " (" + str(incrementor) + ")"
+            output_folder = os.path.join(cmdline.results_dir, folder_name)
+            incrementor += 1
+
+        os.mkdir(output_folder)
+
+        generated_temp_folder = os.path.join(cmdline.temp_dir, folder_name)
+
         program_json = {
             "jar": java_binaries.jar_file,
-            "classpath": [p for p in java_binaries.classpath_jar_files + java_libraries.classpath_jar_files + library_directories if os.path.exists(p)],
-            "gbf": os.path.join(temp_dir, "input_program.gbf"),  # The file should not exist yet. Here we only record the prefered/desired location
-                                                                 # of the file on disk. Analyser will create it, if it does not exist (in the first run)
+            "classpath": class_paths,
+            # The file should not exist yet. Here we only record the prefered/desired location
+            #  of the file on disk. Analyser will create it, if it does not exist (in the first run)
+            "gbf": os.path.join(generated_temp_folder, "input_program.gbf"),
+            "entry-point": fully_qualified_method
+        }
+
+        program_file = os.path.join(output_folder, "program.json")
+        with open(program_file, "w") as program_json_file:
+            json.dump(program_json, program_json_file, sort_keys=True, indent=4)
+
+        command_line_file = os.path.join(output_folder, "command_line.json")
+
+        # Update our copied commandline with accurate / new values.
+        copied_command_line.update({
+            "name": "[" + cmdline.name + "] " + fully_qualified_method,
+            "output-dir": output_folder,
+            "results-dir": output_folder,
+            "temp-dir": generated_temp_folder,
+            "entry-point": fully_qualified_method
+        })
+
+        # Save the commandline file.
+        with open(command_line_file, "w") as commandline_json_file:
+            json.dump(copied_command_line, commandline_json_file, sort_keys=True, indent=4)
+
+        config_file = os.path.join(output_folder, "config.json")
+
+        root_config_json = {
+            "program": program_file,
+            "rules": cmdline.config,
+            "timeout": cmdline.timeout,
+            "verbosity": cmdline.verbosity,
+            "dump_html_summaries": cmdline.dump_html_summaries,
+            "dump_html_statistics": cmdline.dump_html_statistics,
+            "dump_html_slice": cmdline.dump_html_slice,
+            "dump_html_program": cmdline.dump_html_program,
+            "do_not_use_precise_access_paths": cmdline.do_not_use_precise_access_paths,
+            "output-dir": "./",
+            "temp-dir": generated_temp_folder,
+            "data-flow-insensitive-instrumentation": cmdline.data_flow_insensitive_instrumentation,
+            "lazy-methods-context-sensitive": cmdline.lazy_methods_context_sensitive,
+            "verify-csvsa-sparse-domains": cmdline.verify_csvsa_sparse_domains,
+            "use_goto_binary": cmdline.use_goto_binary
         }
-        if entry_point is not None:
-            program_json["entry-point"] = entry_point
-        ofile.write(json.dumps(program_json, sort_keys=True, indent=4))
+
+        # Save the config file.
+        with open(config_file, "w") as root_config_json_file:
+            json.dump(root_config_json, root_config_json_file, sort_keys=True, indent=4)
 
     # Lastly, we complete and return statistics from this stage.
     prof["duration"] = time.time() - prof_start_time

driver/presentation.py

@@ -654,30 +654,6 @@ def build_HTML_interface_to_results_and_statistics(
 
         ofile.write("<p></p>\n")
 
-        ofile.write("<table>\n"
-                    "<caption>Performance info. Times are in seconds.</caption>\n"
-                    "  <tr>\n"
-                    "    <th>Property</th>\n"
-                    "    <th>Value</th>\n"
-                    "  </tr>\n")
-        ofile.write("  <tr>\n")
-        ofile.write("    <td>Duration of collecting Java binaries</td>\n")
-        ofile.write("    <td align=\"center\">" + "{:.3f}".format(prof["collect_java_binaries"]["duration"]) + "</td>\n")
-        ofile.write("  </tr>\n")
-        ofile.write("  <tr>\n")
-        ofile.write("    <td>Duration of reading info of classes (tool java-class-info)</td>\n")
-        ofile.write("    <td align=\"center\">" + "{:.3f}".format(prof["collect_java_binaries"]["java-class-info"]["duration"]) + "</td>\n")
-        ofile.write("  </tr>\n")
-        ofile.write("  <tr>\n")
-        ofile.write("    <td>Number of collected CLASS files</td>\n")
-        ofile.write("    <td align=\"center\">" + str(prof["collect_java_binaries"]["num_classes"]) + "</td>\n")
-        ofile.write("  </tr>\n")
-        ofile.write("  <tr>\n")
-        ofile.write("    <td>Number of collected JAR files for classpath</td>\n")
-        ofile.write("    <td align=\"center\">" + str(prof["collect_java_binaries"]["num_classpath_jar_files"]) + "</td>\n")
-        ofile.write("  </tr>\n")
-        ofile.write("</table>\n")
-
         if cmdline.dump_html_program:
             original_program_root_html_filename = os.path.join(cmdline.results_dir,"goto-program","HTML","index.html")
             ofile.write("<p>\n")