Merge pull request diffblue#542 from diffblue/jd/feature/DI_tool_pipeline_integration

[SEC-239] DI generation pipeline integration

Author: JohnDumbell

CMakeLists.txt

@@ -112,3 +112,4 @@ add_custom_target(models-library ALL
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/LIBRARIES/models/model
 )
 
+add_subdirectory(env-model-generator)

driver/mkbench.py

@@ -1,4 +1,7 @@
 import os
+import re
+import sys
+
 import analyser
 import filecmp
 import shutil
@@ -70,6 +73,101 @@ def _read_info_of_class_files(class_files, configuration, cmdline):
     return classes_info, java_class_info_call_duration
 
 
+def run_di_model_generation(configuration, cmdline):
+    """
+    Runs DI overlay model generation. This attempts to work out what Spring
+    DI modelling is being used by the application under analysis and then create
+    fake Java code that will simulate what the DI would do at run-time.
+
+    At this point we should have a configuration file from java-class-info that
+    will drive generation.
+    """
+
+    # If we don't have anything vaguely looking like a spring library being used, just skip generation.
+    if not next((lib for lib in cmdline.libraries if re.search("spring-[a-zA-Z]*-[0-9]*\.[0-9]*", lib)), None) \
+            or cmdline.skip_di_generation:
+        print("No Spring Framework libraries passed in, cannot generate any DI-related code.", file=sys.stderr)
+        return None
+
+    di_output_file_path = configuration["diConfigurationPath"]
+    detected_entry_points_path = configuration["detectedEntryPointsPath"]
+    di_starting_script = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "env-model-generator", "env-model-generator"))
+
+    # Right now we're only using our metadata generated files, but this should also include XML.
+    spring_files = []
+    if os.path.isfile(di_output_file_path):
+        spring_files.append(di_output_file_path)
+
+    # Clear down any existing files.
+    generated_source_file_path = os.path.join(cmdline.common_dir, "GENERATED_SOURCE")
+    if os.path.exists(generated_source_file_path):
+        os.rmdir(generated_source_file_path)
+    os.mkdir(generated_source_file_path)
+
+    primary_input_file = "".join(spring_files[:1])
+    if not os.path.isfile(primary_input_file):
+        print("No files available to generate DI overlays.")
+        return None
+
+    di_generation_commandline = [di_starting_script, primary_input_file, "--output-path", generated_source_file_path, "--entry-points-input-file", detected_entry_points_path]
+
+    additional_spring_files = spring_files[1:]
+    if additional_spring_files:
+        di_generation_commandline.extend(["--input-file"] + additional_spring_files)
+
+    di_generation_result = utility.call(di_generation_commandline, "DI model generator", cmdline.verbosity, stderr=subprocess.STDOUT)
+    if di_generation_result != 0:
+        return None
+
+    # Quick check to see if we have any output files.
+    if not next((file for root, dirs, files in os.walk(generated_source_file_path) for file in files), None):
+        print("No Java source files emitted by DI generation.")
+        return None
+
+    # Clear down yet more existing files.
+    java_binaries_path = os.path.join(cmdline.common_dir, "GENERATED_BINARIES")
+    if os.path.exists(java_binaries_path):
+        os.rmdir(java_binaries_path)
+    os.mkdir(java_binaries_path)
+
+    # Combine the incoming libraries with our collected classes, this
+    # should hold every reference we need to compile.
+    class_paths = cmdline.libraries + [os.path.join(cmdline.common_dir, "collected_classes.jar")]
+
+    generated_jar_path = os.path.join(cmdline.common_dir, "DI-models.jar")
+
+    ant_format_arguments = {
+        "generated_source_file_path": generated_source_file_path,
+        "class_paths": os.linesep.join(["            <pathelement location=\"" + path + "\"/>" for path in class_paths]),
+        "generated_jar_path": generated_jar_path,
+        "java_binaries_path": java_binaries_path
+    }
+
+    ant_build_statement = """
+<project default="compile">
+  <target name="compile">
+      <javac includeantruntime="true" srcdir="{generated_source_file_path}" destdir="{java_binaries_path}">
+        <classpath>
+{class_paths}
+        </classpath>
+      </javac>
+      <jar destfile="{generated_jar_path}" basedir="{java_binaries_path}"/>
+  </target>
+</project> 
+    """.format(**ant_format_arguments)
+
+    ant_build_path = os.path.join(generated_source_file_path, "build.xml")
+    with open(ant_build_path, 'x') as ant_xml_file:
+        ant_xml_file.write(ant_build_statement)
+
+    javac_result = utility.call(["ant", "-file", ant_build_path], "Java Compiler for DI model overlays", cmdline.verbosity, stderr=subprocess.STDOUT)
+    if javac_result != 0:
+        print("Java compilation of DI-generated source failed.")
+        return None
+
+    return generated_jar_path
+
+
 def collect_java_binaries(cmdline):
     with open(cmdline.prepare_scan) as config_file:
         configuration = json.load(config_file)
@@ -157,9 +255,6 @@ def collect_java_binaries(cmdline):
         print("WARNING: Unable to find any detected entry points. No analysis will be run.")
         return
 
-    with open(entry_points_file) as ep_config_file:
-        ep_config = json.load(ep_config_file)
-
     # Copy the current commandline and transpose into a dictionary.
     copied_command_line = {key.replace("_", "-"): val for key, val in vars(cmdline).items()}
 
@@ -170,14 +265,31 @@ def collect_java_binaries(cmdline):
                    java_libraries.classpath_jar_files +
                    library_directories)
 
+    di_output_path = run_di_model_generation(configuration, cmdline)
+
+    # If DI generation has run we want to target its output jar as our starting point.
+    target_binary = java_binaries.jar_file
+    if di_output_path:
+        class_paths.append(target_binary)
+        class_paths.append(di_output_path) # Appended because we need overlay classes too.
+        target_binary = di_output_path
+    else:
+        print("DI generation failed, continuing without synthetic entry points.", file=sys.stderr)
+
+    with open(entry_points_file) as ep_config_file:
+        ep_config = json.load(ep_config_file)
+
     previously_created_folders = set()
     for ep_data in ep_config["entryPoints"]:
 
         method_data = ep_data["method"]
 
         # We don't add the descriptor here as it's not recognized (and makes folder names too long).
         friendly_method_name = ep_data["className"] + "." + method_data["name"]
-        raw_method_name = ep_data["className"] + "." + method_data["signature"]
+
+        actual_entry_point = \
+            "com.diffblue.security.SyntheticEntryPoints." + method_data["syntheticMethodName"] if di_output_path \
+            else ep_data["className"] + "." + method_data["signature"]
 
         # Try to make sure the folder name isn't invalid.
         folder_name = friendly_method_name.replace(os.path.sep, '.')
@@ -199,12 +311,12 @@ def collect_java_binaries(cmdline):
         generated_temp_folder = os.path.join(cmdline.temp_dir, folder_name)
 
         program_json = {
-            "jar": java_binaries.jar_file,
+            "jar": target_binary,
             "classpath": class_paths,
             # The file should not exist yet. Here we only record the prefered/desired location
             #  of the file on disk. Analyser will create it, if it does not exist (in the first run)
             "gbf": os.path.join(generated_temp_folder, "input_program.gbf"),
-            "entry-point": raw_method_name
+            "entry-point": actual_entry_point
         }
 
         program_file = os.path.join(output_folder, "program.json")
@@ -219,7 +331,7 @@ def collect_java_binaries(cmdline):
             "output-dir": output_folder,
             "results-dir": output_folder,
             "temp-dir": generated_temp_folder,
-            "entry-point": raw_method_name
+            "entry-point": actual_entry_point
         })
 
         # Save the commandline file.

driver/run.py

@@ -195,6 +195,8 @@ def create_parser():
                         help="Path to the configuration file used to do the pre-analysis stage")
     parser.add_argument("--run-scan", type=str, const='', nargs="?",
                         help="Flag for whether a security scan should be run. Target will be the results folder.")
+    parser.add_argument("--skip-di-generation", action='store_true',
+                        help="Turns off DI overlay generation. This overrides the automatic detection of DI usage.")
 
     return parser
 
@@ -432,6 +434,22 @@ def __main():
               analyser.get_missing_binary_error_message())
         return
 
+    def extract_wildcard_paths(path):
+        """
+        If our path is a directory and has a trailing * that isn't related
+        to any concrete item, recursively find all .jar files in that folder
+        and its children.
+        """
+        if not os.path.exists(path) and path.endswith("*") and os.path.isdir(path.strip('*')):
+            return [os.path.join(root, file)
+                    for root, directories, files in os.walk(path.strip('*'))
+                    for file in files if re.search(".*\.jar$", file)]
+        else:
+            return [path]
+
+    # Transform incoming libraries.
+    cmdline.libraries = [path for library_path in cmdline.libraries for path in extract_wildcard_paths(library_path)]
+
     common_libraries = _get_common_libraries(cmdline.models_library_location)
 
     if cmdline.use_models_library:

env-model-generator/CMakeLists.txt

@@ -0,0 +1,19 @@
+project(SECURITY_ANALYSER)
+
+file(GLOB_RECURSE typescript_files "${CMAKE_CURRENT_SOURCE_DIR}/src/*.ts")
+
+string(REGEX REPLACE "\\.ts($|;)" ".js\\1" javascript_files "${typescript_files}")
+string(REPLACE "/src/" "/built/" javascript_files "${javascript_files}")
+
+add_custom_target(env-model-generator ALL
+    DEPENDS ${javascript_files}
+)
+
+add_custom_command(
+    COMMENT "Recompiling env-model-generator."
+    OUTPUT ${javascript_files}
+    DEPENDS ${typescript_files};package.json
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND yarn install
+    COMMAND yarn run build
+)

env-model-generator/env-model-generator

@@ -1,3 +1,4 @@
 #!/bin/bash
-chmod a+x built/env-model-generator.js
-built/env-model-generator.js "$@"
+parent_path=$(dirname $(readlink -f $0))
+chmod a+x "$parent_path/built/env-model-generator.js"
+"$parent_path/built/env-model-generator.js" "$@"

src/java-class-info/entry_point.cpp

@@ -13,6 +13,7 @@
 #include <java_bytecode/java_types.h>
 
 #include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/replace.hpp>
 
 entry_pointt::entry_pointt(
   const std::string &class_name,
@@ -59,6 +60,8 @@ jsont entry_pointt::to_json()
   method_obj[json_namest::return_type] = json_stringt(method.return_type);
   method_obj[json_namest::name] = json_stringt(method.name);
   method_obj[json_namest::signature] = json_stringt(method.signature);
+  method_obj[json_namest::synthetic_method_name] = json_stringt(
+    boost::replace_all_copy(class_name, ".", "_") + "_" + method.name);
 
   return json_entry_point;
 }

src/java-class-info/entry_point.h

@@ -21,6 +21,7 @@ class entry_pointt
   {
   public:
     static constexpr const char *name = "name";
+    static constexpr const char *synthetic_method_name = "syntheticMethodName";
     static constexpr const char *signature = "signature";
     static constexpr const char *type = "type";
     static constexpr const char *arguments = "arguments";