Review amendments

Author: JohnDumbell

CMakeLists.txt

@@ -112,10 +112,4 @@ add_custom_target(models-library ALL
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/LIBRARIES/models/model
 )
 
-add_custom_target(env-model-generator ALL
-    COMMAND yarn install
-    COMMAND yarn run build
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/env-model-generator
-)
-
-
+add_subdirectory(env-model-generator)

driver/mkbench.py

@@ -1,6 +1,6 @@
 import os
 import re
-import subprocess
+import sys
 
 import analyser
 import filecmp
@@ -10,8 +10,6 @@
 import utility
 import subprocess
 
-import xml.dom.minidom
-
 class CollectedJavaBinaries:
     def __init__(self):
         self.class_files = []
@@ -88,16 +86,16 @@ def run_di_model_generation(configuration, cmdline):
     # If we don't have anything vaguely looking like a spring library being used, just skip generation.
     if not next((lib for lib in cmdline.libraries if re.search("spring-[a-zA-Z]*-[0-9]*\.[0-9]*", lib)), None) \
             or cmdline.skip_di_generation:
-        print("No Spring Framework libraries passed in, skipping DI generation.")
-        return
+        print("No Spring Framework libraries passed in, cannot generate any DI-related code.", file=sys.stderr)
+        return None
 
     di_output_file_path = configuration["diConfigurationPath"]
     detected_entry_points_path = configuration["detectedEntryPointsPath"]
-    di_starting_script = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "env-model-generator", "built", "env-model-generator.js"))
+    di_starting_script = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "env-model-generator", "env-model-generator"))
 
     # Right now we're only using our metadata generated files, but this should also include XML.
     spring_files = []
-    if os.path.exists(di_output_file_path) and os.path.isfile(di_output_file_path):
+    if os.path.isfile(di_output_file_path):
         spring_files.append(di_output_file_path)
 
     # Clear down any existing files.
@@ -106,18 +104,25 @@ def run_di_model_generation(configuration, cmdline):
         os.rmdir(generated_source_file_path)
     os.mkdir(generated_source_file_path)
 
-    di_generation_commandline = ["node", di_starting_script, "".join(spring_files[:1]), "--input-file", " ".join(spring_files[1:]), "--output-path", generated_source_file_path, "--entry-points-input-file", detected_entry_points_path]
+    primary_input_file = "".join(spring_files[:1])
+    if not os.path.isfile(primary_input_file):
+        print("No files available to generate DI overlays.")
+        return None
 
-    print("Running commandline: " + " ".join(di_generation_commandline))
-    di_generation_result = subprocess.run(di_generation_commandline, stderr=subprocess.STDOUT)
-    if di_generation_result.returncode != 0:
-        print("DI generation failed.")
-        return
+    di_generation_commandline = [di_starting_script, primary_input_file, "--output-path", generated_source_file_path, "--entry-points-input-file", detected_entry_points_path]
+
+    additional_spring_files = spring_files[1:]
+    if additional_spring_files:
+        di_generation_commandline.extend(["--input-file"] + additional_spring_files)
+
+    di_generation_result = utility.call(di_generation_commandline, "DI model generator", cmdline.verbosity, stderr=subprocess.STDOUT)
+    if di_generation_result != 0:
+        return None
 
     # Quick check to see if we have any output files.
     if not next((file for root, dirs, files in os.walk(generated_source_file_path) for file in files), None):
         print("No Java source files emitted by DI generation.")
-        return
+        return None
 
     # Clear down yet more existing files.
     java_binaries_path = os.path.join(cmdline.common_dir, "GENERATED_BINARIES")
@@ -130,29 +135,35 @@ def run_di_model_generation(configuration, cmdline):
     class_paths = cmdline.libraries + [os.path.join(cmdline.common_dir, "collected_classes.jar")]
 
     generated_jar_path = os.path.join(cmdline.common_dir, "DI-models.jar")
-    ant_build_file = ('<project default="compile">'
-                          '<target name="compile">'
-                              '<javac includeantruntime="true" srcdir="' + generated_source_file_path + '" destdir="' + java_binaries_path + '">'
-                              + '<classpath>' + "".join(["<pathelement location=\"" + path + "\"/>" for path in class_paths]) + '</classpath>' +
-                              '</javac>'
-                              '<jar destfile="' + generated_jar_path + '" basedir="' + java_binaries_path + '"/>'
-                          '</target>'
-                      '</project>')
-
-    # Attempt to prettify output a little.
-    ant_build_file = xml.dom.minidom.parseString(ant_build_file).toprettyxml()
+
+    ant_format_arguments = {
+        "generated_source_file_path": generated_source_file_path,
+        "class_paths": os.linesep.join(["            <pathelement location=\"" + path + "\"/>" for path in class_paths]),
+        "generated_jar_path": generated_jar_path,
+        "java_binaries_path": java_binaries_path
+    }
+
+    ant_build_statement = """
+<project default="compile">
+  <target name="compile">
+      <javac includeantruntime="true" srcdir="{generated_source_file_path}" destdir="{java_binaries_path}">
+        <classpath>
+{class_paths}
+        </classpath>
+      </javac>
+      <jar destfile="{generated_jar_path}" basedir="{java_binaries_path}"/>
+  </target>
+</project> 
+    """.format(**ant_format_arguments)
 
     ant_build_path = os.path.join(generated_source_file_path, "build.xml")
     with open(ant_build_path, 'x') as ant_xml_file:
-        ant_xml_file.write(ant_build_file)
-
-    javac_commandline = ["ant", "-file", ant_build_path, "-lib", os.pathsep.join(class_paths)]
+        ant_xml_file.write(ant_build_statement)
 
-    print("Running commandline: " + " ".join(javac_commandline))
-    javac_result = subprocess.run(javac_commandline, stderr=subprocess.STDOUT)
-    if javac_result.returncode != 0:
+    javac_result = utility.call(["ant", "-file", ant_build_path], "Java Compiler for DI model overlays", cmdline.verbosity, stderr=subprocess.STDOUT)
+    if javac_result != 0:
         print("Java compilation of DI-generated source failed.")
-        return
+        return None
 
     return generated_jar_path
 
@@ -259,9 +270,11 @@ def collect_java_binaries(cmdline):
     # If DI generation has run we want to target its output jar as our starting point.
     target_binary = java_binaries.jar_file
     if di_output_path:
-        class_paths.append(java_binaries.jar_file)
+        class_paths.append(target_binary)
         class_paths.append(di_output_path) # Appended because we need overlay classes too.
         target_binary = di_output_path
+    else:
+        print("DI generation failed, continuing without synthetic entry points.", file=sys.stderr)
 
     with open(entry_points_file) as ep_config_file:
         ep_config = json.load(ep_config_file)

driver/run.py

@@ -196,7 +196,7 @@ def create_parser():
     parser.add_argument("--run-scan", type=str, const='', nargs="?",
                         help="Flag for whether a security scan should be run. Target will be the results folder.")
     parser.add_argument("--skip-di-generation", action='store_true',
-                        help="Turns off DI overlay generation. This overrides the automatic detection of this feature.")
+                        help="Turns off DI overlay generation. This overrides the automatic detection of DI usage.")
 
     return parser
 
@@ -434,14 +434,21 @@ def __main():
               analyser.get_missing_binary_error_message())
         return
 
-    # If we've been passed a path with * on the end, unpack the .jar files in
-    # that directory (and children).
-    detected_jar_files = [os.path.join(root, file)
-                          for path in cmdline.libraries if not os.path.isdir(path) and path.endswith("*") and os.path.isdir(path.strip('*'))
-                          for root, directories, files in os.walk(path.strip('*'))
-                          for file in files if re.search(".*\.jar$", file)]
+    def extract_wildcard_paths(path):
+        """
+        If our path is a directory and has a trailing * that isn't related
+        to any concrete item, recursively find all .jar files in that folder
+        and its children.
+        """
+        if not os.path.exists(path) and path.endswith("*") and os.path.isdir(path.strip('*')):
+            return [os.path.join(root, file)
+                    for root, directories, files in os.walk(path.strip('*'))
+                    for file in files if re.search(".*\.jar$", file)]
+        else:
+            return [path]
 
-    cmdline.libraries = detected_jar_files + [path for path in cmdline.libraries if not (not os.path.isdir(path) and path.endswith("*"))]
+    # Transform incoming libraries.
+    cmdline.libraries = [path for library_path in cmdline.libraries for path in extract_wildcard_paths(library_path)]
 
     common_libraries = _get_common_libraries(cmdline.models_library_location)
 

env-model-generator/CMakeLists.txt

@@ -0,0 +1,19 @@
+project(SECURITY_ANALYSER)
+
+file(GLOB_RECURSE typescript_files "${CMAKE_CURRENT_SOURCE_DIR}/src/*.ts")
+
+string(REGEX REPLACE "\\.ts($|;)" ".js\\1" javascript_files "${typescript_files}")
+string(REPLACE "/src/" "/built/" javascript_files "${javascript_files}")
+
+add_custom_target(env-model-generator ALL
+    DEPENDS ${javascript_files}
+)
+
+add_custom_command(
+    COMMENT "Recompiling env-model-generator."
+    OUTPUT ${javascript_files}
+    DEPENDS ${typescript_files};package.json
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND yarn install
+    COMMAND yarn run build
+)

src/java-class-info/entry_point.cpp

@@ -61,7 +61,7 @@ jsont entry_pointt::to_json()
   method_obj[json_namest::name] = json_stringt(method.name);
   method_obj[json_namest::signature] = json_stringt(method.signature);
   method_obj[json_namest::synthetic_method_name] = json_stringt(
-    boost::replace_all_copy(class_name, ".", "") + "_" + method.name);
+    boost::replace_all_copy(class_name, ".", "_") + "_" + method.name);
 
   return json_entry_point;
 }