Merge pull request diffblue#357 from diffblue/python_driver__improved_traiting_of_models_lib_and_standard_lib

Python driver  improved traiting of models lib and Java standard lib

Author: marek-trtik

benchmarks/LIBRARIES/README.txt

@@ -24,44 +24,37 @@ of the sub-libraries:
 1. Install "Java OpenJDK 8" by using the standard package management system of
    Ubuntu. (e.g. openjdk-8-jdk-headless)
 2. Create a directory "openjdk-8" into this directory.
-3. Locate the file "rt.jar" of "Java OpenJDK 8" on the disc. It typically
-   resides at location:
-      /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar
-4. Copy the found "rt.jar" file into the created directory "openjdk-8".
-   Optional: Instead of copying the "rt.jar", you can alternatively use "jar"
+3. Locate the files "rt.jar" and "jce.jar" of "Java OpenJDK 8" on the disc. They
+   typically reside in the same directory:
+      /usr/lib/jvm/java-8-openjdk-amd64/jre/lib
+4. Copy the found files into the created directory "openjdk-8".
+   Optional: Instead of copying the files, you can alternatively use "jar"
              utility with options "xvf" specified to unpack the content of
-             the "rt.jar" file into the created directory "openjdk-8". This
+             the files into the created directory "openjdk-8". This
              allows you to later remove/adopt/substitute some desired parts
              of the library.
+5. (Optional) Download also source code of the Package (not used by the tool; but might be 
+   useful for degugging purposes):
+        http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/archive/tip.zip/src/share/classes
+   NOTE: the link above was taken from the shell script of the models library:
+        <security-scanner-root>/benchmarks/LIBRARIES/models/model/get-original-jdk.sh [line 10]
 
-Where to find JAR with javax.crypto.*:
-    - This is not in OpenJDK. It is only in the Oracle's JDK, see:
-        https://stackoverflow.com/questions/14935447/why-am-i-getting-package-javax-crypto-does-not-exist
-    - The package can be downloaded from:
-        http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-java-plat-419418.html#JCE-1_2_2-G-JS
+Alternative installation - use of Oracle's JDK:
 
-
-Alternative installation:
-
-1. Download OpenJDK 8 .deb package, e.g. from:
-      http://ftp.uk.debian.org/debian/pool/main/o/openjdk-8/openjdk-8-jre-headless_8u121-b13-1~bpo8+1_amd64.deb
-2. Extract its content into some directory.
-3. Find the "rt.jar" file in the unpacked directory. It typically is in:
-        <your-unpack-dir>/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar
-2. Create a directory "openjdk-8" into this directory (of this README.txt file).
-4. Copy the found "rt.jar" file into the created directory "openjdk-8".
-
-
-IMPORTANT NOTE: In order to analyse Sakai, we need to use Oracle's JDK 8,
-    because these packages are missing in the OpenJDK:
-        javax.crypto.*
-        sun.invoke.*
-        com.sum.jmx.*
-    You can download Oracle's JDK here:
+1. Download Oracle's JDK here:
         http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
-    Then, you need to take these JARs from the directory 'jre/lib':
-        rt.jar
-        jce.jar
+2. Create a directory "oracle-jdk-8" into this directory.
+3. Unpack JAR files "rt.jar" and "jce.jar" from the downloaded package into the 
+   created directory. Both JAR should be located in this directory in the package:
+        jdk1.8.0_161/lib
+
+
+NOTE: You can install both versions of JDK. In that case, the driver script
+            <security-scanner-root>/driver/run.py
+      will first check for presence of OpenJDK version. If NOT present, then
+      Oracle's version is checked for presence. Note though that the option
+      --use-java-runtime-library must be passed to the script in order to
+      enable the checks described above.
 
 
 (2) Apache Tomcat 9

driver/mkbench.py

@@ -91,13 +91,18 @@ def collect_java_binaries(app_path, list_of_classpaths, entry_point, temp_dir, o
     java_binaries = CollectedJavaBinaries()
     _find_java_binaries(app_path, java_binaries)
 
+    library_directories = []
     java_libraries = CollectedJavaBinaries()
     for path in list_of_classpaths:
         if os.path.isdir(path):
             _find_java_binaries(path, java_libraries)
+            library_directories.append(path)
+        else:
+            java_libraries.classpath_jar_files.append(path)
 
     prof["num_classes"] = len(java_binaries.class_files)
-    prof["num_classpath_jar_files"] = len(java_binaries.classpath_jar_files)
+    prof["num_classpath_jar_files"] = len(java_binaries.classpath_jar_files) + len(java_libraries.classpath_jar_files)
+    prof["num_classpath_directories"] = len(library_directories)
 
     # First we read packages of all collected class files.
     classes_info, java_class_info_call_duration = _read_info_of_class_files(
@@ -138,11 +143,13 @@ def collect_java_binaries(app_path, list_of_classpaths, entry_point, temp_dir, o
     if not os.path.isdir(os.path.dirname(output_json)):
         os.makedirs(os.path.dirname(output_json))
     with open(output_json, "w") as ofile:
-        ofile.write(json.dumps({
+        program_json = {
             "jar": java_binaries.jar_file,
-            "classpath": [p for p in java_binaries.classpath_jar_files + java_libraries.classpath_jar_files + list_of_classpaths if os.path.exists(p)],
-            "entry-point": entry_point
-            }, sort_keys=True, indent=4))
+            "classpath": [p for p in java_binaries.classpath_jar_files + java_libraries.classpath_jar_files + library_directories if os.path.exists(p)],
+        }
+        if entry_point is not None:
+            program_json["entry-point"] = entry_point
+        ofile.write(json.dumps(program_json, sort_keys=True, indent=4))
 
     # Lastly, we complete and return statistics from this stage.
     prof["duration"] = time.time() - prof_start_time

driver/run.py

@@ -24,16 +24,26 @@ def get_diffblue_models_library_props():
             props["diffblue_models_library"]["paths"].append(pathname)
         else:
             props["diffblue_models_library"]["error"] = "Cannot access Diffblue Models Library's file " + pathname
-            return props
-        pathname = os.path.join(get_benchmark_library_dir(), "oracle-jdk-8")
-        if os.path.isdir(pathname):
-            props["diffblue_models_library"]["paths"].append(pathname)
-        else:
-            pathname = os.path.join(get_benchmark_library_dir(), "openjdk-8")
-            if os.path.isdir(pathname):
-                props["diffblue_models_library"]["paths"].append(pathname)
-            else:
-                print("Warning: cannot access JDK install directory " + pathname)
+        return props
+
+    def get_java_runtime_library():
+        props = {"java_runtime_library": {"paths": [], "error": None}}
+        pathname = os.path.join(get_benchmark_library_dir(), "openjdk-8")
+        if not os.path.isdir(pathname):
+            pathname = os.path.join(get_benchmark_library_dir(), "oracle-jdk-8")
+            if not os.path.isdir(pathname):
+                props["java_runtime_library"]["error"] = (
+                    "Cannot find Java standard library on neither of the paths: " +
+                    os.path.join(get_benchmark_library_dir(), "openjdk-8") + ", " + pathname
+                    )
+                return props
+        jars_selection = ["rt.jar", "jce.jar"]
+        for jar in jars_selection:
+            jar_full_path = os.path.join(pathname, jar)
+            if os.path.isfile(jar_full_path):
+                props["java_runtime_library"]["paths"].append(jar_full_path)
+        if os.path.join(pathname, "rt.jar") not in props["java_runtime_library"]["paths"]:
+            props["java_runtime_library"]["error"] = "Cannot find 'rt.jar' Java standard library package in the directory " + pathname
         return props
 
     def get_apache_tomcat_props():
@@ -58,6 +68,7 @@ def get_spring_framework_props():
 
     result = {}
     result.update(get_diffblue_models_library_props())
+    result.update(get_java_runtime_library())
     result.update(get_apache_tomcat_props())
     result.update(get_spring_framework_props())
     return result
@@ -119,6 +130,10 @@ def __parse_cmd_line():
                              "during the analysis.")
     parser.add_argument("--use-models-library", action="store_true",
                         help="Add the Diffblue Models Library's JAR file to the classpath of the security-scanner.")
+    parser.add_argument("--use-java-runtime-library", action="store_true",
+                        help="Add the Java standard library to the classpath. First, there will be attempt to add "
+                             "OpenJDK version of the library. If it is not found (e.g. not installed), then the "
+                             "Oracle's version of the library is searched for.")
     parser.add_argument("--use-apache-tomcat", action="store_true",
                         help="Add the Apache Tomcat's JAR files to the classpath of the security-scanner.")
     parser.add_argument("--use-spring-framework", action="store_true",
@@ -161,6 +176,7 @@ def evaluate(cmdline, common_libraries):
         cmdline.input_path,
         cmdline.libraries
             + (common_libraries["diffblue_models_library"]["paths"] if cmdline.use_models_library else [])
+            + (common_libraries["java_runtime_library"]["paths"] if cmdline.use_java_runtime_library else [])
             + (common_libraries["apache_tomcat"]["paths"] if cmdline.use_apache_tomcat else [])
             + (common_libraries["spring_framework"]["paths"] if cmdline.use_spring_framework else []),
         cmdline.entry_point,
@@ -313,6 +329,9 @@ def __main():
     if cmdline.use_models_library and common_libraries["diffblue_models_library"]["error"] is not None:
         print("ERROR: " + common_libraries["diffblue_models_library"]["error"])
         return
+    if cmdline.use_java_runtime_library and common_libraries["java_runtime_library"]["error"] is not None:
+        print("ERROR: " + common_libraries["java_runtime_library"]["error"])
+        return
     if cmdline.use_apache_tomcat and common_libraries["apache_tomcat"]["error"] is not None:
         print("ERROR: " + common_libraries["apache_tomcat"]["error"])
         return