Merge pull request diffblue#151 from diffblue/bugfix/evalautor

Bugfixes in the Evalautor of our benchmarks

Author: marek-trtik

benchmarks/evaluator.py

@@ -2,7 +2,6 @@
 import sys
 import os
 import json
-import urllib
 import zipfile
 import shutil
 import fnmatch
@@ -11,6 +10,15 @@
 def __get_my_dir(): return os.path.dirname(os.path.realpath(__file__))
 
 
+def _download_file(url, dst_pathname):
+    if (sys.version_info > (3, 0)):
+        import urllib.request
+        urllib.request.urlretrieve(url, dst_pathname)
+    else:
+        import urllib
+        urllib.URLopener().retrieve(url, dst_pathname)
+
+
 def __benchmark_installer_libraries():
     try:
         libraries_root_dir = os.path.join(__get_my_dir(), "LIBRARIES")
@@ -23,8 +31,8 @@ def __benchmark_installer_libraries():
         if not os.path.exists(tomcat_dir):
             os.makedirs(temp_dir)
             downloaded_zip_pathname = os.path.join(temp_dir, "__tomcat9__.zip")
-            urllib.URLopener().retrieve(
-                "http://mirror.vorboss.net/apache/tomcat/tomcat-9/v9.0.0.M21/bin/apache-tomcat-9.0.0.M21.zip",
+            _download_file(
+                "http://archive.apache.org/dist/tomcat/tomcat-9/v9.0.0.M21/bin/apache-tomcat-9.0.0.M21.zip",
                 downloaded_zip_pathname
                 )
             with zipfile.ZipFile(downloaded_zip_pathname, 'r') as zip_file:
@@ -49,10 +57,10 @@ def search_for_rt_jar(start_dir):
                 if not java_rt_file:
                     os.makedirs(temp_dir)
                     downloaded_deb_pathname = os.path.join(temp_dir, "__openjdk8__.deb")
-                    urllib.URLopener().retrieve(
+                    _download_file(
                         "http://ftp.uk.debian.org/debian/pool/main/o/openjdk-8/openjdk-8-jre-headless_8u121-b13-1~bpo8+1_amd64.deb",
                         downloaded_deb_pathname
-                    )
+                        )
                     old_cwd = os.getcwd()
                     os.chdir(temp_dir)
                     os.system("ar -x " + downloaded_deb_pathname)
@@ -70,7 +78,7 @@ def search_for_rt_jar(start_dir):
                 shutil.rmtree(temp_dir)
         return None     # no error
     except:
-        print "In '__benchmark_installer_libraries()': Unexpected error:", sys.exc_info()[0]
+        print("In '__benchmark_installer_libraries()': Unexpected error:", sys.exc_info()[0])
 
 
 def __benchmark_installer_TRAINING_diffblue(record):
@@ -123,7 +131,7 @@ def __evaluate_benchmark(record):
         return "Cannot find the installer function '" + record["installer"] + "'."
     error_message = eval(record["installer"])(record)
     if error_message:
-        return error_messages   # failed!
+        return error_message    # failed!
 
     # Now we can run the security-scanner on the installed benchmark
     command = (
@@ -142,7 +150,7 @@ def __evaluate_benchmark(record):
         os.path.join(__get_my_dir(), record["results-dir"], "search_for_error_traces", "error_traces.json")
     if not os.path.exists(error_traces_pathname):
         return "The results file '" + error_traces_pathname + "' does not exist."
-    with open(error_traces_pathname,"rb") as ifile:
+    with open(error_traces_pathname,"r") as ifile:
         error_traces = json.load(ifile)
     return __compare_computed_and_expected_results(error_traces,record)
 
@@ -188,7 +196,7 @@ def __main():
             return 1  # failure (for Travis)
 
         for benchmark_pathname in sorted(benchmarks):
-            with open(benchmark_pathname, "rb") as ifile:
+            with open(benchmark_pathname, "r") as ifile:
                 benchmark = json.load(ifile)
 
             error_message = __benchmark_installer_libraries()
@@ -201,7 +209,11 @@ def __main():
                 return 1    # failure (for Travis)
             print("PASSED:\"" + benchmark["sources-dir"] + "\"")
         return 0    # success (for Travis)
+    except Exception as e:
+        print("ERROR: " + str(e))
+        return 1    # failure (for Travis)
     except:
+        print("ERROR: Unknown exception was raised.")
         return 1    # failure (for Travis)