Merge pull request diffblue#296 from diffblue/smowton/feature/print-command-on-test-failure

Regression tests: print failed driver command on test failure

Author: smowton

regression/end_to_end/driver.py

@@ -3,15 +3,16 @@
 import os.path
 import tempfile
 import regression.utils as utils
-
+import sys
 
 from regression.executable_runner import ExecutableRunner
 
 
 class ErrorTraces:
 
-    def __init__(self, traces):
+    def __init__(self, traces, cmdline):
         self.traces = traces
+        self.cmdline = cmdline
 
     def count_traces(self):
         count = 0
@@ -47,6 +48,22 @@ def trace_goes_through(
                         pass
         return False
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        print("Failure may relate to command: ", self.cmdline, file=sys.stderr)
+
+def quote_pathnames_in_command_line(cmdline):
+    assert isinstance(cmdline, list) and len(cmdline) > 0
+    result = cmdline[:1]
+    for idx in range(1, len(cmdline)):
+        if cmdline[idx - 1] in ["-C", "-I", "-R", "-T"] or \
+           (cmdline[idx - 1] == "--name" and " " in cmdline[idx]):
+            result.append("'" + cmdline[idx] + "'")
+        else:
+            result.append(cmdline[idx])
+    return result
 
 def run_security_analyser_pipeline(
     relative_binary_path,
@@ -84,28 +101,18 @@ def run_security_analyser_pipeline(
     if "SECURITY_ANALYSER_END_TO_END_TESTS_EXTRA_ARGS" in os.environ:
         cmdline.extend(os.environ["SECURITY_ANALYSER_END_TO_END_TESTS_EXTRA_ARGS"].split(","))
 
+    pretty_cmdline = " ".join(quote_pathnames_in_command_line(cmdline))
+
     with utils.working_dir(analyzer_home), \
          utils.temp_dir_deleter(results_dir), \
          utils.temp_dir_deleter(temporary_dir):
 
-        def quote_pathnames_in_command_line(cmdline):
-            assert isinstance(cmdline, list) and len(cmdline) > 0
-            result = cmdline[:1]
-            for idx in range(1, len(cmdline)):
-                if cmdline[idx - 1] in ["-C", "-I", "-R", "-T"] or \
-                   (cmdline[idx - 1] == "--name" and " " in cmdline[idx]):
-                    result.append("'" + cmdline[idx] + "'")
-                else:
-                    result.append(cmdline[idx])
-            return result
-
         executable_runner = ExecutableRunner(cmdline)
         (stdout, stderr, ret) = executable_runner.run()
         if ret != 0:
             raise Exception(
                 "Failed running \"%s\":\nstdout:\n\n%s\nstderr\n\n%s" % \
-                (" ".join(quote_pathnames_in_command_line(cmdline)),
-                 stdout, stderr))
+                (pretty_cmdline, stdout, stderr))
 
         trace_list = \
             os.path.join(
@@ -126,7 +133,6 @@ def quote_pathnames_in_command_line(cmdline):
 
         if "SECURITY_ANALYSER_END_TO_END_TESTS_KEEP_RESULTS" in os.environ:
             print("Test \"%s\" kept results (%s) and temporary directory (%s)" %
-                  (" ".join(quote_pathnames_in_command_line(cmdline)),
-                   results_dir, temporary_dir))
+                  (pretty_cmdline, results_dir, temporary_dir))
 
-        return ErrorTraces(traces)
+        return ErrorTraces(traces, pretty_cmdline)

regression/end_to_end/general001/test_general001.py

@@ -5,9 +5,9 @@
 import os
 
 def test_general001():
-    traces = run_security_analyser_pipeline(
-        "taint_traces_01.jar",
-        "taint_traces_01_rules.json",
-        os.path.realpath(os.path.dirname(__file__)))
-    assert traces.count_traces() > 0
-    assert traces.trace_exists("java::Main.bug:(I)V", 84)
+    with run_security_analyser_pipeline(
+      "taint_traces_01.jar",
+      "taint_traces_01_rules.json",
+      os.path.realpath(os.path.dirname(__file__))) as traces:
+        assert traces.count_traces() > 0
+        assert traces.trace_exists("java::Main.bug:(I)V", 84)

regression/end_to_end/general002/test_general002.py

@@ -5,8 +5,8 @@
 import os
 
 def test_general002():
-    traces = run_security_analyser_pipeline(
-        "taint_traces_02.jar",
-        "taint_traces_02_rules.json",
-        os.path.realpath(os.path.dirname(__file__)))
-    assert traces.count_traces() == 0
+    with run_security_analyser_pipeline(
+      "taint_traces_02.jar",
+      "taint_traces_02_rules.json",
+      os.path.realpath(os.path.dirname(__file__))) as traces:
+        assert traces.count_traces() == 0

regression/end_to_end/general003/test_general003.py

@@ -5,8 +5,8 @@
 import os
 
 def test_general003():
-    traces = run_security_analyser_pipeline(
-        "taint_traces_03.jar",
-        "taint_traces_03_rules.json",
-        os.path.realpath(os.path.dirname(__file__)))
-    assert traces.count_traces() == 0
+    with run_security_analyser_pipeline(
+      "taint_traces_03.jar",
+      "taint_traces_03_rules.json",
+      os.path.realpath(os.path.dirname(__file__))) as traces:
+        assert traces.count_traces() == 0

regression/end_to_end/general004/test_general004.py

@@ -5,11 +5,11 @@
 import os
 
 def test_general004():
-    traces = run_security_analyser_pipeline(
-        "taint_traces_04.jar",
-        "taint_traces_04_rules.json",
-        os.path.realpath(os.path.dirname(__file__)))
-    assert traces.count_traces() > 0
-    assert traces.trace_exists("java::Main.baz0:()V", 19)
-    assert traces.trace_exists("java::Main.baz1:()V", 26)
-    assert traces.trace_exists("java::Main.baz2:(LOther;)V", 32)
+    with run_security_analyser_pipeline(
+      "taint_traces_04.jar",
+      "taint_traces_04_rules.json",
+      os.path.realpath(os.path.dirname(__file__))) as traces:
+        assert traces.count_traces() > 0
+        assert traces.trace_exists("java::Main.baz0:()V", 19)
+        assert traces.trace_exists("java::Main.baz1:()V", 26)
+        assert traces.trace_exists("java::Main.baz2:(LOther;)V", 32)

regression/end_to_end/general005/test_general005.py

@@ -5,10 +5,10 @@
 import os
 
 def test_general005():
-    traces = run_security_analyser_pipeline(
-        "taint_traces_05.jar",
-        "taint_traces_05_rules.json",
-        os.path.realpath(os.path.dirname(__file__)),
-        ["--data-flow-insensitive-instrumentation"])
-    assert traces.count_traces() > 0
-    assert traces.trace_exists("java::DummyAssignmentSubmissionEdit.setSubmittedText:(Ljava/lang/String;)V", 3)
+    with run_security_analyser_pipeline(
+      "taint_traces_05.jar",
+      "taint_traces_05_rules.json",
+      os.path.realpath(os.path.dirname(__file__)),
+      ["--data-flow-insensitive-instrumentation"]) as traces:
+        assert traces.count_traces() > 0
+        assert traces.trace_exists("java::DummyAssignmentSubmissionEdit.setSubmittedText:(Ljava/lang/String;)V", 3)

regression/end_to_end/general006/test_general006.py

@@ -5,11 +5,11 @@
 import os
 
 def test_general006():
-    traces = run_security_analyser_pipeline(
-        "taint_traces_06.jar",
-        "taint_traces_06_rules.json",
-        os.path.realpath(os.path.dirname(__file__)),
-        ["--data-flow-insensitive-instrumentation"])
-    assert traces.count_traces() > 0
-    assert traces.trace_exists("java::taint_test.test.doGet:(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V", 165)
-    assert traces.trace_exists("java::taint_test.test.doGet:(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V", 143)
+    with run_security_analyser_pipeline(
+      "taint_traces_06.jar",
+      "taint_traces_06_rules.json",
+      os.path.realpath(os.path.dirname(__file__)),
+      ["--data-flow-insensitive-instrumentation"]) as traces:
+        assert traces.count_traces() > 0
+        assert traces.trace_exists("java::taint_test.test.doGet:(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V", 165)
+        assert traces.trace_exists("java::taint_test.test.doGet:(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V", 143)

regression/end_to_end/general007/test_general007.py

@@ -5,9 +5,9 @@
 import os
 
 def test_general007():
-    traces = run_security_analyser_pipeline(
-        "taint_traces_07.jar",
-        "taint_traces_07_rules.json",
-        os.path.realpath(os.path.dirname(__file__)))
-    assert traces.count_traces() > 0
-    assert traces.trace_exists("java::training07.test.doGet:(Ltraining07/HttpServletRequest;Ltraining07/HttpServletResponse;)V", 109)
+    with run_security_analyser_pipeline(
+      "taint_traces_07.jar",
+      "taint_traces_07_rules.json",
+      os.path.realpath(os.path.dirname(__file__))) as traces:
+        assert traces.count_traces() > 0
+        assert traces.trace_exists("java::training07.test.doGet:(Ltraining07/HttpServletRequest;Ltraining07/HttpServletResponse;)V", 109)