Added requested regression tests.

Author: marek-trtik

driver/run.py

@@ -294,32 +294,29 @@ def run_scan(cmdline):
 
     instrumented_program_json_path = os.path.join(cmdline.results_dir, "program_slicing", "instrumented_goto_program.json")
 
-    if not os.path.exists(instrumented_program_json_path):
-        print("Instrumented program .json don't exist at " + instrumented_program_json_path + ".")
-        return
-
-    with open(instrumented_program_json_path, "r") as f:
-        instrumented_program = json.load(f)
-    # Instrumented GOTO binaries are specified relative to the results directory
-    # (the security-analyser working directory). Rewrite them with absolute paths:
-    instrumented_program["goto_binary_file"] = os.path.abspath(os.path.join(cmdline.results_dir, instrumented_program["goto_binary_file"]))
-
-    print("Starting program slicing.")
-    prof["program_slicing"] = analyser.run_program_slicing(
-        instrumented_program,
-        os.path.abspath(os.path.join(cmdline.results_dir,"program_slicing")),
-        cmdline.timeout,
-        cmdline.verbosity,
-        cmdline.dump_html_instrumented_goto
-        )
-
-    print("Starting the search for error traces.")
-    prof["search_for_error_traces"] = analyser.run_search_for_error_traces(
-        os.path.abspath(os.path.join(cmdline.results_dir,"program_slicing","sliced_goto_program.json")),
-        os.path.abspath(os.path.join(cmdline.results_dir,"search_for_error_traces")),
-        cmdline.timeout,
-        cmdline.verbosity
-        )
+    if os.path.exists(instrumented_program_json_path):
+        with open(instrumented_program_json_path, "r") as f:
+            instrumented_program = json.load(f)
+        # Instrumented GOTO binaries are specified relative to the results directory
+        # (the security-analyser working directory). Rewrite them with absolute paths:
+        instrumented_program["goto_binary_file"] = os.path.abspath(os.path.join(cmdline.results_dir, instrumented_program["goto_binary_file"]))
+
+        print("Starting program slicing.")
+        prof["program_slicing"] = analyser.run_program_slicing(
+            instrumented_program,
+            os.path.abspath(os.path.join(cmdline.results_dir,"program_slicing")),
+            cmdline.timeout,
+            cmdline.verbosity,
+            cmdline.dump_html_instrumented_goto
+            )
+
+        print("Starting the search for error traces.")
+        prof["search_for_error_traces"] = analyser.run_search_for_error_traces(
+            os.path.abspath(os.path.join(cmdline.results_dir,"program_slicing","sliced_goto_program.json")),
+            os.path.abspath(os.path.join(cmdline.results_dir,"search_for_error_traces")),
+            cmdline.timeout,
+            cmdline.verbosity
+            )
 
     print("Building performance plots.")
     prof_plots = {}

regression/end_to_end/driver.py

@@ -11,8 +11,9 @@
 
 class ErrorTraces:
 
-    def __init__(self, traces, cmdline):
+    def __init__(self, traces, pretty_cmdline, cmdline):
         self.traces = traces
+        self.pretty_cmdline = pretty_cmdline
         self.cmdline = cmdline
 
     def count_traces(self):
@@ -64,12 +65,21 @@ def trace_goes_through(
                         pass
         return False
 
+    def get_cmdline(self):
+        return self.cmdline
+
+    def get_results_dir(self):
+        return self.cmdline[self.cmdline.index("-R") + 1]
+
+    def get_temp_dir(self):
+        return self.cmdline[self.cmdline.index("-T") + 1]
+
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         if exc_value is not None:
-            print("Failure may relate to command: ", self.cmdline, file=sys.stderr)
+            print("Failure may relate to command: ", self.pretty_cmdline, file=sys.stderr)
 
 
 def pretty_print_commandline(cmdline):
@@ -109,7 +119,8 @@ def run_security_analyser_pipeline(
     base_path,
     entry_point,
     load_strategy=LoadStrategy.conventional_lazy_loading,
-    extra_args=None):
+    extra_args=None,
+    keep_results=False):
 
     if extra_args is None:
         extra_args = []
@@ -148,9 +159,9 @@ def run_security_analyser_pipeline(
         cmdline.append("--verify-csvsa-sparse-domains")
 
     with utils.working_dir(analyzer_home), \
-         utils.temp_dir_deleter(results_dir), \
-         utils.temp_dir_deleter(temporary_dir), \
-         utils.temp_dir_deleter(common_dir):
+         utils.temp_dir_deleter(results_dir, keep_results), \
+         utils.temp_dir_deleter(temporary_dir, keep_results), \
+         utils.temp_dir_deleter(common_dir, keep_results):
 
         run_security_driver_script(cmdline)
 
@@ -179,4 +190,4 @@ def run_security_analyser_pipeline(
             print("Test \"%s\" kept results (%s) and temporary directory (%s)" %
                   (pretty_cmdline, results_dir, temporary_dir))
 
-        return ErrorTraces(traces, pretty_cmdline)
+        return ErrorTraces(traces, pretty_cmdline, cmdline)

regression/end_to_end/early_check/build.xml

@@ -0,0 +1,17 @@
+<project name="Main" basedir="." default="compile">
+
+  <property name="root.dir"      value="./"/>
+  <property name="src.dir"       value="${root.dir}/src"/>
+  <property name="classes.dir"   value="${root.dir}/build"/>
+
+  <target name="compile">
+    <antcall target="clean" />
+    <mkdir dir="${classes.dir}"/>
+    <javac srcdir="${src.dir}" destdir="${classes.dir}" includeantruntime="false" debug="on" />
+  </target>
+
+  <target name="clean">
+    <delete dir="${classes.dir}"/>
+  </target>
+
+</project>

regression/end_to_end/early_check/rules.json

@@ -0,0 +1,32 @@
+{
+  "rules":
+    [
+      {
+        "comment": "Obtaining a tainted string.",
+        "class": "Main",
+        "method": "source:()Ljava/lang/String;",
+        "result": {
+          "location": "return_value",
+          "taint": "Tainted string"
+        }
+      },
+      {
+        "comment": "Returning a sanitized string.",
+        "class": "Main",
+        "method": "sanitize:(Ljava/lang/String;)Ljava/lang/String;",
+        "sanitizes": {
+          "location": "returns",
+          "taint": "Clean string"
+        }
+      },
+      {
+        "comment": "Writing a potentially tainted data into the sink",
+        "class": "Main",
+        "method": "sink:(Ljava/lang/String;)V",
+        "sinkTarget": {
+          "location": "arg0",
+          "vulnerability": "Tainted string"
+        }
+      }
+    ]
+}

regression/end_to_end/early_check/src/Main.java

@@ -0,0 +1,42 @@
+public class Main {
+
+  public static String source() {
+    return "Tainted string";
+  }
+
+  public static void sink(String s) {
+  }
+
+  public static String sanitize(String s) {
+    return "sanitised string";
+  }
+
+  // Test cases
+
+  public static void catch_impossible_taint_flow_01() {
+    String s = "no taint";
+    sink(s);
+  }
+
+  public static void catch_impossible_taint_flow_02() {
+    String s = source();
+    s = sanitize(s);
+    sink(s);
+  }
+
+  public static void miss_impossible_taint_flow_01() {
+    String s = source();
+    int i = 0;
+    if (i == 1)
+        sink(s);
+  }
+
+  public static void miss_impossible_taint_flow_02() {
+    String s = source();
+    int i = 20;
+    if (i == 20)
+        s = sanitize(s);
+    sink(s);
+  }
+}
+

regression/end_to_end/early_check/test_early_check.py

@@ -0,0 +1,83 @@
+import fasteners
+import os
+import subprocess
+import pytest
+
+from regression.end_to_end.driver import run_security_analyser_pipeline
+import regression.utils as utils
+
+
+@fasteners.interprocess_locked(os.path.join(os.path.dirname(__file__), ".build_lock"))
+def test_early_check_catch_impossible_taint_flow_01(load_strategy):
+    with utils.working_dir(os.path.abspath(os.path.dirname(__file__))):
+        subprocess.call(["ant"])
+    with run_security_analyser_pipeline(
+            "build",
+            "rules.json",
+            os.path.realpath(os.path.dirname(__file__)),
+            "Main.catch_impossible_taint_flow_01",
+            load_strategy,
+            keep_results=True) as traces:
+        assert traces.count_traces() == 0
+        index_file = os.path.join(traces.get_results_dir(), "Main.catch_impossible_taint_flow_01/index.html")
+        assert os.path.exists(index_file)
+        with open(index_file, "r") as ifile:
+            content = ifile.read()
+        assert "The program is safe. No tainted data may reach any sink." in content
+
+
+@fasteners.interprocess_locked(os.path.join(os.path.dirname(__file__), ".build_lock"))
+def test_early_check_catch_impossible_taint_flow_02(load_strategy):
+    with utils.working_dir(os.path.abspath(os.path.dirname(__file__))):
+        subprocess.call(["ant"])
+    with run_security_analyser_pipeline(
+            "build",
+            "rules.json",
+            os.path.realpath(os.path.dirname(__file__)),
+            "Main.catch_impossible_taint_flow_02",
+            load_strategy,
+            keep_results=True) as traces:
+        assert traces.count_traces() == 0
+        index_file = os.path.join(traces.get_results_dir(), "Main.catch_impossible_taint_flow_02/index.html")
+        assert os.path.exists(index_file)
+        with open(index_file, "r") as ifile:
+            content = ifile.read()
+        assert "The program is safe. No tainted data may reach any sink." in content
+
+
+@fasteners.interprocess_locked(os.path.join(os.path.dirname(__file__), ".build_lock"))
+def test_early_check_miss_impossible_taint_flow_01(load_strategy):
+    with utils.working_dir(os.path.abspath(os.path.dirname(__file__))):
+        subprocess.call(["ant"])
+    with run_security_analyser_pipeline(
+            "build",
+            "rules.json",
+            os.path.realpath(os.path.dirname(__file__)),
+            "Main.miss_impossible_taint_flow_01",
+            load_strategy,
+            keep_results=True) as traces:
+        assert traces.count_traces() == 0
+        index_file = os.path.join(traces.get_results_dir(), "Main.miss_impossible_taint_flow_01/index.html")
+        assert os.path.exists(index_file)
+        with open(index_file, "r") as ifile:
+            content = ifile.read()
+        assert "The program is safe. No tainted data may reach any sink." not in content
+
+
+@fasteners.interprocess_locked(os.path.join(os.path.dirname(__file__), ".build_lock"))
+def test_early_check_miss_impossible_taint_flow_02(load_strategy):
+    with utils.working_dir(os.path.abspath(os.path.dirname(__file__))):
+        subprocess.call(["ant"])
+    with run_security_analyser_pipeline(
+            "build",
+            "rules.json",
+            os.path.realpath(os.path.dirname(__file__)),
+            "Main.miss_impossible_taint_flow_02",
+            load_strategy,
+            keep_results=True) as traces:
+        assert traces.count_traces() == 0
+        index_file = os.path.join(traces.get_results_dir(), "Main.miss_impossible_taint_flow_02/index.html")
+        assert os.path.exists(index_file)
+        with open(index_file, "r") as ifile:
+            content = ifile.read()
+        assert "The program is safe. No tainted data may reach any sink." not in content

regression/utils.py

@@ -14,9 +14,9 @@ def working_dir(new_dir):
 
 
 @contextlib.contextmanager
-def temp_dir_deleter(dirname):
+def temp_dir_deleter(dirname, explicit_keep=False):
     yield
-    if "SECURITY_ANALYSER_END_TO_END_TESTS_KEEP_RESULTS" in os.environ:
+    if explicit_keep is True or "SECURITY_ANALYSER_END_TO_END_TESTS_KEEP_RESULTS" in os.environ:
         return
     try:
         shutil.rmtree(dirname)