Add CSVSA tests folder

The test mainly checks that security scanner doesn't crash.

I've tried to structure it so that we could reuse some of the code to
use pytest to check test.desc files written for test.pl. More work would
be needed to implement this.

Author: Owen Jones

regression/CSVSA/TestBasic/Test$A.class

@@ -0,0 +1,17 @@
+package CSVSA.TestBasic;
+
+public class Test {
+  class A {
+    public void f() {
+    }
+
+    public void g() {
+    }
+  }
+
+  public void tolerate_unreachable_functions() {
+    A a = null;
+    a.f();
+    a.g();
+  }
+}

regression/CSVSA/TestBasic/Test.class

@@ -0,0 +1,15 @@
+from regression.CSVSA.csvsa_driver import CsvsaDriver
+
+
+folder_name = 'TestBasic'
+
+
+def test_tolerate_unreachable_functions():
+    csvsa_driver = CsvsaDriver(folder_name).with_test_function('tolerate_unreachable_functions')
+    csvsa_expectation = csvsa_driver.run()
+
+    csvsa_expectation.check_successful_run()
+    csvsa_expectation.check_does_not_match("a \. f\(\)")
+    csvsa_expectation.check_does_not_match("a \. g\(\)")
+    csvsa_expectation.check_does_not_match("a \. f\(\)")
+    csvsa_expectation.check_does_not_match("a \. g\(\)")

regression/CSVSA/TestBasic/Test.java

@@ -0,0 +1,45 @@
+import os
+
+from regression.executable_runner import ExecutableRunner
+from regression.regex_expectation import RegexExpectation
+
+
+class CsvsaExpectation(RegexExpectation):
+    """Encapsulate the output of CsvsaDriver"""
+
+    def __init__(self, goto_functions, return_code):
+        """Get the state at the end of test_function"""
+        RegexExpectation.__init__(self, goto_functions, return_code, compatibility_mode=False, check_verification=False)
+
+
+class CsvsaDriver:
+    """Run security scanner with CSVSA"""
+
+    def __init__(self, folder_name):
+        """Set the class path and temporary directory"""
+        self.folder_name = folder_name
+        self.with_test_class_name('Test')
+        self.with_test_function('f')
+
+    def with_test_class_name(self, test_class_name):
+        self.class_name = test_class_name
+        self.class_filename = test_class_name + '.class'
+        return self
+
+    def with_test_function(self, test_function_name):
+        self.function_name = test_function_name
+        return self
+
+    def run(self):
+        """Run CSVSA and parse the output before returning it"""
+        fq_class_name = '.'.join(['CSVSA', self.folder_name, self.class_name])
+        fq_function_name = fq_class_name + '.' + self.function_name
+        executable_path = os.path.join(os.environ['SECURITY_SCANNER_HOME'], 'bin', 'security-analyzer')
+        class_path = os.path.join('CSVSA', self.folder_name, self.class_filename)
+        cmd = [executable_path, '--function', fq_function_name, '--lazy-methods-context-sensitive',
+               '--show-goto-functions', class_path]
+
+        executable_runner = ExecutableRunner(cmd)
+        (stdout, _, return_code) = executable_runner.run(pipe_stderr_to_stdout=True)
+
+        return CsvsaExpectation(stdout, return_code)

regression/CSVSA/TestBasic/__init__.py

@@ -13,7 +13,7 @@ def __init__(self, cmd, timeout=600, encoding='utf-8'):
         self.timeout = timeout
         self.encoding = encoding
 
-    def run(self):
+    def run(self, pipe_stderr_to_stdout=False):
         """Run the executable and capture the output
 
         If the process exceeds the timeout then we assert false so that the test fails.
@@ -22,13 +22,14 @@ def run(self):
         """
         # We limit ourselves to the API available in python 3.3 to make life easier on Travis. There are API
         # improvements in python 3.5 and 3.6 which would make this code simpler.
-        proc = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stderr = subprocess.STDOUT if pipe_stderr_to_stdout else subprocess.PIPE
+        proc = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=stderr)
         try:
             (stdout_data, stderr_data) = proc.communicate(timeout=self.timeout)
         except subprocess.TimeoutExpired:
             proc.kill()
             print('The executable ran for longer than {} seconds'.format(self.timeout))
             assert False
-        return (stdout_data.decode(self.encoding),
-                stderr_data.decode(self.encoding),
+        return (stdout_data.decode(self.encoding) if stdout_data else None,
+                stderr_data.decode(self.encoding) if stderr_data else None,
                 proc.returncode)

regression/CSVSA/TestBasic/test_basic.py

@@ -0,0 +1,40 @@
+import re
+
+
+class RegexExpectation:
+    """Encapsulate the output of CsvsaDriver"""
+
+    def __init__(self, text, return_code, compatibility_mode=True, check_verification=False):
+        """Get the state at the end of test_function"""
+        if return_code < 0:
+            self.exit_code = 0
+            self.signal = -return_code
+        else:
+            self.exit_code = return_code
+            self.signal = 0
+        if compatibility_mode:
+            # This makes is easier to use this code with test.desc files written for test.pl
+            text += "\nEXIT={}\nSIGNAL={}\n".format(self.exit_code, self.signal)
+        self.text = text
+        self.split_text = text.splitlines()
+        # check_verification indicates whether a successful run should include "VERIFICATION SUCCESSFUL" in it
+        self.check_verification = check_verification
+
+    def check_successful_run(self):
+        self.check_does_not_match_regex("^warning: ignoring")
+        if self.check_verification:
+            self.check_matches_regex("^VERIFICATION SUCCESSFUL$")
+        assert self.exit_code == 0
+        assert self.signal == 0
+
+    def check_matches(self, x):
+        assert any(line.find(x) != -1 for line in self.split_text)
+
+    def check_does_not_match(self, x):
+        assert all(line.find(x) == -1 for line in self.split_text)
+
+    def check_matches_regex(self, x):
+        assert any(re.search(x, line) for line in self.split_text)
+
+    def check_does_not_match_regex(self, x):
+        assert all(not re.search(x, line) for line in self.split_text)