Merge pull request diffblue#244 from diffblue/owen/tests-for-lvsa

LVSA EVS tests

Author: owen-jones-diffblue

regression/LVSA/TestArray/test_array.py

@@ -24,7 +24,7 @@ def test_lvsa_manually_filled_array_1(tmpdir):
 
     output_value_expectation.check_number_of_values(2)
     output_value_expectation.check_contains_null_object()
-    output_value_expectation.check_contains_most_recent_allocation_dynamic_object()
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=True)
 
 
 def test_lvsa_manually_filled_array_2(tmpdir):
@@ -36,8 +36,8 @@ def test_lvsa_manually_filled_array_2(tmpdir):
 
     output_value_expectation.check_number_of_values(3)
     output_value_expectation.check_contains_null_object()
-    output_value_expectation.check_contains_non_recent_allocation_dynamic_object()
-    output_value_expectation.check_contains_most_recent_allocation_dynamic_object()
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=True)
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=False)
 
 
 def test_lvsa_initialize_array(tmpdir):
@@ -49,4 +49,4 @@ def test_lvsa_initialize_array(tmpdir):
 
     output_value_expectation.check_number_of_values(4)
     output_value_expectation.check_contains_null_object()
-    output_value_expectation.check_contains_most_recent_allocation_dynamic_object(3)
+    output_value_expectation.check_contains_dynamic_object(3, is_most_recent_allocation=True)

regression/LVSA/TestEVS/Test$Node.class

@@ -0,0 +1,33 @@
+package LVSA.TestEVS;
+
+public class Test {
+  public static Node static_node;
+  public static Object static_object;
+
+  public class Node {
+    Object left;
+    Object right;
+  }
+
+  public void write_static_field_to_parameter_field(Node parameter_node) {
+    parameter_node.left = static_node.left;
+  }
+
+  public void write_parameter_field_to_static_field(Node parameter_node) {
+    static_node.left = parameter_node.left;
+  }
+
+  public void write_new_class_to_parameter_field(Node parameter_node) {
+    parameter_node.left = new Object();
+  }
+
+  public void write_new_class_to_static_field() {
+    static_node.left = new Object();
+  }
+
+  public void modified(Node parameter_node) {
+    parameter_node.left = new Object();
+    parameter_node.right = new Object();
+    static_object = parameter_node.left;
+  }
+}

regression/LVSA/TestEVS/Test.class

@@ -0,0 +1,57 @@
+from regression.LVSA.lvsa_driver import LvsaDriver
+
+folder_name = 'TestEVS'
+
+
+def test_write_static_field_to_parameter_field(tmpdir):
+    lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('write_static_field_to_parameter_field')
+    lvsa_expectation = lvsa_driver.run()
+
+    output_value_expectation = lvsa_expectation.check_external_value_set('Node', '.left')
+
+    output_value_expectation.check_number_of_values(2)
+    output_value_expectation.check_contains_external_value_set(expected_number=2)
+
+
+def test_write_parameter_field_to_static_field(tmpdir):
+    lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('write_parameter_field_to_static_field')
+    lvsa_expectation = lvsa_driver.run()
+
+    output_value_expectation = lvsa_expectation.check_external_value_set('Node', '.left')
+
+    output_value_expectation.check_number_of_values(2)
+    output_value_expectation.check_contains_external_value_set(expected_number=2)
+
+
+def test_write_new_class_to_parameter_field(tmpdir):
+    lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('write_new_class_to_parameter_field')
+    lvsa_expectation = lvsa_driver.run()
+
+    output_value_expectation = lvsa_expectation.check_external_value_set('Node', '.left')
+
+    output_value_expectation.check_contains_dynamic_object()
+
+
+def test_write_new_class_to_static_field(tmpdir):
+    lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('write_new_class_to_static_field')
+    lvsa_expectation = lvsa_driver.run()
+
+    output_value_expectation = lvsa_expectation.check_external_value_set('Node', '.left')
+
+    output_value_expectation.check_contains_dynamic_object()
+
+
+def test_modified(tmpdir):
+    lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('modified')
+    lvsa_expectation = lvsa_driver.run()
+
+    output_value_expectation_1 = lvsa_expectation.check_external_value_set('Node', '.left')
+
+    output_value_expectation_1.check_number_of_values(2)
+    output_value_expectation_1.check_contains_dynamic_object(is_most_recent_allocation=True)
+    output_value_expectation_1.check_contains_external_value_set(has_been_read=False)
+
+    output_value_expectation_2 = lvsa_expectation.check_public_static('static_object')
+
+    output_value_expectation_2.check_number_of_values(1)
+    output_value_expectation_2.check_contains_external_value_set(has_been_read=True)

regression/LVSA/TestEVS/Test.java

@@ -11,7 +11,7 @@ def test_outputs_most_recent(tmpdir):
     output_value_expectation = lvsa_expectation.check_output()
 
     output_value_expectation.check_number_of_values(1)
-    output_value_expectation.check_contains_most_recent_allocation_dynamic_object()
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=True)
 
 
 def test_outputs_most_recent_or_null(tmpdir):
@@ -22,7 +22,7 @@ def test_outputs_most_recent_or_null(tmpdir):
 
     output_value_expectation.check_number_of_values(2)
     output_value_expectation.check_contains_null_object()
-    output_value_expectation.check_contains_most_recent_allocation_dynamic_object()
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=True)
 
 
 def test_outputs_most_recent_non_recent_or_null(tmpdir):
@@ -33,8 +33,8 @@ def test_outputs_most_recent_non_recent_or_null(tmpdir):
 
     output_value_expectation.check_number_of_values(3)
     output_value_expectation.check_contains_null_object()
-    output_value_expectation.check_contains_most_recent_allocation_dynamic_object()
-    output_value_expectation.check_contains_non_recent_allocation_dynamic_object()
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=True)
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=False)
 
 
 def test_outputs_non_recent_or_null(tmpdir):
@@ -45,7 +45,7 @@ def test_outputs_non_recent_or_null(tmpdir):
 
     output_value_expectation.check_number_of_values(2)
     output_value_expectation.check_contains_null_object()
-    output_value_expectation.check_contains_non_recent_allocation_dynamic_object()
+    output_value_expectation.check_contains_dynamic_object(is_most_recent_allocation=False)
 
 
 def test_most_recent_gets_strong_update(tmpdir):

regression/LVSA/TestEVS/__init__.py

@@ -4,45 +4,75 @@
 from regression.executable_runner import ExecutableRunner
 
 
-class DynamicObject:
-    """Turn the json for a dynamic object into a python object"""
+class Expr:
+    """Common code for different exprs"""
 
-    def __init__(self, dynamic_object_expr):
-        assert dynamic_object_expr['id'] == 'dynamic_object'
-        self.instance = dynamic_object_expr['sub'][0]['namedSub']['value']
-        self.is_most_recent_allocation = dynamic_object_expr['namedSub']['is_most_recent_allocation']['id']
-        self.type = dynamic_object_expr['namedSub']['type']['id']
+    def __init__(self, expr):
+        self.type = expr['namedSub']['type']['id']
         if self.type == 'symbol':
-            self.type_identifier = dynamic_object_expr['namedSub']['type']['namedSub']['identifier']['id']
+            self.type_identifier = expr['namedSub']['type']['namedSub']['identifier']['id']
         else:
             self.type_identifier = None
 
 
+class DynamicObject(Expr):
+    """Turn the json for a dynamic object expr into a python object"""
+
+    def __init__(self, dynamic_object_expr):
+        assert dynamic_object_expr['id'] == 'dynamic_object'
+        Expr.__init__(self, dynamic_object_expr)
+        self.instance = dynamic_object_expr['sub'][0]['namedSub']['value']
+        # Expect 'most_recent_allocation' or 'any_allocation'
+        self.is_most_recent_allocation = \
+            dynamic_object_expr['namedSub']['is_most_recent_allocation']['id'] == 'most_recent_allocation'
+
+
+class ExternalValueSet(Expr):
+    """Turn the json for an external value set expr into a python object"""
+
+    def __init__(self, evs_expr):
+        assert evs_expr['id'] == 'external_value_set'
+        Expr.__init__(self, evs_expr)
+        # Expect '0' or '1', corresponding to whether the field has ever been read (and hence values from the field
+        # could have been written to the field)
+        self.has_been_read = evs_expr['namedSub']['modified']['id'] == '1'
+        # Expect the parameter name for parameters and 'external_objects' for other things, like static fields
+        self.label = evs_expr['sub'][0]['namedSub']['value']['id']
+        self.is_external_object = self.label == 'external_objects'
+        self.access_paths = evs_expr['sub'][1:]
+
+
 class VariableExpectation:
     """Encapsulate the states that a particular variable can take"""
 
     def __init__(self, var_state):
-        self.var_state = var_state
-        self.null_objects = [x for x in self.var_state['values'] if x['id'] == 'NULL-object']
-        self.dynamic_objects = [DynamicObject(x) for x in self.var_state['values'] if x['id'] == 'dynamic_object']
+        self.var_state = var_state['values']
+        self.null_objects = [x for x in self.var_state if x['id'] == 'NULL-object']
+        self.dynamic_objects = [DynamicObject(x) for x in self.var_state if x['id'] == 'dynamic_object']
+        self.external_value_sets = [ExternalValueSet(x) for x in self.var_state if x['id'] == 'external_value_set']
 
     def check_number_of_values(self, n):
-        assert n == len(self.var_state['values'])
+        assert n == len(self.var_state)
 
     def check_contains_null_object(self, expected_number=1):
         matches = self.null_objects
         assert expected_number == len(matches)
 
-    def check_contains_dynamic_object(self, expected_number=1):
+    def check_contains_dynamic_object(self, expected_number=1, is_most_recent_allocation=None):
         matches = self.dynamic_objects
+        if is_most_recent_allocation is not None:
+            # Expect True or False
+            matches = [x for x in matches if x.is_most_recent_allocation == is_most_recent_allocation]
         assert expected_number == len(matches)
 
-    def check_contains_most_recent_allocation_dynamic_object(self, expected_number=1):
-        matches = [x for x in self.dynamic_objects if x.is_most_recent_allocation == 'most_recent_allocation']
-        assert expected_number == len(matches)
-
-    def check_contains_non_recent_allocation_dynamic_object(self, expected_number=1):
-        matches = [x for x in self.dynamic_objects if x.is_most_recent_allocation == 'any_allocation']
+    def check_contains_external_value_set(self, expected_number=1, is_external_object=None, has_been_read=None):
+        matches = self.external_value_sets
+        if is_external_object is not None:
+            # Expect True or False
+            matches = [x for x in matches if x.is_external_object == is_external_object]
+        if has_been_read is not None:
+            # Expect True or False
+            matches = [x for x in matches if x.has_been_read == has_been_read]
         assert expected_number == len(matches)
 
 
@@ -51,9 +81,16 @@ class LvsaExpectation:
 
     def __init__(self, lvsa_stdout, fq_class, fq_function):
         """Get the state at the end of test_function"""
-        self.fq_class = 'java::' + fq_class
+        self.fq_class = fq_class
+        self.fq_class_with_java_prefix = 'java::' + self.fq_class
         self.target_function = 'java::' + fq_function + ':'
-        temp_all_functions = [x for x in json.loads(lvsa_stdout) if x.get('messageType') == 'LVSA-ALL-FUNCTIONS-DUMP']
+        try:
+            json_output = json.loads(lvsa_stdout)
+        except json.decoder.JSONDecodeError:
+            print('Error: LVSA output is not json\n\n')
+            print (lvsa_stdout)
+            return
+        temp_all_functions = [x for x in json_output if x.get('messageType') == 'LVSA-ALL-FUNCTIONS-DUMP']
         assert len(temp_all_functions) == 1
         all_functions = temp_all_functions[0]['functions']
         temp_function_data = [all_functions[key] for key in all_functions.keys() if key.startswith(
@@ -65,7 +102,7 @@ def __init__(self, lvsa_stdout, fq_class, fq_function):
 
     def check_public_static(self, variable_name, fq_class=None):
         if fq_class is None:
-            fq_class = self.fq_class
+            fq_class = self.fq_class_with_java_prefix
         fq_variable_name = fq_class + '.' + variable_name
         matches = [var_state for var_state in self.state if var_state['id'] == fq_variable_name]
         assert len(matches) == 1
@@ -80,14 +117,22 @@ def check_local_variable(self, variable_name):
         assert len(matches) == 1
         return VariableExpectation(matches[0])
 
+    def check_parameter(self, parameter_name):
+        return self.check_local_variable(parameter_name)
+
     def check_return_value(self):
         matches = [var_state for var_state in self.state if
                    var_state['id'].startswith(self.target_function) and var_state['id'].endswith('#return_value')]
         assert len(matches) == 1
         return VariableExpectation(matches[0])
 
-    def check_value_set(self, variable_name):
-        matches = [var_state for var_state in self.state if var_state['id'].startswith('value_set::')]
+    def check_external_value_set(self, inner_class=None, suffix=None):
+        target_id = 'external_objects_' + self.fq_class.replace('.', '_')
+        if inner_class is not None:
+            target_id += '$' + inner_class
+        matches = [var_state for var_state in self.state if var_state['id'].startswith(target_id)]
+        if suffix is not None:
+            matches = [var_state for var_state in matches if var_state['suffix'] == suffix]
         assert len(matches) == 1
         return VariableExpectation(matches[0])
 
@@ -102,6 +147,7 @@ def __init__(self, temp_dir, folder_name):
         self.class_name = 'Test'
         self.class_filename = 'Test.class'
         self.function_name = 'f'
+        self.max_input_tree_depth = None
 
     def with_test_class_name(self, test_class_name):
         if test_class_name.endswith('.class'):
@@ -116,14 +162,21 @@ def with_test_function(self, test_function_name):
         self.function_name = test_function_name
         return self
 
+    def with_max_input_tree_depth(self, max_input_tree_depth):
+        self.max_input_tree_depth = max_input_tree_depth
+        return self
+
     def run(self):
         """Run LVSA and parse the output before returning it"""
         fq_class_name = '.'.join(['LVSA', self.folder_name, self.class_name])
         fq_function_name = fq_class_name + '.' + self.function_name
         executable_path = os.path.join(os.environ['SECURITY_SCANNER_HOME'], 'bin', 'security-analyzer')
         class_path = os.path.join('LVSA', self.folder_name, self.class_filename)
         cmd = [executable_path, '--local-value-set-analysis', '--lvsa-summary-directory', self.temp_dir_path,
-               '--function', fq_function_name, '--show-value-sets', '--json-ui', class_path]
+               '--function', fq_function_name, '--show-value-sets', '--json-ui']
+        if self.max_input_tree_depth is not None:
+            cmd += ['--java-max-input-tree-depth', str(self.max_input_tree_depth)]
+        cmd.append(class_path)
 
         executable_runner = ExecutableRunner(cmd)
         (stdout, _, _) = executable_runner.run()

regression/LVSA/TestEVS/test_evs.py

@@ -450,6 +450,7 @@ void sec_driver_parse_optionst::help()
     "Java Bytecode frontend options:\n"
     " --classpath dir/jar          set the classpath\n"
     " --main-class class-name      set the name of the main class\n"
+    JAVA_BYTECODE_LANGUAGE_OPTIONS_HELP
     "\n"
     "Program representations:\n"
     " --show-parse-tree            show parse tree\n"

regression/LVSA/TestRecency/test_recency.py

@@ -40,8 +40,8 @@ class optionst;
   "(local-value-set-analysis)(show-value-sets)(lvsa-function):" \
   "(security-scanner):" \
   "(rebuild-taint-cache)" \
-  "(lazy-methods)" \
   UI_MESSAGE_OPTIONS \
+  JAVA_BYTECODE_LANGUAGE_OPTIONS \
   // End of options