Merge pull request diffblue#518 from diffblue/owen-jones-diffblue/change-csvsa-test-mechanism

SEC-581: change csvsa test mechanism (and fix problems in our tests)

Author: owen-jones-diffblue

benchmarks/LIBRARIES/install_apache_tomcat.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+TOMCAT_DIR=apache-tomcat-9
+
+if [[ ! -d $TOMCAT_DIR ]]; then
+    wget http://archive.apache.org/dist/tomcat/tomcat-9/v9.0.0.M21/bin/apache-tomcat-9.0.0.M21.zip
+    unzip apache-tomcat-9.0.0.M21.zip
+    rm apache-tomcat-9.0.0.M21.zip
+    mv apache-tomcat-9.0.0.M21 $TOMCAT_DIR
+    if [[ ! -d $TOMCAT_DIR ]]; then
+        exit 1
+    fi
+fi
+
+exit 0

regression/CMakeLists.txt

@@ -10,12 +10,3 @@ set_tests_properties(regression-tests
     PROPERTIES
     LABELS "CORE;REGRESSION"
     ENVIRONMENT SECURITY_SCANNER_HOME=${CMAKE_BINARY_DIR})
-
-add_test(
-    NAME regression-tests-with-csvsa
-    COMMAND ${PYTHON_EXECUTABLE} -m pytest
-    WORKING_DIRECTORY ${security-scanner_SOURCE_DIR}/regression)
-set_tests_properties(regression-tests-with-csvsa
-    PROPERTIES
-    LABELS "CORE;REGRESSION"
-    ENVIRONMENT "SECURITY_SCANNER_HOME=${CMAKE_BINARY_DIR};SECURITY_REGRESSION_TESTS_USE_CSVSA=1")

regression/LVSA/TestArray/test_array.py

@@ -4,21 +4,21 @@
 folder_name = 'TestArray'
 
 
-def test_lvsa_null_initialized_array(tmpdir):
+def test_lvsa_null_initialized_array(tmpdir, load_strategy):
     """Arrays are automatically null-initialized"""
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('null_initialized_array')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_output()
 
     value_set_expectation.check_number_of_values(1)
     value_set_expectation.check_contains_null_object()
 
 
-def test_lvsa_manually_filled_array_1(tmpdir):
+def test_lvsa_manually_filled_array_1(tmpdir, load_strategy):
     """We cannot tell that the whole array has been given a non-null value"""
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('manually_filled_array_1')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_output()
 
@@ -27,10 +27,10 @@ def test_lvsa_manually_filled_array_1(tmpdir):
     value_set_expectation.check_contains_dynamic_object(is_most_recent_allocation=True)
 
 
-def test_lvsa_manually_filled_array_2(tmpdir):
+def test_lvsa_manually_filled_array_2(tmpdir, load_strategy):
     """We cannot tell that the whole array has been given a non-null value"""
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('manually_filled_array_2')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_output()
 
@@ -40,10 +40,10 @@ def test_lvsa_manually_filled_array_2(tmpdir):
     value_set_expectation.check_contains_dynamic_object(is_most_recent_allocation=False)
 
 
-def test_lvsa_initialize_array(tmpdir):
+def test_lvsa_initialize_array(tmpdir, load_strategy):
     """We use one value set for all indices"""
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('initialize_array')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_output()
 

regression/LVSA/TestBasic/test_basic.py

@@ -4,19 +4,19 @@
 folder_name = 'TestBasic'
 
 
-def test_outputs_null(tmpdir):
+def test_outputs_null(tmpdir, load_strategy):
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('outputs_null')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_output()
 
     value_set_expectation.check_number_of_values(1)
     value_set_expectation.check_contains_null_object()
 
 
-def test_outputs_null_or_dynamic_object(tmpdir):
+def test_outputs_null_or_dynamic_object(tmpdir, load_strategy):
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('outputs_null_or_dynamic_object')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_output()
 
@@ -25,9 +25,9 @@ def test_outputs_null_or_dynamic_object(tmpdir):
     value_set_expectation.check_contains_dynamic_object()
 
 
-def test_outputs_dynamic_object_with_allocated_field(tmpdir):
+def test_outputs_dynamic_object_with_allocated_field(tmpdir, load_strategy):
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('outputs_dynamic_object_with_allocated_field')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_output()
 
@@ -41,9 +41,9 @@ def test_outputs_dynamic_object_with_allocated_field(tmpdir):
     value_set_expectation.check_contains_dynamic_object()
 
 
-def test_only_track_pointers(tmpdir):
+def test_only_track_pointers(tmpdir, load_strategy):
     lvsa_driver = LvsaDriver(tmpdir, folder_name).with_test_function('only_track_pointers')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     fq_class = lvsa_expectation.fq_class_with_java_prefix
     fq_variable_name = fq_class + '.output_int'

regression/LVSA/TestCasts/test_casts.py

@@ -13,12 +13,12 @@ def _get_folder_name():
 
 
 @fasteners.interprocess_locked(os.path.join(os.path.dirname(__file__), ".build_lock"))
-def test_cast_from_child(tmpdir):
+def test_cast_from_child(tmpdir, load_strategy):
     """ Test addition and removal of casts to DO's """
     with utils.working_dir(_get_this_file_dir()):
         os.system("javac Test.java")
     lvsa_driver = driver.LvsaDriver(tmpdir, _get_folder_name()).with_test_function('cast_from_child')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_public_static('interfaceFromChild')
     value_set_expectation.check_number_of_values(1)
@@ -30,12 +30,12 @@ def test_cast_from_child(tmpdir):
 
 
 @fasteners.interprocess_locked(os.path.join(os.path.dirname(__file__), ".build_lock"))
-def test_cast_from_parent(tmpdir):
+def test_cast_from_parent(tmpdir, load_strategy):
     """ Test addition and removal of casts to DO's """
     with utils.working_dir(_get_this_file_dir()):
         os.system("javac Test.java")
     lvsa_driver = driver.LvsaDriver(tmpdir, _get_folder_name()).with_test_function('cast_from_parent')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_public_static('interfaceFromParent')
     value_set_expectation.check_number_of_values(1)
@@ -47,12 +47,12 @@ def test_cast_from_parent(tmpdir):
 
 
 @fasteners.interprocess_locked(os.path.join(os.path.dirname(__file__), ".build_lock"))
-def test_cast_from_interface(tmpdir):
+def test_cast_from_interface(tmpdir, load_strategy):
     """ Test addition and removal of casts to DO's """
     with utils.working_dir(_get_this_file_dir()):
         os.system("javac Test.java")
     lvsa_driver = driver.LvsaDriver(tmpdir, _get_folder_name()).with_test_function('cast_from_interface')
-    lvsa_expectation = lvsa_driver.run()
+    lvsa_expectation = lvsa_driver.run(load_strategy)
 
     value_set_expectation = lvsa_expectation.get_value_set_for_public_static('parentFromInterface')
     value_set_expectation.check_number_of_values(1)