Merge pull request diffblue#472 from diffblue/owen-jones-diffblue/make-csvsa-tests-robust

SEC-518: Make CSVSA tests robust to changing locations

Author: owen-jones-diffblue

regression/CSVSA/TestIncompatibleTypes/test_incompatible_types.py

@@ -1,16 +1,17 @@
-import pytest
 from regression.CSVSA.csvsa_driver import CsvsaDriver
 
 
 folder_name = 'TestIncompatibleTypes'
 
-@pytest.mark.xfail(strict=True)
+
 def test_tolerate_unreachable_functions():
-    pytest.xfail("Callsite numbers vary on CI / local run.")
-    
     csvsa_driver = \
         CsvsaDriver(folder_name).with_test_function('test').with_show_goto_functions(False)
     with csvsa_driver.run() as csvsa_expectation:
         csvsa_expectation.check_successful_run()
-        csvsa_expectation.check_matches("Callsite at 43 may call: java::CSVSA.TestIncompatibleTypes.C.f")
-        csvsa_expectation.check_does_not_match("Callsite at 43 may call: java::CSVSA.TestIncompatibleTypes.B.f")
+        csvsa_expectation.check_matches_regex(
+            "Callsite at [\d]+ \(CSVSA/TestIncompatibleTypes/Test.java:10\) may call: "
+            "java::CSVSA.TestIncompatibleTypes.C.f:\(\)V")
+        csvsa_expectation.check_does_not_match_regex(
+            "Callsite at [\d]+ \(CSVSA/TestIncompatibleTypes/Test.java:10\) may call: "
+            "java::CSVSA.TestIncompatibleTypes.B.f")

regression/CSVSA/TestOpaqueTypePromotion/AbstractTest.class

@@ -1,16 +1,20 @@
-import pytest
-
 from regression.CSVSA.csvsa_driver import CsvsaDriver
-
+import pytest
 folder_name = 'TestOpaqueTypePromotion'
 
 
-@pytest.mark.xfail(strict=True)
+@pytest.mark.xfail(strict=True, reason="Unknown failure")
 def test_opaque_type_promotion():
     csvsa_driver = \
         CsvsaDriver(folder_name).with_test_function('test').with_show_goto_functions(False)
     with csvsa_driver.run() as csvsa_expectation:
         csvsa_expectation.check_successful_run()
-        csvsa_expectation.check_matches("Callsite at 17 may call: java::CSVSA.TestOpaqueTypePromotion.ConcreteTest.f:()V")
-        csvsa_expectation.check_matches("Callsite at 25 (java::CSVSA.TestOpaqueTypePromotion.AbstractTest.f:()V) stubbed")
-        csvsa_expectation.check_matches("Callsite at 43 (java::CSVSA.TestOpaqueTypePromotion.Opaque.f:()V) stubbed")
+        csvsa_expectation.check_matches_regex(
+            "Callsite at [\d]+ \(CSVSA/TestOpaqueTypePromotion/Test.java:10\) may call: "
+            "java::CSVSA.TestOpaqueTypePromotion.ConcreteTest.f:\(\)V")
+        csvsa_expectation.check_matches_regex(
+            "Callsite at [\d]+ \(CSVSA/TestOpaqueTypePromotion/Test.java:14\) for "
+            "java::CSVSA.TestOpaqueTypePromotion.AbstractTest.f:\(\)V stubbed")
+        csvsa_expectation.check_matches_regex(
+            "Callsite at [\d]+ \(CSVSA/TestOpaqueTypePromotion/Test.java:19\) for "
+            "java::CSVSA.TestOpaqueTypePromotion.Opaque.f:\(\)V stubbed")

regression/CSVSA/TestOpaqueTypePromotion/ConcreteTest.class

@@ -650,31 +650,38 @@ void csvsa_function_contextt::print_tree(
   }
   for(const auto &child : callee_nodes)
   {
+    auto inst = prog.instructions.begin();
+    while(inst != prog.instructions.end() &&
+          inst->location_number < child.first)
+    {
+      ++inst;
+    }
+    INVARIANT(
+      inst != prog.instructions.end() && inst->location_number == child.first,
+      "child map key should correspond to a program location number");
+
+    std::ostringstream line_start;
+    line_start << pad_space << "Callsite at " << child.first;
+
+    if(!inst->source_location.get_file().empty())
+    {
+      line_start << " (" << inst->source_location.get_file() << ":"
+                 << inst->source_location.get_line() << ")";
+    }
+
     if(child.second.empty())
     {
-      auto inst = prog.instructions.begin();
-      while(inst != prog.instructions.end() &&
-            inst->location_number < child.first)
-        ++inst;
-      if(inst == prog.instructions.end())
-        throw "Bad entry in child map #1?";
-      if(inst->location_number != child.first)
-        throw "Bad entry in child map #2?";
       const auto &function = to_code_function_call(inst->code).function();
-      out << pad_space << "Callsite at " << child.first << " ("
-          << function.get(ID_C_class) << "." << function.get(ID_component_name)
-          << ") stubbed\n";
+      out << line_start.str() << " for " << function.get(ID_identifier)
+          << " stubbed\n";
     }
     else
     {
       for(const auto &call : child.second)
       {
-        out << pad_space << "Callsite at " << child.first
-            << " may call: " << call.first << "\n";
-        std::ostringstream oss;
-        oss << pad_space << "Callee " << call.first << ":\n";
+        out << line_start.str() << " may call: " << call.first << "\n";
         if(driver.edge_is_recursive(context_number, call.second))
-          oss << " (recursive edge)\n";
+          out << " (recursive edge)\n";
         else
         {
           driver.get_context(call.second)