Merge pull request diffblue#179 from diffblue/feature/refined_separation_of_instrumentation_pipelines

SEC-45: Separating instrumentation pipelines sooner - in instrumentation_props.

Author: marek-trtik

security-scanner/presentation.py

@@ -149,28 +149,38 @@ def build_HTML_interface_to_slicer_instrumentation_props(props,fname):
         ofile.write("<h2>Definition of data types for instrumentation</h2>\n")
 
         ofile.write("<p>\n")
-        ofile.write("This is a list of program types which will be instrumented by shadow variables\n")
-        ofile.write("identified here by the corresponding names of tokens.\n")
+        if props["data_flow_insensitive_instrumentation_applied"]:
+            ofile.write("Control-flow (but not data-flow) sensitive propagation of taint information was used.\n")
+        else:
+            ofile.write("Control- and data-flow sensitive propagation of taint information was used.\n")
         ofile.write("</p>\n")
 
-        ofile.write("<table>\n")
-        ofile.write("  <tr>\n")
-        ofile.write("    <th>Type name</th>\n")
-        ofile.write("    <th>Shadow variables</th>\n")
-        ofile.write("    <th>Create sub-class?</th>\n")
-        ofile.write("  </tr>\n")
-        for dtype in props["datatypes"]:
+        if not props["data_flow_insensitive_instrumentation_applied"]:
+            ofile.write("<h2>Definition of data types for instrumentation</h2>\n")
+
+            ofile.write("<p>\n")
+            ofile.write("This is a list of program types which will be instrumented by shadow variables\n")
+            ofile.write("identified here by the corresponding names of tokens.\n")
+            ofile.write("</p>\n")
+
+            ofile.write("<table>\n")
             ofile.write("  <tr>\n")
-            ofile.write("    <td>" + escape_text_to_HTML(dtype["type_name"]) + "</td>\n")
-            ofile.write("    <td>\n")
-            ofile.write("       <ul>\n")
-            for var in dtype["shadow_vars"]:
-                ofile.write("       <li>" + escape_text_to_HTML(var) + "</li>\n")
-            ofile.write("       </ul>\n")
-            ofile.write("    </td>\n")
-            ofile.write("    <td align=\"center\">" + str(dtype["make_subclass"]) + "</td>\n")
+            ofile.write("    <th>Type name</th>\n")
+            ofile.write("    <th>Shadow variables</th>\n")
+            ofile.write("    <th>Create sub-class?</th>\n")
             ofile.write("  </tr>\n")
-        ofile.write("</table>\n")
+            for dtype in props["datatypes"]:
+                ofile.write("  <tr>\n")
+                ofile.write("    <td>" + escape_text_to_HTML(dtype["type_name"]) + "</td>\n")
+                ofile.write("    <td>\n")
+                ofile.write("       <ul>\n")
+                for var in dtype["shadow_vars"]:
+                    ofile.write("       <li>" + escape_text_to_HTML(var) + "</li>\n")
+                ofile.write("       </ul>\n")
+                ofile.write("    </td>\n")
+                ofile.write("    <td align=\"center\">" + str(dtype["make_subclass"]) + "</td>\n")
+                ofile.write("  </tr>\n")
+            ofile.write("</table>\n")
 
         ofile.write("<h2>Definition of instrumentation statements</h2>\n")
 
@@ -468,7 +478,11 @@ def build_HTML_interface_to_results_and_statistics(
         ofile.write("    <td>" + str(cmdline.timeout) + "</td>\n")
         ofile.write("  </tr>\n")
         ofile.write("  <tr>\n")
-        ofile.write("    <td>Verbosity level fors logging</td>\n")
+        ofile.write("    <td>Use data-flow insensitive propagation of taint information?</td>\n")
+        ofile.write("    <td>" + str(cmdline.data_flow_insensitive_instrumentation) + "</td>\n")
+        ofile.write("  </tr>\n")
+        ofile.write("  <tr>\n")
+        ofile.write("    <td>Verbosity level for logging</td>\n")
         ofile.write("    <td>" + str(cmdline.verbosity) + "</td>\n")
         ofile.write("  </tr>\n")
         ofile.write("  <tr>\n")
@@ -580,6 +594,19 @@ def build_HTML_interface_to_results_and_statistics(
         ofile.write("  </tr>\n")
         ofile.write("</table>\n")
 
+        if cmdline.dump_html_program:
+            original_program_root_html_filename = os.path.join(cmdline.results_dir,"goto-program","HTML","index.html")
+            ofile.write("<p>\n")
+            if os.path.isfile(original_program_root_html_filename):
+                ofile.write("The listing of the original GOTO program can be found "
+                            "<a href=\"" + os.path.relpath(original_program_root_html_filename, cmdline.results_dir) +
+                            "\">here</a>.\n")
+            else:
+                ofile.write("ERROR: Cannot find root HTML file of the saved original GOTO program '\n")
+                ofile.write(original_program_root_html_filename)
+                ofile.write("'.\n")
+            ofile.write("</p>\n")
+
 
         #######################################################################################################
         # PHASE 2

src/taint-slicer/instrumentation_props.cpp

@@ -107,10 +107,13 @@ taint_instrumentation_propst::taint_instrumentation_propst(
   const taint_programt &program,
   const taint_function_idt &_root,
   const std::set<taint_function_idt> &in_functions,
-  const std::set<taint_function_idt> &in_suppressed)
+  const std::set<taint_function_idt> &in_suppressed,
+  const bool use_data_flow_insensitive_instrumentation)
   : root(_root)
   , functions(in_functions)
   , suppressed(in_suppressed)
+  , use_data_flow_insensitive_version(
+      use_data_flow_insensitive_instrumentation)
 {
   // First we compute "valid" nodes of the propagation chaint w.r.t. the root
   // function. We find these nodes by 2 BFSs: one from sources and one from
@@ -141,7 +144,8 @@ taint_instrumentation_propst::taint_instrumentation_propst(
     location_props.push_back(chains.get_nodes().at(nid));
   }
 
-  build_map_from_typenames_to_tokennames(chains, program);
+  if(!use_data_flow_insensitive_instrumentation)
+    build_map_from_typenames_to_tokennames(chains, program);
 }
 
 
@@ -219,7 +223,8 @@ void taint_instrumentation_propst::build_map_from_typenames_to_tokennames(
 void taint_build_instrumentation_props(
   const taint_propagation_chainst &chains,
   const taint_programt &program,
-  std::vector<taint_instrumentation_propst> &output)
+  std::vector<taint_instrumentation_propst> &output,
+  const bool use_data_flow_insensitive_instrumentation)
 {
   // First we collect all functions mentioned in the graph of chains.
   std::set<irep_idt> functions;
@@ -257,49 +262,54 @@ void taint_build_instrumentation_props(
       // First we called all callees including those which should be suppressed.
       find_direct_or_indirect_callees_of_function(
         program.get_call_graph(), root, callees);
-      // Now we compute suppressed functions and erase them from the callees
-      // computed above. We do so in 3 steps.
-      // Step 1: We collect functions which definitelly should be suppressed.
-      //         I.e. those corrensponding to applications of transition rules.
-      for(const auto &node : chains.get_nodes())
+
+      if(use_data_flow_insensitive_instrumentation)
       {
-        if(functions.count(node.get_function_id())!=0UL)
+        // Now we compute suppressed functions and erase them from the callees
+        // computed above. We do so in 3 steps.
+        // Step 1: We collect functions which definitelly should be suppressed
+        //         (those corrensponding to applications of transition rules.)
+        for(const auto &node : chains.get_nodes())
         {
-          goto_programt::instructiont const&  I=*node.get_instruction_id();
-          assert(I.type==FUNCTION_CALL);
-          assert(to_code_function_call(I.code).function().id()==ID_symbol);
-          const std::string full_function_name=as_string(to_symbol_expr(
-            to_code_function_call(I.code).function()).get_identifier());
-          if(callees.count(full_function_name)!=0UL &&
-             program.get_functions().function_map.at(full_function_name)
-                                                 .body_available())
+          if(functions.count(node.get_function_id())!=0UL)
           {
-            suppressed.insert(full_function_name);
+            goto_programt::instructiont const&  I=*node.get_instruction_id();
+            INVARIANT(I.type==FUNCTION_CALL, "");
+            INVARIANT(to_code_function_call(I.code).function().id()==ID_symbol,
+              "It must be a call via function identifier.");
+            const std::string full_function_name=as_string(to_symbol_expr(
+              to_code_function_call(I.code).function()).get_identifier());
+            if(callees.count(full_function_name)!=0UL &&
+               program.get_functions().function_map.at(full_function_name)
+                                                   .body_available())
+            {
+              suppressed.insert(full_function_name);
+            }
           }
         }
-      }
-      // Step 2: We collect a potentially suppressed function into a temporary
-      //         collection "suppressions". These function are all those
-      //         call-graph reachable from functions collected in the step 1.
-      std::unordered_set<irep_idt, dstring_hash> suppressions;
-      for(const auto &fn : suppressed)
-        find_direct_or_indirect_callees_of_function(
-          program.get_call_graph(), fn, suppressions);
-      // Step 3: We copy from "suppressions" to "suppressed" each function
-      //         reachable from the root without passing through any function
-      //         collected in the step 1.
-      while(!suppressions.empty())
-      {
-        std::unordered_set<irep_idt, dstring_hash> ignored_functions(
-          suppressed.cbegin(), suppressed.cend());
-        if(!exists_direct_or_indirect_call(
-          program.get_call_graph(), root, *suppressions.cbegin(),
-          ignored_functions))
+        // Step 2: We collect a potentially suppressed function into a temporary
+        //         collection "suppressions". These function are all those
+        //         call-graph reachable from functions collected in the step 1.
+        std::unordered_set<irep_idt, dstring_hash> suppressions;
+        for(const auto &fn : suppressed)
+          find_direct_or_indirect_callees_of_function(
+            program.get_call_graph(), fn, suppressions);
+        // Step 3: We copy from "suppressions" to "suppressed" each function
+        //         reachable from the root without passing through any function
+        //         collected in the step 1.
+        while(!suppressions.empty())
         {
-          suppressed.insert(as_string(*suppressions.cbegin()));
-          callees.erase(*suppressions.cbegin());
+          std::unordered_set<irep_idt, dstring_hash> ignored_functions(
+            suppressed.cbegin(), suppressed.cend());
+          if(!exists_direct_or_indirect_call(
+            program.get_call_graph(), root, *suppressions.cbegin(),
+            ignored_functions))
+          {
+            suppressed.insert(as_string(*suppressions.cbegin()));
+            callees.erase(*suppressions.cbegin());
+          }
+          suppressions.erase(suppressions.cbegin());
         }
-        suppressions.erase(suppressions.cbegin());
       }
     }
     // We are only interested in callees with body defined (it makes no sense
@@ -319,7 +329,8 @@ void taint_build_instrumentation_props(
       program,
       as_string(root),
       available_functions,
-      suppressed
+      suppressed,
+      use_data_flow_insensitive_instrumentation
     };
     if(!props.get_sources().empty() && !props.get_sinks().empty())
       output.push_back(props);
@@ -328,6 +339,8 @@ void taint_build_instrumentation_props(
 
 void dump_as_json(const taint_instrumentation_propst &props, json_objectt &out)
 {
+  out["data_flow_insensitive_instrumentation_applied"]=jsont::json_boolean(
+    props.data_flow_insensitive_version_applied());
   {
     json_arrayt out_types;
     for(const auto &elem : props.get_datatypes())

src/taint-slicer/instrumentation_props.h

@@ -88,7 +88,8 @@ class taint_instrumentation_propst
     const taint_programt &program,
     const taint_function_idt &_root,
     const std::set<taint_function_idt> &in_functions,
-    const std::set<taint_function_idt> &in_suppressed);
+    const std::set<taint_function_idt> &in_suppressed,
+    const bool use_data_flow_insensitive_instrumentation);
 
   const std::vector<location_propst> &get_location_props() const
   { return location_props; }
@@ -103,6 +104,8 @@ class taint_instrumentation_propst
   { return suppressed; }
   const from_typenames_to_tokennames_mapt &get_datatypes() const
   { return datatypes; }
+  bool data_flow_insensitive_version_applied() const
+  { return use_data_flow_insensitive_version; }
 
 private:
   void build_map_from_typenames_to_tokennames(
@@ -116,6 +119,7 @@ class taint_instrumentation_propst
   std::set<taint_function_idt> functions;
   std::set<taint_function_idt> suppressed;
   from_typenames_to_tokennames_mapt datatypes;
+  bool use_data_flow_insensitive_version;
 };
 
 typedef taint_instrumentation_propst::datatype_infot taint_datatype_infot;
@@ -127,7 +131,8 @@ typedef taint_instrumentation_propst::datatype_infot taint_datatype_infot;
 void taint_build_instrumentation_props(
   const taint_propagation_chainst &chains,
   const taint_programt &program,
-  std::vector<taint_instrumentation_propst> &output);
+  std::vector<taint_instrumentation_propst> &output,
+  const bool use_data_flow_insensitive_instrumentation);
 
 bool is_primitive_type(const typet &type);
 std::string unwrap_type_name(const typet &type, const namespacet &ns);

src/taint-slicer/instrumenter.cpp

@@ -76,12 +76,12 @@ static irept add_shadow_variables_to_type(
 taint_instrumentert::taint_instrumentert(
   const taint_instrumentation_propst &in_props,
   const taint_programt *const in_program,
-  taint_statisticst *const in_statistics,
-  const bool use_data_flow_insensitive_instrumentation)
+  taint_statisticst *const in_statistics)
   : props(in_props)
   , program(in_program)
   , statistics(in_statistics)
-  , use_data_flow_insensitive_version(use_data_flow_insensitive_instrumentation)
+  , use_data_flow_insensitive_version(
+      in_props.data_flow_insensitive_version_applied())
 {
 }
 

src/taint-slicer/instrumenter.h

@@ -46,8 +46,7 @@ class taint_instrumentert
   taint_instrumentert(
     const taint_instrumentation_propst &props,
     const taint_programt * const in_program,
-    taint_statisticst * const in_statistics,
-    const bool use_data_flow_insensitive_instrumentation);
+    taint_statisticst * const in_statistics);
 
   void run();
 
@@ -60,6 +59,9 @@ class taint_instrumentert
   const std::map<taint_tokent::namet, automaton_variable_idt> &
   get_from_tokens_to_vars() const { return from_tokens_to_vars; }
 
+  bool data_flow_insensitive_version_applied() const
+  { return use_data_flow_insensitive_version; }
+
 private:
 
   void instrument_data_types(const taint_instrumentation_propst &props);

src/taint-slicer/slicer.cpp

@@ -80,7 +80,8 @@ void taint_slicert::compute_slice(
   taint_build_instrumentation_props(
     propagation_chains,
     *program,
-    instrumentation_props);
+    instrumentation_props,
+    use_data_flow_insensitive_instrumentation);
 
   // Dump all computed instrumentation props into JSON files.
   for(std::size_t i=0U; i!=instrumentation_props.size(); ++i)
@@ -98,10 +99,7 @@ void taint_slicert::compute_slice(
   for(std::size_t i=0UL, n=instrumentation_props.size(); i!=n; ++i)
   {
     taint_instrumentert instrumenter(
-      instrumentation_props.at(i),
-      program,
-      statistics,
-      use_data_flow_insensitive_instrumentation);
+      instrumentation_props.at(i), program, statistics);
 
     instrumenter.run();