Merge pull request diffblue#362 from diffblue/refined_statistics_of_the_stage_1

Refined statistics of the Phase 1 of the tool.

Author: marek-trtik

driver/analyser.py

@@ -92,14 +92,14 @@ def run_security_analyser(
         get_security_analyser_pathname() + " "
         "--security-scanner '" + root_config_json_fname + "' "
         )
-    prof["calling_goto_analyser"] = {}
-    prof_calling_goto_analyser_start_time = time.time()
+    prof["calling_security_analyser"] = {}
+    prof_calling_security_analyser_start_time = time.time()
     print("Invoking 'security-analyser' ...")
     if verbosity >= 9:
         print("CWD: " + results_dir)
         print("CMD: " + command)
     os.system(command)
-    prof["calling_goto_analyser"]["duration"] = time.time() - prof_calling_goto_analyser_start_time
+    prof["calling_security_analyser"]["duration"] = time.time() - prof_calling_security_analyser_start_time
     os.chdir(old_cwd)
     prof["duration"] = time.time() - prof_start_time
 

driver/run.py

@@ -185,7 +185,7 @@ def evaluate(cmdline, common_libraries):
         cmdline.verbosity
         )
 
-    prof["run_analyser"] = analyser.run_security_analyser(
+    prof["summary_computation"] = analyser.run_security_analyser(
             classes_jar_pathname,
             cmdline.config,
             cmdline.timeout,
@@ -231,7 +231,7 @@ def evaluate(cmdline, common_libraries):
     prof_plots = {}
     prof_plots_start_time = time.time()
 
-    stats_json_fname = os.path.abspath(os.path.join(cmdline.results_dir,"statistics/JSON/statistics.json"))
+    stats_json_fname = os.path.abspath(os.path.join(cmdline.results_dir,"statistics/JSON/statistics_security_analyser.json"))
     stats_plots_dir = os.path.abspath(os.path.join(cmdline.results_dir,"statistics/plots"))
 
     prof_plots["source"] = stats_json_fname
@@ -242,11 +242,13 @@ def evaluate(cmdline, common_libraries):
     prof_plots["duration"] = time.time() - prof_plots_start_time
     prof["plots_build"] = prof_plots
 
-    overall_perf_fname = os.path.abspath(os.path.join(cmdline.results_dir,"overall_performance.json"))
-    print("Saving performance data in JSON format to: " + overall_perf_fname)
-
     prof["duration"] = time.time() - prof_start_time
 
+    overall_perf_fname = os.path.abspath(os.path.join(cmdline.results_dir, "statistics", "JSON", "overall_performance.json"))
+    if not os.path.exists(os.path.dirname(overall_perf_fname)):
+        os.makedirs(os.path.dirname(overall_perf_fname))
+    print("Saving performance data in JSON format to: " + overall_perf_fname)
+
     with open(overall_perf_fname,"w") as overall_perf_file:
         overall_perf_file.write(json.dumps(prof,sort_keys=True,indent=4))
 

src/taint-analysis/taint_security_scanner.cpp

@@ -258,15 +258,19 @@ bool taint_do_security_scan(
       << (summaries_root / "JSON" / "__index.json").native()
       << "')." << messaget::eom;
 
+    statistics.begin_dump_of_taint_json_summaries();
     taint_summaries.flush();
+    statistics.end_dump_of_taint_json_summaries();
 
     logger.status()
       << "Saving summaries of LVSA analysis in JSON format (see '"
       << (boost::filesystem::path(config.get_lvsa_summaries_root_directory()) /
           "__index.json").native()
       << "')." << messaget::eom;
 
+    statistics.begin_dump_of_lvsa_json_summaries();
     lvsa_summaries.flush();
+    statistics.end_dump_of_lvsa_json_summaries();
 
     const boost::filesystem::path stats_directory(
       config.get_statistics_root_directory());
@@ -284,17 +288,14 @@ bool taint_do_security_scan(
     }
 
     const auto json_directory = stats_directory / "JSON";
-    const auto json_stats_file = json_directory / "statistics.json";
+    const auto json_stats_file =
+      json_directory / "statistics_security_analyser.json";
 
     logger.status()
       << "Saving statistics of the whole analysis in JSON format (see '"
       << json_stats_file.native()
       << "')." << messaget::eom;
-    {
-      boost::filesystem::create_directory(json_directory);
-      std::ofstream ostr(json_stats_file.native());
-      taint_dump_statistics_in_JSON(statistics, ostr);
-    }
+    taint_dump_statistics_in_JSON(statistics, json_stats_file);
 
     if(config.is_html_dump_of_program_enabled())
     {

src/taint-analysis/taint_statistics.cpp

@@ -146,45 +146,40 @@ static bool  is_instruction_virtual_dispatch(
 ///
 ///////////////////////////////////////////////////////////////////
 
-
 taint_function_statisticst::taint_function_statisticst(
-    std::size_t const  num_locations_)
-  : num_locations(num_locations_)
-  , num_declarations(0UL)
-  , num_temporaries(0UL)
-  , num_assignments_to_temporaries(0UL)
-  , num_dead_statements(0UL)
-  , num_NONDET_calls(0UL)
-  , num_SKIPs(0UL)
-  , num_GOTOs(0UL)
-  , num_string_builder_lines(0UL)
-  , num_virtual_dispatches(0UL)
-  , num_auxiliary_locations(0UL)
-
-  , sources()
-  , sinks()
-  , sanitisers()
-
-  , time_point_begin_lvsa_analysis()
-  , time_point_end_lvsa_analysis()
-  , num_fixpoint_steps_of_lvsa_analysis(0UL)
-
-  , num_lvsa_uses_of_callee_summaries(0UL)
-  , num_lvsa_of_processed_rules_callee_summaries(0UL)
-
-  , time_point_begin_taint_summaries()
-  , time_point_end_taint_summaries()
-  , num_fixpoint_steps_of_taint_summaries(0UL)
-
-  , summary_input_size(0UL)
-  , summary_output_size(0UL)
-  , summary_domain_size(0UL)
-  , summary_domain_num_abstract_values(0UL)
-
-  , num_usages_of_my_summary(0UL)
-
-  , num_usages_of_callee_summaries(0UL)
-  , num_rules_in_used_callee_summaries(0UL)
+  const std::size_t num_locations_)
+  : num_locations(num_locations_),
+    num_declarations(0UL),
+    num_temporaries(0UL),
+    num_assignments_to_temporaries(0UL),
+    num_dead_statements(0UL),
+    num_NONDET_calls(0UL),
+    num_SKIPs(0UL),
+    num_GOTOs(0UL),
+    num_string_builder_lines(0UL),
+    num_virtual_dispatches(0UL),
+    num_auxiliary_locations(0UL),
+    sources(),
+    sinks(),
+    sanitisers(),
+    num_applications_of_rule_propagation(0UL),
+    num_applications_of_rule_sanitize(0UL),
+    num_applications_of_rule_sink(0UL),
+    time_point_begin_lvsa_analysis(),
+    time_point_end_lvsa_analysis(),
+    num_fixpoint_steps_of_lvsa_analysis(0UL),
+    num_lvsa_uses_of_callee_summaries(0UL),
+    num_lvsa_of_processed_rules_callee_summaries(0UL),
+    time_point_begin_taint_summaries(),
+    time_point_end_taint_summaries(),
+    num_fixpoint_steps_of_taint_summaries(0UL),
+    summary_input_size(0UL),
+    summary_output_size(0UL),
+    summary_domain_size(0UL),
+    summary_domain_num_abstract_values(0UL),
+    num_usages_of_my_summary(0UL),
+    num_usages_of_callee_summaries(0UL),
+    num_rules_in_used_callee_summaries(0UL)
 {}
 
 ///////////////////////////////////////////////////////////////////
@@ -264,6 +259,24 @@ void  taint_function_statisticst::on_get_may(unsigned int const  location)
   sinks.insert(location);
 }
 
+void taint_function_statisticst::on_application_of_propagation_rule(
+  const std::size_t rule_id)
+{
+  ++num_applications_of_rule_propagation;
+}
+
+void taint_function_statisticst::on_application_of_sanitization_rule(
+  const std::size_t rule_id)
+{
+  ++num_applications_of_rule_sanitize;
+}
+
+void taint_function_statisticst::on_application_of_sink_rule(
+  const std::size_t rule_id)
+{
+  ++num_applications_of_rule_sink;
+}
+
 ///////////////////////////////////////////////////////////////////
 /// Queries
 ///////////////////////////////////////////////////////////////////
@@ -534,6 +547,38 @@ void taint_statisticst::on_taint_analysis_use_callee_summary(
   statistics_of_functions.at(callee_name).on_taint_analysis_use_my_summary();
 }
 
+void taint_statisticst::on_taint_analysis_apply_propagation_rule(
+  const std::size_t rule_id,
+  const unsigned int location_number)
+{
+  PRECONDITION(!current_function_name.empty());
+  const auto it = statistics_of_functions.find(current_function_name);
+  PRECONDITION(it != statistics_of_functions.cend());
+  it->second.on_set_may(location_number);
+  it->second.on_application_of_propagation_rule(rule_id);
+}
+
+void taint_statisticst::on_taint_analysis_apply_sanitize_rule(
+  const std::size_t rule_id,
+  const unsigned int location_number)
+{
+  PRECONDITION(!current_function_name.empty());
+  const auto it = statistics_of_functions.find(current_function_name);
+  PRECONDITION(it != statistics_of_functions.cend());
+  it->second.on_clear_may(location_number);
+  it->second.on_application_of_sanitization_rule(rule_id);
+}
+
+void taint_statisticst::on_taint_analysis_apply_sink_rule(
+  const std::size_t rule_id,
+  const unsigned int location_number)
+{
+  PRECONDITION(!current_function_name.empty());
+  const auto it = statistics_of_functions.find(current_function_name);
+  PRECONDITION(it != statistics_of_functions.cend());
+  it->second.on_get_may(location_number);
+  it->second.on_application_of_sink_rule(rule_id);
+}
 
 void taint_statisticst::begin_error_traces_recognition()
 {
@@ -565,6 +610,16 @@ void taint_statisticst::end_dump_of_taint_json_summaries()
   time_point_end_dump_taint_json_summaries = get_current_time();
 }
 
+void taint_statisticst::begin_dump_of_lvsa_json_summaries()
+{
+  time_point_begin_dump_lvsa_json_summaries = get_current_time();
+}
+
+void taint_statisticst::end_dump_of_lvsa_json_summaries()
+{
+  time_point_end_dump_lvsa_json_summaries = get_current_time();
+}
+
 void taint_statisticst::begin_dump_of_taint_html_traces()
 {
   time_point_begin_dump_taint_html_traces = get_current_time();
@@ -640,6 +695,14 @@ taint_statisticst::get_duration_of_dump_of_taint_json_summaries() const
                       time_point_end_dump_taint_json_summaries);
 }
 
+taint_statisticst::durationt
+taint_statisticst::get_duration_of_dump_of_lvsa_json_summaries() const
+{
+  return get_duration(
+    time_point_begin_dump_lvsa_json_summaries,
+    time_point_end_dump_lvsa_json_summaries);
+}
+
 taint_statisticst::durationt
 taint_statisticst::get_duration_of_dump_of_taint_html_traces() const
 {

src/taint-analysis/taint_statistics.h

@@ -68,6 +68,10 @@ class taint_function_statisticst
   void on_clear_may(const unsigned int location);
   void on_get_may(const unsigned int location);
 
+  void on_application_of_propagation_rule(const std::size_t rule_id);
+  void on_application_of_sanitization_rule(const std::size_t rule_id);
+  void on_application_of_sink_rule(const std::size_t rule_id);
+
   ///////////////////////////////////////////////////////////////////
   /// Queries
   ///////////////////////////////////////////////////////////////////
@@ -102,6 +106,19 @@ class taint_function_statisticst
   const std::set<unsigned int>
     &get_locations_of_taint_sanitisers() const noexcept { return sanitisers; }
 
+  std::size_t get_num_applications_of_rule_propagation() const
+  {
+    return num_applications_of_rule_propagation;
+  }
+  std::size_t get_num_applications_of_rule_sanitizer() const
+  {
+    return num_applications_of_rule_sanitize;
+  }
+  std::size_t get_num_applications_of_rule_sink() const
+  {
+    return num_applications_of_rule_sink;
+  }
+
   durationt get_duration_of_lvsa_analysis() const;
   durationt get_duration_of_taint_summaries() const;
 
@@ -151,6 +168,10 @@ class taint_function_statisticst
   std::set<unsigned int> sinks;
   std::set<unsigned int> sanitisers;
 
+  std::size_t num_applications_of_rule_propagation;
+  std::size_t num_applications_of_rule_sanitize;
+  std::size_t num_applications_of_rule_sink;
+
   time_pointt time_point_begin_lvsa_analysis;
   time_pointt time_point_end_lvsa_analysis;
   std::size_t num_fixpoint_steps_of_lvsa_analysis;
@@ -227,6 +248,16 @@ class taint_statisticst
     const std::shared_ptr<taint_summaryt> summary,
     const std::string &callee_name);
 
+  void on_taint_analysis_apply_propagation_rule(
+    const std::size_t rule_id,
+    const unsigned int location_number);
+  void on_taint_analysis_apply_sanitize_rule(
+    const std::size_t rule_id,
+    const unsigned int location_number);
+  void on_taint_analysis_apply_sink_rule(
+    const std::size_t rule_id,
+    const unsigned int location_number);
+
   void begin_error_traces_recognition();
   void end_error_traces_recognition();
 
@@ -236,6 +267,9 @@ class taint_statisticst
   void begin_dump_of_taint_json_summaries();
   void end_dump_of_taint_json_summaries();
 
+  void begin_dump_of_lvsa_json_summaries();
+  void end_dump_of_lvsa_json_summaries();
+
   void begin_dump_of_taint_html_traces();
   void end_dump_of_taint_html_traces();
 
@@ -253,6 +287,7 @@ class taint_statisticst
   durationt get_duration_of_error_traces_recognition() const;
   durationt get_duration_of_dump_of_taint_html_summaries() const;
   durationt get_duration_of_dump_of_taint_json_summaries() const;
+  durationt get_duration_of_dump_of_lvsa_json_summaries() const;
   durationt get_duration_of_dump_of_taint_html_traces() const;
   durationt get_duration_of_dump_of_taint_json_traces() const;
 
@@ -295,6 +330,9 @@ class taint_statisticst
   time_pointt time_point_begin_dump_taint_json_summaries;
   time_pointt time_point_end_dump_taint_json_summaries;
 
+  time_pointt time_point_begin_dump_lvsa_json_summaries;
+  time_pointt time_point_end_dump_lvsa_json_summaries;
+
   time_pointt time_point_begin_dump_taint_html_traces;
   time_pointt time_point_end_dump_taint_html_traces;
 

src/taint-analysis/taint_statistics_dump.h

@@ -21,4 +21,8 @@ void  taint_dump_statistics_in_JSON(
   const taint_statisticst &S,
   std::ostream &ostr);
 
+void taint_dump_statistics_in_JSON(
+  const taint_statisticst &S,
+  const boost::filesystem::path &out_file_pathname);
+
 #endif