added ANALYZE TABLE to the end of acquisition runs (and some whitespace fixes)

Author: melange396

src/acquisition/covidcast/csv_to_database.py

@@ -77,12 +77,12 @@ def upload_archive(
     csv_importer_impl=CsvImporter):
   """Upload CSVs to the database and archive them using the specified handlers.
 
-  :path_details: output from CsvImporter.find*_csv_files 
-  
+  :path_details: output from CsvImporter.find*_csv_files
+
   :database: an open connection to the epidata database
 
   :handlers: functions for archiving (successful, failed) files
-  
+
   :return: the number of modified rows
   """
   archive_as_successful, archive_as_failed = handlers
@@ -130,7 +130,7 @@ def upload_archive(
       archive_as_successful(path_src, filename, source, logger)
     else:
       archive_as_failed(path_src, filename, source,logger)
-  
+
   return total_modified_row_count
 
 
@@ -149,7 +149,7 @@ def main(
   if not path_details:
     logger.info('nothing to do; exiting...')
     return
-  
+
   logger.info("Ingesting CSVs", csv_count = len(path_details))
 
   database = database_impl()
@@ -161,13 +161,12 @@ def main(
       database,
       make_handlers(args.data_dir, args.specific_issue_date),
       logger)
-    logger.info("Finished inserting database rows", row_count = modified_row_count)
-    # the following print statement serves the same function as the logger.info call above
-    # print('inserted/updated %d rows' % modified_row_count)
+    logger.info("Finished inserting/updating database rows", row_count = modified_row_count)
   finally:
+    database.do_analyze()
     # unconditionally commit database changes since CSVs have been archived
     database.disconnect(True)
-  
+
   logger.info(
       "Ingested CSVs into database",
       total_runtime_in_seconds=round(time.time() - start_time, 2))

src/acquisition/covidcast/database.py

@@ -138,6 +138,17 @@ def _reset_load_table_ai_counter(self):
          '1', '1', '1', '1', '1', 1, 1, 1, 1);""")
     self._cursor.execute(f'DELETE FROM epimetric_load')
 
+  def do_analyze(self):
+    """performs and stores key distribution analyses, used for join order and index selection"""
+    # TODO: consider expanding this to update columns' histograms
+    #       https://dev.mysql.com/doc/refman/8.0/en/analyze-table.html#analyze-table-histogram-statistics-analysis
+    self._cursor.execute(
+      f'''ANALYZE TABLE
+          signal_dim, geo_dim,
+          {self.load_table}, {self.history_table}, {self.latest_table}''')
+    output = [self._cursor.column_names] + self._cursor.fetchall()
+    get_structured_logger('do_analyze').info("ANALYZE results: "+str(output))
+
   def insert_or_update_bulk(self, cc_rows):
     return self.insert_or_update_batch(cc_rows)