catching up w/ versioning by adding issue and lag fields

Author: melange396

src/acquisition/covidcast/csv_to_database.py

@@ -78,7 +78,7 @@ def archive_as_successful(path_src, filename, source):
 
     all_rows_valid = False
     try:
-      cc_rows = CovidcastRow.fromCsvRows(csv_rows, source, signal, time_type, geo_type, time_value)
+      cc_rows = CovidcastRow.fromCsvRows(csv_rows, source, signal, time_type, geo_type, time_value, issue, lag)
       rows_list = list(cc_rows)
       if not rows_list:
         raise ValueError("No data")

src/acquisition/covidcast/database.py

@@ -17,19 +17,21 @@ class CovidcastRow():
   """A container for all the values of a single covidcast row."""
 
   @staticmethod
-  def fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value):
+  def fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag):
     return CovidcastRow(source, signal, time_type, geo_type, time_value,
-      row_value.geo_value,
-      row_value.value,
-      row_value.stderr,
-      row_value.sample_size)
+                        row_value.geo_value,
+                        row_value.value,
+                        row_value.stderr,
+                        row_value.sample_size,
+                        issue, lag)
 
   @staticmethod
-  def fromCsvRows(row_values, source, signal, time_type, geo_type, time_value):
+  def fromCsvRows(row_values, source, signal, time_type, geo_type, time_value, issue, lag):
     # NOTE: returns a generator, as row_values is expected to be a generator
-    return (CovidcastRow.fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value) for row_value in row_values)
+    return (CovidcastRow.fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag)
+            for row_value in row_values)
 
-  def __init__(self, source, signal, time_type, geo_type, time_value, geo_value, value, stderr, sample_size):
+  def __init__(self, source, signal, time_type, geo_type, time_value, geo_value, value, stderr, sample_size, issue, lag):
     self.id = None
     self.source = source
     self.signal = signal
@@ -42,6 +44,8 @@ def __init__(self, source, signal, time_type, geo_type, time_value, geo_value, v
     self.sample_size = sample_size  # from CSV row
     self.timestamp2 = 0
     self.direction = None
+    self.issue = issue
+    self.lag = lag
 
 
 class Database:
@@ -98,11 +102,13 @@ def insert_or_update_batch(self, cc_rows, batch_size=0, commit_partial=False):
       INSERT INTO `covidcast`
         (`id`, `source`, `signal`, `time_type`, `geo_type`, `time_value`, `geo_value`,
         `timestamp1`, `value`, `stderr`, `sample_size`,
-        `timestamp2`, `direction`)
+        `timestamp2`, `direction`,
+        `issue`, `lag`)
       VALUES
         (0, %s, %s, %s, %s, %s, %s,
         UNIX_TIMESTAMP(NOW()), %s, %s, %s,
-        0, NULL)
+        0, NULL,
+        %s, %s)
       ON DUPLICATE KEY UPDATE
         `timestamp1` = VALUES(`timestamp1`),
         `value` = VALUES(`value`),
@@ -132,6 +138,8 @@ def insert_or_update_batch(self, cc_rows, batch_size=0, commit_partial=False):
         row.value,
         row.stderr,
         row.sample_size,
+        row.issue,
+        row.lag
       ) for row in cc_rows[start:end]]
 
       result = self._cursor.executemany(sql, args)

tests/acquisition/covidcast/test_csv_to_database.py

@@ -71,7 +71,8 @@ def load_csv_impl(path, *args):
     self.assertEqual(mock_database.insert_or_update_bulk.call_count, 2)
     call_args_list = mock_database.insert_or_update_bulk.call_args_list
     actual_args = [[(a.source, a.signal, a.time_type, a.geo_type, a.time_value,
-                     a.geo_value, a.value, a.stderr, a.sample_size) for a in call.args[0]] for call in call_args_list]
+                     a.geo_value, a.value, a.stderr, a.sample_size, a.issue, a.lag)
+                    for a in call.args[0]] for call in call_args_list]
     expected_args = [
       [('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a1', 'a1', 'a1', 'a1', 20200420, 1),
        ('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a2', 'a2', 'a2', 'a2', 20200420, 1),