Merge pull request #917 from cmu-delphi/ds/remove-wip-tests

Remove remaining wip pieces in tests

Author: melange396

integrations/acquisition/covidcast/test_csv_uploading.py

@@ -101,8 +101,6 @@ def test_uploading(self):
           log_file=log_file_directory +
           "output.log",
           data_dir=data_dir,
-          is_wip_override=False,
-          not_wip_override=False,
           specific_issue_date=False)
     uploader_column_rename = {"geo_id": "geo_value", "val": "value", "se": "stderr", "missing_val": "missing_value", "missing_se": "missing_stderr"}
 
@@ -227,41 +225,6 @@ def test_uploading(self):
       self.setUp()
 
 
-    with self.subTest("Valid wip"):
-      values = pd.DataFrame({
-        "geo_id": ["me", "nd", "wa"],
-        "val": [10.0, 20.0, 30.0],
-        "se": [0.01, 0.02, 0.03],
-        "sample_size": [100.0, 200.0, 300.0],
-        "missing_val": [Nans.NOT_MISSING] * 3,
-        "missing_se": [Nans.NOT_MISSING] * 3,
-        "missing_sample_size": [Nans.NOT_MISSING] * 3
-      })
-      signal_name = "wip_prototype"
-      values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
-
-      # upload CSVs
-      main(args)
-      response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
-
-      expected_values = pd.concat([values, pd.DataFrame({
-        "time_value": [20200419] * 3,
-        "signal": [signal_name] * 3,
-        "direction": [None] * 3
-      })], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
-      expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}
-
-      self.assertEqual(response, expected_response)
-      self.verify_timestamps_and_defaults()
-
-      # Verify that files were archived
-      path = data_dir + f'/archive/successful/src-name/20200419_state_wip_prototype.csv.gz'
-      self.assertIsNotNone(os.stat(path))
-
-      self.tearDown()
-      self.setUp()
-
-
     with self.subTest("Valid signal with name length 32<x<64"):
       values = pd.DataFrame({
         "geo_id": ["pa"],
@@ -272,7 +235,7 @@ def test_uploading(self):
         "missing_se": [Nans.NOT_MISSING],
         "missing_sample_size": [Nans.NOT_MISSING]
       })
-      signal_name = "wip_really_long_name_that_will_be_accepted"
+      signal_name = "really_long_name_that_will_be_accepted"
       values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
 
       # upload CSVs
@@ -303,7 +266,7 @@ def test_uploading(self):
         "missing_se": [Nans.NOT_MISSING],
         "missing_sample_size": [Nans.NOT_MISSING]
       })
-      signal_name = "wip_really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
+      signal_name = "really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
       values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
 
       # upload CSVs

integrations/server/test_covidcast.py

@@ -88,11 +88,11 @@ def test_round_trip(self):
   #      `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`, 
   #      `time_value`, `geo_value`, `value_updated_timestamp`, 
   #      `value`, `stderr`, `sample_size`, `direction_updated_timestamp`, 
-  #      `direction`, `issue`, `lag`, `is_latest_issue`, `is_wip`,`missing_value`,
+  #      `direction`, `issue`, `lag`, `is_latest_issue`, `missing_value`,
   #      `missing_stderr`,`missing_sample_size`) 
   #    VALUES
   #       (0, 'src', 'sig', 'day', 'county', 20200414, '01234',
-  #         123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1, False,
+  #         123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1,
   #         {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING})
   #   ''')
   #   self.cnx.commit()

tests/acquisition/covidcast/test_csv_to_database.py

@@ -2,6 +2,7 @@
 
 # standard library
 import argparse
+from typing import Iterable
 import unittest
 from unittest.mock import MagicMock
 
@@ -27,9 +28,7 @@ def _path_details(self):
       # a file with a data error
       ('path/b.csv', ('src_b', 'sig_b', 'week', 'msa', 202016, 202017, 1)),
       # emulate a file that's named incorrectly
-      ('path/c.csv', None),
-      # another good file w/ wip
-      ('path/d.csv', ('src_d', 'wip_sig_d', 'week', 'msa', 202016, 202017, 1)),
+      ('path/c.csv', None)
     ]
 
   def test_collect_files(self):
@@ -65,15 +64,16 @@ def load_csv_impl(path, *args):
         yield make_row('b1')
         yield None
         yield make_row('b3')
-      elif path == 'path/d.csv':
-        yield make_row('d1')
       else:
         # fail the test for any other path
         raise Exception('unexpected path')
 
+    def iter_len(l: Iterable) -> int:
+      return len(list(l))
+
     data_dir = 'data_dir'
     mock_database = MagicMock()
-    mock_database.insert_or_update_bulk.return_value = 2
+    mock_database.insert_or_update_bulk = MagicMock(wraps=iter_len)
     mock_csv_importer = MagicMock()
     mock_csv_importer.load_csv = load_csv_impl
     mock_file_archiver = MagicMock()
@@ -87,30 +87,28 @@ def load_csv_impl(path, *args):
       mock_logger,
       csv_importer_impl=mock_csv_importer)
 
-    self.assertEqual(modified_row_count, 4)
+    self.assertEqual(modified_row_count, 3)
     # verify that appropriate rows were added to the database
-    self.assertEqual(mock_database.insert_or_update_bulk.call_count, 2)
+    self.assertEqual(mock_database.insert_or_update_bulk.call_count, 1)
     call_args_list = mock_database.insert_or_update_bulk.call_args_list
     actual_args = [[(a.source, a.signal, a.time_type, a.geo_type, a.time_value,
                      a.geo_value, a.value, a.stderr, a.sample_size, a.issue, a.lag)
                     for a in call.args[0]] for call in call_args_list]
     expected_args = [
       [('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a1', 'a1', 'a1', 'a1', 20200420, 1),
        ('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a2', 'a2', 'a2', 'a2', 20200420, 1),
-       ('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)],
-      [('src_d', 'wip_sig_d', 'week', 'msa', 202016, 'd1', 'd1', 'd1', 'd1', 202017, 1)]
+       ('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)]
     ]
     self.assertEqual(actual_args, expected_args)
 
     # verify that two files were successful (a, d) and two failed (b, c)
-    self.assertEqual(mock_file_archiver.archive_file.call_count, 4)
+    self.assertEqual(mock_file_archiver.archive_file.call_count, 3)
     call_args_list = mock_file_archiver.archive_file.call_args_list
     actual_args = [args for (args, kwargs) in call_args_list]
     expected_args = [
       ('path', 'data_dir/archive/successful/src_a', 'a.csv', True),
       ('path', 'data_dir/archive/failed/src_b', 'b.csv', False),
       ('path', 'data_dir/archive/failed/unknown', 'c.csv', False),
-      ('path', 'data_dir/archive/successful/src_d', 'd.csv', True),
     ]
     self.assertEqual(actual_args, expected_args)