From 3d3b8a7013f4b8dc8ee7b7d5c6c03df9a4d6c2a0 Mon Sep 17 00:00:00 2001
From: Dmitry Shemetov <dshemetov@ucdavis.edu>
Date: Thu, 26 May 2022 13:10:31 -0700
Subject: [PATCH 1/3] Remove remaining wip pieces in tests

---
 .../covidcast/test_csv_uploading.py           | 72 +------------------
 integrations/server/test_covidcast.py         |  4 +-
 .../covidcast/test_csv_to_database.py         | 18 ++---
 3 files changed, 9 insertions(+), 85 deletions(-)

diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py
index 29f74f46d..643127c34 100644
--- a/integrations/acquisition/covidcast/test_csv_uploading.py
+++ b/integrations/acquisition/covidcast/test_csv_uploading.py
@@ -102,8 +102,6 @@ def test_uploading(self):
           log_file=log_file_directory +
           "output.log",
           data_dir=data_dir,
-          is_wip_override=False,
-          not_wip_override=False,
           specific_issue_date=False)
     uploader_column_rename = {"geo_id": "geo_value", "val": "value", "se": "stderr", "missing_val": "missing_value", "missing_se": "missing_stderr"}
 
@@ -232,74 +230,6 @@ def test_uploading(self):
       self.setUp()
 
 
-    with self.subTest("Valid wip"):
-      values = pd.DataFrame({
-        "geo_id": ["me", "nd", "wa"],
-        "val": [10.0, 20.0, 30.0],
-        "se": [0.01, 0.02, 0.03],
-        "sample_size": [100.0, 200.0, 300.0],
-        "missing_val": [Nans.NOT_MISSING] * 3,
-        "missing_se": [Nans.NOT_MISSING] * 3,
-        "missing_sample_size": [Nans.NOT_MISSING] * 3
-      })
-      signal_name = "wip_prototype"
-      values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
-
-      # upload CSVs
-      main(args)
-      dbjobs_main()
-      response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
-
-      expected_values = pd.concat([values, pd.DataFrame({
-        "time_value": [20200419] * 3,
-        "signal": [signal_name] * 3,
-        "direction": [None] * 3
-      })], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
-      expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}
-
-      self.assertEqual(response, expected_response)
-      self.verify_timestamps_and_defaults()
-
-      # Verify that files were archived
-      path = data_dir + f'/archive/successful/src-name/20200419_state_wip_prototype.csv.gz'
-      self.assertIsNotNone(os.stat(path))
-
-      self.tearDown()
-      self.setUp()
-
-
-    with self.subTest("Valid signal with name length 32<x<64"):
-      values = pd.DataFrame({
-        "geo_id": ["pa"],
-        "val": [100.0],
-        "se": [5.4],
-        "sample_size": [624.0],
-        "missing_val": [Nans.NOT_MISSING],
-        "missing_se": [Nans.NOT_MISSING],
-        "missing_sample_size": [Nans.NOT_MISSING]
-      })
-      signal_name = "wip_really_long_name_that_will_be_accepted"
-      values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
-
-      # upload CSVs
-      main(args)
-      dbjobs_main()
-      response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
-
-      expected_values = pd.concat([values, pd.DataFrame({
-        "time_value": [20200419],
-        "signal": [signal_name],
-        "direction": [None]
-      })], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
-      expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}
-
-      self.assertEqual(response, expected_response)
-      self.verify_timestamps_and_defaults()
-
-      self.tearDown()
-      self.setUp()
-
-
     with self.subTest("Invalid signal with a too-long name"):
       values = pd.DataFrame({
         "geo_id": ["pa"],
@@ -310,7 +240,7 @@ def test_uploading(self):
         "missing_se": [Nans.NOT_MISSING],
         "missing_sample_size": [Nans.NOT_MISSING]
       })
-      signal_name = "wip_really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
+      signal_name = "really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
       values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
 
       # upload CSVs
diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py
index aa99578d5..3d4e11b76 100644
--- a/integrations/server/test_covidcast.py
+++ b/integrations/server/test_covidcast.py
@@ -154,11 +154,11 @@ def test_round_trip(self):
   #      `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`, 
   #      `time_value`, `geo_value`, `value_updated_timestamp`, 
   #      `value`, `stderr`, `sample_size`, `direction_updated_timestamp`, 
-  #      `direction`, `issue`, `lag`, `is_latest_issue`, `is_wip`,`missing_value`,
+  #      `direction`, `issue`, `lag`, `is_latest_issue`, `missing_value`,
   #      `missing_stderr`,`missing_sample_size`) 
   #    VALUES
   #       (0, 'src', 'sig', 'day', 'county', 20200414, '01234',
-  #         123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1, False,
+  #         123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1,
   #         {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING})
   #   ''')
   #   self.cnx.commit()
diff --git a/tests/acquisition/covidcast/test_csv_to_database.py b/tests/acquisition/covidcast/test_csv_to_database.py
index 2444c5262..e31d35d07 100644
--- a/tests/acquisition/covidcast/test_csv_to_database.py
+++ b/tests/acquisition/covidcast/test_csv_to_database.py
@@ -27,9 +27,7 @@ def _path_details(self):
       # a file with a data error
       ('path/b.csv', ('src_b', 'sig_b', 'week', 'msa', 202016, 202017, 1)),
       # emulate a file that's named incorrectly
-      ('path/c.csv', None),
-      # another good file w/ wip
-      ('path/d.csv', ('src_d', 'wip_sig_d', 'week', 'msa', 202016, 202017, 1)),
+      ('path/c.csv', None)
     ]
 
   def test_collect_files(self):
@@ -65,15 +63,13 @@ def load_csv_impl(path, *args):
         yield make_row('b1')
         yield None
         yield make_row('b3')
-      elif path == 'path/d.csv':
-        yield make_row('d1')
       else:
         # fail the test for any other path
         raise Exception('unexpected path')
 
     data_dir = 'data_dir'
     mock_database = MagicMock()
-    mock_database.insert_or_update_bulk.return_value = 2
+    mock_database.insert_or_update_bulk.return_value = 1
     mock_csv_importer = MagicMock()
     mock_csv_importer.load_csv = load_csv_impl
     mock_file_archiver = MagicMock()
@@ -87,9 +83,9 @@ def load_csv_impl(path, *args):
       mock_logger,
       csv_importer_impl=mock_csv_importer)
 
-    self.assertEqual(modified_row_count, 4)
+    self.assertEqual(modified_row_count, 1)
     # verify that appropriate rows were added to the database
-    self.assertEqual(mock_database.insert_or_update_bulk.call_count, 2)
+    self.assertEqual(mock_database.insert_or_update_bulk.call_count, 1)
     call_args_list = mock_database.insert_or_update_bulk.call_args_list
     actual_args = [[(a.source, a.signal, a.time_type, a.geo_type, a.time_value,
                      a.geo_value, a.value, a.stderr, a.sample_size, a.issue, a.lag)
@@ -97,20 +93,18 @@ def load_csv_impl(path, *args):
     expected_args = [
       [('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a1', 'a1', 'a1', 'a1', 20200420, 1),
        ('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a2', 'a2', 'a2', 'a2', 20200420, 1),
-       ('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)],
-      [('src_d', 'wip_sig_d', 'week', 'msa', 202016, 'd1', 'd1', 'd1', 'd1', 202017, 1)]
+       ('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)]
     ]
     self.assertEqual(actual_args, expected_args)
 
     # verify that two files were successful (a, d) and two failed (b, c)
-    self.assertEqual(mock_file_archiver.archive_file.call_count, 4)
+    self.assertEqual(mock_file_archiver.archive_file.call_count, 3)
     call_args_list = mock_file_archiver.archive_file.call_args_list
     actual_args = [args for (args, kwargs) in call_args_list]
     expected_args = [
       ('path', 'data_dir/archive/successful/src_a', 'a.csv', True),
       ('path', 'data_dir/archive/failed/src_b', 'b.csv', False),
       ('path', 'data_dir/archive/failed/unknown', 'c.csv', False),
-      ('path', 'data_dir/archive/successful/src_d', 'd.csv', True),
     ]
     self.assertEqual(actual_args, expected_args)
 

From c71b3f47bd19534cc95327223be69aeaed7243a4 Mon Sep 17 00:00:00 2001
From: Dmitry Shemetov <dshemetov@ucdavis.edu>
Date: Thu, 26 May 2022 13:13:17 -0700
Subject: [PATCH 2/3] Put long name test back

---
 .../covidcast/test_csv_uploading.py           | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py
index 643127c34..d2e398ba4 100644
--- a/integrations/acquisition/covidcast/test_csv_uploading.py
+++ b/integrations/acquisition/covidcast/test_csv_uploading.py
@@ -230,6 +230,38 @@ def test_uploading(self):
       self.setUp()
 
 
+    with self.subTest("Valid signal with name length 32<x<64"):
+      values = pd.DataFrame({
+        "geo_id": ["pa"],
+        "val": [100.0],
+        "se": [5.4],
+        "sample_size": [624.0],
+        "missing_val": [Nans.NOT_MISSING],
+        "missing_se": [Nans.NOT_MISSING],
+        "missing_sample_size": [Nans.NOT_MISSING]
+      })
+      signal_name = "really_long_name_that_will_be_accepted"
+      values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)
+
+      # upload CSVs
+      main(args)
+      dbjobs_main()
+      response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
+
+      expected_values = pd.concat([values, pd.DataFrame({
+        "time_value": [20200419],
+        "signal": [signal_name],
+        "direction": [None]
+      })], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
+      expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}
+
+      self.assertEqual(response, expected_response)
+      self.verify_timestamps_and_defaults()
+
+      self.tearDown()
+      self.setUp()
+
+
     with self.subTest("Invalid signal with a too-long name"):
       values = pd.DataFrame({
         "geo_id": ["pa"],

From 789a404b274a8fb345565c2a4f39d6fefa1a69ae Mon Sep 17 00:00:00 2001
From: Dmitry Shemetov <dshemetov@ucdavis.edu>
Date: Tue, 30 Aug 2022 16:38:32 -0700
Subject: [PATCH 3/3] Fix mocked tests in test_csv_to_database.py

---
 tests/acquisition/covidcast/test_csv_to_database.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/acquisition/covidcast/test_csv_to_database.py b/tests/acquisition/covidcast/test_csv_to_database.py
index e31d35d07..0b91815fb 100644
--- a/tests/acquisition/covidcast/test_csv_to_database.py
+++ b/tests/acquisition/covidcast/test_csv_to_database.py
@@ -2,6 +2,7 @@
 
 # standard library
 import argparse
+from typing import Iterable
 import unittest
 from unittest.mock import MagicMock
 
@@ -67,9 +68,12 @@ def load_csv_impl(path, *args):
         # fail the test for any other path
         raise Exception('unexpected path')
 
+    def iter_len(l: Iterable) -> int:
+      return len(list(l))
+
     data_dir = 'data_dir'
     mock_database = MagicMock()
-    mock_database.insert_or_update_bulk.return_value = 1
+    mock_database.insert_or_update_bulk = MagicMock(wraps=iter_len)
     mock_csv_importer = MagicMock()
     mock_csv_importer.load_csv = load_csv_impl
     mock_file_archiver = MagicMock()
@@ -83,7 +87,7 @@ def load_csv_impl(path, *args):
       mock_logger,
       csv_importer_impl=mock_csv_importer)
 
-    self.assertEqual(modified_row_count, 1)
+    self.assertEqual(modified_row_count, 3)
     # verify that appropriate rows were added to the database
     self.assertEqual(mock_database.insert_or_update_bulk.call_count, 1)
     call_args_list = mock_database.insert_or_update_bulk.call_args_list