update tests to reflect new func structure

Author: nmdefries

google_symptoms/tests/conftest.py

@@ -54,21 +54,6 @@
 # Set up fake list of dates to fetch.
 dates = [
     "20200726",
-    "20200727",
-    "20200728",
-    "20200729",
-    "20200730",
-    "20200731",
-    "20200801",
-    "20200802",
-    "20200803",
-    "20200804",
-    "20200805",
-    "20200806",
-    "20200807",
-    "20200808",
-    "20200809",
-    "20200810",
     "20200811"
 ]
 
@@ -84,7 +69,7 @@ def run_as_module():
     else:
         makedirs("receiving")
 
-    with mock.patch("delphi_google_symptoms.pull.get_all_dates",
+    with mock.patch("delphi_google_symptoms.pull.get_date_range",
                     return_value=date_list) as mock_all_dates:
         with mock.patch("delphi_google_symptoms.pull.initialize_credentials",
                         return_value=None) as mock_credentials:

google_symptoms/tests/params.json.template

@@ -3,5 +3,6 @@
   "static_file_dir": "../static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
+  "num_export_days": 14,
   "path_to_bigquery_credentials": ""
 }

google_symptoms/tests/test_pull.py

@@ -4,7 +4,8 @@
 from datetime import date, datetime
 import pandas as pd
 
-from delphi_google_symptoms.pull import pull_gs_data, preprocess, get_missing_dates, format_dates_for_query, pull_gs_data_one_geolevel, get_all_dates
+from delphi_google_symptoms.pull import (
+    pull_gs_data, preprocess, format_dates_for_query, pull_gs_data_one_geolevel, get_date_range)
 from delphi_google_symptoms.constants import METRICS, COMBINED_METRIC
 
 good_input = {
@@ -23,24 +24,19 @@
 new_keep_cols = ["geo_id", "timestamp"] + METRICS + [COMBINED_METRIC]
 
 
-# List of faux already-exported CSVs.
-exported_files = [
-    "20201231_whatever.csv",
-    "20210101_whatever.csv",
-    "20210102_whatever.csv",
-    "20210103_whatever.csv",
-    "20210105_not_even_a_csv.txt",
-    "not_the_right_format.csv"
+# Set up fake list of dates to fetch.
+dates = [
+    "20200726",
+    "20200811"
 ]
+date_list = [datetime.strptime(date, "%Y%m%d").date() for date in dates]
 
 
 class TestPullGoogleSymptoms:
     @freeze_time("2021-01-05")
     @mock.patch("pandas_gbq.read_gbq")
     @mock.patch("delphi_google_symptoms.pull.initialize_credentials")
-    @mock.patch("delphi_google_symptoms.pull.listdir")
-    @mock.patch("delphi_google_symptoms.pull.isfile")
-    def test_good_file(self, mock_is_file, mock_get_dates, mock_credentials, mock_read_gbq):
+    def test_good_file(self, mock_credentials, mock_read_gbq):
         # Set up fake data.
         state_data = pd.read_csv(
             good_input["state"], parse_dates=["date"])[keep_cols]
@@ -50,11 +46,9 @@ def test_good_file(self, mock_is_file, mock_get_dates, mock_credentials, mock_re
         # Mocks
         mock_read_gbq.side_effect = [state_data, county_data]
         mock_credentials.return_value = None
-        mock_get_dates.return_value = exported_files
-        mock_is_file = True
 
         dfs = pull_gs_data(
-            "", "receiving", datetime.strptime("20201230", "%Y%m%d"))
+            "", datetime.strptime("20201230", "%Y%m%d"), 0)
 
         for level in ["county", "state"]:
             df = dfs[level]
@@ -88,78 +82,37 @@ def test_invalid_fips(self):
 
 class TestPullHelperFuncs:
     @freeze_time("2021-01-05")
-    @mock.patch("delphi_google_symptoms.pull.listdir")
-    @mock.patch("delphi_google_symptoms.pull.isfile")
-    def test_get_missing_dates(self, mock_is_file, mock_get_dates):
-        mock_get_dates.return_value = exported_files
-        mock_is_file = True
+    def test_get_date_range_recent_export_start_date(self):
+        output = get_date_range(
+            datetime.strptime("20201230", "%Y%m%d"),
+            14
+        )
 
-        output = get_missing_dates(
-            "receiving", datetime.strptime("20201230", "%Y%m%d"))
-
-        expected = [date(2020, 12, 30), date(2021, 1, 4), date(2021, 1, 5)]
-        assert set(output) == set(expected)
-
-    @freeze_time("2021-01-05")
-    @mock.patch("delphi_google_symptoms.pull.listdir")
-    @mock.patch("delphi_google_symptoms.pull.isfile")
-    def test_get_all_dates_recent_export_start_date(self, mock_is_file, mock_get_dates):
-        mock_get_dates.return_value = exported_files
-        mock_is_file = True
-
-        output = get_all_dates(
-            "receiving", datetime.strptime("20201230", "%Y%m%d"))
-
-        expected = [date(2020, 12, 30),
-                    date(2020, 12, 31),
-                    date(2021, 1, 1),
-                    date(2021, 1, 2),
-                    date(2021, 1, 3),
-                    date(2021, 1, 4),
+        expected = [date(2020, 12, 24),
                     date(2021, 1, 5)]
         assert set(output) == set(expected)
 
     @freeze_time("2021-01-05")
-    @mock.patch("delphi_google_symptoms.pull.get_missing_dates")
-    @mock.patch("delphi_google_symptoms.pull.listdir")
-    @mock.patch("delphi_google_symptoms.pull.isfile")
-    def test_get_all_dates(self, mock_is_file, mock_get_dates, mock_missing_dates):
-        mock_get_dates.return_value = exported_files
-        mock_is_file = True
-        mock_missing_dates.return_value = [
-            date(2020, 12, 30), date(2021, 1, 4), date(2021, 1, 5)]
-
-        output = get_all_dates(
-            "receiving", datetime.strptime("20200201", "%Y%m%d"))
+    def test_get_date_range(self):
+        output = get_date_range(
+            datetime.strptime("20200201", "%Y%m%d"),
+            14
+        )
 
-        expected = [date(2020, 12, 24),
-                    date(2020, 12, 25),
-                    date(2020, 12, 26),
-                    date(2020, 12, 27),
-                    date(2020, 12, 28),
-                    date(2020, 12, 29),
-                    date(2020, 12, 30),
-                    date(2020, 12, 31),
-                    date(2021, 1, 1),
-                    date(2021, 1, 2),
-                    date(2021, 1, 3),
-                    date(2021, 1, 4),
-                    date(2021, 1, 5)]
+        expected = [date(2020, 12, 16), date(2021, 1, 5)]
         assert set(output) == set(expected)
 
     def test_format_dates_for_query(self):
-        date_list = [date(2016, 12, 30), date(2020, 12, 30),
-                     date(2021, 1, 4), date(2021, 1, 5)]
+        date_list = [date(2016, 12, 30), date(2021, 1, 5)]
         output = format_dates_for_query(date_list)
-
-        expected = 'timestamp("2020-12-30"), timestamp("2021-01-04"), timestamp("2021-01-05")'
+        expected = ["2016-12-30", "2021-01-05"]
         assert output == expected
 
     @mock.patch("pandas_gbq.read_gbq")
     def test_pull_one_gs_no_dates(self, mock_read_gbq):
         mock_read_gbq.return_value = pd.DataFrame()
 
-        output = pull_gs_data_one_geolevel("state", {})
+        output = pull_gs_data_one_geolevel("state", ["", ""])
         expected = pd.DataFrame(columns=new_keep_cols)
         assert output.equals(expected)