Merge branch 'main' into ndefries/qsf-umd-bugs

Author: nmdefries

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.12
+current_version = 0.3.13
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

.github/workflows/publish-release.yml

@@ -21,6 +21,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           ssh-key: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_SSH }}
+          fetch-depth: 5
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.3
+current_version = 0.3.4
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.3.3"
+__version__ = "0.3.4"

_delphi_utils_python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.3.3",
+    version="0.3.4",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

dsew_community_profile/delphi_dsew_community_profile/pull.py

@@ -408,7 +408,7 @@ def as_cached_filename(params, config):
 def fetch_listing(params):
     """Generate the list of report files to process."""
     export_start_date = params['indicator'].get(
-        'export_start_date', datetime.datetime.fromtimestamp(0).date()
+        'export_start_date', datetime.datetime.utcfromtimestamp(0).date()
     )
 
     listing = requests.get(DOWNLOAD_LISTING).json()['metadata']['attachments']

dsew_community_profile/tests/test_pull.py

@@ -39,6 +39,7 @@ def _set_df_dtypes(df: pd.DataFrame, dtypes: Dict[str, Any]) -> pd.DataFrame:
             df[k] = df[k].astype(v)
     return df
 
+
 class TestPull:
     def test_DatasetTimes(self):
         examples = [
@@ -158,31 +159,35 @@ def test_Dataset_parse_sheet(self):
         # TODO
         pass
 
-    @patch('requests.get')
-    @patch('os.path.exists')
-    def test_fetch_listing(self, mock_listing, mock_exists):
+    def test_fetch_listing(self):
         inst = namedtuple("attachment", "assetId filename publish cache")
         instances = list(chain(*[
             [
-                inst(f"{i}", f"2021010{i}.xlsx", date(2021, 1, i), f"{i}---2021010{i}.xlsx"),
-                inst(f"p{i}", f"2021010{i}.pdf", date(2021, 1, i), f"p{i}---2021010{i}.pdf"),
+                inst(f"{i}", f"2021010{i}.xlsx", date(2021, 1, i), f"2021010{i}--{i}.xlsx"),
+                inst(f"p{i}", f"2021010{i}.pdf", date(2021, 1, i), f"2021010{i}--p{i}.pdf"),
             ]
             for i in [1, 2, 3, 4, 5]
         ]))
 
-        mock_listing.return_value = Mock()
-        mock_listing.return_value.json = Mock(
-            return_value = {
-                'metadata': {
-                    'attachments': [
-                        {"assetId": i.assetId, "filename": i.filename}
-                        for i in instances
-                    ]
-                }
-            }
-        )
-
-        mock_exists.reset_mock(return_value=False)
+        # Solution from https://stackoverflow.com/questions/15753390/
+        #how-can-i-mock-requests-and-the-response
+        def mocked_requests_get(*args, **kwargs):
+            class MockResponse:
+                def __init__(self, json_data):
+                    self.json_data = json_data
+
+                def json(self):
+                    return self.json_data
+
+            return MockResponse({
+                        'metadata': {
+                            'attachments': [
+                                {"assetId": i.assetId, "filename": i.filename}
+                                for i in instances
+                            ]
+                        }
+                    }
+                )
 
         def as_listing(instance):
             return {
@@ -192,15 +197,20 @@ def as_listing(instance):
                 "publish_date": instance.publish
             }
         ex = example(
-            {'indicator':{'reports':'new'}},
+            {'indicator':{'reports':'new', 'input_cache':''}},
             [
                 as_listing(instance)
                 for i, instance in filter(lambda x: x[0]%2 == 0, enumerate(instances))
             ]
         )
 
-        for actual, expected in zip(fetch_listing(ex.given), ex.expected):
-            assert actual == expected
+        with patch('requests.get', side_effect=mocked_requests_get):
+            with patch('os.path.exists', return_value=False):
+                for actual, expected in zip(fetch_listing(ex.given), ex.expected):
+                    assert actual == expected
+
+            with patch('os.path.exists', return_value=True):
+                assert fetch_listing(ex.given) == []
 
     def test_nation_from_state(self):
         geomapper = GeoMapper()

facebook/qsf-tools/generate-changelog.R

@@ -75,7 +75,7 @@ get_codebook <- function(path_to_codebook) {
 
   return(codebook)
 }
-  
+
 # Try to load `path_to_diff`. Check if it is a single CSV or a directory
 # containing a set of CSVs.
 get_diff <- function(path_to_diff) {
@@ -196,7 +196,7 @@ prepare_matrix_base_questions_for_join <- function(qsf_diff, codebook) {
     qsf_diff %>% distinct(variable_name) %>% pull(),
     codebook %>% distinct(variable) %>% pull()
   )
-  
+
   # Add an underscore to the unmatched variable names to create a regex pattern
   matrix_prefixes <- paste0(vars_not_in_codebook, "_")
   names(matrix_prefixes) <- vars_not_in_codebook
@@ -292,7 +292,7 @@ prepare_matrix_base_questions_for_join <- function(qsf_diff, codebook) {
       join_variable_old_wave = coalesce(join_variable_old_wave, variable_name)
     ) %>%
     select(-join_variable)
-  
+
   return(list("diff" = qsf_diff, "vars_not_in_codebook" = vars_not_in_codebook))
 }