Merge pull request #1221 from cmu-delphi/release/indicators_v0.1.12_utils_v0.1.10

Release covidcast-indicators 0.1.12

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.11
+current_version = 0.1.12
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

ansible/templates/usafacts-params-prod.json.j2

@@ -1,6 +1,7 @@
 {
   "common": {
     "export_dir": "/common/covidcast/receiving/usa-facts",
+    "input_dir": "./input-cache",
     "log_filename": "/var/log/indicators/usafacts.log"
   },
   "indicator": {

usafacts/delphi_usafacts/pull.py

@@ -1,10 +1,13 @@
 # -*- coding: utf-8 -*-
 """Functions for pulling data from the USAFacts website."""
+from datetime import date
 import hashlib
 from logging import Logger
+import os
 
 import numpy as np
 import pandas as pd
+import requests
 
 # Columns to drop the the data frame.
 DROP_COLUMNS = [
@@ -14,8 +17,24 @@
     "statefips"
 ]
 
+def fetch(url: str, cache: str) -> pd.DataFrame:
+    """Handle network I/O for fetching raw input data file.
 
-def pull_usafacts_data(base_url: str, metric: str, logger: Logger) -> pd.DataFrame:
+    This is necessary because for some reason pd.read_csv is generating
+    403:Forbidden on the new URLs.
+    """
+    r = requests.get(url)
+    r.raise_for_status()
+    datestamp = date.today().strftime('%Y%m%d')
+    name = url.split('/')[-1].replace('.csv','')
+    os.makedirs(cache, exist_ok=True)
+    filename = os.path.join(cache, f"{datestamp}_{name}.csv")
+    with open(filename, "w") as f:
+        f.write(r.text)
+    return pd.read_csv(filename)
+
+
+def pull_usafacts_data(base_url: str, metric: str, logger: Logger, cache: str=None) -> pd.DataFrame:
     """Pull the latest USA Facts data, and conform it into a dataset.
 
     The output dataset has:
@@ -47,14 +66,17 @@ def pull_usafacts_data(base_url: str, metric: str, logger: Logger) -> pd.DataFra
         Base URL for pulling the USA Facts data
     metric: str
         One of 'confirmed' or 'deaths'. The keys of base_url.
+    logger: Logger
+    cache: str
+        Directory where downloaded csvs should be stashed.
 
     Returns
     -------
     pd.DataFrame
         Dataframe as described above.
     """
     # Read data
-    df = pd.read_csv(base_url.format(metric=metric))
+    df = fetch(base_url.format(metric=metric), cache)
     date_cols = [i for i in df.columns if i.startswith("2")]
     logger.info("data retrieved from source",
                 metric=metric,

usafacts/delphi_usafacts/run.py

@@ -90,9 +90,10 @@ def run_module(params: Dict[str, Dict[str, Any]]):
     else:
         export_start_date = datetime.strptime(export_start_date, "%Y-%m-%d")
     export_dir = params["common"]["export_dir"]
+    input_dir = params["common"]["input_dir"]
     base_url = params["indicator"]["base_url"]
 
-    dfs = {metric: pull_usafacts_data(base_url, metric, logger) for metric in METRICS}
+    dfs = {metric: pull_usafacts_data(base_url, metric, logger, input_dir) for metric in METRICS}
     for metric, geo_res, sensor, smoother in product(
             METRICS, GEO_RESOLUTIONS, SENSORS, SMOOTHERS):
         if "cumulative" in sensor and "seven_day_average" in smoother:

usafacts/input-cache/.gitignore

@@ -0,0 +1,120 @@
+# You should hard commit a prototype for this file, but we
+# want to avoid accidental adding of API tokens and other
+# private data parameters
+params.json
+
+# Do not commit output files
+receiving/*.csv
+
+# Remove macOS files
+.DS_Store
+
+# virtual environment
+dview/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+coverage.xml
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+.static_storage/
+.media/
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

usafacts/params.json.template

@@ -1,6 +1,7 @@
 {
   "common": {
     "export_dir": "./receiving",
+    "input_dir": "./input-cache",
     "log_exceptions": false,
     "log_filename": "./usa-facts.log"
   },

usafacts/tests/test_pull.py

@@ -1,10 +1,13 @@
 import pytest
 import logging
+from unittest.mock import patch
 
 import pandas as pd
 
 from delphi_usafacts.pull import pull_usafacts_data
 
+from test_run import local_fetch
+
 BASE_URL_GOOD = "test_data/small_{metric}_pull.csv"
 
 BASE_URL_BAD = {
@@ -15,6 +18,7 @@
 
 TEST_LOGGER = logging.getLogger()
 
+@patch("delphi_usafacts.pull.fetch", local_fetch)
 class TestPullUSAFacts:
     def test_good_file(self):
         metric = "deaths"

usafacts/tests/test_run.py

@@ -2,16 +2,22 @@
 from itertools import product
 from os import listdir
 from os.path import join
+from unittest.mock import patch
 
 import pandas as pd
 
 from delphi_usafacts.run import run_module
 
+def local_fetch(url, cache):
+    return pd.read_csv(url)
+
+@patch("delphi_usafacts.pull.fetch", local_fetch)
 class TestRun:
     """Tests for the `run_module()` function."""
     PARAMS = {
         "common": {
-            "export_dir": "./receiving"
+            "export_dir": "./receiving",
+            "input_dir": "./input_cache"
         },
         "indicator": {
             "base_url": "./test_data/small_{metric}.csv",