Fix for automated file downloads (#468)

* Fix for automated file downloads

* Add ignores for these message types

* Add null to denom and covid input files

* Remove quotes

Co-authored-by: Brian Clark <[email protected]>

Author: rumackaaron

ansible/templates/changehc-params-prod.json.j2

@@ -2,8 +2,8 @@
   "static_file_dir": "./static",
   "export_dir": "/common/covidcast/receiving/changehc",
   "cache_dir": "./cache",
-  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
-  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
+  "input_denom_file": null,
+  "input_covid_file": null,
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,

changehc/.pylintrc

@@ -7,7 +7,9 @@ disable=logging-format-interpolation,
     # Allow pytest functions to be part of a class.
     no-self-use,
     # Allow pytest classes to have one test.
-    too-few-public-methods
+    too-few-public-methods,
+    # Ignore
+    R0903, C0301, R0914, C0103, W1203, E0611, R0902, R0913, W0105, W0611, W1401
 
 [BASIC]
 

changehc/delphi_changehc/run.py

@@ -26,27 +26,33 @@ def run_module():
 
     logging.basicConfig(level=logging.DEBUG)
 
-    ## download recent files from FTP server
-    logging.info("downloading recent files through SFTP")
-    download(params["cache_dir"], params["ftp_conn"])
-
-    ## get end date from input file
     # the filenames are expected to be in the format:
     # Denominator: "YYYYMMDD_All_Outpatients_By_County.dat.gz"
     # Numerator: "YYYYMMDD_Covid_Outpatients_By_County.dat.gz"
 
+    assert (params["input_denom_file"] is None) == (params["input_covid_file"] is None), \
+        "exactly one of denom and covid files are provided"
+
     if params["drop_date"] is None:
-        dropdate_denom = datetime.strptime(
-            Path(params["input_denom_file"]).name.split("_")[0], "%Y%m%d"
-        )
-
-        dropdate_covid = datetime.strptime(
-            Path(params["input_covid_file"]).name.split("_")[0], "%Y%m%d"
-        )
-        assert dropdate_denom == dropdate_covid, "different drop dates for data files"
-        dropdate_dt = dropdate_denom
+        # files are dropped about 8pm the day after the issue date
+        dropdate_dt = (datetime.now() - timedelta(days=1,hours=20))
+        dropdate_dt = dropdate_dt.replace(hour=0,minute=0,second=0,microsecond=0)
     else:
         dropdate_dt = datetime.strptime(params["drop_date"], "%Y-%m-%d")
+    filedate = dropdate_dt.strftime("%Y%m%d")
+
+    if params["input_denom_file"] is None:
+
+        ## download recent files from FTP server
+        logging.info("downloading recent files through SFTP")
+        download(params["cache_dir"], params["ftp_conn"])
+
+        input_denom_file = "%s/%s_All_Outpatients_By_County.dat.gz" % (params["cache_dir"],filedate)
+        input_covid_file = "%s/%s_Covid_Outpatients_By_County.dat.gz" % (params["cache_dir"],filedate)
+    else:
+        input_denom_file = params["input_denom_file"]
+        input_covid_file = params["input_covid_file"]
+
     dropdate = str(dropdate_dt.date())
 
     # range of estimates to produce
@@ -93,8 +99,8 @@ def run_module():
                 params["se"]
             )
             su_inst.update_sensor(
-                params["input_denom_file"],
-                params["input_covid_file"],
+                input_denom_file,
+                input_covid_file,
                 params["export_dir"]
             )
         logging.info("finished %s", geo)

changehc/params.json.template

@@ -2,8 +2,8 @@
   "static_file_dir": "./static",
   "export_dir": "./receiving",
   "cache_dir": "./cache",
-  "input_denom_file": "./tests/test_data/20200601_All_Outpatients_By_County.dat",
-  "input_covid_file": "./tests/test_data/20200601_Covid_Outpatients_By_County.dat",
+  "input_denom_file": null,
+  "input_covid_file": null,
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,