Merge pull request #861 from cmu-delphi/chng_newfiles

CHNG New file format and location

Author: krivard

changehc/delphi_changehc/config.py

@@ -34,12 +34,12 @@ class Config:
     ID_COLS = [DATE_COL] + [GEO_COL]
     FILT_COLS = ID_COLS + COUNT_COLS
 
-    DENOM_COLS = [GEO_COL, DATE_COL, DENOM_COL]
-    COVID_COLS = [GEO_COL, DATE_COL, COVID_COL]
-    FLU_COLS = [GEO_COL, DATE_COL, FLU_COL]
-    MIXED_COLS = [GEO_COL, DATE_COL, MIXED_COL]
-    FLU_LIKE_COLS = [GEO_COL, DATE_COL, FLU_LIKE_COL]
-    COVID_LIKE_COLS = [GEO_COL, DATE_COL, COVID_LIKE_COL]
+    DENOM_COLS = [DATE_COL, GEO_COL, DENOM_COL]
+    COVID_COLS = [DATE_COL, GEO_COL, COVID_COL]
+    FLU_COLS = [DATE_COL, GEO_COL, FLU_COL]
+    MIXED_COLS = [DATE_COL, GEO_COL, MIXED_COL]
+    FLU_LIKE_COLS = [DATE_COL, GEO_COL, FLU_LIKE_COL]
+    COVID_LIKE_COLS = [DATE_COL, GEO_COL, COVID_LIKE_COL]
 
     DENOM_DTYPES = {DATE_COL: str, DENOM_COL: str, GEO_COL: str}
     COVID_DTYPES = {DATE_COL: str, COVID_COL: str, GEO_COL: str}

changehc/delphi_changehc/download_ftp_files.py

@@ -32,8 +32,8 @@ def get_files_from_dir(sftp, filedate, out_path):
                 not path.exists(path.join(out_path, filename)):
             filepaths_to_download[filename] = path.join(out_path, filename)
 
-    # make sure we don't download more than 2 files per day
-    assert len(filepaths_to_download) <= 2, "more files dropped than expected"
+    # make sure we don't download more than 6 files per day
+    assert len(filepaths_to_download) <= 6, "more files dropped than expected"
 
     # download!
     for infile, outfile in filepaths_to_download.items():
@@ -60,10 +60,7 @@ def download_covid(filedate, out_path, ftp_conn):
                        allow_agent=False, look_for_keys=False)
         sftp = client.open_sftp()
 
-        sftp.chdir('/dailycounts/All_Outpatients_By_County')
-        get_files_from_dir(sftp, filedate, out_path)
-
-        sftp.chdir('/dailycounts/Covid_Outpatients_By_County')
+        sftp.chdir('/countproducts')
         get_files_from_dir(sftp, filedate, out_path)
 
     finally:
@@ -90,19 +87,7 @@ def download_cli(filedate, out_path, ftp_conn):
                        allow_agent=False, look_for_keys=False)
         sftp = client.open_sftp()
 
-        sftp.chdir('/dailycounts/All_Outpatients_By_County')
-        get_files_from_dir(sftp, filedate, out_path)
-
-        sftp.chdir('/dailycounts/Flu_Patient_Count_By_County')
-        get_files_from_dir(sftp, filedate, out_path)
-
-        sftp.chdir('/dailycounts/Mixed_Patient_Count_By_County')
-        get_files_from_dir(sftp, filedate, out_path)
-
-        sftp.chdir('/dailycounts/Flu_Like_Patient_Count_By_County')
-        get_files_from_dir(sftp, filedate, out_path)
-
-        sftp.chdir('/dailycounts/Covid_Like_Patient_Count_By_County')
+        sftp.chdir('/countproducts')
         get_files_from_dir(sftp, filedate, out_path)
 
     finally:

changehc/delphi_changehc/load_data.py

@@ -44,7 +44,7 @@ def load_chng_data(filepath, dropdate, base_geo,
 
     data = pd.read_csv(
         filepath,
-        sep="|",
+        sep=",",
         header=None,
         names=col_names,
         dtype=col_types,

changehc/delphi_changehc/run.py

@@ -31,12 +31,12 @@ def retrieve_files(params, filedate, logger):
         if "cli" in params["indicator"]["types"]:
             download_cli(filedate, params["indicator"]["input_cache_dir"], params["indicator"]["ftp_conn"])
 
-        denom_file = "%s/%s_All_Outpatients_By_County.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
-        covid_file = "%s/%s_Covid_Outpatients_By_County.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
-        flu_file = "%s/%s_Flu_Patient_Count_By_County.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
-        mixed_file = "%s/%s_Mixed_Patient_Count_By_County.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
-        flu_like_file = "%s/%s_Flu_Like_Patient_Count_By_County.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
-        covid_like_file = "%s/%s_Covid_Like_Patient_Count_By_County.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
+        denom_file = "%s/%s_Counts_Products_Denom.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
+        covid_file = "%s/%s_Counts_Products_Covid.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
+        flu_file = "%s/%s_Counts_Products_Flu.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
+        mixed_file = "%s/%s_Counts_Products_Mixed.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
+        flu_like_file = "%s/%s_Counts_Products_Flu_Like.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
+        covid_like_file = "%s/%s_Counts_Products_Covid_Like.dat.gz" % (params["indicator"]["input_cache_dir"],filedate)
     else:
         denom_file = files["denom"]
         covid_file = files["covid"]