Merge pull request #785 from cmu-delphi/changehc_dropdate

changehc: fetch files based on filename not modification time

Author: krivard

changehc/delphi_changehc/download_ftp_files.py

@@ -1,4 +1,4 @@
-"""Download files modified in the last 24 hours from the specified ftp server."""
+"""Download files from the specified ftp server."""
 
 # standard
 import datetime
@@ -16,21 +16,19 @@ def print_callback(filename, bytes_so_far, bytes_total):
         print(f'{filename} transfer: {rough_percent_transferred}%')
 
 
-def get_files_from_dir(sftp, out_path):
-    """Download files from sftp server that have been uploaded in last day.
+def get_files_from_dir(sftp, filedate, out_path):
+    """Download files from sftp server tagged with the specified day.
 
     Args:
         sftp: SFTP Session from Paramiko client
+        filedate: YYYYmmdd string for which the files are named
         out_path: Path to local directory into which to download the files
     """
-    current_time = datetime.datetime.now()
-
     # go through files in recieving dir
     filepaths_to_download = {}
     for fileattr in sftp.listdir_attr():
-        file_time = datetime.datetime.fromtimestamp(fileattr.st_mtime)
         filename = fileattr.filename
-        if current_time - file_time < datetime.timedelta(days=1) and \
+        if fileattr.filename.startswith(filedate) and \
                 not path.exists(path.join(out_path, filename)):
             filepaths_to_download[filename] = path.join(out_path, filename)
 
@@ -43,10 +41,11 @@ def get_files_from_dir(sftp, out_path):
         sftp.get(infile, outfile, callback=callback_for_filename)
 
 
-def download_covid(out_path, ftp_conn):
+def download_covid(filedate, out_path, ftp_conn):
     """Download files necessary to create chng-covid signal from ftp server.
 
     Args:
+        filedate: YYYYmmdd string for which the files are named
         out_path: Path to local directory into which to download the files
         ftp_conn: Dict containing login credentials to ftp server
     """
@@ -62,20 +61,21 @@ def download_covid(out_path, ftp_conn):
         sftp = client.open_sftp()
 
         sftp.chdir('/dailycounts/All_Outpatients_By_County')
-        get_files_from_dir(sftp, out_path)
+        get_files_from_dir(sftp, filedate, out_path)
 
         sftp.chdir('/dailycounts/Covid_Outpatients_By_County')
-        get_files_from_dir(sftp, out_path)
+        get_files_from_dir(sftp, filedate, out_path)
 
     finally:
         if client:
             client.close()
 
 
-def download_cli(out_path, ftp_conn):
+def download_cli(filedate, out_path, ftp_conn):
     """Download files necessary to create chng-cli signal from ftp server.
 
     Args:
+        filedate: YYYYmmdd string for which the files are named
         out_path: Path to local directory into which to download the files
         ftp_conn: Dict containing login credentials to ftp server
     """
@@ -91,19 +91,19 @@ def download_cli(out_path, ftp_conn):
         sftp = client.open_sftp()
 
         sftp.chdir('/dailycounts/All_Outpatients_By_County')
-        get_files_from_dir(sftp, out_path)
+        get_files_from_dir(sftp, filedate, out_path)
 
         sftp.chdir('/dailycounts/Flu_Patient_Count_By_County')
-        get_files_from_dir(sftp, out_path)
+        get_files_from_dir(sftp, filedate, out_path)
 
         sftp.chdir('/dailycounts/Mixed_Patient_Count_By_County')
-        get_files_from_dir(sftp, out_path)
+        get_files_from_dir(sftp, filedate, out_path)
 
         sftp.chdir('/dailycounts/Flu_Like_Patient_Count_By_County')
-        get_files_from_dir(sftp, out_path)
+        get_files_from_dir(sftp, filedate, out_path)
 
         sftp.chdir('/dailycounts/Covid_Like_Patient_Count_By_County')
-        get_files_from_dir(sftp, out_path)
+        get_files_from_dir(sftp, filedate, out_path)
 
     finally:
         if client:

changehc/delphi_changehc/run.py

@@ -25,9 +25,9 @@ def retrieve_files(params, filedate, logger):
         ## download recent files from FTP server
         logger.info("downloading recent files through SFTP")
         if "covid" in params["types"]:
-            download_covid(params["cache_dir"], params["ftp_conn"])
+            download_covid(filedate, params["cache_dir"], params["ftp_conn"])
         if "cli" in params["types"]:
-            download_cli(params["cache_dir"], params["ftp_conn"])
+            download_cli(filedate, params["cache_dir"], params["ftp_conn"])
 
         denom_file = "%s/%s_All_Outpatients_By_County.dat.gz" % (params["cache_dir"],filedate)
         covid_file = "%s/%s_Covid_Outpatients_By_County.dat.gz" % (params["cache_dir"],filedate)

changehc/tests/test_download_ftp_files.py

@@ -37,27 +37,31 @@ def __init__(self, time, name):
     def test_get_files(self, mock_path):
 
         # When one new file is present, one file is downloaded
-        one_new = self.MockSFTP([self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)),"foo")])
-        get_files_from_dir(one_new, "")
+        one_new = self.MockSFTP([
+            self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)), "00001122_foo")
+        ])
+        get_files_from_dir(one_new, "00001122", "")
         assert one_new.num_gets == 1
 
         # When one new file and one old file are present, one file is downloaded
-        one_new_one_old = self.MockSFTP([self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)),"foo"),
-                                         self.FileAttr(dt.timestamp(dt.now()-timedelta(days=10)),"foo")])
-        get_files_from_dir(one_new_one_old, "")
+        one_new_one_old = self.MockSFTP([
+            self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)), "00005566_foo"),
+            self.FileAttr(dt.timestamp(dt.now()-timedelta(days=10)), "00001122_foo")
+        ])
+        get_files_from_dir(one_new_one_old, "00005566", "")
         assert one_new_one_old.num_gets == 1
 
         # When three new files are present, AssertionError
-        new_file1 = self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)),"foo1")
-        new_file2 = self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)),"foo2")
-        new_file3 = self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)),"foo3")
+        new_file1 = self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)), "00001122_foo1")
+        new_file2 = self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)), "00001122_foo2")
+        new_file3 = self.FileAttr(dt.timestamp(dt.now()-timedelta(minutes=1)), "00001122_foo3")
         three_new = self.MockSFTP([new_file1, new_file2, new_file3])
         with pytest.raises(AssertionError):
-            get_files_from_dir(three_new,"")
+            get_files_from_dir(three_new, "00001122", "")
 
         # When the file already exists, no files are downloaded
         mock_path.exists.return_value = True
         one_exists = self.MockSFTP([new_file1])
-        get_files_from_dir(one_new, "")
+        get_files_from_dir(one_new, "00001122", "")
         assert one_exists.num_gets == 0