diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index dc5f843fb..2ce093488 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -19,11 +19,13 @@ def missing_host_key(self, client, hostname, key): return -def print_callback(filename, logger, bytes_so_far, bytes_total): +def print_callback(filename, logger, bytes_so_far, bytes_total, progress_chunks): """Print the callback information.""" rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) - if (rough_percent_transferred % 25) == 0: + if rough_percent_transferred in progress_chunks: logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred) + # Remove progress chunk, so it is not logged again + progress_chunks.remove(rough_percent_transferred) OLD_FILENAME_TIMESTAMP = re.compile( r".*EDI_AGG_INPATIENT_[0-9]_(?P[0-9]*)_(?P[0-9]*)[^0-9]*") @@ -95,7 +97,8 @@ def download(ftp_credentials, out_path, logger): # download! for infile, outfile in filepaths_to_download.items(): - callback_for_filename = functools.partial(print_callback, infile, logger) + callback_for_filename = functools.partial(print_callback, infile, logger, progress_chunks=[0, 25, 50, 75]) sftp.get(infile, outfile, callback=callback_for_filename) + logger.info("Transfer finished", filename=infile, percent=100) client.close() diff --git a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py index efd110d8b..9d51768be 100644 --- a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py +++ b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py @@ -19,12 +19,13 @@ def missing_host_key(self, client, hostname, key): return -def print_callback(filename, logger, bytes_so_far, bytes_total): +def print_callback(filename, logger, bytes_so_far, bytes_total, progress_chunks): """Print the callback information.""" rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) - if (rough_percent_transferred % 25) == 0: + if rough_percent_transferred in progress_chunks: logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred) - + # Remove progress chunk, so it is not logged again + progress_chunks.remove(rough_percent_transferred) OLD_FILENAME_TIMESTAMP = re.compile( r".*EDI_AGG_OUTPATIENT_[0-9]_(?P[0-9]*)_(?P[0-9]*)[^0-9]*") @@ -100,7 +101,8 @@ def download(ftp_credentials, out_path, logger, issue_date=None): # download! for infile, outfile in filepaths_to_download.items(): - callback_for_filename = functools.partial(print_callback, infile, logger) + callback_for_filename = functools.partial(print_callback, infile, logger, progress_chunks=[0, 25, 50, 75]) sftp.get(infile, outfile, callback=callback_for_filename) + logger.info("Transfer finished", filename=infile, percent=100) client.close() diff --git a/doctor_visits/delphi_doctor_visits/input/SYNEDI_AGG_OUTPATIENT_18052020_1455CDT.csv.gz b/doctor_visits/delphi_doctor_visits/input/SYNEDI_AGG_OUTPATIENT_18052020_1455CDT.csv.gz deleted file mode 100644 index b2a52a8f9..000000000 Binary files a/doctor_visits/delphi_doctor_visits/input/SYNEDI_AGG_OUTPATIENT_18052020_1455CDT.csv.gz and /dev/null differ