diff --git a/changehc/Makefile b/changehc/Makefile index 56a71a88c..968732f99 100644 --- a/changehc/Makefile +++ b/changehc/Makefile @@ -13,7 +13,8 @@ install: venv lint: . env/bin/activate; \ - pylint $(dir) + pylint $(dir); \ + pydocstyle $(dir) test: . env/bin/activate ;\ diff --git a/changehc/delphi_changehc/config.py b/changehc/delphi_changehc/config.py index 1cabb1eb4..17b459b9e 100644 --- a/changehc/delphi_changehc/config.py +++ b/changehc/delphi_changehc/config.py @@ -9,8 +9,7 @@ class Config: - """Static configuration variables. - """ + """Static configuration variables.""" ## dates FIRST_DATA_DATE = datetime(2020, 1, 1) @@ -59,9 +58,11 @@ class Config: class Constants: """ - Contains the maximum number of geo units for each geo type + Contains the maximum number of geo units for each geo type. + Used for sanity checks """ + # number of counties in usa, including megacounties NUM_COUNTIES = 3141 + 52 NUM_HRRS = 308 diff --git a/changehc/delphi_changehc/constants.py b/changehc/delphi_changehc/constants.py index c86f78d9b..ab2790a86 100644 --- a/changehc/delphi_changehc/constants.py +++ b/changehc/delphi_changehc/constants.py @@ -1,4 +1,4 @@ -"""Registry for signal names and geo types""" +"""Registry for signal names and geo types.""" SMOOTHED = "smoothed_outpatient_covid" SMOOTHED_ADJ = "smoothed_adj_outpatient_covid" SMOOTHED_CLI = "smoothed_outpatient_cli" diff --git a/changehc/delphi_changehc/download_ftp_files.py b/changehc/delphi_changehc/download_ftp_files.py index eee83366b..75627ce81 100644 --- a/changehc/delphi_changehc/download_ftp_files.py +++ b/changehc/delphi_changehc/download_ftp_files.py @@ -1,5 +1,4 @@ -""" -Downloads files modified in the last 24 hours from the specified ftp server.""" +"""Download files modified in the last 24 hours from the specified ftp server.""" # standard import datetime @@ -11,19 +10,19 @@ def print_callback(filename, bytes_so_far, bytes_total): - """Log file transfer progress""" + """Log file transfer progress.""" rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) if (rough_percent_transferred % 25) == 0: print(f'{filename} transfer: {rough_percent_transferred}%') def get_files_from_dir(sftp, out_path): - """Download files from sftp server that have been uploaded in last day + """Download files from sftp server that have been uploaded in last day. + Args: sftp: SFTP Session from Paramiko client out_path: Path to local directory into which to download the files """ - current_time = datetime.datetime.now() # go through files in recieving dir @@ -45,12 +44,12 @@ def get_files_from_dir(sftp, out_path): def download_covid(out_path, ftp_conn): - """Downloads files necessary to create chng-covid signal from ftp server. + """Download files necessary to create chng-covid signal from ftp server. + Args: out_path: Path to local directory into which to download the files ftp_conn: Dict containing login credentials to ftp server """ - # open client try: client = paramiko.SSHClient() @@ -74,12 +73,12 @@ def download_covid(out_path, ftp_conn): def download_cli(out_path, ftp_conn): - """Downloads files necessary to create chng-cli signal from ftp server. + """Download files necessary to create chng-cli signal from ftp server. + Args: out_path: Path to local directory into which to download the files ftp_conn: Dict containing login credentials to ftp server """ - # open client try: client = paramiko.SSHClient() diff --git a/changehc/delphi_changehc/load_data.py b/changehc/delphi_changehc/load_data.py index 28ad8c4dd..3b6a539db 100644 --- a/changehc/delphi_changehc/load_data.py +++ b/changehc/delphi_changehc/load_data.py @@ -27,7 +27,6 @@ def load_chng_data(filepath, dropdate, base_geo, Returns: cleaned dataframe """ - assert base_geo == "fips", "base unit must be 'fips'" count_flag = False date_flag = False diff --git a/changehc/delphi_changehc/run.py b/changehc/delphi_changehc/run.py index 57a82d4f1..4ed483d01 100644 --- a/changehc/delphi_changehc/run.py +++ b/changehc/delphi_changehc/run.py @@ -19,8 +19,7 @@ from .update_sensor import CHCSensorUpdator def retrieve_files(params, filedate): - """Return filenames of relevant files, downloading them if necessary - """ + """Return filenames of relevant files, downloading them if necessary.""" files = params["input_files"] if files["denom"] is None: @@ -57,8 +56,7 @@ def retrieve_files(params, filedate): def make_asserts(params): - """Assert that for each type, filenames are either all present or all absent - """ + """Assert that for each type, filenames are either all present or all absent.""" files = params["input_files"] if "covid" in params["types"]: assert (files["denom"] is None) == (files["covid"] is None), \ @@ -79,9 +77,7 @@ def make_asserts(params): def run_module(): - """Run the delphi_changehc module. - """ - + """Run the delphi_changehc module.""" params = read_params() logging.basicConfig(level=logging.DEBUG) diff --git a/changehc/delphi_changehc/sensor.py b/changehc/delphi_changehc/sensor.py index 949e12ede..32c23f66b 100644 --- a/changehc/delphi_changehc/sensor.py +++ b/changehc/delphi_changehc/sensor.py @@ -20,19 +20,19 @@ class CHCSensor: - """Sensor class to fit a signal using Covid counts from Change HC outpatient data. - """ + """Sensor class to fit a signal using Covid counts from Change HC outpatient data.""" + smoother = Smoother("savgol", poly_fit_degree=1, gaussian_bandwidth=Config.SMOOTHER_BANDWIDTH) @staticmethod def gauss_smooth(count,total): - """smooth using the left_gauss_linear + """Smooth using the left_gauss_linear. Args: count, total: array - """ + """ count_smooth = CHCSensor.smoother.smooth(count) total_smooth = CHCSensor.smoother.smooth(total) total_clip = np.clip(total_smooth, 0, None) @@ -46,12 +46,12 @@ def backfill( k=Config.MAX_BACKFILL_WINDOW, min_visits_to_fill=Config.MIN_CUM_VISITS): """ - Adjust for backfill (retroactively added observations) by using a - variable length smoother, which starts from the RHS and moves - leftwards (backwards through time). We cumulatively sum the total - visits (denominator), until we have observed some minimum number of - counts, then calculate the sum over that bin. We restrict the - bin size so to avoid inluding long-past values. + Adjust for retroactively added observations (backfill) by using a variable length smoother. + + The smoother starts from the RHS and moves leftwards (backwards through time). + We cumulatively sum the total visits (denominator), until we have observed some minimum number of + counts, then calculate the sum over that bin. We restrict the + bin size so to avoid including long-past values. Args: num: array of covid counts diff --git a/changehc/delphi_changehc/update_sensor.py b/changehc/delphi_changehc/update_sensor.py index 60730aa0f..2489b88da 100644 --- a/changehc/delphi_changehc/update_sensor.py +++ b/changehc/delphi_changehc/update_sensor.py @@ -1,5 +1,6 @@ """ Generate CHC sensors. + Author: Aaron Rumack Created: 2020-10-14 """ @@ -21,6 +22,7 @@ def write_to_csv(output_dict, write_se, out_name, output_path="."): """Write sensor values to csv. + Args: output_dict: dictionary containing sensor rates, se, unique dates, and unique geo_id write_se: boolean to write out standard errors, if true, use an obfuscated name @@ -72,8 +74,7 @@ def write_to_csv(output_dict, write_se, out_name, output_path="."): class CHCSensorUpdator: # pylint: disable=too-many-instance-attributes - """Contains methods to update sensor and write results to csv - """ + """Contains methods to update sensor and write results to csv.""" def __init__(self, startdate, @@ -84,7 +85,8 @@ def __init__(self, weekday, numtype, se): - """Init Sensor Updator + """Init Sensor Updator. + Args: startdate: first sensor date (YYYY-mm-dd) enddate: last sensor date (YYYY-mm-dd) @@ -123,8 +125,7 @@ def __init__(self, self.sensor_dates = None def shift_dates(self): - """shift estimates forward to account for time lag, compute burnindates, sensordates - """ + """Shift estimates forward to account for time lag, compute burnindates, sensordates.""" drange = lambda s, e: pd.date_range(start=s,periods=(e-s).days,freq='D') self.startdate = self.startdate - Config.DAY_SHIFT self.burnindate = self.startdate - Config.BURN_IN_PERIOD @@ -134,7 +135,8 @@ def shift_dates(self): return True def geo_reindex(self, data): - """Reindex based on geography, include all date, geo pairs + """Reindex based on geography, include all date, geo pairs. + Args: data: dataframe, the output of loadcombineddata Returns: @@ -178,6 +180,7 @@ def update_sensor(self, data, outpath): """Generate sensor values, and write to csv format. + Args: data: pd.DataFrame with columns num and den outpath: output path for the csv results diff --git a/changehc/delphi_changehc/weekday.py b/changehc/delphi_changehc/weekday.py index 03b69cba4..e02997482 100644 --- a/changehc/delphi_changehc/weekday.py +++ b/changehc/delphi_changehc/weekday.py @@ -55,7 +55,6 @@ def get_params(data): Return a matrix of parameters: the entire vector of betas, for each time series column in the data. """ - tmp = data.reset_index() denoms = tmp.groupby(Config.DATE_COL).sum()["den"] nums = tmp.groupby(Config.DATE_COL).sum()["num"] @@ -113,7 +112,6 @@ def calc_adjustment(params, sub_data): -- this has the same effect. """ - tmp = sub_data.reset_index() wd_correction = np.zeros((len(tmp["num"])))