diff --git a/quidel/delphi_quidel/constants.py b/quidel/delphi_quidel/constants.py index e660d4f8e..6a905c945 100644 --- a/quidel/delphi_quidel/constants.py +++ b/quidel/delphi_quidel/constants.py @@ -1,4 +1,4 @@ -"""Registry for constants""" +"""Registry for constants.""" # global constants MIN_OBS = 50 # minimum number of observations in order to compute a proportion. MAX_BORROW_OBS = 20 # maximum number of observations can be borrowed in geographical pooling diff --git a/quidel/delphi_quidel/export.py b/quidel/delphi_quidel/export.py index 41ed35e95..4507d5e43 100644 --- a/quidel/delphi_quidel/export.py +++ b/quidel/delphi_quidel/export.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -"""Function to export the dataset in the format expected of the API. -""" +"""Function to export the dataset in the format expected of the API.""" import numpy as np import pandas as pd def export_csv(df, geo_name, sensor, receiving_dir, start_date, end_date): - """Export data set in format expected for injestion by the API + """Export data set in format expected for ingestion by the API. + Parameters ---------- df: pd.DataFrame @@ -23,7 +23,6 @@ def export_csv(df, geo_name, sensor, receiving_dir, start_date, end_date): end_date: datetime.datetime The last date to report """ - df = df.copy() df = df[np.logical_and(df["timestamp"] >= start_date, df["timestamp"] <= end_date)] diff --git a/quidel/delphi_quidel/generate_sensor.py b/quidel/delphi_quidel/generate_sensor.py index ca956732a..8d3fbee8d 100644 --- a/quidel/delphi_quidel/generate_sensor.py +++ b/quidel/delphi_quidel/generate_sensor.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -""" -Functions to help generate sensor for different geographical levels -""" +"""Functions to help generate sensor for different geographical levels.""" import pandas as pd from .data_tools import (fill_dates, raw_positive_prop, smoothed_positive_prop, @@ -11,7 +9,8 @@ def generate_sensor_for_states(state_groups, smooth, device, first_date, last_date): """ - fit over states + Fit over states. + Args: state_groups: pd.groupby.generic.DataFrameGroupBy state_key: "state_id" @@ -70,7 +69,8 @@ def generate_sensor_for_states(state_groups, smooth, device, first_date, last_da def generate_sensor_for_other_geores(state_groups, data, res_key, smooth, device, first_date, last_date): """ - fit over counties/HRRs/MSAs + Fit over counties/HRRs/MSAs. + Args: data: pd.DataFrame res_key: "fips", "cbsa_id" or "hrrnum" diff --git a/quidel/delphi_quidel/handle_wip_sensor.py b/quidel/delphi_quidel/handle_wip_sensor.py index fa3e5540b..73ab71ba0 100644 --- a/quidel/delphi_quidel/handle_wip_sensor.py +++ b/quidel/delphi_quidel/handle_wip_sensor.py @@ -1,8 +1,9 @@ -"""This file checks the wip status of signals""" +"""This file checks the wip status of signals.""" import covidcast def add_prefix(signal_names, wip_signal, prefix): - """Adds prefix to signal if there is a WIP signal + """Add prefix to signal if there is a WIP signal. + Parameters ---------- signal_names: List[str] @@ -18,7 +19,6 @@ def add_prefix(signal_names, wip_signal, prefix): List of signal names wip/non wip signals for further computation """ - if wip_signal is True: return [prefix + signal for signal in signal_names] if isinstance(wip_signal, list): @@ -37,7 +37,8 @@ def add_prefix(signal_names, wip_signal, prefix): def public_signal(signal_): - """Checks if the signal name is already public using COVIDcast + """Check if the signal name is already public using COVIDcast. + Parameters ---------- signal_ : str diff --git a/quidel/delphi_quidel/pull.py b/quidel/delphi_quidel/pull.py index 982a8f30e..02fcdd229 100644 --- a/quidel/delphi_quidel/pull.py +++ b/quidel/delphi_quidel/pull.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Simply downloads email attachments. + Uses this handy package: https://pypi.org/project/imap-tools/ """ import io @@ -26,6 +27,7 @@ def compare_dates(date1, date2, flag): """ Compare two dates. + If op == "l" return the larger date If op == "s" return the smaller date """ @@ -39,9 +41,7 @@ def compare_dates(date1, date2, flag): return date1 def check_whether_date_in_range(search_date, start_date, end_date): - """ - Check whether the search date is in a valid time range - """ + """Check whether the search date is in a valid time range.""" if search_date > end_date: return False if search_date < start_date: @@ -49,10 +49,7 @@ def check_whether_date_in_range(search_date, start_date, end_date): return True def read_historical_data(): - """ - Read historical flu antigen test data stored in - midas /common/quidel-historical-raw - """ + """Read historical flu antigen test data stored in midas /common/quidel-historical-raw.""" pull_dir = "/common/quidel-historical-raw" columns = ['SofiaSerNum', 'TestDate', 'Facility', 'ZipCode', 'FluA', 'FluB', 'StorageDate'] @@ -66,9 +63,10 @@ def read_historical_data(): def regulate_column_names(df, test_type): """ - Regulate column names for flu_ag test data since Quidel changed their - column names multiple times. We want to finalize the column name list - to be: + Regulate column names for flu_ag test data. + + This is done since Quidel changed their column names multiple times. + We want to finalize the column name list to be: ['SofiaSerNum', 'TestDate', 'Facility', 'Zip', 'FluA', 'FluB', 'StorageDate'] """ @@ -88,7 +86,7 @@ def regulate_column_names(df, test_type): def get_from_email(column_names, start_dates, end_dates, mail_server, account, sender, password): """ - Get raw data from email account + Get raw data from email account. Parameters: start_date: datetime.datetime @@ -146,9 +144,7 @@ def get_from_email(column_names, start_dates, end_dates, mail_server, return dfs, time_flag def fix_zipcode(df): - """ - Fix zipcode that is 9 digit instead of 5 digit - """ + """Fix zipcode that is 9 digit instead of 5 digit.""" zipcode5 = [] fixnum = 0 for zipcode in df['Zip'].values: @@ -164,6 +160,8 @@ def fix_zipcode(df): def fix_date(df): """ + Adjust tests based on their test and storage dates. + Quidel antigen tests are labeled with Test Date and Storage Date. In principle, the TestDate should reflect when the test was performed and the StorageDate when the test was logged in the MyVirena cloud storage device. We expect @@ -191,6 +189,7 @@ def preprocess_new_data(start_dates, end_dates, mail_server, account, sender, password, test_mode): """ Pull and pre-process Quidel Antigen Test data from datadrop email. + Drop unnecessary columns. Temporarily consider the positive rate sensor only which is related to number of total tests and number of positive tests. @@ -286,7 +285,7 @@ def preprocess_new_data(start_dates, end_dates, mail_server, account, def check_intermediate_file(cache_dir, pull_start_dates): """ - Check whether there is a cache file containing historical data already + Check whether there is a cache file containing historical data already. Parameters: cache_dir: str @@ -314,8 +313,10 @@ def check_intermediate_file(cache_dir, pull_start_dates): def pull_quidel_data(params): """ - Pull the quidel test data. Decide whether to combine the newly - received data with stored historical records in ./cache + Pull the quidel test data. + + Decide whether to combine the newly received data with + stored historical records in ./cache Parameters: params: dict @@ -372,7 +373,8 @@ def pull_quidel_data(params): def check_export_end_date(input_export_end_dates, _end_date, END_FROM_TODAY_MINUS): """ - Update the export_end_date according to the data received + Update the export_end_date according to the data received. + By default, set the export end date to be the last pulling date - 5 days (END_FROM_TODAY_MINUS = 5). Otherwise, use the required date if it is earlier than the default one. @@ -405,6 +407,8 @@ def check_export_end_date(input_export_end_dates, _end_date, def check_export_start_date(export_start_dates, export_end_dates, EXPORT_DAY_RANGE): """ + Update the export_start_date according to the export_end_date and date range. + Update the export_start_date according to the export_end_date so that it could be export_end_date - EXPORT_DAY_RANGE @@ -439,7 +443,7 @@ def check_export_start_date(export_start_dates, export_end_dates, def update_cache_file(dfs, _end_date, cache_dir): """ - Update cache file. Remove the old one, export the new one + Update cache file. Remove the old one, export the new one. Parameter: df: pd.DataFrame diff --git a/quidel/delphi_quidel/run.py b/quidel/delphi_quidel/run.py index 02bd48ed0..d68fb7ac2 100644 --- a/quidel/delphi_quidel/run.py +++ b/quidel/delphi_quidel/run.py @@ -22,6 +22,7 @@ from .handle_wip_sensor import add_prefix def run_module(): + """Run module for processing Quidel test data.""" params = read_params() cache_dir = params["cache_dir"] export_dir = params["export_dir"]