diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg index e43d90ade..b4df8bcc6 100644 --- a/dev/local/setup.cfg +++ b/dev/local/setup.cfg @@ -27,6 +27,8 @@ packages = delphi.epidata.acquisition.twtr delphi.epidata.acquisition.wiki delphi.epidata.client + delphi.epidata.common + delphi.epidata.common.covid_hosp delphi.epidata.server delphi.epidata.server.covidcast_issues_migration delphi.epidata.server.endpoints diff --git a/src/acquisition/covid_hosp/facility/database.py b/src/acquisition/covid_hosp/facility/database.py index 172f32dc4..0ca583642 100644 --- a/src/acquisition/covid_hosp/facility/database.py +++ b/src/acquisition/covid_hosp/facility/database.py @@ -1,213 +1,16 @@ # first party from delphi.epidata.acquisition.covid_hosp.common.database import Database as BaseDatabase -from delphi.epidata.acquisition.covid_hosp.common.database import Columndef -from delphi.epidata.acquisition.covid_hosp.common.utils import Utils from delphi.epidata.acquisition.covid_hosp.facility.network import Network +from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething class Database(BaseDatabase): - TABLE_NAME = 'covid_hosp_facility' - KEY_COLS = ['hospital_pk', 'collection_week'] - AGGREGATE_KEY_COLS = ['address', 'ccn', 'city', 'fips_code', 'geocoded_hospital_address', 'hhs_ids', 'hospital_name', 'hospital_pk', 'hospital_subtype', 'is_metro_micro', 'state', 'zip'] - # These are 3-tuples of ( - # CSV header name, - # SQL db column name, - # data type - # ) for all the columns in the CSV file. - # Note that the corresponding database column names may be shorter - # due to constraints on the length of column names. See - # /src/ddl/covid_hosp.sql for more information. - ORDERED_CSV_COLUMNS = [ - Columndef('hospital_pk', 'hospital_pk', str), - Columndef('collection_week', 'collection_week', Utils.int_from_date), - Columndef('address', 'address', str), - Columndef('all_adult_hospital_beds_7_day_avg', 'all_adult_hospital_beds_7_day_avg', float), - Columndef('all_adult_hospital_beds_7_day_coverage', 'all_adult_hospital_beds_7_day_coverage', int), - Columndef('all_adult_hospital_beds_7_day_sum', 'all_adult_hospital_beds_7_day_sum', int), - Columndef('all_adult_hospital_inpatient_bed_occupied_7_day_avg', - 'all_adult_hospital_inpatient_bed_occupied_7_day_avg', float), - Columndef('all_adult_hospital_inpatient_bed_occupied_7_day_coverage', - 'all_adult_hospital_inpatient_bed_occupied_7_day_coverage', int), - Columndef('all_adult_hospital_inpatient_bed_occupied_7_day_sum', - 'all_adult_hospital_inpatient_bed_occupied_7_day_sum', int), - Columndef('all_adult_hospital_inpatient_beds_7_day_avg', 'all_adult_hospital_inpatient_beds_7_day_avg', - float), - Columndef('all_adult_hospital_inpatient_beds_7_day_coverage', - 'all_adult_hospital_inpatient_beds_7_day_coverage', int), - Columndef('all_adult_hospital_inpatient_beds_7_day_sum', 'all_adult_hospital_inpatient_beds_7_day_sum', - int), - Columndef('ccn', 'ccn', str), - Columndef('city', 'city', str), - Columndef('fips_code', 'fips_code', str), - Columndef('geocoded_hospital_address', 'geocoded_hospital_address', Utils.limited_geocode), - Columndef('hhs_ids', 'hhs_ids', str), - Columndef('hospital_name', 'hospital_name', str), - Columndef('hospital_subtype', 'hospital_subtype', str), - Columndef('icu_beds_used_7_day_avg', 'icu_beds_used_7_day_avg', float), - Columndef('icu_beds_used_7_day_coverage', 'icu_beds_used_7_day_coverage', int), - Columndef('icu_beds_used_7_day_sum', 'icu_beds_used_7_day_sum', int), - Columndef('icu_patients_confirmed_influenza_7_day_avg', 'icu_patients_confirmed_influenza_7_day_avg', - float), - Columndef('icu_patients_confirmed_influenza_7_day_coverage', - 'icu_patients_confirmed_influenza_7_day_coverage', int), - Columndef('icu_patients_confirmed_influenza_7_day_sum', 'icu_patients_confirmed_influenza_7_day_sum', - int), - Columndef('inpatient_beds_7_day_avg', 'inpatient_beds_7_day_avg', float), - Columndef('inpatient_beds_7_day_coverage', 'inpatient_beds_7_day_coverage', int), - Columndef('inpatient_beds_7_day_sum', 'inpatient_beds_7_day_sum', int), - Columndef('inpatient_beds_used_7_day_avg', 'inpatient_beds_used_7_day_avg', float), - Columndef('inpatient_beds_used_7_day_coverage', 'inpatient_beds_used_7_day_coverage', int), - Columndef('inpatient_beds_used_7_day_sum', 'inpatient_beds_used_7_day_sum', int), - Columndef('is_corrected', 'is_corrected', Utils.parse_bool), - Columndef('is_metro_micro', 'is_metro_micro', Utils.parse_bool), - Columndef('previous_day_admission_adult_covid_confirmed_18-19_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_18_19_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_20-29_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_20_29_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_30-39_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_30_39_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_40-49_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_40_49_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_50-59_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_50_59_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_60-69_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_60_69_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_70-79_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_70_79_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_7_day_coverage', - 'previous_day_admission_adult_covid_confirmed_7_day_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_80+_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_80plus_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_confirmed_unknown_7_day_sum', - 'previous_day_admission_adult_covid_confirmed_unknown_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_18-19_7_day_sum', - 'previous_day_admission_adult_covid_suspected_18_19_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_20-29_7_day_sum', - 'previous_day_admission_adult_covid_suspected_20_29_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_30-39_7_day_sum', - 'previous_day_admission_adult_covid_suspected_30_39_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_40-49_7_day_sum', - 'previous_day_admission_adult_covid_suspected_40_49_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_50-59_7_day_sum', - 'previous_day_admission_adult_covid_suspected_50_59_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_60-69_7_day_sum', - 'previous_day_admission_adult_covid_suspected_60_69_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_70-79_7_day_sum', - 'previous_day_admission_adult_covid_suspected_70_79_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_7_day_coverage', - 'previous_day_admission_adult_covid_suspected_7_day_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_7_day_sum', - 'previous_day_admission_adult_covid_suspected_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_80+_7_day_sum', - 'previous_day_admission_adult_covid_suspected_80plus_7_day_sum', int), - Columndef('previous_day_admission_adult_covid_suspected_unknown_7_day_sum', - 'previous_day_admission_adult_covid_suspected_unknown_7_day_sum', int), - Columndef('previous_day_admission_influenza_confirmed_7_day_sum', - 'previous_day_admission_influenza_confirmed_7_day_sum', int), - Columndef('previous_day_admission_pediatric_covid_confirmed_7_day_coverage', - 'previous_day_admission_pediatric_covid_confirmed_7_day_coverage', int), - Columndef('previous_day_admission_pediatric_covid_confirmed_7_day_sum', - 'previous_day_admission_pediatric_covid_confirmed_7_day_sum', int), - Columndef('previous_day_admission_pediatric_covid_suspected_7_day_coverage', - 'previous_day_admission_pediatric_covid_suspected_7_day_coverage', int), - Columndef('previous_day_admission_pediatric_covid_suspected_7_day_sum', - 'previous_day_admission_pediatric_covid_suspected_7_day_sum', int), - Columndef('previous_day_covid_ED_visits_7_day_sum', 'previous_day_covid_ed_visits_7_day_sum', int), - Columndef('previous_day_total_ED_visits_7_day_sum', 'previous_day_total_ed_visits_7_day_sum', int), - Columndef('previous_week_patients_covid_vaccinated_doses_all_7_day', - 'previous_week_patients_covid_vaccinated_doses_all_7_day', int), - Columndef('previous_week_patients_covid_vaccinated_doses_all_7_day_sum', - 'previous_week_patients_covid_vaccinated_doses_all_7_day_sum', int), - Columndef('previous_week_patients_covid_vaccinated_doses_one_7_day', - 'previous_week_patients_covid_vaccinated_doses_one_7_day', int), - Columndef('previous_week_patients_covid_vaccinated_doses_one_7_day_sum', - 'previous_week_patients_covid_vaccinated_doses_one_7_day_sum', int), - Columndef('previous_week_personnel_covid_vaccinated_doses_administered_7_day', - 'previous_week_personnel_covid_vaccd_doses_administered_7_day', int), - Columndef('previous_week_personnel_covid_vaccinated_doses_administered_7_day_sum', - 'previous_week_personnel_covid_vaccd_doses_administered_7_day_sum', int), - Columndef('staffed_adult_icu_bed_occupancy_7_day_avg', 'staffed_adult_icu_bed_occupancy_7_day_avg', - float), - Columndef('staffed_adult_icu_bed_occupancy_7_day_coverage', - 'staffed_adult_icu_bed_occupancy_7_day_coverage', int), - Columndef('staffed_adult_icu_bed_occupancy_7_day_sum', 'staffed_adult_icu_bed_occupancy_7_day_sum', - int), - Columndef('staffed_icu_adult_patients_confirmed_and_suspected_covid_7_day_avg', - 'staffed_icu_adult_patients_confirmed_suspected_covid_7d_avg', float), - Columndef('staffed_icu_adult_patients_confirmed_and_suspected_covid_7_day_coverage', - 'staffed_icu_adult_patients_confirmed_suspected_covid_7d_cov', int), - Columndef('staffed_icu_adult_patients_confirmed_and_suspected_covid_7_day_sum', - 'staffed_icu_adult_patients_confirmed_suspected_covid_7d_sum', int), - Columndef('staffed_icu_adult_patients_confirmed_covid_7_day_avg', - 'staffed_icu_adult_patients_confirmed_covid_7_day_avg', float), - Columndef('staffed_icu_adult_patients_confirmed_covid_7_day_coverage', - 'staffed_icu_adult_patients_confirmed_covid_7_day_coverage', int), - Columndef('staffed_icu_adult_patients_confirmed_covid_7_day_sum', - 'staffed_icu_adult_patients_confirmed_covid_7_day_sum', int), - Columndef('state', 'state', str), - Columndef('total_adult_patients_hospitalized_confirmed_and_suspected_covid_7_day_avg', - 'total_adult_patients_hosp_confirmed_suspected_covid_7d_avg', float), - Columndef('total_adult_patients_hospitalized_confirmed_and_suspected_covid_7_day_coverage', - 'total_adult_patients_hosp_confirmed_suspected_covid_7d_cov', int), - Columndef('total_adult_patients_hospitalized_confirmed_and_suspected_covid_7_day_sum', - 'total_adult_patients_hosp_confirmed_suspected_covid_7d_sum', int), - Columndef('total_adult_patients_hospitalized_confirmed_covid_7_day_avg', - 'total_adult_patients_hospitalized_confirmed_covid_7_day_avg', float), - Columndef('total_adult_patients_hospitalized_confirmed_covid_7_day_coverage', - 'total_adult_patients_hospitalized_confirmed_covid_7_day_coverage', int), - Columndef('total_adult_patients_hospitalized_confirmed_covid_7_day_sum', - 'total_adult_patients_hospitalized_confirmed_covid_7_day_sum', int), - Columndef('total_beds_7_day_avg', 'total_beds_7_day_avg', float), - Columndef('total_beds_7_day_coverage', 'total_beds_7_day_coverage', int), - Columndef('total_beds_7_day_sum', 'total_beds_7_day_sum', int), - Columndef('total_icu_beds_7_day_avg', 'total_icu_beds_7_day_avg', float), - Columndef('total_icu_beds_7_day_coverage', 'total_icu_beds_7_day_coverage', int), - Columndef('total_icu_beds_7_day_sum', 'total_icu_beds_7_day_sum', int), - Columndef('total_patients_hospitalized_confirmed_influenza_7_day_avg', - 'total_patients_hospitalized_confirmed_influenza_7_day_avg', float), - Columndef('total_patients_hospitalized_confirmed_influenza_7_day_coverage', - 'total_patients_hospitalized_confirmed_influenza_7_day_coverage', int), - Columndef('total_patients_hospitalized_confirmed_influenza_7_day_sum', - 'total_patients_hospitalized_confirmed_influenza_7_day_sum', int), - Columndef('total_patients_hospitalized_confirmed_influenza_and_covid_7_day_avg', - 'total_patients_hosp_confirmed_influenza_and_covid_7d_avg', float), - Columndef('total_patients_hospitalized_confirmed_influenza_and_covid_7_day_coverage', - 'total_patients_hosp_confirmed_influenza_and_covid_7d_cov', int), - Columndef('total_patients_hospitalized_confirmed_influenza_and_covid_7_day_sum', - 'total_patients_hosp_confirmed_influenza_and_covid_7d_sum', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_and_suspected_covid_7_day_avg', - 'total_pediatric_patients_hosp_confirmed_suspected_covid_7d_avg', float), - Columndef('total_pediatric_patients_hospitalized_confirmed_and_suspected_covid_7_day_coverage', - 'total_pediatric_patients_hosp_confirmed_suspected_covid_7d_cov', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_and_suspected_covid_7_day_sum', - 'total_pediatric_patients_hosp_confirmed_suspected_covid_7d_sum', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_covid_7_day_avg', - 'total_pediatric_patients_hospitalized_confirmed_covid_7_day_avg', float), - Columndef('total_pediatric_patients_hospitalized_confirmed_covid_7_day_coverage', - 'total_pediatric_patients_hosp_confirmed_covid_7d_cov', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_covid_7_day_sum', - 'total_pediatric_patients_hospitalized_confirmed_covid_7_day_sum', int), - Columndef('total_personnel_covid_vaccinated_doses_all_7_day', - 'total_personnel_covid_vaccinated_doses_all_7_day', int), - Columndef('total_personnel_covid_vaccinated_doses_all_7_day_sum', - 'total_personnel_covid_vaccinated_doses_all_7_day_sum', int), - Columndef('total_personnel_covid_vaccinated_doses_none_7_day', - 'total_personnel_covid_vaccinated_doses_none_7_day', int), - Columndef('total_personnel_covid_vaccinated_doses_none_7_day_sum', - 'total_personnel_covid_vaccinated_doses_none_7_day_sum', int), - Columndef('total_personnel_covid_vaccinated_doses_one_7_day', - 'total_personnel_covid_vaccinated_doses_one_7_day', int), - Columndef('total_personnel_covid_vaccinated_doses_one_7_day_sum', - 'total_personnel_covid_vaccinated_doses_one_7_day_sum', int), - Columndef('total_staffed_adult_icu_beds_7_day_avg', 'total_staffed_adult_icu_beds_7_day_avg', float), - Columndef('total_staffed_adult_icu_beds_7_day_coverage', 'total_staffed_adult_icu_beds_7_day_coverage', - int), - Columndef('total_staffed_adult_icu_beds_7_day_sum', 'total_staffed_adult_icu_beds_7_day_sum', int), - Columndef('zip', 'zip', str), - ] + chs = CovidHospSomething() + TABLE_NAME = chs.get_ds_table_name('covid_hosp_facility') + KEY_COLS = chs.get_ds_key_cols('covid_hosp_facility') + AGGREGATE_KEY_COLS = chs.get_ds_aggregate_key_cols('covid_hosp_facility') + ORDERED_CSV_COLUMNS = chs.get_ds_ordered_csv_cols('covid_hosp_facility') def __init__(self, *args, **kwargs): super().__init__( diff --git a/src/acquisition/covid_hosp/state_daily/database.py b/src/acquisition/covid_hosp/state_daily/database.py index 6a8228994..1bf7f872e 100644 --- a/src/acquisition/covid_hosp/state_daily/database.py +++ b/src/acquisition/covid_hosp/state_daily/database.py @@ -1,223 +1,18 @@ # first party from delphi.epidata.acquisition.covid_hosp.common.database import Database as BaseDatabase from delphi.epidata.acquisition.covid_hosp.common.database import Columndef -from delphi.epidata.acquisition.covid_hosp.common.utils import Utils from delphi.epidata.acquisition.covid_hosp.state_daily.network import Network +from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething class Database(BaseDatabase): # note we share a database with state_timeseries - TABLE_NAME = 'covid_hosp_state_timeseries' - KEY_COLS = ['state', 'reporting_cutoff_start'] - # These are 3-tuples of (CSV header name, SQL db column name, data type) for - # all the columns in the CSV file. - # Note that the corresponding database column names may be shorter - # due to constraints on the length of column names. See - # /src/ddl/covid_hosp.sql for more information. - # Additionally, all column names below are shared with state_timeseries, - # except for reporting_cutoff_start (here) and date (there). If you need - # to update a column name, do it in both places. - ORDERED_CSV_COLUMNS = [ - Columndef('state', 'state', str), - Columndef('reporting_cutoff_start', 'date', Utils.int_from_date), - Columndef('adult_icu_bed_covid_utilization', 'adult_icu_bed_covid_utilization', float), - Columndef('adult_icu_bed_covid_utilization_coverage', 'adult_icu_bed_covid_utilization_coverage', int), - Columndef('adult_icu_bed_covid_utilization_denominator', 'adult_icu_bed_covid_utilization_denominator', - int), - Columndef('adult_icu_bed_covid_utilization_numerator', 'adult_icu_bed_covid_utilization_numerator', - int), - Columndef('adult_icu_bed_utilization', 'adult_icu_bed_utilization', float), - Columndef('adult_icu_bed_utilization_coverage', 'adult_icu_bed_utilization_coverage', int), - Columndef('adult_icu_bed_utilization_denominator', 'adult_icu_bed_utilization_denominator', int), - Columndef('adult_icu_bed_utilization_numerator', 'adult_icu_bed_utilization_numerator', int), - Columndef('critical_staffing_shortage_anticipated_within_week_no', - 'critical_staffing_shortage_anticipated_within_week_no', int), - Columndef('critical_staffing_shortage_anticipated_within_week_not_reported', - 'critical_staffing_shortage_anticipated_within_week_not_reported', int), - Columndef('critical_staffing_shortage_anticipated_within_week_yes', - 'critical_staffing_shortage_anticipated_within_week_yes', int), - Columndef('critical_staffing_shortage_today_no', 'critical_staffing_shortage_today_no', int), - Columndef('critical_staffing_shortage_today_not_reported', - 'critical_staffing_shortage_today_not_reported', int), - Columndef('critical_staffing_shortage_today_yes', 'critical_staffing_shortage_today_yes', int), - Columndef('deaths_covid', 'deaths_covid', int), - Columndef('deaths_covid_coverage', 'deaths_covid_coverage', int), - Columndef('geocoded_state', 'geocoded_state', str), - Columndef('hospital_onset_covid', 'hospital_onset_covid', int), - Columndef('hospital_onset_covid_coverage', 'hospital_onset_covid_coverage', int), - Columndef('icu_patients_confirmed_influenza', 'icu_patients_confirmed_influenza', int), - Columndef('icu_patients_confirmed_influenza_coverage', 'icu_patients_confirmed_influenza_coverage', - int), - Columndef('inpatient_bed_covid_utilization', 'inpatient_bed_covid_utilization', float), - Columndef('inpatient_bed_covid_utilization_coverage', 'inpatient_bed_covid_utilization_coverage', int), - Columndef('inpatient_bed_covid_utilization_denominator', 'inpatient_bed_covid_utilization_denominator', - int), - Columndef('inpatient_bed_covid_utilization_numerator', 'inpatient_bed_covid_utilization_numerator', - int), - Columndef('inpatient_beds', 'inpatient_beds', int), - Columndef('inpatient_beds_coverage', 'inpatient_beds_coverage', int), - Columndef('inpatient_beds_used', 'inpatient_beds_used', int), - Columndef('inpatient_beds_used_coverage', 'inpatient_beds_used_coverage', int), - Columndef('inpatient_beds_used_covid', 'inpatient_beds_used_covid', int), - Columndef('inpatient_beds_used_covid_coverage', 'inpatient_beds_used_covid_coverage', int), - Columndef('inpatient_beds_utilization', 'inpatient_beds_utilization', float), - Columndef('inpatient_beds_utilization_coverage', 'inpatient_beds_utilization_coverage', int), - Columndef('inpatient_beds_utilization_denominator', 'inpatient_beds_utilization_denominator', int), - Columndef('inpatient_beds_utilization_numerator', 'inpatient_beds_utilization_numerator', int), - Columndef('on_hand_supply_therapeutic_a_casirivimab_imdevimab_courses', - 'on_hand_supply_therapeutic_a_casirivimab_imdevimab_courses', int), - Columndef('on_hand_supply_therapeutic_b_bamlanivimab_courses', - 'on_hand_supply_therapeutic_b_bamlanivimab_courses', int), - Columndef('on_hand_supply_therapeutic_c_bamlanivimab_etesevimab_courses', - 'on_hand_supply_therapeutic_c_bamlanivimab_etesevimab_courses', int), - Columndef('percent_of_inpatients_with_covid', 'percent_of_inpatients_with_covid', float), - Columndef('percent_of_inpatients_with_covid_coverage', 'percent_of_inpatients_with_covid_coverage', - int), - Columndef('percent_of_inpatients_with_covid_denominator', - 'percent_of_inpatients_with_covid_denominator', int), - Columndef('percent_of_inpatients_with_covid_numerator', 'percent_of_inpatients_with_covid_numerator', - int), - Columndef('previous_day_admission_adult_covid_confirmed', - 'previous_day_admission_adult_covid_confirmed', int), - Columndef('previous_day_admission_adult_covid_confirmed_18-19', - 'previous_day_admission_adult_covid_confirmed_18_19', int), - Columndef('previous_day_admission_adult_covid_confirmed_18-19_coverage', - 'previous_day_admission_adult_covid_confirmed_18_19_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_20-29', - 'previous_day_admission_adult_covid_confirmed_20_29', int), - Columndef('previous_day_admission_adult_covid_confirmed_20-29_coverage', - 'previous_day_admission_adult_covid_confirmed_20_29_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_30-39', - 'previous_day_admission_adult_covid_confirmed_30_39', int), - Columndef('previous_day_admission_adult_covid_confirmed_30-39_coverage', - 'previous_day_admission_adult_covid_confirmed_30_39_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_40-49', - 'previous_day_admission_adult_covid_confirmed_40_49', int), - Columndef('previous_day_admission_adult_covid_confirmed_40-49_coverage', - 'previous_day_admission_adult_covid_confirmed_40_49_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_50-59', - 'previous_day_admission_adult_covid_confirmed_50_59', int), - Columndef('previous_day_admission_adult_covid_confirmed_50-59_coverage', - 'previous_day_admission_adult_covid_confirmed_50_59_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_60-69', - 'previous_day_admission_adult_covid_confirmed_60_69', int), - Columndef('previous_day_admission_adult_covid_confirmed_60-69_coverage', - 'previous_day_admission_adult_covid_confirmed_60_69_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_70-79', - 'previous_day_admission_adult_covid_confirmed_70_79', int), - Columndef('previous_day_admission_adult_covid_confirmed_70-79_coverage', - 'previous_day_admission_adult_covid_confirmed_70_79_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_80+', - 'previous_day_admission_adult_covid_confirmed_80plus', int), - Columndef('previous_day_admission_adult_covid_confirmed_80+_coverage', - 'previous_day_admission_adult_covid_confirmed_80plus_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_coverage', - 'previous_day_admission_adult_covid_confirmed_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_unknown', - 'previous_day_admission_adult_covid_confirmed_unknown', int), - Columndef('previous_day_admission_adult_covid_confirmed_unknown_coverage', - 'previous_day_admission_adult_covid_confirmed_unknown_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected', - 'previous_day_admission_adult_covid_suspected', int), - Columndef('previous_day_admission_adult_covid_suspected_18-19', - 'previous_day_admission_adult_covid_suspected_18_19', int), - Columndef('previous_day_admission_adult_covid_suspected_18-19_coverage', - 'previous_day_admission_adult_covid_suspected_18_19_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_20-29', - 'previous_day_admission_adult_covid_suspected_20_29', int), - Columndef('previous_day_admission_adult_covid_suspected_20-29_coverage', - 'previous_day_admission_adult_covid_suspected_20_29_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_30-39', - 'previous_day_admission_adult_covid_suspected_30_39', int), - Columndef('previous_day_admission_adult_covid_suspected_30-39_coverage', - 'previous_day_admission_adult_covid_suspected_30_39_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_40-49', - 'previous_day_admission_adult_covid_suspected_40_49', int), - Columndef('previous_day_admission_adult_covid_suspected_40-49_coverage', - 'previous_day_admission_adult_covid_suspected_40_49_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_50-59', - 'previous_day_admission_adult_covid_suspected_50_59', int), - Columndef('previous_day_admission_adult_covid_suspected_50-59_coverage', - 'previous_day_admission_adult_covid_suspected_50_59_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_60_69', #this is correct; csv header is irregular - 'previous_day_admission_adult_covid_suspected_60_69', int), - Columndef('previous_day_admission_adult_covid_suspected_60-69_coverage', - 'previous_day_admission_adult_covid_suspected_60_69_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_70-79', - 'previous_day_admission_adult_covid_suspected_70_79', int), - Columndef('previous_day_admission_adult_covid_suspected_70-79_coverage', - 'previous_day_admission_adult_covid_suspected_70_79_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_80', - 'previous_day_admission_adult_covid_suspected_80plus', int), - Columndef('previous_day_admission_adult_covid_suspected_80+_coverage', - 'previous_day_admission_adult_covid_suspected_80plus_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_coverage', - 'previous_day_admission_adult_covid_suspected_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_unknown', - 'previous_day_admission_adult_covid_suspected_unknown', int), - Columndef('previous_day_admission_adult_covid_suspected_unknown_coverage', - 'previous_day_admission_adult_covid_suspected_unknown_coverage', int), - Columndef('previous_day_admission_influenza_confirmed', 'previous_day_admission_influenza_confirmed', - int), - Columndef('previous_day_admission_influenza_confirmed_coverage', - 'previous_day_admission_influenza_confirmed_coverage', int), - Columndef('previous_day_admission_pediatric_covid_confirmed', - 'previous_day_admission_pediatric_covid_confirmed', int), - Columndef('previous_day_admission_pediatric_covid_confirmed_coverage', - 'previous_day_admission_pediatric_covid_confirmed_coverage', int), - Columndef('previous_day_admission_pediatric_covid_suspected', - 'previous_day_admission_pediatric_covid_suspected', int), - Columndef('previous_day_admission_pediatric_covid_suspected_coverage', - 'previous_day_admission_pediatric_covid_suspected_coverage', int), - Columndef('previous_day_deaths_covid_and_influenza', 'previous_day_deaths_covid_and_influenza', int), - Columndef('previous_day_deaths_covid_and_influenza_coverage', - 'previous_day_deaths_covid_and_influenza_coverage', int), - Columndef('previous_day_deaths_influenza', 'previous_day_deaths_influenza', int), - Columndef('previous_day_deaths_influenza_coverage', 'previous_day_deaths_influenza_coverage', int), - Columndef('previous_week_therapeutic_a_casirivimab_imdevimab_courses_used', - 'previous_week_therapeutic_a_casirivimab_imdevimab_courses_used', int), - Columndef('previous_week_therapeutic_b_bamlanivimab_courses_used', - 'previous_week_therapeutic_b_bamlanivimab_courses_used', int), - Columndef('previous_week_therapeutic_c_bamlanivimab_etesevimab_courses_used', - 'previous_week_therapeutic_c_bamlanivimab_etesevimab_courses_used', int), - Columndef('staffed_adult_icu_bed_occupancy', 'staffed_adult_icu_bed_occupancy', int), - Columndef('staffed_adult_icu_bed_occupancy_coverage', 'staffed_adult_icu_bed_occupancy_coverage', int), - Columndef('staffed_icu_adult_patients_confirmed_and_suspected_covid', - 'staffed_icu_adult_patients_confirmed_suspected_covid', int), - Columndef('staffed_icu_adult_patients_confirmed_and_suspected_covid_coverage', - 'staffed_icu_adult_patients_confirmed_suspected_covid_coverage', int), - Columndef('staffed_icu_adult_patients_confirmed_covid', 'staffed_icu_adult_patients_confirmed_covid', - int), - Columndef('staffed_icu_adult_patients_confirmed_covid_coverage', - 'staffed_icu_adult_patients_confirmed_covid_coverage', int), - Columndef('total_adult_patients_hospitalized_confirmed_and_suspected_covid', - 'total_adult_patients_hosp_confirmed_suspected_covid', int), - Columndef('total_adult_patients_hospitalized_confirmed_and_suspected_covid_coverage', - 'total_adult_patients_hosp_confirmed_suspected_covid_coverage', int), - Columndef('total_adult_patients_hospitalized_confirmed_covid', - 'total_adult_patients_hosp_confirmed_covid', int), - Columndef('total_adult_patients_hospitalized_confirmed_covid_coverage', - 'total_adult_patients_hosp_confirmed_covid_coverage', int), - Columndef('total_patients_hospitalized_confirmed_influenza', - 'total_patients_hospitalized_confirmed_influenza', int), - Columndef('total_patients_hospitalized_confirmed_influenza_coverage', - 'total_patients_hospitalized_confirmed_influenza_coverage', int), - Columndef('total_patients_hospitalized_confirmed_influenza_covid', - 'total_patients_hospitalized_confirmed_influenza_covid', int), - Columndef('total_patients_hospitalized_confirmed_influenza_covid_coverage', - 'total_patients_hospitalized_confirmed_influenza_covid_coverage', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_and_suspected_covid', - 'total_pediatric_patients_hosp_confirmed_suspected_covid', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_and_suspected_covid_coverage', - 'total_pediatric_patients_hosp_confirmed_suspected_covid_coverage', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_covid', - 'total_pediatric_patients_hosp_confirmed_covid', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_covid_coverage', - 'total_pediatric_patients_hosp_confirmed_covid_coverage', int), - Columndef('total_staffed_adult_icu_beds', 'total_staffed_adult_icu_beds', int), - Columndef('total_staffed_adult_icu_beds_coverage', 'total_staffed_adult_icu_beds_coverage', int), - ] + + chs = CovidHospSomething() + TABLE_NAME = chs.get_ds_table_name('state_daily') + KEY_COLS = chs.get_ds_key_cols('state_daily') + ORDERED_CSV_COLUMNS = chs.get_ds_ordered_csv_cols('state_daily') def __init__(self, *args, **kwargs): super().__init__( diff --git a/src/acquisition/covid_hosp/state_timeseries/database.py b/src/acquisition/covid_hosp/state_timeseries/database.py index 348d9fc0b..1a586babb 100644 --- a/src/acquisition/covid_hosp/state_timeseries/database.py +++ b/src/acquisition/covid_hosp/state_timeseries/database.py @@ -1,222 +1,16 @@ # first party from delphi.epidata.acquisition.covid_hosp.common.database import Database as BaseDatabase from delphi.epidata.acquisition.covid_hosp.common.database import Columndef -from delphi.epidata.acquisition.covid_hosp.common.utils import Utils from delphi.epidata.acquisition.covid_hosp.state_timeseries.network import Network +from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething class Database(BaseDatabase): - TABLE_NAME = 'covid_hosp_state_timeseries' - KEY_COLS = ['state', 'date'] - # These are 3-tuples of (CSV header name, SQL db column name, data type) for - # all the columns in the CSV file. - # Note that the corresponding database column names may be shorter - # due to constraints on the length of column names. See - # /src/ddl/covid_hosp.sql for more information. - # Additionally, all column names below are shared with state_daily, - # except for reporting_cutoff_start (there) and date (here). If you need - # to update a column name, do it in both places. - ORDERED_CSV_COLUMNS = [ - Columndef('state', 'state', str), - Columndef('date', 'date', Utils.int_from_date), - Columndef('adult_icu_bed_covid_utilization', 'adult_icu_bed_covid_utilization', float), - Columndef('adult_icu_bed_covid_utilization_coverage', 'adult_icu_bed_covid_utilization_coverage', int), - Columndef('adult_icu_bed_covid_utilization_denominator', 'adult_icu_bed_covid_utilization_denominator', - int), - Columndef('adult_icu_bed_covid_utilization_numerator', 'adult_icu_bed_covid_utilization_numerator', - int), - Columndef('adult_icu_bed_utilization', 'adult_icu_bed_utilization', float), - Columndef('adult_icu_bed_utilization_coverage', 'adult_icu_bed_utilization_coverage', int), - Columndef('adult_icu_bed_utilization_denominator', 'adult_icu_bed_utilization_denominator', int), - Columndef('adult_icu_bed_utilization_numerator', 'adult_icu_bed_utilization_numerator', int), - Columndef('critical_staffing_shortage_anticipated_within_week_no', - 'critical_staffing_shortage_anticipated_within_week_no', int), - Columndef('critical_staffing_shortage_anticipated_within_week_not_reported', - 'critical_staffing_shortage_anticipated_within_week_not_reported', int), - Columndef('critical_staffing_shortage_anticipated_within_week_yes', - 'critical_staffing_shortage_anticipated_within_week_yes', int), - Columndef('critical_staffing_shortage_today_no', 'critical_staffing_shortage_today_no', int), - Columndef('critical_staffing_shortage_today_not_reported', - 'critical_staffing_shortage_today_not_reported', int), - Columndef('critical_staffing_shortage_today_yes', 'critical_staffing_shortage_today_yes', int), - Columndef('deaths_covid', 'deaths_covid', int), - Columndef('deaths_covid_coverage', 'deaths_covid_coverage', int), - Columndef('geocoded_state', 'geocoded_state', str), - Columndef('hospital_onset_covid', 'hospital_onset_covid', int), - Columndef('hospital_onset_covid_coverage', 'hospital_onset_covid_coverage', int), - Columndef('icu_patients_confirmed_influenza', 'icu_patients_confirmed_influenza', int), - Columndef('icu_patients_confirmed_influenza_coverage', 'icu_patients_confirmed_influenza_coverage', - int), - Columndef('inpatient_bed_covid_utilization', 'inpatient_bed_covid_utilization', float), - Columndef('inpatient_bed_covid_utilization_coverage', 'inpatient_bed_covid_utilization_coverage', int), - Columndef('inpatient_bed_covid_utilization_denominator', 'inpatient_bed_covid_utilization_denominator', - int), - Columndef('inpatient_bed_covid_utilization_numerator', 'inpatient_bed_covid_utilization_numerator', - int), - Columndef('inpatient_beds', 'inpatient_beds', int), - Columndef('inpatient_beds_coverage', 'inpatient_beds_coverage', int), - Columndef('inpatient_beds_used', 'inpatient_beds_used', int), - Columndef('inpatient_beds_used_coverage', 'inpatient_beds_used_coverage', int), - Columndef('inpatient_beds_used_covid', 'inpatient_beds_used_covid', int), - Columndef('inpatient_beds_used_covid_coverage', 'inpatient_beds_used_covid_coverage', int), - Columndef('inpatient_beds_utilization', 'inpatient_beds_utilization', float), - Columndef('inpatient_beds_utilization_coverage', 'inpatient_beds_utilization_coverage', int), - Columndef('inpatient_beds_utilization_denominator', 'inpatient_beds_utilization_denominator', int), - Columndef('inpatient_beds_utilization_numerator', 'inpatient_beds_utilization_numerator', int), - Columndef('on_hand_supply_therapeutic_a_casirivimab_imdevimab_courses', - 'on_hand_supply_therapeutic_a_casirivimab_imdevimab_courses', int), - Columndef('on_hand_supply_therapeutic_b_bamlanivimab_courses', - 'on_hand_supply_therapeutic_b_bamlanivimab_courses', int), - Columndef('on_hand_supply_therapeutic_c_bamlanivimab_etesevimab_courses', - 'on_hand_supply_therapeutic_c_bamlanivimab_etesevimab_courses', int), - Columndef('percent_of_inpatients_with_covid', 'percent_of_inpatients_with_covid', float), - Columndef('percent_of_inpatients_with_covid_coverage', 'percent_of_inpatients_with_covid_coverage', - int), - Columndef('percent_of_inpatients_with_covid_denominator', - 'percent_of_inpatients_with_covid_denominator', int), - Columndef('percent_of_inpatients_with_covid_numerator', 'percent_of_inpatients_with_covid_numerator', - int), - Columndef('previous_day_admission_adult_covid_confirmed', - 'previous_day_admission_adult_covid_confirmed', int), - Columndef('previous_day_admission_adult_covid_confirmed_18-19', - 'previous_day_admission_adult_covid_confirmed_18_19', int), - Columndef('previous_day_admission_adult_covid_confirmed_18-19_coverage', - 'previous_day_admission_adult_covid_confirmed_18_19_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_20-29', - 'previous_day_admission_adult_covid_confirmed_20_29', int), - Columndef('previous_day_admission_adult_covid_confirmed_20-29_coverage', - 'previous_day_admission_adult_covid_confirmed_20_29_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_30-39', - 'previous_day_admission_adult_covid_confirmed_30_39', int), - Columndef('previous_day_admission_adult_covid_confirmed_30-39_coverage', - 'previous_day_admission_adult_covid_confirmed_30_39_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_40-49', - 'previous_day_admission_adult_covid_confirmed_40_49', int), - Columndef('previous_day_admission_adult_covid_confirmed_40-49_coverage', - 'previous_day_admission_adult_covid_confirmed_40_49_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_50-59', - 'previous_day_admission_adult_covid_confirmed_50_59', int), - Columndef('previous_day_admission_adult_covid_confirmed_50-59_coverage', - 'previous_day_admission_adult_covid_confirmed_50_59_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_60-69', - 'previous_day_admission_adult_covid_confirmed_60_69', int), - Columndef('previous_day_admission_adult_covid_confirmed_60-69_coverage', - 'previous_day_admission_adult_covid_confirmed_60_69_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_70-79', - 'previous_day_admission_adult_covid_confirmed_70_79', int), - Columndef('previous_day_admission_adult_covid_confirmed_70-79_coverage', - 'previous_day_admission_adult_covid_confirmed_70_79_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_80+', - 'previous_day_admission_adult_covid_confirmed_80plus', int), - Columndef('previous_day_admission_adult_covid_confirmed_80+_coverage', - 'previous_day_admission_adult_covid_confirmed_80plus_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_coverage', - 'previous_day_admission_adult_covid_confirmed_coverage', int), - Columndef('previous_day_admission_adult_covid_confirmed_unknown', - 'previous_day_admission_adult_covid_confirmed_unknown', int), - Columndef('previous_day_admission_adult_covid_confirmed_unknown_coverage', - 'previous_day_admission_adult_covid_confirmed_unknown_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected', - 'previous_day_admission_adult_covid_suspected', int), - Columndef('previous_day_admission_adult_covid_suspected_18-19', - 'previous_day_admission_adult_covid_suspected_18_19', int), - Columndef('previous_day_admission_adult_covid_suspected_18-19_coverage', - 'previous_day_admission_adult_covid_suspected_18_19_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_20-29', - 'previous_day_admission_adult_covid_suspected_20_29', int), - Columndef('previous_day_admission_adult_covid_suspected_20-29_coverage', - 'previous_day_admission_adult_covid_suspected_20_29_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_30-39', - 'previous_day_admission_adult_covid_suspected_30_39', int), - Columndef('previous_day_admission_adult_covid_suspected_30-39_coverage', - 'previous_day_admission_adult_covid_suspected_30_39_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_40-49', - 'previous_day_admission_adult_covid_suspected_40_49', int), - Columndef('previous_day_admission_adult_covid_suspected_40-49_coverage', - 'previous_day_admission_adult_covid_suspected_40_49_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_50-59', - 'previous_day_admission_adult_covid_suspected_50_59', int), - Columndef('previous_day_admission_adult_covid_suspected_50-59_coverage', - 'previous_day_admission_adult_covid_suspected_50_59_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_60-69', - 'previous_day_admission_adult_covid_suspected_60_69', int), - Columndef('previous_day_admission_adult_covid_suspected_60-69_coverage', - 'previous_day_admission_adult_covid_suspected_60_69_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_70-79', - 'previous_day_admission_adult_covid_suspected_70_79', int), - Columndef('previous_day_admission_adult_covid_suspected_70-79_coverage', - 'previous_day_admission_adult_covid_suspected_70_79_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_80+', - 'previous_day_admission_adult_covid_suspected_80plus', int), - Columndef('previous_day_admission_adult_covid_suspected_80+_coverage', - 'previous_day_admission_adult_covid_suspected_80plus_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_coverage', - 'previous_day_admission_adult_covid_suspected_coverage', int), - Columndef('previous_day_admission_adult_covid_suspected_unknown', - 'previous_day_admission_adult_covid_suspected_unknown', int), - Columndef('previous_day_admission_adult_covid_suspected_unknown_coverage', - 'previous_day_admission_adult_covid_suspected_unknown_coverage', int), - Columndef('previous_day_admission_influenza_confirmed', 'previous_day_admission_influenza_confirmed', - int), - Columndef('previous_day_admission_influenza_confirmed_coverage', - 'previous_day_admission_influenza_confirmed_coverage', int), - Columndef('previous_day_admission_pediatric_covid_confirmed', - 'previous_day_admission_pediatric_covid_confirmed', int), - Columndef('previous_day_admission_pediatric_covid_confirmed_coverage', - 'previous_day_admission_pediatric_covid_confirmed_coverage', int), - Columndef('previous_day_admission_pediatric_covid_suspected', - 'previous_day_admission_pediatric_covid_suspected', int), - Columndef('previous_day_admission_pediatric_covid_suspected_coverage', - 'previous_day_admission_pediatric_covid_suspected_coverage', int), - Columndef('previous_day_deaths_covid_and_influenza', 'previous_day_deaths_covid_and_influenza', int), - Columndef('previous_day_deaths_covid_and_influenza_coverage', - 'previous_day_deaths_covid_and_influenza_coverage', int), - Columndef('previous_day_deaths_influenza', 'previous_day_deaths_influenza', int), - Columndef('previous_day_deaths_influenza_coverage', 'previous_day_deaths_influenza_coverage', int), - Columndef('previous_week_therapeutic_a_casirivimab_imdevimab_courses_used', - 'previous_week_therapeutic_a_casirivimab_imdevimab_courses_used', int), - Columndef('previous_week_therapeutic_b_bamlanivimab_courses_used', - 'previous_week_therapeutic_b_bamlanivimab_courses_used', int), - Columndef('previous_week_therapeutic_c_bamlanivimab_etesevimab_courses_used', - 'previous_week_therapeutic_c_bamlanivimab_etesevimab_courses_used', int), - Columndef('staffed_adult_icu_bed_occupancy', 'staffed_adult_icu_bed_occupancy', int), - Columndef('staffed_adult_icu_bed_occupancy_coverage', 'staffed_adult_icu_bed_occupancy_coverage', int), - Columndef('staffed_icu_adult_patients_confirmed_and_suspected_covid', - 'staffed_icu_adult_patients_confirmed_suspected_covid', int), - Columndef('staffed_icu_adult_patients_confirmed_and_suspected_covid_coverage', - 'staffed_icu_adult_patients_confirmed_suspected_covid_coverage', int), - Columndef('staffed_icu_adult_patients_confirmed_covid', 'staffed_icu_adult_patients_confirmed_covid', - int), - Columndef('staffed_icu_adult_patients_confirmed_covid_coverage', - 'staffed_icu_adult_patients_confirmed_covid_coverage', int), - Columndef('total_adult_patients_hospitalized_confirmed_and_suspected_covid', - 'total_adult_patients_hosp_confirmed_suspected_covid', int), - Columndef('total_adult_patients_hospitalized_confirmed_and_suspected_covid_coverage', - 'total_adult_patients_hosp_confirmed_suspected_covid_coverage', int), - Columndef('total_adult_patients_hospitalized_confirmed_covid', - 'total_adult_patients_hosp_confirmed_covid', int), - Columndef('total_adult_patients_hospitalized_confirmed_covid_coverage', - 'total_adult_patients_hosp_confirmed_covid_coverage', int), - Columndef('total_patients_hospitalized_confirmed_influenza', - 'total_patients_hospitalized_confirmed_influenza', int), - Columndef('total_patients_hospitalized_confirmed_influenza_coverage', - 'total_patients_hospitalized_confirmed_influenza_coverage', int), - Columndef('total_patients_hospitalized_confirmed_influenza_covid', - 'total_patients_hospitalized_confirmed_influenza_covid', int), - Columndef('total_patients_hospitalized_confirmed_influenza_covid_coverage', - 'total_patients_hospitalized_confirmed_influenza_covid_coverage', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_and_suspected_covid', - 'total_pediatric_patients_hosp_confirmed_suspected_covid', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_and_suspected_covid_coverage', - 'total_pediatric_patients_hosp_confirmed_suspected_covid_coverage', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_covid', - 'total_pediatric_patients_hosp_confirmed_covid', int), - Columndef('total_pediatric_patients_hospitalized_confirmed_covid_coverage', - 'total_pediatric_patients_hosp_confirmed_covid_coverage', int), - Columndef('total_staffed_adult_icu_beds', 'total_staffed_adult_icu_beds', int), - Columndef('total_staffed_adult_icu_beds_coverage', 'total_staffed_adult_icu_beds_coverage', int), - ] + chs = CovidHospSomething() + TABLE_NAME = chs.get_ds_table_name('state_timeseries') + KEY_COLS = chs.get_ds_key_cols('state_timeseries') + ORDERED_CSV_COLUMNS = chs.get_ds_ordered_csv_cols('state_timeseries') def __init__(self, *args, **kwargs): super().__init__( diff --git a/src/common/covid_hosp/covid_hosp_schema_io.py b/src/common/covid_hosp/covid_hosp_schema_io.py index 8989a4119..7f2ed0989 100644 --- a/src/common/covid_hosp/covid_hosp_schema_io.py +++ b/src/common/covid_hosp/covid_hosp_schema_io.py @@ -1,8 +1,10 @@ from datetime import datetime +from pathlib import Path import re import sys -# UNCOMMENT: from delphi.epidata.acquisition.covid_hosp.common.utils import Utils +from delphi.epidata.acquisition.covid_hosp.common.utils import Utils +from delphi.epidata.acquisition.covid_hosp.common.database import Columndef # ruamel preserves key ordering, comments, and some formatting for a "round trip" of a yaml file import-->export from ruamel.yaml.main import ( @@ -19,16 +21,14 @@ class CovidHospSomething: - YAML_FILENAME = 'covid_hosp_schemadefs.yaml' - PYTHON_TYPE_MAPPING = { 'int': int, 'float': float, 'str': str, - 'fixedstr', str, - 'bool': int, # UNCOMMENT: Utils.parse_bool, - 'intdate': int, # UNCOMMENT: Utils.int_from_date, - 'geocode': str, # UNCOMMENT: Utils.limited_geocode, + 'fixedstr': str, + 'bool': Utils.parse_bool, + 'intdate': Utils.int_from_date, + 'geocode': Utils.limited_geocode, } SQL_TYPE_MAPPING = { @@ -54,22 +54,22 @@ class CovidHospSomething: MYSQL_COL_NAME_PATTERN = re.compile(r'^[a-zA-Z0-9_]{3,64}$') - def __init__(self): - self.read_schemadefs() - + def __init__(self, yaml_filename=None): + if not yaml_filename: + # the default schema definitions yaml file is a member of the current package + # TODO: use `importlib_resources` instead of generic file Path stuff to load this, once we are past python 3.8 + yaml_filename = str(Path(__file__).parent.absolute()) + "/covid_hosp_schemadefs.yaml" + self.yaml_filename = yaml_filename - def read_schemadefs(self): - # TODO: put the yaml file inside the package structure and access it there, with something like: - # from importlib import resources - # import delphi.epidata.common.covid_hosp - # self.yaml_content = resources.read_text(delphi.epidata.common.covid_hosp, YAML_FILENAME) - with open(CovidHospSomething.YAML_FILENAME, 'r') as yaml_file: + with open(yaml_filename, 'r') as yaml_file: self.yaml_content = yaml_load(yaml_file, preserve_quotes=True) - return self.yaml_content - def write_schemadefs(self, filename=CovidHospSomething.YAML_FILENAME): - with open(filename, 'w') as yaml_file: + def write_schemadefs(self, yaml_filename=None): + if not yaml_filename: + yaml_filename = self.yaml_filename + + with open(yaml_filename, 'w') as yaml_file: # NOTE: `width` specification is to prevent dump from splitting long lines # TODO: consider `block_seq_indent=2` to make list under ORDERED_CSV_COLUMNS look a little better yaml_dump(self.yaml_content, yaml_file, width=200) @@ -93,7 +93,7 @@ def columns(self, ds_name): else: dtype = dtype_cplx col_width = None - yield {'name': name, 'sql_name': sql_name, 'dtype': dtype, 'col_width:' col_width, 'marshaller': CovidHospSomething.PYTHON_TYPE_MAPPING[dtype]} + yield {'name': name, 'sql_name': sql_name, 'dtype': dtype, 'col_width': col_width, 'marshaller': CovidHospSomething.PYTHON_TYPE_MAPPING[dtype]} def add_column(self, ds_name, col_name, dtype, sql_name=None, col_width=None): @@ -119,6 +119,32 @@ def add_column(self, ds_name, col_name, dtype, sql_name=None, col_width=None): self.dataset(ds_name)['ORDERED_CSV_COLUMNS'].append([dtype_cplx, col_name, sql_name]) + def get_ds_table_name(self, ds_name): + return self.dataset(ds_name)['TABLE_NAME'] + + + def get_ds_key_cols(self, ds_name): + return self.dataset(ds_name)['KEY_COLS'] + + + def get_ds_aggregate_key_cols(self, ds_name): + return self.dataset(ds_name).get('AGGREGATE_KEY_COLS', None) + + + def get_ds_ordered_csv_cols(self, ds_name): + return [ + Columndef( + # Original name for the column + column['name'], + # Updated name for the column; same name if it's not specified in the YAML files + column['sql_name'], + # Datatype for the column, ignoring SQL column lengths: e.g. fixedstr:1 -> fixedstr -> str + column['marshaller'] + ) + for column in self.columns(ds_name) + ] + + def detect_changes(self, ds_name): # TODO ds = self.dataset(ds_name) @@ -131,21 +157,22 @@ def detect_changes(self, ds_name): def write_new_definitions(self): + + # write updated yaml file + self.write_schemadefs() + + # then write out the ddl and migration files + today_str = datetime.now().strftime("%Y_%m_%d") - yaml_file = CovidHospSomething.YAML_FILENAME ddl_file = 'covid_hosp.sql' migration_file = f"covid_hosp_v{today_str}.sql" # TODO: do these with proper python `pathlib.Path` objects - repo_root = './' - yaml_file_directory = repo_root + 'src/ddl/' + repo_root = './' # TODO: get this from somewhere ddl_file_directory = repo_root + 'src/ddl/' migration_file_directory = repo_root + 'src/ddl/migrations/' - # write updated yaml file - self.write_schemadefs() - # write newly generated sql definition file with open(ddl_file, 'w') as f: # TODO @@ -158,7 +185,6 @@ def write_new_definitions(self): f.write("\n") # move files into proper locations - # UNCOMMENT: os.system(f"mv -f {yaml_file} {yaml_file_directory}") # UNCOMMENT: os.system("mv -f {ddl_file} {ddl_file_directory}") # UNCOMMENT: os.system("mv -f {migration_file} {migration_file_directory") @@ -168,7 +194,7 @@ def write_new_definitions(self): chs = CovidHospSomething() changed = False - for ds_name in chs.datasets(): + for ds_name in chs.dataset_names(): ds = chs.dataset(ds_name) new_cols = chs.detect_changes(ds_name) if new_cols: diff --git a/src/common/covid_hosp/covid_hosp_schemadefs.yaml b/src/common/covid_hosp/covid_hosp_schemadefs.yaml index 74795a9cf..5cf1642f6 100644 --- a/src/common/covid_hosp/covid_hosp_schemadefs.yaml +++ b/src/common/covid_hosp/covid_hosp_schemadefs.yaml @@ -236,7 +236,6 @@ state_timeseries: - [int, previous_week_therapeutic_a_casirivimab_imdevimab_courses_used, ~] - [int, previous_week_therapeutic_b_bamlanivimab_courses_used, ~] - [int, previous_week_therapeutic_c_bamlanivimab_etesevimab_courses_used, ~] - - [fixedstr:1, record_type, ~] - [int, staffed_adult_icu_bed_occupancy, ~] - [int, staffed_adult_icu_bed_occupancy_coverage, ~] - [int, staffed_icu_adult_patients_confirmed_and_suspected_covid, staffed_icu_adult_patients_confirmed_suspected_covid] @@ -259,7 +258,7 @@ state_timeseries: - [int, total_staffed_adult_icu_beds_coverage, ~] state_daily: - TABLE_NAME: covid_hosp_state_daily + TABLE_NAME: covid_hosp_state_timeseries DATASET_ID: 6xf2-c3ie METADATA_ID: 4cnb-m4rz KEY_COLS: [state, reporting_cutoff_start] diff --git a/tests/common/covid_hosp/test_covid_hosp_schema_io.py b/tests/common/covid_hosp/test_covid_hosp_schema_io.py new file mode 100644 index 000000000..dae06d817 --- /dev/null +++ b/tests/common/covid_hosp/test_covid_hosp_schema_io.py @@ -0,0 +1,34 @@ +from pathlib import Path +import unittest + +from delphi.epidata.acquisition.covid_hosp.common.database import Columndef +from delphi.epidata.acquisition.covid_hosp.common.utils import Utils +from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething + + +# py3tester coverage target (equivalent to `import *`) +__test_target__ = "delphi.epidata.common.covid_hosp.covid_hosp_schema_io" + + +class TestCovidHospSchemaIo(unittest.TestCase): + chs = CovidHospSomething(str(Path(__file__).parent.absolute()) + "/test_covid_hosp_schemadefs.yaml") + + def test_get_ds_info(self): + TABLE_NAME = self.chs.get_ds_table_name('covid_hosp_facility') + KEY_COLS = self.chs.get_ds_key_cols('covid_hosp_facility') + AGGREGATE_KEY_COLS = self.chs.get_ds_aggregate_key_cols('covid_hosp_facility') + ORDERED_CSV_COLUMNS = self.chs.get_ds_ordered_csv_cols('covid_hosp_facility') + + assert TABLE_NAME == "covid_hosp_facility" + assert KEY_COLS == ["hospital_pk", "collection_week"] + assert AGGREGATE_KEY_COLS == ["address", "ccn", "city", "fips_code", "geocoded_hospital_address", "hhs_ids", "hospital_name", "hospital_pk", "hospital_subtype", "is_metro_micro", "state", "zip"] + assert ORDERED_CSV_COLUMNS == [ + Columndef('hospital_pk', 'hospital_pk', str), + Columndef('collection_week', 'collection_week', Utils.int_from_date), + Columndef('reporting_cutoff_start', 'date', Utils.int_from_date), + Columndef('all_adult_hospital_beds_7_day_avg', 'all_adult_hospital_beds_7_day_avg', float), + Columndef('all_adult_hospital_beds_7_day_coverage', 'all_adult_hospital_beds_7_day_coverage', int), + Columndef('fips_code', 'fips_code', str), + Columndef('geocoded_hospital_address', 'geocoded_hospital_address', Utils.limited_geocode), + Columndef('is_corrected', 'is_corrected', Utils.parse_bool) + ] diff --git a/tests/common/covid_hosp/test_covid_hosp_schemadefs.yaml b/tests/common/covid_hosp/test_covid_hosp_schemadefs.yaml new file mode 100644 index 000000000..48b91d747 --- /dev/null +++ b/tests/common/covid_hosp/test_covid_hosp_schemadefs.yaml @@ -0,0 +1,14 @@ +# comment top +covid_hosp_facility: + TABLE_NAME: covid_hosp_facility + KEY_COLS: [hospital_pk, collection_week] + AGGREGATE_KEY_COLS: [address, ccn, city, fips_code, geocoded_hospital_address, hhs_ids, hospital_name, hospital_pk, hospital_subtype, is_metro_micro, state, zip] + ORDERED_CSV_COLUMNS: + - [str:128, hospital_pk, ~] + - [intdate, collection_week, ~] + - [intdate, reporting_cutoff_start, date] + - [float, all_adult_hospital_beds_7_day_avg, ~] + - [int, all_adult_hospital_beds_7_day_coverage, ~] + - [fixedstr:5, fips_code, ~] + - [geocode:32, geocoded_hospital_address, ~] + - [bool, is_corrected, ~]