diff --git a/integrations/test_covid_survey_county_weekly.py b/integrations/test_covid_survey_county_weekly.py new file mode 100644 index 000000000..099f5cf83 --- /dev/null +++ b/integrations/test_covid_survey_county_weekly.py @@ -0,0 +1,132 @@ +"""Integration tests for the `covid_survey_county_weekly` endpoint.""" + +# standard library +import unittest + +# third party +import mysql.connector +import requests + + +# use the local instance of the Epidata API +BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + + +class CovidSurveyCountyWeeklyTests(unittest.TestCase): + """Tests the `covid_survey_county_weekly` endpoint.""" + + def setUp(self): + """Perform per-test setup.""" + + # connect to the `epidata` database and clear the + # `covid_survey_county_weekly` table + cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + cur = cnx.cursor() + cur.execute('truncate table covid_survey_county_weekly') + cnx.commit() + cur.close() + + # make connection and cursor available to test cases + self.cnx = cnx + self.cur = cnx.cursor() + + def tearDown(self): + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + + def test_round_trip(self): + """Make a simple round-trip with some sample data.""" + + # insert dummy data + self.cur.execute(''' + insert into covid_survey_county_weekly values + (0, 202014, '42003', 1.5, 2.5, 3.5, 4.5, 5678.5) + ''') + self.cnx.commit() + + # make the request + response = requests.get(BASE_URL, params={ + 'source': 'covid_survey_county_weekly', + 'counties': '42003', + 'epiweeks': 202014, + }) + response.raise_for_status() + response = response.json() + + # assert that the right data came back + self.assertEqual(response, { + "result": 1, + "epidata": [{ + 'epiweek': 202014, + 'county': '42003', + 'ili': 1.5, + 'ili_stdev': 2.5, + 'cli': 3.5, + 'cli_stdev': 4.5, + 'denominator': 5678.5, + }], + "message": "success", + }) + + def test_privacy_filtering(self): + """Don't return rows with too small of a denominator.""" + + # shared constants + request_params = { + 'source': 'covid_survey_county_weekly', + 'counties': '42003', + 'epiweeks': 202014, + } + + with self.subTest(name='filtered'): + + # insert dummy data + self.cur.execute(''' + insert into covid_survey_county_weekly values + (0, 202014, '42003', 1.5, 2.5, 3.5, 4.5, 99.5) + ''') + self.cnx.commit() + + # make the request + response = requests.get(BASE_URL, params=request_params) + response.raise_for_status() + response = response.json() + + # assert that no data came back + self.assertEqual(response, { + "result": -2, + "message": "no results", + }) + + with self.subTest(name='unfiltered'): + + # amend the denominator + self.cur.execute(''' + update covid_survey_county_weekly set denominator = 100.5 + ''') + self.cnx.commit() + + # make the request + response = requests.get(BASE_URL, params=request_params) + response.raise_for_status() + response = response.json() + + # assert that the right data came back + self.assertEqual(response, { + "result": 1, + "epidata": [{ + 'epiweek': 202014, + 'county': '42003', + 'ili': 1.5, + 'ili_stdev': 2.5, + 'cli': 3.5, + 'cli_stdev': 4.5, + 'denominator': 100.5, + }], + "message": "success", + }) diff --git a/integrations/test_covid_survey_hrr_daily.py b/integrations/test_covid_survey_hrr_daily.py new file mode 100644 index 000000000..21a11bb83 --- /dev/null +++ b/integrations/test_covid_survey_hrr_daily.py @@ -0,0 +1,132 @@ +"""Integration tests for the `covid_survey_hrr_daily` endpoint.""" + +# standard library +import unittest + +# third party +import mysql.connector +import requests + + +# use the local instance of the Epidata API +BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + + +class CovidSurveyHrrDailyTests(unittest.TestCase): + """Tests the `covid_survey_hrr_daily` endpoint.""" + + def setUp(self): + """Perform per-test setup.""" + + # connect to the `epidata` database and clear the `covid_survey_hrr_daily` + # table + cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + cur = cnx.cursor() + cur.execute('truncate table covid_survey_hrr_daily') + cnx.commit() + cur.close() + + # make connection and cursor available to test cases + self.cnx = cnx + self.cur = cnx.cursor() + + def tearDown(self): + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + + def test_round_trip(self): + """Make a simple round-trip with some sample data.""" + + # insert dummy data + self.cur.execute(''' + insert into covid_survey_hrr_daily values + (0, '2020-04-08', 123, 1.5, 2.5, 3.5, 4.5, 5678.5) + ''') + self.cnx.commit() + + # make the request + response = requests.get(BASE_URL, params={ + 'source': 'covid_survey_hrr_daily', + 'hrrs': 123, + 'dates': 20200408, + }) + response.raise_for_status() + response = response.json() + + # assert that the right data came back + self.assertEqual(response, { + "result": 1, + "epidata": [{ + 'date': '2020-04-08', + 'hrr': 123, + 'ili': 1.5, + 'ili_stdev': 2.5, + 'cli': 3.5, + 'cli_stdev': 4.5, + 'denominator': 5678.5, + }], + "message": "success", + }) + + def test_privacy_filtering(self): + """Don't return rows with too small of a denominator.""" + + # shared constants + request_params = { + 'source': 'covid_survey_hrr_daily', + 'hrrs': 123, + 'dates': 20200408, + } + + with self.subTest(name='filtered'): + + # insert dummy data + self.cur.execute(''' + insert into covid_survey_hrr_daily values + (0, '2020-04-08', 123, 1.5, 2.5, 3.5, 4.5, 99.5) + ''') + self.cnx.commit() + + # make the request + response = requests.get(BASE_URL, params=request_params) + response.raise_for_status() + response = response.json() + + # assert that no data came back + self.assertEqual(response, { + "result": -2, + "message": "no results", + }) + + with self.subTest(name='unfiltered'): + + # amend the denominator + self.cur.execute(''' + update covid_survey_hrr_daily set denominator = 100.5 + ''') + self.cnx.commit() + + # make the request + response = requests.get(BASE_URL, params=request_params) + response.raise_for_status() + response = response.json() + + # assert that the right data came back + self.assertEqual(response, { + "result": 1, + "epidata": [{ + 'date': '2020-04-08', + 'hrr': 123, + 'ili': 1.5, + 'ili_stdev': 2.5, + 'cli': 3.5, + 'cli_stdev': 4.5, + 'denominator': 100.5, + }], + "message": "success", + }) diff --git a/integrations/test_fluview.py b/integrations/test_fluview.py new file mode 100644 index 000000000..956c2a045 --- /dev/null +++ b/integrations/test_fluview.py @@ -0,0 +1,82 @@ +"""Integration tests for the `fluview` endpoint.""" + +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata + + +class FluviewTests(unittest.TestCase): + """Tests the `fluview` endpoint.""" + + @classmethod + def setUpClass(cls): + """Perform one-time setup.""" + + # use the local instance of the Epidata API + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + + def setUp(self): + """Perform per-test setup.""" + + # connect to the `epidata` database and clear the `fluview` table + cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + cur = cnx.cursor() + cur.execute('truncate table fluview') + cnx.commit() + cur.close() + + # make connection and cursor available to test cases + self.cnx = cnx + self.cur = cnx.cursor() + + def tearDown(self): + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + + def test_round_trip(self): + """Make a simple round-trip with some sample data.""" + + # insert dummy data + self.cur.execute(''' + insert into fluview values + (0, "2020-04-07", 202021, 202020, "nat", 1, 2, 3, 4, 3.14159, 1.41421, + 10, 11, 12, 13, 14, 15) + ''') + self.cnx.commit() + + # make the request + response = Epidata.fluview('nat', 202020) + + # assert that the right data came back + self.assertEqual(response, { + "result": 1, + "epidata": [{ + "release_date": "2020-04-07", + "region": "nat", + "issue": 202021, + "epiweek": 202020, + "lag": 1, + "num_ili": 2, + "num_patients": 3, + "num_providers": 4, + "num_age_0": 10, + "num_age_1": 11, + "num_age_2": 12, + "num_age_3": 13, + "num_age_4": 14, + "num_age_5": 15, + "wili": 3.14159, + "ili": 1.41421, + }], + "message": "success", + }) diff --git a/src/ddl/covid_survey_county_weekly.sql b/src/ddl/covid_survey_county_weekly.sql new file mode 100644 index 000000000..3677160a8 --- /dev/null +++ b/src/ddl/covid_survey_county_weekly.sql @@ -0,0 +1,48 @@ +/* +Stores a subset of data from the COVID-19 survey, aggregated by weeks and +counties. + ++-------------+------------+------+-----+---------+----------------+ +| Field | Type | Null | Key | Default | Extra | ++-------------+------------+------+-----+---------+----------------+ +| id | int(11) | NO | PRI | NULL | auto_increment | +| epiweek | int(11) | NO | MUL | NULL | | +| county | varchar(5) | NO | | NULL | | +| ili | double | NO | | NULL | | +| ili_stdev | double | NO | | NULL | | +| cli | double | NO | | NULL | | +| cli_stdev | double | NO | | NULL | | +| denominator | double | NO | | NULL | | ++-------------+------------+------+-----+---------+----------------+ + +id: + unique identifier for each record +epiweek: + the epidemiological week during which the survey was submitted +county: + assumed fips 6-4 county code +ili: + estimated percent of sample experiencing influenza-like illness (ILI) +ili_stdev: + standard deviation for the ILI estimate +cli: + estimated percent of sample experiencing codid-19-like illness (CLI) +cli_stdev: + standard deviation for the CLI estimate +denominator: + estimated sample size +*/ + +CREATE TABLE `covid_survey_county_weekly` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `epiweek` int(11) NOT NULL, + `county` varchar(5) NOT NULL, + `ili` double NOT NULL, + `ili_stdev` double NOT NULL, + `cli` double NOT NULL, + `cli_stdev` double NOT NULL, + `denominator` double NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY (`epiweek`, `county`), + KEY (`county`, `epiweek`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/src/ddl/covid_survey_hrr_daily.sql b/src/ddl/covid_survey_hrr_daily.sql new file mode 100644 index 000000000..1a9d81ee6 --- /dev/null +++ b/src/ddl/covid_survey_hrr_daily.sql @@ -0,0 +1,48 @@ +/* +Stores a subset of data from the COVID-19 survey, aggregated by days and +Hospital Referral Regions (HRRs). + ++-------------+---------+------+-----+---------+----------------+ +| Field | Type | Null | Key | Default | Extra | ++-------------+---------+------+-----+---------+----------------+ +| id | int(11) | NO | PRI | NULL | auto_increment | +| date | date | NO | MUL | NULL | | +| hrr | int(11) | NO | | NULL | | +| ili | double | NO | | NULL | | +| ili_stdev | double | NO | | NULL | | +| cli | double | NO | | NULL | | +| cli_stdev | double | NO | | NULL | | +| denominator | double | NO | | NULL | | ++-------------+---------+------+-----+---------+----------------+ + +id: + unique identifier for each record +date: + the date on which the survey was submitted +hrr: + assumed hospital referral region (HRR) identifier +ili: + estimated percent of sample experiencing influenza-like illness (ILI) +ili_stdev: + standard deviation for the ILI estimate +cli: + estimated percent of sample experiencing codid-19-like illness (CLI) +cli_stdev: + standard deviation for the CLI estimate +denominator: + estimated sample size +*/ + +CREATE TABLE `covid_survey_hrr_daily` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `date` date NOT NULL, + `hrr` int(11) NOT NULL, + `ili` double NOT NULL, + `ili_stdev` double NOT NULL, + `cli` double NOT NULL, + `cli_stdev` double NOT NULL, + `denominator` double NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY (`date`, `hrr`), + KEY (`hrr`, `date`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/src/ddl/fluview.sql b/src/ddl/fluview.sql new file mode 100644 index 000000000..c9d12be5e --- /dev/null +++ b/src/ddl/fluview.sql @@ -0,0 +1,88 @@ +/* +Stores ILI data from the CDC. + ++---------------+-------------+------+-----+---------+----------------+ +| Field | Type | Null | Key | Default | Extra | ++---------------+-------------+------+-----+---------+----------------+ +| id | int(11) | NO | PRI | NULL | auto_increment | +| release_date | date | NO | MUL | NULL | | +| issue | int(11) | NO | MUL | NULL | | +| epiweek | int(11) | NO | MUL | NULL | | +| region | varchar(12) | NO | MUL | NULL | | +| lag | int(11) | NO | MUL | NULL | | +| num_ili | int(11) | NO | | NULL | | +| num_patients | int(11) | NO | | NULL | | +| num_providers | int(11) | NO | | NULL | | +| wili | double | NO | | NULL | | +| ili | double | NO | | NULL | | +| num_age_0 | int(11) | YES | | NULL | | +| num_age_1 | int(11) | YES | | NULL | | +| num_age_2 | int(11) | YES | | NULL | | +| num_age_3 | int(11) | YES | | NULL | | +| num_age_4 | int(11) | YES | | NULL | | +| num_age_5 | int(11) | YES | | NULL | | ++---------------+-------------+------+-----+---------+----------------+ + +id: + unique identifier for each record +release_date: + the date when this record was first published by the CDC +issue: + the epiweek of publication (e.g. issue 201453 includes epiweeks up to and + including 2014w53, but not 2015w01 or following) +epiweek: + the epiweek during which the data was collected +region: + the name of the location (e.g. 'nat', 'hhs1', 'cen9', 'pa', 'jfk') +lag: + number of weeks between `epiweek` and `issue` +num_ili: + the number of ILI cases (numerator) +num_patients: + the total number of patients (denominator) +num_providers: + the number of reporting healthcare providers +wili: + weighted percent ILI +ili: + unweighted percent ILI +num_age_0: + number of cases in ages 0-4 +num_age_1: + number of cases in ages 5-24 +num_age_2: + number of cases in ages 25-64 +num_age_3: + number of cases in ages 25-49 +num_age_4: + number of cases in ages 50-64 +num_age_5: + number of cases in ages 65+ +*/ + +CREATE TABLE `fluview` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `release_date` date NOT NULL, + `issue` int(11) NOT NULL, + `epiweek` int(11) NOT NULL, + `region` varchar(12) NOT NULL, + `lag` int(11) NOT NULL, + `num_ili` int(11) NOT NULL, + `num_patients` int(11) NOT NULL, + `num_providers` int(11) NOT NULL, + `wili` double NOT NULL, + `ili` double NOT NULL, + `num_age_0` int(11) DEFAULT NULL, + `num_age_1` int(11) DEFAULT NULL, + `num_age_2` int(11) DEFAULT NULL, + `num_age_3` int(11) DEFAULT NULL, + `num_age_4` int(11) DEFAULT NULL, + `num_age_5` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `issue` (`issue`,`epiweek`,`region`), + KEY `release_date` (`release_date`), + KEY `issue_2` (`issue`), + KEY `epiweek` (`epiweek`), + KEY `region` (`region`), + KEY `lag` (`lag`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/src/server/api.php b/src/server/api.php index 7ddd539e2..503bc604d 100644 --- a/src/server/api.php +++ b/src/server/api.php @@ -30,7 +30,7 @@ require_once('/var/www/html/secrets.php'); // helpers -require_once(__DIR__.'/api_helpers.php'); +require_once(__DIR__ . '/api_helpers.php'); // passwords $AUTH = array( @@ -920,6 +920,64 @@ function get_dengue_nowcast($locations, $epiweeks) { return count($epidata) === 0 ? null : $epidata; } +// queries the `covid_survey_hrr_daily` table. although the data is +// pre-filtered, out of an abundance of caution, rows with `denominator` less +// than 100 are omitted for privacy. +// $dates (required): array of date values/ranges +// $hrrs (required): array of HRR values/ranges +function get_covid_survey_hrr_daily($dates, $hrrs) { + // basic query info + $table = '`covid_survey_hrr_daily` t'; + $fields = "t.`date`, t.`hrr`, t.`ili`, t.`ili_stdev`, t.`cli`, t.`cli_stdev`, t.`denominator`"; + $order = "t.`date` ASC, t.`hrr` ASC"; + // data type of each field + $fields_string = array('date'); + $fields_int = array('hrr'); + $fields_float = array('ili', 'ili_stdev', 'cli', 'cli_stdev', 'denominator'); + // build the date filter + $condition_date = filter_dates('t.`date`', $dates); + // build the HRR filter + $condition_hrr = filter_integers('t.`hrr`', $hrrs); + // build the denominator threshold filter + $condition_denominator = 't.`denominator` >= 100'; + // the query + $query = "SELECT {$fields} FROM {$table} WHERE ({$condition_date}) AND ({$condition_hrr}) AND ({$condition_denominator}) ORDER BY {$order}"; + // get the data from the database + $epidata = array(); + execute_query($query, $epidata, $fields_string, $fields_int, $fields_float); + // return the data + return count($epidata) === 0 ? null : $epidata; +} + +// queries the `covid_survey_county_weekly` table. although the data is +// pre-filtered, out of an abundance of caution, rows with `denominator` less +// than 100 are omitted for privacy. +// $epiweeks (required): array of epiweek values/ranges +// $counties (required): array of FIPS 6-4 county identifiers +function get_covid_survey_county_weekly($epiweeks, $counties) { + // basic query info + $table = '`covid_survey_county_weekly` t'; + $fields = "t.`epiweek`, t.`county`, t.`ili`, t.`ili_stdev`, t.`cli`, t.`cli_stdev`, t.`denominator`"; + $order = "t.`epiweek` ASC, t.`county` ASC"; + // data type of each field + $fields_string = array('county'); + $fields_int = array('epiweek'); + $fields_float = array('ili', 'ili_stdev', 'cli', 'cli_stdev', 'denominator'); + // build the epiweek filter + $condition_epiweek = filter_integers('t.`epiweek`', $epiweeks); + // build the county filter + $condition_county = filter_strings('t.`county`', $counties); + // build the denominator threshold filter + $condition_denominator = 't.`denominator` >= 100'; + // the query + $query = "SELECT {$fields} FROM {$table} WHERE ({$condition_epiweek}) AND ({$condition_county}) AND ({$condition_denominator}) ORDER BY {$order}"; + // get the data from the database + $epidata = array(); + execute_query($query, $epidata, $fields_string, $fields_int, $fields_float); + // return the data + return count($epidata) === 0 ? null : $epidata; +} + // queries a bunch of epidata tables function get_meta() { // query and return metadata @@ -1364,6 +1422,24 @@ function meta_delphi() { $data['message'] = 'unauthenticated'; } } + } else if($source === 'covid_survey_hrr_daily') { + if(require_all($data, array('dates', 'hrrs'))) { + // parse the request + $dates = extract_values($_REQUEST['dates'], 'int'); + $hrrs = extract_values($_REQUEST['hrrs'], 'int'); + // get the data + $epidata = get_covid_survey_hrr_daily($dates, $hrrs); + store_result($data, $epidata); + } + } else if($source === 'covid_survey_county_weekly') { + if(require_all($data, array('epiweeks', 'counties'))) { + // parse the request + $epiweeks = extract_values($_REQUEST['epiweeks'], 'int'); + $counties = extract_values($_REQUEST['counties'], 'str'); + // get the data + $epidata = get_covid_survey_county_weekly($epiweeks, $counties); + store_result($data, $epidata); + } } else { $data['message'] = 'no data source specified'; } diff --git a/src/server/api_helpers.php b/src/server/api_helpers.php index fb4b5b337..3c24aa371 100644 --- a/src/server/api_helpers.php +++ b/src/server/api_helpers.php @@ -1,9 +1,12 @@ \ No newline at end of file +?> diff --git a/src/server/database_config.php b/src/server/database_config.php new file mode 100644 index 000000000..a1a0d67ee --- /dev/null +++ b/src/server/database_config.php @@ -0,0 +1,7 @@ + 'localhost', + 'port' => 3306, +); +?>