Skip to content

Commit 611d200

Browse files
krivardkrivard
authoredAug 5, 2020
Merge pull request #161 from AlSaeed/partitioning-direction-update
Partitioning Direction Update
·
v4.1.33norostat-acquisition
2 parents cdc590a + d87fcb1 commit 611d200

16 files changed

+552
-324
lines changed
 

‎integrations/acquisition/covidcast/test_covidcast_meta_caching.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,14 @@ def test_caching(self):
6666
self.cur.execute('''
6767
insert into covidcast values
6868
(0, 'src', 'sig', 'day', 'state', 20200422, 'pa',
69-
123, 1, 2, 3, 456, 1, 20200422, 0),
69+
123, 1, 2, 3, 456, 1, 20200422, 0, 1),
7070
(0, 'src', 'sig', 'day', 'state', 20200422, 'wa',
71-
789, 1, 2, 3, 456, 1, 20200423, 1)
71+
789, 1, 2, 3, 456, 1, 20200423, 1, 1)
7272
''')
7373
self.cur.execute('''
7474
insert into covidcast values
7575
(100, 'src', 'wip_sig', 'day', 'state', 20200422, 'pa',
76-
456, 4, 5, 6, 789, -1, 20200422, 0)
76+
456, 4, 5, 6, 789, -1, 20200422, 0, 1)
7777
''')
7878

7979
self.cnx.commit()

‎integrations/acquisition/covidcast/test_csv_uploading.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,10 @@ def apply_lag(expected_epidata):
192192
})
193193

194194
# verify timestamps and default values are reasonable
195-
self.cur.execute('select timestamp1, timestamp2, direction from covidcast')
196-
for timestamp1, timestamp2, direction in self.cur:
197-
self.assertGreater(timestamp1, 0)
198-
self.assertEqual(timestamp2, 0)
195+
self.cur.execute('select value_updated_timestamp, direction_updated_timestamp, direction from covidcast')
196+
for value_updated_timestamp, direction_updated_timestamp, direction in self.cur:
197+
self.assertGreater(value_updated_timestamp, 0)
198+
self.assertEqual(direction_updated_timestamp, 0)
199199
self.assertIsNone(direction)
200200

201201
# verify that the CSVs were archived

‎integrations/acquisition/covidcast/test_direction_updating.py

Lines changed: 62 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ def test_uploading(self):
5151
"""Update rows having a stale `direction` field and serve the results."""
5252

5353
# insert some sample data
54+
# src, sig1, 1111:
55+
# direction should be updated to None as there are no historical data for (src, sig1, state).
5456
# CA 20200301:
5557
# timeline should be x=[-2, -1, 0], y=[2, 6, 5] with direction=1
5658
# FL 20200517:
@@ -60,31 +62,77 @@ def test_uploading(self):
6062
# wrong) is fresh
6163
self.cur.execute('''
6264
insert into covidcast values
65+
(0, 'src', 'sig1', 'day', 'state', 20201028, '1111',
66+
123, 2, 0, 0, 0, -1, 20201028, 0, 1),
67+
(0, 'src', 'sig1', 'day', 'state', 20201029, '1111',
68+
123, 6, 0, 0, 0, 0, 20201029, 0, 1),
69+
(0, 'src', 'sig1', 'day', 'state', 20201030, '1111',
70+
123, 5, 0, 0, 0, 1, 20201030, 0, 1),
6371
(0, 'src', 'sig', 'day', 'state', 20200228, 'ca',
64-
123, 2, 0, 0, 0, NULL, 20200228, 0),
72+
123, 2, 0, 0, 0, NULL, 20200228, 0, 1),
6573
(0, 'src', 'sig', 'day', 'state', 20200229, 'ca',
66-
123, 6, 0, 0, 0, NULL, 20200229, 0),
74+
123, 6, 0, 0, 0, NULL, 20200229, 0, 1),
6775
(0, 'src', 'sig', 'day', 'state', 20200301, 'ca',
68-
123, 5, 0, 0, 0, NULL, 20200301, 0),
76+
123, 5, 0, 0, 0, NULL, 20200301, 0, 1),
6977
(0, 'src', 'sig', 'day', 'state', 20200511, 'fl',
70-
123, 1, 0, 0, 0, NULL, 20200511, 0),
78+
123, 1, 0, 0, 0, NULL, 20200511, 0, 1),
7179
(0, 'src', 'sig', 'day', 'state', 20200512, 'fl',
72-
123, 2, 0, 0, 0, NULL, 20200512, 0),
80+
123, 2, 0, 0, 0, NULL, 20200512, 0, 1),
7381
(0, 'src', 'sig', 'day', 'state', 20200517, 'fl',
74-
123, 2, 0, 0, 0, NULL, 20200517, 0),
82+
123, 2, 0, 0, 0, NULL, 20200517, 0, 1),
7583
(0, 'src', 'sig', 'day', 'state', 20200615, 'tx',
76-
123, 9, 0, 0, 456, NULL, 20200615, 0),
84+
123, 9, 0, 0, 456, NULL, 20200615, 0, 1),
7785
(0, 'src', 'sig', 'day', 'state', 20200616, 'tx',
78-
123, 5, 0, 0, 456, NULL, 20200616, 0),
86+
123, 5, 0, 0, 456, NULL, 20200616, 0, 1),
7987
(0, 'src', 'sig', 'day', 'state', 20200617, 'tx',
80-
123, 1, 0, 0, 456, 1, 20200617, 0)
88+
123, 1, 0, 0, 456, 1, 20200617, 0, 1)
8189
''')
8290
self.cnx.commit()
8391

8492
# update direction (only 20200417 has enough history)
85-
args = None
93+
args = get_argument_parser().parse_args('')
8694
main(args)
8795

96+
# The Quick-Fix is working
97+
response = Epidata.covidcast(
98+
'src', 'sig1', 'day', 'state', '20200101-20201231', '*')
99+
100+
self.assertEqual(response, {
101+
'result': 1,
102+
'epidata': [{
103+
'time_value': 20201028,
104+
'geo_value': '1111',
105+
'value': 2,
106+
'stderr': 0,
107+
'sample_size': 0,
108+
'direction': None,
109+
'issue': 20201028,
110+
'lag': 0
111+
},
112+
{
113+
'time_value': 20201029,
114+
'geo_value': '1111',
115+
'value': 6,
116+
'stderr': 0,
117+
'sample_size': 0,
118+
'direction': None,
119+
'issue': 20201029,
120+
'lag': 0
121+
},
122+
{
123+
'time_value': 20201030,
124+
'geo_value': '1111',
125+
'value': 5,
126+
'stderr': 0,
127+
'sample_size': 0,
128+
'direction': None,
129+
'issue': 20201030,
130+
'lag': 0
131+
},
132+
],
133+
'message': 'success',
134+
})
135+
88136
# request data from the API
89137
response = Epidata.covidcast(
90138
'src', 'sig', 'day', 'state', '20200101-20201231', '*')
@@ -188,11 +236,11 @@ def test_uploading(self):
188236
})
189237

190238
# verify secondary timestamps were updated
191-
self.cur.execute('select timestamp2 from covidcast order by id asc')
239+
self.cur.execute('select direction_updated_timestamp from covidcast order by id asc')
192240
timestamps = [t for (t,) in self.cur]
193-
for t in timestamps[:6]:
194-
# first 6 rows had `direction` updated
241+
for t in timestamps[:9]:
242+
# first 9 rows had `direction` updated
195243
self.assertGreater(t, 0)
196-
for t in timestamps[6:]:
244+
for t in timestamps[9:]:
197245
# last 3 rows were not updated
198246
self.assertEqual(t, 456)

‎integrations/acquisition/covidcast/test_fill_is_latest_issue.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
"""Integration tests for covidcast's direction updating."""
2+
3+
# standard library
4+
import unittest
5+
6+
# third party
7+
import mysql.connector
8+
9+
# first party
10+
from delphi.epidata.client.delphi_epidata import Epidata
11+
import delphi.operations.secrets as secrets
12+
13+
# py3tester coverage target (equivalent to `import *`)
14+
__test_target__ = 'delphi.epidata.acquisition.covidcast.fill_is_latest_issue'
15+
16+
17+
class FillIsLatestIssueTests(unittest.TestCase):
18+
"""Tests filling is_latest_issue column"""
19+
20+
def setUp(self):
21+
"""Perform per-test setup."""
22+
23+
# connect to the `epidata` database and clear the `covidcast` table
24+
cnx = mysql.connector.connect(
25+
user='user',
26+
password='pass',
27+
host='delphi_database_epidata',
28+
database='epidata')
29+
cur = cnx.cursor()
30+
cur.execute('truncate table covidcast')
31+
cnx.commit()
32+
cur.close()
33+
34+
# make connection and cursor available to test cases
35+
self.cnx = cnx
36+
self.cur = cnx.cursor()
37+
38+
# use the local instance of the epidata database
39+
secrets.db.host = 'delphi_database_epidata'
40+
secrets.db.epi = ('user', 'pass')
41+
42+
# use the local instance of the Epidata API
43+
Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
44+
45+
def tearDown(self):
46+
"""Perform per-test teardown."""
47+
self.cur.close()
48+
self.cnx.close()
49+
50+
def test_fill_is_latest_issue(self):
51+
"""Update rows having a stale `direction` field and serve the results."""
52+
53+
self.cur.execute('''
54+
insert into covidcast values
55+
(0, 'src', 'sig', 'day', 'state', 20200228, 'ca',
56+
123, 2, 5, 5, 5, NULL, 20200228, 0, 1),
57+
(0, 'src', 'sig', 'day', 'state', 20200228, 'ca',
58+
123, 2, 0, 0, 0, NULL, 20200229, 1, 1),
59+
(0, 'src', 'sig', 'day', 'state', 20200229, 'ca',
60+
123, 6, 0, 0, 0, NULL, 20200301, 1, 1),
61+
(0, 'src', 'sig', 'day', 'state', 20200229, 'ca',
62+
123, 6, 9, 9, 9, NULL, 20200229, 0, 1),
63+
(0, 'src', 'sig', 'day', 'state', 20200301, 'ca',
64+
123, 5, 0, 0, 0, NULL, 20200303, 2, 1),
65+
(0, 'src', 'sig', 'day', 'state', 20200301, 'ca',
66+
123, 5, 5, 5, 5, NULL, 20200302, 1, 1),
67+
(0, 'src', 'sig', 'day', 'state', 20200301, 'ca',
68+
123, 5, 9, 8, 7, NULL, 20200301, 0, 1)
69+
''')
70+
self.cnx.commit()
71+
72+
# fill is_latest_issue
73+
main()
74+
75+
self.cur.execute('''select * from covidcast''')
76+
result = list(self.cur)
77+
expected = [
78+
(1, 'src', 'sig', 'day', 'state', 20200228, 'ca',
79+
123, 2, 5, 5, 5, None, 20200228, 0, bytearray(b'0')),
80+
(2, 'src', 'sig', 'day', 'state', 20200228, 'ca',
81+
123, 2, 0, 0, 0, None, 20200229, 1, bytearray(b'1')),
82+
(3, 'src', 'sig', 'day', 'state', 20200229, 'ca',
83+
123, 6, 0, 0, 0, None, 20200301, 1, bytearray(b'1')),
84+
(4, 'src', 'sig', 'day', 'state', 20200229, 'ca',
85+
123, 6, 9, 9, 9, None, 20200229, 0, bytearray(b'0')),
86+
(5, 'src', 'sig', 'day', 'state', 20200301, 'ca',
87+
123, 5, 0, 0, 0, None, 20200303, 2, bytearray(b'1')),
88+
(6, 'src', 'sig', 'day', 'state', 20200301, 'ca',
89+
123, 5, 5, 5, 5, None, 20200302, 1, bytearray(b'0')),
90+
(7, 'src', 'sig', 'day', 'state', 20200301, 'ca',
91+
123, 5, 9, 8, 7, None, 20200301, 0, bytearray(b'0'))
92+
]
93+
94+
self.assertEqual(result, expected)

‎integrations/client/test_delphi_epidata.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@ def test_covidcast(self):
5050
self.cur.execute('''
5151
insert into covidcast values
5252
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
53-
123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0),
53+
123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 0),
5454
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
55-
456, 5.5, 1.2, 10.5, 789, 0, 20200415, 1),
55+
456, 5.5, 1.2, 10.5, 789, 0, 20200415, 1, 0),
5656
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
57-
345, 6.5, 2.2, 11.5, 678, 0, 20200416, 2)
57+
345, 6.5, 2.2, 11.5, 678, 0, 20200416, 2, 1)
5858
''')
5959
self.cnx.commit()
6060

@@ -157,11 +157,11 @@ def test_covidcast_meta(self):
157157
self.cur.execute('''
158158
insert into covidcast values
159159
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
160-
123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0),
160+
123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 0),
161161
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
162-
345, 6.0, 2.2, 11.5, 678, 0, 20200416, 2),
162+
345, 6.0, 2.2, 11.5, 678, 0, 20200416, 2, 1),
163163
(0, 'src', 'sig', 'day', 'county', 20200415, '01234',
164-
345, 7.0, 2.0, 12.5, 678, 0, 20200416, 1)
164+
345, 7.0, 2.0, 12.5, 678, 0, 20200416, 1, 1)
165165
''')
166166
self.cnx.commit()
167167

‎integrations/server/test_covidcast.py

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def test_round_trip(self):
4545
self.cur.execute('''
4646
insert into covidcast values
4747
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
48-
123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0)
48+
123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1)
4949
''')
5050
self.cnx.commit()
5151

@@ -85,17 +85,17 @@ def test_location_wildcard(self):
8585
self.cur.execute('''
8686
insert into covidcast values
8787
(0, 'src', 'sig', 'day', 'county', 20200414, '11111',
88-
123, 10, 11, 12, 456, 13, 20200414, 0),
88+
123, 10, 11, 12, 456, 13, 20200414, 0, 1),
8989
(0, 'src', 'sig', 'day', 'county', 20200414, '22222',
90-
123, 20, 21, 22, 456, 23, 20200414, 0),
90+
123, 20, 21, 22, 456, 23, 20200414, 0, 1),
9191
(0, 'src', 'sig', 'day', 'county', 20200414, '33333',
92-
123, 30, 31, 32, 456, 33, 20200414, 0),
92+
123, 30, 31, 32, 456, 33, 20200414, 0, 1),
9393
(0, 'src', 'sig', 'day', 'msa', 20200414, '11111',
94-
123, 40, 41, 42, 456, 43, 20200414, 0),
94+
123, 40, 41, 42, 456, 43, 20200414, 0, 1),
9595
(0, 'src', 'sig', 'day', 'msa', 20200414, '22222',
96-
123, 50, 51, 52, 456, 53, 20200414, 0),
96+
123, 50, 51, 52, 456, 53, 20200414, 0, 1),
9797
(0, 'src', 'sig', 'day', 'msa', 20200414, '33333',
98-
123, 60, 61, 62, 456, 634, 20200414, 0)
98+
123, 60, 61, 62, 456, 634, 20200414, 0, 1)
9999
''')
100100
self.cnx.commit()
101101

@@ -155,17 +155,17 @@ def test_location_timeline(self):
155155
self.cur.execute('''
156156
insert into covidcast values
157157
(0, 'src', 'sig', 'day', 'county', 20200411, '01234',
158-
123, 10, 11, 12, 456, 13, 20200413, 2),
158+
123, 10, 11, 12, 456, 13, 20200413, 2, 1),
159159
(0, 'src', 'sig', 'day', 'county', 20200412, '01234',
160-
123, 20, 21, 22, 456, 23, 20200413, 1),
160+
123, 20, 21, 22, 456, 23, 20200413, 1, 1),
161161
(0, 'src', 'sig', 'day', 'county', 20200413, '01234',
162-
123, 30, 31, 32, 456, 33, 20200413, 0),
162+
123, 30, 31, 32, 456, 33, 20200413, 0, 1),
163163
(0, 'src', 'sig', 'day', 'county', 20200411, '11111',
164-
123, 40, 41, 42, 456, 43, 20200413, 2),
164+
123, 40, 41, 42, 456, 43, 20200413, 2, 1),
165165
(0, 'src', 'sig', 'day', 'county', 20200412, '22222',
166-
123, 50, 51, 52, 456, 53, 20200413, 1),
166+
123, 50, 51, 52, 456, 53, 20200413, 1, 1),
167167
(0, 'src', 'sig', 'day', 'county', 20200413, '33333',
168-
123, 60, 61, 62, 456, 63, 20200413, 0)
168+
123, 60, 61, 62, 456, 63, 20200413, 0, 1)
169169
''')
170170
self.cnx.commit()
171171

@@ -225,7 +225,7 @@ def test_unique_key_constraint(self):
225225
self.cur.execute('''
226226
insert into covidcast values
227227
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
228-
0, 0, 0, 0, 0, 0, 20200414, 0)
228+
0, 0, 0, 0, 0, 0, 20200414, 0, 1)
229229
''')
230230
self.cnx.commit()
231231

@@ -234,14 +234,14 @@ def test_unique_key_constraint(self):
234234
self.cur.execute('''
235235
insert into covidcast values
236236
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
237-
1, 1, 1, 1, 1, 1, 20200414, 0)
237+
1, 1, 1, 1, 1, 1, 20200414, 0, 1)
238238
''')
239239

240240
# succeed to insert different dummy data under a different issue
241241
self.cur.execute('''
242242
insert into covidcast values
243243
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
244-
1, 1, 1, 1, 1, 1, 20200415, 1)
244+
1, 1, 1, 1, 1, 1, 20200415, 1, 1)
245245
''')
246246

247247
def test_nullable_columns(self):
@@ -251,7 +251,7 @@ def test_nullable_columns(self):
251251
self.cur.execute('''
252252
insert into covidcast values
253253
(0, 'src', 'sig', 'day', 'county', 20200414, '01234',
254-
123, 0.123, NULL, NULL, 456, NULL, 20200414, 0)
254+
123, 0.123, NULL, NULL, 456, NULL, 20200414, 0, 1)
255255
''')
256256
self.cnx.commit()
257257

@@ -291,15 +291,15 @@ def test_temporal_partitioning(self):
291291
self.cur.execute('''
292292
insert into covidcast values
293293
(0, 'src', 'sig', 'hour', 'state', 2020041714, 'vi',
294-
123, 10, 11, 12, 456, 13, 2020041714, 0),
294+
123, 10, 11, 12, 456, 13, 2020041714, 0, 1),
295295
(0, 'src', 'sig', 'day', 'state', 20200417, 'vi',
296-
123, 20, 21, 22, 456, 23, 20200417, 00),
296+
123, 20, 21, 22, 456, 23, 20200417, 00, 1),
297297
(0, 'src', 'sig', 'week', 'state', 202016, 'vi',
298-
123, 30, 31, 32, 456, 33, 202016, 0),
298+
123, 30, 31, 32, 456, 33, 202016, 0, 1),
299299
(0, 'src', 'sig', 'month', 'state', 202004, 'vi',
300-
123, 40, 41, 42, 456, 43, 202004, 0),
300+
123, 40, 41, 42, 456, 43, 202004, 0, 1),
301301
(0, 'src', 'sig', 'year', 'state', 2020, 'vi',
302-
123, 50, 51, 52, 456, 53, 2020, 0)
302+
123, 50, 51, 52, 456, 53, 2020, 0, 1)
303303
''')
304304
self.cnx.commit()
305305

0 commit comments

Comments
 (0)
Please sign in to comment.