Skip to content

Remove remaining wip pieces in tests #917

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 1, 2022
Merged

Remove remaining wip pieces in tests #917

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3d3b8a7
Remove remaining wip pieces in tests
dshemetov May 26, 2022
c71b3f4
Put long name test back
dshemetov May 26, 2022
789a404
Fix mocked tests in test_csv_to_database.py
dshemetov Aug 30, 2022
09013fa
Merge branch 'v4-schema-revisions-release-prep' into ds/remove-wip-tests
melange396 Sep 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 2 additions & 39 deletions integrations/acquisition/covidcast/test_csv_uploading.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,6 @@ def test_uploading(self):
log_file=log_file_directory +
"output.log",
data_dir=data_dir,
is_wip_override=False,
not_wip_override=False,
specific_issue_date=False)
uploader_column_rename = {"geo_id": "geo_value", "val": "value", "se": "stderr", "missing_val": "missing_value", "missing_se": "missing_stderr"}

Expand Down Expand Up @@ -227,41 +225,6 @@ def test_uploading(self):
self.setUp()


with self.subTest("Valid wip"):
values = pd.DataFrame({
"geo_id": ["me", "nd", "wa"],
"val": [10.0, 20.0, 30.0],
"se": [0.01, 0.02, 0.03],
"sample_size": [100.0, 200.0, 300.0],
"missing_val": [Nans.NOT_MISSING] * 3,
"missing_se": [Nans.NOT_MISSING] * 3,
"missing_sample_size": [Nans.NOT_MISSING] * 3
})
signal_name = "wip_prototype"
values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)

# upload CSVs
main(args)
response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')

expected_values = pd.concat([values, pd.DataFrame({
"time_value": [20200419] * 3,
"signal": [signal_name] * 3,
"direction": [None] * 3
})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}

self.assertEqual(response, expected_response)
self.verify_timestamps_and_defaults()

# Verify that files were archived
path = data_dir + f'/archive/successful/src-name/20200419_state_wip_prototype.csv.gz'
self.assertIsNotNone(os.stat(path))

self.tearDown()
self.setUp()


with self.subTest("Valid signal with name length 32<x<64"):
values = pd.DataFrame({
"geo_id": ["pa"],
Expand All @@ -272,7 +235,7 @@ def test_uploading(self):
"missing_se": [Nans.NOT_MISSING],
"missing_sample_size": [Nans.NOT_MISSING]
})
signal_name = "wip_really_long_name_that_will_be_accepted"
signal_name = "really_long_name_that_will_be_accepted"
values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)

# upload CSVs
Expand Down Expand Up @@ -303,7 +266,7 @@ def test_uploading(self):
"missing_se": [Nans.NOT_MISSING],
"missing_sample_size": [Nans.NOT_MISSING]
})
signal_name = "wip_really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
signal_name = "really_long_name_that_will_get_truncated_lorem_ipsum_dolor_sit_amet"
values.to_csv(source_receiving_dir + f'/20200419_state_{signal_name}.csv', index=False)

# upload CSVs
Expand Down
4 changes: 2 additions & 2 deletions integrations/server/test_covidcast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,11 @@ def test_round_trip(self):
# `covidcast` (`id`, `source`, `signal`, `time_type`, `geo_type`,
# `time_value`, `geo_value`, `value_updated_timestamp`,
# `value`, `stderr`, `sample_size`, `direction_updated_timestamp`,
# `direction`, `issue`, `lag`, `is_latest_issue`, `is_wip`,`missing_value`,
# `direction`, `issue`, `lag`, `is_latest_issue`, `missing_value`,
# `missing_stderr`,`missing_sample_size`)
# VALUES
# (0, 'src', 'sig', 'day', 'county', 20200414, '01234',
# 123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1, False,
# 123, 1.5, 2.5, 3.5, 456, 4, 20200414, 0, 1,
# {Nans.NOT_MISSING}, {Nans.NOT_MISSING}, {Nans.NOT_MISSING})
# ''')
# self.cnx.commit()
Expand Down
22 changes: 10 additions & 12 deletions tests/acquisition/covidcast/test_csv_to_database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

# standard library
import argparse
from typing import Iterable
import unittest
from unittest.mock import MagicMock

Expand All @@ -27,9 +28,7 @@ def _path_details(self):
# a file with a data error
('path/b.csv', ('src_b', 'sig_b', 'week', 'msa', 202016, 202017, 1)),
# emulate a file that's named incorrectly
('path/c.csv', None),
# another good file w/ wip
('path/d.csv', ('src_d', 'wip_sig_d', 'week', 'msa', 202016, 202017, 1)),
('path/c.csv', None)
]

def test_collect_files(self):
Expand Down Expand Up @@ -65,15 +64,16 @@ def load_csv_impl(path, *args):
yield make_row('b1')
yield None
yield make_row('b3')
elif path == 'path/d.csv':
yield make_row('d1')
else:
# fail the test for any other path
raise Exception('unexpected path')

def iter_len(l: Iterable) -> int:
return len(list(l))

data_dir = 'data_dir'
mock_database = MagicMock()
mock_database.insert_or_update_bulk.return_value = 2
mock_database.insert_or_update_bulk = MagicMock(wraps=iter_len)
mock_csv_importer = MagicMock()
mock_csv_importer.load_csv = load_csv_impl
mock_file_archiver = MagicMock()
Expand All @@ -87,30 +87,28 @@ def load_csv_impl(path, *args):
mock_logger,
csv_importer_impl=mock_csv_importer)

self.assertEqual(modified_row_count, 4)
self.assertEqual(modified_row_count, 3)
# verify that appropriate rows were added to the database
self.assertEqual(mock_database.insert_or_update_bulk.call_count, 2)
self.assertEqual(mock_database.insert_or_update_bulk.call_count, 1)
call_args_list = mock_database.insert_or_update_bulk.call_args_list
actual_args = [[(a.source, a.signal, a.time_type, a.geo_type, a.time_value,
a.geo_value, a.value, a.stderr, a.sample_size, a.issue, a.lag)
for a in call.args[0]] for call in call_args_list]
expected_args = [
[('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a1', 'a1', 'a1', 'a1', 20200420, 1),
('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a2', 'a2', 'a2', 'a2', 20200420, 1),
('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)],
[('src_d', 'wip_sig_d', 'week', 'msa', 202016, 'd1', 'd1', 'd1', 'd1', 202017, 1)]
('src_a', 'sig_a', 'day', 'hrr', 20200419, 'a3', 'a3', 'a3', 'a3', 20200420, 1)]
]
self.assertEqual(actual_args, expected_args)

# verify that two files were successful (a, d) and two failed (b, c)
self.assertEqual(mock_file_archiver.archive_file.call_count, 4)
self.assertEqual(mock_file_archiver.archive_file.call_count, 3)
call_args_list = mock_file_archiver.archive_file.call_args_list
actual_args = [args for (args, kwargs) in call_args_list]
expected_args = [
('path', 'data_dir/archive/successful/src_a', 'a.csv', True),
('path', 'data_dir/archive/failed/src_b', 'b.csv', False),
('path', 'data_dir/archive/failed/unknown', 'c.csv', False),
('path', 'data_dir/archive/successful/src_d', 'd.csv', True),
]
self.assertEqual(actual_args, expected_args)

Expand Down