Skip to content

Issue1746 validator no output #1749

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Jan 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 27 additions & 21 deletions _delphi_utils_python/delphi_utils/validator/static.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,28 +90,34 @@ def check_missing_date_files(self, daily_filenames, report):
Returns:
- None
"""
# Create set of all dates seen in CSV names.
unique_dates = {datetime.strptime(
daily_filename[0][0:8], '%Y%m%d').date() for daily_filename in daily_filenames}

# Diff expected and observed dates.
expected_dates = self.params.time_window.date_seq

if len(self.params.max_expected_lag) == 0:
max_expected_lag_overall = 10
else:
max_expected_lag_overall = max(self.params.max_expected_lag.values())

# Only check for date if it should definitely be present,
# i.e if it is more than max_expected_lag since the checking date
expected_dates = [date for date in expected_dates if
((datetime.today().date() - date).days) > max_expected_lag_overall]
check_dateholes = list(set(expected_dates).difference(unique_dates))
check_dateholes.sort()

if check_dateholes:
# Check to see if there are any files in the export directory
# Validator will throw an error if the directory is empty, which can be suppressed
if len(daily_filenames) == 0:
report.add_raised_error(
ValidationFailure("check_missing_date_files",
ValidationFailure("check_empty_filelist",
message="No files found in export directory"))
# Check for missing date only happens when files are found
else:
# Create set of all dates seen in CSV names.
unique_dates = {datetime.strptime(
daily_filename[0][0:8], '%Y%m%d').date() for daily_filename in daily_filenames}
# Diff expected and observed dates.
expected_dates = self.params.time_window.date_seq
if len(self.params.max_expected_lag) == 0:
max_expected_lag_overall = 10
else:
max_expected_lag_overall = max(self.params.max_expected_lag.values())

# Only check for date if it should definitely be present,
# i.e if it is more than max_expected_lag since the checking date
expected_dates = [date for date in expected_dates if
((datetime.today().date() - date).days) > max_expected_lag_overall]
check_dateholes = list(set(expected_dates).difference(unique_dates))
check_dateholes.sort()

if check_dateholes:
report.add_raised_error(
ValidationFailure("check_missing_date_files",
message="Missing dates are observed; if these dates are already "
"in the API they would not be updated"))

Expand Down
6 changes: 5 additions & 1 deletion _delphi_utils_python/delphi_utils/validator/validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ def validate(self):
frames_list = load_all_files(self.export_dir, self.time_window.start_date,
self.time_window.end_date)
self.static_validation.validate(frames_list, report)
all_frames = aggregate_frames(frames_list)
# Check if frames_list is empty before calling aggregate_frames
if len(frames_list) == 0:
all_frames = []
else:
all_frames = aggregate_frames(frames_list)
self.dynamic_validation.validate(all_frames, report)
return report
20 changes: 20 additions & 0 deletions _delphi_utils_python/tests/validator/test_static.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,29 @@ def test_empty_filelist(self):
filenames = list()
validator.check_missing_date_files(filenames, report)

assert len(report.raised_errors) == 1
assert report.raised_errors[0].check_name == "check_empty_filelist"

def test_missing_date_files(self):
params = {
"common": {
"data_source": "",
"span_length": 5,
"end_date": "2020-09-05",
"max_expected_lag": {"all": "1"}
}
}
validator = StaticValidator(params)
report = ValidationReport([])
filenames = [("20200901_county_signal_signal.csv", "match_obj"),
("20200903_county_signal_signal.csv", "match_obj"),
("20200904_county_signal_signal.csv", "match_obj"),
("20200905_county_signal_signal.csv", "match_obj")]
validator.check_missing_date_files(filenames, report)
assert len(report.raised_errors) == 1
assert report.raised_errors[0].check_name == "check_missing_date_files"


def test_same_day(self):
params = {
"common": {
Expand Down