diff --git a/_delphi_utils_python/delphi_utils/validator/static.py b/_delphi_utils_python/delphi_utils/validator/static.py index 48b17b888..d58096d97 100644 --- a/_delphi_utils_python/delphi_utils/validator/static.py +++ b/_delphi_utils_python/delphi_utils/validator/static.py @@ -90,28 +90,34 @@ def check_missing_date_files(self, daily_filenames, report): Returns: - None """ - # Create set of all dates seen in CSV names. - unique_dates = {datetime.strptime( - daily_filename[0][0:8], '%Y%m%d').date() for daily_filename in daily_filenames} - - # Diff expected and observed dates. - expected_dates = self.params.time_window.date_seq - - if len(self.params.max_expected_lag) == 0: - max_expected_lag_overall = 10 - else: - max_expected_lag_overall = max(self.params.max_expected_lag.values()) - - # Only check for date if it should definitely be present, - # i.e if it is more than max_expected_lag since the checking date - expected_dates = [date for date in expected_dates if - ((datetime.today().date() - date).days) > max_expected_lag_overall] - check_dateholes = list(set(expected_dates).difference(unique_dates)) - check_dateholes.sort() - - if check_dateholes: + # Check to see if there are any files in the export directory + # Validator will throw an error if the directory is empty, which can be suppressed + if len(daily_filenames) == 0: report.add_raised_error( - ValidationFailure("check_missing_date_files", + ValidationFailure("check_empty_filelist", + message="No files found in export directory")) + # Check for missing date only happens when files are found + else: + # Create set of all dates seen in CSV names. + unique_dates = {datetime.strptime( + daily_filename[0][0:8], '%Y%m%d').date() for daily_filename in daily_filenames} + # Diff expected and observed dates. + expected_dates = self.params.time_window.date_seq + if len(self.params.max_expected_lag) == 0: + max_expected_lag_overall = 10 + else: + max_expected_lag_overall = max(self.params.max_expected_lag.values()) + + # Only check for date if it should definitely be present, + # i.e if it is more than max_expected_lag since the checking date + expected_dates = [date for date in expected_dates if + ((datetime.today().date() - date).days) > max_expected_lag_overall] + check_dateholes = list(set(expected_dates).difference(unique_dates)) + check_dateholes.sort() + + if check_dateholes: + report.add_raised_error( + ValidationFailure("check_missing_date_files", message="Missing dates are observed; if these dates are already " "in the API they would not be updated")) diff --git a/_delphi_utils_python/delphi_utils/validator/validate.py b/_delphi_utils_python/delphi_utils/validator/validate.py index 9c4861b76..d03d7e3c2 100644 --- a/_delphi_utils_python/delphi_utils/validator/validate.py +++ b/_delphi_utils_python/delphi_utils/validator/validate.py @@ -58,6 +58,10 @@ def validate(self): frames_list = load_all_files(self.export_dir, self.time_window.start_date, self.time_window.end_date) self.static_validation.validate(frames_list, report) - all_frames = aggregate_frames(frames_list) + # Check if frames_list is empty before calling aggregate_frames + if len(frames_list) == 0: + all_frames = [] + else: + all_frames = aggregate_frames(frames_list) self.dynamic_validation.validate(all_frames, report) return report diff --git a/_delphi_utils_python/tests/validator/test_static.py b/_delphi_utils_python/tests/validator/test_static.py index bf270b4fd..ea3d44b1a 100644 --- a/_delphi_utils_python/tests/validator/test_static.py +++ b/_delphi_utils_python/tests/validator/test_static.py @@ -27,9 +27,29 @@ def test_empty_filelist(self): filenames = list() validator.check_missing_date_files(filenames, report) + assert len(report.raised_errors) == 1 + assert report.raised_errors[0].check_name == "check_empty_filelist" + + def test_missing_date_files(self): + params = { + "common": { + "data_source": "", + "span_length": 5, + "end_date": "2020-09-05", + "max_expected_lag": {"all": "1"} + } + } + validator = StaticValidator(params) + report = ValidationReport([]) + filenames = [("20200901_county_signal_signal.csv", "match_obj"), + ("20200903_county_signal_signal.csv", "match_obj"), + ("20200904_county_signal_signal.csv", "match_obj"), + ("20200905_county_signal_signal.csv", "match_obj")] + validator.check_missing_date_files(filenames, report) assert len(report.raised_errors) == 1 assert report.raised_errors[0].check_name == "check_missing_date_files" + def test_same_day(self): params = { "common": {