diff --git a/ansible/templates/google_health-params-prod.json.j2 b/ansible/templates/google_health-params-prod.json.j2 index 35887bd97..848e3c79b 100644 --- a/ansible/templates/google_health-params-prod.json.j2 +++ b/ansible/templates/google_health-params-prod.json.j2 @@ -11,5 +11,7 @@ "aws_access_key_id": "{{ delphi_aws_access_key_id }}", "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}" }, - "bucket_name": "delphi-covidcast-indicator-output" + "bucket_name": "delphi-covidcast-indicator-output", + "test": false, + "test_data_dir": "" } diff --git a/ansible/templates/google_health-params-test.json.j2 b/ansible/templates/google_health-params-test.json.j2 index a9a8d3640..c2fdec945 100644 --- a/ansible/templates/google_health-params-test.json.j2 +++ b/ansible/templates/google_health-params-test.json.j2 @@ -11,5 +11,7 @@ "aws_access_key_id": "{{ delphi_aws_access_key_id }}", "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}" }, - "bucket_name": "delphi-covidcast-indicator-output" + "bucket_name": "delphi-covidcast-indicator-output", + "test": true, + "test_data_dir": "./test_data/{geo_res}_sample.csv" } diff --git a/google_health/delphi_google_health/run.py b/google_health/delphi_google_health/run.py index 1c84f8233..446993134 100644 --- a/google_health/delphi_google_health/run.py +++ b/google_health/delphi_google_health/run.py @@ -107,18 +107,20 @@ def run_module(): start_date=start_date, receiving_dir=export_dir) export_csv(df_msa, MSA, signal, smooth=False, start_date=start_date, receiving_dir=export_dir) - # Diff exports, and make incremental versions - _, common_diffs, new_files = arch_diff.diff_exports() - # Archive changed and new files only - to_archive = [f for f, diff in common_diffs.items() if diff is not None] - to_archive += new_files - _, fails = arch_diff.archive_exports(to_archive) + if not params["test"]: + # Diff exports, and make incremental versions + _, common_diffs, new_files = arch_diff.diff_exports() + + # Archive changed and new files only + to_archive = [f for f, diff in common_diffs.items() if diff is not None] + to_archive += new_files + _, fails = arch_diff.archive_exports(to_archive) - # Filter existing exports to exclude those that failed to archive - succ_common_diffs = {f: diff for f, diff in common_diffs.items() if f not in fails} - arch_diff.filter_exports(succ_common_diffs) + # Filter existing exports to exclude those that failed to archive + succ_common_diffs = {f: diff for f, diff in common_diffs.items() if f not in fails} + arch_diff.filter_exports(succ_common_diffs) - # Report failures: someone should probably look at them - for exported_file in fails: - print(f"Failed to archive '{exported_file}'") + # Report failures: someone should probably look at them + for exported_file in fails: + print(f"Failed to archive '{exported_file}'") diff --git a/google_health/tests/test_run.py b/google_health/tests/test_run.py index 73d118a68..4d033f67e 100644 --- a/google_health/tests/test_run.py +++ b/google_health/tests/test_run.py @@ -12,7 +12,7 @@ class TestRunModule: def test_class(self, run_as_module, wip_signal=read_params()["wip_signal"]): """Tests output file existence.""" - if wip_signal is True: + if wip_signal: assert exists(join("receiving", "20200419_hrr_wip_raw_search.csv")) assert exists(join("receiving", "20200419_msa_wip_raw_search.csv")) assert exists(join("receiving", "20200419_state_wip_raw_search.csv")) @@ -35,7 +35,7 @@ def test_class(self, run_as_module, wip_signal=read_params()["wip_signal"]): def test_match_old_raw_output(self, run_as_module, wip_signal=read_params()["wip_signal"]): """Tests that raw output files don't change over time.""" - if wip_signal is True: + if wip_signal: files = [ "20200419_hrr_wip_raw_search.csv", "20200419_msa_wip_raw_search.csv", @@ -60,7 +60,7 @@ def test_match_old_raw_output(self, run_as_module, wip_signal=read_params()["wip def test_match_old_smoothed_output(self, run_as_module, wip_signal=read_params()["wip_signal"]): """Tests that smooth output files don't change over time.""" - if wip_signal is True: + if wip_signal: files = [ "20200419_hrr_wip_smoothed_search.csv",