File tree Expand file tree Collapse file tree 4 files changed +38
-27
lines changed Expand file tree Collapse file tree 4 files changed +38
-27
lines changed Original file line number Diff line number Diff line change @@ -69,21 +69,20 @@ def run_module():
69
69
csv_export_count = 0
70
70
oldest_final_export_date = None
71
71
params = read_params ()
72
- export_start_date = params ["export_start_date" ]
73
- export_dir = params ["export_dir" ]
74
- base_url = params ["base_url" ]
75
- cache_dir = params ["cache_dir" ]
72
+ export_start_date = params ["indicator" ]["export_start_date" ]
73
+ export_dir = params ["common" ]["export_dir" ]
74
+ base_url = params ["indicator" ]["base_url" ]
76
75
logger = get_structured_logger (
77
- __name__ , filename = params .get ("log_filename" ),
78
- log_exceptions = params .get ("log_exceptions" , True ))
76
+ __name__ , filename = params [ "common" ] .get ("log_filename" ),
77
+ log_exceptions = params [ "common" ] .get ("log_exceptions" , True ))
79
78
80
- if len ( params ["bucket_name" ]) > 0 :
79
+ if params ["archive" ] :
81
80
arch_diff = S3ArchiveDiffer (
82
- cache_dir ,
81
+ params [ "archive" ][ " cache_dir" ] ,
83
82
export_dir ,
84
- params ["bucket_name" ],
83
+ params ["archive" ][ " bucket_name" ],
85
84
"jhu" ,
86
- params ["aws_credentials" ],
85
+ params ["archive" ][ " aws_credentials" ],
87
86
)
88
87
arch_diff .update_cache ()
89
88
else :
Original file line number Diff line number Diff line change 1
1
{
2
- "export_start_date": "2020-02-20",
3
- "static_file_dir": "./static",
4
- "export_dir": "./receiving",
5
- "cache_dir": "./cache",
6
- "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
7
- "aws_credentials": {
8
- "aws_access_key_id": "",
9
- "aws_secret_access_key": ""
2
+ "common": {
3
+ "export_dir": "./receiving",
4
+ "log_exceptions": false
10
5
},
11
- "bucket_name": "",
12
- "log_exceptions": false
6
+ "indicator": {
7
+ "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
8
+ "export_start_date": "2020-02-20",
9
+ "static_file_dir": "./static"
10
+ },
11
+ "archive": {
12
+ "aws_credentials": {
13
+ "aws_access_key_id": "",
14
+ "aws_secret_access_key": ""
15
+ },
16
+ "bucket_name": "",
17
+ "cache_dir": "./cache"
18
+ }
13
19
}
Original file line number Diff line number Diff line change @@ -22,9 +22,9 @@ def run_as_module():
22
22
with mock_s3 ():
23
23
# Create the fake bucket we will be using
24
24
params = read_params ()
25
- aws_credentials = params ["aws_credentials" ]
25
+ aws_credentials = params ["archive" ][ " aws_credentials" ]
26
26
s3_client = Session (** aws_credentials ).client ("s3" )
27
- s3_client .create_bucket (Bucket = params ["bucket_name" ])
27
+ s3_client .create_bucket (Bucket = params ["archive" ][ " bucket_name" ])
28
28
29
29
run_module ()
30
30
Original file line number Diff line number Diff line change 1
1
{
2
- "export_start_date ": "2020-03-03",
3
- "static_file_dir ": "../static",
4
- "export_dir": "./receiving" ,
5
- "cache_dir ": "./cache",
2
+ "common ": {
3
+ "export_dir ": "./receiving"
4
+ } ,
5
+ "indicator ": {
6
6
"base_url": "test_data/small_{metric}.csv",
7
+ "export_start_date": "2020-03-03",
8
+ "static_file_dir": "../static"
9
+ },
10
+ "archive": {
7
11
"aws_credentials": {
8
12
"aws_access_key_id": "FAKE_TEST_ACCESS_KEY_ID",
9
13
"aws_secret_access_key": "FAKE_TEST_SECRET_ACCESS_KEY"
10
14
},
11
- "bucket_name": "test-bucket"
15
+ "bucket_name": "test-bucket",
16
+ "cache_dir": "./cache"
17
+ }
12
18
}
You can’t perform that action at this time.
0 commit comments