Skip to content

Linters #1084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 56 commits into from
Closed

Linters #1084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
c278e71
wiki download indentation
BrainIsDead Feb 7, 2023
5f1cb24
test_exceptions indentation
BrainIsDead Feb 7, 2023
52dd437
dev_test_granular_sensor_authentication indentation and flake8
BrainIsDead Feb 7, 2023
cb66a50
quidel_update indentation and flake8
BrainIsDead Feb 7, 2023
9c7d0ae
simulate_api_response flake8
BrainIsDead Feb 7, 2023
da4da17
delphi_epidata indentation and flake8
BrainIsDead Feb 7, 2023
7783162
src linting
BrainIsDead Feb 9, 2023
459d98f
report_missing_covidcast_meta flake8
BrainIsDead Feb 13, 2023
34502dc
afhsb_csv flake8
BrainIsDead Feb 13, 2023
66948af
afhsb_sql flake8
BrainIsDead Feb 13, 2023
693cc2a
afhsb_update flake8
BrainIsDead Feb 13, 2023
4f74db3
cdc_dropbox_receiver flake8
BrainIsDead Feb 13, 2023
ab31396
cdc_extract flake8
BrainIsDead Feb 13, 2023
8dca39d
cdc_upload flake8
BrainIsDead Feb 13, 2023
382baa5
network flake8
BrainIsDead Feb 13, 2023
2aa6571
test_utils flake8
BrainIsDead Feb 13, 2023
d2d0722
utils flake8
BrainIsDead Feb 13, 2023
6a8f3d2
network flake8
BrainIsDead Feb 13, 2023
6da772b
update flake8
BrainIsDead Feb 13, 2023
c6882ac
database flake8
BrainIsDead Feb 13, 2023
da398d1
update flake8
BrainIsDead Feb 13, 2023
42d9943
network flake8
BrainIsDead Feb 13, 2023
32134d9
update flake8
BrainIsDead Feb 13, 2023
cd0619e
covidcast_meta_cache_updater flake8
BrainIsDead Feb 13, 2023
fd7c668
generate_islatest_fix_sql flake8
BrainIsDead Feb 13, 2023
519cbfc
logger flake8
BrainIsDead Feb 13, 2023
bb3ed61
signal_dash_data_generator flake8
BrainIsDead Feb 13, 2023
e10380d
test_utils flake8
BrainIsDead Feb 13, 2023
6d03506
ecdc_db_update flake8
BrainIsDead Feb 13, 2023
b03d286
ecdc_ili flake8
BrainIsDead Feb 13, 2023
b3dd703
flusurv flake8
BrainIsDead Feb 13, 2023
5b0758c
fluview_locations flake8
BrainIsDead Feb 13, 2023
53cea05
fluview_notify flake8
BrainIsDead Feb 13, 2023
5f791b8
fluview_update flake8
BrainIsDead Feb 13, 2023
42be0a4
fluview flake8
BrainIsDead Feb 13, 2023
472512c
impute_missing_values flake8
BrainIsDead Feb 13, 2023
3d26525
google_health_trends flake8
BrainIsDead Feb 13, 2023
e536a90
kcdc_update flake8
BrainIsDead Feb 13, 2023
db2c276
taiwan_update flake8
BrainIsDead Feb 13, 2023
b2310ed
norostat_raw flake8
BrainIsDead Feb 13, 2023
195838a
paho_db_update flake8
BrainIsDead Feb 13, 2023
27f26ae
paho_download flake8
BrainIsDead Feb 13, 2023
eaa22d6
quidel flake8
BrainIsDead Feb 13, 2023
cc54855
healthtweets flake8
BrainIsDead Feb 13, 2023
ceddf93
pageparser flake8
BrainIsDead Feb 13, 2023
d57dfb8
twitter_update flake8
BrainIsDead Feb 13, 2023
409be6d
wiki_download flake8
BrainIsDead Feb 13, 2023
3928730
flake8 wiki_extract
BrainIsDead Feb 13, 2023
19d1cb8
wiki_util flake8
BrainIsDead Feb 13, 2023
0e993e1
wiki_update flake8
BrainIsDead Feb 13, 2023
a422714
wiki flake8
BrainIsDead Feb 13, 2023
b96108d
_config flake8
BrainIsDead Feb 13, 2023
ba8d3c0
_pandas flake8
BrainIsDead Feb 13, 2023
f16e3c2
sensors flake8
BrainIsDead Feb 13, 2023
0eb38a4
dates flake8
BrainIsDead Feb 13, 2023
8fb4d2c
linters config
BrainIsDead Feb 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[flake8]
max-line-length = 158
exclude =
.env
./.venv
4 changes: 4 additions & 0 deletions .isort.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[settings]
multi_line_output=3
include_trailing_comma=True
line_length=158
20 changes: 12 additions & 8 deletions scripts/report_missing_covidcast_meta.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
from typing import Dict, List, Tuple, Union
from requests import get
import sys
import pandas as pd
from pathlib import Path
from typing import Dict, List, Tuple

import pandas as pd
from requests import get

base_dir = Path(__file__).parent.parent
base_url = 'https://delphi.cmu.edu/epidata'


def is_known_missing(source: str, signal: str) -> bool:
if '7dav_cumulative' in signal:
return True
if source in ('youtube-survey', 'indicator-combination'):
return True
return False


def compute_missing_signals() -> List[Tuple[Tuple[str, str], Dict]]:
defined_meta = get(f"{base_url}/covidcast/meta").json()
defined_signals: Dict[Tuple[str, str], Dict] = {}
Expand All @@ -27,7 +30,7 @@ def compute_missing_signals() -> List[Tuple[Tuple[str, str], Dict]]:
for entry in computed_meta:
computed_signals.setdefault((entry['data_source'], entry['signal']), []).append(entry)

missing_signals: List[Tuple[Tuple[str, str], Dict]] = []
missing_signals: List[Tuple[Tuple[str, str], Dict]] = []

for key, infos in computed_signals.items():
defined_info = defined_signals.get(key)
Expand All @@ -38,9 +41,9 @@ def compute_missing_signals() -> List[Tuple[Tuple[str, str], Dict]]:


def gen_row(source: str, signal: str, info: Dict) -> Dict:
is_weighted = signal.startswith('smoothed_w') and not (signal.startswith('smoothed_wa') or signal.startswith('smoothed_we') or signal.startswith('smoothed_wi') or signal.startswith('smoothed_wo') or signal.startswith('smoothed_wu'))
is_weighted = signal.startswith('smoothed_w') and not signal.startswith(('smoothed_wa', 'smoothed_we', 'smoothed_wi', 'smoothed_wo', 'smoothed_wu'))
base_name = signal.replace('smoothed_w', 'smoothed_') if is_weighted else signal
bool_str = lambda x: 'TRUE' if x else 'FALSE'
bool_str = lambda x: 'TRUE' if x else 'FALSE' # noqa

return {
'Source Subdivision': source,
Expand All @@ -66,6 +69,7 @@ def gen_row(source: str, signal: str, info: Dict) -> Dict:
'Link': 'TODO'
}


def generate_missing_info_hint(missing_signals: List[Tuple[Tuple[str, str], Dict]]) -> None:
missing = pd.DataFrame.from_records([gen_row(s[0], s[1], info) for s, info in missing_signals])

Expand All @@ -76,12 +80,12 @@ def generate_missing_info_hint(missing_signals: List[Tuple[Tuple[str, str], Dict
guessed: pd.DataFrame = pd.concat([current, missing])
guessed.to_csv(base_dir / 'missing_db_signals.csv', index=False)


missing = compute_missing_signals()
if missing:
print(f'found {len(missing)} missing signals')
generate_missing_info_hint(missing)
sys.exit(1)
else:
print(f'all signals found')
print('all signals found')
sys.exit(0)

Loading