From 789fda27f9a1fd1e10441e6cd350075de86a13f6 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Mon, 16 Jan 2023 21:43:06 +0200 Subject: [PATCH] Move parse methods from covidcast --- src/server/_params.py | 175 ++++++++++++++++++ src/server/_query.py | 3 +- src/server/_validate.py | 129 ------------- src/server/endpoints/afhsb.py | 3 +- src/server/endpoints/cdc.py | 3 +- src/server/endpoints/covid_hosp_facility.py | 3 +- .../endpoints/covid_hosp_facility_lookup.py | 3 +- .../endpoints/covid_hosp_state_timeseries.py | 3 +- src/server/endpoints/covidcast.py | 62 +------ src/server/endpoints/covidcast_meta.py | 2 +- src/server/endpoints/covidcast_nowcast.py | 8 +- src/server/endpoints/dengue_nowcast.py | 3 +- src/server/endpoints/dengue_sensors.py | 3 +- src/server/endpoints/ecdc_ili.py | 3 +- src/server/endpoints/flusurv.py | 3 +- src/server/endpoints/fluview.py | 8 +- src/server/endpoints/fluview_clinicial.py | 3 +- src/server/endpoints/gft.py | 3 +- src/server/endpoints/ght.py | 3 +- src/server/endpoints/kcdc_ili.py | 3 +- src/server/endpoints/nidss_dengue.py | 3 +- src/server/endpoints/nidss_flu.py | 3 +- src/server/endpoints/norostat.py | 3 +- src/server/endpoints/nowcast.py | 3 +- src/server/endpoints/paho_dengue.py | 3 +- src/server/endpoints/quidel.py | 3 +- src/server/endpoints/sensors.py | 10 +- src/server/endpoints/twitter.py | 6 +- src/server/endpoints/wiki.py | 3 +- tests/server/test_params.py | 130 +++++++++++++ tests/server/test_validate.py | 131 ------------- 31 files changed, 374 insertions(+), 350 deletions(-) diff --git a/src/server/_params.py b/src/server/_params.py index 53be274f0..dd053121b 100644 --- a/src/server/_params.py +++ b/src/server/_params.py @@ -8,6 +8,7 @@ from ._exceptions import ValidationFailedException from .utils import days_in_range, weeks_in_range, guess_time_value_is_day, guess_time_value_is_week, IntRange, TimeValues, days_to_ranges, weeks_to_ranges +from ._validate import require_any, require_all def _parse_common_multi_arg(key: str) -> List[Tuple[str, Union[bool, Sequence[str]]]]: @@ -308,3 +309,177 @@ def parse_day_or_week_range_arg(key: str) -> TimeSet: if is_week: return TimeSet("week", [parse_week_range_arg(key)]) return TimeSet("day", [parse_day_range_arg(key)]) + + +def _extract_value(key: Union[str, Sequence[str]]) -> Optional[str]: + if isinstance(key, str): + return request.values.get(key) + for k in key: + if k in request.values: + return request.values[k] + return None + + +def _extract_list_value(key: Union[str, Sequence[str]]) -> List[str]: + if isinstance(key, str): + return request.values.getlist(key) + for k in key: + if k in request.values: + return request.values.getlist(k) + return [] + + +def extract_strings(key: Union[str, Sequence[str]]) -> Optional[List[str]]: + s = _extract_list_value(key) + if not s: + # nothing to do + return None + # we can have multiple values + return [v for vs in s for v in vs.split(",")] + + +def extract_integer(key: Union[str, Sequence[str]]) -> Optional[int]: + s = _extract_value(key) + if not s: + # nothing to do + return None + try: + return int(s) + except ValueError: + raise ValidationFailedException(f"{key}: not a number: {s}") + + +def extract_integers(key: Union[str, Sequence[str]]) -> Optional[List[IntRange]]: + parts = extract_strings(key) + if not parts: + # nothing to do + return None + + def _parse_range(part: str): + if "-" not in part: + return int(part) + r = part.split("-", 2) + first = int(r[0]) + last = int(r[1]) + if first == last: + # the first and last numbers are the same, just treat it as a singe value + return first + elif last > first: + # add the range as an array + return (first, last) + # the range is inverted, this is an error + raise ValidationFailedException(f"{key}: the given range is inverted") + + try: + values = [_parse_range(part) for part in parts] + # check for invalid values + return None if any(v is None for v in values) else values + except ValueError as e: + raise ValidationFailedException(f"{key}: not a number: {str(e)}") + + +def parse_date(s: str) -> int: + # parses a given string in format YYYYMMDD or YYYY-MM-DD to a number in the form YYYYMMDD + try: + return int(s.replace("-", "")) + except ValueError: + raise ValidationFailedException(f"not a valid date: {s}") + + +def extract_date(key: Union[str, Sequence[str]]) -> Optional[int]: + s = _extract_value(key) + if not s: + return None + return parse_date(s) + + +def extract_dates(key: Union[str, Sequence[str]]) -> Optional[TimeValues]: + parts = extract_strings(key) + if not parts: + return None + values: TimeValues = [] + + def push_range(first: str, last: str): + first_d = parse_date(first) + last_d = parse_date(last) + if first_d == last_d: + # the first and last numbers are the same, just treat it as a singe value + return first_d + if last_d > first_d: + # add the range as an array + return (first_d, last_d) + # the range is inverted, this is an error + raise ValidationFailedException(f"{key}: the given range is inverted") + + for part in parts: + if "-" not in part and ":" not in part: + # YYYYMMDD + values.append(parse_date(part)) + continue + if ":" in part: + # YYYY-MM-DD:YYYY-MM-DD + range_part = part.split(":", 2) + r = push_range(range_part[0], range_part[1]) + if r is None: + return None + values.append(r) + continue + # YYYY-MM-DD or YYYYMMDD-YYYYMMDD + # split on the dash + range_part = part.split("-") + if len(range_part) == 2: + # YYYYMMDD-YYYYMMDD + r = push_range(range_part[0], range_part[1]) + if r is None: + return None + values.append(r) + continue + # YYYY-MM-DD + values.append(parse_date(part)) + # success, return the list + return values + +def parse_source_signal_sets() -> List[SourceSignalSet]: + ds = request.values.get("data_source") + if ds: + # old version + require_any("signal", "signals", empty=True) + signals = extract_strings(("signals", "signal")) + if len(signals) == 1 and signals[0] == "*": + return [SourceSignalSet(ds, True)] + return [SourceSignalSet(ds, signals)] + + if ":" not in request.values.get("signal", ""): + raise ValidationFailedException("missing parameter: signal or (data_source and signal[s])") + + return parse_source_signal_arg() + + +def parse_geo_sets() -> List[GeoSet]: + geo_type = request.values.get("geo_type") + if geo_type: + # old version + require_any("geo_value", "geo_values", empty=True) + geo_values = extract_strings(("geo_values", "geo_value")) + if len(geo_values) == 1 and geo_values[0] == "*": + return [GeoSet(geo_type, True)] + return [GeoSet(geo_type, geo_values)] + + if ":" not in request.values.get("geo", ""): + raise ValidationFailedException("missing parameter: geo or (geo_type and geo_value[s])") + + return parse_geo_arg() + + +def parse_time_set() -> TimeSet: + time_type = request.values.get("time_type") + if time_type: + # old version + require_all("time_type", "time_values") + time_values = extract_dates("time_values") + return TimeSet(time_type, time_values) + + if ":" not in request.values.get("time", ""): + raise ValidationFailedException("missing parameter: time or (time_type and time_values)") + + return parse_time_arg() diff --git a/src/server/_query.py b/src/server/_query.py index 51ff4e182..c62649d02 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -18,8 +18,7 @@ from ._common import db from ._printer import create_printer, APrinter from ._exceptions import DatabaseErrorException -from ._validate import extract_strings -from ._params import GeoSet, SourceSignalSet, TimeSet +from ._params import extract_strings, GeoSet, SourceSignalSet, TimeSet from .utils import time_values_to_ranges, IntRange, TimeValues diff --git a/src/server/_validate.py b/src/server/_validate.py index ad82b8425..ffdd15232 100644 --- a/src/server/_validate.py +++ b/src/server/_validate.py @@ -55,132 +55,3 @@ def require_any(*values: str, empty=False) -> bool: if request.values.get(value) or (empty and value in request.values): return True raise ValidationFailedException(f"missing parameter: need one of [{', '.join(values)}]") - - -def _extract_value(key: Union[str, Sequence[str]]) -> Optional[str]: - if isinstance(key, str): - return request.values.get(key) - for k in key: - if k in request.values: - return request.values[k] - return None - - -def _extract_list_value(key: Union[str, Sequence[str]]) -> List[str]: - if isinstance(key, str): - return request.values.getlist(key) - for k in key: - if k in request.values: - return request.values.getlist(k) - return [] - - -def extract_strings(key: Union[str, Sequence[str]]) -> Optional[List[str]]: - s = _extract_list_value(key) - if not s: - # nothing to do - return None - # we can have multiple values - return [v for vs in s for v in vs.split(",")] - - -def extract_integer(key: Union[str, Sequence[str]]) -> Optional[int]: - s = _extract_value(key) - if not s: - # nothing to do - return None - try: - return int(s) - except ValueError: - raise ValidationFailedException(f"{key}: not a number: {s}") - - -def extract_integers(key: Union[str, Sequence[str]]) -> Optional[List[IntRange]]: - parts = extract_strings(key) - if not parts: - # nothing to do - return None - - def _parse_range(part: str): - if "-" not in part: - return int(part) - r = part.split("-", 2) - first = int(r[0]) - last = int(r[1]) - if first == last: - # the first and last numbers are the same, just treat it as a singe value - return first - elif last > first: - # add the range as an array - return (first, last) - # the range is inverted, this is an error - raise ValidationFailedException(f"{key}: the given range is inverted") - - try: - values = [_parse_range(part) for part in parts] - # check for invalid values - return None if any(v is None for v in values) else values - except ValueError as e: - raise ValidationFailedException(f"{key}: not a number: {str(e)}") - - -def parse_date(s: str) -> int: - # parses a given string in format YYYYMMDD or YYYY-MM-DD to a number in the form YYYYMMDD - try: - return int(s.replace("-", "")) - except ValueError: - raise ValidationFailedException(f"not a valid date: {s}") - - -def extract_date(key: Union[str, Sequence[str]]) -> Optional[int]: - s = _extract_value(key) - if not s: - return None - return parse_date(s) - - -def extract_dates(key: Union[str, Sequence[str]]) -> Optional[TimeValues]: - parts = extract_strings(key) - if not parts: - return None - values: TimeValues = [] - - def push_range(first: str, last: str): - first_d = parse_date(first) - last_d = parse_date(last) - if first_d == last_d: - # the first and last numbers are the same, just treat it as a singe value - return first_d - if last_d > first_d: - # add the range as an array - return (first_d, last_d) - # the range is inverted, this is an error - raise ValidationFailedException(f"{key}: the given range is inverted") - - for part in parts: - if "-" not in part and ":" not in part: - # YYYYMMDD - values.append(parse_date(part)) - continue - if ":" in part: - # YYYY-MM-DD:YYYY-MM-DD - range_part = part.split(":", 2) - r = push_range(range_part[0], range_part[1]) - if r is None: - return None - values.append(r) - continue - # YYYY-MM-DD or YYYYMMDD-YYYYMMDD - # split on the dash - range_part = part.split("-") - if len(range_part) == 2: - # YYYYMMDD-YYYYMMDD - r = push_range(range_part[0], range_part[1]) - if r is None: - return None - values.append(r) - continue - # YYYY-MM-DD - values.append(parse_date(part)) - # success, return the list - return values diff --git a/src/server/endpoints/afhsb.py b/src/server/endpoints/afhsb.py index 9f05eac9d..69c2d2431 100644 --- a/src/server/endpoints/afhsb.py +++ b/src/server/endpoints/afhsb.py @@ -3,8 +3,9 @@ from flask import Blueprint from .._config import AUTH +from .._params import extract_integers, extract_strings from .._query import execute_queries, filter_integers, filter_strings -from .._validate import check_auth_token, extract_integers, extract_strings, require_all +from .._validate import check_auth_token, require_all # first argument is the endpoint name bp = Blueprint("afhsb", __name__) diff --git a/src/server/endpoints/cdc.py b/src/server/endpoints/cdc.py index 6023d4f16..6b7b9450d 100644 --- a/src/server/endpoints/cdc.py +++ b/src/server/endpoints/cdc.py @@ -1,8 +1,9 @@ from flask import Blueprint from .._config import AUTH, NATION_REGION, REGION_TO_STATE -from .._validate import require_all, extract_strings, extract_integers, check_auth_token +from .._params import extract_strings, extract_integers from .._query import filter_strings, execute_queries, filter_integers +from .._validate import require_all, check_auth_token # first argument is the endpoint name bp = Blueprint("cdc", __name__) diff --git a/src/server/endpoints/covid_hosp_facility.py b/src/server/endpoints/covid_hosp_facility.py index 4bbe863c2..d1c9fad8a 100644 --- a/src/server/endpoints/covid_hosp_facility.py +++ b/src/server/endpoints/covid_hosp_facility.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("covid_hosp_facility", __name__) diff --git a/src/server/endpoints/covid_hosp_facility_lookup.py b/src/server/endpoints/covid_hosp_facility_lookup.py index 0fa94e99e..54a3b9183 100644 --- a/src/server/endpoints/covid_hosp_facility_lookup.py +++ b/src/server/endpoints/covid_hosp_facility_lookup.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_strings, require_any +from .._validate import require_any # first argument is the endpoint name bp = Blueprint("covid_hosp_facility_lookup", __name__) diff --git a/src/server/endpoints/covid_hosp_state_timeseries.py b/src/server/endpoints/covid_hosp_state_timeseries.py index 63540e5c8..a20e74d25 100644 --- a/src/server/endpoints/covid_hosp_state_timeseries.py +++ b/src/server/endpoints/covid_hosp_state_timeseries.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings, extract_date from .._query import execute_query, QueryBuilder -from .._validate import extract_integers, extract_strings, extract_date, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("covid_hosp_state_timeseries", __name__) diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index fa5ebc1e1..05f7cfc3f 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -14,25 +14,23 @@ GeoSet, SourceSignalSet, TimeSet, + extract_date, + extract_dates, + extract_integer, parse_geo_arg, parse_source_signal_arg, - parse_time_arg, parse_day_or_week_arg, parse_day_or_week_range_arg, parse_single_source_signal_arg, parse_single_time_arg, parse_single_geo_arg, + parse_geo_sets, + parse_source_signal_sets, + parse_time_set, ) from .._query import QueryBuilder, execute_query, run_query, parse_row, filter_fields from .._printer import create_printer, CSVPrinter -from .._validate import ( - extract_date, - extract_dates, - extract_integer, - extract_strings, - require_all, - require_any, -) +from .._validate import require_all from .._pandas import as_pandas, print_pandas from .covidcast_utils import compute_trend, compute_trends, compute_correlations, compute_trend_value, CovidcastMetaEntry from ..utils import shift_day_value, day_to_time_value, time_value_to_iso, time_value_to_day, shift_week_value, time_value_to_week, guess_time_value_is_day, week_to_time_value, TimeValues @@ -45,52 +43,6 @@ latest_table = "epimetric_latest_v" history_table = "epimetric_full_v" -def parse_source_signal_sets() -> List[SourceSignalSet]: - ds = request.values.get("data_source") - if ds: - # old version - require_any("signal", "signals", empty=True) - signals = extract_strings(("signals", "signal")) - if len(signals) == 1 and signals[0] == "*": - return [SourceSignalSet(ds, True)] - return [SourceSignalSet(ds, signals)] - - if ":" not in request.values.get("signal", ""): - raise ValidationFailedException("missing parameter: signal or (data_source and signal[s])") - - return parse_source_signal_arg() - - -def parse_geo_sets() -> List[GeoSet]: - geo_type = request.values.get("geo_type") - if geo_type: - # old version - require_any("geo_value", "geo_values", empty=True) - geo_values = extract_strings(("geo_values", "geo_value")) - if len(geo_values) == 1 and geo_values[0] == "*": - return [GeoSet(geo_type, True)] - return [GeoSet(geo_type, geo_values)] - - if ":" not in request.values.get("geo", ""): - raise ValidationFailedException("missing parameter: geo or (geo_type and geo_value[s])") - - return parse_geo_arg() - - -def parse_time_set() -> TimeSet: - time_type = request.values.get("time_type") - if time_type: - # old version - require_all("time_type", "time_values") - time_values = extract_dates("time_values") - return TimeSet(time_type, time_values) - - if ":" not in request.values.get("time", ""): - raise ValidationFailedException("missing parameter: time or (time_type and time_values)") - - return parse_time_arg() - - @bp.route("/", methods=("GET", "POST")) def handle(): source_signal_sets = parse_source_signal_sets() diff --git a/src/server/endpoints/covidcast_meta.py b/src/server/endpoints/covidcast_meta.py index 08e919d24..86eeb8b64 100644 --- a/src/server/endpoints/covidcast_meta.py +++ b/src/server/endpoints/covidcast_meta.py @@ -6,9 +6,9 @@ from sqlalchemy import text from .._common import db +from .._params import extract_strings from .._printer import create_printer from .._query import filter_fields -from .._validate import extract_strings from ..utils.logger import get_structured_logger bp = Blueprint("covidcast_meta", __name__) diff --git a/src/server/endpoints/covidcast_nowcast.py b/src/server/endpoints/covidcast_nowcast.py index ae47259f8..d71ff9404 100644 --- a/src/server/endpoints/covidcast_nowcast.py +++ b/src/server/endpoints/covidcast_nowcast.py @@ -1,11 +1,13 @@ from flask import Blueprint, request -from .._query import execute_query, filter_integers, filter_strings -from .._validate import ( +from .._params import ( extract_date, extract_dates, extract_integer, - extract_strings, + extract_strings +) +from .._query import execute_query, filter_integers, filter_strings +from .._validate import ( require_all, require_any, ) diff --git a/src/server/endpoints/dengue_nowcast.py b/src/server/endpoints/dengue_nowcast.py index cb5747a4d..f77f6bd18 100644 --- a/src/server/endpoints/dengue_nowcast.py +++ b/src/server/endpoints/dengue_nowcast.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("dengue_nowcast", __name__) diff --git a/src/server/endpoints/dengue_sensors.py b/src/server/endpoints/dengue_sensors.py index e1a8fbcf9..0837dc3fc 100644 --- a/src/server/endpoints/dengue_sensors.py +++ b/src/server/endpoints/dengue_sensors.py @@ -1,8 +1,9 @@ from flask import Blueprint from .._config import AUTH +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import check_auth_token, extract_integers, extract_strings, require_all +from .._validate import check_auth_token, require_all # first argument is the endpoint name bp = Blueprint("dengue_sensors", __name__) diff --git a/src/server/endpoints/ecdc_ili.py b/src/server/endpoints/ecdc_ili.py index af932d296..fe83c0d38 100644 --- a/src/server/endpoints/ecdc_ili.py +++ b/src/server/endpoints/ecdc_ili.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integer, extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("ecdc_ili", __name__) diff --git a/src/server/endpoints/flusurv.py b/src/server/endpoints/flusurv.py index a80159d09..97ccc3e59 100644 --- a/src/server/endpoints/flusurv.py +++ b/src/server/endpoints/flusurv.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integer, extract_integers, extract_strings, require_all +from .._validate import require_all bp = Blueprint("flusurv", __name__) diff --git a/src/server/endpoints/fluview.py b/src/server/endpoints/fluview.py index 8b92fa052..75e928c86 100644 --- a/src/server/endpoints/fluview.py +++ b/src/server/endpoints/fluview.py @@ -3,12 +3,14 @@ from flask import Blueprint from .._config import AUTH -from .._query import execute_queries, filter_integers, filter_strings -from .._validate import ( - check_auth_token, +from .._params import ( extract_integer, extract_integers, extract_strings, +) +from .._query import execute_queries, filter_integers, filter_strings +from .._validate import ( + check_auth_token, require_all, ) diff --git a/src/server/endpoints/fluview_clinicial.py b/src/server/endpoints/fluview_clinicial.py index dd095a7d8..e5d201afd 100644 --- a/src/server/endpoints/fluview_clinicial.py +++ b/src/server/endpoints/fluview_clinicial.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integer, extract_integers, extract_strings, require_all +from .._validate import require_all bp = Blueprint("fluview_clinical", __name__) diff --git a/src/server/endpoints/gft.py b/src/server/endpoints/gft.py index ab776c6e5..343f565f4 100644 --- a/src/server/endpoints/gft.py +++ b/src/server/endpoints/gft.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("gft", __name__) diff --git a/src/server/endpoints/ght.py b/src/server/endpoints/ght.py index b30a42abc..ab858e79c 100644 --- a/src/server/endpoints/ght.py +++ b/src/server/endpoints/ght.py @@ -1,8 +1,9 @@ from flask import Blueprint, request from .._config import AUTH +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import check_auth_token, extract_integers, extract_strings, require_all +from .._validate import check_auth_token, require_all # first argument is the endpoint name bp = Blueprint("ght", __name__) diff --git a/src/server/endpoints/kcdc_ili.py b/src/server/endpoints/kcdc_ili.py index 32933eb3e..fc9328898 100644 --- a/src/server/endpoints/kcdc_ili.py +++ b/src/server/endpoints/kcdc_ili.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integer, extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integer, extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("kcdc_ili", __name__) diff --git a/src/server/endpoints/nidss_dengue.py b/src/server/endpoints/nidss_dengue.py index 131f6eb9a..8d7c12624 100644 --- a/src/server/endpoints/nidss_dengue.py +++ b/src/server/endpoints/nidss_dengue.py @@ -2,8 +2,9 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_queries, filter_integers -from .._validate import extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("nidss_dengue", __name__) diff --git a/src/server/endpoints/nidss_flu.py b/src/server/endpoints/nidss_flu.py index 989a41a3d..8eb7d3b56 100644 --- a/src/server/endpoints/nidss_flu.py +++ b/src/server/endpoints/nidss_flu.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integer, extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integer, extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("nidss_flu", __name__) diff --git a/src/server/endpoints/norostat.py b/src/server/endpoints/norostat.py index 9586f8e3f..24867a8d4 100644 --- a/src/server/endpoints/norostat.py +++ b/src/server/endpoints/norostat.py @@ -1,8 +1,9 @@ from flask import Blueprint, request from .._config import AUTH +from .._params import extract_integers from .._query import execute_query, filter_integers, filter_strings -from .._validate import check_auth_token, extract_integers, require_all +from .._validate import check_auth_token, require_all # first argument is the endpoint name bp = Blueprint("norostat", __name__) diff --git a/src/server/endpoints/nowcast.py b/src/server/endpoints/nowcast.py index 77ed84401..77c535ee6 100644 --- a/src/server/endpoints/nowcast.py +++ b/src/server/endpoints/nowcast.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("nowcast", __name__) diff --git a/src/server/endpoints/paho_dengue.py b/src/server/endpoints/paho_dengue.py index 0a50885c2..e793a7c17 100644 --- a/src/server/endpoints/paho_dengue.py +++ b/src/server/endpoints/paho_dengue.py @@ -1,7 +1,8 @@ from flask import Blueprint +from .._params import extract_integer, extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import extract_integer, extract_integers, extract_strings, require_all +from .._validate import require_all # first argument is the endpoint name bp = Blueprint("paho_dengue", __name__) diff --git a/src/server/endpoints/quidel.py b/src/server/endpoints/quidel.py index f96b4dd20..081706190 100644 --- a/src/server/endpoints/quidel.py +++ b/src/server/endpoints/quidel.py @@ -1,8 +1,9 @@ from flask import Blueprint from .._config import AUTH +from .._params import extract_integers, extract_strings from .._query import execute_query, QueryBuilder -from .._validate import check_auth_token, extract_integers, extract_strings, require_all +from .._validate import check_auth_token, require_all # first argument is the endpoint name bp = Blueprint("quidel", __name__) diff --git a/src/server/endpoints/sensors.py b/src/server/endpoints/sensors.py index 68199e2b1..f803dd396 100644 --- a/src/server/endpoints/sensors.py +++ b/src/server/endpoints/sensors.py @@ -1,14 +1,16 @@ from flask import Blueprint from .._config import AUTH, GRANULAR_SENSOR_AUTH_TOKENS, OPEN_SENSORS -from .._validate import ( - require_all, +from .._exceptions import EpiDataException +from .._params import ( extract_strings, extract_integers, - resolve_auth_token, ) from .._query import filter_strings, execute_query, filter_integers -from .._exceptions import EpiDataException +from .._validate import ( + require_all, + resolve_auth_token, +) from typing import List # first argument is the endpoint name diff --git a/src/server/endpoints/twitter.py b/src/server/endpoints/twitter.py index 78b297ef8..41cbe3492 100644 --- a/src/server/endpoints/twitter.py +++ b/src/server/endpoints/twitter.py @@ -1,11 +1,13 @@ from flask import Blueprint, request from .._config import AUTH, NATION_REGION, REGION_TO_STATE +from .._params import ( + extract_integers, + extract_strings, +) from .._query import execute_queries, filter_dates, filter_integers, filter_strings from .._validate import ( check_auth_token, - extract_integers, - extract_strings, require_all, require_any, ) diff --git a/src/server/endpoints/wiki.py b/src/server/endpoints/wiki.py index a6bfcb27f..61139578f 100644 --- a/src/server/endpoints/wiki.py +++ b/src/server/endpoints/wiki.py @@ -1,7 +1,8 @@ from flask import Blueprint, request +from .._params import extract_integers, extract_strings from .._query import execute_query, filter_dates, filter_integers, filter_strings -from .._validate import extract_integers, extract_strings, require_all, require_any +from .._validate import require_all, require_any # first argument is the endpoint name bp = Blueprint("wiki", __name__) diff --git a/tests/server/test_params.py b/tests/server/test_params.py index 810ebfa3c..177ff5cba 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -7,6 +7,11 @@ # from flask.testing import FlaskClient from delphi.epidata.server._common import app from delphi.epidata.server._params import ( + extract_strings, + extract_integers, + extract_integer, + extract_date, + extract_dates, parse_geo_arg, parse_single_geo_arg, parse_source_signal_arg, @@ -366,3 +371,128 @@ def test_parse_day_arg(self): self.assertRaises(ValidationFailedException, parse_day_arg, "time") with app.test_request_context("/?time=week:20121010"): self.assertRaises(ValidationFailedException, parse_day_arg, "time") + + def test_extract_strings(self): + with self.subTest("empty"): + with app.test_request_context("/"): + self.assertIsNone(extract_strings("s")) + with self.subTest("single"): + with app.test_request_context("/?s=a"): + self.assertEqual(extract_strings("s"), ["a"]) + with self.subTest("multiple"): + with app.test_request_context("/?s=a,b"): + self.assertEqual(extract_strings("s"), ["a", "b"]) + with self.subTest("multiple param"): + with app.test_request_context("/?s=a&s=b"): + self.assertEqual(extract_strings("s"), ["a", "b"]) + with self.subTest("multiple param mixed"): + with app.test_request_context("/?s=a&s=b,c"): + self.assertEqual(extract_strings("s"), ["a", "b", "c"]) + + def test_extract_integer(self): + with self.subTest("empty"): + with app.test_request_context("/"): + self.assertIsNone(extract_integer("s")) + with self.subTest("single"): + with app.test_request_context("/?s=1"): + self.assertEqual(extract_integer("s"), 1) + with self.subTest("not a number"): + with app.test_request_context("/?s=a"): + self.assertRaises(ValidationFailedException, lambda: extract_integer("s")) + + def test_extract_integers(self): + with self.subTest("empty"): + with app.test_request_context("/"): + self.assertIsNone(extract_integers("s")) + with self.subTest("single"): + with app.test_request_context("/?s=1"): + self.assertEqual(extract_integers("s"), [1]) + with self.subTest("multiple"): + with app.test_request_context("/?s=1,2"): + self.assertEqual(extract_integers("s"), [1,2]) + with self.subTest("multiple param"): + with app.test_request_context("/?s=1&s=2"): + self.assertEqual(extract_integers("s"), [1,2]) + with self.subTest("multiple param mixed"): + with app.test_request_context("/?s=1&s=2,3"): + self.assertEqual(extract_integers("s"), [1, 2, 3]) + + with self.subTest("not a number"): + with app.test_request_context("/?s=a"): + self.assertRaises(ValidationFailedException, lambda: extract_integers("s")) + + with self.subTest("simple range"): + with app.test_request_context("/?s=1-2"): + self.assertEqual(extract_integers("s"), [(1, 2)]) + with self.subTest("inverted range"): + with app.test_request_context("/?s=2-1"): + self.assertRaises(ValidationFailedException, lambda: extract_integers("s")) + with self.subTest("single range"): + with app.test_request_context("/?s=1-1"): + self.assertEqual(extract_integers("s"), [1]) + + def test_extract_date(self): + with self.subTest("empty"): + with app.test_request_context("/"): + self.assertIsNone(extract_date("s")) + with self.subTest("single"): + with app.test_request_context("/?s=2020-01-01"): + self.assertEqual(extract_date("s"), 20200101) + with app.test_request_context("/?s=20200101"): + self.assertEqual(extract_date("s"), 20200101) + with self.subTest("not a date"): + with app.test_request_context("/?s=abc"): + self.assertRaises(ValidationFailedException, lambda: extract_date("s")) + + def test_extract_dates(self): + with self.subTest("empty"): + with app.test_request_context("/"): + self.assertIsNone(extract_dates("s")) + with self.subTest("single"): + with app.test_request_context("/?s=20200101"): + self.assertEqual(extract_dates("s"), [20200101]) + with self.subTest("multiple"): + with app.test_request_context("/?s=20200101,20200102"): + self.assertEqual(extract_dates("s"), [20200101, 20200102]) + with self.subTest("multiple param"): + with app.test_request_context("/?s=20200101&s=20200102"): + self.assertEqual(extract_dates("s"), [20200101, 20200102]) + with self.subTest("multiple param mixed"): + with app.test_request_context("/?s=20200101&s=20200102,20200103"): + self.assertEqual(extract_dates("s"), [20200101, 20200102, 20200103]) + with self.subTest("single iso"): + with app.test_request_context("/?s=2020-01-01"): + self.assertEqual(extract_dates("s"), [20200101]) + with self.subTest("multiple iso"): + with app.test_request_context("/?s=2020-01-01,2020-01-02"): + self.assertEqual(extract_dates("s"), [20200101, 20200102]) + with self.subTest("multiple param iso"): + with app.test_request_context("/?s=2020-01-01&s=2020-01-02"): + self.assertEqual(extract_dates("s"), [20200101, 20200102]) + with self.subTest("multiple param mixed iso"): + with app.test_request_context("/?s=2020-01-01&s=2020-01-02,2020-01-03"): + self.assertEqual(extract_dates("s"), [20200101, 20200102, 20200103]) + + with self.subTest("not a date"): + with app.test_request_context("/?s=a"): + self.assertRaises(ValidationFailedException, lambda: extract_dates("s")) + + with self.subTest("simple range"): + with app.test_request_context("/?s=20200101-20200102"): + self.assertEqual(extract_dates("s"), [(20200101, 20200102)]) + with self.subTest("inverted range"): + with app.test_request_context("/?s=20200102-20200101"): + self.assertRaises(ValidationFailedException, lambda: extract_dates("s")) + with self.subTest("single range"): + with app.test_request_context("/?s=20200101-20200101"): + self.assertEqual(extract_dates("s"), [20200101]) + + with self.subTest("simple range iso"): + with app.test_request_context("/?s=2020-01-01:2020-01-02"): + self.assertEqual(extract_dates("s"), [(20200101, 20200102)]) + with self.subTest("inverted range iso"): + with app.test_request_context("/?s=2020-01-02:2020-01-01"): + self.assertRaises(ValidationFailedException, lambda: extract_dates("s")) + with self.subTest("single range iso"): + with app.test_request_context("/?s=2020-01-01:2020-01-01"): + self.assertEqual(extract_dates("s"), [20200101]) diff --git a/tests/server/test_validate.py b/tests/server/test_validate.py index c254950ff..ca45c78e2 100644 --- a/tests/server/test_validate.py +++ b/tests/server/test_validate.py @@ -11,11 +11,6 @@ check_auth_token, require_all, require_any, - extract_strings, - extract_integers, - extract_integer, - extract_date, - extract_dates ) from delphi.epidata.server._exceptions import ( ValidationFailedException, @@ -110,129 +105,3 @@ def test_require_any(self): with self.subTest("one options given with is empty but ok"): with app.test_request_context("/?abc="): self.assertTrue(require_any("abc", empty=True)) - - def test_extract_strings(self): - with self.subTest("empty"): - with app.test_request_context("/"): - self.assertIsNone(extract_strings("s")) - with self.subTest("single"): - with app.test_request_context("/?s=a"): - self.assertEqual(extract_strings("s"), ["a"]) - with self.subTest("multiple"): - with app.test_request_context("/?s=a,b"): - self.assertEqual(extract_strings("s"), ["a", "b"]) - with self.subTest("multiple param"): - with app.test_request_context("/?s=a&s=b"): - self.assertEqual(extract_strings("s"), ["a", "b"]) - with self.subTest("multiple param mixed"): - with app.test_request_context("/?s=a&s=b,c"): - self.assertEqual(extract_strings("s"), ["a", "b", "c"]) - - def test_extract_integer(self): - with self.subTest("empty"): - with app.test_request_context("/"): - self.assertIsNone(extract_integer("s")) - with self.subTest("single"): - with app.test_request_context("/?s=1"): - self.assertEqual(extract_integer("s"), 1) - with self.subTest("not a number"): - with app.test_request_context("/?s=a"): - self.assertRaises(ValidationFailedException, lambda: extract_integer("s")) - - def test_extract_integers(self): - with self.subTest("empty"): - with app.test_request_context("/"): - self.assertIsNone(extract_integers("s")) - with self.subTest("single"): - with app.test_request_context("/?s=1"): - self.assertEqual(extract_integers("s"), [1]) - with self.subTest("multiple"): - with app.test_request_context("/?s=1,2"): - self.assertEqual(extract_integers("s"), [1,2]) - with self.subTest("multiple param"): - with app.test_request_context("/?s=1&s=2"): - self.assertEqual(extract_integers("s"), [1,2]) - with self.subTest("multiple param mixed"): - with app.test_request_context("/?s=1&s=2,3"): - self.assertEqual(extract_integers("s"), [1, 2, 3]) - - with self.subTest("not a number"): - with app.test_request_context("/?s=a"): - self.assertRaises(ValidationFailedException, lambda: extract_integers("s")) - - with self.subTest("simple range"): - with app.test_request_context("/?s=1-2"): - self.assertEqual(extract_integers("s"), [(1, 2)]) - with self.subTest("inverted range"): - with app.test_request_context("/?s=2-1"): - self.assertRaises(ValidationFailedException, lambda: extract_integers("s")) - with self.subTest("single range"): - with app.test_request_context("/?s=1-1"): - self.assertEqual(extract_integers("s"), [1]) - - def test_extract_date(self): - with self.subTest("empty"): - with app.test_request_context("/"): - self.assertIsNone(extract_date("s")) - with self.subTest("single"): - with app.test_request_context("/?s=2020-01-01"): - self.assertEqual(extract_date("s"), 20200101) - with app.test_request_context("/?s=20200101"): - self.assertEqual(extract_date("s"), 20200101) - with self.subTest("not a date"): - with app.test_request_context("/?s=abc"): - self.assertRaises(ValidationFailedException, lambda: extract_date("s")) - - def test_extract_dates(self): - with self.subTest("empty"): - with app.test_request_context("/"): - self.assertIsNone(extract_dates("s")) - with self.subTest("single"): - with app.test_request_context("/?s=20200101"): - self.assertEqual(extract_dates("s"), [20200101]) - with self.subTest("multiple"): - with app.test_request_context("/?s=20200101,20200102"): - self.assertEqual(extract_dates("s"), [20200101, 20200102]) - with self.subTest("multiple param"): - with app.test_request_context("/?s=20200101&s=20200102"): - self.assertEqual(extract_dates("s"), [20200101, 20200102]) - with self.subTest("multiple param mixed"): - with app.test_request_context("/?s=20200101&s=20200102,20200103"): - self.assertEqual(extract_dates("s"), [20200101, 20200102, 20200103]) - with self.subTest("single iso"): - with app.test_request_context("/?s=2020-01-01"): - self.assertEqual(extract_dates("s"), [20200101]) - with self.subTest("multiple iso"): - with app.test_request_context("/?s=2020-01-01,2020-01-02"): - self.assertEqual(extract_dates("s"), [20200101, 20200102]) - with self.subTest("multiple param iso"): - with app.test_request_context("/?s=2020-01-01&s=2020-01-02"): - self.assertEqual(extract_dates("s"), [20200101, 20200102]) - with self.subTest("multiple param mixed iso"): - with app.test_request_context("/?s=2020-01-01&s=2020-01-02,2020-01-03"): - self.assertEqual(extract_dates("s"), [20200101, 20200102, 20200103]) - - with self.subTest("not a date"): - with app.test_request_context("/?s=a"): - self.assertRaises(ValidationFailedException, lambda: extract_dates("s")) - - with self.subTest("simple range"): - with app.test_request_context("/?s=20200101-20200102"): - self.assertEqual(extract_dates("s"), [(20200101, 20200102)]) - with self.subTest("inverted range"): - with app.test_request_context("/?s=20200102-20200101"): - self.assertRaises(ValidationFailedException, lambda: extract_dates("s")) - with self.subTest("single range"): - with app.test_request_context("/?s=20200101-20200101"): - self.assertEqual(extract_dates("s"), [20200101]) - - with self.subTest("simple range iso"): - with app.test_request_context("/?s=2020-01-01:2020-01-02"): - self.assertEqual(extract_dates("s"), [(20200101, 20200102)]) - with self.subTest("inverted range iso"): - with app.test_request_context("/?s=2020-01-02:2020-01-01"): - self.assertRaises(ValidationFailedException, lambda: extract_dates("s")) - with self.subTest("single range iso"): - with app.test_request_context("/?s=2020-01-01:2020-01-01"): - self.assertEqual(extract_dates("s"), [20200101]) -