diff --git a/requirements.txt b/requirements.txt index 945ac11ea..8e240e526 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ scipy==1.6.2 tenacity==7.0.0 newrelic epiweeks==2.1.2 +typing-extensions diff --git a/src/server/_params.py b/src/server/_params.py index fa4f63483..9088a8902 100644 --- a/src/server/_params.py +++ b/src/server/_params.py @@ -7,7 +7,7 @@ from ._exceptions import ValidationFailedException -from .utils import days_in_range, weeks_in_range, guess_time_value_is_day +from .utils import days_in_range, weeks_in_range, guess_time_value_is_day, guess_time_value_is_week, TimeValues, days_to_ranges, weeks_to_ranges def _parse_common_multi_arg(key: str) -> List[Tuple[str, Union[bool, Sequence[str]]]]: @@ -108,7 +108,7 @@ def parse_single_source_signal_arg(key: str) -> SourceSignalPair: @dataclass class TimePair: time_type: str - time_values: Union[bool, Sequence[Union[int, Tuple[int, int]]]] + time_values: Union[bool, TimeValues] @property def is_week(self) -> bool: @@ -129,6 +129,16 @@ def count(self) -> float: return sum(1 if isinstance(v, int) else weeks_in_range(v) for v in self.time_values) return sum(1 if isinstance(v, int) else days_in_range(v) for v in self.time_values) + def to_ranges(self): + """ + returns this pair with times converted to ranges + """ + if isinstance(self.time_values, bool): + return TimePair(self.time_type, self.time_values) + if self.time_type == 'week': + return TimePair(self.time_type, weeks_to_ranges(self.time_values)) + return TimePair(self.time_type, days_to_ranges(self.time_values)) + def _verify_range(start: int, end: int) -> Union[int, Tuple[int, int]]: if start == end: @@ -205,8 +215,28 @@ def _parse_time_pair(time_type: str, time_values: Union[bool, Sequence[str]]) -> raise ValidationFailedException(f'time param: {time_type} is not one of "day" or "week"') -def parse_time_arg(key: str = "time") -> List[TimePair]: - return [_parse_time_pair(time_type, time_values) for [time_type, time_values] in _parse_common_multi_arg(key)] +def parse_time_arg(key: str = "time") -> Optional[TimePair]: + time_pairs = [_parse_time_pair(time_type, time_values) for [time_type, time_values] in _parse_common_multi_arg(key)] + + # single value + if len(time_pairs) == 0: + return None + if len(time_pairs) == 1: + return time_pairs[0] + + # make sure 'day' and 'week' aren't mixed + time_types = set(time_pair.time_type for time_pair in time_pairs) + if len(time_types) >= 2: + raise ValidationFailedException(f'{key}: {time_pairs} mixes "day" and "week" time types') + + # merge all time pairs into one + merged = [] + for time_pair in time_pairs: + if time_pair.time_values == True: + return time_pair + else: + merged.extend(time_pair.time_values) + return TimePair(time_pairs[0].time_type, merged).to_ranges() def parse_single_time_arg(key: str) -> TimePair: @@ -255,25 +285,26 @@ def parse_week_range_arg(key: str) -> Tuple[int, int]: raise ValidationFailedException(f"{key} must match YYYYWW-YYYYWW") return r -def parse_day_or_week_arg(key: str, default_value: Optional[int] = None) -> Tuple[int, bool]: +def parse_day_or_week_arg(key: str, default_value: Optional[int] = None) -> TimePair: v = request.values.get(key) if not v: if default_value is not None: - return default_value, guess_time_value_is_day(default_value) + time_type = "day" if guess_time_value_is_day(default_value) else "week" + return TimePair(time_type, [default_value]) raise ValidationFailedException(f"{key} param is required") # format is either YYYY-MM-DD or YYYYMMDD or YYYYMM - is_week = len(v) == 6 + is_week = guess_time_value_is_week(v) if is_week: - return parse_week_arg(key), False - return parse_day_arg(key), True + return TimePair("week", [parse_week_arg(key)]) + return TimePair("day", [parse_day_arg(key)]) -def parse_day_or_week_range_arg(key: str) -> Tuple[Tuple[int, int], bool]: +def parse_day_or_week_range_arg(key: str) -> TimePair: v = request.values.get(key) if not v: raise ValidationFailedException(f"{key} param is required") # format is either YYYY-MM-DD--YYYY-MM-DD or YYYYMMDD-YYYYMMDD or YYYYMM-YYYYMM # so if the first before the - has length 6, it must be a week - is_week = len(v.split('-', 2)[0]) == 6 + is_week = guess_time_value_is_week(v.split('-', 2)[0]) if is_week: - return parse_week_range_arg(key), False - return parse_day_range_arg(key), True + return TimePair("week", [parse_week_range_arg(key)]) + return TimePair("day", [parse_day_range_arg(key)]) diff --git a/src/server/_query.py b/src/server/_query.py index 1029c5e2c..0b34310ab 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -20,9 +20,9 @@ from ._db import metadata from ._printer import create_printer, APrinter from ._exceptions import DatabaseErrorException -from ._validate import DateRange, extract_strings +from ._validate import extract_strings from ._params import GeoPair, SourceSignalPair, TimePair -from .utils import time_values_to_ranges, days_to_ranges, weeks_to_ranges +from .utils import time_values_to_ranges, TimeValues def date_string(value: int) -> str: @@ -36,7 +36,7 @@ def date_string(value: int) -> str: def to_condition( field: str, - value: Union[Tuple[str, str], str, Tuple[int, int], int], + value: Union[str, Tuple[int, int], int], param_key: str, params: Dict[str, Any], formatter=lambda x: x, @@ -52,7 +52,7 @@ def to_condition( def filter_values( field: str, - values: Optional[Sequence[Union[Tuple[str, str], str, Tuple[int, int], int]]], + values: Optional[Sequence[Union[str, Tuple[int, int], int]]], param_key: str, params: Dict[str, Any], formatter=lambda x: x, @@ -68,7 +68,7 @@ def filter_values( def filter_strings( field: str, - values: Optional[Sequence[Union[Tuple[str, str], str]]], + values: Optional[Sequence[str]], param_key: str, params: Dict[str, Any], ): @@ -86,7 +86,7 @@ def filter_integers( def filter_dates( field: str, - values: Optional[Sequence[Union[Tuple[int, int], int]]], + values: Optional[TimeValues], param_key: str, params: Dict[str, Any], ): @@ -171,32 +171,29 @@ def filter_pair(pair: SourceSignalPair, i) -> str: return f"({' OR '.join(parts)})" -def filter_time_pairs( +def filter_time_pair( type_field: str, time_field: str, - values: Sequence[TimePair], + pair: Optional[TimePair], param_key: str, params: Dict[str, Any], ) -> str: """ - returns the SQL sub query to filter by the given time pairs + returns the SQL sub query to filter by the given time pair """ - - def filter_pair(pair: TimePair, i) -> str: - type_param = f"{param_key}_{i}t" - params[type_param] = pair.time_type - if isinstance(pair.time_values, bool) and pair.time_values: - return f"{type_field} = :{type_param}" - ranges = weeks_to_ranges(pair.time_values) if pair.is_week else days_to_ranges(pair.time_values) - return f"({type_field} = :{type_param} AND {filter_integers(time_field, cast(Sequence[Union[int, Tuple[int,int]]], ranges), type_param, params)})" - - parts = [filter_pair(p, i) for i, p in enumerate(values)] - - if not parts: - # something has to be selected + # safety path; should normally not be reached as time pairs are enforced by the API + if not pair: return "FALSE" - return f"({' OR '.join(parts)})" + type_param = f"{param_key}_0t" + params[type_param] = pair.time_type + if isinstance(pair.time_values, bool) and pair.time_values: + parts = f"{type_field} = :{type_param}" + else: + ranges = pair.to_ranges().time_values + parts = f"({type_field} = :{type_param} AND {filter_integers(time_field, ranges, type_param, params)})" + + return f"({parts})" def parse_row( @@ -393,7 +390,7 @@ def where(self, **kvargs: Dict[str, Any]) -> "QueryBuilder": def where_strings( self, field: str, - values: Optional[Sequence[Union[Tuple[str, str], str]]], + values: Optional[Sequence[str]], param_key: Optional[str] = None, ) -> "QueryBuilder": fq_field = f"{self.alias}.{field}" if "." not in field else field @@ -413,16 +410,6 @@ def where_integers( self.conditions.append(filter_integers(fq_field, values, param_key or field, self.params)) return self - def where_dates( - self, - field: str, - values: Optional[Sequence[Union[Tuple[int, int], int]]], - param_key: Optional[str] = None, - ) -> "QueryBuilder": - fq_field = self._fq_field(field) - self.conditions.append(filter_dates(fq_field, values, param_key or field, self.params)) - return self - def where_geo_pairs( self, type_field: str, @@ -463,17 +450,17 @@ def where_source_signal_pairs( ) return self - def where_time_pairs( + def where_time_pair( self, type_field: str, value_field: str, - values: Sequence[TimePair], + values: Optional[TimePair], param_key: Optional[str] = None, ) -> "QueryBuilder": fq_type_field = self._fq_field(type_field) fq_value_field = self._fq_field(value_field) self.conditions.append( - filter_time_pairs( + filter_time_pair( fq_type_field, fq_value_field, values, diff --git a/src/server/_validate.py b/src/server/_validate.py index 3b91e5570..e24644721 100644 --- a/src/server/_validate.py +++ b/src/server/_validate.py @@ -3,6 +3,7 @@ from flask import request from ._exceptions import UnAuthenticatedException, ValidationFailedException +from .utils import TimeValues def resolve_auth_token() -> Optional[str]: @@ -141,14 +142,11 @@ def extract_date(key: Union[str, Sequence[str]]) -> Optional[int]: return parse_date(s) -DateRange = Union[Tuple[int, int], int] - - -def extract_dates(key: Union[str, Sequence[str]]) -> Optional[List[DateRange]]: +def extract_dates(key: Union[str, Sequence[str]]) -> Optional[TimeValues]: parts = extract_strings(key) if not parts: return None - values: List[Union[Tuple[int, int], int]] = [] + values: TimeValues = [] def push_range(first: str, last: str): first_d = parse_date(first) diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index 4a636d891..2a4b43c45 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -35,7 +35,7 @@ ) from .._pandas import as_pandas, print_pandas from .covidcast_utils import compute_trend, compute_trends, compute_correlations, compute_trend_value, CovidcastMetaEntry -from ..utils import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, shift_week_value, week_value_to_week, guess_time_value_is_day, week_to_time_value +from ..utils import shift_day_value, day_to_time_value, time_value_to_iso, time_value_to_day, shift_week_value, time_value_to_week, guess_time_value_is_day, week_to_time_value, TimeValues from .covidcast_utils.model import TimeType, count_signal_time_types, data_sources, create_source_signal_alias_mapper # first argument is the endpoint name @@ -75,13 +75,13 @@ def parse_geo_pairs() -> List[GeoPair]: return parse_geo_arg() -def parse_time_pairs() -> List[TimePair]: +def parse_time_pairs() -> TimePair: time_type = request.values.get("time_type") if time_type: # old version require_all("time_type", "time_values") time_values = extract_dates("time_values") - return [TimePair(time_type, time_values)] + return TimePair(time_type, time_values) if ":" not in request.values.get("time", ""): raise ValidationFailedException("missing parameter: time or (time_type and time_values)") @@ -89,7 +89,7 @@ def parse_time_pairs() -> List[TimePair]: return parse_time_arg() -def _handle_lag_issues_as_of(q: QueryBuilder, issues: Optional[List[Union[Tuple[int, int], int]]] = None, lag: Optional[int] = None, as_of: Optional[int] = None): +def _handle_lag_issues_as_of(q: QueryBuilder, issues: Optional[TimeValues] = None, lag: Optional[int] = None, as_of: Optional[int] = None): if issues: q.retable(history_table) q.where_integers("issue", issues) @@ -115,7 +115,7 @@ def _handle_lag_issues_as_of(q: QueryBuilder, issues: Optional[List[Union[Tuple[ def handle(): source_signal_pairs = parse_source_signal_pairs() source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) - time_pairs = parse_time_pairs() + time_pair = parse_time_pairs() geo_pairs = parse_geo_pairs() as_of = extract_date("as_of") @@ -143,7 +143,7 @@ def handle(): q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pairs("time_type", "time_value", time_pairs) + q.where_time_pair("time_type", "time_value", time_pair) _handle_lag_issues_as_of(q, issues, lag, as_of) @@ -172,8 +172,10 @@ def handle_trend(): source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) geo_pairs = parse_geo_pairs() - time_window, is_day = parse_day_or_week_range_arg("window") - time_value, is_also_day = parse_day_or_week_arg("date") + time_window = parse_day_or_week_range_arg("window") + is_day = time_window.is_day + time_pair = parse_day_or_week_arg("date") + time_value, is_also_day = time_pair.time_values[0], time_pair.is_day if is_day != is_also_day: raise ValidationFailedException("mixing weeks with day arguments") _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) @@ -182,7 +184,7 @@ def handle_trend(): base_shift = extract_integer("basis_shift") if base_shift is None: base_shift = 7 - basis_time_value = shift_time_value(time_value, -1 * base_shift) if is_day else shift_week_value(time_value, -1 * base_shift) + basis_time_value = shift_day_value(time_value, -1 * base_shift) if is_day else shift_week_value(time_value, -1 * base_shift) # build query q = QueryBuilder(latest_table, "t") @@ -195,7 +197,7 @@ def handle_trend(): q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pairs("time_type", "time_value", [TimePair("day" if is_day else "week", [time_window])]) + q.where_time_pair("time_type", "time_value", time_window) # fetch most recent issue fast _handle_lag_issues_as_of(q, None, None, None) @@ -228,7 +230,8 @@ def handle_trendseries(): source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) geo_pairs = parse_geo_pairs() - time_window, is_day = parse_day_or_week_range_arg("window") + time_window = parse_day_or_week_range_arg("window") + is_day = time_window.is_day _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) basis_shift = extract_integer(("basis", "basis_shift")) if basis_shift is None: @@ -245,14 +248,14 @@ def handle_trendseries(): q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pairs("time_type", "time_value", [TimePair("day" if is_day else "week", [time_window])]) + q.where_time_pair("time_type", "time_value", time_window) # fetch most recent issue fast _handle_lag_issues_as_of(q, None, None, None) p = create_printer() - shifter = lambda x: shift_time_value(x, -basis_shift) + shifter = lambda x: shift_day_value(x, -basis_shift) if not is_day: shifter = lambda x: shift_week_value(x, -basis_shift) @@ -283,7 +286,8 @@ def handle_correlation(): daily_signals, weekly_signals = count_signal_time_types(other_pairs + [reference]) source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(other_pairs + [reference]) geo_pairs = parse_geo_arg() - time_window, is_day = parse_day_or_week_range_arg("window") + time_window = parse_day_or_week_range_arg("window") + is_day = time_window.is_day _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) lag = extract_integer("lag") @@ -305,14 +309,14 @@ def handle_correlation(): source_signal_pairs, ) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pairs("time_type", "time_value", [TimePair("day" if is_day else "week", [time_window])]) + q.where_time_pair("time_type", "time_value", time_window) df = as_pandas(str(q), q.params) if is_day: df["time_value"] = to_datetime(df["time_value"], format="%Y%m%d") else: # week but convert to date for simpler shifting - df["time_value"] = to_datetime(df["time_value"].apply(lambda v: week_value_to_week(v).startdate())) + df["time_value"] = to_datetime(df["time_value"].apply(lambda v: time_value_to_week(v).startdate())) p = create_printer() @@ -353,8 +357,10 @@ def handle_export(): source_signal_pairs = [SourceSignalPair(source, [signal])] daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) - start_day, is_day = parse_day_or_week_arg("start_day", 202001 if weekly_signals > 0 else 20200401) - end_day, is_end_day = parse_day_or_week_arg("end_day", 202020 if weekly_signals > 0 else 20200901) + start_pair = parse_day_or_week_arg("start_day", 202001 if weekly_signals > 0 else 20200401) + start_day, is_day = start_pair.time_values[0], start_pair.is_day + end_pair = parse_day_or_week_arg("end_day", 202020 if weekly_signals > 0 else 20200901) + end_day, is_end_day = end_pair.time_values[0], end_pair.is_day if is_day != is_end_day: raise ValidationFailedException("mixing weeks with day arguments") _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) @@ -365,7 +371,7 @@ def handle_export(): if geo_values != "*": geo_values = geo_values.split(",") - as_of, is_as_of_day = parse_day_or_week_arg("as_of") if "as_of" in request.args else (None, is_day) + as_of, is_as_of_day = (parse_day_or_week_arg("as_of").time_values[0], parse_day_or_week_arg("as_of").is_day) if "as_of" in request.args else (None, is_day) if is_day != is_as_of_day: raise ValidationFailedException("mixing weeks with day arguments") @@ -375,12 +381,12 @@ def handle_export(): q.set_fields(["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "source"], [], []) q.set_order("time_value", "geo_value") q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_time_pairs("time_type", "time_value", [TimePair("day" if is_day else "week", [(start_day, end_day)])]) + q.where_time_pair("time_type", "time_value", TimePair("day" if is_day else "week", [(start_day, end_day)])) q.where_geo_pairs("geo_type", "geo_value", [GeoPair(geo_type, True if geo_values == "*" else geo_values)]) _handle_lag_issues_as_of(q, None, None, as_of) - format_date = time_value_to_iso if is_day else lambda x: week_value_to_week(x).cdcformat() + format_date = time_value_to_iso if is_day else lambda x: time_value_to_week(x).cdcformat() # tag as_of in filename, if it was specified as_of_str = "-asof-{as_of}".format(as_of=format_date(as_of)) if as_of is not None else "" filename = "covidcast-{source}-{signal}-{start_day}-to-{end_day}{as_of}".format(source=source, signal=signal, start_day=format_date(start_day), end_day=format_date(end_day), as_of=as_of_str) @@ -456,7 +462,7 @@ def handle_backfill(): q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", [geo_pair]) - q.where_time_pairs("time_type", "time_value", [time_pair]) + q.where_time_pair("time_type", "time_value", time_pair) # no restriction of issues or dates since we want all issues # _handle_lag_issues_as_of(q, issues, lag, as_of) @@ -476,7 +482,7 @@ def gen(rows): for time_value, group in groupby((parse_row(row, fields_string, fields_int, fields_float) for row in rows), lambda row: row["time_value"]): # compute data per time value issues: List[Dict[str, Any]] = [r for r in group] - shifted_time_value = shift_time_value(time_value, reference_anchor_lag) if is_day else shift_week_value(time_value, reference_anchor_lag) + shifted_time_value = shift_day_value(time_value, reference_anchor_lag) if is_day else shift_week_value(time_value, reference_anchor_lag) anchor_row = find_anchor_row(issues, shifted_time_value) for i, row in enumerate(issues): @@ -594,7 +600,8 @@ def handle_coverage(): geo_type = request.args.get("geo_type", "county") if "window" in request.values: - time_window, is_day = parse_day_or_week_range_arg("window") + time_window = parse_day_or_week_range_arg("window") + is_day = time_window.is_day else: now_time = extract_date("latest") last = extract_integer("days") @@ -605,14 +612,14 @@ def handle_coverage(): if last_weeks is None: last_weeks = last or 30 is_day = False - now_week = Week.thisweek() if now_time is None else week_value_to_week(now_time) - time_window = (week_to_time_value(now_week - last_weeks), week_to_time_value(now_week)) + now_week = Week.thisweek() if now_time is None else time_value_to_week(now_time) + time_window = TimePair("week", [(week_to_time_value(now_week - last_weeks), week_to_time_value(now_week))]) else: is_day = True if last is None: last = 30 - now = date.today() if now_time is None else time_value_to_date(now_time) - time_window = (date_to_time_value(now - timedelta(days=last)), date_to_time_value(now)) + now = date.today() if now_time is None else time_value_to_day(now_time) + time_window = TimePair("day", [(day_to_time_value(now - timedelta(days=last)), day_to_time_value(now))]) _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) q = QueryBuilder(latest_table, "c") @@ -631,7 +638,7 @@ def handle_coverage(): else: q.where(geo_type=geo_type) q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_time_pairs("time_type", "time_value", [TimePair("day" if is_day else "week", [time_window])]) + q.where_time_pair("time_type", "time_value", time_window) q.group_by = "c.source, c.signal, c.time_value" q.set_order("source", "signal", "time_value") diff --git a/src/server/utils/__init__.py b/src/server/utils/__init__.py index 3198779d0..efab6c030 100644 --- a/src/server/utils/__init__.py +++ b/src/server/utils/__init__.py @@ -1 +1 @@ -from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, time_values_to_ranges, days_to_ranges, weeks_to_ranges +from .dates import shift_day_value, day_to_time_value, time_value_to_iso, time_value_to_day, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, time_value_to_week, guess_time_value_is_day, guess_time_value_is_week, time_values_to_ranges, days_to_ranges, weeks_to_ranges, TimeValues diff --git a/src/server/utils/dates.py b/src/server/utils/dates.py index ef34a50b9..ffd296c5a 100644 --- a/src/server/utils/dates.py +++ b/src/server/utils/dates.py @@ -7,9 +7,13 @@ ) from datetime import date, timedelta from epiweeks import Week, Year +from typing_extensions import TypeAlias import logging -def time_value_to_date(value: int) -> date: +# Alias for a sequence of date ranges (int, int) or date integers +TimeValues: TypeAlias = Sequence[Union[Tuple[int, int], int]] + +def time_value_to_day(value: int) -> date: year, month, day = value // 10000, (value % 10000) // 100, value % 100 if year < date.min.year: return date.min @@ -17,7 +21,7 @@ def time_value_to_date(value: int) -> date: return date.max return date(year=year, month=month, day=day) -def week_value_to_week(value: int) -> Week: +def time_value_to_week(value: int) -> Week: year, week = value // 100, value % 100 if year < date.min.year: return Week(date.min.year, 1) @@ -33,28 +37,26 @@ def guess_time_value_is_week(value: int) -> bool: # YYYYWW type and not YYYYMMDD return len(str(value)) == 6 -def date_to_time_value(d: date) -> int: +def day_to_time_value(d: date) -> int: return int(d.strftime("%Y%m%d")) - def week_to_time_value(w: Week) -> int: return w.year * 100 + w.week def time_value_to_iso(value: int) -> str: - return time_value_to_date(value).strftime("%Y-%m-%d") - + return time_value_to_day(value).strftime("%Y-%m-%d") -def shift_time_value(time_value: int, days: int) -> int: +def shift_day_value(time_value: int, days: int) -> int: if days == 0: return time_value - d = time_value_to_date(time_value) + d = time_value_to_day(time_value) shifted = d + timedelta(days=days) - return date_to_time_value(shifted) + return day_to_time_value(shifted) def shift_week_value(week_value: int, weeks: int) -> int: if weeks == 0: return week_value - week = week_value_to_week(week_value) + week = time_value_to_week(week_value) shifted = week + weeks return week_to_time_value(shifted) @@ -63,14 +65,14 @@ def days_in_range(range: Tuple[int, int]) -> int: returns the days within this time range """ - start = time_value_to_date(range[0]) - end = time_value_to_date(range[1]) + start = time_value_to_day(range[0]) + end = time_value_to_day(range[1]) delta = end - start return delta.days + 1 # same date should lead to 1 day that will be queried def weeks_in_range(week_range: Tuple[int, int]) -> int: - start = week_value_to_week(week_range[0]) - end = week_value_to_week(week_range[1]) + start = time_value_to_week(week_range[0]) + end = time_value_to_week(week_range[1]) acc = end.week - start.week # accumulate the number of weeks in the years between for y in range(start.year, end.year): @@ -78,7 +80,7 @@ def weeks_in_range(week_range: Tuple[int, int]) -> int: acc += year.totalweeks() return acc + 1 # same week should lead to 1 week that will be queried -def time_values_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]]]) -> Optional[Sequence[Union[Tuple[int, int], int]]]: +def time_values_to_ranges(values: Optional[TimeValues]) -> Optional[TimeValues]: """ Converts a mixed list of dates and date ranges to an optimized list where dates are merged into ranges where possible. e.g. [20200101, 20200102, (20200101, 20200104), 20200106] -> [(20200101, 20200104), 20200106] @@ -96,13 +98,13 @@ def time_values_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]] else: return values -def days_to_ranges(values: Sequence[Union[Tuple[int, int], int]]) -> Sequence[Union[Tuple[int, int], int]]: - return _to_ranges(values, time_value_to_date, date_to_time_value, timedelta(days=1)) +def days_to_ranges(values: TimeValues) -> TimeValues: + return _to_ranges(values, time_value_to_day, day_to_time_value, timedelta(days=1)) -def weeks_to_ranges(values: Sequence[Union[Tuple[int, int], int]]) -> Sequence[Union[Tuple[int, int], int]]: - return _to_ranges(values, week_value_to_week, week_to_time_value, 1) +def weeks_to_ranges(values: TimeValues) -> TimeValues: + return _to_ranges(values, time_value_to_week, week_to_time_value, 1) -def _to_ranges(values: Sequence[Union[Tuple[int, int], int]], value_to_date: Callable, date_to_value: Callable, time_unit: Union[int, timedelta]) -> Sequence[Union[Tuple[int, int], int]]: +def _to_ranges(values: TimeValues, value_to_date: Callable, date_to_value: Callable, time_unit: Union[int, timedelta]) -> TimeValues: try: intervals = [] diff --git a/tests/server/test_params.py b/tests/server/test_params.py index fffea0043..2d22a5d37 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -267,35 +267,38 @@ def test_parse_day_value(self): def test_parse_time_arg(self): with self.subTest("empty"): with app.test_request_context("/"): - self.assertEqual(parse_time_arg(), []) + self.assertEqual(parse_time_arg(), None) with self.subTest("single"): with app.test_request_context("/?time=day:*"): - self.assertEqual(parse_time_arg(), [TimePair("day", True)]) + self.assertEqual(parse_time_arg(), TimePair("day", True)) with app.test_request_context("/?time=day:20201201"): - self.assertEqual(parse_time_arg(), [TimePair("day", [20201201])]) + self.assertEqual(parse_time_arg(), TimePair("day", [20201201])) with self.subTest("single list"): with app.test_request_context("/?time=day:20201201,20201202"): - self.assertEqual(parse_time_arg(), [TimePair("day", [20201201, 20201202])]) + self.assertEqual(parse_time_arg(), TimePair("day", [20201201, 20201202])) with self.subTest("single range"): with app.test_request_context("/?time=day:20201201-20201204"): - self.assertEqual(parse_time_arg(), [TimePair("day", [(20201201, 20201204)])]) + self.assertEqual(parse_time_arg(), TimePair("day", [(20201201, 20201204)])) with self.subTest("multi"): - with app.test_request_context("/?time=day:*;week:*"): - self.assertEqual(parse_time_arg(), [TimePair("day", True), TimePair("week", True)]) - with app.test_request_context("/?time=day:20201201;week:202012"): + with app.test_request_context("/?time=day:*;day:20201201"): self.assertEqual( parse_time_arg(), - [TimePair("day", [20201201]), TimePair("week", [202012])], + TimePair("day", True) ) - with self.subTest("hybrid"): - with app.test_request_context("/?time=day:*;day:20202012;week:202101-202104"): + with app.test_request_context("/?time=week:*;week:202012"): self.assertEqual( parse_time_arg(), - [ - TimePair("day", True), - TimePair("day", [20202012]), - TimePair("week", [(202101, 202104)]), - ], + TimePair("week", True) + ) + with app.test_request_context("/?time=day:20201201;day:20201202-20201205"): + self.assertEqual( + parse_time_arg(), + TimePair("day", [(20201201, 20201205)]) + ) + with app.test_request_context("/?time=week:202012;week:202013-202015"): + self.assertEqual( + parse_time_arg(), + TimePair("week", [(202012, 202015)]) ) with self.subTest("wrong"): @@ -307,6 +310,12 @@ def test_parse_time_arg(self): self.assertRaises(ValidationFailedException, parse_time_arg) with app.test_request_context("/?time=week:20121010"): self.assertRaises(ValidationFailedException, parse_time_arg) + with app.test_request_context("/?time=day:*;week:*"): + self.assertRaisesRegex(ValidationFailedException, "mixes \"day\" and \"week\" time types", parse_time_arg) + with app.test_request_context("/?time=day:20201201;week:202012"): + self.assertRaisesRegex(ValidationFailedException, "mixes \"day\" and \"week\" time types", parse_time_arg) + with app.test_request_context("/?time=day:*;day:20202012;week:202101-202104"): + self.assertRaisesRegex(ValidationFailedException, "mixes \"day\" and \"week\" time types", parse_time_arg) def test_parse_day_range_arg(self): with self.subTest("empty"): diff --git a/tests/server/test_query.py b/tests/server/test_query.py index a59030b75..a1292764f 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -14,7 +14,7 @@ filter_dates, filter_geo_pairs, filter_source_signal_pairs, - filter_time_pairs, + filter_time_pair, ) from delphi.epidata.server._params import ( GeoPair, @@ -245,57 +245,57 @@ def test_filter_source_signal_pairs(self): {"p_0t": "src1", "p_0t_0": "sig2", "p_1t": "src2", "p_1t_0": "srcx"}, ) - def test_filter_time_pairs(self): + def test_filter_time_pair(self): with self.subTest("empty"): params = {} - self.assertEqual(filter_time_pairs("t", "v", [], "p", params), "FALSE") + self.assertEqual(filter_time_pair("t", "v", None, "p", params), "FALSE") self.assertEqual(params, {}) with self.subTest("*"): params = {} self.assertEqual( - filter_time_pairs("t", "v", [TimePair("day", True)], "p", params), + filter_time_pair("t", "v", TimePair("day", True), "p", params), "(t = :p_0t)", ) self.assertEqual(params, {"p_0t": "day"}) with self.subTest("single"): params = {} self.assertEqual( - filter_time_pairs("t", "v", [TimePair("day", [20201201])], "p", params), + filter_time_pair("t", "v", TimePair("day", [20201201]), "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201}) with self.subTest("multi"): params = {} self.assertEqual( - filter_time_pairs("t", "v", [TimePair("day", [20201201, 20201203])], "p", params), + filter_time_pair("t", "v", TimePair("day", [20201201, 20201203]), "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_1": 20201203}) with self.subTest("range"): params = {} self.assertEqual( - filter_time_pairs("t", "v", [TimePair("day", [(20201201, 20201203)])], "p", params), + filter_time_pair("t", "v", TimePair("day", [(20201201, 20201203)]), "p", params), "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_0_2": 20201203}) with self.subTest("dedupe"): params = {} self.assertEqual( - filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200101, (20200101, 20200101), 20200101])], "p", params), + filter_time_pair("t", "v", TimePair("day", [20200101, 20200101, (20200101, 20200101), 20200101]), "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101}) with self.subTest("merge single range"): params = {} self.assertEqual( - filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200102, (20200101, 20200104)])], "p", params), + filter_time_pair("t", "v", TimePair("day", [20200101, 20200102, (20200101, 20200104)]), "p", params), "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_0_2": 20200104}) with self.subTest("merge ranges and singles"): params = {} self.assertEqual( - filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200103, (20200105, 20200107)])], "p", params), + filter_time_pair("t", "v", TimePair("day", [20200101, 20200103, (20200105, 20200107)]), "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1 OR v BETWEEN :p_0t_2 AND :p_0t_2_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_1": 20200103, 'p_0t_2': 20200105, 'p_0t_2_2': 20200107}) diff --git a/tests/server/utils/test_dates.py b/tests/server/utils/test_dates.py index e825bbd9b..5c42e6e16 100644 --- a/tests/server/utils/test_dates.py +++ b/tests/server/utils/test_dates.py @@ -2,21 +2,21 @@ from datetime import date from epiweeks import Week -from delphi.epidata.server.utils.dates import time_value_to_date, date_to_time_value, shift_time_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, week_value_to_week, time_values_to_ranges +from delphi.epidata.server.utils.dates import time_value_to_day, day_to_time_value, shift_day_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, time_value_to_week, time_values_to_ranges class UnitTests(unittest.TestCase): - def test_time_value_to_date(self): - self.assertEqual(time_value_to_date(20201010), date(2020, 10, 10)) - self.assertEqual(time_value_to_date(20190201), date(2019, 2, 1)) + def test_time_value_to_day(self): + self.assertEqual(time_value_to_day(20201010), date(2020, 10, 10)) + self.assertEqual(time_value_to_day(20190201), date(2019, 2, 1)) - def test_date_to_time_value(self): - self.assertEqual(date_to_time_value(date(2020, 10, 10)), 20201010) - self.assertEqual(date_to_time_value(date(2019, 2, 1)), 20190201) + def test_day_to_time_value(self): + self.assertEqual(day_to_time_value(date(2020, 10, 10)), 20201010) + self.assertEqual(day_to_time_value(date(2019, 2, 1)), 20190201) - def test_shift_time_value(self): - self.assertEqual(shift_time_value(20201010, -3), 20201007) - self.assertEqual(shift_time_value(20201010, -12), 20200928) + def test_shift_day_value(self): + self.assertEqual(shift_day_value(20201010, -3), 20201007) + self.assertEqual(shift_day_value(20201010, -12), 20200928) def test_time_value_to_iso(self): self.assertEqual(time_value_to_iso(20201010), "2020-10-10") @@ -33,9 +33,9 @@ def test_weeks_in_range(self): self.assertEqual(weeks_in_range((202001, 202101)), 54) # 2020 has 53 weeks self.assertEqual(weeks_in_range((202101, 202204)), 56) - def test_week_value_to_week(self): - self.assertEqual(week_value_to_week(202021), Week(2020, 21)) - self.assertEqual(week_value_to_week(202101), Week(2021, 1)) + def test_time_value_to_week(self): + self.assertEqual(time_value_to_week(202021), Week(2020, 21)) + self.assertEqual(time_value_to_week(202101), Week(2021, 1)) def test_week_to_time_value(self): self.assertEqual(week_to_time_value(Week(2021, 1)), 202101)