From f43523a892f64a827d6ac9c5100b28fe76959176 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Thu, 22 Dec 2022 14:27:40 +0200 Subject: [PATCH 1/5] Rename *Pair classes to *Set --- src/server/_params.py | 56 ++++----- src/server/_query.py | 18 +-- src/server/endpoints/covidcast.py | 32 +++--- src/server/endpoints/covidcast_utils/model.py | 12 +- tests/server/endpoints/test_covidcast.py | 5 - tests/server/test_params.py | 106 +++++++++--------- tests/server/test_query.py | 42 +++---- 7 files changed, 133 insertions(+), 138 deletions(-) diff --git a/src/server/_params.py b/src/server/_params.py index 0879601e6..551a5ef83 100644 --- a/src/server/_params.py +++ b/src/server/_params.py @@ -48,7 +48,7 @@ def _parse_single_arg(key: str) -> Tuple[str, str]: @dataclass -class GeoPair: +class GeoSet: geo_type: str geo_values: Union[bool, Sequence[str]] @@ -64,20 +64,20 @@ def count(self) -> float: return len(self.geo_values) -def parse_geo_arg(key: str = "geo") -> List[GeoPair]: - return [GeoPair(geo_type, geo_values) for [geo_type, geo_values] in _parse_common_multi_arg(key)] +def parse_geo_arg(key: str = "geo") -> List[GeoSet]: + return [GeoSet(geo_type, geo_values) for [geo_type, geo_values] in _parse_common_multi_arg(key)] -def parse_single_geo_arg(key: str) -> GeoPair: +def parse_single_geo_arg(key: str) -> GeoSet: """ parses a single geo pair with only one value """ r = _parse_single_arg(key) - return GeoPair(r[0], [r[1]]) + return GeoSet(r[0], [r[1]]) @dataclass -class SourceSignalPair: +class SourceSignalSet: source: str signal: Union[bool, Sequence[str]] @@ -93,20 +93,20 @@ def count(self) -> float: return len(self.signal) -def parse_source_signal_arg(key: str = "signal") -> List[SourceSignalPair]: - return [SourceSignalPair(source, signals) for [source, signals] in _parse_common_multi_arg(key)] +def parse_source_signal_arg(key: str = "signal") -> List[SourceSignalSet]: + return [SourceSignalSet(source, signals) for [source, signals] in _parse_common_multi_arg(key)] -def parse_single_source_signal_arg(key: str) -> SourceSignalPair: +def parse_single_source_signal_arg(key: str) -> SourceSignalSet: """ parses a single source signal pair with only one value """ r = _parse_single_arg(key) - return SourceSignalPair(r[0], [r[1]]) + return SourceSignalSet(r[0], [r[1]]) @dataclass -class TimePair: +class TimeSet: time_type: str time_values: Union[bool, TimeValues] @@ -134,10 +134,10 @@ def to_ranges(self): returns this pair with times converted to ranges """ if isinstance(self.time_values, bool): - return TimePair(self.time_type, self.time_values) + return TimeSet(self.time_type, self.time_values) if self.time_type == 'week': - return TimePair(self.time_type, weeks_to_ranges(self.time_values)) - return TimePair(self.time_type, days_to_ranges(self.time_values)) + return TimeSet(self.time_type, weeks_to_ranges(self.time_values)) + return TimeSet(self.time_type, days_to_ranges(self.time_values)) def _verify_range(start: int, end: int) -> IntRange: @@ -204,18 +204,18 @@ def parse_day_value(time_value: str) -> IntRange: raise ValidationFailedException(msg) -def _parse_time_pair(time_type: str, time_values: Union[bool, Sequence[str]]) -> TimePair: +def _parse_time_pair(time_type: str, time_values: Union[bool, Sequence[str]]) -> TimeSet: if isinstance(time_values, bool): - return TimePair(time_type, time_values) + return TimeSet(time_type, time_values) if time_type == "week": - return TimePair("week", [parse_week_value(t) for t in time_values]) + return TimeSet("week", [parse_week_value(t) for t in time_values]) elif time_type == "day": - return TimePair("day", [parse_day_value(t) for t in time_values]) + return TimeSet("day", [parse_day_value(t) for t in time_values]) raise ValidationFailedException(f'time param: {time_type} is not one of "day" or "week"') -def parse_time_arg(key: str = "time") -> Optional[TimePair]: +def parse_time_arg(key: str = "time") -> Optional[TimeSet]: time_pairs = [_parse_time_pair(time_type, time_values) for [time_type, time_values] in _parse_common_multi_arg(key)] # single value @@ -236,10 +236,10 @@ def parse_time_arg(key: str = "time") -> Optional[TimePair]: return time_pair else: merged.extend(time_pair.time_values) - return TimePair(time_pairs[0].time_type, merged).to_ranges() + return TimeSet(time_pairs[0].time_type, merged).to_ranges() -def parse_single_time_arg(key: str) -> TimePair: +def parse_single_time_arg(key: str) -> TimeSet: """ parses a single time pair with only one value """ @@ -285,20 +285,20 @@ def parse_week_range_arg(key: str) -> Tuple[int, int]: raise ValidationFailedException(f"{key} must match YYYYWW-YYYYWW") return r -def parse_day_or_week_arg(key: str, default_value: Optional[int] = None) -> TimePair: +def parse_day_or_week_arg(key: str, default_value: Optional[int] = None) -> TimeSet: v = request.values.get(key) if not v: if default_value is not None: time_type = "day" if guess_time_value_is_day(default_value) else "week" - return TimePair(time_type, [default_value]) + return TimeSet(time_type, [default_value]) raise ValidationFailedException(f"{key} param is required") # format is either YYYY-MM-DD or YYYYMMDD or YYYYMM is_week = guess_time_value_is_week(v) if is_week: - return TimePair("week", [parse_week_arg(key)]) - return TimePair("day", [parse_day_arg(key)]) + return TimeSet("week", [parse_week_arg(key)]) + return TimeSet("day", [parse_day_arg(key)]) -def parse_day_or_week_range_arg(key: str) -> TimePair: +def parse_day_or_week_range_arg(key: str) -> TimeSet: v = request.values.get(key) if not v: raise ValidationFailedException(f"{key} param is required") @@ -306,5 +306,5 @@ def parse_day_or_week_range_arg(key: str) -> TimePair: # so if the first before the - has length 6, it must be a week is_week = guess_time_value_is_week(v.split('-', 2)[0]) if is_week: - return TimePair("week", [parse_week_range_arg(key)]) - return TimePair("day", [parse_day_range_arg(key)]) + return TimeSet("week", [parse_week_range_arg(key)]) + return TimeSet("day", [parse_day_range_arg(key)]) diff --git a/src/server/_query.py b/src/server/_query.py index 81f765a95..1c722175d 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -19,7 +19,7 @@ from ._printer import create_printer, APrinter from ._exceptions import DatabaseErrorException from ._validate import extract_strings -from ._params import GeoPair, SourceSignalPair, TimePair +from ._params import GeoSet, SourceSignalSet, TimeSet from .utils import time_values_to_ranges, IntRange, TimeValues @@ -118,7 +118,7 @@ def filter_fields(generator: Iterable[Dict[str, Any]]): def filter_geo_pairs( type_field: str, value_field: str, - values: Sequence[GeoPair], + values: Sequence[GeoSet], param_key: str, params: Dict[str, Any], ) -> str: @@ -126,7 +126,7 @@ def filter_geo_pairs( returns the SQL sub query to filter by the given geo pairs """ - def filter_pair(pair: GeoPair, i) -> str: + def filter_pair(pair: GeoSet, i) -> str: type_param = f"{param_key}_{i}t" params[type_param] = pair.geo_type if isinstance(pair.geo_values, bool) and pair.geo_values: @@ -145,7 +145,7 @@ def filter_pair(pair: GeoPair, i) -> str: def filter_source_signal_pairs( source_field: str, signal_field: str, - values: Sequence[SourceSignalPair], + values: Sequence[SourceSignalSet], param_key: str, params: Dict[str, Any], ) -> str: @@ -153,7 +153,7 @@ def filter_source_signal_pairs( returns the SQL sub query to filter by the given source signal pairs """ - def filter_pair(pair: SourceSignalPair, i) -> str: + def filter_pair(pair: SourceSignalSet, i) -> str: source_param = f"{param_key}_{i}t" params[source_param] = pair.source if isinstance(pair.signal, bool) and pair.signal: @@ -172,7 +172,7 @@ def filter_pair(pair: SourceSignalPair, i) -> str: def filter_time_pair( type_field: str, time_field: str, - pair: Optional[TimePair], + pair: Optional[TimeSet], param_key: str, params: Dict[str, Any], ) -> str: @@ -410,7 +410,7 @@ def where_geo_pairs( self, type_field: str, value_field: str, - values: Sequence[GeoPair], + values: Sequence[GeoSet], param_key: Optional[str] = None, ) -> "QueryBuilder": fq_type_field = self._fq_field(type_field) @@ -430,7 +430,7 @@ def where_source_signal_pairs( self, type_field: str, value_field: str, - values: Sequence[SourceSignalPair], + values: Sequence[SourceSignalSet], param_key: Optional[str] = None, ) -> "QueryBuilder": fq_type_field = self._fq_field(type_field) @@ -450,7 +450,7 @@ def where_time_pair( self, type_field: str, value_field: str, - values: Optional[TimePair], + values: Optional[TimeSet], param_key: Optional[str] = None, ) -> "QueryBuilder": fq_type_field = self._fq_field(type_field) diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index f7e4564a2..3165bb163 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -11,9 +11,9 @@ from .._common import is_compatibility_mode, db from .._exceptions import ValidationFailedException, DatabaseErrorException from .._params import ( - GeoPair, - SourceSignalPair, - TimePair, + GeoSet, + SourceSignalSet, + TimeSet, parse_geo_arg, parse_source_signal_arg, parse_time_arg, @@ -45,15 +45,15 @@ latest_table = "epimetric_latest_v" history_table = "epimetric_full_v" -def parse_source_signal_pairs() -> List[SourceSignalPair]: +def parse_source_signal_pairs() -> List[SourceSignalSet]: ds = request.values.get("data_source") if ds: # old version require_any("signal", "signals", empty=True) signals = extract_strings(("signals", "signal")) if len(signals) == 1 and signals[0] == "*": - return [SourceSignalPair(ds, True)] - return [SourceSignalPair(ds, signals)] + return [SourceSignalSet(ds, True)] + return [SourceSignalSet(ds, signals)] if ":" not in request.values.get("signal", ""): raise ValidationFailedException("missing parameter: signal or (data_source and signal[s])") @@ -61,15 +61,15 @@ def parse_source_signal_pairs() -> List[SourceSignalPair]: return parse_source_signal_arg() -def parse_geo_pairs() -> List[GeoPair]: +def parse_geo_pairs() -> List[GeoSet]: geo_type = request.values.get("geo_type") if geo_type: # old version require_any("geo_value", "geo_values", empty=True) geo_values = extract_strings(("geo_values", "geo_value")) if len(geo_values) == 1 and geo_values[0] == "*": - return [GeoPair(geo_type, True)] - return [GeoPair(geo_type, geo_values)] + return [GeoSet(geo_type, True)] + return [GeoSet(geo_type, geo_values)] if ":" not in request.values.get("geo", ""): raise ValidationFailedException("missing parameter: geo or (geo_type and geo_value[s])") @@ -77,13 +77,13 @@ def parse_geo_pairs() -> List[GeoPair]: return parse_geo_arg() -def parse_time_pairs() -> TimePair: +def parse_time_pairs() -> TimeSet: time_type = request.values.get("time_type") if time_type: # old version require_all("time_type", "time_values") time_values = extract_dates("time_values") - return TimePair(time_type, time_values) + return TimeSet(time_type, time_values) if ":" not in request.values.get("time", ""): raise ValidationFailedException("missing parameter: time or (time_type and time_values)") @@ -330,7 +330,7 @@ def gen(): @bp.route("/csv", methods=("GET", "POST")) def handle_export(): source, signal = request.values.get("signal", "jhu-csse:confirmed_incidence_num").split(":") - source_signal_pairs = [SourceSignalPair(source, [signal])] + source_signal_pairs = [SourceSignalSet(source, [signal])] daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) start_pair = parse_day_or_week_arg("start_day", 202001 if weekly_signals > 0 else 20200401) @@ -357,8 +357,8 @@ def handle_export(): q.set_fields(["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "source"], [], []) q.set_sort_order("time_value", "geo_value") q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_time_pair("time_type", "time_value", TimePair("day" if is_day else "week", [(start_day, end_day)])) - q.where_geo_pairs("geo_type", "geo_value", [GeoPair(geo_type, True if geo_values == "*" else geo_values)]) + q.where_time_pair("time_type", "time_value", TimeSet("day" if is_day else "week", [(start_day, end_day)])) + q.where_geo_pairs("geo_type", "geo_value", [GeoSet(geo_type, True if geo_values == "*" else geo_values)]) q.apply_as_of_filter(history_table, as_of) @@ -586,13 +586,13 @@ def handle_coverage(): last_weeks = last or 30 is_day = False now_week = Week.thisweek() if now_time is None else time_value_to_week(now_time) - time_window = TimePair("week", [(week_to_time_value(now_week - last_weeks), week_to_time_value(now_week))]) + time_window = TimeSet("week", [(week_to_time_value(now_week - last_weeks), week_to_time_value(now_week))]) else: is_day = True if last is None: last = 30 now = date.today() if now_time is None else time_value_to_day(now_time) - time_window = TimePair("day", [(day_to_time_value(now - timedelta(days=last)), day_to_time_value(now))]) + time_window = TimeSet("day", [(day_to_time_value(now - timedelta(days=last)), day_to_time_value(now))]) _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) q = QueryBuilder(latest_table, "c") diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index bbefe3cd4..1ee300361 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -6,7 +6,7 @@ import pandas as pd import numpy as np -from ..._params import SourceSignalPair +from ..._params import SourceSignalSet class HighValuesAre(str, Enum): @@ -236,7 +236,7 @@ def _load_data_signals(sources: List[DataSource]): data_signals_by_key[(source.db_source, d.signal)] = d -def count_signal_time_types(source_signals: List[SourceSignalPair]) -> Tuple[int, int]: +def count_signal_time_types(source_signals: List[SourceSignalSet]) -> Tuple[int, int]: """ count the number of signals in this query for each time type @returns daily counts, weekly counts @@ -257,9 +257,9 @@ def count_signal_time_types(source_signals: List[SourceSignalPair]) -> Tuple[int return daily, weekly -def create_source_signal_alias_mapper(source_signals: List[SourceSignalPair]) -> Tuple[List[SourceSignalPair], Optional[Callable[[str, str], str]]]: +def create_source_signal_alias_mapper(source_signals: List[SourceSignalSet]) -> Tuple[List[SourceSignalSet], Optional[Callable[[str, str], str]]]: alias_to_data_sources: Dict[str, List[DataSource]] = {} - transformed_pairs: List[SourceSignalPair] = [] + transformed_pairs: List[SourceSignalSet] = [] for pair in source_signals: source = data_source_by_id.get(pair.source) if not source or not source.uses_db_alias: @@ -269,9 +269,9 @@ def create_source_signal_alias_mapper(source_signals: List[SourceSignalPair]) -> alias_to_data_sources.setdefault(source.db_source, []).append(source) if pair.signal is True: # list all signals of this source (*) so resolve to a plain list of all in this alias - transformed_pairs.append(SourceSignalPair(source.db_source, [s.signal for s in source.signals])) + transformed_pairs.append(SourceSignalSet(source.db_source, [s.signal for s in source.signals])) else: - transformed_pairs.append(SourceSignalPair(source.db_source, pair.signal)) + transformed_pairs.append(SourceSignalSet(source.db_source, pair.signal)) if not alias_to_data_sources: # no alias needed diff --git a/tests/server/endpoints/test_covidcast.py b/tests/server/endpoints/test_covidcast.py index b7ecdc263..bb4fee873 100644 --- a/tests/server/endpoints/test_covidcast.py +++ b/tests/server/endpoints/test_covidcast.py @@ -5,11 +5,6 @@ from flask import Response from delphi.epidata.server.main import app -from delphi.epidata.server._params import ( - GeoPair, - TimePair, -) - # py3tester coverage target __test_target__ = "delphi.epidata.server.endpoints.covidcast" diff --git a/tests/server/test_params.py b/tests/server/test_params.py index 2d22a5d37..641b2c429 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -16,9 +16,9 @@ parse_week_value, parse_day_range_arg, parse_day_arg, - GeoPair, - TimePair, - SourceSignalPair, + GeoSet, + TimeSet, + SourceSignalSet, ) from delphi.epidata.server._exceptions import ( ValidationFailedException, @@ -40,44 +40,44 @@ def setUp(self): def test_geo_pair(self): with self.subTest("*"): - p = GeoPair("hrr", True) + p = GeoSet("hrr", True) self.assertTrue(p.matches("hrr", "any")) self.assertFalse(p.matches("msa", "any")) with self.subTest("subset"): - p = GeoPair("hrr", ["a", "b"]) + p = GeoSet("hrr", ["a", "b"]) self.assertTrue(p.matches("hrr", "a")) self.assertTrue(p.matches("hrr", "b")) self.assertFalse(p.matches("hrr", "c")) self.assertFalse(p.matches("msa", "any")) with self.subTest("count"): - self.assertEqual(GeoPair("a", True).count(), inf) - self.assertEqual(GeoPair("a", False).count(), 0) - self.assertEqual(GeoPair("a", ["a", "b"]).count(), 2) + self.assertEqual(GeoSet("a", True).count(), inf) + self.assertEqual(GeoSet("a", False).count(), 0) + self.assertEqual(GeoSet("a", ["a", "b"]).count(), 2) def test_source_signal_pair(self): with self.subTest("*"): - p = SourceSignalPair("src1", True) + p = SourceSignalSet("src1", True) self.assertTrue(p.matches("src1", "any")) self.assertFalse(p.matches("src2", "any")) with self.subTest("subset"): - p = SourceSignalPair("src1", ["a", "b"]) + p = SourceSignalSet("src1", ["a", "b"]) self.assertTrue(p.matches("src1", "a")) self.assertTrue(p.matches("src1", "b")) self.assertFalse(p.matches("src1", "c")) self.assertFalse(p.matches("src2", "any")) with self.subTest("count"): - self.assertEqual(SourceSignalPair("a", True).count(), inf) - self.assertEqual(SourceSignalPair("a", False).count(), 0) - self.assertEqual(SourceSignalPair("a", ["a", "b"]).count(), 2) + self.assertEqual(SourceSignalSet("a", True).count(), inf) + self.assertEqual(SourceSignalSet("a", False).count(), 0) + self.assertEqual(SourceSignalSet("a", ["a", "b"]).count(), 2) def test_time_pair(self): with self.subTest("count"): - self.assertEqual(TimePair("day", True).count(), inf) - self.assertEqual(TimePair("day", False).count(), 0) - self.assertEqual(TimePair("day", [20200202, 20200201]).count(), 2) - self.assertEqual(TimePair("day", [(20200201, 20200202)]).count(), 2) - self.assertEqual(TimePair("day", [(20200201, 20200205)]).count(), 5) - self.assertEqual(TimePair("day", [(20200201, 20200205), 20201212]).count(), 6) + self.assertEqual(TimeSet("day", True).count(), inf) + self.assertEqual(TimeSet("day", False).count(), 0) + self.assertEqual(TimeSet("day", [20200202, 20200201]).count(), 2) + self.assertEqual(TimeSet("day", [(20200201, 20200202)]).count(), 2) + self.assertEqual(TimeSet("day", [(20200201, 20200205)]).count(), 5) + self.assertEqual(TimeSet("day", [(20200201, 20200205), 20201212]).count(), 6) def test_parse_geo_arg(self): with self.subTest("empty"): @@ -85,32 +85,32 @@ def test_parse_geo_arg(self): self.assertEqual(parse_geo_arg(), []) with self.subTest("single"): with app.test_request_context("/?geo=state:*"): - self.assertEqual(parse_geo_arg(), [GeoPair("state", True)]) + self.assertEqual(parse_geo_arg(), [GeoSet("state", True)]) with app.test_request_context("/?geo=state:AK"): - self.assertEqual(parse_geo_arg(), [GeoPair("state", ["ak"])]) + self.assertEqual(parse_geo_arg(), [GeoSet("state", ["ak"])]) with self.subTest("single list"): with app.test_request_context("/?geo=state:AK,TK"): - self.assertEqual(parse_geo_arg(), [GeoPair("state", ["ak", "tk"])]) + self.assertEqual(parse_geo_arg(), [GeoSet("state", ["ak", "tk"])]) with self.subTest("multi"): with app.test_request_context("/?geo=state:*;nation:*"): - self.assertEqual(parse_geo_arg(), [GeoPair("state", True), GeoPair("nation", True)]) + self.assertEqual(parse_geo_arg(), [GeoSet("state", True), GeoSet("nation", True)]) with app.test_request_context("/?geo=state:AK;nation:US"): self.assertEqual( parse_geo_arg(), - [GeoPair("state", ["ak"]), GeoPair("nation", ["us"])], + [GeoSet("state", ["ak"]), GeoSet("nation", ["us"])], ) with app.test_request_context("/?geo=state:AK;state:KY"): self.assertEqual( parse_geo_arg(), - [GeoPair("state", ["ak"]), GeoPair("state", ["ky"])], + [GeoSet("state", ["ak"]), GeoSet("state", ["ky"])], ) with self.subTest("multi list"): with app.test_request_context("/?geo=state:AK,TK;county:42003,40556"): self.assertEqual( parse_geo_arg(), [ - GeoPair("state", ["ak", "tk"]), - GeoPair("county", ["42003", "40556"]), + GeoSet("state", ["ak", "tk"]), + GeoSet("county", ["42003", "40556"]), ], ) with self.subTest("hybrid"): @@ -118,9 +118,9 @@ def test_parse_geo_arg(self): self.assertEqual( parse_geo_arg(), [ - GeoPair("nation", True), - GeoPair("state", ["pa"]), - GeoPair("county", ["42003", "42002"]), + GeoSet("nation", True), + GeoSet("state", ["pa"]), + GeoSet("county", ["42003", "42002"]), ], ) @@ -136,7 +136,7 @@ def test_single_parse_geo_arg(self): self.assertRaises(ValidationFailedException, parse_single_geo_arg, "geo") with self.subTest("single"): with app.test_request_context("/?geo=state:AK"): - self.assertEqual(parse_single_geo_arg("geo"), GeoPair("state", ["ak"])) + self.assertEqual(parse_single_geo_arg("geo"), GeoSet("state", ["ak"])) with self.subTest("single list"): with app.test_request_context("/?geo=state:AK,TK"): self.assertRaises(ValidationFailedException, parse_single_geo_arg, "geo") @@ -155,35 +155,35 @@ def test_parse_source_signal_arg(self): self.assertEqual(parse_source_signal_arg(), []) with self.subTest("single"): with app.test_request_context("/?signal=src1:*"): - self.assertEqual(parse_source_signal_arg(), [SourceSignalPair("src1", True)]) + self.assertEqual(parse_source_signal_arg(), [SourceSignalSet("src1", True)]) with app.test_request_context("/?signal=src1:sig1"): - self.assertEqual(parse_source_signal_arg(), [SourceSignalPair("src1", ["sig1"])]) + self.assertEqual(parse_source_signal_arg(), [SourceSignalSet("src1", ["sig1"])]) with self.subTest("single list"): with app.test_request_context("/?signal=src1:sig1,sig2"): self.assertEqual( parse_source_signal_arg(), - [SourceSignalPair("src1", ["sig1", "sig2"])], + [SourceSignalSet("src1", ["sig1", "sig2"])], ) with self.subTest("multi"): with app.test_request_context("/?signal=src1:*;src2:*"): self.assertEqual( parse_source_signal_arg(), - [SourceSignalPair("src1", True), SourceSignalPair("src2", True)], + [SourceSignalSet("src1", True), SourceSignalSet("src2", True)], ) with app.test_request_context("/?signal=src1:sig1;src2:sig3"): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src1", ["sig1"]), - SourceSignalPair("src2", ["sig3"]), + SourceSignalSet("src1", ["sig1"]), + SourceSignalSet("src2", ["sig3"]), ], ) with app.test_request_context("/?signal=src1:sig1;src1:sig4"): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src1", ["sig1"]), - SourceSignalPair("src1", ["sig4"]), + SourceSignalSet("src1", ["sig1"]), + SourceSignalSet("src1", ["sig4"]), ], ) with self.subTest("multi list"): @@ -191,8 +191,8 @@ def test_parse_source_signal_arg(self): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src1", ["sig1", "sig2"]), - SourceSignalPair("county", ["sig5", "sig6"]), + SourceSignalSet("src1", ["sig1", "sig2"]), + SourceSignalSet("county", ["sig5", "sig6"]), ], ) with self.subTest("hybrid"): @@ -200,9 +200,9 @@ def test_parse_source_signal_arg(self): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src2", True), - SourceSignalPair("src1", ["sig4"]), - SourceSignalPair("src3", ["sig5", "sig6"]), + SourceSignalSet("src2", True), + SourceSignalSet("src1", ["sig4"]), + SourceSignalSet("src3", ["sig5", "sig6"]), ], ) @@ -218,7 +218,7 @@ def test_single_parse_source_signal_arg(self): self.assertRaises(ValidationFailedException, parse_single_source_signal_arg, "signal") with self.subTest("single"): with app.test_request_context("/?signal=src1:sig1"): - self.assertEqual(parse_single_source_signal_arg("signal"), SourceSignalPair("src1", ["sig1"])) + self.assertEqual(parse_single_source_signal_arg("signal"), SourceSignalSet("src1", ["sig1"])) with self.subTest("single list"): with app.test_request_context("/?signal=src1:sig1,sig2"): self.assertRaises(ValidationFailedException, parse_single_source_signal_arg, "signal") @@ -270,35 +270,35 @@ def test_parse_time_arg(self): self.assertEqual(parse_time_arg(), None) with self.subTest("single"): with app.test_request_context("/?time=day:*"): - self.assertEqual(parse_time_arg(), TimePair("day", True)) + self.assertEqual(parse_time_arg(), TimeSet("day", True)) with app.test_request_context("/?time=day:20201201"): - self.assertEqual(parse_time_arg(), TimePair("day", [20201201])) + self.assertEqual(parse_time_arg(), TimeSet("day", [20201201])) with self.subTest("single list"): with app.test_request_context("/?time=day:20201201,20201202"): - self.assertEqual(parse_time_arg(), TimePair("day", [20201201, 20201202])) + self.assertEqual(parse_time_arg(), TimeSet("day", [20201201, 20201202])) with self.subTest("single range"): with app.test_request_context("/?time=day:20201201-20201204"): - self.assertEqual(parse_time_arg(), TimePair("day", [(20201201, 20201204)])) + self.assertEqual(parse_time_arg(), TimeSet("day", [(20201201, 20201204)])) with self.subTest("multi"): with app.test_request_context("/?time=day:*;day:20201201"): self.assertEqual( parse_time_arg(), - TimePair("day", True) + TimeSet("day", True) ) with app.test_request_context("/?time=week:*;week:202012"): self.assertEqual( parse_time_arg(), - TimePair("week", True) + TimeSet("week", True) ) with app.test_request_context("/?time=day:20201201;day:20201202-20201205"): self.assertEqual( parse_time_arg(), - TimePair("day", [(20201201, 20201205)]) + TimeSet("day", [(20201201, 20201205)]) ) with app.test_request_context("/?time=week:202012;week:202013-202015"): self.assertEqual( parse_time_arg(), - TimePair("week", [(202012, 202015)]) + TimeSet("week", [(202012, 202015)]) ) with self.subTest("wrong"): diff --git a/tests/server/test_query.py b/tests/server/test_query.py index a1292764f..f43605ec3 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -17,9 +17,9 @@ filter_time_pair, ) from delphi.epidata.server._params import ( - GeoPair, - TimePair, - SourceSignalPair, + GeoSet, + TimeSet, + SourceSignalSet, ) # py3tester coverage target @@ -138,21 +138,21 @@ def test_filter_geo_pairs(self): with self.subTest("*"): params = {} self.assertEqual( - filter_geo_pairs("t", "v", [GeoPair("state", True)], "p", params), + filter_geo_pairs("t", "v", [GeoSet("state", True)], "p", params), "(t = :p_0t)", ) self.assertEqual(params, {"p_0t": "state"}) with self.subTest("single"): params = {} self.assertEqual( - filter_geo_pairs("t", "v", [GeoPair("state", ["KY"])], "p", params), + filter_geo_pairs("t", "v", [GeoSet("state", ["KY"])], "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "state", "p_0t_0": "KY"}) with self.subTest("multi"): params = {} self.assertEqual( - filter_geo_pairs("t", "v", [GeoPair("state", ["KY", "AK"])], "p", params), + filter_geo_pairs("t", "v", [GeoSet("state", ["KY", "AK"])], "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1)))", ) self.assertEqual(params, {"p_0t": "state", "p_0t_0": "KY", "p_0t_1": "AK"}) @@ -162,7 +162,7 @@ def test_filter_geo_pairs(self): filter_geo_pairs( "t", "v", - [GeoPair("state", True), GeoPair("nation", True)], + [GeoSet("state", True), GeoSet("nation", True)], "p", params, ), @@ -175,7 +175,7 @@ def test_filter_geo_pairs(self): filter_geo_pairs( "t", "v", - [GeoPair("state", ["AK"]), GeoPair("nation", ["US"])], + [GeoSet("state", ["AK"]), GeoSet("nation", ["US"])], "p", params, ), @@ -194,21 +194,21 @@ def test_filter_source_signal_pairs(self): with self.subTest("*"): params = {} self.assertEqual( - filter_source_signal_pairs("t", "v", [SourceSignalPair("src1", True)], "p", params), + filter_source_signal_pairs("t", "v", [SourceSignalSet("src1", True)], "p", params), "(t = :p_0t)", ) self.assertEqual(params, {"p_0t": "src1"}) with self.subTest("single"): params = {} self.assertEqual( - filter_source_signal_pairs("t", "v", [SourceSignalPair("src1", ["sig1"])], "p", params), + filter_source_signal_pairs("t", "v", [SourceSignalSet("src1", ["sig1"])], "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "src1", "p_0t_0": "sig1"}) with self.subTest("multi"): params = {} self.assertEqual( - filter_source_signal_pairs("t", "v", [SourceSignalPair("src1", ["sig1", "sig2"])], "p", params), + filter_source_signal_pairs("t", "v", [SourceSignalSet("src1", ["sig1", "sig2"])], "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1)))", ) self.assertEqual(params, {"p_0t": "src1", "p_0t_0": "sig1", "p_0t_1": "sig2"}) @@ -218,7 +218,7 @@ def test_filter_source_signal_pairs(self): filter_source_signal_pairs( "t", "v", - [SourceSignalPair("src1", True), SourceSignalPair("src2", True)], + [SourceSignalSet("src1", True), SourceSignalSet("src2", True)], "p", params, ), @@ -232,8 +232,8 @@ def test_filter_source_signal_pairs(self): "t", "v", [ - SourceSignalPair("src1", ["sig2"]), - SourceSignalPair("src2", ["srcx"]), + SourceSignalSet("src1", ["sig2"]), + SourceSignalSet("src2", ["srcx"]), ], "p", params, @@ -253,49 +253,49 @@ def test_filter_time_pair(self): with self.subTest("*"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimePair("day", True), "p", params), + filter_time_pair("t", "v", TimeSet("day", True), "p", params), "(t = :p_0t)", ) self.assertEqual(params, {"p_0t": "day"}) with self.subTest("single"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimePair("day", [20201201]), "p", params), + filter_time_pair("t", "v", TimeSet("day", [20201201]), "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201}) with self.subTest("multi"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimePair("day", [20201201, 20201203]), "p", params), + filter_time_pair("t", "v", TimeSet("day", [20201201, 20201203]), "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_1": 20201203}) with self.subTest("range"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimePair("day", [(20201201, 20201203)]), "p", params), + filter_time_pair("t", "v", TimeSet("day", [(20201201, 20201203)]), "p", params), "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_0_2": 20201203}) with self.subTest("dedupe"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimePair("day", [20200101, 20200101, (20200101, 20200101), 20200101]), "p", params), + filter_time_pair("t", "v", TimeSet("day", [20200101, 20200101, (20200101, 20200101), 20200101]), "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101}) with self.subTest("merge single range"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimePair("day", [20200101, 20200102, (20200101, 20200104)]), "p", params), + filter_time_pair("t", "v", TimeSet("day", [20200101, 20200102, (20200101, 20200104)]), "p", params), "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_0_2": 20200104}) with self.subTest("merge ranges and singles"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimePair("day", [20200101, 20200103, (20200105, 20200107)]), "p", params), + filter_time_pair("t", "v", TimeSet("day", [20200101, 20200103, (20200105, 20200107)]), "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1 OR v BETWEEN :p_0t_2 AND :p_0t_2_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_1": 20200103, 'p_0t_2': 20200105, 'p_0t_2_2': 20200107}) From 78be18efa6644ad89d4769d360187553ef98794f Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Thu, 22 Dec 2022 14:49:59 +0200 Subject: [PATCH 2/5] Rename related methods & tests --- src/server/_params.py | 42 +++---- src/server/_query.py | 69 +++++------ src/server/endpoints/covidcast.py | 117 +++++++++--------- src/server/endpoints/covidcast_utils/model.py | 24 ++-- tests/server/test_params.py | 6 +- tests/server/test_query.py | 52 ++++---- 6 files changed, 154 insertions(+), 156 deletions(-) diff --git a/src/server/_params.py b/src/server/_params.py index 551a5ef83..53be274f0 100644 --- a/src/server/_params.py +++ b/src/server/_params.py @@ -57,7 +57,7 @@ def matches(self, geo_type: str, geo_value: str) -> bool: def count(self) -> float: """ - returns the count of items in this pair + returns the count of items in this set """ if isinstance(self.geo_values, bool): return inf if self.geo_values else 0 @@ -70,7 +70,7 @@ def parse_geo_arg(key: str = "geo") -> List[GeoSet]: def parse_single_geo_arg(key: str) -> GeoSet: """ - parses a single geo pair with only one value + parses a single geo set with only one value """ r = _parse_single_arg(key) return GeoSet(r[0], [r[1]]) @@ -86,7 +86,7 @@ def matches(self, source: str, signal: str) -> bool: def count(self) -> float: """ - returns the count of items in this pair + returns the count of items in this set """ if isinstance(self.signal, bool): return inf if self.signal else 0 @@ -99,7 +99,7 @@ def parse_source_signal_arg(key: str = "signal") -> List[SourceSignalSet]: def parse_single_source_signal_arg(key: str) -> SourceSignalSet: """ - parses a single source signal pair with only one value + parses a single source signal set with only one value """ r = _parse_single_arg(key) return SourceSignalSet(r[0], [r[1]]) @@ -121,7 +121,7 @@ def is_day(self) -> bool: def count(self) -> float: """ - returns the count of items in this pair + returns the count of items in this set """ if isinstance(self.time_values, bool): return inf if self.time_values else 0 @@ -131,7 +131,7 @@ def count(self) -> float: def to_ranges(self): """ - returns this pair with times converted to ranges + returns this set with times converted to ranges """ if isinstance(self.time_values, bool): return TimeSet(self.time_type, self.time_values) @@ -204,7 +204,7 @@ def parse_day_value(time_value: str) -> IntRange: raise ValidationFailedException(msg) -def _parse_time_pair(time_type: str, time_values: Union[bool, Sequence[str]]) -> TimeSet: +def _parse_time_set(time_type: str, time_values: Union[bool, Sequence[str]]) -> TimeSet: if isinstance(time_values, bool): return TimeSet(time_type, time_values) @@ -216,35 +216,35 @@ def _parse_time_pair(time_type: str, time_values: Union[bool, Sequence[str]]) -> def parse_time_arg(key: str = "time") -> Optional[TimeSet]: - time_pairs = [_parse_time_pair(time_type, time_values) for [time_type, time_values] in _parse_common_multi_arg(key)] + time_sets = [_parse_time_set(time_type, time_values) for [time_type, time_values] in _parse_common_multi_arg(key)] # single value - if len(time_pairs) == 0: + if len(time_sets) == 0: return None - if len(time_pairs) == 1: - return time_pairs[0] + if len(time_sets) == 1: + return time_sets[0] # make sure 'day' and 'week' aren't mixed - time_types = set(time_pair.time_type for time_pair in time_pairs) + time_types = set(time_set.time_type for time_set in time_sets) if len(time_types) >= 2: - raise ValidationFailedException(f'{key}: {time_pairs} mixes "day" and "week" time types') + raise ValidationFailedException(f'{key}: {time_sets} mixes "day" and "week" time types') - # merge all time pairs into one + # merge all time sets into one merged = [] - for time_pair in time_pairs: - if time_pair.time_values is True: - return time_pair + for time_set in time_sets: + if time_set.time_values is True: + return time_set else: - merged.extend(time_pair.time_values) - return TimeSet(time_pairs[0].time_type, merged).to_ranges() + merged.extend(time_set.time_values) + return TimeSet(time_sets[0].time_type, merged).to_ranges() def parse_single_time_arg(key: str) -> TimeSet: """ - parses a single time pair with only one value + parses a single time set with only one value """ r = _parse_single_arg(key) - return _parse_time_pair(r[0], [r[1]]) + return _parse_time_set(r[0], [r[1]]) def parse_day_range_arg(key: str) -> Tuple[int, int]: diff --git a/src/server/_query.py b/src/server/_query.py index 1c722175d..aaed6c890 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -115,7 +115,7 @@ def filter_fields(generator: Iterable[Dict[str, Any]]): yield filtered -def filter_geo_pairs( +def filter_geo_sets( type_field: str, value_field: str, values: Sequence[GeoSet], @@ -123,17 +123,17 @@ def filter_geo_pairs( params: Dict[str, Any], ) -> str: """ - returns the SQL sub query to filter by the given geo pairs + returns the SQL sub query to filter by the given geo sets """ - def filter_pair(pair: GeoSet, i) -> str: + def filter_set(set: GeoSet, i) -> str: type_param = f"{param_key}_{i}t" - params[type_param] = pair.geo_type - if isinstance(pair.geo_values, bool) and pair.geo_values: + params[type_param] = set.geo_type + if isinstance(set.geo_values, bool) and set.geo_values: return f"{type_field} = :{type_param}" - return f"({type_field} = :{type_param} AND {filter_strings(value_field, cast(Sequence[str], pair.geo_values), type_param, params)})" + return f"({type_field} = :{type_param} AND {filter_strings(value_field, cast(Sequence[str], set.geo_values), type_param, params)})" - parts = [filter_pair(p, i) for i, p in enumerate(values)] + parts = [filter_set(p, i) for i, p in enumerate(values)] if not parts: # something has to be selected @@ -142,7 +142,7 @@ def filter_pair(pair: GeoSet, i) -> str: return f"({' OR '.join(parts)})" -def filter_source_signal_pairs( +def filter_source_signal_sets( source_field: str, signal_field: str, values: Sequence[SourceSignalSet], @@ -150,17 +150,17 @@ def filter_source_signal_pairs( params: Dict[str, Any], ) -> str: """ - returns the SQL sub query to filter by the given source signal pairs + returns the SQL sub query to filter by the given source signal sets """ - def filter_pair(pair: SourceSignalSet, i) -> str: + def filter_set(set: SourceSignalSet, i) -> str: source_param = f"{param_key}_{i}t" - params[source_param] = pair.source - if isinstance(pair.signal, bool) and pair.signal: + params[source_param] = set.source + if isinstance(set.signal, bool) and set.signal: return f"{source_field} = :{source_param}" - return f"({source_field} = :{source_param} AND {filter_strings(signal_field, cast(Sequence[str], pair.signal), source_param, params)})" + return f"({source_field} = :{source_param} AND {filter_strings(signal_field, cast(Sequence[str], set.signal), source_param, params)})" - parts = [filter_pair(p, i) for i, p in enumerate(values)] + parts = [filter_set(p, i) for i, p in enumerate(values)] if not parts: # something has to be selected @@ -169,26 +169,26 @@ def filter_pair(pair: SourceSignalSet, i) -> str: return f"({' OR '.join(parts)})" -def filter_time_pair( +def filter_time_set( type_field: str, time_field: str, - pair: Optional[TimeSet], + set: Optional[TimeSet], param_key: str, params: Dict[str, Any], ) -> str: """ - returns the SQL sub query to filter by the given time pair + returns the SQL sub query to filter by the given time set """ - # safety path; should normally not be reached as time pairs are enforced by the API - if not pair: + # safety path; should normally not be reached as time sets are enforced by the API + if not set: return "FALSE" type_param = f"{param_key}_0t" - params[type_param] = pair.time_type - if isinstance(pair.time_values, bool) and pair.time_values: + params[type_param] = set.time_type + if isinstance(set.time_values, bool) and set.time_values: parts = f"{type_field} = :{type_param}" else: - ranges = pair.to_ranges().time_values + ranges = set.to_ranges().time_values parts = f"({type_field} = :{type_param} AND {filter_integers(time_field, ranges, type_param, params)})" return f"({parts})" @@ -406,17 +406,17 @@ def where_integers( self.conditions.append(filter_integers(fq_field, values, param_key or field, self.params)) return self - def where_geo_pairs( + def apply_geo_filters( self, type_field: str, value_field: str, values: Sequence[GeoSet], - param_key: Optional[str] = None, - ) -> "QueryBuilder": + param_key: Optional[str] = None + ): fq_type_field = self._fq_field(type_field) fq_value_field = self._fq_field(value_field) self.conditions.append( - filter_geo_pairs( + filter_geo_sets( fq_type_field, fq_value_field, values, @@ -424,19 +424,18 @@ def where_geo_pairs( self.params, ) ) - return self - def where_source_signal_pairs( + def apply_source_signal_filters( self, type_field: str, value_field: str, values: Sequence[SourceSignalSet], - param_key: Optional[str] = None, - ) -> "QueryBuilder": + param_key: Optional[str] = None + ): fq_type_field = self._fq_field(type_field) fq_value_field = self._fq_field(value_field) self.conditions.append( - filter_source_signal_pairs( + filter_source_signal_sets( fq_type_field, fq_value_field, values, @@ -444,19 +443,18 @@ def where_source_signal_pairs( self.params, ) ) - return self - def where_time_pair( + def apply_time_filter( self, type_field: str, value_field: str, values: Optional[TimeSet], param_key: Optional[str] = None, - ) -> "QueryBuilder": + ): fq_type_field = self._fq_field(type_field) fq_value_field = self._fq_field(value_field) self.conditions.append( - filter_time_pair( + filter_time_set( fq_type_field, fq_value_field, values, @@ -464,7 +462,6 @@ def where_time_pair( self.params, ) ) - return self def apply_lag_filter(self, history_table: str, lag: Optional[int]): if lag is not None: diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index 3165bb163..045e76490 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -45,7 +45,7 @@ latest_table = "epimetric_latest_v" history_table = "epimetric_full_v" -def parse_source_signal_pairs() -> List[SourceSignalSet]: +def parse_source_signal_sets() -> List[SourceSignalSet]: ds = request.values.get("data_source") if ds: # old version @@ -61,7 +61,7 @@ def parse_source_signal_pairs() -> List[SourceSignalSet]: return parse_source_signal_arg() -def parse_geo_pairs() -> List[GeoSet]: +def parse_geo_sets() -> List[GeoSet]: geo_type = request.values.get("geo_type") if geo_type: # old version @@ -77,7 +77,7 @@ def parse_geo_pairs() -> List[GeoSet]: return parse_geo_arg() -def parse_time_pairs() -> TimeSet: +def parse_time_set() -> TimeSet: time_type = request.values.get("time_type") if time_type: # old version @@ -93,10 +93,10 @@ def parse_time_pairs() -> TimeSet: @bp.route("/", methods=("GET", "POST")) def handle(): - source_signal_pairs = parse_source_signal_pairs() - source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) - time_pair = parse_time_pairs() - geo_pairs = parse_geo_pairs() + source_signal_sets = parse_source_signal_sets() + source_signal_sets, alias_mapper = create_source_signal_alias_mapper(source_signal_sets) + time_set = parse_time_set() + geo_sets = parse_geo_sets() as_of = extract_date("as_of") issues = extract_dates("issues") @@ -121,9 +121,9 @@ def handle(): # data type of each field # build the source, signal, time, and location (type and id) filters - q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pair("time_type", "time_value", time_pair) + q.apply_source_signal_filters("source", "signal", source_signal_sets) + q.apply_geo_filters("geo_type", "geo_value", geo_sets) + q.apply_time_filter("time_type", "time_value", time_set) q.apply_issues_filter(history_table, issues) q.apply_lag_filter(history_table, lag) @@ -149,15 +149,15 @@ def _verify_argument_time_type_matches(is_day_argument: bool, count_daily_signal @bp.route("/trend", methods=("GET", "POST")) def handle_trend(): require_all("window", "date") - source_signal_pairs = parse_source_signal_pairs() - daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) - source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) - geo_pairs = parse_geo_pairs() + source_signal_sets = parse_source_signal_sets() + daily_signals, weekly_signals = count_signal_time_types(source_signal_sets) + source_signal_sets, alias_mapper = create_source_signal_alias_mapper(source_signal_sets) + geo_sets = parse_geo_sets() time_window = parse_day_or_week_range_arg("window") is_day = time_window.is_day - time_pair = parse_day_or_week_arg("date") - time_value, is_also_day = time_pair.time_values[0], time_pair.is_day + time_set = parse_day_or_week_arg("date") + time_value, is_also_day = time_set.time_values[0], time_set.is_day if is_day != is_also_day: raise ValidationFailedException("mixing weeks with day arguments") _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) @@ -177,9 +177,9 @@ def handle_trend(): q.set_fields(fields_string, fields_int, fields_float) q.set_sort_order("geo_type", "geo_value", "source", "signal", "time_value") - q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pair("time_type", "time_value", time_window) + q.apply_source_signal_filters("source", "signal", source_signal_sets) + q.apply_geo_filters("geo_type", "geo_value", geo_sets) + q.apply_time_filter("time_type", "time_value", time_window) p = create_printer() @@ -204,10 +204,10 @@ def gen(rows): @bp.route("/trendseries", methods=("GET", "POST")) def handle_trendseries(): require_all("window") - source_signal_pairs = parse_source_signal_pairs() - daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) - source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) - geo_pairs = parse_geo_pairs() + source_signal_sets = parse_source_signal_sets() + daily_signals, weekly_signals = count_signal_time_types(source_signal_sets) + source_signal_sets, alias_mapper = create_source_signal_alias_mapper(source_signal_sets) + geo_sets = parse_geo_sets() time_window = parse_day_or_week_range_arg("window") is_day = time_window.is_day @@ -225,9 +225,9 @@ def handle_trendseries(): q.set_fields(fields_string, fields_int, fields_float) q.set_sort_order("geo_type", "geo_value", "source", "signal", "time_value") - q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pair("time_type", "time_value", time_window) + q.apply_source_signal_filters("source", "signal", source_signal_sets) + q.apply_geo_filters("geo_type", "geo_value", geo_sets) + q.apply_time_filter("time_type", "time_value", time_window) p = create_printer() @@ -258,10 +258,10 @@ def gen(rows): def handle_correlation(): require_all("reference", "window", "others", "geo") reference = parse_single_source_signal_arg("reference") - other_pairs = parse_source_signal_arg("others") - daily_signals, weekly_signals = count_signal_time_types(other_pairs + [reference]) - source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(other_pairs + [reference]) - geo_pairs = parse_geo_arg() + other_sets = parse_source_signal_arg("others") + daily_signals, weekly_signals = count_signal_time_types(other_sets + [reference]) + source_signal_sets, alias_mapper = create_source_signal_alias_mapper(other_sets + [reference]) + geo_sets = parse_geo_arg() time_window = parse_day_or_week_range_arg("window") is_day = time_window.is_day _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) @@ -279,13 +279,13 @@ def handle_correlation(): q.set_fields(fields_string, fields_int, fields_float) q.set_sort_order("geo_type", "geo_value", "source", "signal", "time_value") - q.where_source_signal_pairs( + q.apply_source_signal_filters( "source", "signal", - source_signal_pairs, + source_signal_sets, ) - q.where_geo_pairs("geo_type", "geo_value", geo_pairs) - q.where_time_pair("time_type", "time_value", time_window) + q.apply_geo_filters("geo_type", "geo_value", geo_sets) + q.apply_time_filter("time_type", "time_value", time_window) df = as_pandas(str(q), q.params) if is_day: @@ -330,13 +330,13 @@ def gen(): @bp.route("/csv", methods=("GET", "POST")) def handle_export(): source, signal = request.values.get("signal", "jhu-csse:confirmed_incidence_num").split(":") - source_signal_pairs = [SourceSignalSet(source, [signal])] - daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) - source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) - start_pair = parse_day_or_week_arg("start_day", 202001 if weekly_signals > 0 else 20200401) - start_day, is_day = start_pair.time_values[0], start_pair.is_day - end_pair = parse_day_or_week_arg("end_day", 202020 if weekly_signals > 0 else 20200901) - end_day, is_end_day = end_pair.time_values[0], end_pair.is_day + source_signal_sets = [SourceSignalSet(source, [signal])] + daily_signals, weekly_signals = count_signal_time_types(source_signal_sets) + source_signal_sets, alias_mapper = create_source_signal_alias_mapper(source_signal_sets) + start_time_set = parse_day_or_week_arg("start_day", 202001 if weekly_signals > 0 else 20200401) + start_day, is_day = start_time_set.time_values[0], start_time_set.is_day + end_time_set = parse_day_or_week_arg("end_day", 202020 if weekly_signals > 0 else 20200901) + end_day, is_end_day = end_time_set.time_values[0], end_time_set.is_day if is_day != is_end_day: raise ValidationFailedException("mixing weeks with day arguments") _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) @@ -355,10 +355,11 @@ def handle_export(): q = QueryBuilder(latest_table, "t") q.set_fields(["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "source"], [], []) + q.set_sort_order("time_value", "geo_value") - q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_time_pair("time_type", "time_value", TimeSet("day" if is_day else "week", [(start_day, end_day)])) - q.where_geo_pairs("geo_type", "geo_value", [GeoSet(geo_type, True if geo_values == "*" else geo_values)]) + q.apply_source_signal_filters("source", "signal", source_signal_sets) + q.apply_time_filter("time_type", "time_value", TimeSet("day" if is_day else "week", [(start_day, end_day)])) + q.apply_geo_filters("geo_type", "geo_value", [GeoSet(geo_type, True if geo_values == "*" else geo_values)]) q.apply_as_of_filter(history_table, as_of) @@ -412,16 +413,16 @@ def handle_backfill(): example query: http://localhost:5000/covidcast/backfill?signal=fb-survey:smoothed_cli&time=day:20200101-20220101&geo=state:ny&anchor_lag=60 """ require_all("geo", "time", "signal") - signal_pair = parse_single_source_signal_arg("signal") - daily_signals, weekly_signals = count_signal_time_types([signal_pair]) - source_signal_pairs, _ = create_source_signal_alias_mapper([signal_pair]) + source_signal_set = parse_single_source_signal_arg("signal") + daily_signals, weekly_signals = count_signal_time_types([source_signal_set]) + source_signal_sets, _ = create_source_signal_alias_mapper([source_signal_set]) # don't need the alias mapper since we don't return the source - time_pair = parse_single_time_arg("time") - is_day = time_pair.is_day + time_set = parse_single_time_arg("time") + is_day = time_set.is_day _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) - geo_pair = parse_single_geo_arg("geo") + get_set = parse_single_geo_arg("geo") reference_anchor_lag = extract_integer("anchor_lag") # in days or weeks if reference_anchor_lag is None: reference_anchor_lag = 60 @@ -436,9 +437,9 @@ def handle_backfill(): q.set_sort_order("time_value", "issue") q.set_fields(fields_string, fields_int, fields_float, ["is_latest_issue"]) - q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_geo_pairs("geo_type", "geo_value", [geo_pair]) - q.where_time_pair("time_type", "time_value", time_pair) + q.apply_source_signal_filters("source", "signal", source_signal_sets) + q.apply_geo_filters("geo_type", "geo_value", [get_set]) + q.apply_time_filter("time_type", "time_value", time_set) p = create_printer() @@ -567,9 +568,9 @@ def handle_coverage(): similar to /signal_dashboard_coverage for a specific signal returns the coverage (number of locations for a given geo_type) """ - source_signal_pairs = parse_source_signal_pairs() - daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) - source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) + source_signal_sets = parse_source_signal_sets() + daily_signals, weekly_signals = count_signal_time_types(source_signal_sets) + source_signal_sets, alias_mapper = create_source_signal_alias_mapper(source_signal_sets) geo_type = request.values.get("geo_type", "county") if "window" in request.values: @@ -610,8 +611,8 @@ def handle_coverage(): q.conditions.append('geo_value not like "%000"') else: q.where(geo_type=geo_type) - q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_time_pair("time_type", "time_value", time_window) + q.apply_source_signal_filters("source", "signal", source_signal_sets) + q.apply_time_filter("time_type", "time_value", time_window) q.group_by = "c.source, c.signal, c.time_value" q.set_sort_order("source", "signal", "time_value") diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 1ee300361..81ce10eef 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -243,11 +243,11 @@ def count_signal_time_types(source_signals: List[SourceSignalSet]) -> Tuple[int, """ weekly = 0 daily = 0 - for pair in source_signals: - if pair.signal == True: + for set in source_signals: + if set.signal == True: continue - for s in pair.signal: - signal = data_signals_by_key.get((pair.source, s)) + for s in set.signal: + signal = data_signals_by_key.get((set.source, s)) if not signal: continue if signal.time_type == TimeType.week: @@ -259,19 +259,19 @@ def count_signal_time_types(source_signals: List[SourceSignalSet]) -> Tuple[int, def create_source_signal_alias_mapper(source_signals: List[SourceSignalSet]) -> Tuple[List[SourceSignalSet], Optional[Callable[[str, str], str]]]: alias_to_data_sources: Dict[str, List[DataSource]] = {} - transformed_pairs: List[SourceSignalSet] = [] - for pair in source_signals: - source = data_source_by_id.get(pair.source) + transformed_sets: List[SourceSignalSet] = [] + for set in source_signals: + source = data_source_by_id.get(set.source) if not source or not source.uses_db_alias: - transformed_pairs.append(pair) + transformed_sets.append(set) continue # uses an alias alias_to_data_sources.setdefault(source.db_source, []).append(source) - if pair.signal is True: + if set.signal is True: # list all signals of this source (*) so resolve to a plain list of all in this alias - transformed_pairs.append(SourceSignalSet(source.db_source, [s.signal for s in source.signals])) + transformed_sets.append(SourceSignalSet(source.db_source, [s.signal for s in source.signals])) else: - transformed_pairs.append(SourceSignalSet(source.db_source, pair.signal)) + transformed_sets.append(SourceSignalSet(source.db_source, set.signal)) if not alias_to_data_sources: # no alias needed @@ -294,4 +294,4 @@ def map_row(source: str, signal: str) -> str: signal_source = possible_data_sources[0] return signal_source.source - return transformed_pairs, map_row + return transformed_sets, map_row diff --git a/tests/server/test_params.py b/tests/server/test_params.py index 641b2c429..810ebfa3c 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -38,7 +38,7 @@ def setUp(self): app.config["WTF_CSRF_ENABLED"] = False app.config["DEBUG"] = False - def test_geo_pair(self): + def test_geo_set(self): with self.subTest("*"): p = GeoSet("hrr", True) self.assertTrue(p.matches("hrr", "any")) @@ -54,7 +54,7 @@ def test_geo_pair(self): self.assertEqual(GeoSet("a", False).count(), 0) self.assertEqual(GeoSet("a", ["a", "b"]).count(), 2) - def test_source_signal_pair(self): + def test_source_signal_set(self): with self.subTest("*"): p = SourceSignalSet("src1", True) self.assertTrue(p.matches("src1", "any")) @@ -70,7 +70,7 @@ def test_source_signal_pair(self): self.assertEqual(SourceSignalSet("a", False).count(), 0) self.assertEqual(SourceSignalSet("a", ["a", "b"]).count(), 2) - def test_time_pair(self): + def test_time_set(self): with self.subTest("count"): self.assertEqual(TimeSet("day", True).count(), inf) self.assertEqual(TimeSet("day", False).count(), 0) diff --git a/tests/server/test_query.py b/tests/server/test_query.py index f43605ec3..53aca5621 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -12,9 +12,9 @@ filter_strings, filter_integers, filter_dates, - filter_geo_pairs, - filter_source_signal_pairs, - filter_time_pair, + filter_geo_sets, + filter_source_signal_sets, + filter_time_set, ) from delphi.epidata.server._params import ( GeoSet, @@ -130,36 +130,36 @@ def test_filter_dates(self): }, ) - def test_filter_geo_pairs(self): + def test_filter_geo_sets(self): with self.subTest("empty"): params = {} - self.assertEqual(filter_geo_pairs("t", "v", [], "p", params), "FALSE") + self.assertEqual(filter_geo_sets("t", "v", [], "p", params), "FALSE") self.assertEqual(params, {}) with self.subTest("*"): params = {} self.assertEqual( - filter_geo_pairs("t", "v", [GeoSet("state", True)], "p", params), + filter_geo_sets("t", "v", [GeoSet("state", True)], "p", params), "(t = :p_0t)", ) self.assertEqual(params, {"p_0t": "state"}) with self.subTest("single"): params = {} self.assertEqual( - filter_geo_pairs("t", "v", [GeoSet("state", ["KY"])], "p", params), + filter_geo_sets("t", "v", [GeoSet("state", ["KY"])], "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "state", "p_0t_0": "KY"}) with self.subTest("multi"): params = {} self.assertEqual( - filter_geo_pairs("t", "v", [GeoSet("state", ["KY", "AK"])], "p", params), + filter_geo_sets("t", "v", [GeoSet("state", ["KY", "AK"])], "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1)))", ) self.assertEqual(params, {"p_0t": "state", "p_0t_0": "KY", "p_0t_1": "AK"}) with self.subTest("multiple pairs"): params = {} self.assertEqual( - filter_geo_pairs( + filter_geo_sets( "t", "v", [GeoSet("state", True), GeoSet("nation", True)], @@ -172,7 +172,7 @@ def test_filter_geo_pairs(self): with self.subTest("multiple pairs with value"): params = {} self.assertEqual( - filter_geo_pairs( + filter_geo_sets( "t", "v", [GeoSet("state", ["AK"]), GeoSet("nation", ["US"])], @@ -186,36 +186,36 @@ def test_filter_geo_pairs(self): {"p_0t": "state", "p_0t_0": "AK", "p_1t": "nation", "p_1t_0": "US"}, ) - def test_filter_source_signal_pairs(self): + def test_filter_source_signal_sets(self): with self.subTest("empty"): params = {} - self.assertEqual(filter_source_signal_pairs("t", "v", [], "p", params), "FALSE") + self.assertEqual(filter_source_signal_sets("t", "v", [], "p", params), "FALSE") self.assertEqual(params, {}) with self.subTest("*"): params = {} self.assertEqual( - filter_source_signal_pairs("t", "v", [SourceSignalSet("src1", True)], "p", params), + filter_source_signal_sets("t", "v", [SourceSignalSet("src1", True)], "p", params), "(t = :p_0t)", ) self.assertEqual(params, {"p_0t": "src1"}) with self.subTest("single"): params = {} self.assertEqual( - filter_source_signal_pairs("t", "v", [SourceSignalSet("src1", ["sig1"])], "p", params), + filter_source_signal_sets("t", "v", [SourceSignalSet("src1", ["sig1"])], "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "src1", "p_0t_0": "sig1"}) with self.subTest("multi"): params = {} self.assertEqual( - filter_source_signal_pairs("t", "v", [SourceSignalSet("src1", ["sig1", "sig2"])], "p", params), + filter_source_signal_sets("t", "v", [SourceSignalSet("src1", ["sig1", "sig2"])], "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1)))", ) self.assertEqual(params, {"p_0t": "src1", "p_0t_0": "sig1", "p_0t_1": "sig2"}) with self.subTest("multiple pairs"): params = {} self.assertEqual( - filter_source_signal_pairs( + filter_source_signal_sets( "t", "v", [SourceSignalSet("src1", True), SourceSignalSet("src2", True)], @@ -228,7 +228,7 @@ def test_filter_source_signal_pairs(self): with self.subTest("multiple pairs with value"): params = {} self.assertEqual( - filter_source_signal_pairs( + filter_source_signal_sets( "t", "v", [ @@ -245,57 +245,57 @@ def test_filter_source_signal_pairs(self): {"p_0t": "src1", "p_0t_0": "sig2", "p_1t": "src2", "p_1t_0": "srcx"}, ) - def test_filter_time_pair(self): + def test_filter_time_set(self): with self.subTest("empty"): params = {} - self.assertEqual(filter_time_pair("t", "v", None, "p", params), "FALSE") + self.assertEqual(filter_time_set("t", "v", None, "p", params), "FALSE") self.assertEqual(params, {}) with self.subTest("*"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimeSet("day", True), "p", params), + filter_time_set("t", "v", TimeSet("day", True), "p", params), "(t = :p_0t)", ) self.assertEqual(params, {"p_0t": "day"}) with self.subTest("single"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimeSet("day", [20201201]), "p", params), + filter_time_set("t", "v", TimeSet("day", [20201201]), "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201}) with self.subTest("multi"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimeSet("day", [20201201, 20201203]), "p", params), + filter_time_set("t", "v", TimeSet("day", [20201201, 20201203]), "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_1": 20201203}) with self.subTest("range"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimeSet("day", [(20201201, 20201203)]), "p", params), + filter_time_set("t", "v", TimeSet("day", [(20201201, 20201203)]), "p", params), "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_0_2": 20201203}) with self.subTest("dedupe"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimeSet("day", [20200101, 20200101, (20200101, 20200101), 20200101]), "p", params), + filter_time_set("t", "v", TimeSet("day", [20200101, 20200101, (20200101, 20200101), 20200101]), "p", params), "((t = :p_0t AND (v = :p_0t_0)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101}) with self.subTest("merge single range"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimeSet("day", [20200101, 20200102, (20200101, 20200104)]), "p", params), + filter_time_set("t", "v", TimeSet("day", [20200101, 20200102, (20200101, 20200104)]), "p", params), "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_0_2": 20200104}) with self.subTest("merge ranges and singles"): params = {} self.assertEqual( - filter_time_pair("t", "v", TimeSet("day", [20200101, 20200103, (20200105, 20200107)]), "p", params), + filter_time_set("t", "v", TimeSet("day", [20200101, 20200103, (20200105, 20200107)]), "p", params), "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1 OR v BETWEEN :p_0t_2 AND :p_0t_2_2)))", ) self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_1": 20200103, 'p_0t_2': 20200105, 'p_0t_2_2': 20200107}) From 888897d6f04d6d08b248cd256e67bf95df9bd3cc Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Thu, 22 Dec 2022 23:49:38 +0200 Subject: [PATCH 3/5] Address review comments --- src/server/_query.py | 26 +++++++++---------- src/server/endpoints/covidcast_utils/model.py | 18 ++++++------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/server/_query.py b/src/server/_query.py index aaed6c890..9cfb71a7b 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -126,12 +126,12 @@ def filter_geo_sets( returns the SQL sub query to filter by the given geo sets """ - def filter_set(set: GeoSet, i) -> str: + def filter_set(gset: GeoSet, i) -> str: type_param = f"{param_key}_{i}t" - params[type_param] = set.geo_type - if isinstance(set.geo_values, bool) and set.geo_values: + params[type_param] = gset.geo_type + if isinstance(gset.geo_values, bool) and gset.geo_values: return f"{type_field} = :{type_param}" - return f"({type_field} = :{type_param} AND {filter_strings(value_field, cast(Sequence[str], set.geo_values), type_param, params)})" + return f"({type_field} = :{type_param} AND {filter_strings(value_field, cast(Sequence[str], gset.geo_values), type_param, params)})" parts = [filter_set(p, i) for i, p in enumerate(values)] @@ -153,12 +153,12 @@ def filter_source_signal_sets( returns the SQL sub query to filter by the given source signal sets """ - def filter_set(set: SourceSignalSet, i) -> str: + def filter_set(ssset: SourceSignalSet, i) -> str: source_param = f"{param_key}_{i}t" - params[source_param] = set.source - if isinstance(set.signal, bool) and set.signal: + params[source_param] = ssset.source + if isinstance(ssset.signal, bool) and ssset.signal: return f"{source_field} = :{source_param}" - return f"({source_field} = :{source_param} AND {filter_strings(signal_field, cast(Sequence[str], set.signal), source_param, params)})" + return f"({source_field} = :{source_param} AND {filter_strings(signal_field, cast(Sequence[str], ssset.signal), source_param, params)})" parts = [filter_set(p, i) for i, p in enumerate(values)] @@ -172,7 +172,7 @@ def filter_set(set: SourceSignalSet, i) -> str: def filter_time_set( type_field: str, time_field: str, - set: Optional[TimeSet], + tset: Optional[TimeSet], param_key: str, params: Dict[str, Any], ) -> str: @@ -180,15 +180,15 @@ def filter_time_set( returns the SQL sub query to filter by the given time set """ # safety path; should normally not be reached as time sets are enforced by the API - if not set: + if not tset: return "FALSE" type_param = f"{param_key}_0t" - params[type_param] = set.time_type - if isinstance(set.time_values, bool) and set.time_values: + params[type_param] = tset.time_type + if isinstance(tset.time_values, bool) and tset.time_values: parts = f"{type_field} = :{type_param}" else: - ranges = set.to_ranges().time_values + ranges = tset.to_ranges().time_values parts = f"({type_field} = :{type_param} AND {filter_integers(time_field, ranges, type_param, params)})" return f"({parts})" diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 81ce10eef..abab0033b 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -243,11 +243,11 @@ def count_signal_time_types(source_signals: List[SourceSignalSet]) -> Tuple[int, """ weekly = 0 daily = 0 - for set in source_signals: - if set.signal == True: + for ssset in source_signals: + if ssset.signal == True: continue - for s in set.signal: - signal = data_signals_by_key.get((set.source, s)) + for s in ssset.signal: + signal = data_signals_by_key.get((ssset.source, s)) if not signal: continue if signal.time_type == TimeType.week: @@ -260,18 +260,18 @@ def count_signal_time_types(source_signals: List[SourceSignalSet]) -> Tuple[int, def create_source_signal_alias_mapper(source_signals: List[SourceSignalSet]) -> Tuple[List[SourceSignalSet], Optional[Callable[[str, str], str]]]: alias_to_data_sources: Dict[str, List[DataSource]] = {} transformed_sets: List[SourceSignalSet] = [] - for set in source_signals: - source = data_source_by_id.get(set.source) + for ssset in source_signals: + source = data_source_by_id.get(ssset.source) if not source or not source.uses_db_alias: - transformed_sets.append(set) + transformed_sets.append(ssset) continue # uses an alias alias_to_data_sources.setdefault(source.db_source, []).append(source) - if set.signal is True: + if ssset.signal is True: # list all signals of this source (*) so resolve to a plain list of all in this alias transformed_sets.append(SourceSignalSet(source.db_source, [s.signal for s in source.signals])) else: - transformed_sets.append(SourceSignalSet(source.db_source, set.signal)) + transformed_sets.append(SourceSignalSet(source.db_source, ssset.signal)) if not alias_to_data_sources: # no alias needed From 189d87ad967f236090007306e60226fbc0a4168a Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Fri, 23 Dec 2022 01:52:29 +0200 Subject: [PATCH 4/5] Fix rebase issues --- src/server/_query.py | 3 +++ src/server/endpoints/covidcast.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/server/_query.py b/src/server/_query.py index 9cfb71a7b..b3964211c 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -424,6 +424,7 @@ def apply_geo_filters( self.params, ) ) + return self def apply_source_signal_filters( self, @@ -443,6 +444,7 @@ def apply_source_signal_filters( self.params, ) ) + return self def apply_time_filter( self, @@ -462,6 +464,7 @@ def apply_time_filter( self.params, ) ) + return self def apply_lag_filter(self, history_table: str, lag: Optional[int]): if lag is not None: diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index 045e76490..fa5ebc1e1 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -422,7 +422,7 @@ def handle_backfill(): is_day = time_set.is_day _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) - get_set = parse_single_geo_arg("geo") + geo_set = parse_single_geo_arg("geo") reference_anchor_lag = extract_integer("anchor_lag") # in days or weeks if reference_anchor_lag is None: reference_anchor_lag = 60 @@ -438,7 +438,7 @@ def handle_backfill(): q.set_fields(fields_string, fields_int, fields_float, ["is_latest_issue"]) q.apply_source_signal_filters("source", "signal", source_signal_sets) - q.apply_geo_filters("geo_type", "geo_value", [get_set]) + q.apply_geo_filters("geo_type", "geo_value", [geo_set]) q.apply_time_filter("time_type", "time_value", time_set) p = create_printer() From 83810e833cb387621128c8d9d13673a96511fb19 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Fri, 23 Dec 2022 01:54:35 +0200 Subject: [PATCH 5/5] QueryBuilder type if return self --- src/server/_query.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/server/_query.py b/src/server/_query.py index b3964211c..51ff4e182 100644 --- a/src/server/_query.py +++ b/src/server/_query.py @@ -411,8 +411,8 @@ def apply_geo_filters( type_field: str, value_field: str, values: Sequence[GeoSet], - param_key: Optional[str] = None - ): + param_key: Optional[str] = None, + ) -> "QueryBuilder": fq_type_field = self._fq_field(type_field) fq_value_field = self._fq_field(value_field) self.conditions.append( @@ -431,8 +431,8 @@ def apply_source_signal_filters( type_field: str, value_field: str, values: Sequence[SourceSignalSet], - param_key: Optional[str] = None - ): + param_key: Optional[str] = None, + ) -> "QueryBuilder": fq_type_field = self._fq_field(type_field) fq_value_field = self._fq_field(value_field) self.conditions.append( @@ -452,7 +452,7 @@ def apply_time_filter( value_field: str, values: Optional[TimeSet], param_key: Optional[str] = None, - ): + ) -> "QueryBuilder": fq_type_field = self._fq_field(type_field) fq_value_field = self._fq_field(value_field) self.conditions.append( @@ -466,20 +466,20 @@ def apply_time_filter( ) return self - def apply_lag_filter(self, history_table: str, lag: Optional[int]): + def apply_lag_filter(self, history_table: str, lag: Optional[int]) -> "QueryBuilder": if lag is not None: self.retable(history_table) # history_table has full spectrum of lag values to search from whereas the latest_table does not self.where(lag=lag) return self - def apply_issues_filter(self, history_table: str, issues: Optional[TimeValues]): + def apply_issues_filter(self, history_table: str, issues: Optional[TimeValues]) -> "QueryBuilder": if issues: self.retable(history_table) self.where_integers("issue", issues) return self - def apply_as_of_filter(self, history_table: str, as_of: Optional[int]): + def apply_as_of_filter(self, history_table: str, as_of: Optional[int]) -> "QueryBuilder": if as_of is not None: self.retable(history_table) sub_condition_asof = "(issue <= :as_of)" @@ -495,7 +495,7 @@ def set_fields(self, *fields: Iterable[str]) -> "QueryBuilder": self.fields = [f"{self.alias}.{field}" for field_list in fields for field in field_list] return self - def set_sort_order(self, *args: str): + def set_sort_order(self, *args: str) -> "QueryBuilder": """ sets the order for the given fields (as key word arguments), True = ASC, False = DESC """