JIT: major feature commit

* add smooth_diff
* add model updates
* add /trend endpoint
* add /trendseries endpoint
* add /csv endpoint
* params with utility functions
* update date utility functions

Author: dshemetov

deploy.json

@@ -32,14 +32,20 @@
       "match": "^.*\\.(py)$",
       "add-header-comment": true
     },
+    {
+      "type": "move",
+      "src": "src/server/utils",
+      "dst": "[[package]]/server/utils/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
     {
       "type": "move",
       "src": "src/server/endpoints/covidcast_utils",
       "dst": "[[package]]/server/endpoints/covidcast_utils/",
       "match": "^.*\\.(py)$",
       "add-header-comment": true
     },
-
     "// acquisition - fluview",
     {
       "type": "move",

integrations/server/test_covidcast_endpoints.py

@@ -1,3 +1,4 @@
+delphi_utils
 epiweeks==2.1.2
 Flask==2.2.2
 itsdangerous<2.1

requirements.api.txt

@@ -219,7 +219,8 @@ def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFram
             df = pd.concat([row.as_dataframe(ignore_fields=ignore_fields) for row in self.rows], ignore_index=True)
             return df[columns]
         else:
-            return pd.DataFrame(columns=columns)
+            df = pd.DataFrame(columns=columns)
+            return set_df_dtypes(df, self._pandas_dtypes)
 
     @property
     def api_row_df(self) -> pd.DataFrame:

src/acquisition/covidcast/covidcast_row.py

@@ -8,6 +8,7 @@
 
 MAX_RESULTS = int(10e6)
 MAX_COMPATIBILITY_RESULTS = int(3650)
+MAX_SMOOTHER_WINDOW = 30
 
 SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "sqlite:///test.db")
 

src/server/_config.py

@@ -1,10 +1,11 @@
-from math import inf
 import re
 from dataclasses import dataclass
+from itertools import groupby
+from math import inf
 from typing import List, Optional, Sequence, Tuple, Union
 
 from flask import request
-
+from more_itertools import flatten
 
 from ._exceptions import ValidationFailedException
 from .utils import days_in_range, weeks_in_range, guess_time_value_is_day, guess_time_value_is_week, TimeValues, days_to_ranges, weeks_to_ranges
@@ -92,9 +93,35 @@ def count(self) -> float:
             return inf if self.signal else 0
         return len(self.signal)
 
+    def add_signal(self, signal: str) -> None:
+        if not isinstance(self.signal, bool):
+            self.signal.append(signal)
+
+    def __hash__(self) -> int:
+        return hash((self.source, self.signal if self.signal is isinstance(self.signal, bool) else tuple(self.signal)))
+
+
+def _combine_source_signal_pairs(source_signal_pairs: List[SourceSignalPair]) -> List[SourceSignalPair]:
+    """Combine SourceSignalPairs with the same source into a single SourceSignalPair object.
+
+    Example:
+    [SourceSignalPair("src", ["sig1", "sig2"]), SourceSignalPair("src", ["sig2", "sig3"])] will be merged
+    into [SourceSignalPair("src", ["sig1", "sig2", "sig3])].
+    """
+    source_signal_pairs_grouped = groupby(sorted(source_signal_pairs, key=lambda x: x.source), lambda x: x.source)
+    source_signal_pairs_combined = []
+    for source, group in source_signal_pairs_grouped:
+        group = list(group)
+        if any(x.signal == True for x in group):
+            combined_signals = True
+        else:
+            combined_signals = sorted(set(flatten(x.signal for x in group)))
+        source_signal_pairs_combined.append(SourceSignalPair(source, combined_signals))
+    return source_signal_pairs_combined
+
 
 def parse_source_signal_arg(key: str = "signal") -> List[SourceSignalPair]:
-    return [SourceSignalPair(source, signals) for [source, signals] in _parse_common_multi_arg(key)]
+    return _combine_source_signal_pairs(SourceSignalPair(source, signals) for [source, signals] in _parse_common_multi_arg(key))
 
 
 def parse_single_source_signal_arg(key: str) -> SourceSignalPair:

src/server/_params.py

@@ -58,7 +58,7 @@ def gen():
                     r = self._print_row(row)
                     if r is not None:
                         yield r
-            except:
+            except Exception as e:
                 get_structured_logger('server_error').error("Exception while executing printer", exception=e)
                 self.result = -1
                 yield self._error(e)

src/server/_printer.py

@@ -98,6 +98,17 @@ def extract_integer(key: Union[str, Sequence[str]]) -> Optional[int]:
         raise ValidationFailedException(f"{key}: not a number: {s}")
 
 
+def extract_float(key: Union[str, Sequence[str]]) -> Optional[float]:
+    s = _extract_value(key)
+    if not s:
+        # nothing to do
+        return None
+    try:
+        return float(s)
+    except ValueError as e:
+        raise ValidationFailedException(f"{key}: not a number: {s}")
+
+
 def extract_integers(key: Union[str, Sequence[str]]) -> Optional[List[IntRange]]:
     parts = extract_strings(key)
     if not parts:
@@ -187,3 +198,13 @@ def push_range(first: str, last: str):
         values.append(parse_date(part))
     # success, return the list
     return values
+
+def extract_bool(key: Union[str, Sequence[str]]) -> Optional[bool]:
+    s = _extract_value(key)
+    if not s:
+        return None
+    if s.lower() == "true":
+        return True
+    if s.lower() == "false":
+        return False
+    raise ValidationFailedException(f"{key}: not a boolean: {s}")