Leonlu2/delete correlation endpoint

integrations/server/test_covidcast_endpoints.py

@@ -182,32 +182,6 @@ def match_row(trend, row):
             self.assertEqual(trend["max_value"], first.value)
             self.assertEqual(trend["max_trend"], "decreasing")
 
-    def test_correlation(self):
-        """Request a signal from the /correlation endpoint."""
-
-        num_rows = 30
-        reference_rows = [CovidcastTestRow.make_default_row(signal="ref", time_value=20200401 + i, value=i) for i in range(num_rows)]
-        first = reference_rows[0]
-        self._insert_rows(reference_rows)
-        other_rows = [CovidcastTestRow.make_default_row(signal="other", time_value=20200401 + i, value=i) for i in range(num_rows)]
-        other = other_rows[0]
-        self._insert_rows(other_rows)
-        max_lag = 3
-
-        out = self._fetch("/correlation", reference=first.signal_pair(), others=other.signal_pair(), geo=first.geo_pair(), window="20200401-20201212", lag=max_lag)
-        self.assertEqual(out["result"], 1)
-        df = pd.DataFrame(out["epidata"])
-        self.assertEqual(len(df), max_lag * 2 + 1)  # -...0...+
-        self.assertEqual(df["geo_type"].unique().tolist(), [first.geo_type])
-        self.assertEqual(df["geo_value"].unique().tolist(), [first.geo_value])
-        self.assertEqual(df["signal_source"].unique().tolist(), [other.source])
-        self.assertEqual(df["signal_signal"].unique().tolist(), [other.signal])
-
-        self.assertEqual(df["lag"].tolist(), list(range(-max_lag, max_lag + 1)))
-        self.assertEqual(df["r2"].unique().tolist(), [1.0])
-        self.assertEqual(df["slope"].unique().tolist(), [1.0])
-        self.assertEqual(df["intercept"].tolist(), [3.0, 2.0, 1.0, 0.0, -1.0, -2.0, -3.0])
-        self.assertEqual(df["samples"].tolist(), [num_rows - abs(l) for l in range(-max_lag, max_lag + 1)])
 
     def test_csv(self):
         """Request a signal from the /csv endpoint."""

src/server/endpoints/covidcast.py

@@ -32,7 +32,7 @@
 from .._printer import create_printer, CSVPrinter
 from .._validate import require_all
 from .._pandas import as_pandas, print_pandas
-from .covidcast_utils import compute_trend, compute_trends, compute_correlations, compute_trend_value, CovidcastMetaEntry
+from .covidcast_utils import compute_trend, compute_trends, compute_trend_value, CovidcastMetaEntry
 from ..utils import shift_day_value, day_to_time_value, time_value_to_iso, time_value_to_day, shift_week_value, time_value_to_week, guess_time_value_is_day, week_to_time_value, TimeValues
 from .covidcast_utils.model import TimeType, count_signal_time_types, data_sources, create_source_signal_alias_mapper
 
@@ -206,79 +206,6 @@ def gen(rows):
     return p(filter_fields(gen(r)))
 
 
-@bp.route("/correlation", methods=("GET", "POST"))
-def handle_correlation():
-    require_all(request, "reference", "window", "others", "geo")
-    reference = parse_single_source_signal_arg("reference")
-    other_sets = parse_source_signal_arg("others")
-    daily_signals, weekly_signals = count_signal_time_types(other_sets + [reference])
-    source_signal_sets, alias_mapper = create_source_signal_alias_mapper(other_sets + [reference])
-    geo_sets = parse_geo_arg()
-    time_window = parse_day_or_week_range_arg("window")
-    is_day = time_window.is_day
-    _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals)
-
-    lag = extract_integer("lag")
-    if lag is None:
-        lag = 28
-
-    # `lag` above is used in post-processing, not in the database query, so we can use latest here
-    q = QueryBuilder(latest_table, "t")
-
-    fields_string = ["geo_type", "geo_value", "source", "signal"]
-    fields_int = ["time_value"]
-    fields_float = ["value"]
-    q.set_fields(fields_string, fields_int, fields_float)
-    q.set_sort_order("geo_type", "geo_value", "source", "signal", "time_value")
-
-    q.apply_source_signal_filters(
-        "source",
-        "signal",
-        source_signal_sets,
-    )
-    q.apply_geo_filters("geo_type", "geo_value", geo_sets)
-    q.apply_time_filter("time_type", "time_value", time_window)
-
-    df = as_pandas(str(q), q.params)
-    if is_day:
-        df["time_value"] = to_datetime(df["time_value"], format="%Y%m%d")
-    else:
-        # week but convert to date for simpler shifting
-        df["time_value"] = to_datetime(df["time_value"].apply(lambda v: time_value_to_week(v).startdate()))
-
-    p = create_printer(request.values.get("format"))
-
-    def prepare_data_frame(df):
-        return df[["time_value", "value"]].set_index("time_value")
-
-    def gen():
-        by_geo = df.groupby(["geo_type", "geo_value"])
-        for (geo_type, geo_value), group in by_geo:
-            # group by source, signal
-            by_signal = group.groupby(["source", "signal"])
-
-            # find reference group
-            # dataframe structure: index=time_value, value=value
-            reference_group = next((prepare_data_frame(group) for (source, signal), group in by_signal if source == reference.source and signal == reference.signal[0]), None)
-
-            if reference_group is None or reference_group.empty:
-                continue  # no data for reference
-
-            # dataframe structure: index=time_value, value=value
-            other_groups = [((source, signal), prepare_data_frame(group)) for (source, signal), group in by_signal if not (source == reference.source and signal == reference.signal[0])]
-            if not other_groups:
-                continue  # no other signals
-
-            for (source, signal), other_group in other_groups:
-                if alias_mapper:
-                    source = alias_mapper(source, signal)
-                for cor in compute_correlations(geo_type, geo_value, source, signal, lag, reference_group, other_group, is_day):
-                    yield cor.asdict()
-
-    # now use a generator for sending the rows and execute all the other queries
-    return p(filter_fields(gen()))
-
-
 @bp.route("/csv", methods=("GET", "POST"))
 def handle_export():
     source, signal = request.values.get("signal", "jhu-csse:confirmed_incidence_num").split(":")

src/server/endpoints/covidcast_utils/__init__.py

@@ -1,3 +1,2 @@
 from .trend import compute_trend, compute_trend_value, compute_trends
-from .correlation import compute_correlations
 from .meta import CovidcastMetaEntry

tests/server/test_pandas.py

@@ -25,7 +25,7 @@ def setUp(self):
     @patch("delphi.epidata.server._pandas.text")
     @patch("pandas.read_sql_query")
     def test_as_pandas(self, mock_read_sql_query, mock_sqlalch_text):
-        with app.test_request_context('/correlation'):
+        with app.test_request_context('covidcast/'):
 
             mock_sqlalch_text.return_value = sentinel.default_limit
             as_pandas("", params=None, db_engine=None)