From 38e533b05a23e503c5d7b7f4acd923d7ca1a6893 Mon Sep 17 00:00:00 2001 From: andrew Date: Wed, 4 Nov 2020 13:39:17 -0800 Subject: [PATCH 1/3] refactor to use smoothing utils --- claims_hosp/delphi_claims_hosp/__init__.py | 1 - claims_hosp/delphi_claims_hosp/indicator.py | 10 +++++++--- claims_hosp/tests/test_indicator.py | 6 +++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/__init__.py b/claims_hosp/delphi_claims_hosp/__init__.py index 8a1e78f6c..00f9ec6c9 100644 --- a/claims_hosp/delphi_claims_hosp/__init__.py +++ b/claims_hosp/delphi_claims_hosp/__init__.py @@ -12,7 +12,6 @@ from . import indicator from . import load_data from . import run -from . import smooth from . import update_indicator from . import weekday diff --git a/claims_hosp/delphi_claims_hosp/indicator.py b/claims_hosp/delphi_claims_hosp/indicator.py index 89f1f3880..ac09df480 100644 --- a/claims_hosp/delphi_claims_hosp/indicator.py +++ b/claims_hosp/delphi_claims_hosp/indicator.py @@ -12,16 +12,20 @@ # third party import numpy as np import pandas as pd +from delphi_utils import Smoother # first party from .config import Config -from .smooth import left_gauss_linear + class ClaimsHospIndicator: """ Class to fit a hospitalizations indicator using CLI counts from claims-based data. """ + smoother = Smoother("savgol", + poly_fit_degree=1, + gaussian_bandwidth=Config.SMOOTHER_BANDWIDTH) @staticmethod def gauss_smooth(num, den): @@ -35,8 +39,8 @@ def gauss_smooth(num, den): tuple: (array of smoothed num, array of smoothed den) """ - num_smooth = left_gauss_linear(num) - den_smooth = left_gauss_linear(den) + num_smooth = ClaimsHospIndicator.smoother.smooth(num) + den_smooth = ClaimsHospIndicator.smoother.smooth(den) den_smooth = np.clip(den_smooth, 0, None) num_smooth = np.clip(num_smooth, 0, den_smooth) return num_smooth, den_smooth diff --git a/claims_hosp/tests/test_indicator.py b/claims_hosp/tests/test_indicator.py index 66fcfbd04..24b08696f 100644 --- a/claims_hosp/tests/test_indicator.py +++ b/claims_hosp/tests/test_indicator.py @@ -1,5 +1,5 @@ # third party -from delphi_utils import read_params +from delphi_utils import read_params, Smoother import numpy as np import numpy.random as nr import pandas as pd @@ -19,6 +19,10 @@ class TestLoadData: fips_data = load_data(DATA_FILEPATH, DROP_DATE, "fips") hrr_data = load_data(DATA_FILEPATH, DROP_DATE, "hrr") + ClaimsHospIndicator.smoother = Smoother("savgol", + poly_fit_degree=1, + gaussian_bandwidth=100, + window_length=10) def test_backwards_pad(self): num0 = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8], dtype=float).reshape(-1, 1) From dc79cae475344ed152fa6bdc84ff84ac1d8636c8 Mon Sep 17 00:00:00 2001 From: andrew Date: Wed, 4 Nov 2020 13:48:10 -0800 Subject: [PATCH 2/3] add comment and use config constant for bandwidth --- claims_hosp/delphi_claims_hosp/smooth.py | 40 ------------------------ claims_hosp/tests/test_indicator.py | 5 +-- claims_hosp/tests/test_smooth.py | 15 --------- 3 files changed, 3 insertions(+), 57 deletions(-) delete mode 100644 claims_hosp/delphi_claims_hosp/smooth.py delete mode 100644 claims_hosp/tests/test_smooth.py diff --git a/claims_hosp/delphi_claims_hosp/smooth.py b/claims_hosp/delphi_claims_hosp/smooth.py deleted file mode 100644 index ed2c378db..000000000 --- a/claims_hosp/delphi_claims_hosp/smooth.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -This file contains a left gauss filter used to smooth a 1-d signal. -Code is courtesy of Addison Hu (minor adjustments by Maria). - -Author: Maria Jahja -Created: 2020-04-16 -Modified: 2020-09-27 - - partially concede few naming changes for pylint - -""" -import numpy as np - -from .config import Config - - -def left_gauss_linear(arr, bandwidth=Config.SMOOTHER_BANDWIDTH): - """ - Smooth the y-values using a local linear left Gaussian filter. - - Args: - arr: one dimensional signal to smooth. - bandwidth: smoothing bandwidth (in terms of variance) - - Returns: a smoothed 1D signal. - - """ - - n_rows = len(arr) - out_arr = np.zeros_like(arr) - X = np.vstack([np.ones(n_rows), np.arange(n_rows)]).T - for idx in range(n_rows): - weights = np.exp(-((np.arange(idx + 1) - idx) ** 2) / bandwidth) - XwX = np.dot(X[: (idx + 1), :].T * weights, X[: (idx + 1), :]) - Xwy = np.dot(X[: (idx + 1), :].T * weights, arr[: (idx + 1)].reshape(-1, 1)) - try: - beta = np.linalg.solve(XwX, Xwy) - out_arr[idx] = np.dot(X[: (idx + 1), :], beta)[-1] - except np.linalg.LinAlgError: - out_arr[idx] = np.nan - return out_arr diff --git a/claims_hosp/tests/test_indicator.py b/claims_hosp/tests/test_indicator.py index 24b08696f..24e3c8604 100644 --- a/claims_hosp/tests/test_indicator.py +++ b/claims_hosp/tests/test_indicator.py @@ -19,10 +19,11 @@ class TestLoadData: fips_data = load_data(DATA_FILEPATH, DROP_DATE, "fips") hrr_data = load_data(DATA_FILEPATH, DROP_DATE, "hrr") + # change smoother window length for test data ClaimsHospIndicator.smoother = Smoother("savgol", poly_fit_degree=1, - gaussian_bandwidth=100, - window_length=10) + gaussian_bandwidth=Config.SMOOTHER_BANDWIDTH, + window_length=20) def test_backwards_pad(self): num0 = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8], dtype=float).reshape(-1, 1) diff --git a/claims_hosp/tests/test_smooth.py b/claims_hosp/tests/test_smooth.py deleted file mode 100644 index 76b71b057..000000000 --- a/claims_hosp/tests/test_smooth.py +++ /dev/null @@ -1,15 +0,0 @@ -# third party -import numpy as np -import pytest - -# first party -from delphi_claims_hosp.smooth import left_gauss_linear - - -class TestLeftGaussSmoother: - def test_gauss_linear(self): - signal = np.ones(10) - assert np.allclose(left_gauss_linear(signal)[1:], signal[1:]) - - signal = np.arange(1, 10) + np.random.normal(0, 1, 9) - assert np.allclose(left_gauss_linear(signal, 0.1)[1:], signal[1:]) From e67970b3442ac1d443417a75961c944287519584 Mon Sep 17 00:00:00 2001 From: chinandrew Date: Wed, 11 Nov 2020 07:38:24 -0800 Subject: [PATCH 3/3] Update claims_hosp/tests/test_indicator.py --- claims_hosp/tests/test_indicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/tests/test_indicator.py b/claims_hosp/tests/test_indicator.py index 82110e1bb..66fcfbd04 100644 --- a/claims_hosp/tests/test_indicator.py +++ b/claims_hosp/tests/test_indicator.py @@ -1,5 +1,5 @@ # third party -from delphi_utils import read_params, Smoother +from delphi_utils import read_params import numpy as np import numpy.random as nr import pandas as pd