Merge branch 'main' into sg-docs

Author: chinandrew

.gitignore

@@ -126,3 +126,7 @@ venv.bak/
 .retry
 .indicators-ansible-vault-pass
 indicators-ansible-vault-pass
+
+# testing_utils
+testing_utils/cache
+testing_utils/*.csv

_delphi_utils_python/.pylintrc

@@ -1,8 +1,21 @@
-[DESIGN]
 
-min-public-methods=1
+[MESSAGES CONTROL]
 
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
 
-[MESSAGES CONTROL]
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
 
-disable=R0801, C0330, E1101, E0611, C0114, C0116, C0103, R0913, R0914, W0702
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

_delphi_utils_python/data_proc/geomap/geo_data_proc.py

@@ -217,6 +217,12 @@ def create_jhu_uid_fips_crosswalk():
             {"jhu_uid": "63072999", "fips": "72000", "weight": 1.0},
         ]
     )
+    cruise_ships = pd.DataFrame(
+        [
+            {"jhu_uid": "84088888", "fips": "88888", "weight": 1.0},
+            {"jhu_uid": "84099999", "fips": "99999", "weight": 1.0},
+        ]
+    )
 
     jhu_df = (
         pd.read_csv(JHU_FIPS_URL, dtype={"UID": str, "FIPS": str})
@@ -234,7 +240,7 @@ def create_jhu_uid_fips_crosswalk():
     # Drop the JHU UIDs that were hand-modified
     dup_ind = jhu_df["jhu_uid"].isin(
         pd.concat(
-            [hand_additions, unassigned_states, out_of_state, puerto_rico_unassigned]
+            [hand_additions, unassigned_states, out_of_state, puerto_rico_unassigned, cruise_ships]
         )["jhu_uid"].values
     )
     jhu_df.drop(jhu_df.index[dup_ind], inplace=True)
@@ -391,6 +397,7 @@ def create_fips_population_table():
     df_pr = df_pr.groupby("fips").sum().reset_index()
     df_pr = df_pr[~df_pr["fips"].isin(census_pop["fips"])]
     census_pop_pr = pd.concat([census_pop, df_pr])
+
     census_pop_pr.to_csv(join(OUTPUT_DIR, FIPS_POPULATION_OUT_FILENAME), index=False)
 
 

_delphi_utils_python/delphi_utils/__init__.py

@@ -7,6 +7,9 @@
 from .archive import ArchiveDiffer, GitArchiveDiffer, S3ArchiveDiffer
 from .export import create_export_csv
 from .utils import read_params
+
 from .geomap import GeoMapper
+from .smooth import Smoother
+from .signal import add_prefix, public_signal
 
 __version__ = "0.1.0"

_delphi_utils_python/delphi_utils/data/jhu_uid_fips_table.csv

@@ -82,8 +82,6 @@ jhu_uid,fips,weight
 63072149,72149,1.0
 63072151,72151,1.0
 63072153,72153,1.0
-84088888,88888,1.0
-84099999,99999,1.0
 84000001,01000,1.0
 84000002,02000,1.0
 84000004,04000,1.0

_delphi_utils_python/delphi_utils/geomap.py

@@ -413,9 +413,10 @@ def replace_geocode(
             df = df.groupby([new_col]).sum().reset_index()
         return df
 
-    def add_population_column(self, data, geocode_type, geocode_col=None):
+    def add_population_column(self, data, geocode_type, geocode_col=None, dropna=True):
         """
-        Appends a population column to a dateframe, based on the FIPS or ZIP code.
+        Appends a population column to a dataframe, based on the FIPS or ZIP code. If no
+        dataframe is provided, the full crosswalk from geocode to population is returned.
 
         Parameters
         ---------
@@ -433,24 +434,26 @@ def add_population_column(self, data, geocode_type, geocode_col=None):
             A dataframe with a population column appended.
         """
         geocode_col = geocode_type if geocode_col is None else geocode_col
+        data = data.copy()
 
         if geocode_type not in ["fips", "zip"]:
             raise ValueError(
                 "Only fips and zip geocodes supported. \
                 For other codes, aggregate those."
             )
 
+        pop_df = self._load_crosswalk(from_code=geocode_type, to_code="pop")
+
         if not is_string_dtype(data[geocode_col]):
             data[geocode_col] = data[geocode_col].astype(str).str.zfill(5)
 
-        pop_df = self._load_crosswalk(from_code=geocode_type, to_code="pop")
-
+        merge_type = "inner" if dropna else "left"
         data_with_pop = (
-            data.copy()
-            .merge(pop_df, left_on=geocode_col, right_on=geocode_type, how="inner")
+            data
+            .merge(pop_df, left_on=geocode_col, right_on=geocode_type, how=merge_type)
             .rename(columns={"pop": "population"})
         )
-        data_with_pop["population"] = data_with_pop["population"].astype(int)
+
         return data_with_pop
 
     @staticmethod

quidel_covidtest/delphi_quidel_covidtest/handle_wip_sensor.py renamed to _delphi_utils_python/delphi_utils/signal.py

@@ -1,13 +1,13 @@
-"""This file checks the wip status of signals"""
+"""Functions for understanding and creating signal names."""
 import covidcast
 
-def add_prefix(signal_names, wip_signal, prefix):
+def add_prefix(signal_names, wip_signal, prefix="wip_"):
     """Adds prefix to signal if there is a WIP signal
     Parameters
     ----------
     signal_names: List[str]
         Names of signals to be exported
-    prefix : 'wip_'
+    prefix : "wip_"
         prefix for new/non public signals
     wip_signal : List[str] or bool
         a list of wip signals: [], OR
@@ -36,11 +36,11 @@ def add_prefix(signal_names, wip_signal, prefix):
     raise ValueError("Supply True | False or '' or [] | list()")
 
 
-def public_signal(signal_):
+def public_signal(signal):
     """Checks if the signal name is already public using COVIDcast
     Parameters
     ----------
-    signal_ : str
+    signal : str
         Name of the signal
     Returns
     -------
@@ -50,6 +50,6 @@ def public_signal(signal_):
     """
     epidata_df = covidcast.metadata()
     for index in range(len(epidata_df)):
-        if epidata_df['signal'][index] == signal_:
+        if epidata_df["signal"][index] == signal:
             return True
     return False