Fix JHU UID, fix tests

dshemetov · dshemetov · commit cdce10e53dfa · 2020-08-27T11:25:21.000-07:00
diff --git a/_delphi_utils_python/data_proc/geomap/geo_data_proc.py b/_delphi_utils_python/data_proc/geomap/geo_data_proc.py
@@ -164,65 +164,71 @@ def create_jhu_uid_fips_crosswalk():
         [
             # Split aggregation of Dukes and Nantucket, Massachusetts
             {
-                "jhu_uid": 84070002,
+                "jhu_uid": "84070002",
                 "fips": "25007",
                 "weight": 16535 / (16535 + 10172),
             },  # Population: 16535
             {
-                "jhu_uid": 84070002,
+                "jhu_uid": "84070002",
                 "fips": "25019",
                 "weight": 10172 / (16535 + 10172),
             },  # 10172
             # Kansas City, Missouri
             {
-                "jhu_uid": 84070003,
+                "jhu_uid": "84070003",
                 "fips": "29095",
                 "weight": 674158 / 1084897,
             },  # Population: 674158
-            {"jhu_uid": 84070003, "fips": "29165", "weight": 89322 / 1084897},  # 89322
-            {"jhu_uid": 84070003, "fips": "29037", "weight": 99478 / 1084897},  # 99478
+            {"jhu_uid": "84070003", "fips": "29165", "weight": 89322 / 1084897},  # 89322
+            {"jhu_uid": "84070003", "fips": "29037", "weight": 99478 / 1084897},  # 99478
             {
-                "jhu_uid": 84070003,
+                "jhu_uid": "84070003",
                 "fips": "29047",
                 "weight": 221939 / 1084897,
             },  # 221939
             # Kusilvak, Alaska
-            {"jhu_uid": 84002158, "fips": "02270", "weight": 1.0},
+            {"jhu_uid": "84002158", "fips": "02270", "weight": 1.0},
             # Oglala Lakota
-            {"jhu_uid": 84046102, "fips": "46113", "weight": 1.0},
+            {"jhu_uid": "84046102", "fips": "46113", "weight": 1.0},
             # Split aggregation of New York County (populations from JHU documentation)
             {
-                "jhu_uid": 84036061,
+                "jhu_uid": "84036061",
                 "fips": "36005",
                 "weight": 1418207 / 8336817,
             },  # Population: 1,418,207
             {
-                "jhu_uid": 84036061,
+                "jhu_uid": "84036061",
                 "fips": "36047",
                 "weight": 2559903 / 8336817,
             },  # 2,559,903
             {
-                "jhu_uid": 84036061,
+                "jhu_uid": "84036061",
                 "fips": "36061",
                 "weight": 1628706 / 8336817,
             },  # 1,628,706
             {
-                "jhu_uid": 84036061,
+                "jhu_uid": "84036061",
                 "fips": "36081",
                 "weight": 2253858 / 8336817,
             },  # 2,253,858
             {
-                "jhu_uid": 84036061,
+                "jhu_uid": "84036061",
                 "fips": "36085",
                 "weight": 476143 / 8336817,
             },  # 476,143
             # Aggregate Utah into a "State FIPS"
-            {"jhu_uid": 84070015, "fips": "49000", "weight": 1.0},
-            {"jhu_uid": 84070016, "fips": "49000", "weight": 1.0},
-            {"jhu_uid": 84070017, "fips": "49000", "weight": 1.0},
-            {"jhu_uid": 84070018, "fips": "49000", "weight": 1.0},
-            {"jhu_uid": 84070019, "fips": "49000", "weight": 1.0},
-            {"jhu_uid": 84070020, "fips": "49000", "weight": 1.0},
+            {"jhu_uid": "84070015", "fips": "49000", "weight": 1.0},
+            {"jhu_uid": "84070016", "fips": "49000", "weight": 1.0},
+            {"jhu_uid": "84070017", "fips": "49000", "weight": 1.0},
+            {"jhu_uid": "84070018", "fips": "49000", "weight": 1.0},
+            {"jhu_uid": "84070019", "fips": "49000", "weight": 1.0},
+            {"jhu_uid": "84070020", "fips": "49000", "weight": 1.0},
+        ]
+    )
+    unassigned_states = pd.DataFrame(
+        [
+            # Map the Unassigned states FIPS to XX000
+            {"jhu_uid": str(x), "fips": str(x)[-2:].ljust(5, "0"), "weight": 1.0} for x in range(84090001, 84090099)
         ]
     )
 
@@ -234,25 +240,25 @@ def create_jhu_uid_fips_crosswalk():
     )
 
     # FIPS Codes that are just two digits long should be zero filled on the right.
-    # These are Guam (66), Northern Mariana Islands (69), Virgin Islands (78),
-    # and Puerto Rico (72).
+    # These are US states and territories Guam (66), Northern Mariana Islands (69),
+    # Virgin Islands (78), and Puerto Rico (72).
     fips_st = jhu_df["fips"].str.len() <= 2
     jhu_df.loc[fips_st, "fips"] = (
-        jhu_df.loc[fips_st, "fips"].astype(str).str.ljust(5, "0")
+        jhu_df.loc[fips_st, "fips"].str.ljust(5, "0")
     )
 
-    # Drop the JHU UIDs that were hand-modified
-    dup_ind = jhu_df["jhu_uid"].isin(hand_additions["jhu_uid"].values)
+    # Drop the FIPS codes in JHU that were hand-modified
+    dup_ind = jhu_df["fips"].isin(hand_additions["fips"].values) | jhu_df["fips"].isin(["02158", "46102"])
     jhu_df.drop(jhu_df.index[dup_ind], inplace=True)
 
-    # Drop the FIPS codes in JHU that were hand-modified
-    dup_ind = jhu_df["fips"].isin(hand_additions["fips"].values)
+    # Drop the JHU UIDs that were hand-modified
+    dup_ind = jhu_df["jhu_uid"].isin(hand_additions["jhu_uid"].values) | jhu_df["jhu_uid"].isin(unassigned_states["jhu_uid"].values) 
     jhu_df.drop(jhu_df.index[dup_ind], inplace=True)
 
     # Add weights of 1.0 to everything not in hand additions, then merge in hand-additions
     # Finally, zero fill FIPS
     jhu_df["weight"] = 1.0
-    jhu_df = pd.concat((jhu_df, hand_additions))
+    jhu_df = pd.concat((jhu_df, hand_additions, unassigned_states))
     jhu_df["fips"] = jhu_df["fips"].astype(int).astype(str).str.zfill(5)
     jhu_df.to_csv(join(OUTPUT_DIR, JHU_FIPS_OUT_FILENAME), index=False)
 
diff --git a/_delphi_utils_python/delphi_utils/data/jhu_uid_fips_table.csv b/_delphi_utils_python/delphi_utils/data/jhu_uid_fips_table.csv
@@ -187,57 +187,6 @@ jhu_uid,fips,weight
 84080054,80054,1.0
 84080055,80055,1.0
 84080056,80056,1.0
-84090001,90001,1.0
-84090002,90002,1.0
-84090004,90004,1.0
-84090005,90005,1.0
-84090006,90006,1.0
-84090008,90008,1.0
-84090009,90009,1.0
-84090010,90010,1.0
-84090011,90011,1.0
-84090012,90012,1.0
-84090013,90013,1.0
-84090015,90015,1.0
-84090016,90016,1.0
-84090017,90017,1.0
-84090018,90018,1.0
-84090019,90019,1.0
-84090020,90020,1.0
-84090021,90021,1.0
-84090022,90022,1.0
-84090023,90023,1.0
-84090024,90024,1.0
-84090025,90025,1.0
-84090026,90026,1.0
-84090027,90027,1.0
-84090028,90028,1.0
-84090029,90029,1.0
-84090030,90030,1.0
-84090031,90031,1.0
-84090032,90032,1.0
-84090033,90033,1.0
-84090034,90034,1.0
-84090035,90035,1.0
-84090036,90036,1.0
-84090037,90037,1.0
-84090038,90038,1.0
-84090039,90039,1.0
-84090040,90040,1.0
-84090041,90041,1.0
-84090042,90042,1.0
-84090044,90044,1.0
-84090045,90045,1.0
-84090046,90046,1.0
-84090047,90047,1.0
-84090048,90048,1.0
-84090049,90049,1.0
-84090050,90050,1.0
-84090051,90051,1.0
-84090053,90053,1.0
-84090054,90054,1.0
-84090055,90055,1.0
-84090056,90056,1.0
 84001001,01001,1.0
 84001003,01003,1.0
 84001005,01005,1.0
@@ -319,7 +268,6 @@ jhu_uid,fips,weight
 84002122,02122,1.0
 84002130,02130,1.0
 84002150,02150,1.0
-84002158,02158,1.0
 84002164,02164,1.0
 84002170,02170,1.0
 84002180,02180,1.0
@@ -2639,7 +2587,6 @@ jhu_uid,fips,weight
 84046097,46097,1.0
 84046099,46099,1.0
 84046101,46101,1.0
-84046102,46102,1.0
 84046103,46103,1.0
 84046105,46105,1.0
 84046107,46107,1.0
@@ -3388,3 +3335,101 @@ jhu_uid,fips,weight
 84070018,49000,1.0
 84070019,49000,1.0
 84070020,49000,1.0
+84090001,01000,1.0
+84090002,02000,1.0
+84090003,03000,1.0
+84090004,04000,1.0
+84090005,05000,1.0
+84090006,06000,1.0
+84090007,07000,1.0
+84090008,08000,1.0
+84090009,09000,1.0
+84090010,10000,1.0
+84090011,11000,1.0
+84090012,12000,1.0
+84090013,13000,1.0
+84090014,14000,1.0
+84090015,15000,1.0
+84090016,16000,1.0
+84090017,17000,1.0
+84090018,18000,1.0
+84090019,19000,1.0
+84090020,20000,1.0
+84090021,21000,1.0
+84090022,22000,1.0
+84090023,23000,1.0
+84090024,24000,1.0
+84090025,25000,1.0
+84090026,26000,1.0
+84090027,27000,1.0
+84090028,28000,1.0
+84090029,29000,1.0
+84090030,30000,1.0
+84090031,31000,1.0
+84090032,32000,1.0
+84090033,33000,1.0
+84090034,34000,1.0
+84090035,35000,1.0
+84090036,36000,1.0
+84090037,37000,1.0
+84090038,38000,1.0
+84090039,39000,1.0
+84090040,40000,1.0
+84090041,41000,1.0
+84090042,42000,1.0
+84090043,43000,1.0
+84090044,44000,1.0
+84090045,45000,1.0
+84090046,46000,1.0
+84090047,47000,1.0
+84090048,48000,1.0
+84090049,49000,1.0
+84090050,50000,1.0
+84090051,51000,1.0
+84090052,52000,1.0
+84090053,53000,1.0
+84090054,54000,1.0
+84090055,55000,1.0
+84090056,56000,1.0
+84090057,57000,1.0
+84090058,58000,1.0
+84090059,59000,1.0
+84090060,60000,1.0
+84090061,61000,1.0
+84090062,62000,1.0
+84090063,63000,1.0
+84090064,64000,1.0
+84090065,65000,1.0
+84090066,66000,1.0
+84090067,67000,1.0
+84090068,68000,1.0
+84090069,69000,1.0
+84090070,70000,1.0
+84090071,71000,1.0
+84090072,72000,1.0
+84090073,73000,1.0
+84090074,74000,1.0
+84090075,75000,1.0
+84090076,76000,1.0
+84090077,77000,1.0
+84090078,78000,1.0
+84090079,79000,1.0
+84090080,80000,1.0
+84090081,81000,1.0
+84090082,82000,1.0
+84090083,83000,1.0
+84090084,84000,1.0
+84090085,85000,1.0
+84090086,86000,1.0
+84090087,87000,1.0
+84090088,88000,1.0
+84090089,89000,1.0
+84090090,90000,1.0
+84090091,91000,1.0
+84090092,92000,1.0
+84090093,93000,1.0
+84090094,94000,1.0
+84090095,95000,1.0
+84090096,96000,1.0
+84090097,97000,1.0
+84090098,98000,1.0
diff --git a/_delphi_utils_python/delphi_utils/geomap.py b/_delphi_utils_python/delphi_utils/geomap.py
@@ -224,14 +224,14 @@ def add_new_code(self, df, from_code, new_code, from_col=None, new_col=None):
 
         if not is_string_dtype(df[from_col]):
             if from_code in ["fips", "zip"]:
-                df[from_col] = df[from_col].astype(str).zfill(5)
+                df[from_col] = df[from_col].astype(str).str.zfill(5)
             else:
                 df[from_col] = df[from_col].astype(str)
 
         crosswalk = self.load_crosswalk(from_code=from_code, to_code=new_code)
         crosswalk = crosswalk.rename(columns={from_code: from_col, new_code: new_col})
 
-        df = df.merge(crosswalk, left_on=from_col, right_on=from_col, how="left")
+        df = df.merge(crosswalk, left_on=from_col, right_on=from_col, how="left").dropna(subset=[new_col])
 
         # Drop extra state columns
         state_codes = ["state_code", "state_id", "state_name"]
@@ -279,8 +279,7 @@ def convert_to_new_code(
 
         df = self.add_new_code(
             df, from_code, new_code, from_col=from_col, new_col=new_col
-        )
-        df.drop(columns=from_col, inplace=True)
+        ).drop(columns=from_col)
 
         if "weight" in df.columns:
             if data_cols is None:
@@ -327,7 +326,7 @@ def convert_fips_to_state_code(
         data = data.copy()
 
         if not is_string_dtype(data[fips_col]):
-            data[fips_col] = data[fips_col].astype(str).zfill(5)
+            data[fips_col] = data[fips_col].astype(str).str.zfill(5)
 
         # Take the first two digits of the FIPS code
         data[state_code_col] = data[fips_col].str[:2]
@@ -425,8 +424,7 @@ def convert_fips_to_zip(
             data[fips_col] = data[fips_col].astype(str).str.zfill(5)
 
         cross = df.rename(columns={"zip": zip_col, "weight": weight_col})
-        data = data.merge(cross, left_on=fips_col, right_on="fips", how="left")
-
+        data = data.merge(cross, left_on=fips_col, right_on="fips", how="left").dropna(subset=[zip_col])
         return data
 
     def convert_state_code_to_state_id(
@@ -617,9 +615,8 @@ def fips_to_state_id(
         data = self.convert_fips_to_state_id(
             data, fips_col=fips_col, state_id_col=state_id_col
         )
-        data.dropna(subset=["state_code"], axis=0, inplace=True)
-        data.drop([fips_col, "state_code"], axis=1, inplace=True)
-        data = data.groupby([date_col, state_id_col], dropna=False).sum()
+        # data.drop([fips_col, "state_code"], axis=1, inplace=True)
+        data = data.groupby([date_col, state_id_col]).sum()
         return data.reset_index()
 
     def fips_to_msa(
@@ -780,7 +777,7 @@ def zip_to_hrr(
 
         data = data[[zip_col, date_col] + count_cols].copy()
         data = self.convert_zip_to_hrr(data, zip_col=zip_col, hrr_col=hrr_col)
-        data = data.groupby([date_col, hrr_col], dropna=False).sum()
+        data = data.groupby([date_col, hrr_col]).sum()
         return data.reset_index()
 
     def convert_jhu_uid_to_fips(
diff --git a/_delphi_utils_python/tests/test_geomap.py b/_delphi_utils_python/tests/test_geomap.py
