category cleaning works

Author: climatebrad

data_cleaner.py

@@ -175,12 +175,13 @@ def simplify_category(data, catname):
 def clean_categories(data):
     """get data categories ready for one-hot-encoding"""
     cat_set = set(CLEAN_CAT_VALUES) | set(CAT_FILL_NA_VALUES) 
-    data = data.astype({cat : 'object' for cat in cat_set}) \
+    cat_set = cat_set.intersection(set(data.columns))
+    data = data.astype({cat : 'object' for cat in cat_set}, errors='ignore') \
                 .replace(CLEAN_CAT_VALUES) \
                 .fillna(CAT_FILL_NA_VALUES) \
                 .dropna(subset=CLEAN_CAT_VALUES.keys()) \
                 .drop(columns=MODEL_IGNORE_COLS, errors='ignore') \
-                .astype({cat : 'category' for cat in cat_set})
+                .astype({cat : 'category' for cat in cat_set}, errors='ignore')
     for catname in REPLACE_REVERSE_DICT:
         data = simplify_category(data, catname)    
     return data

fullmodel.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 81,
+   "execution_count": 103,
    "metadata": {},
    "outputs": [
     {
@@ -19,7 +19,7 @@
        "<module 'data_modeler' from '/Users/brad/Documents/Scripts/flatiron/stop-and-frisk/data_modeler.py'>"
       ]
      },
-     "execution_count": 81,
+     "execution_count": 103,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -53,7 +53,13 @@
      "text": [
       "Loading 2003...\n",
       "Loading 2004...\n",
-      "Loading 2005...\n"
+      "Loading 2005...\n",
+      "Loading 2006...\n",
+      "Loading 2007...\n",
+      "Loading 2008...\n",
+      "Loading 2009...\n",
+      "Loading 2010...\n",
+      "Loading 2011...\n"
      ]
     }
    ],