Added keyword errors {'raise'|'ignore}

Added documenation
Shortened what's new
Removed commas in dictionary for linting compatibility

Author: dickreuter

pandas/io/json.py

@@ -22,10 +22,9 @@
 def to_json(path_or_buf, obj, orient=None, date_format='epoch',
             double_precision=10, force_ascii=True, date_unit='ms',
             default_handler=None, lines=False):
-
     if lines and orient != 'records':
-            raise ValueError(
-                "'lines' keyword only valid when 'orient' is records")
+        raise ValueError(
+            "'lines' keyword only valid when 'orient' is records")
 
     if isinstance(obj, Series):
         s = SeriesWriter(
@@ -53,7 +52,6 @@ def to_json(path_or_buf, obj, orient=None, date_format='epoch',
 
 
 class Writer(object):
-
     def __init__(self, obj, orient, date_format, double_precision,
                  ensure_ascii, date_unit, default_handler=None):
         self.obj = obj
@@ -291,7 +289,6 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
 
 
 class Parser(object):
-
     _STAMP_UNITS = ('s', 'ms', 'us', 'ns')
     _MIN_STAMPS = {
         's': long(31536000),
@@ -492,8 +489,8 @@ def _parse_no_numpy(self):
         if orient == "split":
             decoded = dict((str(k), v)
                            for k, v in compat.iteritems(loads(
-                               json,
-                               precise_float=self.precise_float)))
+                json,
+                precise_float=self.precise_float)))
             self.check_keys_split(decoded)
             self.obj = Series(dtype=None, **decoded)
         else:
@@ -567,8 +564,8 @@ def _parse_no_numpy(self):
         elif orient == "split":
             decoded = dict((str(k), v)
                            for k, v in compat.iteritems(loads(
-                               json,
-                               precise_float=self.precise_float)))
+                json,
+                precise_float=self.precise_float)))
             self.check_keys_split(decoded)
             self.obj = DataFrame(dtype=None, **decoded)
         elif orient == "index":
@@ -595,7 +592,6 @@ def _process_converter(self, f, filt=None):
             new_obj[i] = c
 
         if needs_new_obj:
-
             # possibly handle dup columns
             new_obj = DataFrame(new_obj, index=self.obj.index)
             new_obj.columns = self.obj.columns
@@ -628,9 +624,9 @@ def is_ok(col):
             col_lower = col.lower()
             if (col_lower.endswith('_at') or
                     col_lower.endswith('_time') or
-                    col_lower == 'modified' or
-                    col_lower == 'date' or
-                    col_lower == 'datetime' or
+                        col_lower == 'modified' or
+                        col_lower == 'date' or
+                        col_lower == 'datetime' or
                     col_lower.startswith('timestamp')):
                 return True
             return False
@@ -640,6 +636,7 @@ def is_ok(col):
             lambda col, c: ((self.keep_default_dates and is_ok(col)) or
                             col in convert_dates))
 
+
 # ---------------------------------------------------------------------
 # JSON normalization routines
 
@@ -723,7 +720,7 @@ def nested_to_record(ds, prefix="", level=0):
 
 def json_normalize(data, record_path=None, meta=None,
                    meta_prefix=None,
-                   record_prefix=None):
+                   record_prefix=None, errors='raise'):
     """
     "Normalize" semi-structured JSON data into a flat table
 
@@ -740,6 +737,8 @@ def json_normalize(data, record_path=None, meta=None,
         If True, prefix records with dotted (?) path, e.g. foo.bar.field if
         path to records is ['foo', 'bar']
     meta_prefix : string, default None
+    error: {'raise', 'ignore'}, default 'raise'
+        * ignore: will ignore keyErrors if keys listed in meta are not always present
 
     Returns
     -------
@@ -775,6 +774,7 @@ def json_normalize(data, record_path=None, meta=None,
     4    Cuyahoga        1337   John Kasich     Ohio        OH
 
     """
+
     def _pull_field(js, spec):
         result = js
         if isinstance(spec, list):
@@ -841,8 +841,11 @@ def _recursive_extract(data, path, seen_meta, level=0):
                     else:
                         try:
                             meta_val = _pull_field(obj, val[level:])
-                        except:
-                            meta_val = np.nan
+                        except KeyError as e:
+                            if errors == 'ignore':
+                                meta_val = np.nan
+                            else:
+                                raise KeyError("Try running with errors='ignore' as the following key may not always be present: "+str(e))
                     meta_vals[key].append(meta_val)
 
                 records.extend(recs)

pandas/io/tests/json/test_json_norm.py

@@ -225,7 +225,9 @@ def test_nested_flattens(self):
 
         self.assertEqual(result, expected)
 
+
     def test_json_normalise_fix(self):
+        # issue 14505
         j = {
             "Trades": [{
                 "general": {
@@ -245,7 +247,7 @@ def test_json_normalise_fix(self):
 
                     }
                     ]
-                },
+                }
             }, {
                 "general": {
                     "tradeid": 100,
@@ -262,13 +264,19 @@ def test_json_normalise_fix(self):
 
                     }
                     ]
-                },
+                }
             }
             ]
         }
         j = json_normalize(data=j['Trades'], record_path=[['general', 'stocks']],
-                           meta=[['general', 'tradeid'], ['general', 'trade_version']])
-        self.assertEqual(len(j), 4)
+                           meta=[['general', 'tradeid'], ['general', 'trade_version']], errors='ignore')
+        expected={'general.trade_version': {0: 1.0, 1: 1.0, 2: '', 3: ''},
+         'general.tradeid': {0: 100, 1: 100, 2: 100, 3: 100},
+         'name': {0: 'Apple', 1: 'Google', 2: 'Apple', 3: 'Google'},
+         'price': {0: '0', 1: '0', 2: '0', 3: '0'},
+         'symbol': {0: 'AAPL', 1: 'GOOG', 2: 'AAPL', 3: 'GOOG'}}
+
+        self.assertEqual(j.fillna('').to_dict(), expected)
 
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb',