BUG: fixed json_normalize for subrecords with NoneTypes (pandas-dev#20030) (pandas-dev#20399)

Author: aerymilts

doc/source/whatsnew/v0.23.0.txt

@@ -981,6 +981,7 @@ I/O
 - :class:`Timedelta` now supported in :func:`DataFrame.to_excel` for all Excel file types (:issue:`19242`, :issue:`9155`, :issue:`19900`)
 - Bug in :meth:`pandas.io.stata.StataReader.value_labels` raising an ``AttributeError`` when called on very old files. Now returns an empty dict (:issue:`19417`)
 - Bug in :func:`read_pickle` when unpickling objects with :class:`TimedeltaIndex` or :class:`Float64Index` created with pandas prior to version 0.20 (:issue:`19939`)
+- Bug in :meth:`pandas.io.json.json_normalize` where subrecords are not properly normalized if any subrecords values are NoneType (:issue:`20030`)
 
 Plotting
 ^^^^^^^^

pandas/io/json/normalize.py

@@ -80,6 +80,8 @@ def nested_to_record(ds, prefix="", sep=".", level=0):
                 if level != 0:  # so we skip copying for top level, common case
                     v = new_d.pop(k)
                     new_d[newkey] = v
+                if v is None:  # pop the key if the value is None
+                    new_d.pop(k)
                 continue
             else:
                 v = new_d.pop(k)
@@ -189,7 +191,8 @@ def _pull_field(js, spec):
         data = [data]
 
     if record_path is None:
-        if any(isinstance(x, dict) for x in compat.itervalues(data[0])):
+        if any([[isinstance(x, dict)
+                for x in compat.itervalues(y)] for y in data]):
             # naive normalization, this is idempotent for flat records
             # and potentially will inflate the data considerably for
             # deeply nested structures:

pandas/tests/io/json/test_normalize.py

@@ -54,6 +54,17 @@ def state_data():
          'state': 'Ohio'}]
 
 
+@pytest.fixture
+def author_missing_data():
+    return [
+        {'info': None},
+        {'info':
+            {'created_at': '11/08/1993', 'last_updated': '26/05/2012'},
+            'author_name':
+         {'first': 'Jane', 'last_name': 'Doe'}
+         }]
+
+
 class TestJSONNormalize(object):
 
     def test_simple_records(self):
@@ -226,6 +237,23 @@ def test_non_ascii_key(self):
         result = json_normalize(json.loads(testjson))
         tm.assert_frame_equal(result, expected)
 
+    def test_missing_field(self, author_missing_data):
+        # GH20030: Checks for robustness of json_normalize - should
+        # unnest records where only the first record has a None value
+        result = json_normalize(author_missing_data)
+        ex_data = [
+            {'author_name.first': np.nan,
+             'author_name.last_name': np.nan,
+             'info.created_at': np.nan,
+             'info.last_updated': np.nan},
+            {'author_name.first': 'Jane',
+             'author_name.last_name': 'Doe',
+             'info.created_at': '11/08/1993',
+             'info.last_updated': '26/05/2012'}
+        ]
+        expected = DataFrame(ex_data)
+        tm.assert_frame_equal(result, expected)
+
 
 class TestNestedToRecord(object):
 
@@ -322,3 +350,28 @@ def test_json_normalize_errors(self):
                             ['general', 'trade_version']],
                       errors='raise'
                       )
+
+    def test_nonetype_dropping(self):
+        # GH20030: Checks that None values are dropped in nested_to_record
+        # to prevent additional columns of nans when passed to DataFrame
+        data = [
+            {'info': None,
+             'author_name':
+             {'first': 'Smith', 'last_name': 'Appleseed'}
+             },
+            {'info':
+                {'created_at': '11/08/1993', 'last_updated': '26/05/2012'},
+             'author_name':
+                {'first': 'Jane', 'last_name': 'Doe'}
+             }
+        ]
+        result = nested_to_record(data)
+        expected = [
+            {'author_name.first': 'Smith',
+             'author_name.last_name': 'Appleseed'},
+            {'author_name.first': 'Jane',
+             'author_name.last_name': 'Doe',
+             'info.created_at': '11/08/1993',
+             'info.last_updated': '26/05/2012'}]
+
+        assert result == expected