Fix #21356: JSON nested_to_record Silently Drops Top-Level None Values

Author: daminisatya

pandas/io/json/normalize.py

@@ -80,8 +80,6 @@ def nested_to_record(ds, prefix="", sep=".", level=0):
                 if level != 0:  # so we skip copying for top level, common case
                     v = new_d.pop(k)
                     new_d[newkey] = v
-                elif v is None:  # pop the key if the value is None
-                    new_d.pop(k)
                 continue
             else:
                 v = new_d.pop(k)

pandas/tests/io/json/test_normalize.py

@@ -238,15 +238,16 @@ def test_non_ascii_key(self):
         tm.assert_frame_equal(result, expected)
 
     def test_missing_field(self, author_missing_data):
-        # GH20030: Checks for robustness of json_normalize - should
-        # unnest records where only the first record has a None value
+        # GH20030: Checks for robustness of json_normalize
         result = json_normalize(author_missing_data)
         ex_data = [
-            {'author_name.first': np.nan,
+            {'info': np.nan,
+             'author_name.first': np.nan,
              'author_name.last_name': np.nan,
              'info.created_at': np.nan,
              'info.last_updated': np.nan},
-            {'author_name.first': 'Jane',
+            {'info': None,
+             'author_name.first': 'Jane',
              'author_name.last_name': 'Doe',
              'info.created_at': '11/08/1993',
              'info.last_updated': '26/05/2012'}
@@ -351,9 +352,8 @@ def test_json_normalize_errors(self):
                       errors='raise'
                       )
 
-    def test_nonetype_dropping(self):
-        # GH20030: Checks that None values are dropped in nested_to_record
-        # to prevent additional columns of nans when passed to DataFrame
+    def test_nonetype(self):
+        # GH21356
         data = [
             {'info': None,
              'author_name':
@@ -367,7 +367,8 @@ def test_nonetype_dropping(self):
         ]
         result = nested_to_record(data)
         expected = [
-            {'author_name.first': 'Smith',
+            {'info': None,
+             'author_name.first': 'Smith',
              'author_name.last_name': 'Appleseed'},
             {'author_name.first': 'Jane',
              'author_name.last_name': 'Doe',
@@ -395,6 +396,7 @@ def test_nonetype_top_level_bottom_level(self):
         }
         result = nested_to_record(data)
         expected = {
+            'id': None,
             'location.country.state.id': None,
             'location.country.state.town.info.id': None,
             'location.country.state.town.info.region': None,
@@ -423,6 +425,7 @@ def test_nonetype_multiple_levels(self):
         }
         result = nested_to_record(data)
         expected = {
+            'id': None,
             'location.id': None,
             'location.country.id': None,
             'location.country.state.id': None,