Cleaned up code for table orient in read_json

Author: WillAyd

pandas/io/json/table_schema.py

@@ -80,7 +80,7 @@ def set_default_names(data):
     return data
 
 
-def make_field(arr, dtype=None):
+def convert_pandas_type_to_json_field(arr, dtype=None):
     dtype = dtype or arr.dtype
     if arr.name is None:
         name = 'values'
@@ -108,8 +108,8 @@ def make_field(arr, dtype=None):
     return field
 
 
-def revert_field(field):
-    '''
+def convert_json_field_to_pandas_type(field):
+    """
     Converts a JSON field descriptor into its corresponding NumPy / pandas type
 
     Parameters
@@ -120,9 +120,35 @@ def revert_field(field):
     Returns
     -------
     dtype
-    '''
+
+    Raises
+    -----
+    ValueError
+        If the type of the provided field is unknown or currently unsupported
+
+    Examples
+    --------
+    >>> convert_json_field_to_pandas_type({'name': 'an_int',
+                                           'type': 'integer'})
+    'int64'
+    >>> convert_json_field_to_pandas_type({'name': 'a_categorical',
+                                           'type': 'any',
+                                           'contraints': {'enum': [
+                                                          'a', 'b', 'c']},
+                                           'ordered': True})
+    'CategoricalDtype(categories=['a', 'b', 'c'], ordered=True)'
+    >>> convert_json_field_to_pandas_type({'name': 'a_datetime',
+                                           'type': 'datetime'})
+    'datetime64[ns]'
+    >>> convert_json_field_to_pandas_type({'name': 'a_datetime_with_tz',
+                                           'type': 'datetime',
+                                           'tz': 'US/Central'})
+    'datetime64[ns, US/Central]'
+    """
     typ = field['type']
-    if typ == 'integer':
+    if typ == 'string':
+        return 'object'
+    elif typ == 'integer':
         return 'int64'
     elif typ == 'number':
         return 'float64'
@@ -139,7 +165,10 @@ def revert_field(field):
         if 'constraints' in field and 'ordered' in field:
             return CategoricalDtype(categories=field['constraints']['enum'],
                                     ordered=field['ordered'])
-    return 'object'
+        else:
+            return 'object'
+
+    raise ValueError("Unsupported or invalid field type: {}".format(typ))
 
 
 def build_table_schema(data, index=True, primary_key=None, version=True):
@@ -197,15 +226,15 @@ def build_table_schema(data, index=True, primary_key=None, version=True):
     if index:
         if data.index.nlevels > 1:
             for level in data.index.levels:
-                fields.append(make_field(level))
+                fields.append(convert_pandas_type_to_json_field(level))
         else:
-            fields.append(make_field(data.index))
+            fields.append(convert_pandas_type_to_json_field(data.index))
 
     if data.ndim > 1:
         for column, s in data.iteritems():
-            fields.append(make_field(s))
+            fields.append(convert_pandas_type_to_json_field(s))
     else:
-        fields.append(make_field(data))
+        fields.append(convert_pandas_type_to_json_field(data))
 
     schema['fields'] = fields
     if index and data.index.is_unique and primary_key is None:
@@ -251,7 +280,7 @@ def parse_table_schema(json, precise_float):
     col_order = [field['name'] for field in table['schema']['fields']]
     df = DataFrame(table['data'])[col_order]
 
-    dtypes = {field['name']: revert_field(field)
+    dtypes = {field['name']: convert_json_field_to_pandas_type(field)
               for field in table['schema']['fields']}
 
     # Cannot directly use as_type with timezone data on object; raise for now

pandas/tests/io/json/test_json_table_schema.py

@@ -12,7 +12,8 @@
 from pandas.io.json.table_schema import (
     as_json_table_type,
     build_table_schema,
-    make_field,
+    convert_pandas_type_to_json_field,
+    convert_json_field_to_pandas_type,
     set_default_names)
 import pandas.util.testing as tm
 
@@ -335,62 +336,89 @@ def test_date_format_raises(self):
         self.df.to_json(orient='table', date_format='iso')
         self.df.to_json(orient='table')
 
-    def test_make_field_int(self):
+    def test_convert_pandas_type_to_json_field_int(self):
         data = [1, 2, 3]
         kinds = [pd.Series(data, name='name'), pd.Index(data, name='name')]
         for kind in kinds:
-            result = make_field(kind)
+            result = convert_pandas_type_to_json_field(kind)
             expected = {"name": "name", "type": 'integer'}
             assert result == expected
 
-    def test_make_field_float(self):
+    def test_convert_pandas_type_to_json_field_float(self):
         data = [1., 2., 3.]
         kinds = [pd.Series(data, name='name'), pd.Index(data, name='name')]
         for kind in kinds:
-            result = make_field(kind)
+            result = convert_pandas_type_to_json_field(kind)
             expected = {"name": "name", "type": 'number'}
             assert result == expected
 
-    def test_make_field_datetime(self):
+    def test_convert_pandas_type_to_json_field_datetime(self):
         data = [1., 2., 3.]
         kinds = [pd.Series(pd.to_datetime(data), name='values'),
                  pd.to_datetime(data)]
         for kind in kinds:
-            result = make_field(kind)
+            result = convert_pandas_type_to_json_field(kind)
             expected = {"name": "values", "type": 'datetime'}
             assert result == expected
 
         kinds = [pd.Series(pd.to_datetime(data, utc=True), name='values'),
                  pd.to_datetime(data, utc=True)]
         for kind in kinds:
-            result = make_field(kind)
+            result = convert_pandas_type_to_json_field(kind)
             expected = {"name": "values", "type": 'datetime', "tz": "UTC"}
             assert result == expected
 
         arr = pd.period_range('2016', freq='A-DEC', periods=4)
-        result = make_field(arr)
+        result = convert_pandas_type_to_json_field(arr)
         expected = {"name": "values", "type": 'datetime', "freq": "A-DEC"}
         assert result == expected
 
-    def test_make_field_categorical(self):
+    def test_convert_pandas_type_to_json_field_categorical(self):
         data = ['a', 'b', 'c']
         ordereds = [True, False]
 
         for ordered in ordereds:
             arr = pd.Series(pd.Categorical(data, ordered=ordered), name='cats')
-            result = make_field(arr)
+            result = convert_pandas_type_to_json_field(arr)
             expected = {"name": "cats", "type": "any",
                         "constraints": {"enum": data},
                         "ordered": ordered}
             assert result == expected
 
             arr = pd.CategoricalIndex(data, ordered=ordered, name='cats')
-            result = make_field(arr)
+            result = convert_pandas_type_to_json_field(arr)
             expected = {"name": "cats", "type": "any",
                         "constraints": {"enum": data},
                         "ordered": ordered}
             assert result == expected
 
+    @pytest.mark.parametrize("inp,exp", [
+        ({'type': 'integer'}, 'int64'),
+        ({'type': 'number'}, 'float64'),
+        ({'type': 'boolean'}, 'bool'),
+        ({'type': 'duration'}, 'timedelta64'),
+        ({'type': 'datetime'}, 'datetime64[ns]'),
+        ({'type': 'datetime', 'tz': 'US/Hawaii'}, 'datetime64[ns, US/Hawaii]'),
+        ({'type': 'any'}, 'object'),
+        ({'type': 'any', 'constraints': {'enum': ['a', 'b', 'c']},
+          'ordered': False}, CategoricalDtype(categories=['a', 'b', 'c'],
+                                              ordered=False)),
+        ({'type': 'any', 'constraints': {'enum': ['a', 'b', 'c']},
+          'ordered': True}, CategoricalDtype(categories=['a', 'b', 'c'],
+                                             ordered=True)),
+        ({'type': 'string'}, 'object')])
+    def test_convert_json_field_to_pandas_type(self, inp, exp):
+        field = {'name': 'foo'}
+        field.update(inp)
+        assert convert_json_field_to_pandas_type(field) == exp
+
+    @pytest.mark.parametrize("inp", ["geopoint", "geojson", "fake_type"])
+    def test_convert_json_field_to_pandas_type_raises(self, inp):
+        field = {'type': inp}
+        with tm.assert_raises_regex(ValueError, "Unsupported or invalid field "
+                                    "type: {}".format(inp)):
+            convert_json_field_to_pandas_type(field)
+
     def test_categorical(self):
         s = pd.Series(pd.Categorical(['a', 'b', 'a']))
         s.index.name = 'idx'
@@ -476,92 +504,32 @@ def test_mi_falsey_name(self):
 
 class TestTableOrientReader(object):
 
-    def test_integer(self):
-        df = DataFrame(
-            {'A': [1, 2, 3, 4],
-             },
-            index=pd.Index(range(4), name='idx'))
-        out = df.to_json(orient="table")
-        result = pd.read_json(out, orient="table")
-        tm.assert_frame_equal(df, result)
-
-    def test_object(self):
-        df = DataFrame(
-            {'B': ['a', 'b', 'c', 'c'],
-             },
-            index=pd.Index(range(4), name='idx'))
-        out = df.to_json(orient="table")
-        result = pd.read_json(out, orient="table")
-        tm.assert_frame_equal(df, result)
-
-    def test_date_range(self):
-        df = DataFrame(
-            {'C': pd.date_range('2016-01-01', freq='d', periods=4),
-             },
-            index=pd.Index(range(4), name='idx'))
-
-        out = df.to_json(orient="table")
-        result = pd.read_json(out, orient="table")
-        tm.assert_frame_equal(df, result)
-
-    def test_timedelta_raises(self):
-        df = DataFrame(
-            {'D': pd.timedelta_range('1H', periods=4, freq='T'),
-             },
-            index=pd.Index(range(4), name='idx'))
-
-        out = df.to_json(orient="table")
-        with tm.assert_raises_regex(NotImplementedError, 'can not yet read '
-                                    'ISO-formatted Timedelta data'):
-            pd.read_json(out, orient="table")
-
-    def test_categorical(self):
-        df = DataFrame(
-            {'E': pd.Series(pd.Categorical(['a', 'b', 'c', 'c'])),
-             'F': pd.Series(pd.Categorical(['a', 'b', 'c', 'c'],
-                                           ordered=True)),
-             },
-            index=pd.Index(range(4), name='idx'))
-
+    @pytest.mark.parametrize("vals", [
+        {'ints': [1, 2, 3, 4]},
+        {'objects': ['a', 'b', 'c', 'd']},
+        {'date_ranges': pd.date_range('2016-01-01', freq='d', periods=4)},
+        {'categoricals': pd.Series(pd.Categorical(['a', 'b', 'c', 'c']))},
+        {'ordered_cats': pd.Series(pd.Categorical(['a', 'b', 'c', 'c'],
+                                                  ordered=True))},
+        pytest.param({'floats': [1., 2., 3., 4.]}, marks=pytest.mark.xfail),
+        {'floats': [1.1, 2.2, 3.3, 4.4]},
+        {'bools': [True, False, False, True]}])
+    def test_read_json_table_orient(self, vals):
+        df = DataFrame(vals, index=pd.Index(range(4), name='idx'))
         out = df.to_json(orient="table")
         result = pd.read_json(out, orient="table")
         tm.assert_frame_equal(df, result)
 
-    @pytest.mark.parametrize("float_vals", [
-        pytest.param([1., 2., 3., 4.], marks=pytest.mark.xfail),
-        [1.1, 2.2, 3.3, 4.4]])
-    def test_float(self, float_vals):
-        df = DataFrame(
-            {'G': float_vals,
-             },
-            index=pd.Index(range(4), name='idx'))
-
+    @pytest.mark.parametrize("vals", [
+        {'timedeltas': pd.timedelta_range('1H', periods=4, freq='T')},
+        {'timezones': pd.date_range('2016-01-01', freq='d', periods=4,
+                                    tz='US/Central')}])
+    def test_read_json_table_orient_raises(self, vals):
+        df = DataFrame(vals, index=pd.Index(range(4), name='idx'))
         out = df.to_json(orient="table")
-        result = pd.read_json(out, orient="table", convert_axes=False)
-        tm.assert_frame_equal(df, result)
-
-    def test_timezone_raises(self):
-        df = DataFrame(
-            {'H': pd.date_range('2016-01-01', freq='d', periods=4,
-                                tz='US/Central'),
-             },
-            index=pd.Index(range(4), name='idx'))
-
-        out = df.to_json(orient="table")
-        with tm.assert_raises_regex(NotImplementedError, 'can not yet read '
-                                    'timezone data'):
+        with tm.assert_raises_regex(NotImplementedError, 'can not yet read '):
             pd.read_json(out, orient="table")
 
-    def test_bool(self):
-        df = DataFrame(
-            {'I': [True, False, False, True],
-             },
-            index=pd.Index(range(4), name='idx'))
-
-        out = df.to_json(orient="table")
-        result = pd.read_json(out, orient="table")
-        tm.assert_frame_equal(df, result)
-
     def test_comprehensive(self):
         df = DataFrame(
             {'A': [1, 2, 3, 4],