From 6dfd9763420e12a667e8206e075fffb51a31adb0 Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Mon, 4 Jun 2018 22:05:59 +0300 Subject: [PATCH 01/12] BUG: Fix empty Data frames to JSON round-trippable back to data frames (#21287) --- pandas/io/json/table_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/json/table_schema.py b/pandas/io/json/table_schema.py index 01f7db7d68664..5cea64388bdd7 100644 --- a/pandas/io/json/table_schema.py +++ b/pandas/io/json/table_schema.py @@ -296,7 +296,7 @@ def parse_table_schema(json, precise_float): """ table = loads(json, precise_float=precise_float) col_order = [field['name'] for field in table['schema']['fields']] - df = DataFrame(table['data'])[col_order] + df = DataFrame(table['data'], columns=col_order)[col_order] dtypes = {field['name']: convert_json_field_to_pandas_type(field) for field in table['schema']['fields']} From 466e5a659cb601df9e588e4524da11ed7859e407 Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Mon, 4 Jun 2018 23:33:51 +0300 Subject: [PATCH 02/12] Add test and whatsnew --- doc/source/whatsnew/v0.23.1.txt | 2 +- pandas/tests/io/json/test_json_table_schema.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.1.txt b/doc/source/whatsnew/v0.23.1.txt index f2bc81eea186b..cce0448cd334f 100644 --- a/doc/source/whatsnew/v0.23.1.txt +++ b/doc/source/whatsnew/v0.23.1.txt @@ -92,7 +92,7 @@ I/O - Bug in IO methods specifying ``compression='zip'`` which produced uncompressed zip archives (:issue:`17778`, :issue:`21144`) - Bug in :meth:`DataFrame.to_stata` which prevented exporting DataFrames to buffers and most file-like objects (:issue:`21041`) -- +- Bug in IO JSON methods reading empty JSON schema back to DataFrame caused an error (:issue:`21287`) Plotting ^^^^^^^^ diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index 49b39c17238ae..2c14c37b98663 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -12,6 +12,7 @@ from pandas.io.json.table_schema import ( as_json_table_type, build_table_schema, + parse_table_schema, convert_pandas_type_to_json_field, convert_json_field_to_pandas_type, set_default_names) @@ -86,6 +87,15 @@ def test_multiindex(self): assert result == expected +class TestParseSchema(object): + + def test_empty_json_data(self): + # GH21287 + df = pd.DataFrame([], columns=['a', 'b', 'c']) + json = df.to_json(None, orient='table') + df = parse_table_schema(json, True) + assert df.empty + class TestTableSchemaType(object): @pytest.mark.parametrize('int_type', [ From db3a7389d71611d0482457fe27bb84dfd830ac07 Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Mon, 4 Jun 2018 23:37:23 +0300 Subject: [PATCH 03/12] Empty line between test classes --- pandas/tests/io/json/test_json_table_schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index 2c14c37b98663..2627483b4b704 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -96,6 +96,7 @@ def test_empty_json_data(self): df = parse_table_schema(json, True) assert df.empty + class TestTableSchemaType(object): @pytest.mark.parametrize('int_type', [ From 58443017d87e24e2fcfb25fe82266f099c3a52a4 Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Tue, 5 Jun 2018 22:04:52 +0300 Subject: [PATCH 04/12] Changes based on review comments --- doc/source/whatsnew/v0.23.1.txt | 2 +- pandas/tests/io/json/test_json_table_schema.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v0.23.1.txt b/doc/source/whatsnew/v0.23.1.txt index cce0448cd334f..a9797792bcf9b 100644 --- a/doc/source/whatsnew/v0.23.1.txt +++ b/doc/source/whatsnew/v0.23.1.txt @@ -92,7 +92,7 @@ I/O - Bug in IO methods specifying ``compression='zip'`` which produced uncompressed zip archives (:issue:`17778`, :issue:`21144`) - Bug in :meth:`DataFrame.to_stata` which prevented exporting DataFrames to buffers and most file-like objects (:issue:`21041`) -- Bug in IO JSON methods reading empty JSON schema back to DataFrame caused an error (:issue:`21287`) +- Bug in IO JSON method :func:`read_json` reading empty JSON schema back to :class:DataFrame caused an error (:issue:`21287`) Plotting ^^^^^^^^ diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index 2627483b4b704..a73959a41d75f 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -87,16 +87,6 @@ def test_multiindex(self): assert result == expected -class TestParseSchema(object): - - def test_empty_json_data(self): - # GH21287 - df = pd.DataFrame([], columns=['a', 'b', 'c']) - json = df.to_json(None, orient='table') - df = parse_table_schema(json, True) - assert df.empty - - class TestTableSchemaType(object): @pytest.mark.parametrize('int_type', [ @@ -571,3 +561,11 @@ def test_multiindex(self, index_names): out = df.to_json(orient="table") result = pd.read_json(out, orient="table") tm.assert_frame_equal(df, result) + + def test_empty_frame_roundtrip(self): + # GH 21287 + df = pd.DataFrame([], columns=['a', 'b', 'c']) + expected = df.copy() + out = df.to_json(None, orient='table') + result = pd.read_json(out, orient='table') + tm.assert_frame_equal(expected, result) From fd8fa93e87d8db33c8437d0a7037958acccc4690 Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Tue, 5 Jun 2018 22:35:53 +0300 Subject: [PATCH 05/12] Fix whatsnew + PEP --- doc/source/whatsnew/v0.23.1.txt | 2 +- pandas/tests/io/json/test_json_table_schema.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.23.1.txt b/doc/source/whatsnew/v0.23.1.txt index a9797792bcf9b..8deec8bf9b50f 100644 --- a/doc/source/whatsnew/v0.23.1.txt +++ b/doc/source/whatsnew/v0.23.1.txt @@ -92,7 +92,7 @@ I/O - Bug in IO methods specifying ``compression='zip'`` which produced uncompressed zip archives (:issue:`17778`, :issue:`21144`) - Bug in :meth:`DataFrame.to_stata` which prevented exporting DataFrames to buffers and most file-like objects (:issue:`21041`) -- Bug in IO JSON method :func:`read_json` reading empty JSON schema back to :class:DataFrame caused an error (:issue:`21287`) +- Bug in IO JSON :func:`read_json`reading empty JSON schema with ``orient='table'`` back to :class:DataFrame caused an error (:issue:`21287`) Plotting ^^^^^^^^ diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index a73959a41d75f..c1d9723cc0fe0 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -561,11 +561,11 @@ def test_multiindex(self, index_names): out = df.to_json(orient="table") result = pd.read_json(out, orient="table") tm.assert_frame_equal(df, result) - + def test_empty_frame_roundtrip(self): # GH 21287 df = pd.DataFrame([], columns=['a', 'b', 'c']) expected = df.copy() out = df.to_json(None, orient='table') result = pd.read_json(out, orient='table') - tm.assert_frame_equal(expected, result) + pd.testing.assert_frame_equal(expected, result) From 2f347c0809dd6b0a25873c32968d29012b67dece Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Wed, 6 Jun 2018 00:05:22 +0300 Subject: [PATCH 06/12] Prevent empty data from being coerced to float64 --- pandas/io/json/json.py | 3 ++- pandas/tests/io/json/test_json_table_schema.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/io/json/json.py b/pandas/io/json/json.py index 1627b2f4d3ec3..b6aebe5be6868 100644 --- a/pandas/io/json/json.py +++ b/pandas/io/json/json.py @@ -658,6 +658,7 @@ def _try_convert_types(self): def _try_convert_data(self, name, data, use_dtypes=True, convert_dates=True): + print(data) """ try to parse a ndarray like into a column by inferring dtype """ # don't try to coerce, unless a force conversion @@ -686,7 +687,7 @@ def _try_convert_data(self, name, data, use_dtypes=True, result = False - if data.dtype == 'object': + if len(data) and data.dtype == 'object': # try float try: diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index c1d9723cc0fe0..6ce932f6ab9b1 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -1,5 +1,6 @@ """Tests for Table Schema integration.""" import json +import io from collections import OrderedDict import numpy as np @@ -566,6 +567,6 @@ def test_empty_frame_roundtrip(self): # GH 21287 df = pd.DataFrame([], columns=['a', 'b', 'c']) expected = df.copy() - out = df.to_json(None, orient='table') + out = df.to_json(orient='table') result = pd.read_json(out, orient='table') - pd.testing.assert_frame_equal(expected, result) + tm.assert_frame_equal(expected, result) From 28d6e05f55f290d5dd1d0e06568a4cdb3aef090b Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Wed, 6 Jun 2018 00:07:36 +0300 Subject: [PATCH 07/12] Remove debugging messages --- pandas/io/json/json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/json/json.py b/pandas/io/json/json.py index b6aebe5be6868..2631222e4c63d 100644 --- a/pandas/io/json/json.py +++ b/pandas/io/json/json.py @@ -658,7 +658,6 @@ def _try_convert_types(self): def _try_convert_data(self, name, data, use_dtypes=True, convert_dates=True): - print(data) """ try to parse a ndarray like into a column by inferring dtype """ # don't try to coerce, unless a force conversion From 743c08f6c3fc1c21c94b756aea6cdd73a139915d Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Wed, 6 Jun 2018 00:10:40 +0300 Subject: [PATCH 08/12] Remove obsolete imports from tests --- pandas/tests/io/json/test_json_table_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index 6ce932f6ab9b1..a91f56d210765 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -1,6 +1,5 @@ """Tests for Table Schema integration.""" import json -import io from collections import OrderedDict import numpy as np @@ -13,7 +12,6 @@ from pandas.io.json.table_schema import ( as_json_table_type, build_table_schema, - parse_table_schema, convert_pandas_type_to_json_field, convert_json_field_to_pandas_type, set_default_names) From 2461b90a14b7e9276e06be09aa96a9cbf8f39686 Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Wed, 6 Jun 2018 21:36:23 +0300 Subject: [PATCH 09/12] Loosen test type checks, remove length check from JSON parser --- pandas/io/json/json.py | 2 +- pandas/tests/io/json/test_json_table_schema.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/io/json/json.py b/pandas/io/json/json.py index 2631222e4c63d..1627b2f4d3ec3 100644 --- a/pandas/io/json/json.py +++ b/pandas/io/json/json.py @@ -686,7 +686,7 @@ def _try_convert_data(self, name, data, use_dtypes=True, result = False - if len(data) and data.dtype == 'object': + if data.dtype == 'object': # try float try: diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index a91f56d210765..816023c349f0c 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -567,4 +567,6 @@ def test_empty_frame_roundtrip(self): expected = df.copy() out = df.to_json(orient='table') result = pd.read_json(out, orient='table') - tm.assert_frame_equal(expected, result) + # TODO: When DF coercion issue (GH ) is resolved, tighten type checks + tm.assert_frame_equal(expected, result, + check_dtype=False, check_index_type=False) From 03a2b8a8950efe6455274c94f8534c3d9786d875 Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Wed, 6 Jun 2018 22:10:29 +0300 Subject: [PATCH 10/12] Add GH issue number to TODO comment --- pandas/tests/io/json/test_json_table_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index 816023c349f0c..a1526a0c948ef 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -567,6 +567,6 @@ def test_empty_frame_roundtrip(self): expected = df.copy() out = df.to_json(orient='table') result = pd.read_json(out, orient='table') - # TODO: When DF coercion issue (GH ) is resolved, tighten type checks + # TODO: After DF coercion issue (GH 21345) is resolved, tighten type checks tm.assert_frame_equal(expected, result, check_dtype=False, check_index_type=False) From fc15ba025ec40afbbc2547a567ec4de2e13a32dc Mon Sep 17 00:00:00 2001 From: Pyry Kovanen <> Date: Thu, 7 Jun 2018 21:21:48 +0300 Subject: [PATCH 11/12] Parametrize JSON roundtrip test with xfail mark --- pandas/tests/io/json/test_json_table_schema.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index a1526a0c948ef..b6483d0e978ba 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -561,12 +561,15 @@ def test_multiindex(self, index_names): result = pd.read_json(out, orient="table") tm.assert_frame_equal(df, result) - def test_empty_frame_roundtrip(self): + @pytest.mark.parametrize("strict_check", [ + pytest.param(True, marks=pytest.mark.xfail), False]) + def test_empty_frame_roundtrip(self, strict_check): # GH 21287 df = pd.DataFrame([], columns=['a', 'b', 'c']) expected = df.copy() out = df.to_json(orient='table') result = pd.read_json(out, orient='table') - # TODO: After DF coercion issue (GH 21345) is resolved, tighten type checks + # TODO: When DF coercion issue (#21345) is resolved tighten type checks tm.assert_frame_equal(expected, result, - check_dtype=False, check_index_type=False) + check_dtype=strict_check, + check_index_type=strict_check) From 8d5f1270f130317112bc0cdb652a6d21b0fafa94 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Fri, 8 Jun 2018 19:38:43 -0400 Subject: [PATCH 12/12] fix whatsnew --- doc/source/whatsnew/v0.23.1.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.1.txt b/doc/source/whatsnew/v0.23.1.txt index 11a70c697a3dc..6118732768cb0 100644 --- a/doc/source/whatsnew/v0.23.1.txt +++ b/doc/source/whatsnew/v0.23.1.txt @@ -131,4 +131,4 @@ Bug Fixes **Other** -- Tab completion on :class:`Index` in IPython no longer outputs deprecation warnings (:issue:`21125`) \ No newline at end of file +- Tab completion on :class:`Index` in IPython no longer outputs deprecation warnings (:issue:`21125`)