From 715118ed92fdf5addcd7d12ba915776c87b49dca Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Thu, 6 Feb 2020 08:52:53 -0500 Subject: [PATCH 1/4] BUG: Fixed encoding of pd.NA with to_json GH31615 --- doc/source/whatsnew/v1.0.2.rst | 5 +++-- pandas/_libs/src/ujson/python/objToJSON.c | 12 ++++++++++++ pandas/tests/io/json/test_pandas.py | 8 ++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.0.2.rst b/doc/source/whatsnew/v1.0.2.rst index 07a837829c384..182ad7231983e 100644 --- a/doc/source/whatsnew/v1.0.2.rst +++ b/doc/source/whatsnew/v1.0.2.rst @@ -25,8 +25,9 @@ Fixed regressions Bug fixes ~~~~~~~~~ -- -- +**I/O** + +- Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`) .. --------------------------------------------------------------------------- diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 0fc146d25459b..8cfc20ffd2c1c 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -53,6 +53,7 @@ static PyTypeObject *cls_dataframe; static PyTypeObject *cls_series; static PyTypeObject *cls_index; static PyTypeObject *cls_nat; +static PyTypeObject *cls_na; PyObject *cls_timedelta; npy_int64 get_nat(void) { return NPY_MIN_INT64; } @@ -149,6 +150,7 @@ int PdBlock_iterNext(JSOBJ, JSONTypeContext *); void *initObjToJSON(void) { PyObject *mod_pandas; PyObject *mod_nattype; + PyObject *mod_natype; PyObject *mod_decimal = PyImport_ImportModule("decimal"); type_decimal = (PyTypeObject *)PyObject_GetAttrString(mod_decimal, "Decimal"); @@ -174,6 +176,12 @@ void *initObjToJSON(void) { Py_DECREF(mod_nattype); } + mod_natype = PyImport_ImportModule("pandas._libs.missing"); + if (mod_natype) { + cls_na = (PyTypeObject *)PyObject_GetAttrString(mod_natype, "NAType"); + Py_DECREF(mod_natype); + } + /* Initialise numpy API */ import_array(); // GH 31463 @@ -1789,6 +1797,10 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { "%R (0d array) is not JSON serializable at the moment", obj); goto INVALID; + } else if (PyObject_TypeCheck(obj, cls_na)) { + PRINTMARK(); + tc->type = JT_NULL; + return; } ISITERABLE: diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 602022a21c4a6..f24a9537a6511 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1671,3 +1671,11 @@ def test_to_s3(self, s3_resource): assert target_file in ( obj.key for obj in s3_resource.Bucket("pandas-test").objects.all() ) + + @pytest.mark.parametrize( + "dataframe,expected", [(pd.DataFrame([[pd.NA]]), '{"0":{"0":null}}',)], + ) + def test_json_pandas_na(self, dataframe, expected): + # GH 31615 + result = dataframe.to_json() + assert result == expected From c472a3abca5b26c0525a6e6b5dec7f166e502ac6 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Thu, 6 Feb 2020 11:57:42 -0500 Subject: [PATCH 2/4] DOC/TST: Update tests and docs for np.NA fix --- doc/source/whatsnew/v1.0.2.rst | 3 ++- pandas/tests/io/json/test_pandas.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.0.2.rst b/doc/source/whatsnew/v1.0.2.rst index 182ad7231983e..09d655ed340b4 100644 --- a/doc/source/whatsnew/v1.0.2.rst +++ b/doc/source/whatsnew/v1.0.2.rst @@ -25,7 +25,8 @@ Fixed regressions Bug fixes ~~~~~~~~~ -**I/O** +I/O +^^^ - Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index f24a9537a6511..9650756104bbf 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1673,7 +1673,11 @@ def test_to_s3(self, s3_resource): ) @pytest.mark.parametrize( - "dataframe,expected", [(pd.DataFrame([[pd.NA]]), '{"0":{"0":null}}',)], + "dataframe,expected", + [ + (pd.DataFrame([[pd.NA]]), '{"0":{"0":null}}',), + (pd.DataFrame({pd.NA: ["a"]}), '{"":{"0":"a"}}',), + ], ) def test_json_pandas_na(self, dataframe, expected): # GH 31615 From cda6f07b7628983c37ee4fcb77a350615f45a701 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 7 Feb 2020 08:20:00 -0500 Subject: [PATCH 3/4] DOC/TST: Fixes for PR --- doc/source/whatsnew/v1.0.2.rst | 3 +-- pandas/tests/io/json/test_pandas.py | 32 ++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/doc/source/whatsnew/v1.0.2.rst b/doc/source/whatsnew/v1.0.2.rst index 09d655ed340b4..182ad7231983e 100644 --- a/doc/source/whatsnew/v1.0.2.rst +++ b/doc/source/whatsnew/v1.0.2.rst @@ -25,8 +25,7 @@ Fixed regressions Bug fixes ~~~~~~~~~ -I/O -^^^ +**I/O** - Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 9650756104bbf..be7d38c2c232f 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1672,14 +1672,26 @@ def test_to_s3(self, s3_resource): obj.key for obj in s3_resource.Bucket("pandas-test").objects.all() ) - @pytest.mark.parametrize( - "dataframe,expected", - [ - (pd.DataFrame([[pd.NA]]), '{"0":{"0":null}}',), - (pd.DataFrame({pd.NA: ["a"]}), '{"":{"0":"a"}}',), - ], - ) - def test_json_pandas_na(self, dataframe, expected): + def test_json_pandas_na(self): # GH 31615 - result = dataframe.to_json() - assert result == expected + result = pd.DataFrame([[pd.NA]]).to_json() + assert result == '{"0":{"0":null}}' + + def test_json_pandas_na_label(self): + # GH 31615 + result = pd.DataFrame({pd.NA: ["a"]}).to_json() + assert result == '{"":{"0":"a"}}' + + def test_json_pandas_nulls(self, nulls_fixture): + # GH 31615 + result = pd.DataFrame([[nulls_fixture]]).to_json() + assert result == '{"0":{"0":null}}' + + def test_json_pandas_nulls_labels(self, nulls_fixture): + # GH 31615 + result = pd.DataFrame({nulls_fixture: ["a"]}).to_json() + if nulls_fixture is pd.NaT: + null_string = "null" + else: + null_string = str(nulls_fixture) + assert result == '{"' + null_string + '":{"0":"a"}}' From 2f3853e621d2168423c616bd34ab67d9465ef21b Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 7 Feb 2020 17:03:39 -0500 Subject: [PATCH 4/4] TST: remove label tests --- pandas/tests/io/json/test_pandas.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index be7d38c2c232f..f2d35bfb3b5ae 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1677,21 +1677,7 @@ def test_json_pandas_na(self): result = pd.DataFrame([[pd.NA]]).to_json() assert result == '{"0":{"0":null}}' - def test_json_pandas_na_label(self): - # GH 31615 - result = pd.DataFrame({pd.NA: ["a"]}).to_json() - assert result == '{"":{"0":"a"}}' - def test_json_pandas_nulls(self, nulls_fixture): # GH 31615 result = pd.DataFrame([[nulls_fixture]]).to_json() assert result == '{"0":{"0":null}}' - - def test_json_pandas_nulls_labels(self, nulls_fixture): - # GH 31615 - result = pd.DataFrame({nulls_fixture: ["a"]}).to_json() - if nulls_fixture is pd.NaT: - null_string = "null" - else: - null_string = str(nulls_fixture) - assert result == '{"' + null_string + '":{"0":"a"}}'