From bb9c174453d1bc0198bc21514fa36b0b8b07f93c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 May 2019 19:44:29 -0700 Subject: [PATCH 01/34] Added test --- pandas/tests/io/json/test_pandas.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 8b140263b12bc..ef0c483842564 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1317,3 +1317,23 @@ def test_read_timezone_information(self): index=DatetimeIndex(['2019-01-01 11:00:00'], tz='UTC')) assert_series_equal(result, expected) + + def test_to_json_indent(self): + # GH 12004 + df = pd.DataFrame([ + ['foo', 'bar'], ['baz', 'qux'] + ], columns=['a', 'b']) + + result = df.to_json(indent=4) + expected = """{ + "a": { + "0": "foo", + "1": "baz" + }, + "b": { + "0": "bar", + "1": "qux" + } +}""" + + assert result == expected From 4c84106a87b0139e864145443ce72eb38e7e8cca Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 May 2019 19:57:49 -0700 Subject: [PATCH 02/34] Vendored ujson changes --- pandas/_libs/src/ujson/lib/ultrajson.h | 4 ++++ pandas/_libs/src/ujson/lib/ultrajsonenc.c | 24 +++++++++++++++++++++++ pandas/_libs/src/ujson/python/objToJSON.c | 8 +++++--- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/src/ujson/lib/ultrajson.h b/pandas/_libs/src/ujson/lib/ultrajson.h index 0470fef450dde..51c482da67943 100644 --- a/pandas/_libs/src/ujson/lib/ultrajson.h +++ b/pandas/_libs/src/ujson/lib/ultrajson.h @@ -244,6 +244,10 @@ typedef struct __JSONObjectEncoder { If true, '<', '>', and '&' characters will be encoded as \u003c, \u003e, and \u0026, respectively. If false, no special encoding will be used. */ int encodeHTMLChars; + /* + Configuration for spaces of indent */ + int indent; + /* Set to an error message if error occurred */ const char *errorMsg; diff --git a/pandas/_libs/src/ujson/lib/ultrajsonenc.c b/pandas/_libs/src/ujson/lib/ultrajsonenc.c index 2d6c823a45515..23a3905fd1509 100644 --- a/pandas/_libs/src/ujson/lib/ultrajsonenc.c +++ b/pandas/_libs/src/ujson/lib/ultrajsonenc.c @@ -722,6 +722,20 @@ FASTCALL_ATTR INLINE_PREFIX void FASTCALL_MSVC strreverse(char *begin, while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux; } +void Buffer_AppendIndentNewlineUnchecked(JSONObjectEncoder *enc) +{ + if (enc->indent > 0) Buffer_AppendCharUnchecked(enc, '\n'); +} + +void Buffer_AppendIndentUnchecked(JSONObjectEncoder *enc, JSINT32 value) +{ + int i; + if (enc->indent > 0) + while (value-- > 0) + for (i = 0; i < enc->indent; i++) + Buffer_AppendCharUnchecked(enc, ' '); +} + void Buffer_AppendIntUnchecked(JSONObjectEncoder *enc, JSINT32 value) { char *wstr; JSUINT32 uvalue = (value < 0) ? -value : value; @@ -954,6 +968,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, enc->iterBegin(obj, &tc); Buffer_AppendCharUnchecked(enc, '['); + Buffer_AppendIndentNewlineUnchecked (enc); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -961,16 +976,20 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, #ifndef JSON_NO_EXTRA_WHITESPACE Buffer_AppendCharUnchecked(buffer, ' '); #endif + Buffer_AppendIndentNewlineUnchecked (enc); } iterObj = enc->iterGetValue(obj, &tc); enc->level++; + Buffer_AppendIndentUnchecked (enc, enc->level); encode(iterObj, enc, NULL, 0); count++; } enc->iterEnd(obj, &tc); + Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentUnchecked (enc, enc->level); Buffer_Reserve(enc, 2); Buffer_AppendCharUnchecked(enc, ']'); break; @@ -981,6 +1000,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, enc->iterBegin(obj, &tc); Buffer_AppendCharUnchecked(enc, '{'); + Buffer_AppendIndentNewlineUnchecked (enc); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -988,17 +1008,21 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, #ifndef JSON_NO_EXTRA_WHITESPACE Buffer_AppendCharUnchecked(enc, ' '); #endif + Buffer_AppendIndentNewlineUnchecked (enc); } iterObj = enc->iterGetValue(obj, &tc); objName = enc->iterGetName(obj, &tc, &szlen); enc->level++; + Buffer_AppendIndentUnchecked (enc, enc->level); encode(iterObj, enc, objName, szlen); count++; } enc->iterEnd(obj, &tc); + Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentUnchecked (enc, enc->level); Buffer_Reserve(enc, 2); Buffer_AppendCharUnchecked(enc, '}'); break; diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 52788f85ff71e..2d475bcb732ed 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -2268,7 +2268,7 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = { "obj", "ensure_ascii", "double_precision", "encode_html_chars", "orient", "date_unit", "iso_dates", "default_handler", - NULL}; + "indent", NULL}; char buffer[65536]; char *ret; @@ -2281,6 +2281,7 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { char *sdateFormat = NULL; PyObject *oisoDates = 0; PyObject *odefHandler = 0; + int indent = 0; PyObjectEncoder pyEncoder = {{ Object_beginTypeContext, @@ -2302,6 +2303,7 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { idoublePrecision, 1, // forceAscii 0, // encodeHTMLChars + 0, // indent }}; JSONObjectEncoder *encoder = (JSONObjectEncoder *)&pyEncoder; @@ -2326,10 +2328,10 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { PRINTMARK(); - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OiOssOO", kwlist, &oinput, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OiOssOOi", kwlist, &oinput, &oensureAscii, &idoublePrecision, &oencodeHTMLChars, &sOrient, &sdateFormat, - &oisoDates, &odefHandler)) { + &oisoDates, &odefHandler, &indent)) { return NULL; } From b99f42b126f203e3e09b8a894d69028f9bdbc807 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 May 2019 20:23:40 -0700 Subject: [PATCH 03/34] Pass indent to ext module --- pandas/_libs/src/ujson/python/objToJSON.c | 2 ++ pandas/core/generic.py | 9 +++++-- pandas/io/json/json.py | 33 +++++++++++++---------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 2d475bcb732ed..91d5dcbddd7d7 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -2397,6 +2397,8 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { pyEncoder.defaultHandler = odefHandler; } + encoder->indent = indent; + pyEncoder.originalOutputFormat = pyEncoder.outputFormat; PRINTMARK(); ret = JSON_EncodeObject(oinput, encoder, buffer, sizeof(buffer)); diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 76910f425836e..00c2bcbbac7e2 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2178,7 +2178,7 @@ def to_excel(self, excel_writer, sheet_name="Sheet1", na_rep="", def to_json(self, path_or_buf=None, orient=None, date_format=None, double_precision=10, force_ascii=True, date_unit='ms', default_handler=None, lines=False, compression='infer', - index=True): + index=True, indent=0): """ Convert the object to a JSON string. @@ -2260,6 +2260,11 @@ def to_json(self, path_or_buf=None, orient=None, date_format=None, .. versionadded:: 0.23.0 + indent : integer, default 0 + Length of whitespace used to indent each record. + + .. versionadded:: 0.25.0 + Returns ------- None or str @@ -2325,7 +2330,7 @@ def to_json(self, path_or_buf=None, orient=None, date_format=None, force_ascii=force_ascii, date_unit=date_unit, default_handler=default_handler, lines=lines, compression=compression, - index=index) + index=index, indent=indent) def to_hdf(self, path_or_buf, key, **kwargs): """ diff --git a/pandas/io/json/json.py b/pandas/io/json/json.py index ee9d9e000d7e3..b3cca412f9c5f 100644 --- a/pandas/io/json/json.py +++ b/pandas/io/json/json.py @@ -33,7 +33,7 @@ def to_json(path_or_buf, obj, orient=None, date_format='epoch', double_precision=10, force_ascii=True, date_unit='ms', default_handler=None, lines=False, compression='infer', - index=True): + index=True, indent=0): if not index and orient not in ['split', 'table']: raise ValueError("'index=False' is only valid when 'orient' is " @@ -59,7 +59,7 @@ def to_json(path_or_buf, obj, orient=None, date_format='epoch', obj, orient=orient, date_format=date_format, double_precision=double_precision, ensure_ascii=force_ascii, date_unit=date_unit, default_handler=default_handler, - index=index).write() + index=index, indent=indent).write() if lines: s = _convert_to_line_delimits(s) @@ -78,7 +78,8 @@ def to_json(path_or_buf, obj, orient=None, date_format='epoch', class Writer: def __init__(self, obj, orient, date_format, double_precision, - ensure_ascii, date_unit, index, default_handler=None): + ensure_ascii, date_unit, index, default_handler=None, + indent=0): self.obj = obj if orient is None: @@ -91,6 +92,7 @@ def __init__(self, obj, orient, date_format, double_precision, self.date_unit = date_unit self.default_handler = default_handler self.index = index + self.indent = indent self.is_copy = None self._format_axes() @@ -101,10 +103,11 @@ def _format_axes(self): def write(self): return self._write(self.obj, self.orient, self.double_precision, self.ensure_ascii, self.date_unit, - self.date_format == 'iso', self.default_handler) + self.date_format == 'iso', self.default_handler, + self.indent) def _write(self, obj, orient, double_precision, ensure_ascii, - date_unit, iso_dates, default_handler): + date_unit, iso_dates, default_handler, indent): return dumps( obj, orient=orient, @@ -112,7 +115,8 @@ def _write(self, obj, orient, double_precision, ensure_ascii, ensure_ascii=ensure_ascii, date_unit=date_unit, iso_dates=iso_dates, - default_handler=default_handler + default_handler=default_handler, + indent=indent ) @@ -125,11 +129,11 @@ def _format_axes(self): "'{orient}'".format(orient=self.orient)) def _write(self, obj, orient, double_precision, ensure_ascii, - date_unit, iso_dates, default_handler): + date_unit, iso_dates, default_handler, indent): if not self.index and orient == 'split': obj = {"name": obj.name, "data": obj.values} return super()._write(obj, orient, double_precision, ensure_ascii, - date_unit, iso_dates, default_handler) + date_unit, iso_dates, default_handler, indent) class FrameWriter(Writer): @@ -149,19 +153,20 @@ def _format_axes(self): "'{orient}'.".format(orient=self.orient)) def _write(self, obj, orient, double_precision, ensure_ascii, - date_unit, iso_dates, default_handler): + date_unit, iso_dates, default_handler, indent): if not self.index and orient == 'split': obj = obj.to_dict(orient='split') del obj["index"] return super()._write(obj, orient, double_precision, ensure_ascii, - date_unit, iso_dates, default_handler) + date_unit, iso_dates, default_handler, indent) class JSONTableWriter(FrameWriter): _default_orient = 'records' def __init__(self, obj, orient, date_format, double_precision, - ensure_ascii, date_unit, index, default_handler=None): + ensure_ascii, date_unit, index, default_handler=None, + indent=0): """ Adds a `schema` attribute with the Table Schema, resets the index (can't do in caller, because the schema inference needs @@ -170,7 +175,7 @@ def __init__(self, obj, orient, date_format, double_precision, """ super().__init__(obj, orient, date_format, double_precision, ensure_ascii, date_unit, index, - default_handler=default_handler) + default_handler=default_handler, indent=indent) if date_format != 'iso': msg = ("Trying to write with `orient='table'` and " @@ -211,9 +216,9 @@ def __init__(self, obj, orient, date_format, double_precision, self.index = index def _write(self, obj, orient, double_precision, ensure_ascii, - date_unit, iso_dates, default_handler): + date_unit, iso_dates, default_handler, indent): data = super()._write(obj, orient, double_precision, ensure_ascii, - date_unit, iso_dates, default_handler) + date_unit, iso_dates, default_handler, indent) serialized = '{{"schema": {schema}, "data": {data}}}'.format( schema=dumps(self.schema), data=data) return serialized From 367b494e255eeb43c47dafb8e84dd81c4cf01198 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 18 May 2019 21:29:30 -0700 Subject: [PATCH 04/34] hacked together working example --- pandas/_libs/src/ujson/lib/ultrajsonenc.c | 10 +++------- pandas/tests/io/json/test_pandas.py | 12 ++++++------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/pandas/_libs/src/ujson/lib/ultrajsonenc.c b/pandas/_libs/src/ujson/lib/ultrajsonenc.c index 23a3905fd1509..1201294f3aa21 100644 --- a/pandas/_libs/src/ujson/lib/ultrajsonenc.c +++ b/pandas/_libs/src/ujson/lib/ultrajsonenc.c @@ -968,7 +968,6 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, enc->iterBegin(obj, &tc); Buffer_AppendCharUnchecked(enc, '['); - Buffer_AppendIndentNewlineUnchecked (enc); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -976,7 +975,6 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, #ifndef JSON_NO_EXTRA_WHITESPACE Buffer_AppendCharUnchecked(buffer, ' '); #endif - Buffer_AppendIndentNewlineUnchecked (enc); } iterObj = enc->iterGetValue(obj, &tc); @@ -988,8 +986,6 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, } enc->iterEnd(obj, &tc); - Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level); Buffer_Reserve(enc, 2); Buffer_AppendCharUnchecked(enc, ']'); break; @@ -1001,6 +997,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, Buffer_AppendCharUnchecked(enc, '{'); Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentUnchecked (enc, enc->level + 2); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -1009,21 +1006,20 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, Buffer_AppendCharUnchecked(enc, ' '); #endif Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentUnchecked (enc, enc->level + 2); } iterObj = enc->iterGetValue(obj, &tc); objName = enc->iterGetName(obj, &tc, &szlen); enc->level++; - Buffer_AppendIndentUnchecked (enc, enc->level); encode(iterObj, enc, objName, szlen); count++; } enc->iterEnd(obj, &tc); Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level); - Buffer_Reserve(enc, 2); + Buffer_AppendIndentUnchecked (enc, enc->level + 1); Buffer_AppendCharUnchecked(enc, '}'); break; } diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index ef0c483842564..4fa31bd36a4e4 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1326,13 +1326,13 @@ def test_to_json_indent(self): result = df.to_json(indent=4) expected = """{ - "a": { - "0": "foo", - "1": "baz" + "a":{ + "0":"foo", + "1":"baz" }, - "b": { - "0": "bar", - "1": "qux" + "b":{ + "0":"bar", + "1":"qux" } }""" From 43ab17bf3ab793b9c29134bb1eef089aa2fb799f Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 24 Aug 2019 12:27:20 +0100 Subject: [PATCH 05/34] Expanded test and clean implementation --- pandas/_libs/src/ujson/lib/ultrajsonenc.c | 6 +++--- pandas/tests/io/json/test_pandas.py | 26 ++++++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pandas/_libs/src/ujson/lib/ultrajsonenc.c b/pandas/_libs/src/ujson/lib/ultrajsonenc.c index abb900ce13127..a4b891a4314bc 100644 --- a/pandas/_libs/src/ujson/lib/ultrajsonenc.c +++ b/pandas/_libs/src/ujson/lib/ultrajsonenc.c @@ -1003,7 +1003,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, Buffer_AppendCharUnchecked(enc, '{'); Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level + 2); + Buffer_AppendIndentUnchecked (enc, enc->level); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -1012,7 +1012,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, Buffer_AppendCharUnchecked(enc, ' '); #endif Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level + 2); + Buffer_AppendIndentUnchecked (enc, enc->level); } iterObj = enc->iterGetValue(obj, &tc); @@ -1025,7 +1025,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, enc->iterEnd(obj, &tc); Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level + 1); + Buffer_AppendIndentUnchecked (enc, enc->level); Buffer_AppendCharUnchecked(enc, '}'); break; } diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 3e7526335d826..fd7d5abf735a5 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1648,22 +1648,24 @@ def test_tuple_labels(self, orient, expected): result = df.to_json(orient=orient) assert result == expected - def test_to_json_indent(self): + @pytest.mark.parametrize("indent", [1, 2, 4]) + def test_to_json_indent(self, indent): # GH 12004 df = pd.DataFrame([ ['foo', 'bar'], ['baz', 'qux'] ], columns=['a', 'b']) - result = df.to_json(indent=4) - expected = """{ - "a":{ - "0":"foo", - "1":"baz" - }, - "b":{ - "0":"bar", - "1":"qux" - } -}""" + result = df.to_json(indent=indent) + spaces = " " * indent + expected = """{{ +"a":{{ +{spaces}"0":"foo", +{spaces}"1":"baz" +{spaces}}}, +"b":{{ +{spaces}"0":"bar", +{spaces}"1":"qux" +{spaces}}} +}}""".format(spaces=spaces) assert result == expected From 592be66833d824b7aa8ad5b94e1b7ac3c398faca Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 24 Aug 2019 12:28:28 +0100 Subject: [PATCH 06/34] Cleaned up docs and whitespace --- pandas/_libs/src/ujson/lib/ultrajson.h | 2 +- pandas/core/generic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/src/ujson/lib/ultrajson.h b/pandas/_libs/src/ujson/lib/ultrajson.h index 973886456d2c6..05c3ae4096ad5 100644 --- a/pandas/_libs/src/ujson/lib/ultrajson.h +++ b/pandas/_libs/src/ujson/lib/ultrajson.h @@ -246,7 +246,7 @@ typedef struct __JSONObjectEncoder { /* Configuration for spaces of indent */ - int indent; + int indent; /* Set to an error message if error occurred */ diff --git a/pandas/core/generic.py b/pandas/core/generic.py index f6c6e29899efe..cfbe1df5efea0 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2333,7 +2333,7 @@ def to_json( indent : integer, default 0 Length of whitespace used to indent each record. - .. versionadded:: 0.25.0 + .. versionadded:: 1.0.0 Returns ------- From 5c0e8a30ecaaf347c228efa785c826c36373f7ef Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 25 Aug 2019 16:05:06 -0700 Subject: [PATCH 07/34] Annotated to_json in pandas.core.generic --- pandas/core/generic.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index cfbe1df5efea0..e3176c2c22ecb 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -7,7 +7,7 @@ import pickle import re from textwrap import dedent -from typing import Callable, Dict, FrozenSet, List, Optional, Set +from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Union import warnings import weakref @@ -50,7 +50,7 @@ from pandas.core.dtypes.missing import isna, notna import pandas as pd -from pandas._typing import Dtype +from pandas._typing import Dtype, FilePathOrBuffer from pandas.core import missing, nanops import pandas.core.algorithms as algos from pandas.core.base import PandasObject, SelectionMixin @@ -122,6 +122,9 @@ def _single_replace(self, to_replace, method, inplace, limit): return result +bool_t = bool # Need alias because NDFrame has def bool: + + class NDFrame(PandasObject, SelectionMixin): """ N-dimensional analogue of DataFrame. Store multi-dimensional in a @@ -2239,18 +2242,18 @@ def to_excel( def to_json( self, - path_or_buf=None, - orient=None, - date_format=None, - double_precision=10, - force_ascii=True, - date_unit="ms", - default_handler=None, - lines=False, - compression="infer", - index=True, - indent=0, - ): + path_or_buf: Optional[FilePathOrBuffer] = None, + orient: Optional[str] = None, + date_format: Optional[str] = None, + double_precision: int = 10, + force_ascii: bool_t = True, + date_unit: str = "ms", + default_handler: Optional[Callable[[Any], Union[int, str, List, Dict]]] = None, + lines: bool_t = False, + compression: Optional[str] = "infer", + index: bool_t = True, + indent: int = 0, + ) -> Optional[str]: """ Convert the object to a JSON string. From d11e47ffacda654825d689aabcd80323bb422dbc Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 25 Aug 2019 16:29:04 -0700 Subject: [PATCH 08/34] Simple annotations for to_json --- pandas/io/json/_json.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 2a7b8bb6cf4d2..6adb08b84a8a6 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -1,6 +1,7 @@ from io import StringIO from itertools import islice import os +from typing import Any, Callable, Dict, List, Optional, Type, Union import numpy as np @@ -36,16 +37,16 @@ def to_json( path_or_buf, obj, - orient=None, - date_format="epoch", - double_precision=10, - force_ascii=True, - date_unit="ms", - default_handler=None, - lines=False, - compression="infer", - index=True, - indent=0, + orient: Optional[str] = None, + date_format: str = "epoch", + double_precision: int = 10, + force_ascii: bool =True, + date_unit: str = "ms", + default_handler: Optional[Callable[[Any], Union[int, str, List, Dict]]]=None, + lines: bool =False, + compression: Optional[str] = "infer", + index: bool = True, + indent: int = 0, ): if not index and orient not in ["split", "table"]: @@ -60,7 +61,7 @@ def to_json( if orient == "table" and isinstance(obj, Series): obj = obj.to_frame(name=obj.name or "values") if orient == "table" and isinstance(obj, DataFrame): - writer = JSONTableWriter + writer = JSONTableWriter # type: Type["Writer"] elif isinstance(obj, Series): writer = SeriesWriter elif isinstance(obj, DataFrame): From 21672ed0436fa839a606124fa255a6314329d4f3 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 25 Aug 2019 16:35:38 -0700 Subject: [PATCH 09/34] More hints in _json --- pandas/io/json/_json.py | 80 +++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 6adb08b84a8a6..f80450ecbc962 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -32,6 +32,8 @@ TABLE_SCHEMA_VERSION = "0.20.0" +Serializable = Union[int, str, List, Dict] + # interface to/from def to_json( @@ -42,7 +44,7 @@ def to_json( double_precision: int = 10, force_ascii: bool =True, date_unit: str = "ms", - default_handler: Optional[Callable[[Any], Union[int, str, List, Dict]]]=None, + default_handler: Optional[Callable[[Any], Serializable]]=None, lines: bool =False, compression: Optional[str] = "infer", index: bool = True, @@ -101,19 +103,19 @@ class Writer: def __init__( self, obj, - orient, - date_format, - double_precision, - ensure_ascii, - date_unit, - index, - default_handler=None, - indent=0, + orient: Optional[str], + date_format: str, + double_precision: int, + ensure_ascii: bool, + date_unit: str, + index: bool, + default_handler: Optional[Callable[[Any], Serializable]]=None, + indent: int = 0, ): self.obj = obj if orient is None: - orient = self._default_orient + orient = self._default_orient # type: ignore self.orient = orient self.date_format = date_format @@ -145,13 +147,13 @@ def write(self): def _write( self, obj, - orient, - double_precision, - ensure_ascii, - date_unit, - iso_dates, - default_handler, - indent, + orient: Optional[str], + double_precision: int, + ensure_ascii: bool, + date_unit: str, + iso_dates: bool, + default_handler: Optional[Callable[[Any], Serializable]], + indent: int, ): return dumps( obj, @@ -178,13 +180,13 @@ def _format_axes(self): def _write( self, obj, - orient, - double_precision, - ensure_ascii, - date_unit, - iso_dates, - default_handler, - indent, + orient: Optional[str], + double_precision: int, + ensure_ascii: bool, + date_unit: str, + iso_dates: bool, + default_handler: Optional[Callable[[Any], Serializable]], + indent: int, ): if not self.index and orient == "split": obj = {"name": obj.name, "data": obj.values} @@ -225,13 +227,13 @@ def _format_axes(self): def _write( self, obj, - orient, - double_precision, - ensure_ascii, - date_unit, - iso_dates, - default_handler, - indent, + orient: Optional[str], + double_precision: int, + ensure_ascii: bool, + date_unit: str, + iso_dates: bool, + default_handler: Optional[Callable[[Any], Serializable]], + indent: int, ): if not self.index and orient == "split": obj = obj.to_dict(orient="split") @@ -254,14 +256,14 @@ class JSONTableWriter(FrameWriter): def __init__( self, obj, - orient, - date_format, - double_precision, - ensure_ascii, - date_unit, - index, - default_handler=None, - indent=0, + orient: Optional[str], + date_format: str, + double_precision: int, + ensure_ascii: bool, + date_unit: str, + index: bool, + default_handler: Optional[Callable[[Any], Serializable]]=None, + indent: int=0, ): """ Adds a `schema` attribute with the Table Schema, resets From 79c1cbc4c4c24d6088d136e910a1887684bccda7 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 25 Aug 2019 16:36:29 -0700 Subject: [PATCH 10/34] blackify --- pandas/io/json/_json.py | 15 +++++++-------- pandas/tests/io/json/test_pandas.py | 8 ++++---- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index f80450ecbc962..c8592626bd843 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -42,10 +42,10 @@ def to_json( orient: Optional[str] = None, date_format: str = "epoch", double_precision: int = 10, - force_ascii: bool =True, + force_ascii: bool = True, date_unit: str = "ms", - default_handler: Optional[Callable[[Any], Serializable]]=None, - lines: bool =False, + default_handler: Optional[Callable[[Any], Serializable]] = None, + lines: bool = False, compression: Optional[str] = "infer", index: bool = True, indent: int = 0, @@ -99,7 +99,6 @@ def to_json( class Writer: - def __init__( self, obj, @@ -109,7 +108,7 @@ def __init__( ensure_ascii: bool, date_unit: str, index: bool, - default_handler: Optional[Callable[[Any], Serializable]]=None, + default_handler: Optional[Callable[[Any], Serializable]] = None, indent: int = 0, ): self.obj = obj @@ -262,8 +261,8 @@ def __init__( ensure_ascii: bool, date_unit: str, index: bool, - default_handler: Optional[Callable[[Any], Serializable]]=None, - indent: int=0, + default_handler: Optional[Callable[[Any], Serializable]] = None, + indent: int = 0, ): """ Adds a `schema` attribute with the Table Schema, resets @@ -343,7 +342,7 @@ def _write( date_unit, iso_dates, default_handler, - indent + indent, ) serialized = '{{"schema": {schema}, "data": {data}}}'.format( schema=dumps(self.schema), data=data diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index fd7d5abf735a5..b8e18823d89d2 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1651,9 +1651,7 @@ def test_tuple_labels(self, orient, expected): @pytest.mark.parametrize("indent", [1, 2, 4]) def test_to_json_indent(self, indent): # GH 12004 - df = pd.DataFrame([ - ['foo', 'bar'], ['baz', 'qux'] - ], columns=['a', 'b']) + df = pd.DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["a", "b"]) result = df.to_json(indent=indent) spaces = " " * indent @@ -1666,6 +1664,8 @@ def test_to_json_indent(self, indent): {spaces}"0":"bar", {spaces}"1":"qux" {spaces}}} -}}""".format(spaces=spaces) +}}""".format( + spaces=spaces + ) assert result == expected From 2da6fbfaf9812c8ed527beb8b076553fd5d6d6e8 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 27 Aug 2019 10:12:17 -0700 Subject: [PATCH 11/34] Reused Scalar variable --- pandas/_typing.py | 2 +- pandas/core/generic.py | 4 ++-- pandas/io/json/_json.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index 837a7a89e0b83..936f1ccde2049 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -25,6 +25,6 @@ FilePathOrBuffer = Union[str, Path, IO[AnyStr]] FrameOrSeries = TypeVar("FrameOrSeries", "Series", "DataFrame") -Scalar = Union[str, int, float] +Scalar = Union[str, int, float, bool] Axis = Union[str, int] Ordered = Optional[bool] diff --git a/pandas/core/generic.py b/pandas/core/generic.py index d280e0d5ec5cc..c3034889df1a3 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -61,7 +61,7 @@ from pandas.core.dtypes.missing import isna, notna import pandas as pd -from pandas._typing import Dtype, FilePathOrBuffer +from pandas._typing import Dtype, FilePathOrBuffer, Scalar from pandas.core import missing, nanops import pandas.core.algorithms as algos from pandas.core.base import PandasObject, SelectionMixin @@ -2259,7 +2259,7 @@ def to_json( double_precision: int = 10, force_ascii: bool_t = True, date_unit: str = "ms", - default_handler: Optional[Callable[[Any], Union[int, str, List, Dict]]] = None, + default_handler: Optional[Callable[[Any], Union[Scalar, List, Dict]]] = None, lines: bool_t = False, compression: Optional[str] = "infer", index: bool_t = True, diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index c8592626bd843..348a658c20658 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -7,6 +7,7 @@ import pandas._libs.json as json from pandas._libs.tslibs import iNaT +from pandas._typing import Scalar from pandas.errors import AbstractMethodError from pandas.core.dtypes.common import ensure_str, is_period_dtype @@ -32,7 +33,7 @@ TABLE_SCHEMA_VERSION = "0.20.0" -Serializable = Union[int, str, List, Dict] +Serializable = Union[Scalar, List, Dict] # interface to/from From a4f740afdb0bbf2c1124ca4685855b5a3672d04b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 27 Aug 2019 11:15:26 -0700 Subject: [PATCH 12/34] Fixed vendored changes --- pandas/_libs/src/ujson/lib/ultrajsonenc.c | 8 +++++--- pandas/tests/io/json/test_pandas.py | 12 ++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/src/ujson/lib/ultrajsonenc.c b/pandas/_libs/src/ujson/lib/ultrajsonenc.c index a4b891a4314bc..1bcb6a08651f4 100644 --- a/pandas/_libs/src/ujson/lib/ultrajsonenc.c +++ b/pandas/_libs/src/ujson/lib/ultrajsonenc.c @@ -974,6 +974,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, enc->iterBegin(obj, &tc); Buffer_AppendCharUnchecked(enc, '['); + Buffer_AppendIndentNewlineUnchecked (enc); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -981,6 +982,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, #ifndef JSON_NO_EXTRA_WHITESPACE Buffer_AppendCharUnchecked(buffer, ' '); #endif + Buffer_AppendIndentNewlineUnchecked (enc); } iterObj = enc->iterGetValue(obj, &tc); @@ -992,7 +994,8 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, } enc->iterEnd(obj, &tc); - Buffer_Reserve(enc, 2); + Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentUnchecked (enc, enc->level); Buffer_AppendCharUnchecked(enc, ']'); break; } @@ -1003,7 +1006,6 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, Buffer_AppendCharUnchecked(enc, '{'); Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -1012,13 +1014,13 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, Buffer_AppendCharUnchecked(enc, ' '); #endif Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level); } iterObj = enc->iterGetValue(obj, &tc); objName = enc->iterGetName(obj, &tc, &szlen); enc->level++; + Buffer_AppendIndentUnchecked (enc, enc->level); encode(iterObj, enc, objName, szlen); count++; } diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index b8e18823d89d2..aeb002224f849 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1656,13 +1656,13 @@ def test_to_json_indent(self, indent): result = df.to_json(indent=indent) spaces = " " * indent expected = """{{ -"a":{{ -{spaces}"0":"foo", -{spaces}"1":"baz" +{spaces}"a":{{ +{spaces}{spaces}"0":"foo", +{spaces}{spaces}"1":"baz" {spaces}}}, -"b":{{ -{spaces}"0":"bar", -{spaces}"1":"qux" +{spaces}"b":{{ +{spaces}{spaces}"0":"bar", +{spaces}{spaces}"1":"qux" {spaces}}} }}""".format( spaces=spaces From cd0c9e6a8c0710ae34bfeb4d45578990202d19ae Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 27 Aug 2019 11:19:36 -0700 Subject: [PATCH 13/34] Replaced tabs with spaces --- pandas/_libs/src/ujson/lib/ultrajsonenc.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/src/ujson/lib/ultrajsonenc.c b/pandas/_libs/src/ujson/lib/ultrajsonenc.c index 1bcb6a08651f4..9ba677c70eb74 100644 --- a/pandas/_libs/src/ujson/lib/ultrajsonenc.c +++ b/pandas/_libs/src/ujson/lib/ultrajsonenc.c @@ -974,7 +974,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, enc->iterBegin(obj, &tc); Buffer_AppendCharUnchecked(enc, '['); - Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentNewlineUnchecked (enc); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -982,20 +982,20 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, #ifndef JSON_NO_EXTRA_WHITESPACE Buffer_AppendCharUnchecked(buffer, ' '); #endif - Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentNewlineUnchecked (enc); } iterObj = enc->iterGetValue(obj, &tc); enc->level++; - Buffer_AppendIndentUnchecked (enc, enc->level); + Buffer_AppendIndentUnchecked (enc, enc->level); encode(iterObj, enc, NULL, 0); count++; } enc->iterEnd(obj, &tc); Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level); + Buffer_AppendIndentUnchecked (enc, enc->level); Buffer_AppendCharUnchecked(enc, ']'); break; } @@ -1005,7 +1005,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, enc->iterBegin(obj, &tc); Buffer_AppendCharUnchecked(enc, '{'); - Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentNewlineUnchecked (enc); while (enc->iterNext(obj, &tc)) { if (count > 0) { @@ -1013,21 +1013,21 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, #ifndef JSON_NO_EXTRA_WHITESPACE Buffer_AppendCharUnchecked(enc, ' '); #endif - Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentNewlineUnchecked (enc); } iterObj = enc->iterGetValue(obj, &tc); objName = enc->iterGetName(obj, &tc, &szlen); enc->level++; - Buffer_AppendIndentUnchecked (enc, enc->level); + Buffer_AppendIndentUnchecked (enc, enc->level); encode(iterObj, enc, objName, szlen); count++; } enc->iterEnd(obj, &tc); - Buffer_AppendIndentNewlineUnchecked (enc); - Buffer_AppendIndentUnchecked (enc, enc->level); + Buffer_AppendIndentNewlineUnchecked (enc); + Buffer_AppendIndentUnchecked (enc, enc->level); Buffer_AppendCharUnchecked(enc, '}'); break; } From c7403594f3db02b53fd4488c6be742c932136f49 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 27 Aug 2019 11:53:22 -0700 Subject: [PATCH 14/34] isort fixup --- pandas/io/json/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 348a658c20658..81352dd5db501 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -7,12 +7,12 @@ import pandas._libs.json as json from pandas._libs.tslibs import iNaT -from pandas._typing import Scalar from pandas.errors import AbstractMethodError from pandas.core.dtypes.common import ensure_str, is_period_dtype from pandas import DataFrame, MultiIndex, Series, isna, to_datetime +from pandas._typing import Scalar from pandas.core.reshape.concat import concat from pandas.io.common import ( From 2b5cb507162b8bb8f5ca1f653138a6578811476b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 27 Aug 2019 14:15:37 -0700 Subject: [PATCH 15/34] whatsnew --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 7fe358d3820f2..1a685990e37c6 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -21,7 +21,7 @@ including other versions of pandas. Enhancements ~~~~~~~~~~~~ -- +- :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`). - .. _whatsnew_1000.enhancements.other: From b8705857bafca6d84149cd2aecc61b277f92429e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 10:29:19 -0600 Subject: [PATCH 16/34] Added tests for all orients --- pandas/tests/io/json/test_pandas.py | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index aeb002224f849..a46df3121e85e 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1669,3 +1669,87 @@ def test_to_json_indent(self, indent): ) assert result == expected + + @pytest.mark.parametrize("orient,expected", [ + ("split", """{ + "columns":[ + "a", + "b" + ], + "index":[ + 0, + 1 + ], + "data":[ + [ + "foo", + "bar" + ], + [ + "baz", + "qux" + ] + ] +}"""), + ("records", """[ + { + "a":"foo", + "b":"bar" + }, + { + "a":"baz", + "b":"qux" + } +]"""), + ("index", """{ + "0":{ + "a":"foo", + "b":"bar" + }, + "1":{ + "a":"baz", + "b":"qux" + } +}"""), + ("columns", """{ + "a":{ + "0":"foo", + "1":"baz" + }, + "b":{ + "0":"bar", + "1":"qux" + } +}"""), + ("values", """[ + [ + "foo", + "bar" + ], + [ + "baz", + "qux" + ] +]"""), + + # TODO custom schema build is inconsistent with indent for table orient + ("table", """{"schema": {"fields":\ +[{"name":"index","type":"integer"},{"name":"a","type":"string"},\ +{"name":"b","type":"string"}],"primaryKey":["index"],"pandas_version":"0.20.0"}\ +, "data": [ + { + "index":0, + "a":"foo", + "b":"bar" + }, + { + "index":1, + "a":"baz", + "b":"qux" + } +]}""")]) + def test_json_indent_all_orients(self, orient, expected): + # GH 12004 + df = pd.DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["a", "b"]) + result = df.to_json(orient=orient, indent=4) + assert result == expected From ba7f044fd61464b3ba82cc399d296fff51044093 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 10:59:56 -0600 Subject: [PATCH 17/34] Fixed table schema --- pandas/io/json/_json.py | 17 +++++++++-- pandas/tests/io/json/test_pandas.py | 47 ++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 81352dd5db501..88781fded9c61 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -1,6 +1,7 @@ from io import StringIO from itertools import islice import os +import textwrap from typing import Any, Callable, Dict, List, Optional, Type, Union import numpy as np @@ -345,8 +346,20 @@ def _write( default_handler, indent, ) - serialized = '{{"schema": {schema}, "data": {data}}}'.format( - schema=dumps(self.schema), data=data + schema = dumps(self.schema, indent=indent) + + spaces = " " * indent + if indent: + line_break = "\n" + data = textwrap.indent(data, spaces).strip() + schema = textwrap.indent(schema, spaces).strip() + spacer = "\n{}".format(spaces) + else: + line_break = "" + + serialized = '{{{line_break}{spaces}"schema":{schema},\ +{line_break}{spaces}"data":{data}{line_break}}}'.format( + line_break=line_break, spaces=spaces, schema=schema, data=data ) return serialized diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index a46df3121e85e..df9d6f5196727 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1733,21 +1733,40 @@ def test_to_json_indent(self, indent): ]"""), # TODO custom schema build is inconsistent with indent for table orient - ("table", """{"schema": {"fields":\ -[{"name":"index","type":"integer"},{"name":"a","type":"string"},\ -{"name":"b","type":"string"}],"primaryKey":["index"],"pandas_version":"0.20.0"}\ -, "data": [ - { - "index":0, - "a":"foo", - "b":"bar" + ("table", """{ + "schema":{ + "fields":[ + { + "name":"index", + "type":"integer" + }, + { + "name":"a", + "type":"string" + }, + { + "name":"b", + "type":"string" + } + ], + "primaryKey":[ + "index" + ], + "pandas_version":"0.20.0" }, - { - "index":1, - "a":"baz", - "b":"qux" - } -]}""")]) + "data":[ + { + "index":0, + "a":"foo", + "b":"bar" + }, + { + "index":1, + "a":"baz", + "b":"qux" + } + ] +}""")]) def test_json_indent_all_orients(self, orient, expected): # GH 12004 df = pd.DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["a", "b"]) From df589e334062a6ec3b4058450fab060357e76737 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 10:29:19 -0600 Subject: [PATCH 18/34] merge conflict fixup --- pandas/tests/io/json/test_pandas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index df9d6f5196727..4eb489cfcbb67 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1732,7 +1732,6 @@ def test_to_json_indent(self, indent): ] ]"""), - # TODO custom schema build is inconsistent with indent for table orient ("table", """{ "schema":{ "fields":[ From 517377b32941b5f4d84e3c2da60571cc1b3ce0f2 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 13:30:44 -0700 Subject: [PATCH 19/34] Simplified logic for building table schema --- pandas/io/json/_json.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 88781fded9c61..decbcb8e8a0cb 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from io import StringIO from itertools import islice import os @@ -336,8 +337,10 @@ def _write( default_handler, indent, ): - data = super()._write( - obj, + table_obj = OrderedDict((("schema", self.schema), ("data", obj))) + breakpoint() + serialized = super()._write( + table_obj, orient, double_precision, ensure_ascii, @@ -346,21 +349,7 @@ def _write( default_handler, indent, ) - schema = dumps(self.schema, indent=indent) - - spaces = " " * indent - if indent: - line_break = "\n" - data = textwrap.indent(data, spaces).strip() - schema = textwrap.indent(schema, spaces).strip() - spacer = "\n{}".format(spaces) - else: - line_break = "" - serialized = '{{{line_break}{spaces}"schema":{schema},\ -{line_break}{spaces}"data":{data}{line_break}}}'.format( - line_break=line_break, spaces=spaces, schema=schema, data=data - ) return serialized From 4aec9d7c484b107c76d895878d5b1620bf8eadc9 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 14:09:36 -0700 Subject: [PATCH 20/34] Fixed test, removed breakpoint --- pandas/io/json/_json.py | 1 - pandas/tests/io/json/test_json_table_schema.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index decbcb8e8a0cb..4c1536a9cb0dc 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -338,7 +338,6 @@ def _write( indent, ): table_obj = OrderedDict((("schema", self.schema), ("data", obj))) - breakpoint() serialized = super()._write( table_obj, orient, diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index b2fc9ec217ca6..836b5a3cc8d00 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -596,7 +596,8 @@ def test_timestamp_in_columns(self): ) result = df.to_json(orient="table") js = json.loads(result) - assert js["schema"]["fields"][1]["name"] == 1451606400000 + assert js["schema"]["fields"][1]["name"] == "2016-01-01T00:00:00.000Z" + # TODO - below expectation is not correct; see GH 28256 assert js["schema"]["fields"][2]["name"] == 10000 @pytest.mark.parametrize( From c896b8a173be755af8ad4eb6fa07d15d2d6aa6f7 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 14:11:42 -0700 Subject: [PATCH 21/34] Added whatsnew for fixed issue --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 54e82dba022b9..e9b6550f2014f 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -162,7 +162,7 @@ I/O - :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`) - Bug in :meth:`DataFrame.to_json` where using a Tuple as a column or index value and using ``orient="columns"`` or ``orient="index"`` would produce invalid JSON (:issue:`20500`) -- +- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in iso format with ``orient="table"`` (:issue:`28130`) Plotting ^^^^^^^^ From b046061f836b6578a3ed3548d6bd71c5d75109b1 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 14:26:03 -0700 Subject: [PATCH 22/34] Ran clang-format on objToJSON.c --- pandas/_libs/src/ujson/python/objToJSON.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index ddc22c800042d..fc621b16b3c54 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -2373,10 +2373,16 @@ char *Object_iterGetName(JSOBJ obj, JSONTypeContext *tc, size_t *outLen) { } PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { - static char *kwlist[] = { - "obj", "ensure_ascii", "double_precision", "encode_html_chars", - "orient", "date_unit", "iso_dates", "default_handler", - "indent", NULL}; + static char *kwlist[] = {"obj", + "ensure_ascii", + "double_precision", + "encode_html_chars", + "orient", + "date_unit", + "iso_dates", + "default_handler", + "indent", + NULL}; char buffer[65536]; char *ret; @@ -2411,7 +2417,7 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { idoublePrecision, 1, // forceAscii 0, // encodeHTMLChars - 0, // indent + 0, // indent }}; JSONObjectEncoder *encoder = (JSONObjectEncoder *)&pyEncoder; @@ -2436,8 +2442,8 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) { PRINTMARK(); - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OiOssOOi", kwlist, &oinput, - &oensureAscii, &idoublePrecision, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OiOssOOi", kwlist, + &oinput, &oensureAscii, &idoublePrecision, &oencodeHTMLChars, &sOrient, &sdateFormat, &oisoDates, &odefHandler, &indent)) { return NULL; From ae933099f4c753a88262829275e842f24cb6f056 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 2 Sep 2019 17:11:50 -0700 Subject: [PATCH 23/34] lint fixups --- pandas/io/json/_json.py | 1 - pandas/tests/io/json/test_pandas.py | 49 ++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 4c1536a9cb0dc..006d04cfd6a2d 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -2,7 +2,6 @@ from io import StringIO from itertools import islice import os -import textwrap from typing import Any, Callable, Dict, List, Optional, Type, Union import numpy as np diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 4eb489cfcbb67..02ef1a9d031e8 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1670,8 +1670,12 @@ def test_to_json_indent(self, indent): assert result == expected - @pytest.mark.parametrize("orient,expected", [ - ("split", """{ + @pytest.mark.parametrize( + "orient,expected", + [ + ( + "split", + """{ "columns":[ "a", "b" @@ -1690,8 +1694,11 @@ def test_to_json_indent(self, indent): "qux" ] ] -}"""), - ("records", """[ +}""", + ), + ( + "records", + """[ { "a":"foo", "b":"bar" @@ -1700,8 +1707,11 @@ def test_to_json_indent(self, indent): "a":"baz", "b":"qux" } -]"""), - ("index", """{ +]""", + ), + ( + "index", + """{ "0":{ "a":"foo", "b":"bar" @@ -1710,8 +1720,11 @@ def test_to_json_indent(self, indent): "a":"baz", "b":"qux" } -}"""), - ("columns", """{ +}""", + ), + ( + "columns", + """{ "a":{ "0":"foo", "1":"baz" @@ -1720,8 +1733,11 @@ def test_to_json_indent(self, indent): "0":"bar", "1":"qux" } -}"""), - ("values", """[ +}""", + ), + ( + "values", + """[ [ "foo", "bar" @@ -1730,9 +1746,11 @@ def test_to_json_indent(self, indent): "baz", "qux" ] -]"""), - - ("table", """{ +]""", + ), + ( + "table", + """{ "schema":{ "fields":[ { @@ -1765,7 +1783,10 @@ def test_to_json_indent(self, indent): "b":"qux" } ] -}""")]) +}""", + ), + ], + ) def test_json_indent_all_orients(self, orient, expected): # GH 12004 df = pd.DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["a", "b"]) From ccb9823779a81cf48c4ea2893aeef603306fca8b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 3 Sep 2019 08:30:21 -0700 Subject: [PATCH 24/34] whitespace fixup --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 3396cd7e79d29..c501bff75f5e4 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -163,7 +163,7 @@ I/O - :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`) - Bug in :meth:`DataFrame.to_json` where using a Tuple as a column or index value and using ``orient="columns"`` or ``orient="index"`` would produce invalid JSON (:issue:`20500`) - Improve infinity parsing. :meth:`read_csv` now interprets ``Infinity``, ``+Infinity``, ``-Infinity`` as floating point values (:issue:`10065`) -- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in iso format with ``orient="table"`` (:issue:`28130`) +- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in iso format with ``orient="table"`` (:issue:`28130`) Plotting ^^^^^^^^ From 7d757e473aa9efa1637f4dab97bf9ea4c0a0632a Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 3 Sep 2019 13:16:36 -0700 Subject: [PATCH 25/34] Py35 compat --- .../tests/io/json/test_json_table_schema.py | 37 ++++++++++++++++--- pandas/tests/io/json/test_pandas.py | 8 +++- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index 836b5a3cc8d00..c19f977c7f475 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -9,6 +9,7 @@ import pandas as pd from pandas import DataFrame +from pandas.compat import PY35 import pandas.util.testing as tm from pandas.io.json._table_schema import ( @@ -234,7 +235,11 @@ def test_build_series(self): ), ] ) - assert result == expected + + if PY35: + assert sorted(result.items()) == sorted(expected.items()) + else: + assert result == expected def test_to_json(self): df = self.df.copy() @@ -323,7 +328,11 @@ def test_to_json(self): ), ] expected = OrderedDict([("schema", schema), ("data", data)]) - assert result == expected + + if PY35: + assert sorted(result.items()) == sorted(expected.items()) + else: + assert result == expected def test_to_json_float_index(self): data = pd.Series(1, index=[1.0, 2.0]) @@ -352,7 +361,11 @@ def test_to_json_float_index(self): ), ] ) - assert result == expected + + if PY35: + assert sorted(result.items()) == sorted(expected.items()) + else: + assert result == expected def test_to_json_period_index(self): idx = pd.period_range("2016", freq="Q-JAN", periods=2) @@ -372,7 +385,11 @@ def test_to_json_period_index(self): OrderedDict([("index", "2016-02-01T00:00:00.000Z"), ("values", 1)]), ] expected = OrderedDict([("schema", schema), ("data", data)]) - assert result == expected + + if PY35: + assert sorted(result.items()) == sorted(expected.items()) + else: + assert result == expected def test_to_json_categorical_index(self): data = pd.Series(1, pd.CategoricalIndex(["a", "b"])) @@ -406,7 +423,11 @@ def test_to_json_categorical_index(self): ), ] ) - assert result == expected + + if PY35: + assert sorted(result.items()) == sorted(expected.items()) + else: + assert result == expected def test_date_format_raises(self): with pytest.raises(ValueError): @@ -542,7 +563,11 @@ def test_categorical(self): ), ] ) - assert result == expected + + if PY35: + assert sorted(result.items()) == sorted(expected.items()) + else: + assert result == expected @pytest.mark.parametrize( "idx,nm,prop", diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 02ef1a9d031e8..e9401134022d6 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from pandas.compat import is_platform_32bit +from pandas.compat import is_platform_32bit, PY35 import pandas.util._test_decorators as td import pandas as pd @@ -1791,4 +1791,8 @@ def test_json_indent_all_orients(self, orient, expected): # GH 12004 df = pd.DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["a", "b"]) result = df.to_json(orient=orient, indent=4) - assert result == expected + + if PY35: + assert json.loads(result) == json.loads(expected) + else: + assert result == expected From 65315c338b64a5d37dad321530df9fe2bd387147 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 3 Sep 2019 16:01:00 -0700 Subject: [PATCH 26/34] isort and lint fixups --- doc/source/whatsnew/v1.0.0.rst | 2 +- pandas/tests/io/json/test_json_table_schema.py | 3 ++- pandas/tests/io/json/test_pandas.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 1439121fc07fb..21e098657fd63 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -21,7 +21,7 @@ including other versions of pandas. Enhancements ~~~~~~~~~~~~ - :meth:`DataFrame.to_latex` now accepts ``caption`` and ``label`` arguments (:issue:`25436`) -- :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`) +- :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`) - .. _whatsnew_1000.enhancements.other: diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index c19f977c7f475..b8396991705ea 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -5,11 +5,12 @@ import numpy as np import pytest +from pandas.compat import PY35 + from pandas.core.dtypes.dtypes import CategoricalDtype, DatetimeTZDtype, PeriodDtype import pandas as pd from pandas import DataFrame -from pandas.compat import PY35 import pandas.util.testing as tm from pandas.io.json._table_schema import ( diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index e9401134022d6..40458517c949d 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from pandas.compat import is_platform_32bit, PY35 +from pandas.compat import PY35, is_platform_32bit import pandas.util._test_decorators as td import pandas as pd From 0b440e03c5390ebc2f2f1b984970b705e5d7be89 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 5 Sep 2019 17:24:41 -0700 Subject: [PATCH 27/34] Changed default indent to None --- pandas/core/generic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index b08b8d832cfd3..68f86fc2647f5 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2263,7 +2263,7 @@ def to_json( lines: bool_t = False, compression: Optional[str] = "infer", index: bool_t = True, - indent: int = 0, + indent: Optional[int] = None, ) -> Optional[str]: """ Convert the object to a JSON string. @@ -2344,7 +2344,7 @@ def to_json( .. versionadded:: 0.23.0 - indent : integer, default 0 + indent : integer, optional Length of whitespace used to indent each record. .. versionadded:: 1.0.0 @@ -2409,6 +2409,12 @@ def to_json( date_format = "iso" elif date_format is None: date_format = "epoch" + + if indent is None: + int_indent = 0 + else: + int_indent = indent + return json.to_json( path_or_buf=path_or_buf, obj=self, @@ -2421,7 +2427,7 @@ def to_json( lines=lines, compression=compression, index=index, - indent=indent, + indent=int_indent, ) def to_hdf(self, path_or_buf, key, **kwargs): From 0024f411aaa9d5a2de296c63a514ecf4ffba6129 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 5 Sep 2019 17:30:57 -0700 Subject: [PATCH 28/34] Validate int input --- pandas/core/generic.py | 8 ++++++++ pandas/tests/io/json/test_pandas.py | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 68f86fc2647f5..004af7e6d6caf 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2359,6 +2359,13 @@ def to_json( -------- read_json + Notes + ----- + The behavior of ``indent=0`` varies from the stdlib, which does not + indent the output but does insert newlines. Currently, ``indent=0`` + and the default ``indent=None`` are equivalent in pandas, though this + may change in a future release. + Examples -------- @@ -2410,6 +2417,7 @@ def to_json( elif date_format is None: date_format = "epoch" + config.is_nonnegative_int(indent) if indent is None: int_indent = 0 else: diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 40458517c949d..5c7cc0f8b1943 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1796,3 +1796,7 @@ def test_json_indent_all_orients(self, orient, expected): assert json.loads(result) == json.loads(expected) else: assert result == expected + + def test_json_negative_indent_raises(self): + with pytest.raises(ValueError, match="must be a nonnegative integer"): + pd.DataFrame().to_json(indent=-1) From b894b8c0b754dcd398218862dbf4ca4f59124ef6 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 11 Sep 2019 09:03:02 -0700 Subject: [PATCH 29/34] Added helper function for Py35 compat --- .../tests/io/json/test_json_table_schema.py | 38 +++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/pandas/tests/io/json/test_json_table_schema.py b/pandas/tests/io/json/test_json_table_schema.py index b8396991705ea..569e299860614 100644 --- a/pandas/tests/io/json/test_json_table_schema.py +++ b/pandas/tests/io/json/test_json_table_schema.py @@ -22,6 +22,14 @@ ) +def assert_results_equal(result, expected): + """Helper function for comparing deserialized JSON with Py35 compat.""" + if PY35: + assert sorted(result.items()) == sorted(expected.items()) + else: + assert result == expected + + class TestBuildSchema: def setup_method(self, method): self.df = DataFrame( @@ -237,10 +245,7 @@ def test_build_series(self): ] ) - if PY35: - assert sorted(result.items()) == sorted(expected.items()) - else: - assert result == expected + assert_results_equal(result, expected) def test_to_json(self): df = self.df.copy() @@ -330,10 +335,7 @@ def test_to_json(self): ] expected = OrderedDict([("schema", schema), ("data", data)]) - if PY35: - assert sorted(result.items()) == sorted(expected.items()) - else: - assert result == expected + assert_results_equal(result, expected) def test_to_json_float_index(self): data = pd.Series(1, index=[1.0, 2.0]) @@ -363,10 +365,7 @@ def test_to_json_float_index(self): ] ) - if PY35: - assert sorted(result.items()) == sorted(expected.items()) - else: - assert result == expected + assert_results_equal(result, expected) def test_to_json_period_index(self): idx = pd.period_range("2016", freq="Q-JAN", periods=2) @@ -387,10 +386,7 @@ def test_to_json_period_index(self): ] expected = OrderedDict([("schema", schema), ("data", data)]) - if PY35: - assert sorted(result.items()) == sorted(expected.items()) - else: - assert result == expected + assert_results_equal(result, expected) def test_to_json_categorical_index(self): data = pd.Series(1, pd.CategoricalIndex(["a", "b"])) @@ -425,10 +421,7 @@ def test_to_json_categorical_index(self): ] ) - if PY35: - assert sorted(result.items()) == sorted(expected.items()) - else: - assert result == expected + assert_results_equal(result, expected) def test_date_format_raises(self): with pytest.raises(ValueError): @@ -565,10 +558,7 @@ def test_categorical(self): ] ) - if PY35: - assert sorted(result.items()) == sorted(expected.items()) - else: - assert result == expected + assert_results_equal(result, expected) @pytest.mark.parametrize( "idx,nm,prop", From dab8df1405ed00223a9b9bce55cc934e011d69de Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 13 Sep 2019 13:28:02 -0700 Subject: [PATCH 30/34] Reverted line removal --- doc/source/whatsnew/v1.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 4c321aabbd27d..61cc63886ec78 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -21,6 +21,7 @@ including other versions of pandas. Enhancements ~~~~~~~~~~~~ + .. _whatsnew_1000.enhancements.other: Other enhancements From 966fadbde572d3e44080b30d54a20650d204714e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Sep 2019 20:41:42 -0700 Subject: [PATCH 31/34] Whatsnew fix --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index e47e193a544ff..b453b935f02dd 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -194,7 +194,7 @@ I/O - Improve infinity parsing. :meth:`read_csv` now interprets ``Infinity``, ``+Infinity``, ``-Infinity`` as floating point values (:issue:`10065`) - Bug in :meth:`DataFrame.to_csv` where values were truncated when the length of ``na_rep`` was shorter than the text input data. (:issue:`25099`) - Bug in :func:`DataFrame.to_string` where values were truncated using display options instead of outputting the full content (:issue:`9784`) -- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in iso format with ``orient="table"`` (:issue:`28130`) +- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in ISO format with ``orient="table"`` (:issue:`28130`) Plotting ^^^^^^^^ From c8efda6cd4f9bf821bafbdba5e9c08cb222584e1 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Sep 2019 20:47:26 -0700 Subject: [PATCH 32/34] Simplified indent code --- pandas/core/generic.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 04f38e4dc7b2f..0dadbe443f153 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2415,10 +2415,7 @@ def to_json( date_format = "epoch" config.is_nonnegative_int(indent) - if indent is None: - int_indent = 0 - else: - int_indent = indent + indent = indent or 0 return json.to_json( path_or_buf=path_or_buf, @@ -2432,7 +2429,7 @@ def to_json( lines=lines, compression=compression, index=index, - indent=int_indent, + indent=indent, ) def to_hdf(self, path_or_buf, key, **kwargs): From 5067eb704828eae477f91f3be448f14db1071904 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Sep 2019 20:53:08 -0700 Subject: [PATCH 33/34] Added comment for func --- pandas/_libs/src/ujson/lib/ultrajsonenc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/_libs/src/ujson/lib/ultrajsonenc.c b/pandas/_libs/src/ujson/lib/ultrajsonenc.c index 9ba677c70eb74..51c9b9244ecfc 100644 --- a/pandas/_libs/src/ujson/lib/ultrajsonenc.c +++ b/pandas/_libs/src/ujson/lib/ultrajsonenc.c @@ -733,6 +733,8 @@ void Buffer_AppendIndentNewlineUnchecked(JSONObjectEncoder *enc) if (enc->indent > 0) Buffer_AppendCharUnchecked(enc, '\n'); } +// This function could be refactored to only accept enc as an argument, +// but this is a straight vendor from ujson source void Buffer_AppendIndentUnchecked(JSONObjectEncoder *enc, JSINT32 value) { int i; From f376f121e20e25848a305e31c501500b8a205f2c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 16 Sep 2019 08:23:24 -0700 Subject: [PATCH 34/34] whitespace doc fixup --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index b453b935f02dd..c379d1a2a160d 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -194,7 +194,7 @@ I/O - Improve infinity parsing. :meth:`read_csv` now interprets ``Infinity``, ``+Infinity``, ``-Infinity`` as floating point values (:issue:`10065`) - Bug in :meth:`DataFrame.to_csv` where values were truncated when the length of ``na_rep`` was shorter than the text input data. (:issue:`25099`) - Bug in :func:`DataFrame.to_string` where values were truncated using display options instead of outputting the full content (:issue:`9784`) -- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in ISO format with ``orient="table"`` (:issue:`28130`) +- Bug in :meth:`DataFrame.to_json` where a datetime column label would not be written out in ISO format with ``orient="table"`` (:issue:`28130`) Plotting ^^^^^^^^