Merge pull request #9130 from Komnomnomnom/json-block-support

PERF: json support for blocks GH9037

Author: jreback

doc/source/whatsnew/v0.16.0.txt

@@ -28,6 +28,22 @@ Backwards incompatible API changes
 .. _whatsnew_0160.api_breaking:
 
 - ``Index.duplicated`` now returns `np.array(dtype=bool)` rather than `Index(dtype=object)` containing `bool` values. (:issue:`8875`)
+- ``DataFrame.to_json`` now returns accurate type serialisation for each column for frames of mixed dtype (:issue:`9037`)
+
+  Previously data was coerced to a common dtype before serialisation, which for
+  example resulted in integers being serialised to floats:
+
+  .. code-block:: python
+
+    In [2]: pd.DataFrame({'i': [1,2], 'f': [3.0, 4.2]}).to_json()
+    Out[2]: '{"f":{"0":3.0,"1":4.2},"i":{"0":1.0,"1":2.0}}'
+
+  Now each column is serialised using its correct dtype:
+
+  .. code-block:: python
+
+    In [2]:  pd.DataFrame({'i': [1,2], 'f': [3.0, 4.2]}).to_json()
+    Out[2]: '{"f":{"0":3.0,"1":4.2},"i":{"0":1,"1":2}}'
 
 Deprecations
 ~~~~~~~~~~~~
@@ -46,10 +62,10 @@ Performance
 .. _whatsnew_0160.performance:
 
 - Fixed a performance regression for ``.loc`` indexing with an array or list-like (:issue:`9126`:).
+- ``DataFrame.to_json`` 30x performance improvement for mixed dtype frames. (:issue:`9037`)
 - Performance improvements in ``MultiIndex.duplicated`` by working with labels instead of values (:issue:`9125`)
 - Improved the speed of `nunique` by calling `unique` instead of `value_counts` (:issue:`9129`, :issue:`7771`)
 
-
 Bug Fixes
 ~~~~~~~~~
 

pandas/core/generic.py

@@ -2160,20 +2160,12 @@ def as_blocks(self):
         Convert the frame to a dict of dtype -> Constructor Types that each has
         a homogeneous dtype.
 
-        are presented in sorted order unless a specific list of columns is
-        provided.
-
         NOTE: the dtypes of the blocks WILL BE PRESERVED HERE (unlike in
               as_matrix)
 
-        Parameters
-        ----------
-        columns : array-like
-            Specific column order
-
         Returns
         -------
-        values : a list of Object
+        values : a dict of dtype -> Constructor Types
         """
         self._consolidate_inplace()
 

pandas/io/tests/test_json/test_pandas.py

@@ -1,5 +1,5 @@
 # pylint: disable-msg=W0612,E1101
-from pandas.compat import range, lrange, StringIO
+from pandas.compat import range, lrange, StringIO, OrderedDict
 from pandas import compat
 import os
 
@@ -337,14 +337,44 @@ def test_v12_compat(self):
 
         v12_json = os.path.join(self.dirpath, 'tsframe_v012.json')
         df_unser = pd.read_json(v12_json)
-        df_unser = pd.read_json(v12_json)
         assert_frame_equal(df, df_unser)
 
         df_iso = df.drop(['modified'], axis=1)
         v12_iso_json = os.path.join(self.dirpath, 'tsframe_iso_v012.json')
         df_unser_iso = pd.read_json(v12_iso_json)
         assert_frame_equal(df_iso, df_unser_iso)
 
+    def test_blocks_compat_GH9037(self):
+        index = pd.date_range('20000101', periods=10, freq='H')
+        df_mixed = DataFrame(OrderedDict(
+            float_1=[-0.92077639, 0.77434435, 1.25234727, 0.61485564,
+                     -0.60316077, 0.24653374, 0.28668979, -2.51969012,
+                     0.95748401, -1.02970536],
+            int_1=[19680418, 75337055, 99973684, 65103179, 79373900,
+                   40314334, 21290235,  4991321, 41903419, 16008365],
+            str_1=['78c608f1', '64a99743', '13d2ff52', 'ca7f4af2', '97236474',
+                   'bde7e214', '1a6bde47', 'b1190be5', '7a669144', '8d64d068'],
+            float_2=[-0.0428278, -1.80872357,  3.36042349, -0.7573685,
+                     -0.48217572, 0.86229683, 1.08935819, 0.93898739,
+                     -0.03030452, 1.43366348],
+            str_2=['14f04af9', 'd085da90', '4bcfac83', '81504caf', '2ffef4a9',
+                   '08e2f5c4', '07e1af03', 'addbd4a7', '1f6a09ba', '4bfc4d87'],
+            int_2=[86967717, 98098830, 51927505, 20372254, 12601730, 20884027,
+                   34193846, 10561746, 24867120, 76131025]
+        ), index=index)
+
+        # JSON deserialisation always creates unicode strings
+        df_mixed.columns = df_mixed.columns.astype('unicode')
+
+        df_roundtrip = pd.read_json(df_mixed.to_json(orient='split'),
+                                    orient='split')
+        assert_frame_equal(df_mixed, df_roundtrip,
+                           check_index_type=True,
+                           check_column_type=True,
+                           check_frame_type=True,
+                           by_blocks=True,
+                           check_exact=True)
+
     def test_series_non_unique_index(self):
         s = Series(['a', 'b'], index=[1, 1])
 

pandas/src/datetime_helper.h

@@ -13,8 +13,11 @@ void mangle_nat(PyObject *val) {
 }
 
 npy_int64 get_long_attr(PyObject *o, const char *attr) {
+  npy_int64 long_val;
   PyObject *value = PyObject_GetAttrString(o, attr);
-  return PyLong_Check(value) ? PyLong_AsLongLong(value) : PyInt_AS_LONG(value);
+  long_val = (PyLong_Check(value) ? PyLong_AsLongLong(value) : PyInt_AS_LONG(value));
+  Py_DECREF(value);
+  return long_val;
 }
 
 npy_float64 total_seconds(PyObject *td) {