Merge remote-tracking branch 'upstream/master' into fix-melt

Author: Joshua Bradt

doc/source/whatsnew/v0.20.0.txt

@@ -356,6 +356,59 @@ New Behavior:
    In [11]: index.memory_usage(deep=True)
    Out[11]: 260
 
+.. _whatsnew_0200.api_breaking.groupby_describe:
+
+Groupby Describe Formatting
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The output formatting of ``groupby.describe()`` now labels the ``describe()`` metrics in the columns instead of the index.
+This format is consistent with ``groupby.agg()`` when applying multiple functions at once. (:issue:`4792`)
+
+Previous Behavior:
+
+.. code-block:: ipython
+
+   In [1]: df = pd.DataFrame({'A': [1, 1, 2, 2], 'B': [1, 2, 3, 4]})
+
+   In [2]: df.groupby('A').describe()
+   Out[2]:
+                   B
+   A
+   1 count  2.000000
+     mean   1.500000
+     std    0.707107
+     min    1.000000
+     25%    1.250000
+     50%    1.500000
+     75%    1.750000
+     max    2.000000
+   2 count  2.000000
+     mean   3.500000
+     std    0.707107
+     min    3.000000
+     25%    3.250000
+     50%    3.500000
+     75%    3.750000
+     max    4.000000
+
+   In [3]: df.groupby('A').agg([np.mean, np.std, np.min, np.max])
+   Out[3]:
+        B
+     mean       std amin amax
+   A
+   1  1.5  0.707107    1    2
+   2  3.5  0.707107    3    4
+
+New Behavior:
+
+.. ipython:: python
+
+   df = pd.DataFrame({'A': [1, 1, 2, 2], 'B': [1, 2, 3, 4]})
+
+   df.groupby('A').describe()
+
+   df.groupby('A').agg([np.mean, np.std, np.min, np.max])
+
 .. _whatsnew_0200.api:
 
 Other API Changes
@@ -371,6 +424,7 @@ Other API Changes
 - ``pandas.api.types.is_datetime64_ns_dtype`` will now report ``True`` on a tz-aware dtype, similar to ``pandas.api.types.is_datetime64_any_dtype``
 - ``DataFrame.asof()`` will return a null filled ``Series`` instead the scalar ``NaN`` if a match is not found (:issue:`15118`)
 - The :func:`pd.read_gbq` method now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no more casted to ``int64`` which also caused precision lost (:issue: `14064`, :issue:`14305`).
+- Reorganization of timeseries development tests (:issue:`14854`)
 
 .. _whatsnew_0200.deprecations:
 
@@ -385,7 +439,7 @@ Deprecations
 - ``TimedeltaIndex.searchsorted()``, ``DatetimeIndex.searchsorted()``, and ``PeriodIndex.searchsorted()`` have deprecated the ``key`` parameter in favor of ``value`` (:issue:`12662`)
 - ``DataFrame.astype()`` has deprecated the ``raise_on_error`` parameter in favor of ``errors`` (:issue:`14878`)
 - ``Series.sortlevel`` and ``DataFrame.sortlevel`` have been deprecated in favor of ``Series.sort_index`` and ``DataFrame.sort_index`` (:issue:`15099`)
-
+- importing ``concat`` from ``pandas.tools.merge`` has been deprecated in favor of imports from the ``pandas`` namespace. This should only affect explict imports (:issue:`15358`)
 
 .. _whatsnew_0200.prior_deprecations:
 
@@ -484,6 +538,8 @@ Bug Fixes
 - Bug in ``pd.pivot_table()`` where no error was raised when values argument was not in the columns (:issue:`14938`)
 
 - Bug in ``.to_json()`` where ``lines=True`` and contents (keys or values) contain escaped characters (:issue:`15096`)
+- Bug in ``.to_json()`` causing single byte ascii characters to be expanded to four byte unicode (:issue:`15344`)
+- Bug in ``.read_json()`` for Python 2 where ``lines=True`` and contents contain non-ascii unicode characters (:issue:`15132`)
 - Bug in ``.rolling/expanding()`` functions where ``count()`` was not counting ``np.Inf``, nor handling ``object`` dtypes (:issue:`12541`)
 - Bug in ``DataFrame.resample().median()`` if duplicate column names are present (:issue:`14233`)
 
@@ -507,7 +563,6 @@ Bug Fixes
 - Bug in ``DataFrame.fillna()`` where the argument ``downcast`` was ignored when fillna value was of type ``dict`` (:issue:`15277`)
 
 
-- Bug in ``.read_json()`` for Python 2 where ``lines=True`` and contents contain non-ascii unicode characters (:issue:`15132`)
 
 - Bug in ``pd.read_csv()`` with ``float_precision='round_trip'`` which caused a segfault when a text entry is parsed (:issue:`15140`)
 

pandas/__init__.py

@@ -42,10 +42,10 @@
 from pandas.sparse.api import *
 from pandas.stats.api import *
 from pandas.tseries.api import *
-from pandas.io.api import *
 from pandas.computation.api import *
 
-from pandas.tools.merge import (merge, concat, ordered_merge,
+from pandas.tools.concat import concat
+from pandas.tools.merge import (merge, ordered_merge,
                                 merge_ordered, merge_asof)
 from pandas.tools.pivot import pivot_table, crosstab
 from pandas.tools.plotting import scatter_matrix, plot_params
@@ -54,6 +54,8 @@
 from pandas.core.reshape import melt
 from pandas.util.print_versions import show_versions
 
+from pandas.io.api import *
+
 # define the testing framework
 import pandas.util.testing
 from pandas.util.nosetester import NoseTester

pandas/core/base.py

@@ -472,7 +472,7 @@ def _aggregate(self, arg, *args, **kwargs):
 
                 arg = new_arg
 
-            from pandas.tools.merge import concat
+            from pandas.tools.concat import concat
 
             def _agg_1dim(name, how, subset=None):
                 """
@@ -579,7 +579,7 @@ def _agg(arg, func):
         return result, True
 
     def _aggregate_multiple_funcs(self, arg, _level):
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
 
         if self.axis != 0:
             raise NotImplementedError("axis other than 0 is not supported")

pandas/core/categorical.py

@@ -1907,7 +1907,7 @@ def describe(self):
         counts = self.value_counts(dropna=False)
         freqs = counts / float(counts.sum())
 
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
         result = concat([counts, freqs], axis=1)
         result.columns = ['counts', 'freqs']
         result.index.name = 'categories'

pandas/core/frame.py

@@ -4402,7 +4402,7 @@ def append(self, other, ignore_index=False, verify_integrity=False):
             if (self.columns.get_indexer(other.columns) >= 0).all():
                 other = other.loc[:, self.columns]
 
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
         if isinstance(other, (list, tuple)):
             to_concat = [self] + other
         else:
@@ -4532,7 +4532,8 @@ def join(self, other, on=None, how='left', lsuffix='', rsuffix='',
 
     def _join_compat(self, other, on=None, how='left', lsuffix='', rsuffix='',
                      sort=False):
-        from pandas.tools.merge import merge, concat
+        from pandas.tools.merge import merge
+        from pandas.tools.concat import concat
 
         if isinstance(other, Series):
             if other.name is None:
@@ -4636,7 +4637,7 @@ def round(self, decimals=0, *args, **kwargs):
         Series.round
 
         """
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
 
         def _dict_round(df, decimals):
             for col, vals in df.iteritems():
@@ -5306,7 +5307,7 @@ def isin(self, values):
         """
         if isinstance(values, dict):
             from collections import defaultdict
-            from pandas.tools.merge import concat
+            from pandas.tools.concat import concat
             values = defaultdict(list, values)
             return concat((self.iloc[:, [i]].isin(values[col])
                            for i, col in enumerate(self.columns)), axis=1)

pandas/core/groupby.py

@@ -80,7 +80,6 @@
     'mean', 'sum', 'min', 'max',
     'cumcount',
     'resample',
-    'describe',
     'rank', 'quantile',
     'fillna',
     'mad',
@@ -854,7 +853,7 @@ def _wrap_applied_output(self, *args, **kwargs):
         raise AbstractMethodError(self)
 
     def _concat_objects(self, keys, values, not_indexed_same=False):
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
 
         def reset_identity(values):
             # reset the identities of the components
@@ -1138,6 +1137,16 @@ def ohlc(self):
         return self._apply_to_column_groupbys(
             lambda x: x._cython_agg_general('ohlc'))
 
+    @Appender(DataFrame.describe.__doc__)
+    @Substitution(name='groupby')
+    @Appender(_doc_template)
+    def describe(self, **kwargs):
+        self._set_group_selection()
+        result = self.apply(lambda x: x.describe(**kwargs))
+        if self.axis == 1:
+            return result.T
+        return result.unstack()
+
     @Substitution(name='groupby')
     @Appender(_doc_template)
     def resample(self, rule, *args, **kwargs):
@@ -3039,6 +3048,14 @@ def nlargest(self, n=5, keep='first'):
     def nsmallest(self, n=5, keep='first'):
         return self.apply(lambda x: x.nsmallest(n=n, keep=keep))
 
+    @Appender(Series.describe.__doc__)
+    def describe(self, **kwargs):
+        self._set_group_selection()
+        result = self.apply(lambda x: x.describe(**kwargs))
+        if self.axis == 1:
+            return result.T
+        return result.unstack()
+
     def value_counts(self, normalize=False, sort=True, ascending=False,
                      bins=None, dropna=True):
 
@@ -3507,7 +3524,7 @@ def first_non_None_value(values):
                         # still a series
                         # path added as of GH 5545
                         elif all_indexed_same:
-                            from pandas.tools.merge import concat
+                            from pandas.tools.concat import concat
                             return concat(values)
 
                     if not all_indexed_same:
@@ -3540,7 +3557,7 @@ def first_non_None_value(values):
                         else:
                             # GH5788 instead of stacking; concat gets the
                             # dtypes correct
-                            from pandas.tools.merge import concat
+                            from pandas.tools.concat import concat
                             result = concat(values, keys=key_index,
                                             names=key_index.names,
                                             axis=self.axis).unstack()
@@ -3588,7 +3605,7 @@ def first_non_None_value(values):
                                         not_indexed_same=not_indexed_same)
 
     def _transform_general(self, func, *args, **kwargs):
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
 
         applied = []
         obj = self._obj_with_exclusions
@@ -3980,7 +3997,7 @@ def _iterate_column_groupbys(self):
                                          exclusions=self.exclusions)
 
     def _apply_to_column_groupbys(self, func):
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
         return concat(
             (func(col_groupby) for _, col_groupby
              in self._iterate_column_groupbys()),
@@ -4061,7 +4078,7 @@ def groupby_series(obj, col=None):
         if isinstance(obj, Series):
             results = groupby_series(obj)
         else:
-            from pandas.tools.merge import concat
+            from pandas.tools.concat import concat
             results = [groupby_series(obj[col], col) for col in obj.columns]
             results = concat(results, axis=1)
 

pandas/core/panel.py

@@ -1282,7 +1282,7 @@ def join(self, other, how='left', lsuffix='', rsuffix=''):
         -------
         joined : Panel
         """
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
 
         if isinstance(other, Panel):
             join_major, join_minor = self._get_join_index(other, how)

pandas/core/reshape.py

@@ -1204,7 +1204,7 @@ def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False,
     --------
     Series.str.get_dummies
     """
-    from pandas.tools.merge import concat
+    from pandas.tools.concat import concat
     from itertools import cycle
 
     if isinstance(data, DataFrame):

pandas/core/series.py

@@ -1588,7 +1588,7 @@ def append(self, to_append, ignore_index=False, verify_integrity=False):
 
 
         """
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
 
         if isinstance(to_append, (list, tuple)):
             to_concat = [self] + to_append

pandas/formats/format.py

@@ -165,7 +165,7 @@ def __init__(self, series, buf=None, length=True, header=True, index=True,
         self._chk_truncate()
 
     def _chk_truncate(self):
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
         max_rows = self.max_rows
         truncate_v = max_rows and (len(self.series) > max_rows)
         series = self.series
@@ -406,7 +406,7 @@ def _chk_truncate(self):
         Checks whether the frame should be truncated. If so, slices
         the frame up.
         """
-        from pandas.tools.merge import concat
+        from pandas.tools.concat import concat
 
         # Column of which first element is used to determine width of a dot col
         self.tr_size_col = -1

pandas/io/gbq.py

@@ -10,9 +10,7 @@
 import numpy as np
 
 from distutils.version import StrictVersion
-from pandas import compat
-from pandas.core.api import DataFrame
-from pandas.tools.merge import concat
+from pandas import compat, DataFrame, concat
 from pandas.core.common import PandasError
 from pandas.compat import lzip, bytes_to_str
 

pandas/io/pytables.py

@@ -26,13 +26,12 @@
 
 import pandas as pd
 from pandas import (Series, DataFrame, Panel, Panel4D, Index,
-                    MultiIndex, Int64Index, isnull)
+                    MultiIndex, Int64Index, isnull, concat,
+                    SparseSeries, SparseDataFrame, PeriodIndex,
+                    DatetimeIndex, TimedeltaIndex)
 from pandas.core import config
 from pandas.io.common import _stringify_path
-from pandas.sparse.api import SparseSeries, SparseDataFrame
 from pandas.sparse.array import BlockIndex, IntIndex
-from pandas.tseries.api import PeriodIndex, DatetimeIndex
-from pandas.tseries.tdi import TimedeltaIndex
 from pandas.core.base import StringMixin
 from pandas.formats.printing import adjoin, pprint_thing
 from pandas.core.common import _asarray_tuplesafe, PerformanceWarning
@@ -42,7 +41,6 @@
                                    _block2d_to_blocknd,
                                    _factor_indexer, _block_shape)
 from pandas.core.index import _ensure_index
-from pandas.tools.merge import concat
 from pandas import compat
 from pandas.compat import u_safe as u, PY3, range, lrange, string_types, filter
 from pandas.core.config import get_option

pandas/io/tests/json/test_pandas.py

@@ -1044,3 +1044,13 @@ def roundtrip(s, encoding='latin-1'):
 
         for s in examples:
             roundtrip(s)
+
+    def test_data_frame_size_after_to_json(self):
+        # GH15344
+        df = DataFrame({'a': [str(1)]})
+
+        size_before = df.memory_usage(index=True, deep=True).sum()
+        df.to_json()
+        size_after = df.memory_usage(index=True, deep=True).sum()
+
+        self.assertEqual(size_before, size_after)

pandas/src/ujson/python/objToJSON.c

@@ -402,6 +402,16 @@ static void *PyStringToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
 static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
                              size_t *_outLen) {
     PyObject *obj = (PyObject *)_obj;
+
+#if (PY_VERSION_HEX >= 0x03030000)
+    if (PyUnicode_IS_COMPACT_ASCII(obj)) {
+        Py_ssize_t len;
+        char *data = PyUnicode_AsUTF8AndSize(obj, &len);
+        *_outLen = len;
+        return data;
+    }
+#endif
+
     PyObject *newObj = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(obj),
                                             PyUnicode_GET_SIZE(obj), NULL);