Merge remote-tracking branch 'upstream/master'

Author: Carlos Souza

doc/source/whatsnew/v0.20.0.txt

@@ -750,6 +750,62 @@ New Behavior:
    TypeError: Cannot compare 2014-01-01 00:00:00 of
    type <class 'pandas.tslib.Timestamp'> to string column
 
+.. _whatsnew_0200.api_breaking.index_order:
+
+Index.intersection and inner join now preserve the order of the left Index
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+`:meth:Index.intersection` now preserves the order of the calling ``Index`` (left)
+instead of the other ``Index`` (right) (:issue:`15582`). This affects the inner
+joins (`:meth:DataFrame.join` and `:func:merge`) and the ``.align`` methods.
+
+- ``Index.intersection``
+
+  .. ipython:: python
+
+     left = pd.Index([2, 1, 0])
+     left
+     right = pd.Index([1, 2, 3])
+     right
+
+  Previous Behavior:
+
+  .. code-block:: ipython
+
+     In [4]: left.intersection(right)
+     Out[4]: Int64Index([1, 2], dtype='int64')
+
+  New Behavior:
+
+  .. ipython:: python
+
+     left.intersection(right)
+
+- ``DataFrame.join`` and ``pd.merge``
+
+  .. ipython:: python
+
+     left = pd.DataFrame({'a': [20, 10, 0]}, index=[2, 1, 0])
+     left
+     right = pd.DataFrame({'b': [100, 200, 300]}, index=[1, 2, 3])
+     right
+
+  Previous Behavior:
+
+  .. code-block:: ipython
+
+     In [4]: left.join(right, how='inner')
+     Out[4]:
+         a    b
+     1  10  100
+     2  20  200
+
+  New Behavior:
+
+  .. ipython:: python
+
+     left.join(right, how='inner')
+
 
 .. _whatsnew_0200.api:
 
@@ -887,6 +943,7 @@ Bug Fixes
 
 - Compat for 32-bit platforms for ``.qcut/cut``; bins will now be ``int64`` dtype (:issue:`14866`)
 
+- Properly set ``__name__`` and ``__qualname__`` for ``Groupby.*`` functions (:issue:`14620`)
 - Bug in ``.at`` when selecting from a tz-aware column (:issue:`15822`)
 - Bug in the display of ``.info()`` where a qualifier (+) would always be displayed with a ``MultiIndex`` that contains only non-strings (:issue:`15245`)
 - Bug in ``.replace()`` may result in incorrect dtypes. (:issue:`12747`, :issue:`15765`)
@@ -983,6 +1040,7 @@ Bug Fixes
 
 - Bug in ``DataFrame.to_stata()`` and ``StataWriter`` which produces incorrectly formatted files to be produced for some locales (:issue:`13856`)
 - Bug in ``StataReader`` and ``StataWriter`` which allows invalid encodings (:issue:`15723`)
+- Bug with ``sort=True`` in ``DataFrame.join`` and ``pd.merge`` when joining on indexes (:issue:`15582`)
 
 - Bug in ``pd.concat()`` in which concatting with an empty dataframe with ``join='inner'`` was being improperly handled (:issue:`15328`)
 - Bug in ``groupby.agg()`` incorrectly localizing timezone on ``datetime`` (:issue:`15426`, :issue:`10668`, :issue:`13046`)

pandas/core/frame.py

@@ -124,10 +124,14 @@
 ----------%s
 right : DataFrame
 how : {'left', 'right', 'outer', 'inner'}, default 'inner'
-    * left: use only keys from left frame (SQL: left outer join)
-    * right: use only keys from right frame (SQL: right outer join)
-    * outer: use union of keys from both frames (SQL: full outer join)
-    * inner: use intersection of keys from both frames (SQL: inner join)
+    * left: use only keys from left frame, similar to a SQL left outer join;
+      preserve key order
+    * right: use only keys from right frame, similar to a SQL right outer join;
+      preserve key order
+    * outer: use union of keys from both frames, similar to a SQL full outer
+      join; sort keys lexicographically
+    * inner: use intersection of keys from both frames, similar to a SQL inner
+      join; preserve the order of the left keys
 on : label or list
     Field names to join on. Must be found in both DataFrames. If on is
     None and not merging on indexes, then it merges on the intersection of
@@ -147,7 +151,8 @@
     Use the index from the right DataFrame as the join key. Same caveats as
     left_index
 sort : boolean, default False
-    Sort the join keys lexicographically in the result DataFrame
+    Sort the join keys lexicographically in the result DataFrame. If False,
+    the order of the join keys depends on the join type (how keyword)
 suffixes : 2-length sequence (tuple, list, ...)
     Suffix to apply to overlapping column names in the left and right
     side, respectively
@@ -4472,16 +4477,18 @@ def join(self, other, on=None, how='left', lsuffix='', rsuffix='',
             * left: use calling frame's index (or column if on is specified)
             * right: use other frame's index
             * outer: form union of calling frame's index (or column if on is
-                specified) with other frame's index
+              specified) with other frame's index, and sort it
+              lexicographically
             * inner: form intersection of calling frame's index (or column if
-                on is specified) with other frame's index
+              on is specified) with other frame's index, preserving the order
+              of the calling's one
         lsuffix : string
             Suffix to use from left frame's overlapping columns
         rsuffix : string
             Suffix to use from right frame's overlapping columns
         sort : boolean, default False
             Order result DataFrame lexicographically by the join key. If False,
-            preserves the index order of the calling (left) DataFrame
+            the order of the join key depends on the join type (how keyword)
 
         Notes
         -----

pandas/core/groupby.py

@@ -12,8 +12,8 @@
 )
 
 from pandas import compat
-from pandas.compat.numpy import function as nv
-from pandas.compat.numpy import _np_version_under1p8
+from pandas.compat.numpy import function as nv, _np_version_under1p8
+from pandas.compat import set_function_name
 
 from pandas.types.common import (is_numeric_dtype,
                                  is_timedelta64_dtype, is_datetime64_dtype,
@@ -172,64 +172,6 @@
                                 'cummin', 'cummax'])
 
 
-def _groupby_function(name, alias, npfunc, numeric_only=True,
-                      _convert=False):
-
-    _local_template = "Compute %(f)s of group values"
-
-    @Substitution(name='groupby', f=name)
-    @Appender(_doc_template)
-    @Appender(_local_template)
-    def f(self, **kwargs):
-        if 'numeric_only' not in kwargs:
-            kwargs['numeric_only'] = numeric_only
-        self._set_group_selection()
-        try:
-            return self._cython_agg_general(alias, alt=npfunc, **kwargs)
-        except AssertionError as e:
-            raise SpecificationError(str(e))
-        except Exception:
-            result = self.aggregate(lambda x: npfunc(x, axis=self.axis))
-            if _convert:
-                result = result._convert(datetime=True)
-            return result
-
-        f.__name__ = name
-
-    return f
-
-
-def _first_compat(x, axis=0):
-
-    def _first(x):
-
-        x = np.asarray(x)
-        x = x[notnull(x)]
-        if len(x) == 0:
-            return np.nan
-        return x[0]
-
-    if isinstance(x, DataFrame):
-        return x.apply(_first, axis=axis)
-    else:
-        return _first(x)
-
-
-def _last_compat(x, axis=0):
-    def _last(x):
-
-        x = np.asarray(x)
-        x = x[notnull(x)]
-        if len(x) == 0:
-            return np.nan
-        return x[-1]
-
-    if isinstance(x, DataFrame):
-        return x.apply(_last, axis=axis)
-    else:
-        return _last(x)
-
-
 class Grouper(object):
     """
     A Grouper allows the user to specify a groupby instruction for a target
@@ -1184,14 +1126,76 @@ def size(self):
             result.name = getattr(self, 'name', None)
         return result
 
-    sum = _groupby_function('sum', 'add', np.sum)
-    prod = _groupby_function('prod', 'prod', np.prod)
-    min = _groupby_function('min', 'min', np.min, numeric_only=False)
-    max = _groupby_function('max', 'max', np.max, numeric_only=False)
-    first = _groupby_function('first', 'first', _first_compat,
-                              numeric_only=False, _convert=True)
-    last = _groupby_function('last', 'last', _last_compat, numeric_only=False,
-                             _convert=True)
+    @classmethod
+    def _add_numeric_operations(cls):
+        """ add numeric operations to the GroupBy generically """
+
+        def groupby_function(name, alias, npfunc,
+                             numeric_only=True, _convert=False):
+
+            _local_template = "Compute %(f)s of group values"
+
+            @Substitution(name='groupby', f=name)
+            @Appender(_doc_template)
+            @Appender(_local_template)
+            def f(self, **kwargs):
+                if 'numeric_only' not in kwargs:
+                    kwargs['numeric_only'] = numeric_only
+                self._set_group_selection()
+                try:
+                    return self._cython_agg_general(
+                        alias, alt=npfunc, **kwargs)
+                except AssertionError as e:
+                    raise SpecificationError(str(e))
+                except Exception:
+                    result = self.aggregate(
+                        lambda x: npfunc(x, axis=self.axis))
+                    if _convert:
+                        result = result._convert(datetime=True)
+                    return result
+
+            set_function_name(f, name, cls)
+
+            return f
+
+        def first_compat(x, axis=0):
+
+            def first(x):
+
+                x = np.asarray(x)
+                x = x[notnull(x)]
+                if len(x) == 0:
+                    return np.nan
+                return x[0]
+
+            if isinstance(x, DataFrame):
+                return x.apply(first, axis=axis)
+            else:
+                return first(x)
+
+        def last_compat(x, axis=0):
+
+            def last(x):
+
+                x = np.asarray(x)
+                x = x[notnull(x)]
+                if len(x) == 0:
+                    return np.nan
+                return x[-1]
+
+            if isinstance(x, DataFrame):
+                return x.apply(last, axis=axis)
+            else:
+                return last(x)
+
+        cls.sum = groupby_function('sum', 'add', np.sum)
+        cls.prod = groupby_function('prod', 'prod', np.prod)
+        cls.min = groupby_function('min', 'min', np.min, numeric_only=False)
+        cls.max = groupby_function('max', 'max', np.max, numeric_only=False)
+        cls.first = groupby_function('first', 'first', first_compat,
+                                     numeric_only=False, _convert=True)
+        cls.last = groupby_function('last', 'last', last_compat,
+                                    numeric_only=False, _convert=True)
 
     @Substitution(name='groupby')
     @Appender(_doc_template)
@@ -1604,6 +1608,9 @@ def tail(self, n=5):
         return self._selected_obj[mask]
 
 
+GroupBy._add_numeric_operations()
+
+
 @Appender(GroupBy.__doc__)
 def groupby(obj, by, **kwds):
     if isinstance(obj, Series):

pandas/indexes/base.py

@@ -2089,8 +2089,8 @@ def intersection(self, other):
         """
         Form the intersection of two Index objects.
 
-        This returns a new Index with elements common to the index and `other`.
-        Sortedness of the result is not guaranteed.
+        This returns a new Index with elements common to the index and `other`,
+        preserving the order of the calling index.
 
         Parameters
         ----------
@@ -2128,15 +2128,15 @@ def intersection(self, other):
                 pass
 
         try:
-            indexer = Index(self._values).get_indexer(other._values)
+            indexer = Index(other._values).get_indexer(self._values)
             indexer = indexer.take((indexer != -1).nonzero()[0])
         except:
             # duplicates
-            indexer = Index(self._values).get_indexer_non_unique(
-                other._values)[0].unique()
+            indexer = Index(other._values).get_indexer_non_unique(
+                self._values)[0].unique()
             indexer = indexer[indexer != -1]
 
-        taken = self.take(indexer)
+        taken = other.take(indexer)
         if self.name != other.name:
             taken.name = None
         return taken
@@ -2831,8 +2831,7 @@ def _reindex_non_unique(self, target):
         new_index = self._shallow_copy_with_infer(new_labels, freq=None)
         return new_index, indexer, new_indexer
 
-    def join(self, other, how='left', level=None, return_indexers=False):
-        """
+    _index_shared_docs['join'] = """
         *this is an internal non-public method*
 
         Compute join_index and indexers to conform data
@@ -2844,11 +2843,20 @@ def join(self, other, how='left', level=None, return_indexers=False):
         how : {'left', 'right', 'inner', 'outer'}
         level : int or level name, default None
         return_indexers : boolean, default False
+        sort : boolean, default False
+            Sort the join keys lexicographically in the result Index. If False,
+            the order of the join keys depends on the join type (how keyword)
+
+            .. versionadded:: 0.20.0
 
         Returns
         -------
         join_index, (left_indexer, right_indexer)
         """
+
+    @Appender(_index_shared_docs['join'])
+    def join(self, other, how='left', level=None, return_indexers=False,
+             sort=False):
         from .multi import MultiIndex
         self_is_mi = isinstance(self, MultiIndex)
         other_is_mi = isinstance(other, MultiIndex)
@@ -2929,6 +2937,9 @@ def join(self, other, how='left', level=None, return_indexers=False):
         elif how == 'outer':
             join_index = self.union(other)
 
+        if sort:
+            join_index = join_index.sort_values()
+
         if return_indexers:
             if join_index is self:
                 lindexer = None