BUG: construct MultiIndex identically from levels/labels when concatting

closes pandas-dev#15622
closes pandas-dev#15687
closes pandas-dev#14015

Author: jreback

doc/source/whatsnew/v0.20.0.txt

@@ -818,6 +818,7 @@ Bug Fixes
 - Bug in ``pd.qcut()`` with a single quantile and an array with identical values (:issue:`15431`)
 - Compat with SciPy 0.19.0 for testing on ``.interpolate()`` (:issue:`15662`)
 
+- Bug in ``DataFrame.sort_index()`` that would not sort a lexsorted, but non monotonic ``MultiIndex`` (:issue:`15622`, :issue:`15687`, :issue:`14015`)
 
 - Bug in the display of ``.info()`` where a qualifier (+) would always be displayed with a ``MultiIndex`` that contains only non-strings (:issue:`15245`)
 

pandas/core/frame.py

@@ -3389,8 +3389,7 @@ def sort_index(self, axis=0, level=None, ascending=True, inplace=False,
 
             # make sure that the axis is lexsorted to start
             # if not we need to reconstruct to get the correct indexer
-            if not labels.is_lexsorted():
-                labels = MultiIndex.from_tuples(labels.values)
+            labels = labels._reconstruct_as_sorted()
 
             indexer = lexsort_indexer(labels.labels, orders=ascending,
                                       na_position=na_position)
@@ -3410,8 +3409,9 @@ def sort_index(self, axis=0, level=None, ascending=True, inplace=False,
             indexer = nargsort(labels, kind=kind, ascending=ascending,
                                na_position=na_position)
 
+        baxis = self._get_block_manager_axis(axis)
         new_data = self._data.take(indexer,
-                                   axis=self._get_block_manager_axis(axis),
+                                   axis=baxis,
                                    convert=False, verify=False)
 
         if inplace:

pandas/core/groupby.py

@@ -1807,6 +1807,13 @@ def get_group_levels(self):
         'ohlc': lambda *args: ['open', 'high', 'low', 'close']
     }
 
+    def _is_builtin_func(self, arg):
+        """
+        if we define an builtin function for this argument, return it,
+        otherwise return the arg
+        """
+        return SelectionMixin._builtin_table.get(arg, arg)
+
     def _get_cython_function(self, kind, how, values, is_numeric):
 
         dtype_str = values.dtype.name
@@ -2032,7 +2039,7 @@ def _aggregate_series_fast(self, obj, func):
         # avoids object / Series creation overhead
         dummy = obj._get_values(slice(None, 0)).to_dense()
         indexer = get_group_index_sorter(group_index, ngroups)
-        obj = obj.take(indexer, convert=False)
+        obj = obj.take(indexer, convert=False).to_dense()
         group_index = algorithms.take_nd(
             group_index, indexer, allow_fill=False)
         grouper = lib.SeriesGrouper(obj, func, group_index, ngroups,

pandas/core/reshape.py

@@ -22,8 +22,8 @@
 from pandas.sparse.libsparse import IntIndex
 
 from pandas.core.categorical import Categorical, _factorize_from_iterable
-from pandas.core.sorting import (get_group_index, compress_group_index,
-                                 decons_obs_group_ids)
+from pandas.core.sorting import (get_group_index, get_compressed_ids,
+                                 compress_group_index, decons_obs_group_ids)
 
 import pandas.core.algorithms as algos
 from pandas._libs import algos as _algos, reshape as _reshape
@@ -494,11 +494,6 @@ def _unstack_frame(obj, level, fill_value=None):
         return unstacker.get_result()
 
 
-def get_compressed_ids(labels, sizes):
-    ids = get_group_index(labels, sizes, sort=True, xnull=False)
-    return compress_group_index(ids, sort=True)
-
-
 def stack(frame, level=-1, dropna=True):
     """
     Convert DataFrame to Series with multi-level Index. Columns become the

pandas/core/sorting.py

@@ -93,6 +93,11 @@ def maybe_lift(lab, size):  # pormote nan values
     return loop(list(labels), list(shape))
 
 
+def get_compressed_ids(labels, sizes):
+    ids = get_group_index(labels, sizes, sort=True, xnull=False)
+    return compress_group_index(ids, sort=True)
+
+
 def is_int64_overflow_possible(shape):
     the_prod = long(1)
     for x in shape:

pandas/indexes/multi.py

@@ -1175,9 +1175,34 @@ def from_product(cls, iterables, sortorder=None, names=None):
 
         labels, levels = _factorize_from_iterables(iterables)
         labels = cartesian_product(labels)
+        return MultiIndex(levels, labels, sortorder=sortorder, names=names)
 
-        return MultiIndex(levels=levels, labels=labels, sortorder=sortorder,
-                          names=names)
+    def _reconstruct_as_sorted(self):
+        """
+        reconstruct the MultiIndex, such that we are
+        monotonically sorted; this will also ensure that
+        we are lexsorted
+        """
+        if self.is_lexsorted() and self.is_monotonic:
+            return self
+
+        new_levels = []
+        new_labels = []
+        for lev, lab in zip(self.levels, self.labels):
+
+            if lev.is_monotonic:
+                new_levels.append(lev)
+                new_labels.append(lab)
+                continue
+
+            indexer = lev.argsort()
+            lev = lev.take(indexer)
+            lab = algos.take_1d(indexer, lab)
+
+            new_levels.append(lev)
+            new_labels.append(lab)
+
+        return MultiIndex(new_levels, new_labels, names=self.names)
 
     @property
     def nlevels(self):