diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index ee1f1b7be1b86..13a4a4531e6d0 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -104,7 +104,6 @@ Other enhancements - Let :meth:`DataFrame.to_feather` accept a non-default :class:`Index` and non-string column names (:issue:`51787`) - Performance improvement in :func:`read_csv` (:issue:`52632`) with ``engine="c"`` - :meth:`Categorical.from_codes` has gotten a ``validate`` parameter (:issue:`50975`) -- :meth:`DataFrame.stack` gained the ``sort`` keyword to dictate whether the resulting :class:`MultiIndex` levels are sorted (:issue:`15105`) - :meth:`DataFrame.unstack` gained the ``sort`` keyword to dictate whether the resulting :class:`MultiIndex` levels are sorted (:issue:`15105`) - :meth:`DataFrameGroupby.agg` and :meth:`DataFrameGroupby.transform` now support grouping by multiple keys when the index is not a :class:`MultiIndex` for ``engine="numba"`` (:issue:`53486`) - :meth:`Series.explode` now supports pyarrow-backed list types (:issue:`53602`) @@ -501,7 +500,8 @@ Reshaping - Bug in :meth:`DataFrame.idxmin` and :meth:`DataFrame.idxmax`, where the axis dtype would be lost for empty frames (:issue:`53265`) - Bug in :meth:`DataFrame.merge` not merging correctly when having ``MultiIndex`` with single level (:issue:`52331`) - Bug in :meth:`DataFrame.stack` losing extension dtypes when columns is a :class:`MultiIndex` and frame contains mixed dtypes (:issue:`45740`) -- Bug in :meth:`DataFrame.stack` sorting columns lexicographically (:issue:`53786`) +- Bug in :meth:`DataFrame.stack` sorting columns lexicographically in rare cases (:issue:`53786`) +- Bug in :meth:`DataFrame.stack` sorting index lexicographically in rare cases (:issue:`53824`) - Bug in :meth:`DataFrame.transpose` inferring dtype for object column (:issue:`51546`) - Bug in :meth:`Series.combine_first` converting ``int64`` dtype to ``float64`` and losing precision on very large integers (:issue:`51764`) - diff --git a/pandas/core/frame.py b/pandas/core/frame.py index c7e8f74ff7849..d12c07b3caca4 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -9010,7 +9010,7 @@ def pivot_table( sort=sort, ) - def stack(self, level: IndexLabel = -1, dropna: bool = True, sort: bool = True): + def stack(self, level: IndexLabel = -1, dropna: bool = True): """ Stack the prescribed level(s) from columns to index. @@ -9036,8 +9036,6 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True, sort: bool = True): axis can create combinations of index and column values that are missing from the original dataframe. See Examples section. - sort : bool, default True - Whether to sort the levels of the resulting MultiIndex. Returns ------- @@ -9137,15 +9135,15 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True, sort: bool = True): >>> df_multi_level_cols2.stack(0) kg m - cat height NaN 2.0 - weight 1.0 NaN - dog height NaN 4.0 - weight 3.0 NaN + cat weight 1.0 NaN + height NaN 2.0 + dog weight 3.0 NaN + height NaN 4.0 >>> df_multi_level_cols2.stack([0, 1]) - cat height m 2.0 - weight kg 1.0 - dog height m 4.0 - weight kg 3.0 + cat weight kg 1.0 + height m 2.0 + dog weight kg 3.0 + height m 4.0 dtype: float64 **Dropping missing values** @@ -9181,9 +9179,9 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True, sort: bool = True): ) if isinstance(level, (tuple, list)): - result = stack_multiple(self, level, dropna=dropna, sort=sort) + result = stack_multiple(self, level, dropna=dropna) else: - result = stack(self, level, dropna=dropna, sort=sort) + result = stack(self, level, dropna=dropna) return result.__finalize__(self, method="stack") diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py index 5deaa41e2f63c..b0c74745511c4 100644 --- a/pandas/core/reshape/reshape.py +++ b/pandas/core/reshape/reshape.py @@ -1,6 +1,5 @@ from __future__ import annotations -import itertools from typing import ( TYPE_CHECKING, cast, @@ -499,7 +498,7 @@ def unstack(obj: Series | DataFrame, level, fill_value=None, sort: bool = True): if isinstance(obj.index, MultiIndex): return _unstack_frame(obj, level, fill_value=fill_value, sort=sort) else: - return obj.T.stack(dropna=False, sort=sort) + return obj.T.stack(dropna=False) elif not isinstance(obj.index, MultiIndex): # GH 36113 # Give nicer error messages when unstack a Series whose @@ -572,7 +571,7 @@ def _unstack_extension_series( return result -def stack(frame: DataFrame, level=-1, dropna: bool = True, sort: bool = True): +def stack(frame: DataFrame, level=-1, dropna: bool = True): """ Convert DataFrame to Series with multi-level Index. Columns become the second level of the resulting hierarchical index @@ -594,9 +593,7 @@ def factorize(index): level_num = frame.columns._get_level_number(level) if isinstance(frame.columns, MultiIndex): - return _stack_multi_columns( - frame, level_num=level_num, dropna=dropna, sort=sort - ) + return _stack_multi_columns(frame, level_num=level_num, dropna=dropna) elif isinstance(frame.index, MultiIndex): new_levels = list(frame.index.levels) new_codes = [lab.repeat(K) for lab in frame.index.codes] @@ -649,13 +646,13 @@ def factorize(index): return frame._constructor_sliced(new_values, index=new_index) -def stack_multiple(frame: DataFrame, level, dropna: bool = True, sort: bool = True): +def stack_multiple(frame: DataFrame, level, dropna: bool = True): # If all passed levels match up to column names, no # ambiguity about what to do if all(lev in frame.columns.names for lev in level): result = frame for lev in level: - result = stack(result, lev, dropna=dropna, sort=sort) + result = stack(result, lev, dropna=dropna) # Otherwise, level numbers may change as each successive level is stacked elif all(isinstance(lev, int) for lev in level): @@ -668,7 +665,7 @@ def stack_multiple(frame: DataFrame, level, dropna: bool = True, sort: bool = Tr while level: lev = level.pop(0) - result = stack(result, lev, dropna=dropna, sort=sort) + result = stack(result, lev, dropna=dropna) # Decrement all level numbers greater than current, as these # have now shifted down by one level = [v if v <= lev else v - 1 for v in level] @@ -694,7 +691,14 @@ def _stack_multi_column_index(columns: MultiIndex) -> MultiIndex: # Remove duplicate tuples in the MultiIndex. tuples = zip(*levs) - unique_tuples = (key for key, _ in itertools.groupby(tuples)) + seen = set() + # mypy doesn't like our trickery to get `set.add` to work in a comprehension + # error: "add" of "set" does not return a value + unique_tuples = ( + key + for key in tuples + if not (key in seen or seen.add(key)) # type: ignore[func-returns-value] + ) new_levs = zip(*unique_tuples) # The dtype of each level must be explicitly set to avoid inferring the wrong type. @@ -710,7 +714,7 @@ def _stack_multi_column_index(columns: MultiIndex) -> MultiIndex: def _stack_multi_columns( - frame: DataFrame, level_num: int = -1, dropna: bool = True, sort: bool = True + frame: DataFrame, level_num: int = -1, dropna: bool = True ) -> DataFrame: def _convert_level_number(level_num: int, columns: Index): """ @@ -740,23 +744,12 @@ def _convert_level_number(level_num: int, columns: Index): roll_columns = roll_columns.swaplevel(lev1, lev2) this.columns = mi_cols = roll_columns - if not mi_cols._is_lexsorted() and sort: - # Workaround the edge case where 0 is one of the column names, - # which interferes with trying to sort based on the first - # level - level_to_sort = _convert_level_number(0, mi_cols) - this = this.sort_index(level=level_to_sort, axis=1) - mi_cols = this.columns - - mi_cols = cast(MultiIndex, mi_cols) new_columns = _stack_multi_column_index(mi_cols) # time to ravel the values new_data = {} level_vals = mi_cols.levels[-1] level_codes = unique(mi_cols.codes[-1]) - if sort: - level_codes = np.sort(level_codes) level_vals_nan = level_vals.insert(len(level_vals), None) level_vals_used = np.take(level_vals_nan, level_codes) @@ -764,7 +757,9 @@ def _convert_level_number(level_num: int, columns: Index): drop_cols = [] for key in new_columns: try: - loc = this.columns.get_loc(key) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", PerformanceWarning) + loc = this.columns.get_loc(key) except KeyError: drop_cols.append(key) continue @@ -774,9 +769,12 @@ def _convert_level_number(level_num: int, columns: Index): # but if unsorted can get a boolean # indexer if not isinstance(loc, slice): - slice_len = len(loc) + slice_len = loc.sum() else: slice_len = loc.stop - loc.start + if loc.step is not None: + # Integer division using ceiling instead of floor + slice_len = -(slice_len // -loc.step) if slice_len != levsize: chunk = this.loc[:, this.columns[loc]] diff --git a/pandas/tests/frame/test_stack_unstack.py b/pandas/tests/frame/test_stack_unstack.py index a48728a778877..ffdcb06ee2847 100644 --- a/pandas/tests/frame/test_stack_unstack.py +++ b/pandas/tests/frame/test_stack_unstack.py @@ -1099,7 +1099,7 @@ def test_stack_preserve_categorical_dtype(self, ordered, labels): "labels,data", [ (list("xyz"), [10, 11, 12, 13, 14, 15]), - (list("zyx"), [14, 15, 12, 13, 10, 11]), + (list("zyx"), [10, 11, 12, 13, 14, 15]), ], ) def test_stack_multi_preserve_categorical_dtype(self, ordered, labels, data): @@ -1107,10 +1107,10 @@ def test_stack_multi_preserve_categorical_dtype(self, ordered, labels, data): cidx = pd.CategoricalIndex(labels, categories=sorted(labels), ordered=ordered) cidx2 = pd.CategoricalIndex(["u", "v"], ordered=ordered) midx = MultiIndex.from_product([cidx, cidx2]) - df = DataFrame([sorted(data)], columns=midx) + df = DataFrame([data], columns=midx) result = df.stack([0, 1]) - s_cidx = pd.CategoricalIndex(sorted(labels), ordered=ordered) + s_cidx = pd.CategoricalIndex(labels, ordered=ordered) expected = Series(data, index=MultiIndex.from_product([[0], s_cidx, cidx2])) tm.assert_series_equal(result, expected) @@ -1400,8 +1400,8 @@ def test_unstack_non_slice_like_blocks(using_array_manager): tm.assert_frame_equal(res, expected) -def test_stack_sort_false(): - # GH 15105 +def test_stack_nosort(): + # GH 15105, GH 53825 data = [[1, 2, 3.0, 4.0], [2, 3, 4.0, 5.0], [3, 4, np.nan, np.nan]] df = DataFrame( data, @@ -1409,7 +1409,7 @@ def test_stack_sort_false(): levels=[["B", "A"], ["x", "y"]], codes=[[0, 0, 1, 1], [0, 1, 0, 1]] ), ) - result = df.stack(level=0, sort=False) + result = df.stack(level=0) expected = DataFrame( {"x": [1.0, 3.0, 2.0, 4.0, 3.0], "y": [2.0, 4.0, 3.0, 5.0, 4.0]}, index=MultiIndex.from_arrays([[0, 0, 1, 1, 2], ["B", "A", "B", "A", "B"]]), @@ -1421,15 +1421,15 @@ def test_stack_sort_false(): data, columns=MultiIndex.from_arrays([["B", "B", "A", "A"], ["x", "y", "x", "y"]]), ) - result = df.stack(level=0, sort=False) + result = df.stack(level=0) tm.assert_frame_equal(result, expected) -def test_stack_sort_false_multi_level(): - # GH 15105 +def test_stack_nosort_multi_level(): + # GH 15105, GH 53825 idx = MultiIndex.from_tuples([("weight", "kg"), ("height", "m")]) df = DataFrame([[1.0, 2.0], [3.0, 4.0]], index=["cat", "dog"], columns=idx) - result = df.stack([0, 1], sort=False) + result = df.stack([0, 1]) expected_index = MultiIndex.from_tuples( [ ("cat", "weight", "kg"), @@ -1999,13 +1999,12 @@ def __init__(self, *args, **kwargs) -> None: ), ) @pytest.mark.parametrize("stack_lev", range(2)) - @pytest.mark.parametrize("sort", [True, False]) - def test_stack_order_with_unsorted_levels(self, levels, stack_lev, sort): + def test_stack_order_with_unsorted_levels(self, levels, stack_lev): # GH#16323 # deep check for 1-row case columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)]) - df_stacked = df.stack(stack_lev, sort=sort) + df_stacked = df.stack(stack_lev) for row in df.index: for col in df.columns: expected = df.loc[row, col] @@ -2037,7 +2036,7 @@ def test_stack_order_with_unsorted_levels_multi_row_2(self): stack_lev = 1 columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)], index=[1, 0, 2, 3]) - result = df.stack(stack_lev, sort=True) + result = df.stack(stack_lev) expected_index = MultiIndex( levels=[[0, 1, 2, 3], [0, 1]], codes=[[1, 1, 0, 0, 2, 2, 3, 3], [1, 0, 1, 0, 1, 0, 1, 0]],