Merge pull request #1 from pandas-dev/main

UPDATE

Author: zain581

.pre-commit-config.yaml

@@ -226,6 +226,13 @@ repos:
         entry: python scripts/no_bool_in_generic.py
         language: python
         files: ^pandas/core/generic\.py$
+    -   id: no-return-exception
+        name: Use raise instead of return for exceptions
+        language: pygrep
+        entry: 'return [A-Za-z]+(Error|Exit|Interrupt|Exception|Iteration)'
+        files: ^pandas/
+        types: [python]
+        exclude: ^pandas/tests/
     -   id: pandas-errors-documented
         name: Ensure pandas errors are documented in doc/source/reference/testing.rst
         entry: python scripts/pandas_errors_documented.py

doc/source/whatsnew/v1.6.0.rst

@@ -220,6 +220,7 @@ Missing
 
 MultiIndex
 ^^^^^^^^^^
+- Bug in :meth:`MultiIndex.argsort` raising ``TypeError`` when index contains :attr:`NA` (:issue:`48495`)
 - Bug in :meth:`MultiIndex.difference` losing extension array dtype (:issue:`48606`)
 - Bug in :class:`MultiIndex.set_levels` raising ``IndexError`` when setting empty level (:issue:`48636`)
 - Bug in :meth:`MultiIndex.unique` losing extension array dtype (:issue:`48335`)

environment.yml

@@ -100,7 +100,7 @@ dependencies:
   - natsort  # DataFrame.sort_values doctest
   - numpydoc
   - pandas-dev-flaker=0.5.0
-  - pydata-sphinx-theme
+  - pydata-sphinx-theme<0.11
   - pytest-cython  # doctest
   - sphinx
   - sphinx-panels

pandas/core/indexes/base.py

@@ -554,7 +554,7 @@ def __new__(
             return klass._simple_new(arr, name)
 
         elif is_scalar(data):
-            raise cls._scalar_data_error(data)
+            raise cls._raise_scalar_data_error(data)
         elif hasattr(data, "__array__"):
             return Index(np.asarray(data), dtype=dtype, copy=copy, name=name, **kwargs)
         else:
@@ -4386,14 +4386,23 @@ def is_int(v):
         return indexer
 
     @final
-    def _invalid_indexer(self, form: str_t, key) -> TypeError:
+    def _raise_invalid_indexer(
+        self,
+        form: str_t,
+        key,
+        reraise: lib.NoDefault | None | Exception = lib.no_default,
+    ) -> None:
         """
-        Consistent invalid indexer message.
+        Raise consistent invalid indexer message.
         """
-        return TypeError(
+        msg = (
             f"cannot do {form} indexing on {type(self).__name__} with these "
             f"indexers [{key}] of type {type(key).__name__}"
         )
+        if reraise is not lib.no_default:
+            raise TypeError(msg) from reraise
+        else:
+            raise TypeError(msg)
 
     # --------------------------------------------------------------------
     # Reindex Methods
@@ -5279,10 +5288,10 @@ def where(self, cond, other=None) -> Index:
     # construction helpers
     @final
     @classmethod
-    def _scalar_data_error(cls, data):
+    def _raise_scalar_data_error(cls, data):
         # We return the TypeError so that we can raise it from the constructor
         #  in order to keep mypy happy
-        return TypeError(
+        raise TypeError(
             f"{cls.__name__}(...) must be called with a collection of some "
             f"kind, {repr(data)} was passed"
         )
@@ -6674,15 +6683,15 @@ def _maybe_cast_listlike_indexer(self, target) -> Index:
         return ensure_index(target)
 
     @final
-    def _validate_indexer(self, form: str_t, key, kind: str_t):
+    def _validate_indexer(self, form: str_t, key, kind: str_t) -> None:
         """
         If we are positional indexer, validate that we have appropriate
         typed bounds must be an integer.
         """
         assert kind in ["getitem", "iloc"]
 
         if key is not None and not is_integer(key):
-            raise self._invalid_indexer(form, key)
+            self._raise_invalid_indexer(form, key)
 
     def _maybe_cast_slice_bound(self, label, side: str_t, kind=no_default):
         """
@@ -6714,7 +6723,7 @@ def _maybe_cast_slice_bound(self, label, side: str_t, kind=no_default):
         # datetimelike Indexes
         # reject them, if index does not contain label
         if (is_float(label) or is_integer(label)) and label not in self:
-            raise self._invalid_indexer("slice", label)
+            self._raise_invalid_indexer("slice", label)
 
         return label
 

pandas/core/indexes/category.py

@@ -230,7 +230,7 @@ def __new__(
             data = []
 
         if is_scalar(data):
-            raise cls._scalar_data_error(data)
+            cls._raise_scalar_data_error(data)
 
         data = Categorical(
             data, categories=categories, ordered=ordered, dtype=dtype, copy=copy

pandas/core/indexes/datetimelike.py

@@ -314,12 +314,12 @@ def _maybe_cast_slice_bound(self, label, side: str, kind=lib.no_default):
                 # DTI -> parsing.DateParseError
                 # TDI -> 'unit abbreviation w/o a number'
                 # PI -> string cannot be parsed as datetime-like
-                raise self._invalid_indexer("slice", label) from err
+                self._raise_invalid_indexer("slice", label, err)
 
             lower, upper = self._parsed_string_to_bounds(reso, parsed)
             return lower if side == "left" else upper
         elif not isinstance(label, self._data._recognized_scalars):
-            raise self._invalid_indexer("slice", label)
+            self._raise_invalid_indexer("slice", label)
 
         return label
 

pandas/core/indexes/datetimes.py

@@ -334,7 +334,7 @@ def __new__(
     ) -> DatetimeIndex:
 
         if is_scalar(data):
-            raise cls._scalar_data_error(data)
+            cls._raise_scalar_data_error(data)
 
         # - Cases checked above all return/raise before reaching here - #
 

pandas/core/indexes/multi.py

@@ -1952,7 +1952,7 @@ def _lexsort_depth(self) -> int:
             return self.sortorder
         return _lexsort_depth(self.codes, self.nlevels)
 
-    def _sort_levels_monotonic(self) -> MultiIndex:
+    def _sort_levels_monotonic(self, raise_if_incomparable: bool = False) -> MultiIndex:
         """
         This is an *internal* function.
 
@@ -1999,7 +1999,8 @@ def _sort_levels_monotonic(self) -> MultiIndex:
                     # indexer to reorder the levels
                     indexer = lev.argsort()
                 except TypeError:
-                    pass
+                    if raise_if_incomparable:
+                        raise
                 else:
                     lev = lev.take(indexer)
 
@@ -2245,9 +2246,9 @@ def append(self, other):
 
     def argsort(self, *args, **kwargs) -> npt.NDArray[np.intp]:
         if len(args) == 0 and len(kwargs) == 0:
-            # np.lexsort is significantly faster than self._values.argsort()
-            values = [self._get_level_values(i) for i in reversed(range(self.nlevels))]
-            return np.lexsort(values)
+            # lexsort is significantly faster than self._values.argsort()
+            target = self._sort_levels_monotonic(raise_if_incomparable=True)
+            return lexsort_indexer(target._get_codes_for_sorting())
         return self._values.argsort(*args, **kwargs)
 
     @Appender(_index_shared_docs["repeat"] % _index_doc_kwargs)

pandas/core/indexes/numeric.py

@@ -140,7 +140,7 @@ def _ensure_array(cls, data, dtype, copy: bool):
         if not isinstance(data, (np.ndarray, Index)):
             # Coerce to ndarray if not already ndarray or Index
             if is_scalar(data):
-                raise cls._scalar_data_error(data)
+                cls._raise_scalar_data_error(data)
 
             # other iterable of some kind
             if not isinstance(data, (ABCSeries, list, tuple)):

pandas/core/indexes/period.py

@@ -240,7 +240,7 @@ def __new__(
             # range-based.
             if not fields:
                 # test_pickle_compat_construction
-                raise cls._scalar_data_error(None)
+                cls._raise_scalar_data_error(None)
 
             data, freq2 = PeriodArray._generate_range(None, None, None, freq, fields)
             # PeriodArray._generate range does validation that fields is

pandas/core/indexes/timedeltas.py

@@ -128,7 +128,7 @@ def __new__(
         name = maybe_extract_name(name, data, cls)
 
         if is_scalar(data):
-            raise cls._scalar_data_error(data)
+            cls._raise_scalar_data_error(data)
 
         if unit in {"Y", "y", "M"}:
             raise ValueError(

pandas/core/internals/managers.py

@@ -1038,7 +1038,7 @@ def _verify_integrity(self) -> None:
         tot_items = sum(len(x.mgr_locs) for x in self.blocks)
         for block in self.blocks:
             if block.shape[1:] != mgr_shape[1:]:
-                raise construction_error(tot_items, block.shape[1:], self.axes)
+                raise_construction_error(tot_items, block.shape[1:], self.axes)
         if len(self.items) != tot_items:
             raise AssertionError(
                 "Number of manager items must equal union of "
@@ -2145,7 +2145,7 @@ def create_block_manager_from_blocks(
     except ValueError as err:
         arrays = [blk.values for blk in blocks]
         tot_items = sum(arr.shape[0] for arr in arrays)
-        raise construction_error(tot_items, arrays[0].shape[1:], axes, err)
+        raise_construction_error(tot_items, arrays[0].shape[1:], axes, err)
 
     if consolidate:
         mgr._consolidate_inplace()
@@ -2172,13 +2172,13 @@ def create_block_manager_from_column_arrays(
         blocks = _form_blocks(arrays, consolidate)
         mgr = BlockManager(blocks, axes, verify_integrity=False)
     except ValueError as e:
-        raise construction_error(len(arrays), arrays[0].shape, axes, e)
+        raise_construction_error(len(arrays), arrays[0].shape, axes, e)
     if consolidate:
         mgr._consolidate_inplace()
     return mgr
 
 
-def construction_error(
+def raise_construction_error(
     tot_items: int,
     block_shape: Shape,
     axes: list[Index],
@@ -2198,10 +2198,10 @@ def construction_error(
     # We return the exception object instead of raising it so that we
     #  can raise it in the caller; mypy plays better with that
     if passed == implied and e is not None:
-        return e
+        raise e
     if block_shape[0] == 0:
-        return ValueError("Empty data passed with indices specified.")
-    return ValueError(f"Shape of passed values is {passed}, indices imply {implied}")
+        raise ValueError("Empty data passed with indices specified.")
+    raise ValueError(f"Shape of passed values is {passed}, indices imply {implied}")
 
 
 # -----------------------------------------------------------------------

pandas/io/pytables.py

@@ -1658,13 +1658,6 @@ def _create_storer(
         if value is not None and not isinstance(value, (Series, DataFrame)):
             raise TypeError("value must be None, Series, or DataFrame")
 
-        def error(t):
-            # return instead of raising so mypy can tell where we are raising
-            return TypeError(
-                f"cannot properly create the storer for: [{t}] [group->"
-                f"{group},value->{type(value)},format->{format}"
-            )
-
         pt = _ensure_decoded(getattr(group._v_attrs, "pandas_type", None))
         tt = _ensure_decoded(getattr(group._v_attrs, "table_type", None))
 
@@ -1699,7 +1692,10 @@ def error(t):
             try:
                 cls = _STORER_MAP[pt]
             except KeyError as err:
-                raise error("_STORER_MAP") from err
+                raise TypeError(
+                    f"cannot properly create the storer for: [_STORER_MAP] [group->"
+                    f"{group},value->{type(value)},format->{format}"
+                ) from err
             return cls(self, group, encoding=encoding, errors=errors)
 
         # existing node (and must be a table)
@@ -1732,7 +1728,10 @@ def error(t):
         try:
             cls = _TABLE_MAP[tt]
         except KeyError as err:
-            raise error("_TABLE_MAP") from err
+            raise TypeError(
+                f"cannot properly create the storer for: [_TABLE_MAP] [group->"
+                f"{group},value->{type(value)},format->{format}"
+            ) from err
 
         return cls(self, group, encoding=encoding, errors=errors)
 

pandas/tests/indexes/multi/test_sorting.py

@@ -14,6 +14,7 @@
     Index,
     MultiIndex,
     RangeIndex,
+    Timestamp,
 )
 import pandas._testing as tm
 from pandas.core.indexes.frozen import FrozenList
@@ -280,3 +281,26 @@ def test_remove_unused_levels_with_nan():
     result = idx.levels
     expected = FrozenList([["a", np.nan], [4]])
     assert str(result) == str(expected)
+
+
+def test_sort_values_nan():
+    # GH48495, GH48626
+    midx = MultiIndex(levels=[["A", "B", "C"], ["D"]], codes=[[1, 0, 2], [-1, -1, 0]])
+    result = midx.sort_values()
+    expected = MultiIndex(
+        levels=[["A", "B", "C"], ["D"]], codes=[[0, 1, 2], [-1, -1, 0]]
+    )
+    tm.assert_index_equal(result, expected)
+
+
+def test_sort_values_incomparable():
+    # GH48495
+    mi = MultiIndex.from_arrays(
+        [
+            [1, Timestamp("2000-01-01")],
+            [3, 4],
+        ]
+    )
+    match = "'<' not supported between instances of 'Timestamp' and 'int'"
+    with pytest.raises(TypeError, match=match):
+        mi.sort_values()

pandas/tests/indexing/multiindex/test_sorted.py

@@ -2,9 +2,11 @@
 import pytest
 
 from pandas import (
+    NA,
     DataFrame,
     MultiIndex,
     Series,
+    array,
 )
 import pandas._testing as tm
 
@@ -86,6 +88,36 @@ def test_sort_values_key(self):
 
         tm.assert_frame_equal(result, expected)
 
+    def test_argsort_with_na(self):
+        # GH48495
+        arrays = [
+            array([2, NA, 1], dtype="Int64"),
+            array([1, 2, 3], dtype="Int64"),
+        ]
+        index = MultiIndex.from_arrays(arrays)
+        result = index.argsort()
+        expected = np.array([2, 0, 1], dtype=np.intp)
+        tm.assert_numpy_array_equal(result, expected)
+
+    def test_sort_values_with_na(self):
+        # GH48495
+        arrays = [
+            array([2, NA, 1], dtype="Int64"),
+            array([1, 2, 3], dtype="Int64"),
+        ]
+        index = MultiIndex.from_arrays(arrays)
+        index = index.sort_values()
+        result = DataFrame(range(3), index=index)
+
+        arrays = [
+            array([1, 2, NA], dtype="Int64"),
+            array([3, 1, 2], dtype="Int64"),
+        ]
+        index = MultiIndex.from_arrays(arrays)
+        expected = DataFrame(range(3), index=index)
+
+        tm.assert_frame_equal(result, expected)
+
     def test_frame_getitem_not_sorted(self, multiindex_dataframe_random_data):
         frame = multiindex_dataframe_random_data
         df = frame.T