Merge remote-tracking branch 'upstream/main' into ref/is_range_indexer/step

Author: mroeschke

doc/source/whatsnew/v3.0.0.rst

@@ -272,11 +272,11 @@ Performance improvements
 - Performance improvement in :meth:`Index.join` by propagating cached attributes in cases where the result matches one of the inputs (:issue:`57023`)
 - Performance improvement in :meth:`Index.take` when ``indices`` is a full range indexer from zero to length of index (:issue:`56806`)
 - Performance improvement in :meth:`MultiIndex.equals` for equal length indexes (:issue:`56990`)
-- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`)
+- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask or integers returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`)
 - Performance improvement in :meth:`RangeIndex.append` when appending the same index (:issue:`57252`)
-- Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`)
-- Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`)
-- Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`)
+- Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`, :issue:`57752`)
+- Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`, :issue:`57752`)
+- Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`, :issue:`57752`)
 - Performance improvement in ``DataFrameGroupBy.__len__`` and ``SeriesGroupBy.__len__`` (:issue:`57595`)
 - Performance improvement in indexing operations for string dtypes (:issue:`56997`)
 
@@ -289,6 +289,7 @@ Bug fixes
 - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
 - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
 - Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`)
+- Fixed bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
 - Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
 - Fixed bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
 
@@ -393,6 +394,7 @@ Other
 ^^^^^
 - Bug in :class:`DataFrame` when passing a ``dict`` with a NA scalar and ``columns`` that would always return ``np.nan`` (:issue:`57205`)
 - Bug in :func:`tseries.api.guess_datetime_format` would fail to infer time format when "%Y" == "%H%M" (:issue:`57452`)
+- Bug in :func:`unique` on :class:`Index` not always returning :class:`Index` (:issue:`57043`)
 - Bug in :meth:`DataFrame.sort_index` when passing ``axis="columns"`` and ``ignore_index=True`` and ``ascending=False`` not returning a :class:`RangeIndex` columns (:issue:`57293`)
 - Bug in :meth:`DataFrame.where` where using a non-bool type array in the function would return a ``ValueError`` instead of a ``TypeError`` (:issue:`56330`)
 - Bug in Dataframe Interchange Protocol implementation was returning incorrect results for data buffers' associated dtype, for string and datetime columns (:issue:`54781`)

environment.yml

@@ -62,6 +62,7 @@ dependencies:
   # downstream packages
   - dask-core
   - seaborn-base
+  - dask-expr
 
   # local testing dependencies
   - moto

pandas/core/algorithms.py

@@ -439,6 +439,10 @@ def unique_with_mask(values, mask: npt.NDArray[np.bool_] | None = None):
         # Dispatch to extension dtype's unique.
         return values.unique()
 
+    if isinstance(values, ABCIndex):
+        # Dispatch to Index's unique.
+        return values.unique()
+
     original = values
     hashtable, values = _get_hashtable_algo(values)
 

pandas/core/groupby/generic.py

@@ -1642,8 +1642,11 @@ def _wrap_applied_output(
         first_not_none = next(com.not_none(*values), None)
 
         if first_not_none is None:
-            # GH9684 - All values are None, return an empty frame.
-            return self.obj._constructor()
+            # GH9684 - All values are None, return an empty frame
+            # GH57775 - Ensure that columns and dtypes from original frame are kept.
+            result = self.obj._constructor(columns=data.columns)
+            result = result.astype(data.dtypes)
+            return result
         elif isinstance(first_not_none, DataFrame):
             return self._concat_objects(
                 values,

pandas/core/groupby/groupby.py

@@ -1636,6 +1636,14 @@ def apply(self, func, *args, include_groups: bool = True, **kwargs) -> NDFrameT:
         a    5
         b    2
         dtype: int64
+
+        Example 4: The function passed to ``apply`` returns ``None`` for one of the
+        group. This group is filtered from the result:
+
+        >>> g1.apply(lambda x: None if x.iloc[0, 0] == 3 else x, include_groups=False)
+           B  C
+        0  1  4
+        1  2  6
         """
         if isinstance(func, str):
             if hasattr(self, func):

pandas/core/indexes/base.py

@@ -4235,7 +4235,6 @@ def join(
 
         return self._join_via_get_indexer(other, how, sort)
 
-    @final
     def _join_empty(
         self, other: Index, how: JoinHow, sort: bool
     ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:

pandas/core/indexes/multi.py

@@ -2063,7 +2063,7 @@ def remove_unused_levels(self) -> MultiIndex:
 
         >>> mi2 = mi[2:].remove_unused_levels()
         >>> mi2.levels
-        (Index([1], dtype='int64'), Index(['a', 'b'], dtype='object'))
+        (RangeIndex(start=1, stop=2, step=1), Index(['a', 'b'], dtype='object'))
         """
         new_levels = []
         new_codes = []

pandas/core/indexes/range.py

@@ -472,9 +472,15 @@ def _shallow_copy(self, values, name: Hashable = no_default):
 
         if values.dtype.kind == "f":
             return Index(values, name=name, dtype=np.float64)
-        if values.dtype.kind == "i" and values.ndim == 1 and len(values) > 1:
+        if values.dtype.kind == "i" and values.ndim == 1:
             # GH 46675 & 43885: If values is equally spaced, return a
             # more memory-compact RangeIndex instead of Index with 64-bit dtype
+            if len(values) == 0:
+                return type(self)._simple_new(_empty_range, name=name)
+            elif len(values) == 1:
+                start = values[0]
+                new_range = range(start, start + self.step, self.step)
+                return type(self)._simple_new(new_range, name=name)
             diff = values[1] - values[0]
             if not missing.isna(diff) and lib.is_range(values, diff):
                 new_range = range(values[0], values[-1] + diff, diff)
@@ -889,12 +895,19 @@ def symmetric_difference(
             result = result.rename(result_name)
         return result
 
+    def _join_empty(
+        self, other: Index, how: JoinHow, sort: bool
+    ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
+        if other.dtype.kind == "i":
+            other = self._shallow_copy(other._values, name=other.name)
+        return super()._join_empty(other, how=how, sort=sort)
+
     def _join_monotonic(
         self, other: Index, how: JoinHow = "left"
     ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
         # This currently only gets called for the monotonic increasing case
         if not isinstance(other, type(self)):
-            maybe_ri = self._shallow_copy(other._values)
+            maybe_ri = self._shallow_copy(other._values, name=other.name)
             if not isinstance(maybe_ri, type(self)):
                 return super()._join_monotonic(other, how=how)
             other = maybe_ri
@@ -1070,6 +1083,8 @@ def __getitem__(self, key):
         """
         Conserve RangeIndex type for scalar and slice keys.
         """
+        if key is Ellipsis:
+            key = slice(None)
         if isinstance(key, slice):
             return self._getitem_slice(key)
         elif is_integer(key):
@@ -1089,17 +1104,20 @@ def __getitem__(self, key):
             )
         elif com.is_bool_indexer(key):
             if isinstance(getattr(key, "dtype", None), ExtensionDtype):
-                np_key = key.to_numpy(dtype=bool, na_value=False)
+                key = key.to_numpy(dtype=bool, na_value=False)
             else:
-                np_key = np.asarray(key, dtype=bool)
-            check_array_indexer(self._range, np_key)  # type: ignore[arg-type]
+                key = np.asarray(key, dtype=bool)
+            check_array_indexer(self._range, key)  # type: ignore[arg-type]
             # Short circuit potential _shallow_copy check
-            if np_key.all():
+            if key.all():
                 return self._simple_new(self._range, name=self.name)
-            elif not np_key.any():
+            elif not key.any():
                 return self._simple_new(_empty_range, name=self.name)
-            return self.take(np.flatnonzero(np_key))
-        return super().__getitem__(key)
+            key = np.flatnonzero(key)
+        try:
+            return self.take(key)
+        except (TypeError, ValueError):
+            return super().__getitem__(key)
 
     def _getitem_slice(self, slobj: slice) -> Self:
         """

pandas/core/internals/__init__.py

@@ -35,8 +35,6 @@ def __getattr__(name: str):
         return create_block_manager_from_blocks
 
     if name in [
-        "NumericBlock",
-        "ObjectBlock",
         "Block",
         "ExtensionBlock",
         "DatetimeTZBlock",
@@ -49,25 +47,17 @@ def __getattr__(name: str):
             # on hard-coding stacklevel
             stacklevel=2,
         )
-        if name == "NumericBlock":
-            from pandas.core.internals.blocks import NumericBlock
-
-            return NumericBlock
-        elif name == "DatetimeTZBlock":
+        if name == "DatetimeTZBlock":
             from pandas.core.internals.blocks import DatetimeTZBlock
 
             return DatetimeTZBlock
         elif name == "ExtensionBlock":
             from pandas.core.internals.blocks import ExtensionBlock
 
             return ExtensionBlock
-        elif name == "Block":
+        else:
             from pandas.core.internals.blocks import Block
 
             return Block
-        else:
-            from pandas.core.internals.blocks import ObjectBlock
-
-            return ObjectBlock
 
     raise AttributeError(f"module 'pandas.core.internals' has no attribute '{name}'")

pandas/core/internals/blocks.py

@@ -2148,18 +2148,6 @@ def is_numeric(self) -> bool:  # type: ignore[override]
         return kind in "fciub"
 
 
-class NumericBlock(NumpyBlock):
-    # this Block type is kept for backwards-compatibility
-    # TODO(3.0): delete and remove deprecation in __init__.py.
-    __slots__ = ()
-
-
-class ObjectBlock(NumpyBlock):
-    # this Block type is kept for backwards-compatibility
-    # TODO(3.0): delete and remove deprecation in __init__.py.
-    __slots__ = ()
-
-
 class NDArrayBackedExtensionBlock(EABackedBlock):
     """
     Block backed by an NDArrayBackedExtensionArray

pandas/core/methods/selectn.py

@@ -213,7 +213,7 @@ def compute(self, method: str) -> DataFrame:
                     f"cannot use method {method!r} with this dtype"
                 )
 
-        def get_indexer(current_indexer, other_indexer):
+        def get_indexer(current_indexer: Index, other_indexer: Index) -> Index:
             """
             Helper function to concat `current_indexer` and `other_indexer`
             depending on `method`

pandas/core/methods/to_dict.py

@@ -155,7 +155,8 @@ def to_dict(
             stacklevel=find_stack_level(),
         )
     # GH16122
-    into_c = com.standardize_mapping(into)
+    # error: Call to untyped function "standardize_mapping" in typed context
+    into_c = com.standardize_mapping(into)  # type: ignore[no-untyped-call]
 
     #  error: Incompatible types in assignment (expression has type "str",
     # variable has type "Literal['dict', 'list', 'series', 'split', 'tight',

pandas/core/ops/invalid.py

@@ -7,17 +7,26 @@
 import operator
 from typing import (
     TYPE_CHECKING,
+    Any,
     Callable,
     NoReturn,
 )
 
 import numpy as np
 
 if TYPE_CHECKING:
-    from pandas._typing import npt
+    from pandas._typing import (
+        ArrayLike,
+        Scalar,
+        npt,
+    )
 
 
-def invalid_comparison(left, right, op) -> npt.NDArray[np.bool_]:
+def invalid_comparison(
+    left: ArrayLike,
+    right: ArrayLike | Scalar,
+    op: Callable[[Any, Any], bool],
+) -> npt.NDArray[np.bool_]:
     """
     If a comparison has mismatched types and is not necessarily meaningful,
     follow python3 conventions by:
@@ -59,7 +68,7 @@ def make_invalid_op(name: str) -> Callable[..., NoReturn]:
     invalid_op : function
     """
 
-    def invalid_op(self, other=None) -> NoReturn:
+    def invalid_op(self: object, other: object = None) -> NoReturn:
         typ = type(self).__name__
         raise TypeError(f"cannot perform {name} with this index type: {typ}")
 

pandas/io/common.py

@@ -278,7 +278,7 @@ def stringify_path(
     return _expand_user(filepath_or_buffer)
 
 
-def urlopen(*args, **kwargs):
+def urlopen(*args: Any, **kwargs: Any) -> Any:
     """
     Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of
     the stdlib.
@@ -972,7 +972,7 @@ def __init__(
         mode: Literal["r", "a", "w", "x"] = "r",
         fileobj: ReadBuffer[bytes] | WriteBuffer[bytes] | None = None,
         archive_name: str | None = None,
-        **kwargs,
+        **kwargs: Any,
     ) -> None:
         super().__init__()
         self.archive_name = archive_name
@@ -1025,7 +1025,7 @@ def __init__(
         file: FilePath | ReadBuffer[bytes] | WriteBuffer[bytes],
         mode: str,
         archive_name: str | None = None,
-        **kwargs,
+        **kwargs: Any,
     ) -> None:
         super().__init__()
         mode = mode.replace("b", "")

pandas/io/excel/_odswriter.py

@@ -18,6 +18,8 @@
 )
 
 if TYPE_CHECKING:
+    from odf.opendocument import OpenDocumentSpreadsheet
+
     from pandas._typing import (
         ExcelWriterIfSheetExists,
         FilePath,
@@ -37,12 +39,12 @@ def __init__(
         path: FilePath | WriteExcelBuffer | ExcelWriter,
         engine: str | None = None,
         date_format: str | None = None,
-        datetime_format=None,
+        datetime_format: str | None = None,
         mode: str = "w",
         storage_options: StorageOptions | None = None,
         if_sheet_exists: ExcelWriterIfSheetExists | None = None,
         engine_kwargs: dict[str, Any] | None = None,
-        **kwargs,
+        **kwargs: Any,
     ) -> None:
         from odf.opendocument import OpenDocumentSpreadsheet
 
@@ -63,7 +65,7 @@ def __init__(
         self._style_dict: dict[str, str] = {}
 
     @property
-    def book(self):
+    def book(self) -> OpenDocumentSpreadsheet:
         """
         Book instance of class odf.opendocument.OpenDocumentSpreadsheet.
 
@@ -149,7 +151,7 @@ def _write_cells(
             for row_nr in range(max(rows.keys()) + 1):
                 wks.addElement(rows[row_nr])
 
-    def _make_table_cell_attributes(self, cell) -> dict[str, int | str]:
+    def _make_table_cell_attributes(self, cell: ExcelCell) -> dict[str, int | str]:
         """Convert cell attributes to OpenDocument attributes
 
         Parameters
@@ -171,7 +173,7 @@ def _make_table_cell_attributes(self, cell) -> dict[str, int | str]:
             attributes["numbercolumnsspanned"] = cell.mergeend
         return attributes
 
-    def _make_table_cell(self, cell) -> tuple[object, Any]:
+    def _make_table_cell(self, cell: ExcelCell) -> tuple[object, Any]:
         """Convert cell data to an OpenDocument spreadsheet cell
 
         Parameters

pandas/io/formats/css.py

@@ -36,7 +36,9 @@ def _side_expander(prop_fmt: str) -> Callable:
         function: Return to call when a 'border(-{side}): {value}' string is encountered
     """
 
-    def expand(self, prop: str, value: str) -> Generator[tuple[str, str], None, None]:
+    def expand(
+        self: CSSResolver, prop: str, value: str
+    ) -> Generator[tuple[str, str], None, None]:
         """
         Expand shorthand property into side-specific property (top, right, bottom, left)
 
@@ -81,7 +83,9 @@ def _border_expander(side: str = "") -> Callable:
     if side != "":
         side = f"-{side}"
 
-    def expand(self, prop: str, value: str) -> Generator[tuple[str, str], None, None]:
+    def expand(
+        self: CSSResolver, prop: str, value: str
+    ) -> Generator[tuple[str, str], None, None]:
         """
         Expand border into color, style, and width tuples
 
@@ -343,7 +347,9 @@ def _update_other_units(self, props: dict[str, str]) -> dict[str, str]:
                     )
         return props
 
-    def size_to_pt(self, in_val, em_pt=None, conversions=UNIT_RATIOS) -> str:
+    def size_to_pt(
+        self, in_val: str, em_pt: float | None = None, conversions: dict = UNIT_RATIOS
+    ) -> str:
         def _error() -> str:
             warnings.warn(
                 f"Unhandled size: {in_val!r}",