Merge branch 'main' into fix_to_string_backslash

Author: GianlucaFicarelli

asv_bench/benchmarks/dtypes.py

@@ -24,7 +24,7 @@
 
 
 class Dtypes:
-    params = _dtypes + list(map(lambda dt: dt.name, _dtypes))
+    params = _dtypes + [dt.name for dt in _dtypes]
     param_names = ["dtype"]
 
     def time_pandas_dtype(self, dtype):

doc/source/whatsnew/v2.0.2.rst

@@ -13,19 +13,22 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
-- Bug in :func:`DataFrame.to_string` with ``header=True`` that printed a backslash at the end of the first row of the data, instead of the headers, when the DataFrame doesn't fit the line width (:issue:`53054`)
+- Fixed regression in :meth:`DataFrame.loc` losing :class:`MultiIndex` name when enlarging object (:issue:`53053`)
+- Fixed regression in :meth:`DataFrame.to_string` printing a backslash at the end of the first row of data, instead of headers, when the DataFrame doesn't fit the line width (:issue:`53054`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_202.bug_fixes:
 
 Bug fixes
 ~~~~~~~~~
+- Bug in :func:`api.interchange.from_dataframe` was raising ``IndexError`` on empty categorical data (:issue:`53077`)
 - Bug in :func:`api.interchange.from_dataframe` was returning :class:`DataFrame`'s of incorrect sizes when called on slices (:issue:`52824`)
 - Bug in :func:`api.interchange.from_dataframe` was unnecessarily raising on bitmasks (:issue:`49888`)
+- Bug in :func:`to_timedelta` was raising ``ValueError`` with ``pandas.NA`` (:issue:`52909`)
 - Bug in :meth:`DataFrame.convert_dtypes` ignores ``convert_*`` keywords when set to False ``dtype_backend="pyarrow"`` (:issue:`52872`)
 - Bug in :meth:`Series.describe` treating pyarrow-backed timestamps and timedeltas as categorical data (:issue:`53001`)
 - Bug in :meth:`pd.array` raising for ``NumPy`` array and ``pa.large_string`` or ``pa.large_binary`` (:issue:`52590`)
--
+
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_202.other:

doc/source/whatsnew/v2.1.0.rst

@@ -285,6 +285,7 @@ Performance improvements
 - Performance improvement accessing :attr:`arrays.IntegerArrays.dtype` & :attr:`arrays.FloatingArray.dtype` (:issue:`52998`)
 - Performance improvement in :class:`Series` reductions (:issue:`52341`)
 - Performance improvement in :func:`concat` when ``axis=1`` and objects have different indexes (:issue:`52541`)
+- Performance improvement in :meth:`DataFrame.loc` when selecting rows and columns (:issue:`53014`)
 - Performance improvement in :meth:`Series.corr` and :meth:`Series.cov` for extension dtypes (:issue:`52502`)
 - Performance improvement in :meth:`Series.to_numpy` when dtype is a numpy float dtype and ``na_value`` is ``np.nan`` (:issue:`52430`)
 - Performance improvement in :meth:`~arrays.ArrowExtensionArray.to_numpy` (:issue:`52525`)
@@ -307,6 +308,8 @@ Datetimelike
 - Bug in :func:`date_range` when ``freq`` was a :class:`DateOffset` with ``nanoseconds`` (:issue:`46877`)
 - Bug in :meth:`Timestamp.round` with values close to the implementation bounds returning incorrect results instead of raising ``OutOfBoundsDatetime`` (:issue:`51494`)
 - Bug in :meth:`arrays.DatetimeArray.map` and :meth:`DatetimeIndex.map`, where the supplied callable operated array-wise instead of element-wise (:issue:`51977`)
+- Bug in parsing datetime strings with weekday but no day e.g. "2023 Sept Thu" incorrectly raising ``AttributeError`` instead of ``ValueError`` (:issue:`52659`)
+-
 
 Timedelta
 ^^^^^^^^^

pandas/_libs/tslibs/offsets.pyx

@@ -15,8 +15,6 @@ from cpython.datetime cimport (
 
 import_datetime()
 
-from dateutil.easter import easter
-from dateutil.relativedelta import relativedelta
 import numpy as np
 
 cimport numpy as cnp
@@ -348,6 +346,8 @@ cdef _determine_offset(kwds):
         kwds_no_nanos["microseconds"] = kwds_no_nanos.get("microseconds", 0) + micro
 
     if all(k in kwds_use_relativedelta for k in kwds_no_nanos):
+        from dateutil.relativedelta import relativedelta
+
         return relativedelta(**kwds_no_nanos), True
 
     raise ValueError(
@@ -3691,6 +3691,8 @@ cdef class Easter(SingleConstructorOffset):
 
     @apply_wraps
     def _apply(self, other: datetime) -> datetime:
+        from dateutil.easter import easter
+
         current_easter = easter(other.year)
         current_easter = datetime(
             current_easter.year, current_easter.month, current_easter.day
@@ -3721,6 +3723,9 @@ cdef class Easter(SingleConstructorOffset):
     def is_on_offset(self, dt: datetime) -> bool:
         if self.normalize and not _is_normalized(dt):
             return False
+
+        from dateutil.easter import easter
+
         return date(dt.year, dt.month, dt.day) == easter(dt.year)
 
 

pandas/_libs/tslibs/parsing.pyx

@@ -46,7 +46,6 @@ from dateutil.parser import (
     DEFAULTPARSER,
     parse as du_parse,
 )
-from dateutil.relativedelta import relativedelta
 from dateutil.tz import (
     tzlocal as _dateutil_tzlocal,
     tzoffset,
@@ -692,7 +691,11 @@ cdef datetime dateutil_parse(
         ) from err
 
     if res.weekday is not None and not res.day:
-        ret = ret + relativedelta.relativedelta(weekday=res.weekday)
+        # GH#52659
+        raise ValueError(
+            "Parsing datetimes with weekday but no day information is "
+            "not supported"
+        )
     if not ignoretz:
         if res.tzname and res.tzname in time.tzname:
             # GH#50791

pandas/_libs/tslibs/timedeltas.pyx

@@ -34,6 +34,7 @@ import_datetime()
 
 
 cimport pandas._libs.tslibs.util as util
+from pandas._libs.missing cimport checknull_with_nat_and_na
 from pandas._libs.tslibs.base cimport ABCTimestamp
 from pandas._libs.tslibs.conversion cimport (
     cast_from_unit,
@@ -341,8 +342,7 @@ cdef convert_to_timedelta64(object ts, str unit):
     Return an ns based int64
     """
     # Caller is responsible for checking unit not in ["Y", "y", "M"]
-
-    if checknull_with_nat(ts):
+    if checknull_with_nat_and_na(ts):
         return np.timedelta64(NPY_NAT, "ns")
     elif isinstance(ts, _Timedelta):
         # already in the proper format
@@ -1808,7 +1808,7 @@ class Timedelta(_Timedelta):
             # unit=None is de-facto 'ns'
             unit = parse_timedelta_unit(unit)
             value = convert_to_timedelta64(value, unit)
-        elif checknull_with_nat(value):
+        elif checknull_with_nat_and_na(value):
             return NaT
         else:
             raise ValueError(

pandas/_typing.py

@@ -155,7 +155,7 @@
 
 RandomState = Union[
     int,
-    ArrayLike,
+    np.ndarray,
     np.random.Generator,
     np.random.BitGenerator,
     np.random.RandomState,

pandas/core/array_algos/putmask.py

@@ -136,7 +136,7 @@ def setitem_datetimelike_compat(values: np.ndarray, num_set: int, other):
     other : Any
     """
     if values.dtype == object:
-        dtype, _ = infer_dtype_from(other, pandas_dtype=True)
+        dtype, _ = infer_dtype_from(other)
 
         if isinstance(dtype, np.dtype) and dtype.kind in "mM":
             # https://github.com/numpy/numpy/issues/12550

pandas/core/arrays/arrow/array.py

@@ -18,22 +18,6 @@
 import numpy as np
 
 from pandas._libs import lib
-from pandas._typing import (
-    ArrayLike,
-    AxisInt,
-    Dtype,
-    FillnaOptions,
-    Iterator,
-    NpDtype,
-    PositionalIndexer,
-    Scalar,
-    Self,
-    SortKind,
-    TakeIndexer,
-    TimeAmbiguous,
-    TimeNonexistent,
-    npt,
-)
 from pandas.compat import (
     pa_version_under7p0,
     pa_version_under8p0,
@@ -140,8 +124,22 @@ def floordiv_compat(
 
 if TYPE_CHECKING:
     from pandas._typing import (
+        ArrayLike,
+        AxisInt,
+        Dtype,
+        FillnaOptions,
+        Iterator,
+        NpDtype,
         NumpySorter,
         NumpyValueArrayLike,
+        PositionalIndexer,
+        Scalar,
+        Self,
+        SortKind,
+        TakeIndexer,
+        TimeAmbiguous,
+        TimeNonexistent,
+        npt,
     )
 
     from pandas import Series
@@ -805,8 +803,9 @@ def fillna(
             fallback_performancewarning()
             return super().fillna(value=value, method=method, limit=limit)
 
-        if is_array_like(value):
-            value = cast(ArrayLike, value)
+        if isinstance(value, (np.ndarray, ExtensionArray)):
+            # Similar to check_value_size, but we do not mask here since we may
+            #  end up passing it to the super() method.
             if len(value) != len(self):
                 raise ValueError(
                     f"Length of 'value' does not match. Got ({len(value)}) "

pandas/core/arrays/base.py

@@ -38,7 +38,7 @@
     validate_insert_loc,
 )
 
-from pandas.core.dtypes.cast import maybe_cast_to_extension_array
+from pandas.core.dtypes.cast import maybe_cast_pointwise_result
 from pandas.core.dtypes.common import (
     is_list_like,
     is_scalar,
@@ -1957,7 +1957,7 @@ def _maybe_convert(arr):
                     # https://github.com/pandas-dev/pandas/issues/22850
                     # We catch all regular exceptions here, and fall back
                     # to an ndarray.
-                    res = maybe_cast_to_extension_array(type(self), arr)
+                    res = maybe_cast_pointwise_result(arr, self.dtype, same_dtype=False)
                     if not isinstance(res, type(self)):
                         # exception raised in _from_sequence; ensure we have ndarray
                         res = np.asarray(arr)

pandas/core/arrays/masked.py

@@ -75,7 +75,11 @@
 from pandas.core.array_algos.quantile import quantile_with_mask
 from pandas.core.arraylike import OpsMixin
 from pandas.core.arrays.base import ExtensionArray
-from pandas.core.construction import ensure_wrapped_if_datetimelike
+from pandas.core.construction import (
+    array as pd_array,
+    ensure_wrapped_if_datetimelike,
+    extract_array,
+)
 from pandas.core.indexers import check_array_indexer
 from pandas.core.ops import invalid_comparison
 
@@ -645,20 +649,8 @@ def _arith_method(self, other, op):
             and len(other) == len(self)
         ):
             # Try inferring masked dtype instead of casting to object
-            inferred_dtype = lib.infer_dtype(other, skipna=True)
-            if inferred_dtype == "integer":
-                from pandas.core.arrays import IntegerArray
-
-                other = IntegerArray._from_sequence(other)
-            elif inferred_dtype in ["floating", "mixed-integer-float"]:
-                from pandas.core.arrays import FloatingArray
-
-                other = FloatingArray._from_sequence(other)
-
-            elif inferred_dtype in ["boolean"]:
-                from pandas.core.arrays import BooleanArray
-
-                other = BooleanArray._from_sequence(other)
+            other = pd_array(other)
+            other = extract_array(other, extract_numpy=True)
 
         if isinstance(other, BaseMaskedArray):
             other, omask = other._data, other._mask

pandas/core/common.py

@@ -33,11 +33,9 @@
 
 from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
 from pandas.core.dtypes.common import (
-    is_array_like,
     is_bool_dtype,
     is_integer,
 )
-from pandas.core.dtypes.dtypes import ExtensionDtype
 from pandas.core.dtypes.generic import (
     ABCExtensionArray,
     ABCIndex,
@@ -120,9 +118,7 @@ def is_bool_indexer(key: Any) -> bool:
     check_array_indexer : Check that `key` is a valid array to index,
         and convert to an ndarray.
     """
-    if isinstance(key, (ABCSeries, np.ndarray, ABCIndex)) or (
-        is_array_like(key) and isinstance(key.dtype, ExtensionDtype)
-    ):
+    if isinstance(key, (ABCSeries, np.ndarray, ABCIndex, ABCExtensionArray)):
         if key.dtype == np.object_:
             key_array = np.asarray(key)
 
@@ -420,7 +416,7 @@ def random_state(state: np.random.Generator) -> np.random.Generator:
 
 @overload
 def random_state(
-    state: int | ArrayLike | np.random.BitGenerator | np.random.RandomState | None,
+    state: int | np.ndarray | np.random.BitGenerator | np.random.RandomState | None,
 ) -> np.random.RandomState:
     ...
 
@@ -445,24 +441,8 @@ def random_state(state: RandomState | None = None):
     np.random.RandomState or np.random.Generator. If state is None, returns np.random
 
     """
-    if (
-        is_integer(state)
-        or is_array_like(state)
-        or isinstance(state, np.random.BitGenerator)
-    ):
-        # error: Argument 1 to "RandomState" has incompatible type "Optional[Union[int,
-        # Union[ExtensionArray, ndarray[Any, Any]], Generator, RandomState]]"; expected
-        # "Union[None, Union[Union[_SupportsArray[dtype[Union[bool_, integer[Any]]]],
-        # Sequence[_SupportsArray[dtype[Union[bool_, integer[Any]]]]],
-        # Sequence[Sequence[_SupportsArray[dtype[Union[bool_, integer[Any]]]]]],
-        # Sequence[Sequence[Sequence[_SupportsArray[dtype[Union[bool_,
-        # integer[Any]]]]]]],
-        # Sequence[Sequence[Sequence[Sequence[_SupportsArray[dtype[Union[bool_,
-        # integer[Any]]]]]]]]], Union[bool, int, Sequence[Union[bool, int]],
-        # Sequence[Sequence[Union[bool, int]]], Sequence[Sequence[Sequence[Union[bool,
-        # int]]]], Sequence[Sequence[Sequence[Sequence[Union[bool, int]]]]]]],
-        # BitGenerator]"
-        return np.random.RandomState(state)  # type: ignore[arg-type]
+    if is_integer(state) or isinstance(state, (np.ndarray, np.random.BitGenerator)):
+        return np.random.RandomState(state)
     elif isinstance(state, np.random.RandomState):
         return state
     elif isinstance(state, np.random.Generator):

pandas/core/computation/expr.py

@@ -192,7 +192,7 @@ def _filter_nodes(superclass, all_nodes=_all_nodes):
     return frozenset(node_names)
 
 
-_all_node_names = frozenset(map(lambda x: x.__name__, _all_nodes))
+_all_node_names = frozenset(x.__name__ for x in _all_nodes)
 _mod_nodes = _filter_nodes(ast.mod)
 _stmt_nodes = _filter_nodes(ast.stmt)
 _expr_nodes = _filter_nodes(ast.expr)