Merge remote-tracking branch 'upstream/main' into bisect

Author: simonjayhawkins

.github/workflows/macos-windows.yml

@@ -23,7 +23,7 @@ jobs:
     defaults:
       run:
         shell: bash -el {0}
-    timeout-minutes: 90
+    timeout-minutes: 120
     strategy:
       matrix:
         os: [macos-latest, windows-latest]

doc/redirects.csv

@@ -761,6 +761,7 @@ generated/pandas.IntervalIndex.mid,../reference/api/pandas.IntervalIndex.mid
 generated/pandas.IntervalIndex.overlaps,../reference/api/pandas.IntervalIndex.overlaps
 generated/pandas.IntervalIndex.right,../reference/api/pandas.IntervalIndex.right
 generated/pandas.IntervalIndex.set_closed,../reference/api/pandas.IntervalIndex.set_closed
+generated/pandas.IntervalIndex.set_inclusive,../reference/api/pandas.IntervalIndex.set_inclusive
 generated/pandas.IntervalIndex.to_tuples,../reference/api/pandas.IntervalIndex.to_tuples
 generated/pandas.IntervalIndex.values,../reference/api/pandas.IntervalIndex.values
 generated/pandas.Interval.left,../reference/api/pandas.Interval.left

doc/source/reference/arrays.rst

@@ -352,6 +352,7 @@ A collection of intervals may be stored in an :class:`arrays.IntervalArray`.
       arrays.IntervalArray.contains
       arrays.IntervalArray.overlaps
       arrays.IntervalArray.set_closed
+      arrays.IntervalArray.set_inclusive
       arrays.IntervalArray.to_tuples
 
 

doc/source/reference/indexing.rst

@@ -251,6 +251,7 @@ IntervalIndex components
    IntervalIndex.get_loc
    IntervalIndex.get_indexer
    IntervalIndex.set_closed
+   IntervalIndex.set_inclusive
    IntervalIndex.contains
    IntervalIndex.overlaps
    IntervalIndex.to_tuples

doc/source/user_guide/groupby.rst

@@ -839,10 +839,10 @@ Alternatively, the built-in methods could be used to produce the same outputs.
 
 .. ipython:: python
 
-   max = ts.groupby(lambda x: x.year).transform("max")
-   min = ts.groupby(lambda x: x.year).transform("min")
+   max_ts = ts.groupby(lambda x: x.year).transform("max")
+   min_ts = ts.groupby(lambda x: x.year).transform("min")
 
-   max - min
+   max_ts - min_ts
 
 Another common data transform is to replace missing data with the group mean.
 

doc/source/whatsnew/v1.5.0.rst

@@ -277,7 +277,7 @@ Other enhancements
 - Allow reading compressed SAS files with :func:`read_sas` (e.g., ``.sas7bdat.gz`` files)
 - :meth:`DatetimeIndex.astype` now supports casting timezone-naive indexes to ``datetime64[s]``, ``datetime64[ms]``, and ``datetime64[us]``, and timezone-aware indexes to the corresponding ``datetime64[unit, tzname]`` dtypes (:issue:`47579`)
 - :class:`Series` reducers (e.g. ``min``, ``max``, ``sum``, ``mean``) will now successfully operate when the dtype is numeric and ``numeric_only=True`` is provided; previously this would raise a ``NotImplementedError`` (:issue:`47500`)
--
+- :meth:`RangeIndex.union` now can return a :class:`RangeIndex` instead of a :class:`Int64Index` if the resulting values are equally spaced (:issue:`47557`, :issue:`43885`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -762,7 +762,7 @@ Other Deprecations
 - Deprecated the ``closed`` argument in :class:`IntervalIndex` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`IntervalDtype` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`.IntervalArray` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
-- Deprecated the ``closed`` argument in :class:`IntervalTree` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated :meth:`.IntervalArray.set_closed` and :meth:`.IntervalIndex.set_closed` in favor of ``set_inclusive``; In a future version ``set_closed`` will get removed (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`ArrowInterval` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated allowing ``unit="M"`` or ``unit="Y"`` in :class:`Timestamp` constructor with a non-round float value (:issue:`47267`)
 - Deprecated the ``display.column_space`` global configuration option (:issue:`7576`)
@@ -885,6 +885,7 @@ Indexing
 - Bug when setting a value too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`, :issue:`32878`)
 - Bug in :meth:`loc.__setitem__` treating ``range`` keys as positional instead of label-based (:issue:`45479`)
 - Bug in :meth:`Series.__setitem__` when setting ``boolean`` dtype values containing ``NA`` incorrectly raising instead of casting to ``boolean`` dtype (:issue:`45462`)
+- Bug in :meth:`Series.loc` raising with boolean indexer containing ``NA`` when :class:`Index` did not match (:issue:`46551`)
 - Bug in :meth:`Series.__setitem__` where setting :attr:`NA` into a numeric-dtype :class:`Series` would incorrectly upcast to object-dtype rather than treating the value as ``np.nan`` (:issue:`44199`)
 - Bug in :meth:`DataFrame.loc` when setting values to a column and right hand side is a dictionary (:issue:`47216`)
 - Bug in :meth:`DataFrame.loc` when setting a :class:`DataFrame` not aligning index in some cases (:issue:`47578`)
@@ -1008,7 +1009,7 @@ Reshaping
 - Bug in :func:`concat` with identical key leads to error when indexing :class:`MultiIndex` (:issue:`46519`)
 - Bug in :meth:`DataFrame.join` with a list when using suffixes to join DataFrames with duplicate column names (:issue:`46396`)
 - Bug in :meth:`DataFrame.pivot_table` with ``sort=False`` results in sorted index (:issue:`17041`)
--
+- Bug in :meth:`concat` when ``axis=1`` and ``sort=False`` where the resulting Index was a :class:`Int64Index` instead of a :class:`RangeIndex` (:issue:`46675`)
 
 Sparse
 ^^^^^^

pandas/_config/localization.py

@@ -39,7 +39,8 @@ def set_locale(
     particular locale, without globally setting the locale. This probably isn't
     thread-safe.
     """
-    current_locale = locale.getlocale()
+    # getlocale is not always compliant with setlocale, use setlocale. GH#46595
+    current_locale = locale.setlocale(lc_var)
 
     try:
         locale.setlocale(lc_var, new_locale)

pandas/_libs/interval.pyi

@@ -12,7 +12,7 @@ import numpy.typing as npt
 
 from pandas._libs import lib
 from pandas._typing import (
-    IntervalClosedType,
+    IntervalInclusiveType,
     Timedelta,
     Timestamp,
 )
@@ -56,25 +56,25 @@ class IntervalMixin:
 
 def _warning_interval(
     inclusive, closed
-) -> tuple[IntervalClosedType, lib.NoDefault]: ...
+) -> tuple[IntervalInclusiveType, lib.NoDefault]: ...
 
 class Interval(IntervalMixin, Generic[_OrderableT]):
     @property
     def left(self: Interval[_OrderableT]) -> _OrderableT: ...
     @property
     def right(self: Interval[_OrderableT]) -> _OrderableT: ...
     @property
-    def inclusive(self) -> IntervalClosedType: ...
+    def inclusive(self) -> IntervalInclusiveType: ...
     @property
-    def closed(self) -> IntervalClosedType: ...
+    def closed(self) -> IntervalInclusiveType: ...
     mid: _MidDescriptor
     length: _LengthDescriptor
     def __init__(
         self,
         left: _OrderableT,
         right: _OrderableT,
-        inclusive: IntervalClosedType = ...,
-        closed: IntervalClosedType = ...,
+        inclusive: IntervalInclusiveType = ...,
+        closed: IntervalInclusiveType = ...,
     ) -> None: ...
     def __hash__(self) -> int: ...
     @overload
@@ -151,14 +151,14 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
 
 def intervals_to_interval_bounds(
     intervals: np.ndarray, validate_closed: bool = ...
-) -> tuple[np.ndarray, np.ndarray, str]: ...
+) -> tuple[np.ndarray, np.ndarray, IntervalInclusiveType]: ...
 
 class IntervalTree(IntervalMixin):
     def __init__(
         self,
         left: np.ndarray,
         right: np.ndarray,
-        inclusive: IntervalClosedType = ...,
+        inclusive: IntervalInclusiveType = ...,
         leaf_size: int = ...,
     ) -> None: ...
     @property

pandas/_libs/intervaltree.pxi.in

@@ -8,8 +8,6 @@ import warnings
 from pandas._libs import lib
 from pandas._libs.algos import is_monotonic
 
-from pandas._libs.interval import _warning_interval
-
 ctypedef fused int_scalar_t:
     int64_t
     float64_t
@@ -42,18 +40,13 @@ cdef class IntervalTree(IntervalMixin):
         object _is_overlapping, _left_sorter, _right_sorter
         Py_ssize_t _na_count
 
-    def __init__(self, left, right, inclusive: str | None = None, closed: None | lib.NoDefault = lib.no_default, leaf_size=100):
+    def __init__(self, left, right, inclusive: str | None = None, leaf_size=100):
         """
         Parameters
         ----------
         left, right : np.ndarray[ndim=1]
             Left and right bounds for each interval. Assumed to contain no
             NaNs.
-        closed : {'left', 'right', 'both', 'neither'}, optional
-            Whether the intervals are closed on the left-side, right-side, both
-            or neither. Defaults to 'right'.
-
-            .. deprecated:: 1.5.0
 
         inclusive : {"both", "neither", "left", "right"}, optional
             Whether the intervals are closed on the left-side, right-side, both
@@ -66,8 +59,6 @@ cdef class IntervalTree(IntervalMixin):
             to brute-force search. Tune this parameter to optimize query
             performance.
         """
-        inclusive, closed = _warning_interval(inclusive, closed)
-
         if inclusive is None:
             inclusive = "right"
 
@@ -119,7 +110,7 @@ cdef class IntervalTree(IntervalMixin):
         if self._is_overlapping is not None:
             return self._is_overlapping
 
-        # <= when both sides closed since endpoints can overlap
+        # <= when inclusive on both sides since endpoints can overlap
         op = le if self.inclusive == 'both' else lt
 
         # overlap if start of current interval < end of previous interval
@@ -263,7 +254,7 @@ cdef class IntervalNode:
 
 
 # we need specialized nodes and leaves to optimize for different dtype and
-# closed values
+# inclusive values
 
 {{py:
 

pandas/_libs/tslibs/timedeltas.pyx

@@ -950,14 +950,18 @@ cdef _timedelta_from_value_and_reso(int64_t value, NPY_DATETIMEUNIT reso):
     cdef:
         _Timedelta td_base
 
+    # For millisecond and second resos, we cannot actually pass int(value) because
+    #  many cases would fall outside of the pytimedelta implementation bounds.
+    #  We pass 0 instead, and override seconds, microseconds, days.
+    #  In principle we could pass 0 for ns and us too.
     if reso == NPY_FR_ns:
         td_base = _Timedelta.__new__(Timedelta, microseconds=int(value) // 1000)
     elif reso == NPY_DATETIMEUNIT.NPY_FR_us:
         td_base = _Timedelta.__new__(Timedelta, microseconds=int(value))
     elif reso == NPY_DATETIMEUNIT.NPY_FR_ms:
-        td_base = _Timedelta.__new__(Timedelta, milliseconds=int(value))
+        td_base = _Timedelta.__new__(Timedelta, milliseconds=0)
     elif reso == NPY_DATETIMEUNIT.NPY_FR_s:
-        td_base = _Timedelta.__new__(Timedelta, seconds=int(value))
+        td_base = _Timedelta.__new__(Timedelta, seconds=0)
     # Other resolutions are disabled but could potentially be implemented here:
     # elif reso == NPY_DATETIMEUNIT.NPY_FR_m:
     #    td_base = _Timedelta.__new__(Timedelta, minutes=int(value))
@@ -977,6 +981,34 @@ cdef _timedelta_from_value_and_reso(int64_t value, NPY_DATETIMEUNIT reso):
     return td_base
 
 
+class MinMaxReso:
+    """
+    We need to define min/max/resolution on both the Timedelta _instance_
+    and Timedelta class.  On an instance, these depend on the object's _reso.
+    On the class, we default to the values we would get with nanosecond _reso.
+    """
+    def __init__(self, name):
+        self._name = name
+
+    def __get__(self, obj, type=None):
+        if self._name == "min":
+            val = np.iinfo(np.int64).min + 1
+        elif self._name == "max":
+            val = np.iinfo(np.int64).max
+        else:
+            assert self._name == "resolution"
+            val = 1
+
+        if obj is None:
+            # i.e. this is on the class, default to nanos
+            return Timedelta(val)
+        else:
+            return Timedelta._from_value_and_reso(val, obj._reso)
+
+    def __set__(self, obj, value):
+        raise AttributeError(f"{self._name} is not settable.")
+
+
 # Similar to Timestamp/datetime, this is a construction requirement for
 # timedeltas that we need to do object instantiation in python. This will
 # serve as a C extension type that shadows the Python class, where we do any
@@ -990,6 +1022,36 @@ cdef class _Timedelta(timedelta):
 
     # higher than np.ndarray and np.matrix
     __array_priority__ = 100
+    min = MinMaxReso("min")
+    max = MinMaxReso("max")
+    resolution = MinMaxReso("resolution")
+
+    @property
+    def days(self) -> int:  # TODO(cython3): make cdef property
+        # NB: using the python C-API PyDateTime_DELTA_GET_DAYS will fail
+        #  (or be incorrect)
+        self._ensure_components()
+        return self._d
+
+    @property
+    def seconds(self) -> int:  # TODO(cython3): make cdef property
+        # NB: using the python C-API PyDateTime_DELTA_GET_SECONDS will fail
+        #  (or be incorrect)
+        self._ensure_components()
+        return self._h * 3600 + self._m * 60 + self._s
+
+    @property
+    def microseconds(self) -> int:  # TODO(cython3): make cdef property
+        # NB: using the python C-API PyDateTime_DELTA_GET_MICROSECONDS will fail
+        #  (or be incorrect)
+        self._ensure_components()
+        return self._ms * 1000 + self._us
+
+    def total_seconds(self) -> float:
+        """Total seconds in the duration."""
+        # We need to override bc we overrided days/seconds/microseconds
+        # TODO: add nanos/1e9?
+        return self.days * 24 * 3600 + self.seconds + self.microseconds / 1_000_000
 
     @property
     def freq(self) -> None:
@@ -1979,9 +2041,3 @@ cdef _broadcast_floordiv_td64(
         res = res.astype('f8')
         res[mask] = np.nan
     return res
-
-
-# resolution in ns
-Timedelta.min = Timedelta(np.iinfo(np.int64).min + 1)
-Timedelta.max = Timedelta(np.iinfo(np.int64).max)
-Timedelta.resolution = Timedelta(nanoseconds=1)

pandas/_typing.py

@@ -314,7 +314,7 @@ def closed(self) -> bool:
 
 # Interval closed type
 IntervalLeftRight = Literal["left", "right"]
-IntervalClosedType = Union[IntervalLeftRight, Literal["both", "neither"]]
+IntervalInclusiveType = Union[IntervalLeftRight, Literal["both", "neither"]]
 
 # datetime and NaTType
 DatetimeNaTType = Union[datetime, "NaTType"]

pandas/core/arrays/arrow/_arrow_utils.py

@@ -6,6 +6,7 @@
 import numpy as np
 import pyarrow
 
+from pandas._typing import IntervalInclusiveType
 from pandas.errors import PerformanceWarning
 from pandas.util._decorators import deprecate_kwarg
 from pandas.util._exceptions import find_stack_level
@@ -107,11 +108,11 @@ def to_pandas_dtype(self):
 
 class ArrowIntervalType(pyarrow.ExtensionType):
     @deprecate_kwarg(old_arg_name="closed", new_arg_name="inclusive")
-    def __init__(self, subtype, inclusive: str) -> None:
+    def __init__(self, subtype, inclusive: IntervalInclusiveType) -> None:
         # attributes need to be set first before calling
         # super init (as that calls serialize)
         assert inclusive in VALID_CLOSED
-        self._closed = inclusive
+        self._closed: IntervalInclusiveType = inclusive
         if not isinstance(subtype, pyarrow.DataType):
             subtype = pyarrow.type_for_alias(str(subtype))
         self._subtype = subtype
@@ -124,11 +125,11 @@ def subtype(self):
         return self._subtype
 
     @property
-    def inclusive(self) -> str:
+    def inclusive(self) -> IntervalInclusiveType:
         return self._closed
 
     @property
-    def closed(self):
+    def closed(self) -> IntervalInclusiveType:
         warnings.warn(
             "Attribute `closed` is deprecated in favor of `inclusive`.",
             FutureWarning,