Merge branch 'main' into bug-range

Author: jreback

asv_bench/benchmarks/frame_methods.py

@@ -611,6 +611,9 @@ def time_frame_duplicated(self):
     def time_frame_duplicated_wide(self):
         self.df2.duplicated()
 
+    def time_frame_duplicated_subset(self):
+        self.df.duplicated(subset=["a"])
+
 
 class XS:
 

doc/source/development/contributing_environment.rst

@@ -222,7 +222,7 @@ Consult the docs for setting up pyenv `here <https://github.com/pyenv/pyenv>`__.
    pyenv virtualenv <version> <name-to-give-it>
 
    # For instance:
-   pyenv virtualenv 3.7.6 pandas-dev
+   pyenv virtualenv 3.9.10 pandas-dev
 
    # Activate the virtualenv
    pyenv activate pandas-dev

doc/source/getting_started/comparison/comparison_with_sql.rst

@@ -18,7 +18,7 @@ structure.
 
     url = (
         "https://raw.github.com/pandas-dev"
-        "/pandas/master/pandas/tests/io/data/csv/tips.csv"
+        "/pandas/main/pandas/tests/io/data/csv/tips.csv"
     )
     tips = pd.read_csv(url)
     tips

doc/source/whatsnew/v1.3.0.rst

@@ -811,7 +811,7 @@ Other Deprecations
 - Deprecated allowing scalars to be passed to the :class:`Categorical` constructor (:issue:`38433`)
 - Deprecated constructing :class:`CategoricalIndex` without passing list-like data (:issue:`38944`)
 - Deprecated allowing subclass-specific keyword arguments in the :class:`Index` constructor, use the specific subclass directly instead (:issue:`14093`, :issue:`21311`, :issue:`22315`, :issue:`26974`)
-- Deprecated the :meth:`astype` method of datetimelike (``timedelta64[ns]``, ``datetime64[ns]``, ``Datetime64TZDtype``, ``PeriodDtype``) to convert to integer dtypes, use ``values.view(...)`` instead (:issue:`38544`)
+- Deprecated the :meth:`astype` method of datetimelike (``timedelta64[ns]``, ``datetime64[ns]``, ``Datetime64TZDtype``, ``PeriodDtype``) to convert to integer dtypes, use ``values.view(...)`` instead (:issue:`38544`). This deprecation was later reverted in pandas 1.4.0.
 - Deprecated :meth:`MultiIndex.is_lexsorted` and :meth:`MultiIndex.lexsort_depth`, use :meth:`MultiIndex.is_monotonic_increasing` instead (:issue:`32259`)
 - Deprecated keyword ``try_cast`` in :meth:`Series.where`, :meth:`Series.mask`, :meth:`DataFrame.where`, :meth:`DataFrame.mask`; cast results manually if desired (:issue:`38836`)
 - Deprecated comparison of :class:`Timestamp` objects with ``datetime.date`` objects.  Instead of e.g. ``ts <= mydate`` use ``ts <= pd.Timestamp(mydate)`` or ``ts.date() <= mydate`` (:issue:`36131`)

doc/source/whatsnew/v1.4.0.rst

@@ -31,9 +31,11 @@ enhancement2
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
+- :meth:`MultiIndex.to_frame` now supports the argument ``allow_duplicates`` and raises on duplicate labels if it is missing or False (:issue:`45245`)
 - :class:`StringArray` now accepts array-likes containing nan-likes (``None``, ``np.nan``) for the ``values`` parameter in its constructor in addition to strings and :attr:`pandas.NA`. (:issue:`40839`)
 - Improved the rendering of ``categories`` in :class:`CategoricalIndex` (:issue:`45218`)
 - :meth:`to_numeric` now preserves float64 arrays when downcasting would generate values not representable in float32 (:issue:`43693`)
+- :meth:`.GroupBy.min` and :meth:`.GroupBy.max` now supports `Numba <https://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`45428`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -145,6 +147,7 @@ Other Deprecations
 - Deprecated behavior of :meth:`SparseArray.astype`, :meth:`Series.astype`, and :meth:`DataFrame.astype` with :class:`SparseDtype` when passing a non-sparse ``dtype``. In a future version, this will cast to that non-sparse dtype instead of wrapping it in a :class:`SparseDtype` (:issue:`34457`)
 - Deprecated behavior of :meth:`DatetimeIndex.intersection` and :meth:`DatetimeIndex.symmetric_difference` (``union`` behavior was already deprecated in version 1.3.0) with mixed timezones; in a future version both will be cast to UTC instead of object dtype (:issue:`39328`, :issue:`45357`)
 - Deprecated :meth:`DataFrame.iteritems`, :meth:`Series.iteritems`, :meth:`HDFStore.iteritems` in favor of :meth:`DataFrame.items`, :meth:`Series.items`, :meth:`HDFStore.items`  (:issue:`45321`)
+- Deprecated the ``__array_wrap__`` method of DataFrame and Series, rely on standard numpy ufuncs instead (:issue:`45451`)
 -
 
 
@@ -153,7 +156,7 @@ Other Deprecations
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
--
+- Performance improvement in :meth:`DataFrame.duplicated` when subset consists of only one column (:issue:`45236`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -203,7 +206,7 @@ Strings
 
 Interval
 ^^^^^^^^
--
+- Bug in :meth:`IntervalArray.__setitem__` when setting ``np.nan`` into an integer-backed array raising ``ValueError`` instead of ``TypeError`` (:issue:`45484`)
 -
 
 Indexing
@@ -213,6 +216,7 @@ Indexing
 - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`)
 - Bug when setting a value too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`, :issue:`32878`)
 - Bug in :meth:`loc.__setitem__` treating ``range`` keys as positional instead of label-based (:issue:`45479`)
+- Bug in :meth:`Series.__setitem__` where setting :attr:`NA` into a numeric-dtpye :class:`Series` would incorrectly upcast to object-dtype rather than treating the value as ``np.nan`` (:issue:`44199`)>>>>>>> main
 -
 
 Missing
@@ -228,7 +232,7 @@ MultiIndex
 I/O
 ^^^
 - Bug in :meth:`DataFrame.to_stata` where no error is raised if the :class:`DataFrame` contains ``-np.inf`` (:issue:`45350`)
--
+- Bug in :meth:`DataFrame.info` where a new line at the end of the output is omitted when called on an empty :class:`DataFrame` (:issue:`45494`)
 
 Period
 ^^^^^^

doc/source/whatsnew/v1.5.0.rst

@@ -199,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # refs/heads/ and refs/tags/ prefixes that would let us distinguish
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
+        # "stabilization", as well as "HEAD" and "main".
         tags = {r for r in refs if re.search(r"\d", r)}
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))

pandas/_version.py

@@ -126,6 +126,8 @@ def putmask_without_repeat(
     mask : np.ndarray[bool]
     new : Any
     """
+    new = setitem_datetimelike_compat(values, mask.sum(), new)
+
     if getattr(new, "ndim", 0) >= 1:
         new = new.astype(values.dtype, copy=False)
 

pandas/core/array_algos/putmask.py

@@ -427,7 +427,7 @@ def __contains__(self, item: object) -> bool | np.bool_:
             if not self._can_hold_na:
                 return False
             elif item is self.dtype.na_value or isinstance(item, self.dtype.type):
-                return self._hasnans
+                return self._hasna
             else:
                 return False
         else:
@@ -606,7 +606,7 @@ def isna(self) -> np.ndarray | ExtensionArraySupportsAnyAll:
         raise AbstractMethodError(self)
 
     @property
-    def _hasnans(self) -> bool:
+    def _hasna(self) -> bool:
         # GH#22680
         """
         Equivalent to `self.isna().any()`.
@@ -628,6 +628,16 @@ def _values_for_argsort(self) -> np.ndarray:
         See Also
         --------
         ExtensionArray.argsort : Return the indices that would sort this array.
+
+        Notes
+        -----
+        The caller is responsible for *not* modifying these values in-place, so
+        it is safe for implementors to give views on `self`.
+
+        Functions that use this (e.g. ExtensionArray.argsort) should ignore
+        entries with missing values in the original array (according to `self.isna()`).
+        This means that the corresponding entries in the returned array don't need to
+        be modified to sort correctly.
         """
         # Note: this is used in `ExtensionArray.argsort`.
         return np.array(self)
@@ -698,7 +708,7 @@ def argmin(self, skipna: bool = True) -> int:
         ExtensionArray.argmax
         """
         validate_bool_kwarg(skipna, "skipna")
-        if not skipna and self._hasnans:
+        if not skipna and self._hasna:
             raise NotImplementedError
         return nargminmax(self, "argmin")
 
@@ -722,7 +732,7 @@ def argmax(self, skipna: bool = True) -> int:
         ExtensionArray.argmin
         """
         validate_bool_kwarg(skipna, "skipna")
-        if not skipna and self._hasnans:
+        if not skipna and self._hasna:
             raise NotImplementedError
         return nargminmax(self, "argmax")
 
@@ -1534,6 +1544,9 @@ def _empty(cls, shape: Shape, dtype: ExtensionDtype):
         ExtensionDtype.empty
             ExtensionDtype.empty is the 'official' public version of this API.
         """
+        # Implementer note: while ExtensionDtype.empty is the public way to
+        # call this method, it is still required to implement this `_empty`
+        # method as well (it is called internally in pandas)
         obj = cls._from_sequence([], dtype=dtype)
 
         taker = np.broadcast_to(np.intp(-1), shape)

pandas/core/arrays/base.py

@@ -421,24 +421,6 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
         # coerce
         return self.to_numpy(dtype=dtype, na_value=na_value, copy=False)
 
-    def _values_for_argsort(self) -> np.ndarray:
-        """
-        Return values for sorting.
-
-        Returns
-        -------
-        ndarray
-            The transformed values should maintain the ordering between values
-            within the array.
-
-        See Also
-        --------
-        ExtensionArray.argsort : Return the indices that would sort this array.
-        """
-        data = self._data.copy()
-        data[self._mask] = -1
-        return data
-
     def _logical_method(self, other, op):
 
         assert op.__name__ in {"or_", "ror_", "and_", "rand_", "xor", "rxor"}

pandas/core/arrays/boolean.py

@@ -430,14 +430,6 @@ def astype(self, dtype, copy: bool = True):
         elif is_integer_dtype(dtype):
             # we deliberately ignore int32 vs. int64 here.
             # See https://github.com/pandas-dev/pandas/issues/24381 for more.
-            warnings.warn(
-                f"casting {self.dtype} values to int64 with .astype(...) is "
-                "deprecated and will raise in a future version. "
-                "Use .view(...) instead.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
-
             values = self.asi8
 
             if is_unsigned_integer_dtype(dtype):
@@ -849,7 +841,7 @@ def _isnan(self) -> npt.NDArray[np.bool_]:
         return self.asi8 == iNaT
 
     @property  # NB: override with cache_readonly in immutable subclasses
-    def _hasnans(self) -> bool:
+    def _hasna(self) -> bool:
         """
         return if I have any nans; enables various perf speedups
         """
@@ -874,7 +866,7 @@ def _maybe_mask_results(
 
         This is an internal routine.
         """
-        if self._hasnans:
+        if self._hasna:
             if convert:
                 result = result.astype(convert)
             if fill_value is None:
@@ -1133,7 +1125,7 @@ def _add_timedelta_arraylike(self, other):
         new_values = checked_add_with_arr(
             self_i8, other_i8, arr_mask=self._isnan, b_mask=other._isnan
         )
-        if self._hasnans or other._hasnans:
+        if self._hasna or other._hasna:
             mask = self._isnan | other._isnan
             np.putmask(new_values, mask, iNaT)
 

pandas/core/arrays/datetimelike.py

@@ -738,7 +738,7 @@ def _sub_datetime_arraylike(self, other):
         other_i8 = other.asi8
         arr_mask = self._isnan | other._isnan
         new_values = checked_add_with_arr(self_i8, -other_i8, arr_mask=arr_mask)
-        if self._hasnans or other._hasnans:
+        if self._hasna or other._hasna:
             np.putmask(new_values, arr_mask, iNaT)
         return new_values.view("timedelta64[ns]")
 
@@ -1356,7 +1356,7 @@ def isocalendar(self) -> DataFrame:
         iso_calendar_df = DataFrame(
             sarray, columns=["year", "week", "day"], dtype="UInt32"
         )
-        if self._hasnans:
+        if self._hasna:
             iso_calendar_df.iloc[self._isnan] = None
         return iso_calendar_df
 

pandas/core/arrays/datetimes.py

@@ -244,9 +244,6 @@ def _coerce_to_array(
     ) -> tuple[np.ndarray, np.ndarray]:
         return coerce_to_array(value, dtype=dtype, copy=copy)
 
-    def _values_for_argsort(self) -> np.ndarray:
-        return self._data
-
 
 _dtype_docstring = """
 An ExtensionDtype for {dtype} data.

pandas/core/arrays/floating.py

@@ -301,25 +301,6 @@ def _coerce_to_array(
     ) -> tuple[np.ndarray, np.ndarray]:
         return coerce_to_array(value, dtype=dtype, copy=copy)
 
-    def _values_for_argsort(self) -> np.ndarray:
-        """
-        Return values for sorting.
-
-        Returns
-        -------
-        ndarray
-            The transformed values should maintain the ordering between values
-            within the array.
-
-        See Also
-        --------
-        ExtensionArray.argsort : Return the indices that would sort this array.
-        """
-        data = self._data.copy()
-        if self._mask.any():
-            data[self._mask] = data.min() - 1
-        return data
-
 
 _dtype_docstring = """
 An ExtensionDtype for {dtype} integer data.

pandas/core/arrays/integer.py

@@ -1140,7 +1140,9 @@ def _validate_setitem_value(self, value):
             return self._validate_listlike(value)
 
         if needs_float_conversion:
-            raise ValueError("Cannot set float NaN to integer-backed IntervalArray")
+            # GH#45484 TypeError, not ValueError, matches what we get with
+            #  non-NA un-holdable value.
+            raise TypeError("Cannot set float NaN to integer-backed IntervalArray")
         return value_left, value_right
 
     def value_counts(self, dropna: bool = True):

pandas/core/arrays/interval.py

@@ -757,6 +757,10 @@ def factorize(self, na_sentinel: int = -1) -> tuple[np.ndarray, ExtensionArray]:
         uniques_ea = type(self)(uniques, np.zeros(len(uniques), dtype=bool))
         return codes, uniques_ea
 
+    @doc(ExtensionArray._values_for_argsort)
+    def _values_for_argsort(self) -> np.ndarray:
+        return self._data
+
     def value_counts(self, dropna: bool = True) -> Series:
         """
         Returns a Series containing counts of each unique value.

pandas/core/arrays/masked.py

@@ -548,7 +548,7 @@ def _time_shift(self, periods: int, freq=None) -> PeriodArray:
                 f"{type(self).__name__}._time_shift"
             )
         values = self.asi8 + periods * self.freq.n
-        if self._hasnans:
+        if self._hasna:
             values[self._isnan] = iNaT
         return type(self)(values, freq=self.freq)
 
@@ -618,7 +618,7 @@ def asfreq(self, freq=None, how: str = "E") -> PeriodArray:
 
         new_data = period_asfreq_arr(ordinal, base1, base2, end)
 
-        if self._hasnans:
+        if self._hasna:
             new_data[self._isnan] = iNaT
 
         return type(self)(new_data, freq=freq)
@@ -645,7 +645,7 @@ def _format_native_types(
         else:
             formatter = lambda dt: str(dt)
 
-        if self._hasnans:
+        if self._hasna:
             mask = self._isnan
             values[mask] = na_rep
             imask = ~mask
@@ -712,7 +712,7 @@ def _sub_period(self, other):
         new_data = asi8 - other.ordinal
         new_data = np.array([self.freq * x for x in new_data])
 
-        if self._hasnans:
+        if self._hasna:
             new_data[self._isnan] = NaT
 
         return new_data
@@ -739,7 +739,7 @@ def _sub_period_array(self, other):
         )
 
         new_values = np.array([self.freq.base * x for x in new_values])
-        if self._hasnans or other._hasnans:
+        if self._hasna or other._hasna:
             mask = self._isnan | other._isnan
             new_values[mask] = NaT
         return new_values

pandas/core/arrays/period.py

@@ -82,7 +82,7 @@ def _field_accessor(name: str, alias: str, docstring: str):
     def f(self) -> np.ndarray:
         values = self.asi8
         result = get_timedelta_field(values, alias)
-        if self._hasnans:
+        if self._hasna:
             result = self._maybe_mask_results(
                 result, fill_value=None, convert="float64"
             )
@@ -911,7 +911,7 @@ def components(self) -> DataFrame:
             "microseconds",
             "nanoseconds",
         ]
-        hasnans = self._hasnans
+        hasnans = self._hasna
         if hasnans:
 
             def f(x):