Merge branch 'main' into bug-range

Author: jbrockmendel

.github/workflows/code-checks.yml

@@ -74,7 +74,7 @@ jobs:
 
     - name: Install pyright
       # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g [email protected].202
+      run: npm install -g [email protected].212
 
     - name: Build Pandas
       id: build

.pre-commit-config.yaml

@@ -84,8 +84,8 @@ repos:
         pass_filenames: false
         types: [python]
         stages: [manual]
-        # note: keep version in sync with .github/workflows/ci.yml
-        additional_dependencies: ['[email protected].202']
+        # note: keep version in sync with .github/workflows/code-checks.yml
+        additional_dependencies: ['[email protected].212']
 -   repo: local
     hooks:
     -   id: flake8-rst

asv_bench/benchmarks/frame_methods.py

@@ -611,6 +611,9 @@ def time_frame_duplicated(self):
     def time_frame_duplicated_wide(self):
         self.df2.duplicated()
 
+    def time_frame_duplicated_subset(self):
+        self.df.duplicated(subset=["a"])
+
 
 class XS:
 

doc/source/development/contributing_environment.rst

@@ -222,7 +222,7 @@ Consult the docs for setting up pyenv `here <https://github.com/pyenv/pyenv>`__.
    pyenv virtualenv <version> <name-to-give-it>
 
    # For instance:
-   pyenv virtualenv 3.7.6 pandas-dev
+   pyenv virtualenv 3.9.10 pandas-dev
 
    # Activate the virtualenv
    pyenv activate pandas-dev

doc/source/getting_started/comparison/comparison_with_sql.rst

@@ -18,7 +18,7 @@ structure.
 
     url = (
         "https://raw.github.com/pandas-dev"
-        "/pandas/master/pandas/tests/io/data/csv/tips.csv"
+        "/pandas/main/pandas/tests/io/data/csv/tips.csv"
     )
     tips = pd.read_csv(url)
     tips

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.4
 .. toctree::
    :maxdepth: 2
 
+   v1.4.1
    v1.4.0
 
 Version 1.3

doc/source/whatsnew/v1.3.0.rst

@@ -811,7 +811,7 @@ Other Deprecations
 - Deprecated allowing scalars to be passed to the :class:`Categorical` constructor (:issue:`38433`)
 - Deprecated constructing :class:`CategoricalIndex` without passing list-like data (:issue:`38944`)
 - Deprecated allowing subclass-specific keyword arguments in the :class:`Index` constructor, use the specific subclass directly instead (:issue:`14093`, :issue:`21311`, :issue:`22315`, :issue:`26974`)
-- Deprecated the :meth:`astype` method of datetimelike (``timedelta64[ns]``, ``datetime64[ns]``, ``Datetime64TZDtype``, ``PeriodDtype``) to convert to integer dtypes, use ``values.view(...)`` instead (:issue:`38544`)
+- Deprecated the :meth:`astype` method of datetimelike (``timedelta64[ns]``, ``datetime64[ns]``, ``Datetime64TZDtype``, ``PeriodDtype``) to convert to integer dtypes, use ``values.view(...)`` instead (:issue:`38544`). This deprecation was later reverted in pandas 1.4.0.
 - Deprecated :meth:`MultiIndex.is_lexsorted` and :meth:`MultiIndex.lexsort_depth`, use :meth:`MultiIndex.is_monotonic_increasing` instead (:issue:`32259`)
 - Deprecated keyword ``try_cast`` in :meth:`Series.where`, :meth:`Series.mask`, :meth:`DataFrame.where`, :meth:`DataFrame.mask`; cast results manually if desired (:issue:`38836`)
 - Deprecated comparison of :class:`Timestamp` objects with ``datetime.date`` objects.  Instead of e.g. ``ts <= mydate`` use ``ts <= pd.Timestamp(mydate)`` or ``ts.date() <= mydate`` (:issue:`36131`)

doc/source/whatsnew/v1.4.0.rst

@@ -0,0 +1,45 @@
+.. _whatsnew_141:
+
+What's new in 1.4.1 (February ??, 2022)
+---------------------------------------
+
+These are the changes in pandas 1.4.1. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_141.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_141.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_141.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_141.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.4.0..v1.4.1|HEAD

doc/source/whatsnew/v1.4.1.rst

@@ -31,9 +31,11 @@ enhancement2
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
+- :meth:`MultiIndex.to_frame` now supports the argument ``allow_duplicates`` and raises on duplicate labels if it is missing or False (:issue:`45245`)
 - :class:`StringArray` now accepts array-likes containing nan-likes (``None``, ``np.nan``) for the ``values`` parameter in its constructor in addition to strings and :attr:`pandas.NA`. (:issue:`40839`)
 - Improved the rendering of ``categories`` in :class:`CategoricalIndex` (:issue:`45218`)
 - :meth:`to_numeric` now preserves float64 arrays when downcasting would generate values not representable in float32 (:issue:`43693`)
+- :meth:`.GroupBy.min` and :meth:`.GroupBy.max` now supports `Numba <https://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`45428`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -70,9 +72,10 @@ If installed, we now require:
 +-----------------+-----------------+----------+---------+
 | Package         | Minimum Version | Required | Changed |
 +=================+=================+==========+=========+
-|                 |                 |    X     |    X    |
+| mypy (dev)      | 0.931           |          |    X    |
 +-----------------+-----------------+----------+---------+
 
+
 For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
 Optional libraries below the lowest tested version may still work, but are not considered supported.
@@ -145,6 +148,7 @@ Other Deprecations
 - Deprecated behavior of :meth:`SparseArray.astype`, :meth:`Series.astype`, and :meth:`DataFrame.astype` with :class:`SparseDtype` when passing a non-sparse ``dtype``. In a future version, this will cast to that non-sparse dtype instead of wrapping it in a :class:`SparseDtype` (:issue:`34457`)
 - Deprecated behavior of :meth:`DatetimeIndex.intersection` and :meth:`DatetimeIndex.symmetric_difference` (``union`` behavior was already deprecated in version 1.3.0) with mixed timezones; in a future version both will be cast to UTC instead of object dtype (:issue:`39328`, :issue:`45357`)
 - Deprecated :meth:`DataFrame.iteritems`, :meth:`Series.iteritems`, :meth:`HDFStore.iteritems` in favor of :meth:`DataFrame.items`, :meth:`Series.items`, :meth:`HDFStore.items`  (:issue:`45321`)
+- Deprecated the ``__array_wrap__`` method of DataFrame and Series, rely on standard numpy ufuncs instead (:issue:`45451`)
 -
 
 
@@ -153,7 +157,7 @@ Other Deprecations
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
--
+- Performance improvement in :meth:`DataFrame.duplicated` when subset consists of only one column (:issue:`45236`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -203,7 +207,7 @@ Strings
 
 Interval
 ^^^^^^^^
--
+- Bug in :meth:`IntervalArray.__setitem__` when setting ``np.nan`` into an integer-backed array raising ``ValueError`` instead of ``TypeError`` (:issue:`45484`)
 -
 
 Indexing
@@ -213,6 +217,7 @@ Indexing
 - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`)
 - Bug when setting a value too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`, :issue:`32878`)
 - Bug in :meth:`loc.__setitem__` treating ``range`` keys as positional instead of label-based (:issue:`45479`)
+- Bug in :meth:`Series.__setitem__` where setting :attr:`NA` into a numeric-dtpye :class:`Series` would incorrectly upcast to object-dtype rather than treating the value as ``np.nan`` (:issue:`44199`)
 -
 
 Missing
@@ -228,7 +233,7 @@ MultiIndex
 I/O
 ^^^
 - Bug in :meth:`DataFrame.to_stata` where no error is raised if the :class:`DataFrame` contains ``-np.inf`` (:issue:`45350`)
--
+- Bug in :meth:`DataFrame.info` where a new line at the end of the output is omitted when called on an empty :class:`DataFrame` (:issue:`45494`)
 
 Period
 ^^^^^^
@@ -242,7 +247,7 @@ Plotting
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
--
+- Bug in :meth:`DataFrame.resample` ignoring ``closed="right"`` on :class:`TimedeltaIndex` (:issue:`45414`)
 -
 
 Reshaping

doc/source/whatsnew/v1.5.0.rst

@@ -24,7 +24,7 @@ dependencies:
   - flake8-bugbear=21.3.2  # used by flake8, find likely bugs
   - flake8-comprehensions=3.7.0  # used by flake8, linting of unnecessary comprehensions
   - isort>=5.2.1  # check that imports are in the right order
-  - mypy=0.930
+  - mypy=0.931
   - pre-commit>=2.9.2
   - pycodestyle  # used by flake8
   - pyupgrade

environment.yml

@@ -228,6 +228,7 @@ static PyObject *get_values(PyObject *obj) {
             PyErr_Clear();
         } else if (PyObject_HasAttrString(values, "__array__")) {
             // We may have gotten a Categorical or Sparse array so call np.array
+            Py_DECREF(values);
             values = PyObject_CallMethod(values, "__array__", NULL);
         } else if (!PyArray_CheckExact(values)) {
             // Didn't get a numpy array, so keep trying

pandas/_libs/src/ujson/python/objToJSON.c

@@ -199,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # refs/heads/ and refs/tags/ prefixes that would let us distinguish
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
+        # "stabilization", as well as "HEAD" and "main".
         tags = {r for r in refs if re.search(r"\d", r)}
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))

pandas/_version.py

@@ -126,6 +126,8 @@ def putmask_without_repeat(
     mask : np.ndarray[bool]
     new : Any
     """
+    new = setitem_datetimelike_compat(values, mask.sum(), new)
+
     if getattr(new, "ndim", 0) >= 1:
         new = new.astype(values.dtype, copy=False)
 

pandas/core/array_algos/putmask.py

@@ -427,7 +427,7 @@ def __contains__(self, item: object) -> bool | np.bool_:
             if not self._can_hold_na:
                 return False
             elif item is self.dtype.na_value or isinstance(item, self.dtype.type):
-                return self._hasnans
+                return self._hasna
             else:
                 return False
         else:
@@ -606,7 +606,7 @@ def isna(self) -> np.ndarray | ExtensionArraySupportsAnyAll:
         raise AbstractMethodError(self)
 
     @property
-    def _hasnans(self) -> bool:
+    def _hasna(self) -> bool:
         # GH#22680
         """
         Equivalent to `self.isna().any()`.
@@ -628,6 +628,16 @@ def _values_for_argsort(self) -> np.ndarray:
         See Also
         --------
         ExtensionArray.argsort : Return the indices that would sort this array.
+
+        Notes
+        -----
+        The caller is responsible for *not* modifying these values in-place, so
+        it is safe for implementors to give views on `self`.
+
+        Functions that use this (e.g. ExtensionArray.argsort) should ignore
+        entries with missing values in the original array (according to `self.isna()`).
+        This means that the corresponding entries in the returned array don't need to
+        be modified to sort correctly.
         """
         # Note: this is used in `ExtensionArray.argsort`.
         return np.array(self)
@@ -698,7 +708,7 @@ def argmin(self, skipna: bool = True) -> int:
         ExtensionArray.argmax
         """
         validate_bool_kwarg(skipna, "skipna")
-        if not skipna and self._hasnans:
+        if not skipna and self._hasna:
             raise NotImplementedError
         return nargminmax(self, "argmin")
 
@@ -722,7 +732,7 @@ def argmax(self, skipna: bool = True) -> int:
         ExtensionArray.argmin
         """
         validate_bool_kwarg(skipna, "skipna")
-        if not skipna and self._hasnans:
+        if not skipna and self._hasna:
             raise NotImplementedError
         return nargminmax(self, "argmax")
 
@@ -1534,6 +1544,9 @@ def _empty(cls, shape: Shape, dtype: ExtensionDtype):
         ExtensionDtype.empty
             ExtensionDtype.empty is the 'official' public version of this API.
         """
+        # Implementer note: while ExtensionDtype.empty is the public way to
+        # call this method, it is still required to implement this `_empty`
+        # method as well (it is called internally in pandas)
         obj = cls._from_sequence([], dtype=dtype)
 
         taker = np.broadcast_to(np.intp(-1), shape)

pandas/core/arrays/base.py

@@ -421,24 +421,6 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
         # coerce
         return self.to_numpy(dtype=dtype, na_value=na_value, copy=False)
 
-    def _values_for_argsort(self) -> np.ndarray:
-        """
-        Return values for sorting.
-
-        Returns
-        -------
-        ndarray
-            The transformed values should maintain the ordering between values
-            within the array.
-
-        See Also
-        --------
-        ExtensionArray.argsort : Return the indices that would sort this array.
-        """
-        data = self._data.copy()
-        data[self._mask] = -1
-        return data
-
     def _logical_method(self, other, op):
 
         assert op.__name__ in {"or_", "ror_", "and_", "rand_", "xor", "rxor"}

pandas/core/arrays/boolean.py

@@ -430,14 +430,6 @@ def astype(self, dtype, copy: bool = True):
         elif is_integer_dtype(dtype):
             # we deliberately ignore int32 vs. int64 here.
             # See https://github.com/pandas-dev/pandas/issues/24381 for more.
-            warnings.warn(
-                f"casting {self.dtype} values to int64 with .astype(...) is "
-                "deprecated and will raise in a future version. "
-                "Use .view(...) instead.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
-
             values = self.asi8
 
             if is_unsigned_integer_dtype(dtype):
@@ -849,7 +841,7 @@ def _isnan(self) -> npt.NDArray[np.bool_]:
         return self.asi8 == iNaT
 
     @property  # NB: override with cache_readonly in immutable subclasses
-    def _hasnans(self) -> bool:
+    def _hasna(self) -> bool:
         """
         return if I have any nans; enables various perf speedups
         """
@@ -874,7 +866,7 @@ def _maybe_mask_results(
 
         This is an internal routine.
         """
-        if self._hasnans:
+        if self._hasna:
             if convert:
                 result = result.astype(convert)
             if fill_value is None:
@@ -1133,7 +1125,7 @@ def _add_timedelta_arraylike(self, other):
         new_values = checked_add_with_arr(
             self_i8, other_i8, arr_mask=self._isnan, b_mask=other._isnan
         )
-        if self._hasnans or other._hasnans:
+        if self._hasna or other._hasna:
             mask = self._isnan | other._isnan
             np.putmask(new_values, mask, iNaT)
 

pandas/core/arrays/datetimelike.py

@@ -738,7 +738,7 @@ def _sub_datetime_arraylike(self, other):
         other_i8 = other.asi8
         arr_mask = self._isnan | other._isnan
         new_values = checked_add_with_arr(self_i8, -other_i8, arr_mask=arr_mask)
-        if self._hasnans or other._hasnans:
+        if self._hasna or other._hasna:
             np.putmask(new_values, arr_mask, iNaT)
         return new_values.view("timedelta64[ns]")
 
@@ -1356,7 +1356,7 @@ def isocalendar(self) -> DataFrame:
         iso_calendar_df = DataFrame(
             sarray, columns=["year", "week", "day"], dtype="UInt32"
         )
-        if self._hasnans:
+        if self._hasna:
             iso_calendar_df.iloc[self._isnan] = None
         return iso_calendar_df
 

pandas/core/arrays/datetimes.py

@@ -244,9 +244,6 @@ def _coerce_to_array(
     ) -> tuple[np.ndarray, np.ndarray]:
         return coerce_to_array(value, dtype=dtype, copy=copy)
 
-    def _values_for_argsort(self) -> np.ndarray:
-        return self._data
-
 
 _dtype_docstring = """
 An ExtensionDtype for {dtype} data.