Merge branch 'main' into pandas-devgh-50703

Author: lpizzinidev

.github/workflows/python-dev.yml

@@ -73,7 +73,7 @@ jobs:
       run: |
         python --version
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
+        python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
         python -m pip install versioneer[toml]
         python -m pip install python-dateutil pytz cython hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17

doc/source/whatsnew/v2.0.0.rst

@@ -83,34 +83,40 @@ be set to ``"pyarrow"`` to return pyarrow-backed, nullable :class:`ArrowDtype` (
         df_pyarrow = pd.read_csv(data, use_nullable_dtypes=True, engine="pyarrow")
     df_pyarrow.dtypes
 
-Copy on write improvements
+Copy-on-Write improvements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A new lazy copy mechanism that defers the copy until the object in question is modified
-was added to the following methods:
-
-- :meth:`DataFrame.reset_index` / :meth:`Series.reset_index`
-- :meth:`DataFrame.set_index` / :meth:`Series.set_index`
-- :meth:`DataFrame.set_axis` / :meth:`Series.set_axis`
-- :meth:`DataFrame.rename_axis` / :meth:`Series.rename_axis`
-- :meth:`DataFrame.rename_columns`
-- :meth:`DataFrame.reindex` / :meth:`Series.reindex`
-- :meth:`DataFrame.reindex_like` / :meth:`Series.reindex_like`
-- :meth:`DataFrame.assign`
-- :meth:`DataFrame.drop`
-- :meth:`DataFrame.dropna` / :meth:`Series.dropna`
-- :meth:`DataFrame.select_dtypes`
-- :meth:`DataFrame.align` / :meth:`Series.align`
-- :meth:`Series.to_frame`
-- :meth:`DataFrame.rename` / :meth:`Series.rename`
-- :meth:`DataFrame.add_prefix` / :meth:`Series.add_prefix`
-- :meth:`DataFrame.add_suffix` / :meth:`Series.add_suffix`
-- :meth:`DataFrame.drop_duplicates` / :meth:`Series.drop_duplicates`
-- :meth:`DataFrame.reorder_levels` / :meth:`Series.reorder_levels`
-
-These methods return views when copy on write is enabled, which provides a significant
-performance improvement compared to the regular execution (:issue:`49473`). Copy on write
-can be enabled through
+- A new lazy copy mechanism that defers the copy until the object in question is modified
+  was added to the following methods:
+
+  - :meth:`DataFrame.reset_index` / :meth:`Series.reset_index`
+  - :meth:`DataFrame.set_index` / :meth:`Series.set_index`
+  - :meth:`DataFrame.set_axis` / :meth:`Series.set_axis`
+  - :meth:`DataFrame.rename_axis` / :meth:`Series.rename_axis`
+  - :meth:`DataFrame.rename_columns`
+  - :meth:`DataFrame.reindex` / :meth:`Series.reindex`
+  - :meth:`DataFrame.reindex_like` / :meth:`Series.reindex_like`
+  - :meth:`DataFrame.assign`
+  - :meth:`DataFrame.drop`
+  - :meth:`DataFrame.dropna` / :meth:`Series.dropna`
+  - :meth:`DataFrame.select_dtypes`
+  - :meth:`DataFrame.align` / :meth:`Series.align`
+  - :meth:`Series.to_frame`
+  - :meth:`DataFrame.rename` / :meth:`Series.rename`
+  - :meth:`DataFrame.add_prefix` / :meth:`Series.add_prefix`
+  - :meth:`DataFrame.add_suffix` / :meth:`Series.add_suffix`
+  - :meth:`DataFrame.drop_duplicates` / :meth:`Series.drop_duplicates`
+  - :meth:`DataFrame.reorder_levels` / :meth:`Series.reorder_levels`
+
+  These methods return views when Copy-on-Write is enabled, which provides a significant
+  performance improvement compared to the regular execution (:issue:`49473`).
+
+- Accessing a single column of a DataFrame as a Series (e.g. ``df["col"]``) now always
+  returns a new object every time it is constructed when Copy-on-Write is enabled (not
+  returning multiple times an identical, cached Series object). This ensures that those
+  Series objects correctly follow the Copy-on-Write rules (:issue:`49450`)
+
+Copy-on-Write can be enabled through
 
 .. code-block:: python
 
@@ -563,8 +569,7 @@ Deprecations
 ~~~~~~~~~~~~
 - Deprecated argument ``infer_datetime_format`` in :func:`to_datetime` and :func:`read_csv`, as a strict version of it is now the default (:issue:`48621`)
 - Deprecated :func:`pandas.io.sql.execute`(:issue:`50185`)
--
-
+- :meth:`Index.is_integer` has been deprecated. Use :func:`pandas.api.types.is_integer_dtype` instead (:issue:`50042`)
 - :meth:`Index.is_floating` has been deprecated. Use :func:`pandas.api.types.is_float_dtype` instead (:issue:`50042`)
 
 .. ---------------------------------------------------------------------------

pandas/_testing/asserters.py

@@ -15,6 +15,7 @@
     is_bool,
     is_categorical_dtype,
     is_extension_array_dtype,
+    is_integer_dtype,
     is_interval_dtype,
     is_number,
     is_numeric_dtype,
@@ -1335,7 +1336,7 @@ def assert_indexing_slices_equivalent(ser: Series, l_slc: slice, i_slc: slice) -
 
     assert_series_equal(ser.loc[l_slc], expected)
 
-    if not ser.index.is_integer():
+    if not is_integer_dtype(ser.index):
         # For integer indices, .loc and plain getitem are position-based.
         assert_series_equal(ser[l_slc], expected)
 

pandas/core/frame.py

@@ -35,7 +35,10 @@
 import numpy as np
 from numpy import ma
 
-from pandas._config import get_option
+from pandas._config import (
+    get_option,
+    using_copy_on_write,
+)
 
 from pandas._libs import (
     algos as libalgos,
@@ -4153,6 +4156,10 @@ def _clear_item_cache(self) -> None:
 
     def _get_item_cache(self, item: Hashable) -> Series:
         """Return the cached item, item represents a label indexer."""
+        if using_copy_on_write():
+            loc = self.columns.get_loc(item)
+            return self._ixs(loc, axis=1)
+
         cache = self._item_cache
         res = cache.get(item)
         if res is None:

pandas/core/generic.py

@@ -3676,6 +3676,8 @@ def _maybe_update_cacher(
         verify_is_copy : bool, default True
             Provide is_copy checks.
         """
+        if using_copy_on_write():
+            return
 
         if verify_is_copy:
             self._check_setitem_copy(t="referent")

pandas/core/groupby/grouper.py

@@ -14,6 +14,8 @@
 
 import numpy as np
 
+from pandas._config import using_copy_on_write
+
 from pandas._typing import (
     ArrayLike,
     Axis,
@@ -887,6 +889,13 @@ def is_in_axis(key) -> bool:
     def is_in_obj(gpr) -> bool:
         if not hasattr(gpr, "name"):
             return False
+        if using_copy_on_write():
+            # For the CoW case, we need an equality check as the identity check
+            # no longer works (each Series from column access is a new object)
+            try:
+                return gpr.equals(obj[gpr.name])
+            except (AttributeError, KeyError, IndexError, InvalidIndexError):
+                return False
         try:
             return gpr is obj[gpr.name]
         except (KeyError, IndexError, InvalidIndexError):

pandas/core/indexes/base.py

@@ -98,6 +98,7 @@
     is_float_dtype,
     is_hashable,
     is_integer,
+    is_integer_dtype,
     is_interval_dtype,
     is_iterator,
     is_list_like,
@@ -2188,7 +2189,7 @@ def is_boolean(self) -> bool:
 
         See Also
         --------
-        is_integer : Check if the Index only consists of integers.
+        is_integer : Check if the Index only consists of integers (deprecated).
         is_floating : Check if the Index is a floating type (deprecated).
         is_numeric : Check if the Index only consists of numeric data.
         is_object : Check if the Index is of the object dtype.
@@ -2216,6 +2217,9 @@ def is_integer(self) -> bool:
         """
         Check if the Index only consists of integers.
 
+        .. deprecated:: 2.0.0
+            Use `pandas.api.types.is_integer_dtype` instead.
+
         Returns
         -------
         bool
@@ -2244,6 +2248,12 @@ def is_integer(self) -> bool:
         >>> idx.is_integer()
         False
         """
+        warnings.warn(
+            f"{type(self).__name__}.is_integer is deprecated. "
+            "Use pandas.api.types.is_integer_dtype instead.",
+            FutureWarning,
+            stacklevel=find_stack_level(),
+        )
         return self.inferred_type in ["integer"]
 
     @final
@@ -2266,7 +2276,7 @@ def is_floating(self) -> bool:
         See Also
         --------
         is_boolean : Check if the Index only consists of booleans.
-        is_integer : Check if the Index only consists of integers.
+        is_integer : Check if the Index only consists of integers (deprecated).
         is_numeric : Check if the Index only consists of numeric data.
         is_object : Check if the Index is of the object dtype.
         is_categorical : Check if the Index holds categorical data.
@@ -2311,7 +2321,7 @@ def is_numeric(self) -> bool:
         See Also
         --------
         is_boolean : Check if the Index only consists of booleans.
-        is_integer : Check if the Index only consists of integers.
+        is_integer : Check if the Index only consists of integers (deprecated).
         is_floating : Check if the Index is a floating type (deprecated).
         is_object : Check if the Index is of the object dtype.
         is_categorical : Check if the Index holds categorical data.
@@ -2354,7 +2364,7 @@ def is_object(self) -> bool:
         See Also
         --------
         is_boolean : Check if the Index only consists of booleans.
-        is_integer : Check if the Index only consists of integers.
+        is_integer : Check if the Index only consists of integers (deprecated).
         is_floating : Check if the Index is a floating type (deprecated).
         is_numeric : Check if the Index only consists of numeric data.
         is_categorical : Check if the Index holds categorical data.
@@ -2395,7 +2405,7 @@ def is_categorical(self) -> bool:
         --------
         CategoricalIndex : Index for categorical data.
         is_boolean : Check if the Index only consists of booleans.
-        is_integer : Check if the Index only consists of integers.
+        is_integer : Check if the Index only consists of integers (deprecated).
         is_floating : Check if the Index is a floating type (deprecated).
         is_numeric : Check if the Index only consists of numeric data.
         is_object : Check if the Index is of the object dtype.
@@ -2438,7 +2448,7 @@ def is_interval(self) -> bool:
         --------
         IntervalIndex : Index for Interval objects.
         is_boolean : Check if the Index only consists of booleans.
-        is_integer : Check if the Index only consists of integers.
+        is_integer : Check if the Index only consists of integers (deprecated).
         is_floating : Check if the Index is a floating type (deprecated).
         is_numeric : Check if the Index only consists of numeric data.
         is_object : Check if the Index is of the object dtype.
@@ -3877,7 +3887,7 @@ def is_int(v):
 
         if kind == "getitem":
             # called from the getitem slicers, validate that we are in fact integers
-            if self.is_integer() or is_index_slice:
+            if is_integer_dtype(self.dtype) or is_index_slice:
                 # Note: these checks are redundant if we know is_index_slice
                 self._validate_indexer("slice", key.start, "getitem")
                 self._validate_indexer("slice", key.stop, "getitem")

pandas/core/series.py

@@ -1242,6 +1242,8 @@ def _set_as_cached(self, item, cacher) -> None:
         Set the _cacher attribute on the calling object with a weakref to
         cacher.
         """
+        if using_copy_on_write():
+            return
         self._cacher = (item, weakref.ref(cacher))
 
     def _clear_item_cache(self) -> None:
@@ -1265,6 +1267,10 @@ def _maybe_update_cacher(
         """
         See NDFrame._maybe_update_cacher.__doc__
         """
+        # for CoW, we never want to update the parent DataFrame cache
+        # if the Series changed, but don't keep track of any cacher
+        if using_copy_on_write():
+            return
         cacher = getattr(self, "_cacher", None)
         if cacher is not None:
             assert self.ndim == 1
@@ -1274,13 +1280,7 @@ def _maybe_update_cacher(
             # a copy
             if ref is None:
                 del self._cacher
-            # for CoW, we never want to update the parent DataFrame cache
-            # if the Series changed, and always pop the cached item
-            elif (
-                not using_copy_on_write()
-                and len(self) == len(ref)
-                and self.name in ref.columns
-            ):
+            elif len(self) == len(ref) and self.name in ref.columns:
                 # GH#42530 self.name must be in ref.columns
                 # to ensure column still in dataframe
                 # otherwise, either self or ref has swapped in new arrays

pandas/tests/copy_view/test_indexing.py

@@ -820,6 +820,46 @@ def test_column_as_series_set_with_upcast(using_copy_on_write, using_array_manag
         tm.assert_frame_equal(df, df_orig)
 
 
+@pytest.mark.parametrize(
+    "method",
+    [
+        lambda df: df["a"],
+        lambda df: df.loc[:, "a"],
+        lambda df: df.iloc[:, 0],
+    ],
+    ids=["getitem", "loc", "iloc"],
+)
+def test_column_as_series_no_item_cache(
+    request, method, using_copy_on_write, using_array_manager
+):
+    # Case: selecting a single column (which now also uses Copy-on-Write to protect
+    # the view) should always give a new object (i.e. not make use of a cache)
+    df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [0.1, 0.2, 0.3]})
+    df_orig = df.copy()
+
+    s1 = method(df)
+    s2 = method(df)
+
+    is_iloc = request.node.callspec.id == "iloc"
+    if using_copy_on_write or is_iloc:
+        assert s1 is not s2
+    else:
+        assert s1 is s2
+
+    if using_copy_on_write or using_array_manager:
+        s1.iloc[0] = 0
+    else:
+        with pd.option_context("chained_assignment", "warn"):
+            with tm.assert_produces_warning(SettingWithCopyWarning):
+                s1.iloc[0] = 0
+
+    if using_copy_on_write:
+        tm.assert_series_equal(s2, df_orig["a"])
+        tm.assert_frame_equal(df, df_orig)
+    else:
+        assert s2.iloc[0] == 0
+
+
 # TODO add tests for other indexing methods on the Series
 
 

pandas/tests/frame/indexing/test_xs.py

@@ -36,7 +36,8 @@ def four_level_index_dataframe():
 
 
 class TestXS:
-    def test_xs(self, float_frame, datetime_frame):
+    def test_xs(self, float_frame, datetime_frame, using_copy_on_write):
+        float_frame_orig = float_frame.copy()
         idx = float_frame.index[5]
         xs = float_frame.xs(idx)
         for item, value in xs.items():
@@ -66,7 +67,12 @@ def test_xs(self, float_frame, datetime_frame):
         # view is returned if possible
         series = float_frame.xs("A", axis=1)
         series[:] = 5
-        assert (expected == 5).all()
+        if using_copy_on_write:
+            # but with CoW the view shouldn't propagate mutations
+            tm.assert_series_equal(float_frame["A"], float_frame_orig["A"])
+            assert not (expected == 5).all()
+        else:
+            assert (expected == 5).all()
 
     def test_xs_corner(self):
         # pathological mixed-type reordering case

pandas/tests/frame/methods/test_cov_corr.py

@@ -206,7 +206,7 @@ def test_corr_nullable_integer(self, nullable_column, other_column, method):
         expected = DataFrame(np.ones((2, 2)), columns=["a", "b"], index=["a", "b"])
         tm.assert_frame_equal(result, expected)
 
-    def test_corr_item_cache(self):
+    def test_corr_item_cache(self, using_copy_on_write):
         # Check that corr does not lead to incorrect entries in item_cache
 
         df = DataFrame({"A": range(10)})
@@ -217,11 +217,16 @@ def test_corr_item_cache(self):
 
         _ = df.corr(numeric_only=True)
 
-        # Check that the corr didn't break link between ser and df
-        ser.values[0] = 99
-        assert df.loc[0, "A"] == 99
-        assert df["A"] is ser
-        assert df.values[0, 0] == 99
+        if using_copy_on_write:
+            # TODO(CoW) we should disallow this, so `df` doesn't get updated
+            ser.values[0] = 99
+            assert df.loc[0, "A"] == 99
+        else:
+            # Check that the corr didn't break link between ser and df
+            ser.values[0] = 99
+            assert df.loc[0, "A"] == 99
+            assert df["A"] is ser
+            assert df.values[0, 0] == 99
 
     @pytest.mark.parametrize("length", [2, 20, 200, 2000])
     def test_corr_for_constant_columns(self, length):