Merge remote-tracking branch 'upstream/main' into tst/cln/unneeded_pa_checking

Author: mroeschke

.github/workflows/macos-windows.yml

@@ -18,8 +18,6 @@ env:
   PANDAS_CI: 1
   PYTEST_TARGET: pandas
   PATTERN: "not slow and not db and not network and not single_cpu"
-  ERROR_ON_WARNINGS: "1"
-
 
 permissions:
   contents: read

.github/workflows/ubuntu.yml

@@ -40,7 +40,6 @@ jobs:
           - name: "Minimum Versions"
             env_file: actions-38-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
-            error_on_warnings: "0"
           - name: "Locale: it_IT"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -65,12 +64,10 @@ jobs:
             env_file: actions-310.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
-            error_on_warnings: "0"
           - name: "Data Manager"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_data_manager: "array"
-            error_on_warnings: "0"
           - name: "Pypy"
             env_file: actions-pypy-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -79,7 +76,6 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
-            error_on_warnings: "0"
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "8"
@@ -99,7 +95,6 @@ jobs:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
-      ERROR_ON_WARNINGS: ${{ matrix.error_on_warnings || '1' }}
       LANG: ${{ matrix.lang || '' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}

asv_bench/benchmarks/frame_methods.py

@@ -17,6 +17,21 @@
 from .pandas_vb_common import tm
 
 
+class Clip:
+    params = [
+        ["float64", "Float64", "float64[pyarrow]"],
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        data = np.random.randn(100_000, 10)
+        df = DataFrame(data, dtype=dtype)
+        self.df = df
+
+    def time_clip(self, dtype):
+        self.df.clip(-1.0, 1.0)
+
+
 class GetNumericData:
     def setup(self):
         self.df = DataFrame(np.random.randn(10000, 25))

ci/run_tests.sh

@@ -30,13 +30,6 @@ if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
 fi
 
-if [[ "$ERROR_ON_WARNINGS" == "1" ]]; then
-  for pth in $(find pandas -name '*.py' -not -path "pandas/tests/*" | sed -e 's/\.py//g' -e 's/\/__init__//g' -e 's/\//./g');
-    do
-        PYTEST_CMD="$PYTEST_CMD -W error:::$pth"
-    done
-fi
-
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
 

doc/source/whatsnew/v2.0.0.rst

@@ -261,6 +261,8 @@ Copy-on-Write improvements
 - :meth:`DataFrame.replace` will now respect the Copy-on-Write mechanism
   when ``inplace=True``.
 
+- :meth:`DataFrame.transpose` will now respect the Copy-on-Write mechanism.
+
 - Arithmetic operations that can be inplace, e.g. ``ser *= 2`` will now respect the
   Copy-on-Write mechanism.
 

doc/source/whatsnew/v2.1.0.rst

@@ -100,7 +100,7 @@ Deprecations
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
--
+- Performance improvement in :meth:`DataFrame.clip` and :meth:`Series.clip` (:issue:`51472`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/_testing/__init__.py

@@ -215,6 +215,7 @@
     FLOAT_PYARROW_DTYPES_STR_REPR = [
         str(ArrowDtype(typ)) for typ in FLOAT_PYARROW_DTYPES
     ]
+    DECIMAL_PYARROW_DTYPES = [pa.decimal128(7, 3)]
     STRING_PYARROW_DTYPES = [pa.string()]
     BINARY_PYARROW_DTYPES = [pa.binary()]
 
@@ -239,6 +240,7 @@
     ALL_PYARROW_DTYPES = (
         ALL_INT_PYARROW_DTYPES
         + FLOAT_PYARROW_DTYPES
+        + DECIMAL_PYARROW_DTYPES
         + STRING_PYARROW_DTYPES
         + BINARY_PYARROW_DTYPES
         + TIME_PYARROW_DTYPES

pandas/_testing/_warnings.py

@@ -5,6 +5,7 @@
     nullcontext,
 )
 import re
+import sys
 from typing import (
     Generator,
     Literal,
@@ -163,6 +164,17 @@ def _assert_caught_no_extra_warnings(
 
     for actual_warning in caught_warnings:
         if _is_unexpected_warning(actual_warning, expected_warning):
+            # GH#38630 pytest.filterwarnings does not suppress these.
+            if actual_warning.category == ResourceWarning:
+                # GH 44732: Don't make the CI flaky by filtering SSL-related
+                # ResourceWarning from dependencies
+                if "unclosed <ssl.SSLSocket" in str(actual_warning.message):
+                    continue
+                # GH 44844: Matplotlib leaves font files open during the entire process
+                # upon import. Don't make CI flaky if ResourceWarning raised
+                # due to these open files.
+                if any("matplotlib" in mod for mod in sys.modules):
+                    continue
             extra_warnings.append(
                 (
                     actual_warning.category.__name__,

pandas/core/arrays/arrow/array.py

@@ -1091,6 +1091,7 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
             pa.types.is_integer(pa_type)
             or pa.types.is_floating(pa_type)
             or pa.types.is_duration(pa_type)
+            or pa.types.is_decimal(pa_type)
         ):
             # pyarrow only supports any/all for boolean dtype, we allow
             #  for other dtypes, matching our non-pyarrow behavior

pandas/core/arrays/arrow/dtype.py

@@ -201,7 +201,7 @@ def construct_from_string(cls, string: str) -> ArrowDtype:
         try:
             pa_dtype = pa.type_for_alias(base_type)
         except ValueError as err:
-            has_parameters = re.search(r"\[.*\]", base_type)
+            has_parameters = re.search(r"[\[\(].*[\]\)]", base_type)
             if has_parameters:
                 # Fallback to try common temporal types
                 try:

pandas/core/frame.py

@@ -3545,10 +3545,14 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
         if self._can_fast_transpose:
             # Note: tests pass without this, but this improves perf quite a bit.
             new_vals = self._values.T
-            if copy:
+            if copy and not using_copy_on_write():
                 new_vals = new_vals.copy()
 
-            result = self._constructor(new_vals, index=self.columns, columns=self.index)
+            result = self._constructor(
+                new_vals, index=self.columns, columns=self.index, copy=False
+            )
+            if using_copy_on_write() and len(self) > 0:
+                result._mgr.add_references(self._mgr)  # type: ignore[arg-type]
 
         elif (
             self._is_homogeneous_type and dtypes and is_extension_array_dtype(dtypes[0])

pandas/core/generic.py

@@ -4015,10 +4015,10 @@ class  animal  locomotion
 
         Get values at several indexes
 
-        >>> df.xs(('mammal', 'dog'))
-                    num_legs  num_wings
-        locomotion
-        walks              4          0
+        >>> df.xs(('mammal', 'dog', 'walks'))
+        num_legs     4
+        num_wings    0
+        Name: (mammal, dog, walks), dtype: int64
 
         Get values at specified index and level
 
@@ -7985,24 +7985,33 @@ def _clip_with_scalar(self, lower, upper, inplace: bool_t = False):
         ):
             raise ValueError("Cannot use an NA value as a clip threshold")
 
-        result = self
-        mask = isna(self._values)
+        mgr = self._mgr
 
-        with np.errstate(all="ignore"):
+        if inplace:
+            # cond (for putmask) identifies values to be updated.
+            # exclude boundary as values at the boundary should be no-ops.
             if upper is not None:
-                subset = self <= upper
-                result = result.where(subset, upper, axis=None, inplace=False)
+                cond = self > upper
+                mgr = mgr.putmask(mask=cond, new=upper, align=False)
             if lower is not None:
-                subset = self >= lower
-                result = result.where(subset, lower, axis=None, inplace=False)
-
-        if np.any(mask):
-            result[mask] = np.nan
+                cond = self < lower
+                mgr = mgr.putmask(mask=cond, new=lower, align=False)
+        else:
+            # cond (for where) identifies values to be left as-is.
+            # include boundary as values at the boundary should be no-ops.
+            mask = isna(self)
+            if upper is not None:
+                cond = mask | (self <= upper)
+                mgr = mgr.where(other=upper, cond=cond, align=False)
+            if lower is not None:
+                cond = mask | (self >= lower)
+                mgr = mgr.where(other=lower, cond=cond, align=False)
 
+        result = self._constructor(mgr)
         if inplace:
             return self._update_inplace(result)
         else:
-            return result
+            return result.__finalize__(self)
 
     @final
     def _clip_with_one_bound(self, threshold, method, axis, inplace):

pandas/core/indexes/base.py

@@ -810,7 +810,11 @@ def _engine(
         target_values = self._get_engine_target()
         if isinstance(target_values, ExtensionArray):
             if isinstance(target_values, (BaseMaskedArray, ArrowExtensionArray)):
-                return _masked_engines[target_values.dtype.name](target_values)
+                try:
+                    return _masked_engines[target_values.dtype.name](target_values)
+                except KeyError:
+                    # Not supported yet e.g. decimal
+                    pass
             elif self._engine_type is libindex.ObjectEngine:
                 return libindex.ExtensionEngine(target_values)
 
@@ -4948,6 +4952,8 @@ def _get_engine_target(self) -> ArrayLike:
             and not (
                 isinstance(self._values, ArrowExtensionArray)
                 and is_numeric_dtype(self.dtype)
+                # Exclude decimal
+                and self.dtype.kind != "O"
             )
         ):
             # TODO(ExtensionIndex): remove special-case, just use self._values

pandas/core/internals/managers.py

@@ -253,6 +253,9 @@ def add_references(self, mgr: BaseBlockManager) -> None:
         Adds the references from one manager to another. We assume that both
         managers have the same block structure.
         """
+        if len(self.blocks) != len(mgr.blocks):
+            # If block structure changes, then we made a copy
+            return
         for i, blk in enumerate(self.blocks):
             blk.refs = mgr.blocks[i].refs
             # Argument 1 to "add_reference" of "BlockValuesRefs" has incompatible type

pandas/io/sql.py

@@ -632,36 +632,9 @@ def read_sql(
     >>> pd.read_sql('test_data', 'postgres:///db_name')  # doctest:+SKIP
 
     Apply date parsing to columns through the ``parse_dates`` argument
-
-    >>> pd.read_sql('SELECT int_column, date_column FROM test_data',
-    ...             conn,
-    ...             parse_dates=["date_column"])
-       int_column date_column
-    0           0  2012-10-11
-    1           1  2010-12-11
-
     The ``parse_dates`` argument calls ``pd.to_datetime`` on the provided columns.
     Custom argument values for applying ``pd.to_datetime`` on a column are specified
     via a dictionary format:
-    1. Ignore errors while parsing the values of "date_column"
-
-    >>> pd.read_sql('SELECT int_column, date_column FROM test_data',
-    ...             conn,
-    ...             parse_dates={"date_column": {"errors": "ignore"}})
-       int_column date_column
-    0           0  2012-10-11
-    1           1  2010-12-11
-
-    2. Apply a dayfirst date parsing order on the values of "date_column"
-
-    >>> pd.read_sql('SELECT int_column, date_column FROM test_data',
-    ...             conn,
-    ...             parse_dates={"date_column": {"dayfirst": True}})
-       int_column date_column
-    0           0  2012-11-10
-    1           1  2010-11-12
-
-    3. Apply custom formatting when date parsing the values of "date_column"
 
     >>> pd.read_sql('SELECT int_column, date_column FROM test_data',
     ...             conn,

pandas/tests/copy_view/test_clip.py

@@ -12,13 +12,13 @@ def test_clip_inplace_reference(using_copy_on_write):
     view = df[:]
     df.clip(lower=2, inplace=True)
 
-    # Clip not actually inplace right now but could be
-    assert not np.shares_memory(get_array(df, "a"), arr_a)
-
     if using_copy_on_write:
+        assert not np.shares_memory(get_array(df, "a"), arr_a)
         assert df._mgr._has_no_reference(0)
         assert view._mgr._has_no_reference(0)
         tm.assert_frame_equal(df_copy, view)
+    else:
+        assert np.shares_memory(get_array(df, "a"), arr_a)
 
 
 def test_clip_inplace_reference_no_op(using_copy_on_write):
@@ -28,22 +28,20 @@ def test_clip_inplace_reference_no_op(using_copy_on_write):
     view = df[:]
     df.clip(lower=0, inplace=True)
 
+    assert np.shares_memory(get_array(df, "a"), arr_a)
+
     if using_copy_on_write:
-        assert np.shares_memory(get_array(df, "a"), arr_a)
         assert not df._mgr._has_no_reference(0)
         assert not view._mgr._has_no_reference(0)
         tm.assert_frame_equal(df_copy, view)
-    else:
-        assert not np.shares_memory(get_array(df, "a"), arr_a)
 
 
 def test_clip_inplace(using_copy_on_write):
     df = DataFrame({"a": [1.5, 2, 3]})
     arr_a = get_array(df, "a")
     df.clip(lower=2, inplace=True)
 
-    # Clip not actually inplace right now but could be
-    assert not np.shares_memory(get_array(df, "a"), arr_a)
+    assert np.shares_memory(get_array(df, "a"), arr_a)
 
     if using_copy_on_write:
         assert df._mgr._has_no_reference(0)

pandas/tests/copy_view/test_methods.py

@@ -1593,3 +1593,37 @@ def test_inplace_arithmetic_series_with_reference(using_copy_on_write):
         tm.assert_series_equal(ser_orig, view)
     else:
         assert np.shares_memory(get_array(ser), get_array(view))
+
+
+@pytest.mark.parametrize("copy", [True, False])
+def test_transpose(using_copy_on_write, copy, using_array_manager):
+    df = DataFrame({"a": [1, 2, 3], "b": 1})
+    df_orig = df.copy()
+    result = df.transpose(copy=copy)
+
+    if not copy and not using_array_manager or using_copy_on_write:
+        assert np.shares_memory(get_array(df, "a"), get_array(result, 0))
+    else:
+        assert not np.shares_memory(get_array(df, "a"), get_array(result, 0))
+
+    result.iloc[0, 0] = 100
+    if using_copy_on_write:
+        tm.assert_frame_equal(df, df_orig)
+
+
+def test_transpose_different_dtypes(using_copy_on_write):
+    df = DataFrame({"a": [1, 2, 3], "b": 1.5})
+    df_orig = df.copy()
+    result = df.T
+
+    assert not np.shares_memory(get_array(df, "a"), get_array(result, 0))
+    result.iloc[0, 0] = 100
+    if using_copy_on_write:
+        tm.assert_frame_equal(df, df_orig)
+
+
+def test_transpose_ea_single_column(using_copy_on_write):
+    df = DataFrame({"a": [1, 2, 3]}, dtype="Int64")
+    result = df.T
+
+    assert not np.shares_memory(get_array(df, "a"), get_array(result, 0))