Upgrade pyarrow minimum version to 7.0 (#51151)

* Upgrade ci minimum version to 7.0

* Fix

* Remove

* Add cond back in

* Remove super

* Remove if

* Fix merge

Author: phofl

ci/deps/actions-38-minimum_versions.yaml

@@ -43,7 +43,7 @@ dependencies:
   - openpyxl=3.0.7
   - pandas-gbq=0.15.0
   - psycopg2=2.8.6
-  - pyarrow=6.0.0
+  - pyarrow=7.0.0
   - pymysql=1.0.2
   - pyreadstat=1.1.2
   - pytables=3.6.1

doc/source/getting_started/install.rst

@@ -441,7 +441,7 @@ PyTables                  3.6.1              hdf5             HDF5-based reading
 blosc                     1.21.0             hdf5             Compression for HDF5; only available on ``conda``
 zlib                                         hdf5             Compression for HDF5
 fastparquet               0.6.3              -                Parquet reading / writing (pyarrow is default)
-pyarrow                   6.0.0              parquet, feather Parquet, ORC, and feather reading / writing
+pyarrow                   7.0.0              parquet, feather Parquet, ORC, and feather reading / writing
 pyreadstat                1.1.2              spss             SPSS files (.sav) reading
 odfpy                     1.4.1              excel            Open document format (.odf, .ods, .odt) reading / writing
 ========================= ================== ================ =============================================================

doc/source/whatsnew/v2.0.0.rst

@@ -668,7 +668,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | Package         | Minimum Version | Changed |
 +=================+=================+=========+
-| pyarrow         | 6.0.0           |    X    |
+| pyarrow         | 7.0.0           |    X    |
 +-----------------+-----------------+---------+
 | matplotlib      | 3.6.1           |    X    |
 +-----------------+-----------------+---------+

pandas/_testing/__init__.py

@@ -29,7 +29,7 @@
     Frequency,
     NpDtype,
 )
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 
 from pandas.core.dtypes.common import (
     is_float_dtype,
@@ -191,7 +191,7 @@
     ]
 ]
 
-if not pa_version_under6p0:
+if not pa_version_under7p0:
     import pyarrow as pa
 
     UNSIGNED_INT_PYARROW_DTYPES = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()]

pandas/compat/__init__.py

@@ -27,7 +27,6 @@
     np_version_under1p21,
 )
 from pandas.compat.pyarrow import (
-    pa_version_under6p0,
     pa_version_under7p0,
     pa_version_under8p0,
     pa_version_under9p0,
@@ -157,7 +156,6 @@ def get_lzma_file() -> type[pandas.compat.compressors.LZMAFile]:
 __all__ = [
     "is_numpy_dev",
     "np_version_under1p21",
-    "pa_version_under6p0",
     "pa_version_under7p0",
     "pa_version_under8p0",
     "pa_version_under9p0",

pandas/compat/_optional.py

@@ -31,7 +31,7 @@
     "pandas_gbq": "0.15.0",
     "psycopg2": "2.8.6",  # (dt dec pq3 ext lo64)
     "pymysql": "1.0.2",
-    "pyarrow": "6.0.0",
+    "pyarrow": "7.0.0",
     "pyreadstat": "1.1.2",
     "pytest": "7.0.0",
     "pyxlsb": "1.0.8",

pandas/compat/pyarrow.py

@@ -9,13 +9,11 @@
 
     _pa_version = pa.__version__
     _palv = Version(_pa_version)
-    pa_version_under6p0 = _palv < Version("6.0.0")
     pa_version_under7p0 = _palv < Version("7.0.0")
     pa_version_under8p0 = _palv < Version("8.0.0")
     pa_version_under9p0 = _palv < Version("9.0.0")
     pa_version_under10p0 = _palv < Version("10.0.0")
 except ImportError:
-    pa_version_under6p0 = True
     pa_version_under7p0 = True
     pa_version_under8p0 = True
     pa_version_under9p0 = True

pandas/core/arrays/arrow/array.py

@@ -26,7 +26,6 @@
     npt,
 )
 from pandas.compat import (
-    pa_version_under6p0,
     pa_version_under7p0,
     pa_version_under8p0,
     pa_version_under9p0,
@@ -54,7 +53,7 @@
     validate_indices,
 )
 
-if not pa_version_under6p0:
+if not pa_version_under7p0:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -199,8 +198,8 @@ class ArrowExtensionArray(OpsMixin, ExtensionArray):
     _dtype: ArrowDtype
 
     def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
-        if pa_version_under6p0:
-            msg = "pyarrow>=6.0.0 is required for PyArrow backed ArrowExtensionArray."
+        if pa_version_under7p0:
+            msg = "pyarrow>=7.0.0 is required for PyArrow backed ArrowExtensionArray."
             raise ImportError(msg)
         if isinstance(values, pa.Array):
             self._data = pa.chunked_array([values])
@@ -529,11 +528,6 @@ def _argmin_max(self, skipna: bool, method: str) -> int:
             # let ExtensionArray.arg{max|min} raise
             return getattr(super(), f"arg{method}")(skipna=skipna)
 
-        if pa_version_under6p0:
-            raise NotImplementedError(
-                f"arg{method} only implemented for pyarrow version >= 6.0"
-            )
-
         data = self._data
         if pa.types.is_duration(data.type):
             data = data.cast(pa.int64())
@@ -567,11 +561,7 @@ def dropna(self: ArrowExtensionArrayT) -> ArrowExtensionArrayT:
         -------
         ArrowExtensionArray
         """
-        if pa_version_under6p0:
-            fallback_performancewarning(version="6")
-            return super().dropna()
-        else:
-            return type(self)(pc.drop_null(self._data))
+        return type(self)(pc.drop_null(self._data))
 
     @doc(ExtensionArray.fillna)
     def fillna(
@@ -1293,9 +1283,6 @@ def _mode(self: ArrowExtensionArrayT, dropna: bool = True) -> ArrowExtensionArra
         same type as self
             Sorted, if possible.
         """
-        if pa_version_under6p0:
-            raise NotImplementedError("mode only supported for pyarrow version >= 6.0")
-
         pa_type = self._data.type
         if pa.types.is_temporal(pa_type):
             nbits = pa_type.bit_width

pandas/core/arrays/arrow/dtype.py

@@ -5,15 +5,15 @@
 import numpy as np
 
 from pandas._typing import DtypeObj
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 from pandas.util._decorators import cache_readonly
 
 from pandas.core.dtypes.base import (
     StorageExtensionDtype,
     register_extension_dtype,
 )
 
-if not pa_version_under6p0:
+if not pa_version_under7p0:
     import pyarrow as pa
 
 
@@ -66,8 +66,8 @@ class ArrowDtype(StorageExtensionDtype):
 
     def __init__(self, pyarrow_dtype: pa.DataType) -> None:
         super().__init__("pyarrow")
-        if pa_version_under6p0:
-            raise ImportError("pyarrow>=6.0.0 is required for ArrowDtype")
+        if pa_version_under7p0:
+            raise ImportError("pyarrow>=7.0.0 is required for ArrowDtype")
         if not isinstance(pyarrow_dtype, pa.DataType):
             raise ValueError(
                 f"pyarrow_dtype ({pyarrow_dtype}) must be an instance "

pandas/core/arrays/string_.py

@@ -21,7 +21,7 @@
     npt,
     type_t,
 )
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import doc
 
@@ -115,9 +115,9 @@ def __init__(self, storage=None) -> None:
             raise ValueError(
                 f"Storage must be 'python' or 'pyarrow'. Got {storage} instead."
             )
-        if storage == "pyarrow" and pa_version_under6p0:
+        if storage == "pyarrow" and pa_version_under7p0:
             raise ImportError(
-                "pyarrow>=6.0.0 is required for PyArrow backed StringArray."
+                "pyarrow>=7.0.0 is required for PyArrow backed StringArray."
             )
         self.storage = storage
 

pandas/core/arrays/string_arrow.py

@@ -17,7 +17,7 @@
     Scalar,
     npt,
 )
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 
 from pandas.core.dtypes.common import (
     is_bool_dtype,
@@ -40,7 +40,7 @@
 )
 from pandas.core.strings.object_array import ObjectStringArrayMixin
 
-if not pa_version_under6p0:
+if not pa_version_under7p0:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -50,8 +50,8 @@
 
 
 def _chk_pyarrow_available() -> None:
-    if pa_version_under6p0:
-        msg = "pyarrow>=6.0.0 is required for PyArrow backed ArrowExtensionArray."
+    if pa_version_under7p0:
+        msg = "pyarrow>=7.0.0 is required for PyArrow backed ArrowExtensionArray."
         raise ImportError(msg)
 
 

pandas/tests/arrays/string_/test_string.py

@@ -5,7 +5,6 @@
 import numpy as np
 import pytest
 
-from pandas.compat import pa_version_under6p0
 import pandas.util._test_decorators as td
 
 from pandas.core.dtypes.common import is_dtype_equal
@@ -358,11 +357,6 @@ def test_reduce_missing(skipna, dtype):
 @pytest.mark.parametrize("method", ["min", "max"])
 @pytest.mark.parametrize("skipna", [True, False])
 def test_min_max(method, skipna, dtype, request):
-    if dtype.storage == "pyarrow" and pa_version_under6p0:
-        reason = "'ArrowStringArray' object has no attribute 'max'"
-        mark = pytest.mark.xfail(raises=TypeError, reason=reason)
-        request.node.add_marker(mark)
-
     arr = pd.Series(["a", "b", "c", None], dtype=dtype)
     result = getattr(arr, method)(skipna=skipna)
     if skipna:
@@ -375,7 +369,7 @@ def test_min_max(method, skipna, dtype, request):
 @pytest.mark.parametrize("method", ["min", "max"])
 @pytest.mark.parametrize("box", [pd.Series, pd.array])
 def test_min_max_numpy(method, box, dtype, request):
-    if dtype.storage == "pyarrow" and (pa_version_under6p0 or box is pd.array):
+    if dtype.storage == "pyarrow" and box is pd.array:
         if box is pd.array:
             reason = "'<=' not supported between instances of 'str' and 'NoneType'"
         else:

pandas/tests/arrays/string_/test_string_arrow.py

@@ -4,7 +4,7 @@
 import numpy as np
 import pytest
 
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 
 import pandas as pd
 import pandas._testing as tm
@@ -15,8 +15,8 @@
 from pandas.core.arrays.string_arrow import ArrowStringArray
 
 skip_if_no_pyarrow = pytest.mark.skipif(
-    pa_version_under6p0,
-    reason="pyarrow>=6.0.0 is required for PyArrow backed StringArray",
+    pa_version_under7p0,
+    reason="pyarrow>=7.0.0 is required for PyArrow backed StringArray",
 )
 
 
@@ -119,11 +119,11 @@ def test_from_sequence_wrong_dtype_raises():
 
 
 @pytest.mark.skipif(
-    not pa_version_under6p0,
+    not pa_version_under7p0,
     reason="pyarrow is installed",
 )
 def test_pyarrow_not_installed_raises():
-    msg = re.escape("pyarrow>=6.0.0 is required for PyArrow backed")
+    msg = re.escape("pyarrow>=7.0.0 is required for PyArrow backed")
 
     with pytest.raises(ImportError, match=msg):
         StringDtype(storage="pyarrow")