Fix ArrowExtensionArray and add whatsnew

Author: rhshadrach

doc/source/whatsnew/v2.3.0.rst

@@ -50,10 +50,20 @@ Notable bug fixes
 
 These are bug fixes that might have notable behavior changes.
 
-.. _whatsnew_230.notable_bug_fixes.notable_bug_fix1:
+.. _whatsnew_230.notable_bug_fixes.string_comparisons:
 
-notable_bug_fix1
-^^^^^^^^^^^^^^^^
+Comparisons between different string dtypes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In previous versions, comparing Series of different string dtypes (e.g. ``pd.StringDtype("pyarrow", na_value=pd.NA)`` against ``pd.StringDtype("python", na_value=np.nan)``) would result in inconsistent resulting dtype or incorrectly raise. pandas will now use the hierarchy
+
+    object < (python, NaN) < (pyarrow, NaN) < (python, NA) < (pyarrow, NA)
+
+in determining the result dtype when there are different string dtypes compared. Some examples:
+
+- When ``pd.StringDtype("pyarrow", na_value=pd.NA)`` is compared against any other string dtype, the result will always be ``boolean[pyarrow]``.
+- When ``pd.StringDtype("python", na_value=pd.NA)`` is compared against ``pd.StringDtype("pyarrow", na_value=np.nan)``, the result will be ``boolean``, the NumPy-backed nullable extension array.
+- When ``pd.StringDtype("python", na_value=pd.NA)`` is compared against ``pd.StringDtype("python", na_value=np.nan)``, the result will be ``boolean``, the NumPy-backed nullable extension array.
 
 .. _whatsnew_230.api_changes:
 

pandas/core/arrays/string_arrow.py

@@ -474,7 +474,7 @@ def value_counts(self, dropna: bool = True) -> Series:
 
     def _cmp_method(self, other, op):
         if (
-            isinstance(other, BaseStringArray)
+            isinstance(other, (BaseStringArray, ArrowExtensionArray))
             and self.dtype.na_value is not libmissing.NA
             and other.dtype.na_value is libmissing.NA
         ):

pandas/tests/arrays/string_/test_string.py

@@ -15,6 +15,7 @@
     pa_version_under12p0,
     pa_version_under19p0,
 )
+import pandas.util._test_decorators as td
 
 from pandas.core.dtypes.common import is_dtype_equal
 
@@ -371,6 +372,26 @@ def test_comparison_methods_array(comparison_op, dtype, dtype2):
         tm.assert_extension_array_equal(result, expected)
 
 
+@td.skip_if_no("pyarrow")
+def test_comparison_methods_array_arrow_extension(comparison_op, dtype2):
+    # Test pd.ArrowDtype(pa.string()) against other string arrays
+    import pyarrow as pa
+
+    op_name = f"__{comparison_op.__name__}__"
+    dtype = pd.ArrowDtype(pa.string())
+    a = pd.array(["a", None, "c"], dtype=dtype)
+    other = pd.array([None, None, "c"], dtype=dtype2)
+    result = comparison_op(a, other)
+
+    # ensure operation is commutative
+    result2 = comparison_op(other, a)
+    tm.assert_equal(result, result2)
+
+    expected = pd.array([None, None, True], dtype="bool[pyarrow]")
+    expected[-1] = getattr(other[-1], op_name)(a[-1])
+    tm.assert_extension_array_equal(result, expected)
+
+
 def test_comparison_methods_list(comparison_op, dtype):
     op_name = f"__{comparison_op.__name__}__"
 

pandas/tests/extension/test_string.py

@@ -240,10 +240,8 @@ def test_arith_series_with_array(
         if (
             using_infer_string
             and all_arithmetic_operators == "__radd__"
-            and (
-                dtype.na_value is pd.NA
-                and not (not HAS_PYARROW and dtype.storage == "python")
-            )
+            and dtype.na_value is pd.NA
+            and (HAS_PYARROW or dtype.storage != "pyarrow")
         ):
             # TODO(infer_string)
             mark = pytest.mark.xfail(