Add pyarrow support to python engine in read_csv (#50318)

Author: phofl

doc/source/whatsnew/v2.0.0.rst

@@ -42,7 +42,7 @@ The ``use_nullable_dtypes`` keyword argument has been expanded to the following
 Additionally a new global configuration, ``mode.nullable_backend`` can now be used in conjunction with the parameter ``use_nullable_dtypes=True`` in the following functions
 to select the nullable dtypes implementation.
 
-* :func:`read_csv` (with ``engine="pyarrow"``)
+* :func:`read_csv` (with ``engine="pyarrow"`` or ``engine="python"``)
 * :func:`read_excel`
 * :func:`read_parquet`
 * :func:`read_orc`

pandas/io/parsers/readers.py

@@ -562,6 +562,7 @@ def _read(
     elif (
         kwds.get("use_nullable_dtypes", False)
         and get_option("mode.nullable_backend") == "pyarrow"
+        and kwds.get("engine") == "c"
     ):
         raise NotImplementedError(
             f"use_nullable_dtypes=True and engine={kwds['engine']} with "

pandas/tests/io/parser/dtypes/test_dtypes_basic.py

@@ -492,13 +492,14 @@ def test_use_nullable_dtypes_pyarrow_backend(all_parsers, request):
     # GH#36712
     pa = pytest.importorskip("pyarrow")
     parser = all_parsers
+    engine = parser.engine
 
     data = """a,b,c,d,e,f,g,h,i,j
 1,2.5,True,a,,,,,12-31-2019,
 3,4.5,False,b,6,7.5,True,a,12-31-2019,
 """
     with pd.option_context("mode.nullable_backend", "pyarrow"):
-        if parser.engine != "pyarrow":
+        if engine == "c":
             request.node.add_marker(
                 pytest.mark.xfail(
                     raises=NotImplementedError,
@@ -517,7 +518,10 @@ def test_use_nullable_dtypes_pyarrow_backend(all_parsers, request):
                 "e": pd.Series([pd.NA, 6], dtype="int64[pyarrow]"),
                 "f": pd.Series([pd.NA, 7.5], dtype="float64[pyarrow]"),
                 "g": pd.Series([pd.NA, True], dtype="bool[pyarrow]"),
-                "h": pd.Series(["", "a"], dtype=pd.ArrowDtype(pa.string())),
+                "h": pd.Series(
+                    [pd.NA if engine == "python" else "", "a"],
+                    dtype=pd.ArrowDtype(pa.string()),
+                ),
                 "i": pd.Series([Timestamp("2019-12-31")] * 2),
                 "j": pd.Series([pd.NA, pd.NA], dtype="null[pyarrow]"),
             }