BUG: Fix some more arrow CSV tests

doc/source/whatsnew/v2.1.0.rst

@@ -199,7 +199,8 @@ Period
 - Bug in :class:`PeriodDtype` constructor failing to raise ``TypeError`` when no argument is passed or when ``None`` is passed (:issue:`27388`)
 - Bug in :class:`PeriodDtype` constructor raising ``ValueError`` instead of ``TypeError`` when an invalid type is passed (:issue:`51790`)
 - Bug in :meth:`arrays.PeriodArray.map` and :meth:`PeriodIndex.map`, where the supplied callable operated array-wise instead of element-wise (:issue:`51977`)
--
+- Bug in :func:`read_csv` not processing empty strings as a null value, with ``engine="pyarrow"`` (:issue:`52087 `)
+- Bug in :func:`read_csv` returning ``object`` dtype columns instead of ``float64`` dtype columns with ``engine="pyarrow"`` for columns that are all null with ``engine="pyarrow"`` (:issue:`52087 `)
 
 Plotting
 ^^^^^^^^

pandas/io/_util.py

@@ -8,6 +8,8 @@
 def _arrow_dtype_mapping() -> dict:
     pa = import_optional_dependency("pyarrow")
     return {
+        # All nulls should still give Float64 not object
+        pa.null(): pd.Float64Dtype(),
         pa.int8(): pd.Int8Dtype(),
         pa.int16(): pd.Int16Dtype(),
         pa.int32(): pd.Int32Dtype(),

pandas/io/parsers/arrow_parser_wrapper.py

@@ -80,6 +80,7 @@ def _get_pyarrow_options(self) -> None:
                 "decimal_point",
             )
         }
+        self.convert_options["strings_can_be_null"] = "" in self.kwds["null_values"]
         self.read_options = {
             "autogenerate_column_names": self.header is None,
             "skip_rows": self.header
@@ -149,6 +150,7 @@ def read(self) -> DataFrame:
         DataFrame
             The DataFrame created from the CSV file.
         """
+        pa = import_optional_dependency("pyarrow")
         pyarrow_csv = import_optional_dependency("pyarrow.csv")
         self._get_pyarrow_options()
 
@@ -158,6 +160,19 @@ def read(self) -> DataFrame:
             parse_options=pyarrow_csv.ParseOptions(**self.parse_options),
             convert_options=pyarrow_csv.ConvertOptions(**self.convert_options),
         )
+
+        # Convert all pa.null() cols -> float64
+        # TODO: There has to be a better way... right?
+        cols_to_convert = []
+        new_schema = table.schema
+        for i, type in enumerate(table.schema.types):
+            if pa.types.is_null(type):
+                cols_to_convert.append(i)
+        for i in cols_to_convert:
+            new_schema = new_schema.set(i, new_schema.field(i).with_type(pa.float64()))
+
+        table = table.cast(new_schema)
+
         if self.kwds["dtype_backend"] == "pyarrow":
             frame = table.to_pandas(types_mapper=pd.ArrowDtype)
         elif self.kwds["dtype_backend"] == "numpy_nullable":

pandas/io/parsers/readers.py

@@ -1458,8 +1458,11 @@ def _get_options_with_defaults(self, engine: CSVEngine) -> dict[str, Any]:
                 value = kwds[argname]
 
                 if engine != "c" and value != default:
+                    # TODO: Refactor this logic, its pretty convoluted
                     if "python" in engine and argname not in _python_unsupported:
                         pass
+                    elif "pyarrow" in engine and argname not in _pyarrow_unsupported:
+                        pass
                     else:
                         raise ValueError(
                             f"The {repr(argname)} option is not supported with the "

pandas/tests/io/parser/test_na_values.py

@@ -20,7 +20,6 @@
 xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
 
 
-@skip_pyarrow
 def test_string_nas(all_parsers):
     parser = all_parsers
     data = """A,B,C
@@ -36,7 +35,6 @@ def test_string_nas(all_parsers):
     tm.assert_frame_equal(result, expected)
 
 
-@skip_pyarrow
 def test_detect_string_na(all_parsers):
     parser = all_parsers
     data = """A,B
@@ -89,7 +87,6 @@ def test_non_string_na_values(all_parsers, data, na_values):
     tm.assert_frame_equal(result, expected)
 
 
-@skip_pyarrow
 def test_default_na_values(all_parsers):
     _NA_VALUES = {
         "-1.#IND",
@@ -138,6 +135,7 @@ def f(i, v):
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: needs skiprows list support in pyarrow
 @skip_pyarrow
 @pytest.mark.parametrize("na_values", ["baz", ["baz"]])
 def test_custom_na_values(all_parsers, na_values):
@@ -172,6 +170,7 @@ def test_bool_na_values(all_parsers):
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: Needs pyarrow support for dictionary in na_values
 @skip_pyarrow
 def test_na_value_dict(all_parsers):
     data = """A,B,C
@@ -191,7 +190,6 @@ def test_na_value_dict(all_parsers):
     tm.assert_frame_equal(df, expected)
 
 
-@skip_pyarrow
 @pytest.mark.parametrize(
     "index_col,expected",
     [
@@ -225,6 +223,7 @@ def test_na_value_dict_multi_index(all_parsers, index_col, expected):
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: xfail components of this test, the first one passes
 @skip_pyarrow
 @pytest.mark.parametrize(
     "kwargs,expected",
@@ -287,7 +286,6 @@ def test_na_values_keep_default(all_parsers, kwargs, expected):
     tm.assert_frame_equal(result, expected)
 
 
-@skip_pyarrow
 def test_no_na_values_no_keep_default(all_parsers):
     # see gh-4318: passing na_values=None and
     # keep_default_na=False yields 'None" as a na_value
@@ -314,6 +312,7 @@ def test_no_na_values_no_keep_default(all_parsers):
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: Blocked on na_values dict support in pyarrow
 @skip_pyarrow
 def test_no_keep_default_na_dict_na_values(all_parsers):
     # see gh-19227
@@ -326,6 +325,7 @@ def test_no_keep_default_na_dict_na_values(all_parsers):
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: Blocked on na_values dict support in pyarrow
 @skip_pyarrow
 def test_no_keep_default_na_dict_na_scalar_values(all_parsers):
     # see gh-19227
@@ -338,6 +338,7 @@ def test_no_keep_default_na_dict_na_scalar_values(all_parsers):
     tm.assert_frame_equal(df, expected)
 
 
+# TODO: Blocked on na_values dict support in pyarrow
 @skip_pyarrow
 @pytest.mark.parametrize("col_zero_na_values", [113125, "113125"])
 def test_no_keep_default_na_dict_na_values_diff_reprs(all_parsers, col_zero_na_values):
@@ -368,6 +369,7 @@ def test_no_keep_default_na_dict_na_values_diff_reprs(all_parsers, col_zero_na_v
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: Empty null_values doesn't work properly on pyarrow
 @skip_pyarrow
 @pytest.mark.parametrize(
     "na_filter,row_data",
@@ -390,6 +392,7 @@ def test_na_values_na_filter_override(all_parsers, na_filter, row_data):
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: Arrow parse error
 @skip_pyarrow
 def test_na_trailing_columns(all_parsers):
     parser = all_parsers
@@ -418,6 +421,7 @@ def test_na_trailing_columns(all_parsers):
     tm.assert_frame_equal(result, expected)
 
 
+# TODO: xfail the na_values dict case
 @skip_pyarrow
 @pytest.mark.parametrize(
     "na_values,row_data",
@@ -495,7 +499,6 @@ def test_empty_na_values_no_default_with_index(all_parsers):
     tm.assert_frame_equal(result, expected)
 
 
-@skip_pyarrow
 @pytest.mark.parametrize(
     "na_filter,index_data", [(False, ["", "5"]), (True, [np.nan, 5.0])]
 )