Merge remote-tracking branch 'upstream/main' into tst/ref/test_sql

Author: mroeschke

doc/source/whatsnew/v2.2.0.rst

@@ -314,6 +314,7 @@ MultiIndex
 I/O
 ^^^
 - Bug in :func:`read_csv` where ``on_bad_lines="warn"`` would write to ``stderr`` instead of raise a Python warning. This now yields a :class:`.errors.ParserWarning` (:issue:`54296`)
+- Bug in :func:`read_csv` with ``engine="pyarrow"`` where ``usecols`` wasn't working with a csv with no headers (:issue:`54459`)
 - Bug in :func:`read_excel`, with ``engine="xlrd"`` (``xls`` files) erroring when file contains NaNs/Infs (:issue:`54564`)
 - Bug in :func:`to_excel`, with ``OdsWriter`` (``ods`` files) writing boolean/string value (:issue:`54994`)
 

pandas/io/parsers/arrow_parser_wrapper.py

@@ -130,6 +130,12 @@ def handle_warning(invalid_row):
             )
         }
         self.convert_options["strings_can_be_null"] = "" in self.kwds["null_values"]
+        # autogenerated column names are prefixed with 'f' in pyarrow.csv
+        if self.header is None and "include_columns" in self.convert_options:
+            self.convert_options["include_columns"] = [
+                f"f{n}" for n in self.convert_options["include_columns"]
+            ]
+
         self.read_options = {
             "autogenerate_column_names": self.header is None,
             "skip_rows": self.header

pandas/tests/frame/test_arithmetic.py

@@ -22,14 +22,12 @@
 )
 import pandas._testing as tm
 from pandas.core.computation import expressions as expr
-from pandas.core.computation.expressions import (
-    _MIN_ELEMENTS,
-    NUMEXPR_INSTALLED,
-)
+from pandas.core.computation.expressions import _MIN_ELEMENTS
 from pandas.tests.frame.common import (
     _check_mixed_float,
     _check_mixed_int,
 )
+from pandas.util.version import Version
 
 
 @pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
@@ -501,10 +499,19 @@ def test_floordiv_axis0(self):
         result2 = df.floordiv(ser.values, axis=0)
         tm.assert_frame_equal(result2, expected)
 
-    @pytest.mark.skipif(not NUMEXPR_INSTALLED, reason="numexpr not installed")
     @pytest.mark.parametrize("opname", ["floordiv", "pow"])
-    def test_floordiv_axis0_numexpr_path(self, opname):
+    def test_floordiv_axis0_numexpr_path(self, opname, request):
         # case that goes through numexpr and has to fall back to masked_arith_op
+        ne = pytest.importorskip("numexpr")
+        if (
+            Version(ne.__version__) >= Version("2.8.7")
+            and opname == "pow"
+            and "python" in request.node.callspec.id
+        ):
+            request.node.add_marker(
+                pytest.mark.xfail(reason="https://github.com/pydata/numexpr/issues/454")
+            )
+
         op = getattr(operator, opname)
 
         arr = np.arange(_MIN_ELEMENTS + 100).reshape(_MIN_ELEMENTS // 100 + 1, -1) * 100

pandas/tests/io/parser/test_header.py

@@ -684,3 +684,21 @@ def test_header_delim_whitespace(all_parsers):
     result = parser.read_csv(StringIO(data), delim_whitespace=True)
     expected = DataFrame({"a,b": ["1,2", "3,4"]})
     tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_no_header_pyarrow(pyarrow_parser_only):
+    parser = pyarrow_parser_only
+    data = """
+a,i,x
+b,j,y
+"""
+    result = parser.read_csv(
+        StringIO(data),
+        header=None,
+        usecols=[0, 1],
+        dtype="string[pyarrow]",
+        dtype_backend="pyarrow",
+        engine="pyarrow",
+    )
+    expected = DataFrame([["a", "i"], ["b", "j"]], dtype="string[pyarrow]")
+    tm.assert_frame_equal(result, expected)