Backport PR #52211 on branch 2.0.x (CI: Test pyarrow nightly instead of intermediate versions) (#52313)

Backport PR #52211: CI: Test pyarrow nightly instead of intermediate versions

Author: mroeschke

.github/actions/setup-conda/action.yml

@@ -9,20 +9,9 @@ inputs:
   extra-specs:
     description: Extra packages to install
     required: false
-  pyarrow-version:
-    description: If set, overrides the PyArrow version in the Conda environment to the given string.
-    required: false
 runs:
   using: composite
   steps:
-    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
-      run: |
-        grep -q '  - pyarrow' ${{ inputs.environment-file }}
-        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
-        cat ${{ inputs.environment-file }}
-      shell: bash
-      if: ${{ inputs.pyarrow-version }}
-
     - name: Install ${{ inputs.environment-file }}
       uses: mamba-org/provision-with-micromamba@v12
       with:

.github/workflows/macos-windows.yml

@@ -51,7 +51,6 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ci/deps/${{ matrix.env_file }}
-        pyarrow-version: ${{ matrix.os == 'macos-latest' && '9' || '' }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas

.github/workflows/ubuntu.yml

@@ -27,7 +27,6 @@ jobs:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml, actions-311.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
-        pyarrow_version: ["8", "9", "10"]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -75,21 +74,11 @@ jobs:
             # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
             # There are some warnings failing the build with -werror
             pandas_ci: "0"
-        exclude:
-          - env_file: actions-38.yaml
-            pyarrow_version: "8"
-          - env_file: actions-38.yaml
-            pyarrow_version: "9"
-          - env_file: actions-39.yaml
-            pyarrow_version: "8"
-          - env_file: actions-39.yaml
-            pyarrow_version: "9"
-          - env_file: actions-310.yaml
-            pyarrow_version: "8"
-          - env_file: actions-310.yaml
-            pyarrow_version: "9"
+          - name: "Pyarrow Nightly"
+            env_file: actions-311-pyarrownightly.yaml
+            pattern: "not slow and not network and not single_cpu"
       fail-fast: false
-    name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }}
+    name: ${{ matrix.name || matrix.env_file }}
     env:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
@@ -107,7 +96,7 @@ jobs:
       COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
       cancel-in-progress: true
 
     services:
@@ -166,7 +155,6 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ${{ env.ENV_FILE }}
-        pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
       id: build

ci/deps/actions-310.yaml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2
   - pymysql
   - pytables
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat
   - python-snappy
   - pyxlsb

ci/deps/actions-311-pyarrownightly.yaml

@@ -0,0 +1,29 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+
+  # build dependencies
+  - versioneer[toml]
+  - cython>=0.29.33
+
+  # test dependencies
+  - pytest>=7.0.0
+  - pytest-cov
+  - pytest-xdist>=2.2.0
+  - hypothesis>=6.34.2
+  - pytest-asyncio>=0.17.0
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+  - pytz
+  - pip
+
+  - pip:
+    - "tzdata>=2022.1"
+    - "--extra-index-url https://pypi.fury.io/arrow-nightlies/"
+    - "--prefer-binary"
+    - "--pre"
+    - "pyarrow"

ci/deps/actions-311.yaml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2
   - pymysql
   # - pytables>=3.8.0  # first version that supports 3.11
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat
   - python-snappy
   - pyxlsb

ci/deps/actions-38-downstream_compat.yaml

@@ -39,7 +39,7 @@ dependencies:
   - openpyxl<3.1.1
   - odfpy
   - psycopg2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql
   - pyreadstat
   - pytables

ci/deps/actions-38.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql
   - pyreadstat
   - pytables

ci/deps/actions-39.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pandas-gbq
   - psycopg2
   - pymysql
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat
   - pytables
   - python-snappy

ci/deps/circle-38-arm64.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql
   # Not provided on ARM
   #- pyreadstat

environment.yml

@@ -42,7 +42,7 @@ dependencies:
   - odfpy
   - py
   - psycopg2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql
   - pyreadstat
   - pytables

pandas/tests/arrays/string_/test_string.py

@@ -12,6 +12,7 @@
 import pandas as pd
 import pandas._testing as tm
 from pandas.core.arrays.string_arrow import ArrowStringArray
+from pandas.util.version import Version
 
 
 @pytest.fixture
@@ -406,15 +407,14 @@ def test_fillna_args(dtype, request):
         arr.fillna(value=1)
 
 
-@td.skip_if_no("pyarrow")
 def test_arrow_array(dtype):
     # protocol added in 0.15.0
-    import pyarrow as pa
+    pa = pytest.importorskip("pyarrow")
 
     data = pd.array(["a", "b", "c"], dtype=dtype)
     arr = pa.array(data)
     expected = pa.array(list(data), type=pa.string(), from_pandas=True)
-    if dtype.storage == "pyarrow":
+    if dtype.storage == "pyarrow" and Version(pa.__version__) <= Version("11.0.0"):
         expected = pa.chunked_array(expected)
 
     assert arr.equals(expected)

pandas/tests/io/test_parquet.py

@@ -1019,7 +1019,10 @@ def test_read_dtype_backend_pyarrow_config_index(self, pa):
             {"a": [1, 2]}, index=pd.Index([3, 4], name="test"), dtype="int64[pyarrow]"
         )
         expected = df.copy()
+        import pyarrow
 
+        if Version(pyarrow.__version__) > Version("11.0.0"):
+            expected.index = expected.index.astype("int64[pyarrow]")
         check_round_trip(
             df,
             engine=pa,

pandas/tests/util/test_show_versions.py

@@ -65,7 +65,7 @@ def test_show_versions_console(capsys):
     assert re.search(r"numpy\s*:\s[0-9]+\..*\n", result)
 
     # check optional dependency
-    assert re.search(r"pyarrow\s*:\s([0-9\.]+|None)\n", result)
+    assert re.search(r"pyarrow\s*:\s([0-9]+.*|None)\n", result)
 
 
 def test_json_output_match(capsys, tmpdir):

requirements-dev.txt

@@ -31,7 +31,7 @@ openpyxl<3.1.1
 odfpy
 py
 psycopg2-binary
-pyarrow
+pyarrow>=7.0.0
 pymysql
 pyreadstat
 tables