From 85e58bd541510baf51d24a616c9ed933a9ca91cf Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 15 Mar 2022 19:04:46 -0700 Subject: [PATCH 1/6] CI: Test Pyarrow Less --- .github/workflows/posix.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 4380e8dfa2e57..ba8462cdb6ffb 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -26,9 +26,6 @@ jobs: matrix: env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml] pattern: ["not single_cpu", "single_cpu"] - # Don't test pyarrow v2/3: Causes timeouts in read_csv engine - # even if tests are skipped/xfailed - pyarrow_version: ["5", "7"] include: - env_file: actions-38-downstream_compat.yaml pattern: "not slow and not network and not single_cpu" @@ -62,6 +59,13 @@ jobs: pandas_testing_mode: "deprecate" test_args: "-W error" name: "Numpy Dev" + # Test lowest supported pyarrow + - env_file: actions-38.yaml + pattern: "not single_cpu" + name: "Min Pyarrow" + - env_file: actions-38.yaml + pattern: "single_cpu" + name: "Min Pyarrow (single)" fail-fast: false name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} env: From fda3a0057e0cc20f0b98a48e38f8792fbd526383 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 15 Mar 2022 19:09:09 -0700 Subject: [PATCH 2/6] fixups --- .github/workflows/posix.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index ba8462cdb6ffb..f577dbe5232a6 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -62,12 +62,14 @@ jobs: # Test lowest supported pyarrow - env_file: actions-38.yaml pattern: "not single_cpu" + pyarrow_version: 5 name: "Min Pyarrow" - env_file: actions-38.yaml pattern: "single_cpu" + pyarrow_version: 5 name: "Min Pyarrow (single)" fail-fast: false - name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} + name: ${{ matrix.name || format('{0} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} env: ENV_FILE: ci/deps/${{ matrix.env_file }} PATTERN: ${{ matrix.pattern }} From a5eac9bf78d882d7db97e51c1638ab2c0d984d7f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 16 Mar 2022 18:40:39 -0700 Subject: [PATCH 3/6] comments --- .github/workflows/posix.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index f577dbe5232a6..7d8e213f474dd 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -60,6 +60,8 @@ jobs: test_args: "-W error" name: "Numpy Dev" # Test lowest supported pyarrow + # Don't test pyarrow v2/3: Causes timeouts in read_csv engine + # even if tests are skipped/xfailed - env_file: actions-38.yaml pattern: "not single_cpu" pyarrow_version: 5 @@ -69,7 +71,7 @@ jobs: pyarrow_version: 5 name: "Min Pyarrow (single)" fail-fast: false - name: ${{ matrix.name || format('{0} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} + name: ${{ matrix.name || format('{0} {1}', matrix.env_file, matrix.pattern) }} env: ENV_FILE: ci/deps/${{ matrix.env_file }} PATTERN: ${{ matrix.pattern }} From 5cb5f9ec593c08ed438defb60180958e7df1af1e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 16 Mar 2022 20:15:58 -0700 Subject: [PATCH 4/6] test latest pyarrow --- ci/deps/actions-310.yaml | 6 +++--- ci/deps/actions-38.yaml | 3 ++- ci/deps/actions-39.yaml | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 37e7ea04a348a..3570151aca35f 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -31,8 +31,7 @@ dependencies: - jinja2 - lxml - matplotlib - # TODO: uncomment after numba supports py310 - #- numba + - numba - numexpr - openpyxl - odfpy @@ -40,7 +39,8 @@ dependencies: - psycopg2 - pymysql - pytables - - pyarrow + # Workaround mamba bug that installs pyarrow 2.0 + - pyarrow>2 - pyreadstat - python-snappy - pyxlsb diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 9c46eca4ab989..11619f5c363b5 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -37,7 +37,8 @@ dependencies: - odfpy - pandas-gbq - psycopg2 - - pyarrow + # Workaround mamba bug that installs pyarrow 2.0 + - pyarrow>2 - pymysql - pyreadstat - pytables diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 89b647372d7bc..bc3f178f07bb2 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -38,7 +38,8 @@ dependencies: - pandas-gbq - psycopg2 - pymysql - - pyarrow + # Workaround mamba bug that installs pyarrow 2.0 + - pyarrow>2 - pyreadstat - pytables - python-snappy From 73b7c30d7e5bea664b8acc644422540e5dea18ea Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 23 Mar 2022 08:11:08 -0700 Subject: [PATCH 5/6] update --- .github/workflows/posix.yml | 10 +++++++++- ci/deps/actions-310.yaml | 4 ++-- ci/deps/actions-38.yaml | 4 ++-- ci/deps/actions-39.yaml | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 7d8e213f474dd..94964f9aedf39 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -59,7 +59,7 @@ jobs: pandas_testing_mode: "deprecate" test_args: "-W error" name: "Numpy Dev" - # Test lowest supported pyarrow + # Test lowest & highest supported pyarrow # Don't test pyarrow v2/3: Causes timeouts in read_csv engine # even if tests are skipped/xfailed - env_file: actions-38.yaml @@ -70,6 +70,14 @@ jobs: pattern: "single_cpu" pyarrow_version: 5 name: "Min Pyarrow (single)" + - env_file: actions-38.yaml + pattern: "not single_cpu" + pyarrow_version: 7 + name: "Latest Pyarrow" + - env_file: actions-38.yaml + pattern: "single_cpu" + pyarrow_version: 7 + name: "Latest Pyarrow (single)" fail-fast: false name: ${{ matrix.name || format('{0} {1}', matrix.env_file, matrix.pattern) }} env: diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 3570151aca35f..82c741384ac62 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -39,8 +39,8 @@ dependencies: - psycopg2 - pymysql - pytables - # Workaround mamba bug that installs pyarrow 2.0 - - pyarrow>2 + # Installed separately since mamba doesn't get latest version properly + #- pyarrow - pyreadstat - python-snappy - pyxlsb diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 11619f5c363b5..ee75b443882e6 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -37,8 +37,8 @@ dependencies: - odfpy - pandas-gbq - psycopg2 - # Workaround mamba bug that installs pyarrow 2.0 - - pyarrow>2 + # Installed separately since mamba doesn't get latest version properly + #- pyarrow - pymysql - pyreadstat - pytables diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index bc3f178f07bb2..3086d55e89e38 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -38,8 +38,8 @@ dependencies: - pandas-gbq - psycopg2 - pymysql - # Workaround mamba bug that installs pyarrow 2.0 - - pyarrow>2 + # Installed separately since mamba doesn't get latest version properly + #- pyarrow - pyreadstat - pytables - python-snappy From 815127dca1798d71744c1a86fa643d0080233f2a Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 26 Mar 2022 10:51:49 -0700 Subject: [PATCH 6/6] try something --- .github/workflows/posix.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index ec5682601f4a9..9ad80ea96de31 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -65,18 +65,18 @@ jobs: - env_file: actions-38.yaml pattern: "not single_cpu" pyarrow_version: 5 - name: "Min Pyarrow" + name: "Pyarrow 5" - env_file: actions-38.yaml pattern: "single_cpu" pyarrow_version: 5 - name: "Min Pyarrow (single)" + name: "Pyarrow 5 (single)" - env_file: actions-38.yaml pattern: "not single_cpu" - pyarrow_version: 7 + pyarrow_version: latest name: "Latest Pyarrow" - env_file: actions-38.yaml pattern: "single_cpu" - pyarrow_version: 7 + pyarrow_version: latest name: "Latest Pyarrow (single)" fail-fast: false name: ${{ matrix.name || format('{0} {1}', matrix.env_file, matrix.pattern) }} @@ -165,7 +165,13 @@ jobs: if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support - name: Upgrade Arrow version - run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} + run: | + conda remove --force pyarrow + if [[ ${{matrix.pyarrow_version}} == 'latest' ]]; then + conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow + else + conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} + fi if: ${{ matrix.pyarrow_version }} - name: Setup PyPy