From 49a570f8c60f0d81934e7cd9368a803049101de1 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 16:16:37 -0600 Subject: [PATCH 01/14] CI: Draft of CircleCI for ARM --- .circleci/config.yml | 18 ++++++++++++++++++ ci/setup_env.sh | 15 ++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000000..6008af17fa680 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,18 @@ +version: 2.1 + +jobs: + test-arm: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.medium + steps: + - run: echo "Hello, Arm!" + - run: uname + - run: uname -a + - run: ci/setup_env.sh + - run: ci/run_tests.sh + +workflows: + build: + jobs: + - test-arm diff --git a/ci/setup_env.sh b/ci/setup_env.sh index c36422884f2ec..81800e69862c3 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -22,28 +22,25 @@ if [ -d "$MINICONDA_DIR" ]; then fi echo "Install Miniconda" +DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" UNAME_OS=$(uname) if [[ "$UNAME_OS" == 'Linux' ]]; then if [[ "$BITS32" == "yes" ]]; then - CONDA_OS="Linux-x86" + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" else - CONDA_OS="Linux-x86_64" + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" fi elif [[ "$UNAME_OS" == 'Darwin' ]]; then - CONDA_OS="MacOSX-x86_64" + CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" +elif [[ "$UNAME_OS" == 'arm64' ]]; then + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh" else echo "OS $UNAME_OS not supported" exit 1 fi -if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh" -else - CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -fi wget -q $CONDA_URL -O miniconda.sh chmod +x miniconda.sh - # Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3. ./miniconda.sh -b -p $MINICONDA_DIR From 7cd7aaf2075b2ad55db75ec6ab123ca26b0dfa8f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 16:35:11 -0600 Subject: [PATCH 02/14] Fixes to circle ci build --- .circleci/config.yml | 3 ++- ci/setup_env.sh | 23 +++++++---------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6008af17fa680..387144e946dc3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,8 @@ jobs: steps: - run: echo "Hello, Arm!" - run: uname - - run: uname -a + - run: uname -m + - checkout - run: ci/setup_env.sh - run: ci/run_tests.sh diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 81800e69862c3..a84d8cb337772 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -12,38 +12,29 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then echo fi -MINICONDA_DIR="$HOME/miniconda3" - - -if [ -d "$MINICONDA_DIR" ]; then - echo - echo "rm -rf "$MINICONDA_DIR"" - rm -rf "$MINICONDA_DIR" -fi echo "Install Miniconda" DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" -UNAME_OS=$(uname) -if [[ "$UNAME_OS" == 'Linux' ]]; then +if [[ "$(uname)" == 'Linux' ]]; then if [[ "$BITS32" == "yes" ]]; then CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" else CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" fi -elif [[ "$UNAME_OS" == 'Darwin' ]]; then +elif [[ "$(uname)" == 'Darwin' ]]; then CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" -elif [[ "$UNAME_OS" == 'arm64' ]]; then +elif [[ "$(uname -m)" == 'aarch64' ]]; then CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh" else - echo "OS $UNAME_OS not supported" + echo "OS $(uname) not supported" exit 1 fi - wget -q $CONDA_URL -O miniconda.sh chmod +x miniconda.sh -# Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3. -./miniconda.sh -b -p $MINICONDA_DIR +MINICONDA_DIR="$HOME/miniconda3" +rf -rf $MINICONDA_DIR +./miniconda.sh -b -p $MINICONDA_DIR export PATH=$MINICONDA_DIR/bin:$PATH echo From b3a00f1ea6240bfe79f1885d4d379f84891c450e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 16:38:57 -0600 Subject: [PATCH 03/14] Fix typo --- ci/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index a84d8cb337772..f3a443833f05e 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -33,7 +33,7 @@ wget -q $CONDA_URL -O miniconda.sh chmod +x miniconda.sh MINICONDA_DIR="$HOME/miniconda3" -rf -rf $MINICONDA_DIR +rm -rf $MINICONDA_DIR ./miniconda.sh -b -p $MINICONDA_DIR export PATH=$MINICONDA_DIR/bin:$PATH From b3db6bd46805de12e6228d91eca9b9a6be97969e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 16:52:01 -0600 Subject: [PATCH 04/14] Clean up and debug info --- .circleci/config.yml | 5 +---- ci/deps/{travis-37-arm64.yaml => circle-37-arm64.yaml} | 0 ci/setup_env.sh | 1 + 3 files changed, 2 insertions(+), 4 deletions(-) rename ci/deps/{travis-37-arm64.yaml => circle-37-arm64.yaml} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 387144e946dc3..cd5abc32d044e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,11 +6,8 @@ jobs: image: ubuntu-2004:202101-01 resource_class: arm.medium steps: - - run: echo "Hello, Arm!" - - run: uname - - run: uname -m - checkout - - run: ci/setup_env.sh + - run: ENV_FILE="ci/deps/circle-37-arm64.yaml" ci/setup_env.sh - run: ci/run_tests.sh workflows: diff --git a/ci/deps/travis-37-arm64.yaml b/ci/deps/circle-37-arm64.yaml similarity index 100% rename from ci/deps/travis-37-arm64.yaml rename to ci/deps/circle-37-arm64.yaml diff --git a/ci/setup_env.sh b/ci/setup_env.sh index f3a443833f05e..0282ff55d9571 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -29,6 +29,7 @@ else echo "OS $(uname) not supported" exit 1 fi +echo "Downloading $CONDA_URL" wget -q $CONDA_URL -O miniconda.sh chmod +x miniconda.sh From 6e9457b3e499ae106aff76eb88cfb028d47fd4a1 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 16:58:32 -0600 Subject: [PATCH 05/14] Fixing arm downloading right conda --- ci/setup_env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 0282ff55d9571..c17860cbd11ee 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -15,7 +15,9 @@ fi echo "Install Miniconda" DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" -if [[ "$(uname)" == 'Linux' ]]; then +if [[ "$(uname -m)" == 'aarch64' ]]; then + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh" +elif [[ "$(uname)" == 'Linux' ]]; then if [[ "$BITS32" == "yes" ]]; then CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" else @@ -23,8 +25,6 @@ if [[ "$(uname)" == 'Linux' ]]; then fi elif [[ "$(uname)" == 'Darwin' ]]; then CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" -elif [[ "$(uname -m)" == 'aarch64' ]]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh" else echo "OS $(uname) not supported" exit 1 From 32ec5323f69be41b55e08f0c2e683aadf8fc327c Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 17:16:09 -0600 Subject: [PATCH 06/14] passing parameters to run tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cd5abc32d044e..740865adb73f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: steps: - checkout - run: ENV_FILE="ci/deps/circle-37-arm64.yaml" ci/setup_env.sh - - run: ci/run_tests.sh + - run: PYTEST_WORKERS="auto" PATTERN="not slow and not network and not clipboard and not arm_slow" ci/run_tests.sh workflows: build: From 78adf49cf1a427d39efec01c187bfceb88e5d9f3 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 17:30:48 -0600 Subject: [PATCH 07/14] Improving cirle ci config --- .circleci/config.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 740865adb73f1..1e44d05972799 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,12 +5,11 @@ jobs: machine: image: ubuntu-2004:202101-01 resource_class: arm.medium + environment: + ENV_FILE: ci/deps/circle-37-arm64.yaml + PYTEST_WORKERS: auto + PATTERN: "not slow and not network and not clipboard and not arm_slow" steps: - checkout - - run: ENV_FILE="ci/deps/circle-37-arm64.yaml" ci/setup_env.sh - - run: PYTEST_WORKERS="auto" PATTERN="not slow and not network and not clipboard and not arm_slow" ci/run_tests.sh - -workflows: - build: - jobs: - - test-arm + - run: ci/setup_env.sh + - run: PATH=$HOME/miniconda3/bin:$PATH ci/run_tests.sh From 5acb47501484490a0b91f70915fab73ad37d1c0a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 17:33:45 -0600 Subject: [PATCH 08/14] Restoring workflow --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e44d05972799..74912eb7666a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,3 +13,8 @@ jobs: - checkout - run: ci/setup_env.sh - run: PATH=$HOME/miniconda3/bin:$PATH ci/run_tests.sh + +workflows: + test: + jobs: + - test-arm From a7c7c790359178637d738355196ab0bc94e24f46 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 18:27:56 -0600 Subject: [PATCH 09/14] Adding debug info --- ci/run_tests.sh | 2 ++ ci/setup_env.sh | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 0d6f26d8c29f8..526d690cbc4a6 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +echo $PATH + # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set) # https://github.com/pytest-dev/pytest/issues/920 # https://github.com/pytest-dev/pytest/issues/1075 diff --git a/ci/setup_env.sh b/ci/setup_env.sh index c17860cbd11ee..b1b426353bcb0 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -148,3 +148,9 @@ else echo "not using dbs on non-linux Travis builds or Azure Pipelines" fi echo "done" + + +echo "PATH" +echo $PATH +echo "which pytest" +which pytest From fccf727a3f750cd0c56c53ec7e3ae78249d8c5a4 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 30 May 2021 18:41:28 -0600 Subject: [PATCH 10/14] Fixing PATH, removing debug info --- .circleci/config.yml | 2 +- ci/run_tests.sh | 2 -- ci/setup_env.sh | 6 ------ 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74912eb7666a7..e34f8dc826aa5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: steps: - checkout - run: ci/setup_env.sh - - run: PATH=$HOME/miniconda3/bin:$PATH ci/run_tests.sh + - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh workflows: test: diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 526d690cbc4a6..0d6f26d8c29f8 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,7 +1,5 @@ #!/bin/bash -e -echo $PATH - # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set) # https://github.com/pytest-dev/pytest/issues/920 # https://github.com/pytest-dev/pytest/issues/1075 diff --git a/ci/setup_env.sh b/ci/setup_env.sh index b1b426353bcb0..c17860cbd11ee 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -148,9 +148,3 @@ else echo "not using dbs on non-linux Travis builds or Azure Pipelines" fi echo "done" - - -echo "PATH" -echo $PATH -echo "which pytest" -which pytest From 7df1f52d2600b85d4503ccf7fe0bb0167c681863 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 31 May 2021 18:26:24 -0600 Subject: [PATCH 11/14] Updating conda arm url --- ci/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index c17860cbd11ee..e6bd9950331ca 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -16,7 +16,7 @@ fi echo "Install Miniconda" DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" if [[ "$(uname -m)" == 'aarch64' ]]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh" + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" elif [[ "$(uname)" == 'Linux' ]]; then if [[ "$BITS32" == "yes" ]]; then CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" From 8ebadbf07498a3ea7fb5b602d98203ac62e3aaae Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 31 May 2021 21:16:44 -0600 Subject: [PATCH 12/14] Xfailing broken ARM tests --- pandas/compat/__init__.py | 12 ++++++++++++ pandas/tests/indexes/interval/test_astype.py | 3 +++ pandas/tests/window/test_rolling.py | 2 ++ 3 files changed, 17 insertions(+) diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index 8d64bf8852946..369832e9bc05c 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -92,6 +92,18 @@ def is_platform_mac() -> bool: return sys.platform == "darwin" +def is_platform_arm() -> bool: + """ + Checking if he running platform use ARM architecture. + + Returns + ------- + bool + True if the running platform uses ARM architecture. + """ + return platform.machine() in ("arm64", "aarch64") + + def import_lzma(): """ Importing the `lzma` module. diff --git a/pandas/tests/indexes/interval/test_astype.py b/pandas/tests/indexes/interval/test_astype.py index cac145aa30fd0..bdb9c3f97e798 100644 --- a/pandas/tests/indexes/interval/test_astype.py +++ b/pandas/tests/indexes/interval/test_astype.py @@ -3,6 +3,8 @@ import numpy as np import pytest +from pandas.compat import is_platform_arm + from pandas.core.dtypes.dtypes import ( CategoricalDtype, IntervalDtype, @@ -168,6 +170,7 @@ def test_subtype_integer_with_non_integer_borders(self, subtype): ) tm.assert_index_equal(result, expected) + @pytest.mark.xfail(is_platform_arm(), reason="GH 41740") def test_subtype_integer_errors(self): # float64 -> uint64 fails with negative values index = interval_range(-10.0, 10.0) diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py index c28d54dd9fbfb..17a6d9216ca92 100644 --- a/pandas/tests/window/test_rolling.py +++ b/pandas/tests/window/test_rolling.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from pandas.compat import is_platform_arm from pandas.errors import UnsupportedFunctionCall from pandas import ( @@ -1072,6 +1073,7 @@ def test_rolling_sem(frame_or_series): tm.assert_series_equal(result, expected) +@pytest.mark.xfail(is_platform_arm(), reason="GH 41740") @pytest.mark.parametrize( ("func", "third_value", "values"), [ From c0fd82d82ea14eb8a97c360b5d900ecf8d50e590 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 31 May 2021 22:14:56 -0600 Subject: [PATCH 13/14] Fixing xpass --- pandas/tests/tools/test_to_numeric.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/tools/test_to_numeric.py b/pandas/tests/tools/test_to_numeric.py index eecb9492f29e3..643a5617abbeb 100644 --- a/pandas/tests/tools/test_to_numeric.py +++ b/pandas/tests/tools/test_to_numeric.py @@ -4,6 +4,8 @@ from numpy import iinfo import pytest +from pandas.compat import is_platform_arm + import pandas as pd from pandas import ( DataFrame, @@ -750,7 +752,7 @@ def test_to_numeric_from_nullable_string(values, nullable_string_dtype, expected "UInt64", "signed", "UInt64", - marks=pytest.mark.xfail(reason="GH38798"), + marks=pytest.mark.xfail(not is_platform_arm(), reason="GH38798"), ), ([1, 1], "Int64", "unsigned", "UInt8"), ([1.0, 1.0], "Float32", "unsigned", "UInt8"), From e8e2cf984939a4089b32a1e8fa0cd74a93fd7f24 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 1 Jun 2021 06:41:15 -0600 Subject: [PATCH 14/14] Removing CircleCI config --- .circleci/config.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e34f8dc826aa5..0000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 2.1 - -jobs: - test-arm: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.medium - environment: - ENV_FILE: ci/deps/circle-37-arm64.yaml - PYTEST_WORKERS: auto - PATTERN: "not slow and not network and not clipboard and not arm_slow" - steps: - - checkout - - run: ci/setup_env.sh - - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh - -workflows: - test: - jobs: - - test-arm