From d127721f25ee7063125f6b6f615dc4a7902297df Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 16 Nov 2019 15:35:56 +0000 Subject: [PATCH 01/53] Adding db build to actions --- .github/workflows/tests.yml | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000000..25943f11710e9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,46 @@ +name: Tests + +on: + push: + branches: master + pull_request: + branches: master + + +jobs: + tests: + name: Run tests + runs-on: ${{ matrix.os }} + services: + postgres: + image: postgres:10.8 + strategy: + matrix: + os: [ubuntu-latest] + include: + - os: ubuntu-latest + env_file: travis-36-cov.yaml + pattern: "((not slow and not network) or (single and db))" + pandas_testing_mode: deprecate + coverage: true + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Setup environment and build pandas + run: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/setup_env.sh + with: + ENV_FILE: ci/deps/${{ matrix.env_file }} + + - name: Run tests + run: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + ci/run_tests.sh + with: + PATTERN: ${{ matrix.pattern }} + PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} + COVERAGE: ${{ matrix.coverage }} From 3c3f25d39e5ec5d1b3a2da932003384f2988c13b Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 16 Nov 2019 15:38:53 +0000 Subject: [PATCH 02/53] Replacing with by env --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25943f11710e9..a9e533327c04f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: run: | export PATH=$HOME/miniconda3/bin:$PATH ci/setup_env.sh - with: + env: ENV_FILE: ci/deps/${{ matrix.env_file }} - name: Run tests @@ -40,7 +40,7 @@ jobs: export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/run_tests.sh - with: + env: PATTERN: ${{ matrix.pattern }} PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} COVERAGE: ${{ matrix.coverage }} From f1352545f4526fc861e57004418a88ee493ccec8 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 18:48:58 +0000 Subject: [PATCH 03/53] Adding mysql service --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9e533327c04f..83ccc23dc5bfa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,8 @@ jobs: services: postgres: image: postgres:10.8 + mysql: + image: mysql:8.0 strategy: matrix: os: [ubuntu-latest] From 91ab51d34a3d87e2b15bd130ebecbb16b01f6e8d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 18:50:15 +0000 Subject: [PATCH 04/53] Fixing indentation --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 83ccc23dc5bfa..9adfa641c89e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,8 +14,8 @@ jobs: services: postgres: image: postgres:10.8 - mysql: - image: mysql:8.0 + mysql: + image: mysql:8.0 strategy: matrix: os: [ubuntu-latest] From e374623d9fca33588afd59067d4d61c514022829 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 19:36:30 +0000 Subject: [PATCH 05/53] Adding ports and healthchecks to db services --- .github/workflows/tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9adfa641c89e8..eaae2451e6959 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,8 +14,16 @@ jobs: services: postgres: image: postgres:10.8 + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + # (copied from https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml) + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 mysql: image: mysql:8.0 + ports: + - '8888:3306' + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 strategy: matrix: os: [ubuntu-latest] From 54e1f3d98361c0e674af8a2ae3caad6659f33583 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 19:43:47 +0000 Subject: [PATCH 06/53] disabling mysql health checks --- .github/workflows/tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eaae2451e6959..8aa95918a47a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,8 +22,9 @@ jobs: mysql: image: mysql:8.0 ports: - - '8888:3306' - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + - 8888:3306 + # healthy checks not passing, no idea why + # options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 strategy: matrix: os: [ubuntu-latest] From f235e9bebfcbe4d7355bc7bbf9e3b3dcc5a6c0a6 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 20:08:52 +0000 Subject: [PATCH 07/53] using latest versions of dbs --- .github/workflows/tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8aa95918a47a4..dc1348efd12b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,14 +13,12 @@ jobs: runs-on: ${{ matrix.os }} services: postgres: - image: postgres:10.8 + image: postgres:latest ports: - 5432:5432 - # needed because the postgres container does not provide a healthcheck - # (copied from https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml) options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 mysql: - image: mysql:8.0 + image: mysql:latest ports: - 8888:3306 # healthy checks not passing, no idea why From be4f8272c6434f54338f90f1a77136f41c7bc451 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 20:33:20 +0000 Subject: [PATCH 08/53] Adding database creation --- .github/workflows/tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dc1348efd12b5..042f3107d1084 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,12 @@ jobs: - name: Checkout uses: actions/checkout@v1 + - name: Create databases + run: | + mysql -e 'create database pandas_nosetest;' + psql -c 'create database pandas_nosetest;' -U postgres + if: ${{ matrix.coverage }} + - name: Setup environment and build pandas run: | export PATH=$HOME/miniconda3/bin:$PATH From 2677a0e768b9ebdfc16732a89befb5196da07a4e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 20:42:27 +0000 Subject: [PATCH 09/53] Fixing how to provide an env variable to if --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 042f3107d1084..3c33e7326ddf2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,7 +41,7 @@ jobs: run: | mysql -e 'create database pandas_nosetest;' psql -c 'create database pandas_nosetest;' -U postgres - if: ${{ matrix.coverage }} + if: matrix.coverage - name: Setup environment and build pandas run: | From 211a4262ff38dce33ac055ea5d61c092380fc1fe Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 17 Nov 2019 23:59:27 +0000 Subject: [PATCH 10/53] Testing mysql --- .github/workflows/tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3c33e7326ddf2..d5d2cf86cb520 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,8 +21,9 @@ jobs: image: mysql:latest ports: - 8888:3306 - # healthy checks not passing, no idea why - # options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + env: + MYSQL_ROOT_PASSWORD: password + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 strategy: matrix: os: [ubuntu-latest] @@ -39,7 +40,7 @@ jobs: - name: Create databases run: | - mysql -e 'create database pandas_nosetest;' + mysql -e 'create database pandas_nosetest;' || mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} --user root --password password -e "CREATE DATABASE pandas_nosetest;" psql -c 'create database pandas_nosetest;' -U postgres if: matrix.coverage From 525f3a3344d6596dcda9c296f984745b6947f650 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 00:13:19 +0000 Subject: [PATCH 11/53] More mysql tests --- .github/workflows/tests.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d5d2cf86cb520..c55ae0d7ef3a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,10 +38,12 @@ jobs: - name: Checkout uses: actions/checkout@v1 - - name: Create databases - run: | - mysql -e 'create database pandas_nosetest;' || mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} --user root --password password -e "CREATE DATABASE pandas_nosetest;" - psql -c 'create database pandas_nosetest;' -U postgres + - name: Create PostgreSQL database + run: psql -c 'CREATE DATABASE pandas_nosetest;' -U postgres + if: matrix.coverage + + - name: Create MySQL database + run: mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} --user root --password password -e "CREATE DATABASE pandas_nosetest;" if: matrix.coverage - name: Setup environment and build pandas From b355d2fb4af87fcaec4487fd6d1f4b48f85511c0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 00:21:44 +0000 Subject: [PATCH 12/53] Trying to create dbs --- .github/workflows/tests.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c55ae0d7ef3a3..78d4accace18b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,12 +39,21 @@ jobs: uses: actions/checkout@v1 - name: Create PostgreSQL database - run: psql -c 'CREATE DATABASE pandas_nosetest;' -U postgres + run: | + sudo apt-get install -y postgresql-client + psql -c 'CREATE DATABASE pandas_nosetest;' -U postgres if: matrix.coverage - name: Create MySQL database - run: mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} --user root --password password -e "CREATE DATABASE pandas_nosetest;" - if: matrix.coverage + run: | + sudo apt-get install -y mysql-client + mysql --host 127.0.0.1 \ + --port ${{ job.services.mysql.ports['3306'] }} \ + --user root \ + --password password \ + --execute "CREATE DATABASE pandas_nosetest;" + # if: matrix.coverage + if: true - name: Setup environment and build pandas run: | From 75c9dcd76cbbe89f058d230981b2e0b18394cbac Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 00:45:59 +0000 Subject: [PATCH 13/53] Working on making the databases work --- .github/workflows/tests.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 78d4accace18b..01567d8df723a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: mysql: image: mysql:latest ports: - - 8888:3306 + - 3306:3306 env: MYSQL_ROOT_PASSWORD: password options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 @@ -41,8 +41,12 @@ jobs: - name: Create PostgreSQL database run: | sudo apt-get install -y postgresql-client - psql -c 'CREATE DATABASE pandas_nosetest;' -U postgres - if: matrix.coverage + psql --host 127.0.0.1 + --port ${{ job.services.postgres.ports['5432'] }} \ + --username postgres \ + --command 'CREATE DATABASE pandas_nosetest;' + # if: matrix.coverage + if: true - name: Create MySQL database run: | From 4ea0cd95d05961432cacff1723b2ddea179c60e0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 01:05:37 +0000 Subject: [PATCH 14/53] Adding jobs to existing build --- .github/workflows/ci.yml | 131 ++++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 77 --------------------- 2 files changed, 131 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aa31e0ed3ab0..fc2c1ac22ff7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,88 @@ env: # PATH: $HOME/miniconda3/bin:$PATH jobs: + tests: + name: Run tests + runs-on: ${{ matrix.os }} + + services: + postgres: + image: postgres:latest + ports: + - 5432/tcp + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + mysql: + image: mysql:latest + ports: + - 3306/tcp + env: + MYSQL_ROOT_PASSWORD: password + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + + strategy: + matrix: + os: [ubuntu-latest] + include: + - os: ubuntu-latest + env_file: travis-36-cov.yaml + pattern: "((not slow and not network) or (single and db))" + pandas_testing_mode: deprecate + coverage: true + + - os: ubuntu-latest + env_file: travis-36-locale.yaml + pattern: "((not slow and not network) or (single and db))" + locale_override: zh_CN.UTF-8 + + - os: ubuntu-latest + env_file: travis-37.yaml + pattern: "(not slow and not network)" + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Create PostgreSQL database + run: | + sudo apt-get install -y postgresql-client + psql --host 127.0.0.1 + --port ${{ job.services.postgres.ports['5432'] }} \ + --username postgres \ + --command 'CREATE DATABASE pandas_nosetest;' + # if: matrix.coverage + if: true + + - name: Create MySQL database + run: | + sudo apt-get install -y mysql-client + mysql --host 127.0.0.1 \ + --port ${{ job.services.mysql.ports['3306'] }} \ + --user root \ + --password password \ + --execute "CREATE DATABASE pandas_nosetest;" + # if: matrix.coverage + if: true + + - name: Setup environment and build pandas + run: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/setup_env.sh + env: + ENV_FILE: ci/deps/${{ matrix.env_file }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + - name: Run tests + run: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + ci/run_tests.sh + env: + PATTERN: ${{ matrix.pattern }} + PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} + COVERAGE: ${{ matrix.coverage }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + checks: name: Checks runs-on: ubuntu-latest @@ -101,3 +183,52 @@ jobs: echo "Benchmarks did not run, no changes detected" fi if: true + + web_and_docs: + name: Web and docs + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v1 + + - name: Setup environment and build pandas + run: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/setup_env.sh + + - name: Build website + run: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + python web/pandas_web.py web/pandas --target-path=web/build + + - name: Build documentation + run: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + # Next we should simply have `doc/make.py --warnings-are-errors`, + # everything else is required because the ipython directive doesn't fail the build on errors + # (https://github.com/ipython/ipython/issues/11547) + doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]} + echo "Sphinx exit status: $SPHINX_RET" + ls + grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) + echo "Grep to detect IPython code block errors exit: $IPY_RET" + exit $(( $SPHINX_RET + $IPY_RET )) + + - name: Merge website and docs + run: | + mkdir -p pandas_web/docs + cp -r web/build/* pandas_web/ + cp -r doc/build/html/* pandas_web/docs/ + + - name: Create artifact + run: | + tar -czvf pandas_web.tar.gz pandas_web + + - name: Upload artifact + uses: actions/upload-artifact@master + with: + name: pandas_web + path: pandas_web.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 01567d8df723a..0000000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Tests - -on: - push: - branches: master - pull_request: - branches: master - - -jobs: - tests: - name: Run tests - runs-on: ${{ matrix.os }} - services: - postgres: - image: postgres:latest - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - mysql: - image: mysql:latest - ports: - - 3306:3306 - env: - MYSQL_ROOT_PASSWORD: password - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 - strategy: - matrix: - os: [ubuntu-latest] - include: - - os: ubuntu-latest - env_file: travis-36-cov.yaml - pattern: "((not slow and not network) or (single and db))" - pandas_testing_mode: deprecate - coverage: true - - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Create PostgreSQL database - run: | - sudo apt-get install -y postgresql-client - psql --host 127.0.0.1 - --port ${{ job.services.postgres.ports['5432'] }} \ - --username postgres \ - --command 'CREATE DATABASE pandas_nosetest;' - # if: matrix.coverage - if: true - - - name: Create MySQL database - run: | - sudo apt-get install -y mysql-client - mysql --host 127.0.0.1 \ - --port ${{ job.services.mysql.ports['3306'] }} \ - --user root \ - --password password \ - --execute "CREATE DATABASE pandas_nosetest;" - # if: matrix.coverage - if: true - - - name: Setup environment and build pandas - run: | - export PATH=$HOME/miniconda3/bin:$PATH - ci/setup_env.sh - env: - ENV_FILE: ci/deps/${{ matrix.env_file }} - - - name: Run tests - run: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/run_tests.sh - env: - PATTERN: ${{ matrix.pattern }} - PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} - COVERAGE: ${{ matrix.coverage }} From 87b4197b12476149f70444307c44d2166240b9c5 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 01:20:31 +0000 Subject: [PATCH 15/53] Fixing typo in postgres call, better matrix --- .github/workflows/ci.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc2c1ac22ff7a..572229c6778b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,20 +33,23 @@ jobs: strategy: matrix: - os: [ubuntu-latest] + name: ['main', 'zh-locale + db', 'py37'] include: - - os: ubuntu-latest + - name: 'main' + os: ubuntu-latest env_file: travis-36-cov.yaml pattern: "((not slow and not network) or (single and db))" pandas_testing_mode: deprecate coverage: true - - os: ubuntu-latest + - name: 'zh-locale' + os: ubuntu-latest env_file: travis-36-locale.yaml pattern: "((not slow and not network) or (single and db))" locale_override: zh_CN.UTF-8 - - os: ubuntu-latest + - name: 'py37' + os: ubuntu-latest env_file: travis-37.yaml pattern: "(not slow and not network)" @@ -57,7 +60,7 @@ jobs: - name: Create PostgreSQL database run: | sudo apt-get install -y postgresql-client - psql --host 127.0.0.1 + psql --host 127.0.0.1 \ --port ${{ job.services.postgres.ports['5432'] }} \ --username postgres \ --command 'CREATE DATABASE pandas_nosetest;' From 267c6c3c89cb0b1906f8c8a05bbc35063b80cb07 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 01:27:33 +0000 Subject: [PATCH 16/53] Debugging matrix and grep problems --- .github/workflows/ci.yml | 8 ++++---- ci/code_checks.sh | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 572229c6778b7..e292c3c9c1a61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,22 +33,22 @@ jobs: strategy: matrix: - name: ['main', 'zh-locale + db', 'py37'] + kind: ['main', 'zh-locale + db', 'py37'] include: - - name: 'main' + - kind: 'main' os: ubuntu-latest env_file: travis-36-cov.yaml pattern: "((not slow and not network) or (single and db))" pandas_testing_mode: deprecate coverage: true - - name: 'zh-locale' + - kind: 'zh-locale' os: ubuntu-latest env_file: travis-36-locale.yaml pattern: "((not slow and not network) or (single and db))" locale_override: zh_CN.UTF-8 - - name: 'py37' + - kind: 'py37' os: ubuntu-latest env_file: travis-37.yaml pattern: "(not slow and not network)" diff --git a/ci/code_checks.sh b/ci/code_checks.sh index d5566c522ac64..f229f1e886136 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -190,9 +190,9 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" ".. ipython ::" doc/source RET=$(($RET + $?)) ; echo $MSG "DONE" -    MSG='Check for extra blank lines after the class definition' ; echo $MSG -    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' . -    RET=$(($RET + $?)) ; echo $MSG "DONE" +#    MSG='Check for extra blank lines after the class definition' ; echo $MSG +#    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' . +#    RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG set -o pipefail From e32bbd8af7eda7263792d5192422a2dfb7013212 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 01:37:11 +0000 Subject: [PATCH 17/53] More debugging and trials --- .github/workflows/ci.yml | 8 ++++---- ci/code_checks.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e292c3c9c1a61..55e4eb77644d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,22 +33,22 @@ jobs: strategy: matrix: - kind: ['main', 'zh-locale + db', 'py37'] + name: [py36-cov, py36-locale-zh, py37] include: - - kind: 'main' + - name: py36-cov os: ubuntu-latest env_file: travis-36-cov.yaml pattern: "((not slow and not network) or (single and db))" pandas_testing_mode: deprecate coverage: true - - kind: 'zh-locale' + - name: py36-locale-zh os: ubuntu-latest env_file: travis-36-locale.yaml pattern: "((not slow and not network) or (single and db))" locale_override: zh_CN.UTF-8 - - kind: 'py37' + - name: py37 os: ubuntu-latest env_file: travis-37.yaml pattern: "(not slow and not network)" diff --git a/ci/code_checks.sh b/ci/code_checks.sh index f229f1e886136..1dd7de569b236 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -190,7 +190,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" ".. ipython ::" doc/source RET=$(($RET + $?)) ; echo $MSG "DONE" -#    MSG='Check for extra blank lines after the class definition' ; echo $MSG +    MSG='Check for extra blank lines after the class definition' ; echo $MSG #    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' . #    RET=$(($RET + $?)) ; echo $MSG "DONE" From f92bd8b77720c3241a80e2dd8a989b35c3c675d0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 01:52:50 +0000 Subject: [PATCH 18/53] More debugging --- .github/workflows/ci.yml | 169 +++++++++++++++++++-------------------- ci/code_checks.sh | 4 +- 2 files changed, 87 insertions(+), 86 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55e4eb77644d0..f2f8b0eeb5564 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,91 +12,6 @@ env: # PATH: $HOME/miniconda3/bin:$PATH jobs: - tests: - name: Run tests - runs-on: ${{ matrix.os }} - - services: - postgres: - image: postgres:latest - ports: - - 5432/tcp - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - - mysql: - image: mysql:latest - ports: - - 3306/tcp - env: - MYSQL_ROOT_PASSWORD: password - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 - - strategy: - matrix: - name: [py36-cov, py36-locale-zh, py37] - include: - - name: py36-cov - os: ubuntu-latest - env_file: travis-36-cov.yaml - pattern: "((not slow and not network) or (single and db))" - pandas_testing_mode: deprecate - coverage: true - - - name: py36-locale-zh - os: ubuntu-latest - env_file: travis-36-locale.yaml - pattern: "((not slow and not network) or (single and db))" - locale_override: zh_CN.UTF-8 - - - name: py37 - os: ubuntu-latest - env_file: travis-37.yaml - pattern: "(not slow and not network)" - - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Create PostgreSQL database - run: | - sudo apt-get install -y postgresql-client - psql --host 127.0.0.1 \ - --port ${{ job.services.postgres.ports['5432'] }} \ - --username postgres \ - --command 'CREATE DATABASE pandas_nosetest;' - # if: matrix.coverage - if: true - - - name: Create MySQL database - run: | - sudo apt-get install -y mysql-client - mysql --host 127.0.0.1 \ - --port ${{ job.services.mysql.ports['3306'] }} \ - --user root \ - --password password \ - --execute "CREATE DATABASE pandas_nosetest;" - # if: matrix.coverage - if: true - - - name: Setup environment and build pandas - run: | - export PATH=$HOME/miniconda3/bin:$PATH - ci/setup_env.sh - env: - ENV_FILE: ci/deps/${{ matrix.env_file }} - LOCALE_OVERRIDE: ${{ matrix.locale_override }} - - - name: Run tests - run: | - export PATH=$HOME/miniconda3/bin:$PATH - source activate pandas-dev - ci/run_tests.sh - env: - PATTERN: ${{ matrix.pattern }} - PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} - COVERAGE: ${{ matrix.coverage }} - LOCALE_OVERRIDE: ${{ matrix.locale_override }} - checks: name: Checks runs-on: ubuntu-latest @@ -235,3 +150,87 @@ jobs: with: name: pandas_web path: pandas_web.tar.gz + + tests: + name: Tests + runs-on: ${{ matrix.os }} + + services: + postgres: + image: postgres:latest + ports: + - 5432/tcp + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + mysql: + image: mysql:latest + ports: + - 3306/tcp + env: + MYSQL_ROOT_PASSWORD: password + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + name: [py36-cov, py36-locale-zh, py37] + include: + - name: py36-cov + os: ubuntu-latest + env_file: travis-36-cov.yaml + pattern: "((not slow and not network) or (single and db))" + pandas_testing_mode: deprecate + coverage: true + + - name: py36-locale-zh + os: ubuntu-latest + env_file: travis-36-locale.yaml + pattern: "((not slow and not network) or (single and db))" + locale_override: zh_CN.UTF-8 + + - name: py37 + os: ubuntu-latest + env_file: travis-37.yaml + pattern: "(not slow and not network)" + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Create PostgreSQL database + run: | + sudo apt-get install -y postgresql-client + psql --host 127.0.0.1 \ + --port ${{ job.services.postgres.ports['5432'] }} \ + --username postgres \ + --command 'CREATE DATABASE pandas_nosetest;' + if: matrix.coverage + + - name: Create MySQL database + run: | + mysql --host 127.0.0.1 \ + --port ${{ job.services.mysql.ports['3306'] }} \ + --user pandas \ + --password password \ + --execute "CREATE DATABASE pandas_nosetest;" + # if: matrix.coverage + if: true + + - name: Setup environment and build pandas + run: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/setup_env.sh + env: + ENV_FILE: ci/deps/${{ matrix.env_file }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + - name: Run tests + run: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas-dev + ci/run_tests.sh + env: + PATTERN: ${{ matrix.pattern }} + PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} + COVERAGE: ${{ matrix.coverage }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 1dd7de569b236..0347fd9db5712 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -173,6 +173,8 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ RET=$(($RET + $?)) ; echo $MSG "DONE" +    MSG='Check for extra blank lines after the class definition' ; echo $MSG + MSG='Check for incorrect sphinx directives' ; echo $MSG invgrep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source RET=$(($RET + $?)) ; echo $MSG "DONE" @@ -190,7 +192,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" ".. ipython ::" doc/source RET=$(($RET + $?)) ; echo $MSG "DONE" -    MSG='Check for extra blank lines after the class definition' ; echo $MSG +#    MSG='Check for extra blank lines after the class definition' ; echo $MSG #    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' . #    RET=$(($RET + $?)) ; echo $MSG "DONE" From f20c3d02fd0590e90e78f869da2dd1f8b78ca643 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 01:59:08 +0000 Subject: [PATCH 19/53] More debugging --- .github/workflows/ci.yml | 5 +++-- ci/code_checks.sh | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2f8b0eeb5564..e1702cd2598e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,9 +208,10 @@ jobs: - name: Create MySQL database run: | - mysql --host 127.0.0.1 \ + netstat -an + mysql --host localhost \ --port ${{ job.services.mysql.ports['3306'] }} \ - --user pandas \ + --user root \ --password password \ --execute "CREATE DATABASE pandas_nosetest;" # if: matrix.coverage diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 0347fd9db5712..d6eab312da70d 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -173,7 +173,9 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ RET=$(($RET + $?)) ; echo $MSG "DONE" +    OTHERVAR='Check for extra blank lines after the class definition' ; echo $OTHERVAR     MSG='Check for extra blank lines after the class definition' ; echo $MSG +    MSG='Check' ; echo $MSG MSG='Check for incorrect sphinx directives' ; echo $MSG invgrep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source From 3a80742133e40441322e007ac71ebc7beb9e5a4a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 02:04:09 +0000 Subject: [PATCH 20/53] More debugging --- .github/workflows/ci.yml | 7 +++---- ci/code_checks.sh | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1702cd2598e1..1cb60bc83f80f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,14 +208,13 @@ jobs: - name: Create MySQL database run: | - netstat -an + echo ${{ job.services.mysql.ports['3306'] }} + netstat -an | grep LISTEN mysql --host localhost \ --port ${{ job.services.mysql.ports['3306'] }} \ --user root \ - --password password \ --execute "CREATE DATABASE pandas_nosetest;" - # if: matrix.coverage - if: true + if: matrix.coverage - name: Setup environment and build pandas run: | diff --git a/ci/code_checks.sh b/ci/code_checks.sh index d6eab312da70d..0c8078b5b4580 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -174,6 +174,8 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then RET=$(($RET + $?)) ; echo $MSG "DONE"     OTHERVAR='Check for extra blank lines after the class definition' ; echo $OTHERVAR + MSG='Check for incorrect sphinx directives' ; echo $MSG + MSG='Check for extra blank lines' ; echo $MSG     MSG='Check for extra blank lines after the class definition' ; echo $MSG     MSG='Check' ; echo $MSG From 449865d6ac2223e42a1119aff110be0fcf8adf38 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 02:14:16 +0000 Subject: [PATCH 21/53] Copying exact configuration from rails blog --- .github/workflows/ci.yml | 23 ++++++++++++++++++----- ci/code_checks.sh | 7 ++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cb60bc83f80f..1c305d1a7ebe4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,13 +162,21 @@ jobs: - 5432/tcp options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + #mysql: + # image: mysql:latest + # ports: + # - 3306/tcp + # env: + # MYSQL_ROOT_PASSWORD: password + # options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + mysql: - image: mysql:latest - ports: - - 3306/tcp + image: mysql:5.7 env: - MYSQL_ROOT_PASSWORD: password - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + MYSQL_ROOT_PASSWORD: root + ports: + - 3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 strategy: fail-fast: false @@ -197,6 +205,11 @@ jobs: - name: Checkout uses: actions/checkout@v1 + - name: Verify MySQL connection from host + run: | + sudo apt-get install -y mysql-client libmysqlclient-dev + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -proot -e "SHOW GRANTS FOR 'root'@'localhost'" + - name: Create PostgreSQL database run: | sudo apt-get install -y postgresql-client diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 0c8078b5b4580..44e8f2cde71cc 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -173,11 +173,16 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ RET=$(($RET + $?)) ; echo $MSG "DONE" -    OTHERVAR='Check for extra blank lines after the class definition' ; echo $OTHERVAR + MSG='XXXXXXXXXXXXXXXXXXXXXXX' ; echo $MSG MSG='Check for incorrect sphinx directives' ; echo $MSG MSG='Check for extra blank lines' ; echo $MSG + MSG='Check for extra blank lines after' ; echo $MSG + MSG='Check for extra blank lines after the' ; echo $MSG + MSG='Check for extra blank lines after the class' ; echo $MSG + MSG='Check for extra blank lines after the class definition' ; echo $MSG     MSG='Check for extra blank lines after the class definition' ; echo $MSG     MSG='Check' ; echo $MSG + MSG='XXXXXXXXXXXXXXXXXXXXXXX' ; echo $MSG MSG='Check for incorrect sphinx directives' ; echo $MSG invgrep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source From 6d830d0422338fad93b26d7d31f13bd3464fd366 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 02:38:20 +0000 Subject: [PATCH 22/53] Some more debugging --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++-- ci/code_checks.sh | 13 +----------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c305d1a7ebe4..cadd1aeb10c6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,20 @@ jobs: fi if: true + - name: Create build output artifact + run: | + echo $STEPS_CONTEXT > build_output.json + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + if: job.status == 'failure' + + - name: Upload build output artifact + uses: actions/upload-artifact@master + with: + name: build_output_checks + path: build_output.json + if: job.status == 'failure' + web_and_docs: name: Web and docs runs-on: ubuntu-latest @@ -151,6 +165,20 @@ jobs: name: pandas_web path: pandas_web.tar.gz + - name: Create build output artifact + run: | + echo $STEPS_CONTEXT > build_output.json + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + if: job.status == 'failure' + + - name: Upload build output artifact + uses: actions/upload-artifact@master + with: + name: build_output_web_and_docs + path: build_output.json + if: job.status == 'failure' + tests: name: Tests runs-on: ${{ matrix.os }} @@ -223,9 +251,10 @@ jobs: run: | echo ${{ job.services.mysql.ports['3306'] }} netstat -an | grep LISTEN - mysql --host localhost \ - --port ${{ job.services.mysql.ports['3306'] }} \ + mysql --host 127.0.0.1 \ + --port ${{ job.services.mysql.ports['3306'] }} \ # should quotes be removed? --user root \ + --password root \ --execute "CREATE DATABASE pandas_nosetest;" if: matrix.coverage @@ -247,3 +276,15 @@ jobs: PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} COVERAGE: ${{ matrix.coverage }} LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + + finalize: + name: Finalize build + runs-on: ubuntu-latest + needs: [checks, web_and_docs, tests] + steps: + + - name: Report build status + run: | + echo "Download the artifacts with the output of each job" + echo "Create a comment in the PR with the errors" diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 44e8f2cde71cc..241adcdcd7823 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -173,17 +173,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/ RET=$(($RET + $?)) ; echo $MSG "DONE" - MSG='XXXXXXXXXXXXXXXXXXXXXXX' ; echo $MSG - MSG='Check for incorrect sphinx directives' ; echo $MSG - MSG='Check for extra blank lines' ; echo $MSG - MSG='Check for extra blank lines after' ; echo $MSG - MSG='Check for extra blank lines after the' ; echo $MSG - MSG='Check for extra blank lines after the class' ; echo $MSG - MSG='Check for extra blank lines after the class definition' ; echo $MSG -    MSG='Check for extra blank lines after the class definition' ; echo $MSG -    MSG='Check' ; echo $MSG - MSG='XXXXXXXXXXXXXXXXXXXXXXX' ; echo $MSG - MSG='Check for incorrect sphinx directives' ; echo $MSG invgrep -R --include="*.py" --include="*.pyx" --include="*.rst" -E "\.\. (autosummary|contents|currentmodule|deprecated|function|image|important|include|ipython|literalinclude|math|module|note|raw|seealso|toctree|versionadded|versionchanged|warning):[^:]" ./pandas ./doc/source RET=$(($RET + $?)) ; echo $MSG "DONE" @@ -201,7 +190,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -R --include="*.rst" ".. ipython ::" doc/source RET=$(($RET + $?)) ; echo $MSG "DONE" -#    MSG='Check for extra blank lines after the class definition' ; echo $MSG + MSG='Check for extra blank lines after the class definition' ; echo $MSG #    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' . #    RET=$(($RET + $?)) ; echo $MSG "DONE" From 02061c1c944d21e8dd7f676380d5c4bf4c99a2cc Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 02:43:30 +0000 Subject: [PATCH 23/53] Rewriting the mysterious failing code in code_checks.sh --- ci/code_checks.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 241adcdcd7823..edd8fcd418c47 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -191,8 +191,8 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check for extra blank lines after the class definition' ; echo $MSG -#    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' . -#    RET=$(($RET + $?)) ; echo $MSG "DONE" + invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' . + RET=$(($RET + $?)) ; echo $MSG "DONE" MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG set -o pipefail From 89ed13064885604b20f44b383f3142fbf21d4459 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 02:52:32 +0000 Subject: [PATCH 24/53] Another mysql test --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cadd1aeb10c6b..b5ad094ad3339 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -237,6 +237,7 @@ jobs: run: | sudo apt-get install -y mysql-client libmysqlclient-dev mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -proot -e "SHOW GRANTS FOR 'root'@'localhost'" + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -proot -e "CREATE DATABASE pandas_nosetest;" - name: Create PostgreSQL database run: | From 9540ff3d602527ff8dad210c8461334ce140357e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 03:04:36 +0000 Subject: [PATCH 25/53] Getting closer to get mysql working --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5ad094ad3339..0c547d7acb6bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: mysql: image: mysql:5.7 env: - MYSQL_ROOT_PASSWORD: root + MYSQL_ROOT_PASSWORD: password ports: - 3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 @@ -237,7 +237,7 @@ jobs: run: | sudo apt-get install -y mysql-client libmysqlclient-dev mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -proot -e "SHOW GRANTS FOR 'root'@'localhost'" - mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -proot -e "CREATE DATABASE pandas_nosetest;" + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} --username root --password password --execute "CREATE DATABASE pandas_nosetest;" - name: Create PostgreSQL database run: | From b149ce8d323e306b44b98f8d95d24ad96078f508 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 03:14:44 +0000 Subject: [PATCH 26/53] Fixing password --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c547d7acb6bf..f7b38fe67533a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -236,7 +236,7 @@ jobs: - name: Verify MySQL connection from host run: | sudo apt-get install -y mysql-client libmysqlclient-dev - mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -proot -e "SHOW GRANTS FOR 'root'@'localhost'" + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -ppassword -e "SHOW GRANTS FOR 'root'@'localhost'" mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} --username root --password password --execute "CREATE DATABASE pandas_nosetest;" - name: Create PostgreSQL database From 3beafd29f4f7e3448c7b2e05274b5b256c37d9f6 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 03:18:27 +0000 Subject: [PATCH 27/53] s/username/user --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7b38fe67533a..1c1c7a072732c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -237,7 +237,7 @@ jobs: run: | sudo apt-get install -y mysql-client libmysqlclient-dev mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -ppassword -e "SHOW GRANTS FOR 'root'@'localhost'" - mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} --username root --password password --execute "CREATE DATABASE pandas_nosetest;" + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} --user root --password password --execute "CREATE DATABASE pandas_nosetest;" - name: Create PostgreSQL database run: | From 71f8b40d81c9e0f55cfaf300b828ef324be118a2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 03:21:52 +0000 Subject: [PATCH 28/53] Checking why password doesn't work --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c1c7a072732c..53a00014072aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -237,7 +237,7 @@ jobs: run: | sudo apt-get install -y mysql-client libmysqlclient-dev mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -ppassword -e "SHOW GRANTS FOR 'root'@'localhost'" - mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} --user root --password password --execute "CREATE DATABASE pandas_nosetest;" + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} --user root -ppassword --execute "CREATE DATABASE pandas_nosetest;" - name: Create PostgreSQL database run: | From 5cd9dc64f7adf6fd0adfee3123a1454b40e1c5c6 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 03:25:02 +0000 Subject: [PATCH 29/53] This should fix the mysql database creation --- .github/workflows/ci.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53a00014072aa..ffede4575bcd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,19 +243,17 @@ jobs: run: | sudo apt-get install -y postgresql-client psql --host 127.0.0.1 \ - --port ${{ job.services.postgres.ports['5432'] }} \ + --port ${{ job.services.postgres.ports[5432] }} \ --username postgres \ --command 'CREATE DATABASE pandas_nosetest;' if: matrix.coverage - name: Create MySQL database run: | - echo ${{ job.services.mysql.ports['3306'] }} - netstat -an | grep LISTEN mysql --host 127.0.0.1 \ - --port ${{ job.services.mysql.ports['3306'] }} \ # should quotes be removed? + --port ${{ job.services.mysql.ports[3306] }} \ --user root \ - --password root \ + --ppassword \ --execute "CREATE DATABASE pandas_nosetest;" if: matrix.coverage From 346e5c25c84c85ebf488ed856966e635a264bac5 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 03:28:11 +0000 Subject: [PATCH 30/53] Fixing typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffede4575bcd0..ccef8714de03c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,7 +253,7 @@ jobs: mysql --host 127.0.0.1 \ --port ${{ job.services.mysql.ports[3306] }} \ --user root \ - --ppassword \ + -ppassword \ --execute "CREATE DATABASE pandas_nosetest;" if: matrix.coverage From 988ea6b3f3af7ca56616ed64ad9e4a21e75ccb21 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 03:31:56 +0000 Subject: [PATCH 31/53] This should leave everything in place regarding dbs --- .github/workflows/ci.yml | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccef8714de03c..942c0aa2bdfea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,21 +190,13 @@ jobs: - 5432/tcp options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - #mysql: - # image: mysql:latest - # ports: - # - 3306/tcp - # env: - # MYSQL_ROOT_PASSWORD: password - # options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 - mysql: - image: mysql:5.7 + image: mysql:latest + ports: + - 3306/tcp env: MYSQL_ROOT_PASSWORD: password - ports: - - 3306 - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 strategy: fail-fast: false @@ -233,12 +225,6 @@ jobs: - name: Checkout uses: actions/checkout@v1 - - name: Verify MySQL connection from host - run: | - sudo apt-get install -y mysql-client libmysqlclient-dev - mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -ppassword -e "SHOW GRANTS FOR 'root'@'localhost'" - mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} --user root -ppassword --execute "CREATE DATABASE pandas_nosetest;" - - name: Create PostgreSQL database run: | sudo apt-get install -y postgresql-client From a81fd0a93e4fa5744a36add8073e982dd176411e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 12:03:18 +0000 Subject: [PATCH 32/53] Adding all builds to actions --- .github/workflows/ci.yml | 68 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 942c0aa2bdfea..75d6f5571da34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,25 +201,85 @@ jobs: strategy: fail-fast: false matrix: - name: [py36-cov, py36-locale-zh, py37] + name: [py36-cov, py36-locale-zh, py37, py37-locale, py36-minimum-versions, py36-macos, py36-32bits, py36-locale-slow, py36-np15-windows, py37-np141-windows, py37-numpydev, py36-locale-slow-old-np] include: + + # db tests + - name: py36-cov os: ubuntu-latest env_file: travis-36-cov.yaml - pattern: "((not slow and not network) or (single and db))" + pattern: "(not slow and not network) or (single and db)" pandas_testing_mode: deprecate coverage: true - name: py36-locale-zh os: ubuntu-latest env_file: travis-36-locale.yaml - pattern: "((not slow and not network) or (single and db))" + pattern: "(not slow and not network) or (single and db)" locale_override: zh_CN.UTF-8 + # regular tests + - name: py37 os: ubuntu-latest env_file: travis-37.yaml - pattern: "(not slow and not network)" + pattern: "not slow and not network" + + - name: py37-locale + os: ubuntu-latest + env_file: azure-37-locale.yaml + pattern: "not slow and not network" + locale_override: zh_CN.UTF-8 + + - name: py36-minimum-versions + os: ubuntu-latest + env_file: azure-36-minimum_versions.yaml + pattern: "not slow and not network" + + - name: py36-macos + os: macos-latest + env_file: azure-macos-36.yaml + pattern: "not slow and not network" + + - name: py36-32bits + os: ubuntu-latest + env_file: azure-36-32bit.yaml + pattern: "not slow and not network" + + - name: py36-locale-slow # TODO: this is not slow, see pattern + os: ubuntu-latest + env_file: azure-36-locale_slow.yaml + pattern: "not slow and not network" + locale_override: it_IT.UTF-8 + + - name: py36-np15-windows + os: windows-latest + env_file: azure-windows-36.yaml + pattern: "not slow and not network" + + - name: py37-np141-windows + os: windows-latest + env_file: azure-windows-37.yaml + pattern: "not slow and not network" + + # TODO: This is commented in azure - https://github.com/pandas-dev/pandas/issues/29432 + - name: py37-numpydev + os: ubuntu-latest + env_file: azure-37-numpydev.yaml + pattern: "not slow and not network" + pandas_testing_mode: deprecate + # TODO env vars pending to add and use in the workflow + # TEST_ARGS: "-W error" + # EXTRA_APT: "xsel" + + # slow tests + + - name: py36-locale-slow-old-np + os: ubuntu-latest + env_file: azure-36-locale.yaml + pattern: "slow" + locale_override: zh_CN.UTF-8 steps: - name: Checkout From c2626535ad2e152de4fab724d1dc0c3914aae52c Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 18 Nov 2019 20:46:54 +0000 Subject: [PATCH 33/53] Making progress on the finalize task --- .github/workflows/ci.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75d6f5571da34..8b4ce10bd4dc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,10 +326,30 @@ jobs: finalize: name: Finalize build runs-on: ubuntu-latest + if: true needs: [checks, web_and_docs, tests] steps: - - name: Report build status + - name: Download checks artifact + uses: actions/download-artifact@master + with: + name: build_output_checks + path: build_output_checks.json + + - name: Extract errors from checks artifact + shell: python run: | - echo "Download the artifacts with the output of each job" - echo "Create a comment in the PR with the errors" + import os + import json + data = json.load(open('build_output_checks.json')) + # TODO: Replace sample errors by actual errors in data + errors = ['This will be replaced', + 'By an actual list of errors'] + msg = ('Hi @{os.environ["AUTHOR"]}, thanks for your contribution.' + 'We found the next list of errors in your pull request:') + json.dump({'body': ['\n- '.join([msg, *errors])}, open('payload.json')]) + + + - name: + run: | + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$( Date: Mon, 18 Nov 2019 20:56:27 +0000 Subject: [PATCH 34/53] Fixing Python code in yaml --- .github/workflows/ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b4ce10bd4dc8..c4f8282bb8430 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -343,12 +343,10 @@ jobs: import json data = json.load(open('build_output_checks.json')) # TODO: Replace sample errors by actual errors in data - errors = ['This will be replaced', - 'By an actual list of errors'] - msg = ('Hi @{os.environ["AUTHOR"]}, thanks for your contribution.' - 'We found the next list of errors in your pull request:') - json.dump({'body': ['\n- '.join([msg, *errors])}, open('payload.json')]) - + errors = ['This will be replaced', 'By an actual list of errors'] + msg = ('Hi @{os.environ["AUTHOR"]}, thanks for your contribution. We found the next list of errors in your pull request:') + # FIXME: The colon makes this break + # json.dump({'body': ['\n- '.join([msg, *errors])}, open('payload.json')]) - name: run: | From ee07631658a63327c85381a8bedae21b016f2fa8 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 19 Nov 2019 01:20:58 +0000 Subject: [PATCH 35/53] Trying to remove colon to fix syntax error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4f8282bb8430..74d8319aaeb3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -344,7 +344,7 @@ jobs: data = json.load(open('build_output_checks.json')) # TODO: Replace sample errors by actual errors in data errors = ['This will be replaced', 'By an actual list of errors'] - msg = ('Hi @{os.environ["AUTHOR"]}, thanks for your contribution. We found the next list of errors in your pull request:') + msg = ('Hi @{os.environ["AUTHOR"]}, thanks for your contribution. We found the next list of errors in your pull request.') # FIXME: The colon makes this break # json.dump({'body': ['\n- '.join([msg, *errors])}, open('payload.json')]) From 457635b392144e5e3e403d0fac8aa16303df0846 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 19 Nov 2019 17:41:42 +0000 Subject: [PATCH 36/53] Removing Python code which was causing problems --- .github/workflows/ci.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74d8319aaeb3e..5a795a1109969 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,16 +337,10 @@ jobs: path: build_output_checks.json - name: Extract errors from checks artifact - shell: python run: | - import os - import json - data = json.load(open('build_output_checks.json')) - # TODO: Replace sample errors by actual errors in data - errors = ['This will be replaced', 'By an actual list of errors'] - msg = ('Hi @{os.environ["AUTHOR"]}, thanks for your contribution. We found the next list of errors in your pull request.') - # FIXME: The colon makes this break - # json.dump({'body': ['\n- '.join([msg, *errors])}, open('payload.json')]) + export MSG="Hi @$AUTHOR, thanks for your contribution. We found the next list of errors in your pull request:" + export MSG="$MSG\n- [Actual messages from the log are not yet extracted](https://github.com/pandas-dev/pandas/actions)" + echo "{\"body\": \"$MSG\"}" > payload.json - name: run: | From 7482cd3856b51609cd9151f114b26841623425b2 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 19 Nov 2019 18:31:10 +0000 Subject: [PATCH 37/53] Fixing indentation --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a795a1109969..daf3eb98c177a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -339,9 +339,10 @@ jobs: - name: Extract errors from checks artifact run: | export MSG="Hi @$AUTHOR, thanks for your contribution. We found the next list of errors in your pull request:" + # TODO get actual errors export MSG="$MSG\n- [Actual messages from the log are not yet extracted](https://github.com/pandas-dev/pandas/actions)" echo "{\"body\": \"$MSG\"}" > payload.json - - name: - run: | - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$( Date: Fri, 22 Nov 2019 01:35:31 +0000 Subject: [PATCH 38/53] Using different jobs for db tests and the rest, and setting conda path just once per job --- .github/workflows/ci.yml | 125 +++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daf3eb98c177a..04c30f527001b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,17 +6,15 @@ on: pull_request: branches: master -env: - ENV_FILE: environment.yml - # TODO: remove export PATH=... in each step once this works - # PATH: $HOME/miniconda3/bin:$PATH - jobs: checks: name: Checks runs-on: ubuntu-latest steps: + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + - name: Checkout uses: actions/checkout@v1 @@ -26,62 +24,53 @@ jobs: - name: Setup environment and build pandas run: | - export PATH=$HOME/miniconda3/bin:$PATH ci/setup_env.sh if: true - name: Linting run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh lint if: true - name: Dependencies consistency run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh dependencies if: true - name: Checks on imported code run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh code if: true - name: Running doctests run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh doctests if: true - name: Docstring validation run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh docstrings if: true - name: Typing validation run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/code_checks.sh typing if: true - name: Testing docstring validation script run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev pytest --capture=no --strict scripts if: true - name: Running benchmarks run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev cd asv_bench asv check -E existing @@ -121,23 +110,22 @@ jobs: runs-on: ubuntu-latest steps: + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + - name: Checkout uses: actions/checkout@v1 - name: Setup environment and build pandas - run: | - export PATH=$HOME/miniconda3/bin:$PATH - ci/setup_env.sh + run: ci/setup_env.sh - name: Build website run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev # Next we should simply have `doc/make.py --warnings-are-errors`, # everything else is required because the ipython directive doesn't fail the build on errors @@ -183,42 +171,12 @@ jobs: name: Tests runs-on: ${{ matrix.os }} - services: - postgres: - image: postgres:latest - ports: - - 5432/tcp - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - - mysql: - image: mysql:latest - ports: - - 3306/tcp - env: - MYSQL_ROOT_PASSWORD: password - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 - strategy: fail-fast: false matrix: - name: [py36-cov, py36-locale-zh, py37, py37-locale, py36-minimum-versions, py36-macos, py36-32bits, py36-locale-slow, py36-np15-windows, py37-np141-windows, py37-numpydev, py36-locale-slow-old-np] + name: [py37, py37-locale, py36-minimum-versions, py36-macos, py36-32bits, py36-locale-slow, py36-np15-windows, py37-np141-windows, py37-numpydev, py36-locale-slow-old-np] include: - # db tests - - - name: py36-cov - os: ubuntu-latest - env_file: travis-36-cov.yaml - pattern: "(not slow and not network) or (single and db)" - pandas_testing_mode: deprecate - coverage: true - - - name: py36-locale-zh - os: ubuntu-latest - env_file: travis-36-locale.yaml - pattern: "(not slow and not network) or (single and db)" - locale_override: zh_CN.UTF-8 - # regular tests - name: py37 @@ -282,6 +240,71 @@ jobs: locale_override: zh_CN.UTF-8 steps: + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + + - name: Checkout + uses: actions/checkout@v1 + + - name: Setup environment and build pandas + run: | + ci/setup_env.sh + env: + ENV_FILE: ci/deps/${{ matrix.env_file }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + - name: Run tests + run: | + source activate pandas-dev + ci/run_tests.sh + env: + PATTERN: ${{ matrix.pattern }} + PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} + COVERAGE: ${{ matrix.coverage }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + db_tests: + name: DB Tests + runs-on: ${{ matrix.os }} + + services: + postgres: + image: postgres:latest + ports: + - 5432/tcp + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + mysql: + image: mysql:latest + ports: + - 3306/tcp + env: + MYSQL_ROOT_PASSWORD: password + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + name: [py36-cov, py36-locale-zh] + include: + + - name: py36-cov + os: ubuntu-latest + env_file: travis-36-cov.yaml + pattern: "(not slow and not network) or (single and db)" + pandas_testing_mode: deprecate + coverage: true + + - name: py36-locale-zh + os: ubuntu-latest + env_file: travis-36-locale.yaml + pattern: "(not slow and not network) or (single and db)" + locale_override: zh_CN.UTF-8 + + steps: + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + - name: Checkout uses: actions/checkout@v1 @@ -305,7 +328,6 @@ jobs: - name: Setup environment and build pandas run: | - export PATH=$HOME/miniconda3/bin:$PATH ci/setup_env.sh env: ENV_FILE: ci/deps/${{ matrix.env_file }} @@ -313,7 +335,6 @@ jobs: - name: Run tests run: | - export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev ci/run_tests.sh env: From b3a52e7edb37bc386f7f54e5d9fb14a5ecf7e6ca Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 22 Nov 2019 01:47:40 +0000 Subject: [PATCH 39/53] Restoing env removed by mistake --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04c30f527001b..a8b071bcbc350 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,9 @@ on: pull_request: branches: master +env: + ENV_FILE: environment.yml + jobs: checks: name: Checks From df925f71d7f250baf9623aa9764044c52d81ee6f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 24 Nov 2019 00:39:03 +0000 Subject: [PATCH 40/53] Trying to set path after checkout --- .github/workflows/ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8b071bcbc350..9b87b2ca73927 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,15 +243,14 @@ jobs: locale_override: zh_CN.UTF-8 steps: - - name: Setting conda path - run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" - - name: Checkout uses: actions/checkout@v1 + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + - name: Setup environment and build pandas - run: | - ci/setup_env.sh + run: ci/setup_env.sh env: ENV_FILE: ci/deps/${{ matrix.env_file }} LOCALE_OVERRIDE: ${{ matrix.locale_override }} From 8c86bab1837faec3bd9561ea9c537099e942c8f5 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 24 Nov 2019 22:39:15 +0000 Subject: [PATCH 41/53] First version using rclone, and debug information added --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++- pandas/tests/config/test_localization.py | 6 +++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b87b2ca73927..e3f9398227c64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,37 @@ jobs: path: build_output.json if: job.status == 'failure' + - name: Install rclone + run: sudo apt install rclone -y + + - name: Set up Rclone + run: | + export RCLONE_CONFIG_PATH=`rclone config file | tail -n1` + mkdir -p `dirname $RCLONE_CONFIG_PATH` + echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH + echo "type = swift" >> $RCLONE_CONFIG_PATH + echo "env_auth = false" >> $RCLONE_CONFIG_PATH + echo "auth_version = 3" >> $RCLONE_CONFIG_PATH + echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH + echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH + echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH + echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH + echo "domain = default" >> $RCLONE_CONFIG_PATH + echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH + echo "key = ${{ secrets.web_hosting_password }}" >> $RCLONE_CONFIG_PATH + echo "region = BHS" >> $RCLONE_CONFIG_PATH + + - name: Sync web + run: | + if [[ "${{ github.event_name }}" == "push" ]]; then + export REMOTE_PATH="." + else + export REMOTE_PATH="pr/$PR_NUMBER" + fi + rclone sync pandas_web ovh_cloud_pandas_web:$REMOTE_PATH + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + tests: name: Tests runs-on: ${{ matrix.os }} @@ -350,7 +381,8 @@ jobs: name: Finalize build runs-on: ubuntu-latest if: true - needs: [checks, web_and_docs, tests] + # TODO run when the other jobs finish: + # needs: [checks, web_and_docs, tests] steps: - name: Download checks artifact diff --git a/pandas/tests/config/test_localization.py b/pandas/tests/config/test_localization.py index 20a5be0c8a289..3270be0db35d1 100644 --- a/pandas/tests/config/test_localization.py +++ b/pandas/tests/config/test_localization.py @@ -68,6 +68,12 @@ def test_set_locale(): lang, enc = "it_CH", "UTF-8" elif locale_override == "C": lang, enc = "en_US", "ascii" + elif len(locale_override.split(".")) != 2: + raise ValueError( + "Unknown format for LOCALE_OVERRIDE, " + "expected country_VARIANT.ENCODING " + f'(e.g. en_US.UTF-8), found "{locale_override}' + ) else: lang, enc = locale_override.split(".") From 9343ea68f247b8292b2ca119bb74ab5ab806d336 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 24 Nov 2019 23:25:08 +0000 Subject: [PATCH 42/53] Several fixes (locale, 32 bits, docs, db) --- .github/workflows/ci.yml | 11 ++++++----- pandas/tests/config/test_localization.py | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3f9398227c64..a8b9165beec56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,9 +135,10 @@ jobs: # (https://github.com/ipython/ipython/issues/11547) doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]} echo "Sphinx exit status: $SPHINX_RET" - ls - grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) - echo "Grep to detect IPython code block errors exit: $IPY_RET" + # TODO: Very strange, but this grep seems to be making the job fail without output + # grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) + # echo "Grep to detect IPython code block errors exit: $IPY_RET" + IPY_RET=0 exit $(( $SPHINX_RET + $IPY_RET )) - name: Merge website and docs @@ -238,6 +239,7 @@ jobs: os: ubuntu-latest env_file: azure-36-32bit.yaml pattern: "not slow and not network" + bits32: "yes" - name: py36-locale-slow # TODO: this is not slow, see pattern os: ubuntu-latest @@ -285,6 +287,7 @@ jobs: env: ENV_FILE: ci/deps/${{ matrix.env_file }} LOCALE_OVERRIDE: ${{ matrix.locale_override }} + BITS32: ${{ matrix.bits32 }} - name: Run tests run: | @@ -348,7 +351,6 @@ jobs: --port ${{ job.services.postgres.ports[5432] }} \ --username postgres \ --command 'CREATE DATABASE pandas_nosetest;' - if: matrix.coverage - name: Create MySQL database run: | @@ -357,7 +359,6 @@ jobs: --user root \ -ppassword \ --execute "CREATE DATABASE pandas_nosetest;" - if: matrix.coverage - name: Setup environment and build pandas run: | diff --git a/pandas/tests/config/test_localization.py b/pandas/tests/config/test_localization.py index 3270be0db35d1..8a62b97dc5bb1 100644 --- a/pandas/tests/config/test_localization.py +++ b/pandas/tests/config/test_localization.py @@ -62,9 +62,9 @@ def test_set_locale(): # getlocale() returned (None, None). pytest.skip("Current locale is not set.") - locale_override = os.environ.get("LOCALE_OVERRIDE", None) + locale_override = os.environ.get("LOCALE_OVERRIDE") - if locale_override is None: + if not locale_override: lang, enc = "it_CH", "UTF-8" elif locale_override == "C": lang, enc = "en_US", "ascii" @@ -72,7 +72,7 @@ def test_set_locale(): raise ValueError( "Unknown format for LOCALE_OVERRIDE, " "expected country_VARIANT.ENCODING " - f'(e.g. en_US.UTF-8), found "{locale_override}' + f"(e.g. en_US.UTF-8), found {locale_override!r}" ) else: lang, enc = locale_override.split(".") From b4970b300938f2b181fb75a17a241755c720213d Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 25 Nov 2019 00:40:24 +0000 Subject: [PATCH 43/53] Improvements to the docs job --- .github/workflows/ci.yml | 53 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8b9165beec56..0830941b5de98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,16 +130,14 @@ jobs: - name: Build documentation run: | source activate pandas-dev - # Next we should simply have `doc/make.py --warnings-are-errors`, - # everything else is required because the ipython directive doesn't fail the build on errors - # (https://github.com/ipython/ipython/issues/11547) - doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]} - echo "Sphinx exit status: $SPHINX_RET" - # TODO: Very strange, but this grep seems to be making the job fail without output - # grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; IPY_RET=$(( $? != 1 )) - # echo "Grep to detect IPython code block errors exit: $IPY_RET" - IPY_RET=0 - exit $(( $SPHINX_RET + $IPY_RET )) + doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} + + - name: Check ipython directive errors + run: | + # This can be removed when the ipython directive fails when there are errors, + # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) + grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log + exit $((! $?)) - name: Merge website and docs run: | @@ -157,26 +155,12 @@ jobs: name: pandas_web path: pandas_web.tar.gz - - name: Create build output artifact - run: | - echo $STEPS_CONTEXT > build_output.json - env: - STEPS_CONTEXT: ${{ toJson(steps) }} - if: job.status == 'failure' - - - name: Upload build output artifact - uses: actions/upload-artifact@master - with: - name: build_output_web_and_docs - path: build_output.json - if: job.status == 'failure' - - name: Install rclone run: sudo apt install rclone -y - name: Set up Rclone run: | - export RCLONE_CONFIG_PATH=`rclone config file | tail -n1` + RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf mkdir -p `dirname $RCLONE_CONFIG_PATH` echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH echo "type = swift" >> $RCLONE_CONFIG_PATH @@ -202,6 +186,20 @@ jobs: env: PR_NUMBER: ${{ github.event.pull_request.number }} + - name: Create build output artifact + run: | + echo $STEPS_CONTEXT > build_output.json + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + if: job.status == 'failure' + + - name: Upload build output artifact + uses: actions/upload-artifact@master + with: + name: build_output_web_and_docs + path: build_output.json + if: job.status == 'failure' + tests: name: Tests runs-on: ${{ matrix.os }} @@ -382,8 +380,8 @@ jobs: name: Finalize build runs-on: ubuntu-latest if: true - # TODO run when the other jobs finish: - # needs: [checks, web_and_docs, tests] + # TODO run when all the other jobs finish: + needs: [checks, web_and_docs] steps: - name: Download checks artifact @@ -393,6 +391,7 @@ jobs: path: build_output_checks.json - name: Extract errors from checks artifact + if: true # TODO temporary for debugging run: | export MSG="Hi @$AUTHOR, thanks for your contribution. We found the next list of errors in your pull request:" # TODO get actual errors From dc4d8050824031640d6faae553715c0f910c2932 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 25 Nov 2019 01:44:18 +0000 Subject: [PATCH 44/53] Couple more fixes (windows bash, ipython grep) --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0830941b5de98..37ab3b6b472f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,8 +136,7 @@ jobs: run: | # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log - exit $((! $?)) + grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; exit $((! $?)) - name: Merge website and docs run: | @@ -281,6 +280,7 @@ jobs: run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" - name: Setup environment and build pandas + shell: bash run: ci/setup_env.sh env: ENV_FILE: ci/deps/${{ matrix.env_file }} @@ -288,6 +288,7 @@ jobs: BITS32: ${{ matrix.bits32 }} - name: Run tests + shell: bash run: | source activate pandas-dev ci/run_tests.sh From f0a403069a2efee9f7baa13d6d2c183215e2a7d4 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 26 Nov 2019 00:49:55 +0000 Subject: [PATCH 45/53] Trying to fix grep of ipython directive --- .github/workflows/ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37ab3b6b472f0..c1d5807b5fa17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,11 +132,10 @@ jobs: source activate pandas-dev doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} + # This can be removed when the ipython directive fails when there are errors, + # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - name: Check ipython directive errors - run: | - # This can be removed when the ipython directive fails when there are errors, - # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log ; exit $((! $?)) + run: ! grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log - name: Merge website and docs run: | From a202e94a2ecf45304b41e2b09e189b966f32cbd9 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 26 Nov 2019 00:51:37 +0000 Subject: [PATCH 46/53] bash command in quotes --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1d5807b5fa17..82f8c081f7771 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - name: Check ipython directive errors - run: ! grep -B1 "^<<<-------------------------------------------------------------------------$" sphinx.log + run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" - name: Merge website and docs run: | From ca23a170e488d4f1ce2a8abedea9e4ccabb8a086 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 26 Nov 2019 01:24:37 +0000 Subject: [PATCH 47/53] Trying more fixes for grep, downloading conda for windows, and small clean up --- .github/workflows/ci.yml | 2 +- ci/setup_env.sh | 47 ++++++++-------------------------------- 2 files changed, 10 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82f8c081f7771..96a8842ce2bed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - name: Check ipython directive errors - run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" + run: bash -c "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" - name: Merge website and docs run: | diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 3d79c0cfd7000..f0e3a0177b110 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -24,18 +24,21 @@ fi echo "Install Miniconda" UNAME_OS=$(uname) if [[ "$UNAME_OS" == 'Linux' ]]; then - if [[ "$BITS32" == "yes" ]]; then - CONDA_OS="Linux-x86" - else - CONDA_OS="Linux-x86_64" - fi + CONDA_OS="Linux" elif [[ "$UNAME_OS" == 'Darwin' ]]; then - CONDA_OS="MacOSX-x86_64" + CONDA_OS="MacOSX" +elif [[ "${UNAME_OS:0:4}" == 'MINGW64' ]]; then + CONDA_OS="Windows" else echo "OS $UNAME_OS not supported" exit 1 fi +CONDA_OS="${CONDA_OS}-x86" +if [[ "$BITS32" != "yes" ]]; then + CONDA_OS="${CONDA_OS}_64" +fi + wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh chmod +x miniconda.sh ./miniconda.sh -b @@ -56,29 +59,6 @@ conda update -n base conda echo "conda info -a" conda info -a -echo -echo "set the compiler cache to work" -if [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then - echo "Using ccache" - export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH - GCC=$(which gcc) - echo "gcc: $GCC" - CCACHE=$(which ccache) - echo "ccache: $CCACHE" - export CC='ccache gcc' -elif [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then - echo "Install ccache" - brew install ccache > /dev/null 2>&1 - echo "Using ccache" - export PATH=/usr/local/opt/ccache/libexec:$PATH - gcc=$(which gcc) - echo "gcc: $gcc" - CCACHE=$(which ccache) - echo "ccache: $CCACHE" -else - echo "Not using ccache" -fi - echo "source deactivate" source deactivate @@ -139,13 +119,4 @@ echo echo "conda list" conda list -# Install DB for Linux -if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - echo "installing dbs" - mysql -e 'create database pandas_nosetest;' - psql -c 'create database pandas_nosetest;' -U postgres -else - echo "not using dbs on non-linux Travis builds or Azure Pipelines" -fi - echo "done" From 0a3e7921b91da6cceb204d937be8fe23a5b2ee3f Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 26 Nov 2019 01:29:44 +0000 Subject: [PATCH 48/53] Fixed typo detecting windows OS --- ci/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index f0e3a0177b110..d0da4c5c12f40 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -27,7 +27,7 @@ if [[ "$UNAME_OS" == 'Linux' ]]; then CONDA_OS="Linux" elif [[ "$UNAME_OS" == 'Darwin' ]]; then CONDA_OS="MacOSX" -elif [[ "${UNAME_OS:0:4}" == 'MINGW64' ]]; then +elif [[ "${UNAME_OS:0:7}" == 'MINGW64' ]]; then CONDA_OS="Windows" else echo "OS $UNAME_OS not supported" From 41dd0c71fb7c57f00190ab9f87e97ceb55b626f9 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 26 Nov 2019 17:11:08 +0000 Subject: [PATCH 49/53] Temporary fixing error in master --- .github/workflows/ci.yml | 2 +- environment.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96a8842ce2bed..f6fe87eab22cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - name: Check ipython directive errors - run: bash -c "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" + run: ! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log - name: Merge website and docs run: | diff --git a/environment.yml b/environment.yml index 848825c37a160..c3b8a8f338883 100644 --- a/environment.yml +++ b/environment.yml @@ -78,7 +78,8 @@ dependencies: - fastparquet>=0.3.2 # pandas.read_parquet, DataFrame.to_parquet - html5lib # pandas.read_html - lxml # pandas.read_html - - openpyxl # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile + # FIXME: Avoiding CI failure in master here, but unpin openpyxl when master is fixed + - openpyxl=3.0.1 # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile - pyarrow>=0.13.1 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather - pyqt>=5.9.2 # pandas.read_clipboard - pytables>=3.4.2 # pandas.read_hdf, DataFrame.to_hdf From b40501c5c83cd8e9a71243a8d85b24f407ffe814 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 26 Nov 2019 17:31:33 +0000 Subject: [PATCH 50/53] Quoting command, needed when starts with exclamation mark --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6fe87eab22cc..82f8c081f7771 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: # This can be removed when the ipython directive fails when there are errors, # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - name: Check ipython directive errors - run: ! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log + run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" - name: Merge website and docs run: | From 79bfd82f9c7db251712043ff8316151d39af560e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Wed, 27 Nov 2019 21:32:35 +0000 Subject: [PATCH 51/53] Fixing path of pr docs, adding clean up task --- .github/workflows/ci.yml | 9 ++++----- .github/workflows/clean_pr.yml | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/clean_pr.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82f8c081f7771..3dd480dd7d559 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,15 +170,14 @@ jobs: echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH echo "domain = default" >> $RCLONE_CONFIG_PATH echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH - echo "key = ${{ secrets.web_hosting_password }}" >> $RCLONE_CONFIG_PATH + echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH echo "region = BHS" >> $RCLONE_CONFIG_PATH - name: Sync web run: | - if [[ "${{ github.event_name }}" == "push" ]]; then - export REMOTE_PATH="." - else - export REMOTE_PATH="pr/$PR_NUMBER" + REMOTE_PATH="dev" + if [[ "${{ github.event_name }}" == "pr" ]]; then + REMOTE_PATH="$REMOTE_PATH/pr/$PR_NUMBER" fi rclone sync pandas_web ovh_cloud_pandas_web:$REMOTE_PATH env: diff --git a/.github/workflows/clean_pr.yml b/.github/workflows/clean_pr.yml new file mode 100644 index 0000000000000..245f7550ea2e7 --- /dev/null +++ b/.github/workflows/clean_pr.yml @@ -0,0 +1,34 @@ +name: Clean on PR closed + +on: + pull_request: + types: [closed] + +jobs: + clean_dev_docs: + name: Clean dev docs + runs-on: ubuntu-latest + steps: + + - name: Set up Rclone + run: | + RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf + mkdir -p `dirname $RCLONE_CONFIG_PATH` + echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH + echo "type = swift" >> $RCLONE_CONFIG_PATH + echo "env_auth = false" >> $RCLONE_CONFIG_PATH + echo "auth_version = 3" >> $RCLONE_CONFIG_PATH + echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH + echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH + echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH + echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH + echo "domain = default" >> $RCLONE_CONFIG_PATH + echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH + echo "key = ${{ secrets.ovh_object_storage_key }}" >> $RCLONE_CONFIG_PATH + echo "region = BHS" >> $RCLONE_CONFIG_PATH + + - name: Clean PR path + run: rclone delete ovh_cloud_pandas_web:dev/pr/$PR_NUMBER + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + From 4647b0d6efeec61f7c0ffe1cd32fee7ced6468e9 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 1 Dec 2019 17:23:31 +0000 Subject: [PATCH 52/53] Removing changes to localization tests, addressed in a separate PR, and removing action to clean up PR docs --- .github/workflows/clean_pr.yml | 34 ------------------------ pandas/tests/config/test_localization.py | 10 ++----- 2 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 .github/workflows/clean_pr.yml diff --git a/.github/workflows/clean_pr.yml b/.github/workflows/clean_pr.yml deleted file mode 100644 index 245f7550ea2e7..0000000000000 --- a/.github/workflows/clean_pr.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Clean on PR closed - -on: - pull_request: - types: [closed] - -jobs: - clean_dev_docs: - name: Clean dev docs - runs-on: ubuntu-latest - steps: - - - name: Set up Rclone - run: | - RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf - mkdir -p `dirname $RCLONE_CONFIG_PATH` - echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH - echo "type = swift" >> $RCLONE_CONFIG_PATH - echo "env_auth = false" >> $RCLONE_CONFIG_PATH - echo "auth_version = 3" >> $RCLONE_CONFIG_PATH - echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH - echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH - echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH - echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH - echo "domain = default" >> $RCLONE_CONFIG_PATH - echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH - echo "key = ${{ secrets.ovh_object_storage_key }}" >> $RCLONE_CONFIG_PATH - echo "region = BHS" >> $RCLONE_CONFIG_PATH - - - name: Clean PR path - run: rclone delete ovh_cloud_pandas_web:dev/pr/$PR_NUMBER - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - diff --git a/pandas/tests/config/test_localization.py b/pandas/tests/config/test_localization.py index 8a62b97dc5bb1..20a5be0c8a289 100644 --- a/pandas/tests/config/test_localization.py +++ b/pandas/tests/config/test_localization.py @@ -62,18 +62,12 @@ def test_set_locale(): # getlocale() returned (None, None). pytest.skip("Current locale is not set.") - locale_override = os.environ.get("LOCALE_OVERRIDE") + locale_override = os.environ.get("LOCALE_OVERRIDE", None) - if not locale_override: + if locale_override is None: lang, enc = "it_CH", "UTF-8" elif locale_override == "C": lang, enc = "en_US", "ascii" - elif len(locale_override.split(".")) != 2: - raise ValueError( - "Unknown format for LOCALE_OVERRIDE, " - "expected country_VARIANT.ENCODING " - f"(e.g. en_US.UTF-8), found {locale_override!r}" - ) else: lang, enc = locale_override.split(".") From 76aca4b5afbe93a1e032526b2f85a4d634d2acbd Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 1 Dec 2019 17:25:58 +0000 Subject: [PATCH 53/53] Removing duplicated set-path --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50e0e0b27b29c..268b3f9693be2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,9 +15,6 @@ jobs: runs-on: ubuntu-latest steps: - - name: Setting conda path - run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" - - name: Checkout uses: actions/checkout@v1