Backport PR #38344: CI: Move sql builds from Travis to Github actions (#38734)

Co-authored-by: Fangchen Li <[email protected]>

Author: meeseeksmachine

.github/workflows/database.yml

@@ -0,0 +1,180 @@
+name: Database
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches:
+      - master
+      - 1.2.x
+
+env:
+  PYTEST_WORKERS: "auto"
+  PANDAS_CI: 1
+  PATTERN: ((not slow and not network and not clipboard) or (single and db))
+
+jobs:
+  Linux_py37_locale:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      ENV_FILE: ci/deps/actions-37-locale.yaml
+      LOCALE_OVERRIDE: zh_CN.UTF-8
+
+    services:
+      mysql:
+        image: mysql
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+          MYSQL_DATABASE: pandas
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 3306:3306
+
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: pandas
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Cache conda
+      uses: actions/cache@v1
+      env:
+        CACHE_NUMBER: 0
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
+          hashFiles('${{ env.ENV_FILE }}') }}
+
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ${{ env.ENV_FILE }}
+        use-only-tar-bz2: true
+
+    - name: Environment Detail
+      run: |
+        conda info
+        conda list
+
+    - name: Build Pandas
+      run: |
+        python setup.py build_ext -j 2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
+
+    - name: Test
+      run: ci/run_tests.sh
+      if: always()
+
+    - name: Publish test results
+      uses: actions/upload-artifact@master
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Print skipped tests
+      run: python ci/print_skipped.py
+
+  Linux_py37_cov:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      ENV_FILE: ci/deps/actions-37-cov.yaml
+      PANDAS_TESTING_MODE: deprecate
+      COVERAGE: true
+
+    services:
+      mysql:
+        image: mysql
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+          MYSQL_DATABASE: pandas
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 3306:3306
+
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: pandas
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Cache conda
+      uses: actions/cache@v1
+      env:
+        CACHE_NUMBER: 0
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
+          hashFiles('${{ env.ENV_FILE }}') }}
+
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ${{ env.ENV_FILE }}
+        use-only-tar-bz2: true
+
+    - name: Environment Detail
+      run: |
+        conda info
+        conda list
+
+    - name: Build Pandas
+      run: |
+        python setup.py build_ext -j 2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
+
+    - name: Test
+      run: ci/run_tests.sh
+      if: always()
+
+    - name: Publish test results
+      uses: actions/upload-artifact@master
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Print skipped tests
+      run: python ci/print_skipped.py

.travis.yml

@@ -16,13 +16,13 @@ services:
 # travis cache --delete inside the project directory from the travis command line client
 # The cache directories will be deleted if anything in ci/ changes in a commit
 cache:
+  apt: true
   ccache: true
   directories:
     - $HOME/.cache # cython cache
 
 env:
   global:
-    - PYTEST_WORKERS="auto"
     # create a github personal access token
     # cd pandas-dev/pandas
     # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas
@@ -35,25 +35,10 @@ matrix:
   fast_finish: true
 
   include:
-    - env:
-        - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
-      services:
-        - mysql
-        - postgresql
-
     - arch: arm64
       env:
         - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
-    - env:
-        # Enabling Deprecations when running tests
-        # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs
-        # See pandas/_testing.py for more details.
-        - JOB="3.7, coverage" ENV_FILE="ci/deps/travis-37-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
-      services:
-        - mysql
-        - postgresql
-
   allow_failures:
     # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
     - arch: arm64

ci/deps/travis-37-cov.yaml renamed to ci/deps/actions-37-cov.yaml

@@ -15,7 +15,7 @@ dependencies:
   - beautifulsoup4
   - botocore>=1.11
   - dask
-  - fastparquet>=0.3.2
+  - fastparquet>=0.4.0
   - fsspec>=0.7.4
   - gcsfs>=0.6.0
   - geopandas

ci/deps/travis-37-locale.yaml renamed to ci/deps/actions-37-locale.yaml

@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.7.*
@@ -18,9 +17,9 @@ dependencies:
 
   # optional
   - beautifulsoup4
-  - blosc=1.15.0
+  - blosc=1.17.0
   - python-blosc
-  - fastparquet=0.3.2
+  - fastparquet=0.4.0
   - html5lib
   - ipython
   - jinja2
@@ -43,5 +42,5 @@ dependencies:
 
   # sql
   - psycopg2=2.7
-  - pymysql=0.7.11
+  - pymysql=0.8.1
   - sqlalchemy=1.3.0

ci/deps/azure-38-slow.yaml

@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.8.*

ci/deps/azure-windows-38.yaml

@@ -15,7 +15,7 @@ dependencies:
   # pandas dependencies
   - blosc
   - bottleneck
-  - fastparquet>=0.3.2
+  - fastparquet>=0.4.0
   - flask
   - fsspec>=0.8.0
   - matplotlib=3.1.3

doc/source/getting_started/install.rst

@@ -263,12 +263,12 @@ Jinja2                    2.10               Conditional formatting with DataFra
 PyQt4                                        Clipboard I/O
 PyQt5                                        Clipboard I/O
 PyTables                  3.5.1              HDF5-based reading / writing
-SQLAlchemy                1.2.8              SQL support for databases other than sqlite
+SQLAlchemy                1.3.0              SQL support for databases other than sqlite
 SciPy                     1.12.0             Miscellaneous statistical functions
 xlsxwriter                1.0.2              Excel writing
-blosc                     1.15.0             Compression for HDF5
+blosc                     1.17.0             Compression for HDF5
 fsspec                    0.7.4              Handling files aside from local and HTTP
-fastparquet               0.3.2              Parquet reading / writing
+fastparquet               0.4.0              Parquet reading / writing
 gcsfs                     0.6.0              Google Cloud Storage access
 html5lib                  1.0.1              HTML parser for read_html (see :ref:`note <optional_html>`)
 lxml                      4.3.0              HTML parser for read_html (see :ref:`note <optional_html>`)
@@ -278,7 +278,7 @@ openpyxl                  2.6.0              Reading / writing for xlsx files
 pandas-gbq                0.12.0             Google Big Query access
 psycopg2                  2.7                PostgreSQL engine for sqlalchemy
 pyarrow                   0.15.0             Parquet, ORC, and feather reading / writing
-pymysql                   0.7.11             MySQL engine for sqlalchemy
+pymysql                   0.8.1              MySQL engine for sqlalchemy
 pyreadstat                                   SPSS files (.sav) reading
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O

doc/source/whatsnew/v1.2.1.rst

@@ -23,6 +23,13 @@ Fixed regressions
 
 Bug fixes
 ~~~~~~~~~
+
+I/O
+^^^
+
+- Bumped minimum fastparquet version to 0.4.0 to avoid ``AttributeError`` from numba (:issue:`38344`)
+- Bumped minimum pymysql version to 0.8.1 to avoid test failures (:issue:`38344`)
+
 -
 -
 

pandas/tests/io/test_sql.py

@@ -1194,7 +1194,7 @@ def test_sql_open_close(self):
 
     @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed")
     def test_con_string_import_error(self):
-        conn = "mysql://root@localhost/pandas_nosetest"
+        conn = "mysql://root@localhost/pandas"
         msg = "Using URI string without sqlalchemy installed"
         with pytest.raises(ImportError, match=msg):
             sql.read_sql("SELECT * FROM iris", conn)
@@ -1931,11 +1931,12 @@ class _TestMySQLAlchemy:
     """
 
     flavor = "mysql"
+    port = 3306
 
     @classmethod
     def connect(cls):
         return sqlalchemy.create_engine(
-            f"mysql+{cls.driver}://root@localhost/pandas_nosetest",
+            f"mysql+{cls.driver}://root@localhost:{cls.port}/pandas",
             connect_args=cls.connect_args,
         )
 
@@ -2000,11 +2001,12 @@ class _TestPostgreSQLAlchemy:
     """
 
     flavor = "postgresql"
+    port = 5432
 
     @classmethod
     def connect(cls):
         return sqlalchemy.create_engine(
-            f"postgresql+{cls.driver}://postgres@localhost/pandas_nosetest"
+            f"postgresql+{cls.driver}://postgres:postgres@localhost:{cls.port}/pandas"
         )
 
     @classmethod
@@ -2620,7 +2622,7 @@ class TestXMySQL(MySQLMixIn):
     @pytest.fixture(autouse=True, scope="class")
     def setup_class(cls):
         pymysql = pytest.importorskip("pymysql")
-        pymysql.connect(host="localhost", user="root", passwd="", db="pandas_nosetest")
+        pymysql.connect(host="localhost", user="root", passwd="", db="pandas")
         try:
             pymysql.connect(read_default_group="pandas")
         except pymysql.ProgrammingError as err:
@@ -2640,7 +2642,7 @@ def setup_class(cls):
     @pytest.fixture(autouse=True)
     def setup_method(self, request, datapath):
         pymysql = pytest.importorskip("pymysql")
-        pymysql.connect(host="localhost", user="root", passwd="", db="pandas_nosetest")
+        pymysql.connect(host="localhost", user="root", passwd="", db="pandas")
         try:
             pymysql.connect(read_default_group="pandas")
         except pymysql.ProgrammingError as err: