Backport PR #43468 on branch 1.3.x (CI: split Windows Azure tests in half)

.github/workflows/ci.yml

@@ -2,11 +2,13 @@ name: CI
 
 on:
   push:
-    branches: [master]
+    branches:
+      - master
+      - 1.3.x
   pull_request:
     branches:
       - master
-      - 1.2.x
+      - 1.3.x
 
 env:
   ENV_FILE: environment.yml
@@ -123,15 +125,15 @@ jobs:
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
-      if: github.event_name == 'push'
+      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
     - name: Upload web
       run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' --exclude='Pandas_Cheat_Sheet*' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
-      if: github.event_name == 'push'
+      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
     - name: Upload dev docs
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
-      if: github.event_name == 'push'
+      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs
@@ -163,6 +165,7 @@ jobs:
         PANDAS_DATA_MANAGER: array
         PATTERN: ${{ matrix.pattern }}
         PYTEST_WORKERS: "auto"
+        PYTEST_TARGET: pandas
       run: |
         source activate pandas-dev
         ci/run_tests.sh

.github/workflows/database.yml

@@ -6,7 +6,7 @@ on:
   pull_request:
     branches:
       - master
-      - 1.2.x
+      - 1.3.x
     paths-ignore:
       - "doc/**"
 
@@ -99,7 +99,7 @@ jobs:
       run: python ci/print_skipped.py
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2
       with:
         flags: unittests
         name: codecov-pandas

.github/workflows/posix.yml

@@ -2,11 +2,13 @@ name: Posix
 
 on:
   push:
-    branches: [master]
+    branches:
+      - master
+      - 1.3.x
   pull_request:
     branches:
       - master
-      - 1.2.x
+      - 1.3.x
     paths-ignore:
       - "doc/**"
 
@@ -43,6 +45,7 @@ jobs:
       LC_ALL: ${{ matrix.settings[4] }}
       PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
       TEST_ARGS: ${{ matrix.settings[6] }}
+      PYTEST_TARGET:  pandas
 
     steps:
     - name: Checkout
@@ -90,7 +93,7 @@ jobs:
       run: python ci/print_skipped.py
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2
       with:
         flags: unittests
         name: codecov-pandas

.github/workflows/pre-commit.yml

@@ -3,7 +3,9 @@ name: pre-commit
 on:
   pull_request:
   push:
-    branches: [master]
+    branches:
+      - master
+      - 1.3.x
 
 jobs:
   pre-commit:

.github/workflows/python-dev.yml

@@ -4,12 +4,21 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
+      - 1.3.x
     paths-ignore:
       - "doc/**"
 
+env:
+  PYTEST_WORKERS: "auto"
+  PANDAS_CI: 1
+  PATTERN: "not slow and not network and not clipboard"
+  COVERAGE: true
+  PYTEST_TARGET:  pandas
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -29,10 +38,10 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        pip install git+https://github.com/numpy/numpy.git
+        pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         pip install git+https://github.com/pytest-dev/pytest.git
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest-xdist
+        pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov
         pip list
 
     - name: Build Pandas
@@ -46,7 +55,8 @@ jobs:
 
     - name: Test with pytest
       run: |
-        coverage run -m pytest -m 'not slow and not network and not clipboard' pandas
+        ci/run_tests.sh
+      # GH 41935
       continue-on-error: true
 
     - name: Publish test results
@@ -65,7 +75,7 @@ jobs:
         coverage report -m
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2
       with:
         flags: unittests
         name: codecov-pandas

.github/workflows/sdist.yml

@@ -0,0 +1,64 @@
+name: sdist
+
+on:
+  push:
+    branches:
+      - master
+      - 1.3.x
+  pull_request:
+    branches:
+      - master
+      - 1.3.x
+    paths-ignore:
+      - "doc/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.8", "3.9"]
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+
+        # GH 39416
+        pip install numpy
+
+    - name: Build pandas sdist
+      run: |
+        pip list
+        python setup.py sdist --formats=gztar
+
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: pandas-sdist
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install pandas from sdist
+      run: |
+        conda list
+        python -m pip install dist/*.gz
+
+    - name: Import pandas
+      run: |
+        cd ..
+        conda list
+        python -c "import pandas; pandas.show_versions();"

.pre-commit-config.yaml

@@ -110,7 +110,7 @@ repos:
         entry: python scripts/generate_pip_deps_from_conda.py
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
-        additional_dependencies: [pyyaml]
+        additional_dependencies: [pyyaml, toml]
     -   id: sync-flake8-versions
         name: Check flake8 version is synced across flake8, yesqa, and environment.yml
         language: python

MANIFEST.in

@@ -17,18 +17,19 @@ global-exclude *.h5
 global-exclude *.html
 global-exclude *.json
 global-exclude *.jsonl
+global-exclude *.msgpack
 global-exclude *.pdf
 global-exclude *.pickle
 global-exclude *.png
 global-exclude *.pptx
-global-exclude *.pyc
-global-exclude *.pyd
 global-exclude *.ods
 global-exclude *.odt
+global-exclude *.orc
 global-exclude *.sas7bdat
 global-exclude *.sav
 global-exclude *.so
 global-exclude *.xls
+global-exclude *.xlsb
 global-exclude *.xlsm
 global-exclude *.xlsx
 global-exclude *.xpt
@@ -39,6 +40,13 @@ global-exclude .DS_Store
 global-exclude .git*
 global-exclude \#*
 
+global-exclude *.c
+global-exclude *.cpp
+global-exclude *.h
+
+global-exclude *.py[ocd]
+global-exclude *.pxi
+
 # GH 39321
 # csv_dir_path fixture checks the existence of the directory
 # exclude the whole directory to avoid running related tests in sdist
@@ -47,3 +55,6 @@ prune pandas/tests/io/parser/data
 include versioneer.py
 include pandas/_version.py
 include pandas/io/formats/templates/*.tpl
+
+graft pandas/_libs/src
+graft pandas/_libs/tslibs/src

asv_bench/benchmarks/algos/isin.py

@@ -1,7 +1,5 @@
 import numpy as np
 
-from pandas.compat.numpy import np_version_under1p20
-
 from pandas import (
     Categorical,
     NaT,
@@ -280,10 +278,6 @@ class IsInLongSeriesLookUpDominates:
     def setup(self, dtype, MaxNumber, series_type):
         N = 10 ** 7
 
-        # https://github.com/pandas-dev/pandas/issues/39844
-        if not np_version_under1p20 and dtype in ("Int64", "Float64"):
-            raise NotImplementedError
-
         if series_type == "random_hits":
             array = np.random.randint(0, MaxNumber, N)
         if series_type == "random_misses":
@@ -294,7 +288,8 @@ def setup(self, dtype, MaxNumber, series_type):
             array = np.arange(N) + MaxNumber
 
         self.series = Series(array).astype(dtype)
-        self.values = np.arange(MaxNumber).astype(dtype)
+
+        self.values = np.arange(MaxNumber).astype(dtype.lower())
 
     def time_isin(self, dtypes, MaxNumber, series_type):
         self.series.isin(self.values)
@@ -310,18 +305,24 @@ class IsInLongSeriesValuesDominate:
     def setup(self, dtype, series_type):
         N = 10 ** 7
 
-        # https://github.com/pandas-dev/pandas/issues/39844
-        if not np_version_under1p20 and dtype in ("Int64", "Float64"):
-            raise NotImplementedError
-
         if series_type == "random":
             vals = np.random.randint(0, 10 * N, N)
         if series_type == "monotone":
             vals = np.arange(N)
 
-        self.values = vals.astype(dtype)
+        self.values = vals.astype(dtype.lower())
         M = 10 ** 6 + 1
         self.series = Series(np.arange(M)).astype(dtype)
 
     def time_isin(self, dtypes, series_type):
         self.series.isin(self.values)
+
+
+class IsInWithLongTupples:
+    def setup(self):
+        t = tuple(range(1000))
+        self.series = Series([t] * 1000)
+        self.values = [t]
+
+    def time_isin(self):
+        self.series.isin(self.values)

asv_bench/benchmarks/frame_methods.py

@@ -232,6 +232,22 @@ def time_to_html_mixed(self):
         self.df2.to_html()
 
 
+class ToDict:
+    params = [["dict", "list", "series", "split", "records", "index"]]
+    param_names = ["orient"]
+
+    def setup(self, orient):
+        data = np.random.randint(0, 1000, size=(10000, 4))
+        self.int_df = DataFrame(data)
+        self.datetimelike_df = self.int_df.astype("timedelta64[ns]")
+
+    def time_to_dict_ints(self, orient):
+        self.int_df.to_dict(orient=orient)
+
+    def time_to_dict_datetimelike(self, orient):
+        self.datetimelike_df.to_dict(orient=orient)
+
+
 class ToNumpy:
     def setup(self):
         N = 10000

azure-pipelines.yml

@@ -3,29 +3,33 @@ trigger:
   branches:
     include:
     - master
-    - 1.2.x
+    - 1.3.x
   paths:
     exclude:
     - 'doc/*'
 
 pr:
-- master
-- 1.2.x
+  autoCancel: true
+  branches:
+    include:
+    - master
+    - 1.3.x
 
 variables:
   PYTEST_WORKERS: auto
+  PYTEST_TARGET:  pandas
 
 jobs:
 # Mac and Linux use the same template
 - template: ci/azure/posix.yml
   parameters:
     name: macOS
-    vmImage: macOS-10.14
+    vmImage: macOS-10.15
 
 - template: ci/azure/windows.yml
   parameters:
     name: Windows
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
 - job: py37_32bit
   pool: