Merge branch 'pandas-dev:master' into write-to

Author: feefladder

.github/workflows/ci.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches:
       - master
-      - 1.2.x
       - 1.3.x
 
 env:
@@ -24,7 +23,7 @@ jobs:
 
     concurrency:
       group: ${{ github.ref }}-checks
-      cancel-in-progress: true
+      cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     steps:
     - name: Checkout

.github/workflows/database.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches:
       - master
-      - 1.2.x
       - 1.3.x
     paths-ignore:
       - "doc/**"
@@ -19,20 +18,20 @@ env:
   COVERAGE: true
 
 jobs:
-  Linux_py37_IO:
+  Linux_py38_IO:
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
 
     strategy:
       matrix:
-        ENV_FILE: [ci/deps/actions-37-db-min.yaml, ci/deps/actions-37-db.yaml]
+        ENV_FILE: [ci/deps/actions-38-db-min.yaml, ci/deps/actions-38-db.yaml]
       fail-fast: false
 
     concurrency:
       group: ${{ github.ref }}-${{ matrix.ENV_FILE }}
-      cancel-in-progress: true
+      cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     services:
       mysql:

.github/workflows/posix.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches:
       - master
-      - 1.2.x
       - 1.3.x
     paths-ignore:
       - "doc/**"
@@ -25,14 +24,13 @@ jobs:
     strategy:
       matrix:
         settings: [
-          [actions-37-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-37.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-37-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-37-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
+          [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
           [actions-38.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-38-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
+          [actions-39-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
           [actions-39.yaml, "not slow and not network and not clipboard", "", "", "", "", ""]
         ]
       fail-fast: false
@@ -47,7 +45,7 @@ jobs:
       TEST_ARGS: ${{ matrix.settings[6] }}
     concurrency:
       group: ${{ github.ref }}-${{ matrix.settings[0] }}
-      cancel-in-progress: true
+      cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     steps:
     - name: Checkout

.github/workflows/pre-commit.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     concurrency:
       group: ${{ github.ref }}-pre-commit
-      cancel-in-progress: true
+      cancel-in-progress: ${{github.event_name == 'pull_request'}}
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.github/workflows/python-dev.yml

@@ -4,12 +4,19 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
     paths-ignore:
       - "doc/**"
 
+env:
+  PYTEST_WORKERS: "auto"
+  PANDAS_CI: 1
+  PATTERN: "not slow and not network and not clipboard"
+  COVERAGE: true
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -18,7 +25,7 @@ jobs:
 
     concurrency:
       group: ${{ github.ref }}-dev
-      cancel-in-progress: true
+      cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     steps:
     - uses: actions/checkout@v2
@@ -36,7 +43,7 @@ jobs:
         pip install git+https://github.com/numpy/numpy.git
         pip install git+https://github.com/pytest-dev/pytest.git
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest-xdist
+        pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov
         pip list
 
     - name: Build Pandas
@@ -50,7 +57,8 @@ jobs:
 
     - name: Test with pytest
       run: |
-        coverage run -m pytest -m 'not slow and not network and not clipboard' pandas
+        ci/run_tests.sh
+      # GH 41935
       continue-on-error: true
 
     - name: Publish test results

.github/workflows/sdist.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches:
       - master
-      - 1.2.x
       - 1.3.x
     paths-ignore:
       - "doc/**"
@@ -23,7 +22,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.8", "3.9"]
+    concurrency:
+      group: ${{github.ref}}-${{matrix.python-version}}-sdist
+      cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     steps:
     - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -53,14 +53,14 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.9.0
+    rev: 5.9.2
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.19.4
+    rev: v2.21.0
     hooks:
     -   id: pyupgrade
-        args: [--py37-plus]
+        args: [--py38-plus]
 -   repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.9.0
     hooks:

asv_bench/benchmarks/algos/isin.py

@@ -1,6 +1,9 @@
 import numpy as np
 
-from pandas.compat.numpy import np_version_under1p20
+try:
+    from pandas.compat import np_version_under1p20
+except ImportError:
+    from pandas.compat.numpy import _np_version_under1p20 as np_version_under1p20
 
 from pandas import (
     Categorical,

asv_bench/benchmarks/frame_methods.py

@@ -232,6 +232,22 @@ def time_to_html_mixed(self):
         self.df2.to_html()
 
 
+class ToDict:
+    params = [["dict", "list", "series", "split", "records", "index"]]
+    param_names = ["orient"]
+
+    def setup(self, orient):
+        data = np.random.randint(0, 1000, size=(10000, 4))
+        self.int_df = DataFrame(data)
+        self.datetimelike_df = self.int_df.astype("timedelta64[ns]")
+
+    def time_to_dict_ints(self, orient):
+        self.int_df.to_dict(orient=orient)
+
+    def time_to_dict_datetimelike(self, orient):
+        self.datetimelike_df.to_dict(orient=orient)
+
+
 class ToNumpy:
     def setup(self):
         N = 10000

asv_bench/benchmarks/groupby.py

@@ -832,4 +832,18 @@ def function(values):
         self.grouper.agg(function, engine="cython")
 
 
+class Sample:
+    def setup(self):
+        N = 10 ** 3
+        self.df = DataFrame({"a": np.zeros(N)})
+        self.groups = np.arange(0, N)
+        self.weights = np.ones(N)
+
+    def time_sample(self):
+        self.df.groupby(self.groups).sample(n=1)
+
+    def time_sample_weights(self):
+        self.df.groupby(self.groups).sample(n=1, weights=self.weights)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/io/csv.py

@@ -291,7 +291,8 @@ class ReadCSVFloatPrecision(StringIORewind):
 
     def setup(self, sep, decimal, float_precision):
         floats = [
-            "".join(random.choice(string.digits) for _ in range(28)) for _ in range(15)
+            "".join([random.choice(string.digits) for _ in range(28)])
+            for _ in range(15)
         ]
         rows = sep.join([f"0{decimal}" + "{}"] * 3) + "\n"
         data = rows * 5
@@ -395,7 +396,7 @@ class ReadCSVCachedParseDates(StringIORewind):
     param_names = ["do_cache", "engine"]
 
     def setup(self, do_cache, engine):
-        data = ("\n".join(f"10/{year}" for year in range(2000, 2100)) + "\n") * 10
+        data = ("\n".join([f"10/{year}" for year in range(2000, 2100)]) + "\n") * 10
         self.StringIO_input = StringIO(data)
 
     def time_read_csv_cached(self, do_cache, engine):

asv_bench/benchmarks/rolling.py

@@ -296,5 +296,8 @@ def time_apply(self, method):
             table_method_func, raw=True, engine="numba"
         )
 
+    def time_ewm_mean(self, method):
+        self.df.ewm(1, method=method).mean(engine="numba")
+
 
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

azure-pipelines.yml

@@ -3,7 +3,6 @@ trigger:
   branches:
     include:
     - master
-    - 1.2.x
     - 1.3.x
   paths:
     exclude:
@@ -22,14 +21,14 @@ jobs:
 - template: ci/azure/posix.yml
   parameters:
     name: macOS
-    vmImage: macOS-10.14
+    vmImage: macOS-10.15
 
 - template: ci/azure/windows.yml
   parameters:
     name: Windows
-    vmImage: vs2017-win2016
+    vmImage: windows-2019
 
-- job: py37_32bit
+- job: py38_32bit
   pool:
     vmImage: ubuntu-18.04
 
@@ -38,7 +37,7 @@ jobs:
         docker pull quay.io/pypa/manylinux2014_i686
         docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
         /bin/bash -xc "cd pandas && \
-        /opt/python/cp37-cp37m/bin/python -m venv ~/virtualenvs/pandas-dev && \
+        /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel setuptools && \
         pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
@@ -52,4 +51,4 @@ jobs:
       inputs:
         testResultsFiles: '**/test-*.xml'
         failTaskOnFailedTests: true
-        testRunTitle: 'Publish test results for Python 3.7-32 bit full Linux'
+        testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux'

ci/azure/posix.yml

@@ -9,9 +9,9 @@ jobs:
   strategy:
     matrix:
       ${{ if eq(parameters.name, 'macOS') }}:
-        py37_macos:
-          ENV_FILE: ci/deps/azure-macos-37.yaml
-          CONDA_PY: "37"
+        py38_macos:
+          ENV_FILE: ci/deps/azure-macos-38.yaml
+          CONDA_PY: "38"
           PATTERN: "not slow and not network"
 
   steps:

ci/azure/windows.yml

@@ -8,41 +8,39 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py37_np17:
-        ENV_FILE: ci/deps/azure-windows-37.yaml
-        CONDA_PY: "37"
-        PATTERN: "not slow and not network"
-
       py38_np18:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
+        PATTERN: "not slow and not network"
+        PYTEST_WORKERS: 2  # GH-42236
+
+      py39:
+        ENV_FILE: ci/deps/azure-windows-39.yaml
+        CONDA_PY: "39"
         PATTERN: "not slow and not network and not high_memory"
+        PYTEST_WORKERS: 2  # GH-42236
 
   steps:
     - powershell: |
         Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
         Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
       displayName: 'Add conda to PATH'
-
     - script: conda update -q -n base conda
       displayName: 'Update conda'
 
     - bash: |
         conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
       displayName: 'Create anaconda environment'
-
     - bash: |
         source activate pandas-dev
         conda list
         python setup.py build_ext -q -j 4
         python -m pip install --no-build-isolation -e .
       displayName: 'Build'
-
     - bash: |
         source activate pandas-dev
         ci/run_tests.sh
       displayName: 'Test'
-
     - task: PublishTestResults@2
       condition: succeededOrFailed()
       inputs: