Merge branch 'master' into issue4889

Author: Dr-Irv

.github/workflows/ci.yml

@@ -4,7 +4,9 @@ on:
   push:
     branches: master
   pull_request:
-    branches: master
+    branches:
+      - master
+      - 1.1.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/pre-commit.yml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - uses: pre-commit/[email protected]

.github/workflows/stale-pr.yml

@@ -0,0 +1,21 @@
+name: "Stale PRs"
+on:
+  schedule:
+  # * is a special character in YAML so you have to quote this string
+  - cron: "0 */6 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity."
+        skip-stale-pr-message: true
+        stale-pr-label: "Stale"
+        exempt-pr-labels: "Needs Review,Blocked,Needs Discussion"
+        days-before-stale: 30
+        days-before-close: -1
+        remove-stale-when-updated: false
+        debug-only: false

.pre-commit-config.yaml

@@ -1,44 +1,67 @@
 repos:
 -   repo: https://github.com/python/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
     -   id: black
-        language_version: python3
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.7
+    rev: 3.8.3
     hooks:
     -   id: flake8
-        language: python_venv
         additional_dependencies: [flake8-comprehensions>=3.1.0]
     -   id: flake8
         name: flake8-pyx
-        language: python_venv
         files: \.(pyx|pxd)$
         types:
           - file
         args: [--append-config=flake8/cython.cfg]
     -   id: flake8
         name: flake8-pxd
-        language: python_venv
         files: \.pxi\.in$
         types:
           - file
         args: [--append-config=flake8/cython-template.cfg]
--   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.2.2
     hooks:
     -   id: isort
-        language: python_venv
         exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.730
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.2
     hooks:
-     -  id: mypy
-        args:
-          # As long as a some files are excluded from check-untyped-defs
-          # we have to exclude it from the pre-commit hook as the configuration
-          # is based on modules but the hook runs on files.
-          - --no-check-untyped-defs
-          - --follow-imports
-          - skip
-        files: pandas/
+    -   id: pyupgrade
+        args: [--py37-plus]
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.6.0
+    hooks:
+      - id: rst-backticks
+        # these exclusions should be removed and the files fixed
+        exclude: (?x)(
+          text\.rst|
+          timeseries\.rst|
+          visualization\.rst|
+          missing_data\.rst|
+          options\.rst|
+          reshaping\.rst|
+          scale\.rst|
+          merging\.rst|
+          cookbook\.rst|
+          enhancingperf\.rst|
+          groupby\.rst|
+          io\.rst|
+          overview\.rst|
+          panel\.rst|
+          plotting\.rst|
+          10min\.rst|
+          basics\.rst|
+          categorical\.rst|
+          contributing\.rst|
+          contributing_docstring\.rst|
+          extending\.rst|
+          ecosystem\.rst|
+          comparison_with_sql\.rst|
+          install\.rst|
+          calculate_statistics\.rst|
+          combine_dataframes\.rst|
+          v0\.|
+          v1\.0\.|
+          v1\.1\.[012])

.travis.yml

@@ -1,6 +1,15 @@
 language: python
 python: 3.7
 
+addons:
+  apt:
+    update: true
+    packages:
+      - xvfb
+
+services:
+  - xvfb
+
 # To turn off cached cython files and compiler cache
 # set NOCACHE-true
 # To delete caches go to https://travis-ci.org/OWNER/REPOSITORY/caches or run
@@ -10,30 +19,27 @@ cache:
   ccache: true
   directories:
     - $HOME/.cache # cython cache
-    - $HOME/.ccache # compiler cache
 
 env:
   global:
-    # Variable for test workers
     - PYTEST_WORKERS="auto"
     # create a github personal access token
     # cd pandas-dev/pandas
     # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas
     - secure: "EkWLZhbrp/mXJOx38CHjs7BnjXafsqHtwxPQrqWy457VDFWhIY1DMnIR/lOWG+a20Qv52sCsFtiZEmMfUjf0pLGXOqurdxbYBGJ7/ikFLk9yV2rDwiArUlVM9bWFnFxHvdz9zewBH55WurrY4ShZWyV+x2dWjjceWG5VpWeI6sA="
 
 git:
-  # for cloning
   depth: false
 
 matrix:
   fast_finish: true
 
   include:
-    # In allowed failures
     - dist: bionic
       python: 3.9-dev
       env:
         - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
+
     - env:
         - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
@@ -42,10 +48,10 @@ matrix:
 
     - arch: arm64
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
+        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
     - env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
+        - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
         - mysql
         - postgresql
@@ -54,17 +60,10 @@ matrix:
         # Enabling Deprecations when running tests
         # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs
         # See pandas/_testing.py for more details.
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
+        - JOB="3.7, coverage" ENV_FILE="ci/deps/travis-37-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
       services:
         - mysql
         - postgresql
-  allow_failures:
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
-    - dist: bionic
-      env:
-        - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
 
 
 before_install:
@@ -78,12 +77,6 @@ before_install:
   - uname -a
   - git --version
   - ./ci/check_git_tags.sh
-  # Because travis runs on Google Cloud and has a /etc/boto.cfg,
-  # it breaks moto import, see:
-  # https://github.com/spulec/moto/issues/1771
-  # https://github.com/boto/boto/issues/3741
-  # This overrides travis and tells it to look nowhere.
-  - export BOTO_CONFIG=/dev/null
 
 install:
   - echo "install start"

Makefile

@@ -25,3 +25,16 @@ doc:
 	cd doc; \
 	python make.py clean; \
 	python make.py html
+
+check:
+	python3 scripts/validate_unwanted_patterns.py \
+		--validation-type="private_function_across_module" \
+		--included-file-extensions="py" \
+		--excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored \
+		pandas/
+
+	python3 scripts/validate_unwanted_patterns.py \
+		--validation-type="private_import_across_module" \
+		--included-file-extensions="py" \
+		--excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored,doc/
+		pandas/

README.md

@@ -32,7 +32,7 @@ its way towards this goal.
 Here are just a few of the things that pandas does well:
 
   - Easy handling of [**missing data**][missing-data] (represented as
-    `NaN`) in floating point as well as non-floating point data
+    `NaN`, `NA`, or `NaT`) in floating point as well as non-floating point data
   - Size mutability: columns can be [**inserted and
     deleted**][insertion-deletion] from DataFrame and higher dimensional
     objects

asv_bench/asv.conf.json

@@ -26,7 +26,7 @@
     // The Pythons you'd like to test against.  If not provided, defaults
     // to the current version of Python used to run `asv`.
     // "pythons": ["2.7", "3.4"],
-    "pythons": ["3.6"],
+    "pythons": ["3.8"],
 
     // The matrix of dependencies to test.  Each key is the name of a
     // package (in PyPI) and the values are version numbers.  An empty
@@ -39,7 +39,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.16"],
+        "Cython": ["0.29.21"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

asv_bench/benchmarks/arithmetic.py

@@ -125,7 +125,7 @@ def setup(self, op):
         arr1 = np.random.randn(n_rows, int(n_cols / 2)).astype("f8")
         arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("f4")
         df = pd.concat(
-            [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True,
+            [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True
         )
         # should already be the case, but just to be sure
         df._consolidate_inplace()

asv_bench/benchmarks/frame_ctor.py

@@ -6,7 +6,7 @@
 from .pandas_vb_common import tm
 
 try:
-    from pandas.tseries.offsets import Nano, Hour
+    from pandas.tseries.offsets import Hour, Nano
 except ImportError:
     # For compatibility with older versions
     from pandas.core.datetools import *  # noqa

asv_bench/benchmarks/frame_methods.py

@@ -219,6 +219,46 @@ def time_to_html_mixed(self):
         self.df2.to_html()
 
 
+class ToNumpy:
+    def setup(self):
+        N = 10000
+        M = 10
+        self.df_tall = DataFrame(np.random.randn(N, M))
+        self.df_wide = DataFrame(np.random.randn(M, N))
+        self.df_mixed_tall = self.df_tall.copy()
+        self.df_mixed_tall["foo"] = "bar"
+        self.df_mixed_tall[0] = period_range("2000", periods=N)
+        self.df_mixed_tall[1] = range(N)
+        self.df_mixed_wide = self.df_wide.copy()
+        self.df_mixed_wide["foo"] = "bar"
+        self.df_mixed_wide[0] = period_range("2000", periods=M)
+        self.df_mixed_wide[1] = range(M)
+
+    def time_to_numpy_tall(self):
+        self.df_tall.to_numpy()
+
+    def time_to_numpy_wide(self):
+        self.df_wide.to_numpy()
+
+    def time_to_numpy_mixed_tall(self):
+        self.df_mixed_tall.to_numpy()
+
+    def time_to_numpy_mixed_wide(self):
+        self.df_mixed_wide.to_numpy()
+
+    def time_values_tall(self):
+        self.df_tall.values
+
+    def time_values_wide(self):
+        self.df_wide.values
+
+    def time_values_mixed_tall(self):
+        self.df_mixed_tall.values
+
+    def time_values_mixed_wide(self):
+        self.df_mixed_wide.values
+
+
 class Repr:
     def setup(self):
         nrows = 10000

asv_bench/benchmarks/gil.py

@@ -7,14 +7,14 @@
 
 try:
     from pandas import (
-        rolling_median,
+        rolling_kurt,
+        rolling_max,
         rolling_mean,
+        rolling_median,
         rolling_min,
-        rolling_max,
-        rolling_var,
         rolling_skew,
-        rolling_kurt,
         rolling_std,
+        rolling_var,
     )
 
     have_rolling_methods = True