Merge remote-tracking branch 'upstream/master' into yanglinlee-json_improve

Author: TomAugspurger

.github/FUNDING.yml

@@ -1 +1,2 @@
 custom: https://pandas.pydata.org/donate.html
+tidelift: pypi/pandas

.github/SECURITY.md

@@ -0,0 +1 @@
+To report a security vulnerability to pandas, please go to https://tidelift.com/security and see the instructions there.

.pre-commit-config.yaml

@@ -9,6 +9,7 @@ repos:
       hooks:
           - id: flake8
             language: python_venv
+            additional_dependencies: [flake8-comprehensions]
     - repo: https://github.com/pre-commit/mirrors-isort
       rev: v4.3.20
       hooks:

.travis.yml

@@ -21,7 +21,7 @@ env:
 
 git:
     # for cloning
-    depth: 2000
+    depth: false
 
 matrix:
     fast_finish: true
@@ -63,7 +63,7 @@ before_install:
   - pwd
   - uname -a
   - git --version
-  - git tag
+  - ./ci/check_git_tags.sh
   # Because travis runs on Google Cloud and has a /etc/boto.cfg,
   # it breaks moto import, see:
   # https://github.com/spulec/moto/issues/1771

README.md

@@ -233,3 +233,5 @@ You can also triage issues which may include reproducing bug reports, or asking
 Or maybe through using pandas you have an idea of your own or are looking for something in the documentation and thinking ‘this can be improved’...you can do something about it!
 
 Feel free to ask questions on the [mailing list](https://groups.google.com/forum/?fromgroups#!forum/pydata) or on [Gitter](https://gitter.im/pydata/pandas).
+
+As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/pandas/blob/master/.github/CODE_OF_CONDUCT.md)

asv_bench/benchmarks/index_object.py

@@ -1,3 +1,4 @@
+import gc
 import numpy as np
 import pandas.util.testing as tm
 from pandas import (
@@ -225,4 +226,21 @@ def time_intersection_both_duplicate(self, N):
         self.intv.intersection(self.intv2)
 
 
+class GC:
+    params = [1, 2, 5]
+
+    def create_use_drop(self):
+        idx = Index(list(range(1000 * 1000)))
+        idx._engine
+
+    def peakmem_gc_instances(self, N):
+        try:
+            gc.disable()
+
+            for _ in range(N):
+                self.create_use_drop()
+        finally:
+            gc.enable()
+
+
 from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/reshape.py

@@ -214,6 +214,7 @@ def setup(self, bins):
         self.datetime_series = pd.Series(
             np.random.randint(N, size=N), dtype="datetime64[ns]"
         )
+        self.interval_bins = pd.IntervalIndex.from_breaks(np.linspace(0, N, bins))
 
     def time_cut_int(self, bins):
         pd.cut(self.int_series, bins)
@@ -239,6 +240,14 @@ def time_qcut_timedelta(self, bins):
     def time_qcut_datetime(self, bins):
         pd.qcut(self.datetime_series, bins)
 
+    def time_cut_interval(self, bins):
+        # GH 27668
+        pd.cut(self.int_series, self.interval_bins)
+
+    def peakmem_cut_interval(self, bins):
+        # GH 27668
+        pd.cut(self.int_series, self.interval_bins)
+
 
 class Explode:
     param_names = ["n_rows", "max_list_length"]

azure-pipelines.yml

@@ -22,80 +22,67 @@ jobs:
   timeoutInMinutes: 90
   steps:
   - script: |
-      # XXX next command should avoid redefining the path in every step, but
-      # made the process crash as it couldn't find deactivate
-      #echo '##vso[task.prependpath]$HOME/miniconda3/bin'
+      echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
       echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
       echo '##vso[task.setvariable variable=AZURE]true'
     displayName: 'Setting environment variables'
 
   # Do not require a conda environment
-  - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
-      ci/code_checks.sh patterns
+  - script: ci/code_checks.sh patterns
     displayName: 'Looking for unwanted patterns'
     condition: true
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       sudo apt-get install -y libc6-dev-i386
       ci/setup_env.sh
     displayName: 'Setup environment and build pandas'
     condition: true
 
   # Do not require pandas
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       ci/code_checks.sh lint
     displayName: 'Linting'
     condition: true
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       ci/code_checks.sh dependencies
     displayName: 'Dependencies consistency'
     condition: true
 
   # Require pandas
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       ci/code_checks.sh code
     displayName: 'Checks on imported code'
     condition: true
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       ci/code_checks.sh doctests
     displayName: 'Running doctests'
     condition: true
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       ci/code_checks.sh docstrings
     displayName: 'Docstring validation'
     condition: true
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       ci/code_checks.sh typing
     displayName: 'Typing validation'
     condition: true
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       pytest --capture=no --strict scripts
-    displayName: 'Testing docstring validaton script'
+    displayName: 'Testing docstring validation script'
     condition: true
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       cd asv_bench
       asv check -E existing
@@ -124,16 +111,15 @@ jobs:
   steps:
   - script: |
       echo '##vso[task.setvariable variable=ENV_FILE]environment.yml'
+      echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
     displayName: 'Setting environment variables'
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       sudo apt-get install -y libc6-dev-i386
       ci/setup_env.sh
     displayName: 'Setup environment and build pandas'
 
   - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev
       # Next we should simply have `doc/make.py --warnings-are-errors`, everything else is required because the ipython directive doesn't fail the build on errors (https://github.com/ipython/ipython/issues/11547)
       doc/make.py --warnings-are-errors | tee sphinx.log ; SPHINX_RET=${PIPESTATUS[0]}

ci/azure/posix.yml

@@ -56,17 +56,15 @@ jobs:
   steps:
     - script: |
         if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386 $EXTRA_APT; fi
+        echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
         echo "Creating Environment"
         ci/setup_env.sh
       displayName: 'Setup environment and build pandas'
     - script: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/run_tests.sh
       displayName: 'Test'
-    - script: |
-        export PATH=$HOME/miniconda3/bin:$PATH
-        source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+    - script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: 'test-data-*.xml'
@@ -97,7 +95,6 @@ jobs:
         }
       displayName: 'Check for test failures'
     - script: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         python ci/print_skipped.py
       displayName: 'Print skipped tests'

ci/azure/windows.yml

@@ -17,7 +17,9 @@ jobs:
         CONDA_PY: "37"
 
   steps:
-    - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    - powershell: |
+        Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+        Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
       displayName: 'Add conda to PATH'
     - script: conda update -q -n base conda
       displayName: Update conda
@@ -52,7 +54,6 @@ jobs:
         }
       displayName: 'Check for test failures'
     - script: |
-        export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         python ci/print_skipped.py
       displayName: 'Print skipped tests'

ci/check_git_tags.sh

@@ -0,0 +1,28 @@
+set -e
+
+if [[ ! $(git tag) ]]; then
+    echo "No git tags in clone, please sync your git tags with upstream using:"
+    echo "    git fetch --tags upstream"
+    echo "    git push --tags origin"
+    echo ""
+    echo "If the issue persists, the clone depth needs to be increased in .travis.yml"
+    exit 1
+fi
+
+# This will error if there are no tags and we omit --always
+DESCRIPTION=$(git describe --long --tags)
+echo "$DESCRIPTION"
+
+if [[ "$DESCRIPTION" == *"untagged"* ]]; then
+    echo "Unable to determine most recent tag, aborting build"
+    exit 1
+else
+    if [[ "$DESCRIPTION" != *"g"* ]]; then
+	# A good description will have the hash prefixed by g, a bad one will be
+	# just the hash
+	echo "Unable to determine most recent tag, aborting build"
+	exit 1
+    else
+	echo "$(git tag)"
+    fi
+fi

ci/code_checks.sh

@@ -263,8 +263,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL03,GL04,GL05,GL06,GL07,GL09,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA05
+    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/azure-36-locale.yaml

@@ -20,8 +20,8 @@ dependencies:
   - xlsxwriter=0.9.8
   - xlwt=1.2.0
   # universal
-  - pytest>=4.0.2,<5.0.0
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - hypothesis>=3.58.0

ci/deps/azure-37-locale.yaml

@@ -26,8 +26,8 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - pip

ci/deps/azure-37-numpydev.yaml

@@ -6,7 +6,8 @@ dependencies:
   - pytz
   - Cython>=0.28.2
   # universal
-  - pytest>=4.0.2
+  # pytest < 5 until defaults has pytest-xdist>=1.29.0
+  - pytest>=4.0.2,<5.0
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0

ci/deps/azure-macos-35.yaml

@@ -25,8 +25,8 @@ dependencies:
   - pip:
     - pyreadstat
     # universal
-    - pytest==4.5.0
-    - pytest-xdist
+    - pytest>=5.0.1
+    - pytest-xdist>=1.29.0
     - pytest-mock
     - hypothesis>=3.58.0
     # https://github.com/pandas-dev/pandas/issues/27421

ci/deps/azure-windows-36.yaml

@@ -23,8 +23,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - hypothesis>=3.58.0

ci/deps/azure-windows-37.yaml

@@ -26,8 +26,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - hypothesis>=3.58.0

ci/deps/travis-36-cov.yaml

@@ -39,8 +39,8 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-cov
   - pytest-mock
   - hypothesis>=3.58.0

ci/deps/travis-36-locale.yaml

@@ -29,13 +29,13 @@ dependencies:
   - s3fs=0.0.8
   - scipy
   - sqlalchemy=1.1.4
-  - xarray=0.8.2
+  - xarray=0.10
   - xlrd
   - xlsxwriter
   - xlwt
   # universal
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pip
   - pip:

ci/deps/travis-36-slow.yaml

@@ -25,8 +25,8 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest>=4.0.2,<5.0.0
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - moto
   - hypothesis>=3.58.0

ci/deps/travis-37.yaml

@@ -13,8 +13,8 @@ dependencies:
   - pyarrow
   - pytz
   # universal
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - hypothesis>=3.58.0
   - s3fs