Merge remote-tracking branch 'upstream/master' into multi-index-join

Author: harisbal

.circleci/config.yml

@@ -1,10 +1,6 @@
 version: 2
 jobs:
-
-  # --------------------------------------------------------------------------
-  # 1. py36_locale
-  # --------------------------------------------------------------------------
-  py36_locale:
+  build:
     docker:
       - image: continuumio/miniconda:latest
       # databases configuration
@@ -21,7 +17,7 @@ jobs:
 
     environment:
       JOB: "3.6_LOCALE"
-      ENV_FILE: "ci/circle-36-locale.yaml"
+      ENV_FILE: "ci/deps/circle-36-locale.yaml"
       LOCALE_OVERRIDE: "zh_CN.UTF-8"
       MINICONDA_DIR: /home/ubuntu/miniconda3
     steps:
@@ -34,9 +30,3 @@ jobs:
       - run:
           name: test
           command: ./ci/circle/run_circle.sh  --skip-slow --skip-network
-
-workflows:
-  version: 2
-  build_and_test:
-    jobs:
-      - py36_locale

.travis.yml

@@ -34,28 +34,28 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network"
 
     - dist: trusty
       env:
-        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
+        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
       addons:
         apt:
           packages:
           - language-pack-zh-hans
     - dist: trusty
       env:
-        - JOB="2.7" ENV_FILE="ci/travis-27.yaml" TEST_ARGS="--skip-slow"
+        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" TEST_ARGS="--skip-slow"
       addons:
         apt:
           packages:
           - python-gtk2
     - dist: trusty
       env:
-        - JOB="3.6, lint, coverage" ENV_FILE="ci/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
+        - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
     - dist: trusty
       env:
-        - JOB="3.7, NumPy dev" ENV_FILE="ci/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
+        - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
@@ -64,19 +64,19 @@ matrix:
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
+        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
 
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6, doc" ENV_FILE="ci/travis-36-doc.yaml" DOC=true
+        - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
     allow_failures:
       - dist: trusty
         env:
-          - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
+          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
       - dist: trusty
         env:
-          - JOB="3.6, doc" ENV_FILE="ci/travis-36-doc.yaml" DOC=true
+          - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
 
 before_install:
   - echo "before_install"

asv_bench/benchmarks/reshape.py

@@ -49,21 +49,33 @@ def time_unstack(self):
 
 class Unstack(object):
 
-    def setup(self):
+    params = ['int', 'category']
+
+    def setup(self, dtype):
         m = 100
         n = 1000
 
         levels = np.arange(m)
         index = MultiIndex.from_product([levels] * 2)
         columns = np.arange(n)
-        values = np.arange(m * m * n).reshape(m * m, n)
+        if dtype == 'int':
+            values = np.arange(m * m * n).reshape(m * m, n)
+        else:
+            # the category branch is ~20x slower than int. So we
+            # cut down the size a bit. Now it's only ~3x slower.
+            n = 50
+            columns = columns[:n]
+            indices = np.random.randint(0, 52, size=(m * m, n))
+            values = np.take(list(string.ascii_letters), indices)
+            values = [pd.Categorical(v) for v in values.T]
+
         self.df = DataFrame(values, index, columns)
         self.df2 = self.df.iloc[:-1]
 
-    def time_full_product(self):
+    def time_full_product(self, dtype):
         self.df.unstack()
 
-    def time_without_last_row(self):
+    def time_without_last_row(self, dtype):
         self.df2.unstack()
 
 

ci/azure/linux.yml

@@ -10,20 +10,20 @@ jobs:
     maxParallel: 11
     matrix:
       py27_np_19:
-        ENV_FILE: ci/azure-27-compat.yaml
+        ENV_FILE: ci/deps/azure-27-compat.yaml
         CONDA_PY: "27"
         CONDA_ENV: pandas
         TEST_ARGS: "--skip-slow --skip-network"
 
       py36_locale:
-        ENV_FILE: ci/azure-37-locale.yaml
+        ENV_FILE: ci/deps/azure-37-locale.yaml
         CONDA_PY: "37"
         CONDA_ENV: pandas
         TEST_ARGS: "--skip-slow --skip-network"
         LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
       py36_locale_slow:
-        ENV_FILE: ci/azure-36-locale_slow.yaml
+        ENV_FILE: ci/deps/azure-36-locale_slow.yaml
         CONDA_PY: "36"
         CONDA_ENV: pandas
         TEST_ARGS: "--only-slow --skip-network"
@@ -53,4 +53,29 @@ jobs:
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: 'test-data-*.xml'
-        testRunTitle: 'Linux'
+        testRunTitle: 'Linux'
+    - powershell: |
+        $junitXml = "test-data-single.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-single"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+
+        $junitXmlMulti = "test-data-multiple.xml"
+        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-multi"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/azure/macos.yml

@@ -10,7 +10,7 @@ jobs:
     maxParallel: 11
     matrix:
       py35_np_120:
-        ENV_FILE: ci/azure-macos-35.yaml
+        ENV_FILE: ci/deps/azure-macos-35.yaml
         CONDA_PY: "35"
         CONDA_ENV: pandas
         TEST_ARGS: "--skip-slow --skip-network"
@@ -41,3 +41,28 @@ jobs:
       inputs:
         testResultsFiles: 'test-data-*.xml'
         testRunTitle: 'MacOS-35'
+    - powershell: |
+        $junitXml = "test-data-single.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-single"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+
+        $junitXmlMulti = "test-data-multiple.xml"
+        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-multi"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/azure/windows-py27.yml

@@ -10,7 +10,7 @@ jobs:
     maxParallel: 11
     matrix:
       py36_np121:
-        ENV_FILE: ci/azure-windows-27.yaml
+        ENV_FILE: ci/deps/azure-windows-27.yaml
         CONDA_PY: "27"
         CONDA_ENV: pandas
 
@@ -43,3 +43,16 @@ jobs:
       inputs:
         testResultsFiles: 'test-data.xml'
         testRunTitle: 'Windows 27'
+    - powershell: |
+        $junitXml = "test-data.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/azure/windows.yml

@@ -10,7 +10,7 @@ jobs:
     maxParallel: 11
     matrix:
       py36_np14:
-        ENV_FILE: ci/azure-windows-36.yaml
+        ENV_FILE: ci/deps/azure-windows-36.yaml
         CONDA_PY: "36"
         CONDA_ENV: pandas
 
@@ -34,3 +34,16 @@ jobs:
       inputs:
         testResultsFiles: 'test-data.xml'
         testRunTitle: 'Windows 36'
+    - powershell: |
+        $junitXml = "test-data.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures

ci/code_checks.sh

@@ -44,6 +44,13 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     flake8 pandas/_libs --filename=*.pxi.in,*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    echo "flake8-rst --version"
+    flake8-rst --version
+
+    MSG='Linting code-blocks in .rst documentation' ; echo $MSG
+    flake8-rst doc/source --filename=*.rst
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     # Check that cython casting is of the form `<type>obj` as opposed to `<type> obj`;
     # it doesn't make a difference, but we want to be internally consistent.
     # Note: this grep pattern is (intended to be) equivalent to the python
@@ -64,6 +71,9 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    echo "isort --version-number"
+    isort --version-number
+
     # Imports - Check formatting using isort see setup.cfg for settings
     MSG='Check import format using isort ' ; echo $MSG
     isort --recursive --check-only pandas

ci/azure-27-compat.yaml renamed to ci/deps/azure-27-compat.yaml

@@ -12,7 +12,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy=1.12.0
-  - openpyxl
+  - openpyxl=2.5.5
   - pytables
   - python=3.5*
   - pytz

ci/azure-36-locale_slow.yaml renamed to ci/deps/azure-36-locale_slow.yaml

@@ -9,6 +9,7 @@ dependencies:
   - fastparquet
   - flake8>=3.5
   - flake8-comprehensions
+  - flake8-rst
   - gcsfs
   - geopandas
   - html5lib

ci/azure-37-locale.yaml renamed to ci/deps/azure-37-locale.yaml

@@ -7,6 +7,7 @@ dependencies:
   - NumPy
   - flake8
   - flake8-comprehensions
+  - flake8-rst
   - hypothesis>=3.58.0
   - isort
   - moto

ci/azure-macos-35.yaml renamed to ci/deps/azure-macos-35.yaml

@@ -13,7 +13,7 @@ conda list
 @rem Clean up any left-over from a previous build
 conda remove --all -q -y -n %CONDA_ENV%
 @rem Scipy, CFFI, jinja2 and IPython are optional dependencies, but exercised in the test suite
-conda env create -n %CONDA_ENV% --file=ci\azure-windows-%CONDA_PY%.yaml
+conda env create -n %CONDA_ENV% --file=ci\deps\azure-windows-%CONDA_PY%.yaml
 
 call activate %CONDA_ENV%
 conda list

ci/azure-windows-27.yaml renamed to ci/deps/azure-windows-27.yaml

@@ -4,6 +4,7 @@ Cython>=0.28.2
 NumPy
 flake8
 flake8-comprehensions
+flake8-rst
 hypothesis>=3.58.0
 isort
 moto