Merge remote-tracking branch 'upstream/master' into ea-repr

Author: TomAugspurger

.circleci/config.yml

@@ -34,5 +34,5 @@ jobs:
           command: |
             export PATH="$MINICONDA_DIR/bin:$PATH"
             source activate pandas-dev
-            echo "pytest --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas"
-            pytest       --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas
+            echo "pytest -m "not slow and not network" --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml pandas"
+            pytest       -m "not slow and not network" --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml pandas

.travis.yml

@@ -34,28 +34,28 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network"
 
     - dist: trusty
       env:
-        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
+        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" PATTERN="slow"
       addons:
         apt:
           packages:
           - language-pack-zh-hans
     - dist: trusty
       env:
-        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" TEST_ARGS="--skip-slow"
+        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="not slow"
       addons:
         apt:
           packages:
           - python-gtk2
     - dist: trusty
       env:
-        - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
+        - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
     - dist: trusty
       env:
-        - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
+        - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" PATTERN="not slow and not network" TEST_ARGS="-W error" PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
@@ -64,7 +64,7 @@ matrix:
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
+        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
 
     # In allow_failures
     - dist: trusty
@@ -73,7 +73,7 @@ matrix:
     allow_failures:
       - dist: trusty
         env:
-          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
+          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
       - dist: trusty
         env:
           - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
@@ -107,20 +107,16 @@ script:
   - echo "script start"
   - source activate pandas-dev
   - ci/run_build_docs.sh
-  - ci/script_single.sh
-  - ci/script_multi.sh
+  - ci/run_tests.sh
   - ci/code_checks.sh
 
-after_success:
-  -  ci/upload_coverage.sh
-
 after_script:
   - echo "after_script start"
   - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
   - if [ -e test-data-single.xml ]; then
-    ci/print_skipped.py test-data-single.xml;
+        ci/print_skipped.py test-data-single.xml;
     fi
   - if [ -e test-data-multiple.xml ]; then
-    ci/print_skipped.py test-data-multiple.xml;
+        ci/print_skipped.py test-data-multiple.xml;
     fi
   - echo "after_script done"

asv_bench/benchmarks/categoricals.py

@@ -46,6 +46,8 @@ def setup(self):
         self.values_some_nan = list(np.tile(self.categories + [np.nan], N))
         self.values_all_nan = [np.nan] * len(self.values)
         self.values_all_int8 = np.ones(N, 'int8')
+        self.categorical = pd.Categorical(self.values, self.categories)
+        self.series = pd.Series(self.categorical)
 
     def time_regular(self):
         pd.Categorical(self.values, self.categories)
@@ -68,6 +70,12 @@ def time_all_nan(self):
     def time_from_codes_all_int8(self):
         pd.Categorical.from_codes(self.values_all_int8, self.categories)
 
+    def time_existing_categorical(self):
+        pd.Categorical(self.categorical)
+
+    def time_existing_series(self):
+        pd.Categorical(self.series)
+
 
 class ValueCounts(object):
 

ci/README.txt

@@ -12,18 +12,18 @@ jobs:
       py27_np_120:
         ENV_FILE: ci/deps/azure-27-compat.yaml
         CONDA_PY: "27"
-        TEST_ARGS: "--skip-slow --skip-network"
+        PATTERN: "not slow and not network"
 
       py37_locale:
         ENV_FILE: ci/deps/azure-37-locale.yaml
         CONDA_PY: "37"
-        TEST_ARGS: "--skip-slow --skip-network"
+        PATTERN: "not slow and not network"
         LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
       py36_locale_slow:
         ENV_FILE: ci/deps/azure-36-locale_slow.yaml
         CONDA_PY: "36"
-        TEST_ARGS: "--only-slow --skip-network"
+        PATTERN: "not slow and not network"
         LOCALE_OVERRIDE: "it_IT.UTF-8"
 
   steps:
@@ -43,9 +43,7 @@ jobs:
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
-        ci/script_single.sh
-        ci/script_multi.sh
-        echo "[Test done]"
+        ci/run_tests.sh
       displayName: 'Test'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH

ci/azure/linux.yml

@@ -12,7 +12,7 @@ jobs:
       py35_np_120:
         ENV_FILE: ci/deps/azure-macos-35.yaml
         CONDA_PY: "35"
-        TEST_ARGS: "--skip-slow --skip-network"
+        PATTERN: "not slow and not network"
 
   steps:
     - script: |
@@ -31,9 +31,7 @@ jobs:
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
-        ci/script_single.sh
-        ci/script_multi.sh
-        echo "[Test done]"
+        ci/run_tests.sh
       displayName: 'Test'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH

ci/azure/macos.yml

@@ -37,7 +37,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest --junitxml=test-data.xml --skip-slow --skip-network pandas -n 2 -r sxX --strict --durations=10 %*
+        pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

ci/azure/windows-py27.yml

@@ -28,7 +28,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest --junitxml=test-data.xml --skip-slow --skip-network pandas -n 2 -r sxX --strict --durations=10 %*
+        pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

ci/azure/windows.yml

@@ -147,7 +147,7 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Doctests frame.py' ; echo $MSG
     pytest -q --doctest-modules pandas/core/frame.py \
-        -k"-axes -combine -itertuples -join -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack"
+        -k"-axes -combine -itertuples -join -pivot_table -quantile -query -reindex -reindex_axis -round"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests series.py' ; echo $MSG

ci/code_checks.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+if [ "$DOC" ]; then
+    echo "We are not running pytest as this is a doc-build"
+    exit 0
+fi
+
+# Workaround for pytest-xdist flaky collection order
+# https://github.com/pytest-dev/pytest/issues/920
+# https://github.com/pytest-dev/pytest/issues/1075
+export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    export LC_ALL="$LOCALE_OVERRIDE"
+    export LANG="$LOCALE_OVERRIDE"
+    PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
+    if [[ "$LOCALE_OVERIDE" != "$PANDAS_LOCALE" ]]; then
+        echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
+        # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
+        # exit 1
+    fi
+fi
+if [[ "not network" == *"$PATTERN"* ]]; then
+    export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
+fi
+
+
+if [ -n "$PATTERN" ]; then
+    PATTERN=" and $PATTERN"
+fi
+
+for TYPE in single multiple
+do
+    if [ "$COVERAGE" ]; then
+        COVERAGE_FNAME="/tmp/coc-$TYPE.xml"
+        COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
+    fi
+
+    TYPE_PATTERN=$TYPE
+    NUM_JOBS=1
+    if [[ "$TYPE_PATTERN" == "multiple" ]]; then
+        TYPE_PATTERN="not single"
+        NUM_JOBS=2
+    fi
+
+    pytest -m "$TYPE_PATTERN$PATTERN" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas
+
+    if [[ "$COVERAGE" && $? == 0 ]]; then
+        echo "uploading coverage for $TYPE tests"
+        bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+    fi
+done