Merge remote-tracking branch 'upstream/master' into docfix-multiindex-set_levels

Author: hweecat

.github/workflows/ci.yml

@@ -23,53 +23,53 @@ jobs:
 
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
-      if: true
+      if: always()
 
     - name: Setup environment and build pandas
       run: ci/setup_env.sh
-      if: true
+      if: always()
 
     - name: Linting
       run: |
         source activate pandas-dev
         ci/code_checks.sh lint
-      if: true
+      if: always()
 
     - name: Dependencies consistency
       run: |
         source activate pandas-dev
         ci/code_checks.sh dependencies
-      if: true
+      if: always()
 
     - name: Checks on imported code
       run: |
         source activate pandas-dev
         ci/code_checks.sh code
-      if: true
+      if: always()
 
     - name: Running doctests
       run: |
         source activate pandas-dev
         ci/code_checks.sh doctests
-      if: true
+      if: always()
 
     - name: Docstring validation
       run: |
         source activate pandas-dev
         ci/code_checks.sh docstrings
-      if: true
+      if: always()
 
     - name: Typing validation
       run: |
         source activate pandas-dev
         ci/code_checks.sh typing
-      if: true
+      if: always()
 
     - name: Testing docstring validation script
       run: |
         source activate pandas-dev
         pytest --capture=no --strict scripts
-      if: true
+      if: always()
 
     - name: Running benchmarks
       run: |
@@ -87,11 +87,73 @@ jobs:
         else
             echo "Benchmarks did not run, no changes detected"
         fi
-      if: true
+      if: always()
 
     - name: Publish benchmarks artifact
       uses: actions/upload-artifact@master
       with:
         name: Benchmarks log
         path: asv_bench/benchmarks.log
       if: failure()
+
+  web_and_docs:
+    name: Web and docs
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Setting conda path
+      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+
+    - name: Build website
+      run: |
+        source activate pandas-dev
+        python web/pandas_web.py web/pandas --target-path=web/build
+
+    - name: Build documentation
+      run: |
+        source activate pandas-dev
+        doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]}
+
+    # This can be removed when the ipython directive fails when there are errors,
+    # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)
+    - name: Check ipython directive errors
+      run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
+
+    - name: Merge website and docs
+      run: |
+        mkdir -p pandas_web/docs
+        cp -r web/build/* pandas_web/
+        cp -r doc/build/html/* pandas_web/docs/
+      if: github.event_name == 'push'
+
+    - name: Install Rclone
+      run: sudo apt install rclone -y
+      if: github.event_name == 'push'
+
+    - name: Set up Rclone
+      run: |
+        RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf
+        mkdir -p `dirname $RCLONE_CONFIG_PATH`
+        echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH
+        echo "type = swift" >> $RCLONE_CONFIG_PATH
+        echo "env_auth = false" >> $RCLONE_CONFIG_PATH
+        echo "auth_version = 3" >> $RCLONE_CONFIG_PATH
+        echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH
+        echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH
+        echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH
+        echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH
+        echo "domain = default" >> $RCLONE_CONFIG_PATH
+        echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH
+        echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH
+        echo "region = BHS" >> $RCLONE_CONFIG_PATH
+      if: github.event_name == 'push'
+
+    - name: Sync web
+      run: rclone sync pandas_web ovh_cloud_pandas_web:dev
+      if: github.event_name == 'push'

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 3.5
+python: 3.7
 
 # To turn off cached cython files and compiler cache
 # set NOCACHE-true
@@ -48,17 +48,12 @@ matrix:
         - mysql
         - postgresql
 
-    # In allow_failures
     - env:
         - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
       services:
         - mysql
         - postgresql
 
-    allow_failures:
-      - env:
-          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
-
 before_install:
   - echo "before_install"
   # set non-blocking IO on travis

ci/azure/posix.yml

@@ -19,18 +19,24 @@ jobs:
           ENV_FILE: ci/deps/azure-36-minimum_versions.yaml
           CONDA_PY: "36"
           PATTERN: "not slow and not network"
+
         py36_locale_slow_old_np:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
           PATTERN: "slow"
-          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          # pandas does not use the language (zh_CN), but should support diferent encodings (utf8)
+          # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
+          LANG: "zh_CN.utf8"
+          LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans"
 
         py36_locale:
           ENV_FILE: ci/deps/azure-36-locale.yaml
           CONDA_PY: "36"
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "it_IT.UTF-8"
+          LANG: "it_IT.utf8"
+          LC_ALL: "it_IT.utf8"
+          EXTRA_APT: "language-pack-it"
 
         py36_32bit:
           ENV_FILE: ci/deps/azure-36-32bit.yaml
@@ -42,7 +48,9 @@ jobs:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          LANG: "zh_CN.utf8"
+          LC_ALL: "zh_CN.utf8"
+          EXTRA_APT: "language-pack-zh-hans"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml
@@ -54,10 +62,16 @@ jobs:
 
   steps:
     - script: |
-        if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386 $EXTRA_APT; fi
-        echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
-        echo "Creating Environment"
-        ci/setup_env.sh
+        if [ "$(uname)" == "Linux" ]; then
+          sudo apt-get update
+          sudo apt-get install -y libc6-dev-i386 $EXTRA_APT
+        fi
+      displayName: 'Install extra packages'
+
+    - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
+      displayName: 'Set conda path'
+
+    - script: ci/setup_env.sh
       displayName: 'Setup environment and build pandas'
 
     - script: |

ci/azure/windows.yml

@@ -34,7 +34,7 @@ jobs:
     - bash: |
         source activate pandas-dev
         conda list
-        python setup.py build_ext -q -i
+        python setup.py build_ext -q -i -j 4
         python -m pip install --no-build-isolation -e .
       displayName: 'Build'
 

ci/deps/azure-36-locale_slow.yaml

@@ -13,7 +13,7 @@ dependencies:
   - pytest-azurepipelines
 
   # pandas dependencies
-  - beautifulsoup4==4.6.0
+  - beautifulsoup4=4.6.0
   - bottleneck=1.2.*
   - lxml
   - matplotlib=2.2.2

ci/run_tests.sh

@@ -5,17 +5,6 @@
 # https://github.com/pytest-dev/pytest/issues/1075
 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
 
-if [ -n "$LOCALE_OVERRIDE" ]; then
-    export LC_ALL="$LOCALE_OVERRIDE"
-    export LANG="$LOCALE_OVERRIDE"
-    PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
-    if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then
-        echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
-        # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
-        # exit 1
-    fi
-fi
-
 if [[ "not network" == *"$PATTERN"* ]]; then
     export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
 fi

ci/setup_env.sh

@@ -1,15 +1,15 @@
 #!/bin/bash -e
 
 # edit the locale file if needed
-if [ -n "$LOCALE_OVERRIDE" ]; then
+if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo "Adding locale to the first line of pandas/__init__.py"
     rm -f pandas/__init__.pyc
-    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n"
+    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n"
     sed -i "$SEDC" pandas/__init__.py
+
     echo "[head -4 pandas/__init__.py]"
     head -4 pandas/__init__.py
     echo
-    sudo locale-gen "$LOCALE_OVERRIDE"
 fi
 
 MINICONDA_DIR="$HOME/miniconda3"

doc/source/getting_started/10min.rst

@@ -697,8 +697,9 @@ Plotting
 
 See the :ref:`Plotting <visualization>` docs.
 
+We use the standard convention for referencing the matplotlib API:
+
 .. ipython:: python
-   :suppress:
 
    import matplotlib.pyplot as plt
    plt.close('all')

doc/source/user_guide/integer_na.rst

@@ -15,6 +15,10 @@ Nullable integer data type
    IntegerArray is currently experimental. Its API or implementation may
    change without warning.
 
+.. versionchanged:: 1.0.0
+
+   Now uses :attr:`pandas.NA` as the missing value rather
+   than :attr:`numpy.nan`.
 
 In :ref:`missing_data`, we saw that pandas primarily uses ``NaN`` to represent
 missing data. Because ``NaN`` is a float, this forces an array of integers with
@@ -23,6 +27,9 @@ much. But if your integer column is, say, an identifier, casting to float can
 be problematic. Some integers cannot even be represented as floating point
 numbers.
 
+Construction
+------------
+
 Pandas can represent integer data with possibly missing values using
 :class:`arrays.IntegerArray`. This is an :ref:`extension types <extending.extension-types>`
 implemented within pandas.
@@ -39,6 +46,12 @@ NumPy's ``'int64'`` dtype:
 
    pd.array([1, 2, np.nan], dtype="Int64")
 
+All NA-like values are replaced with :attr:`pandas.NA`.
+
+.. ipython:: python
+
+   pd.array([1, 2, np.nan, None, pd.NA], dtype="Int64")
+
 This array can be stored in a :class:`DataFrame` or :class:`Series` like any
 NumPy array.
 
@@ -78,6 +91,9 @@ with the dtype.
    In the future, we may provide an option for :class:`Series` to infer a
    nullable-integer dtype.
 
+Operations
+----------
+
 Operations involving an integer array will behave similar to NumPy arrays.
 Missing values will be propagated, and the data will be coerced to another
 dtype if needed.
@@ -123,3 +139,15 @@ Reduction and groupby operations such as 'sum' work as well.
 
    df.sum()
    df.groupby('B').A.sum()
+
+Scalar NA Value
+---------------
+
+:class:`arrays.IntegerArray` uses :attr:`pandas.NA` as its scalar
+missing value. Slicing a single element that's missing will return
+:attr:`pandas.NA`
+
+.. ipython:: python
+
+   a = pd.array([1, None], dtype="Int64")
+   a[1]