Merging from master

Author: datapythonista

.gitignore

@@ -101,6 +101,7 @@ asv_bench/pandas/
 # Documentation generated files #
 #################################
 doc/source/generated
+doc/source/api/generated
 doc/source/_static
 doc/source/vbench
 doc/source/vbench.rst

.travis.yml

@@ -34,23 +34,23 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network and not db"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
 
     - dist: trusty
       env:
-        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="not slow"
+        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="(not slow or (single and db))"
       addons:
         apt:
           packages:
           - python-gtk2
 
     - dist: trusty
       env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="not slow and not network" LOCALE_OVERRIDE="zh_CN.UTF-8"
+        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8"
 
     - dist: trusty
       env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
 
     # In allow_failures
     - dist: trusty

LICENSES/DATEUTIL_LICENSE

@@ -0,0 +1,54 @@
+Copyright 2017- Paul Ganssle <[email protected]>
+Copyright 2017- dateutil contributors (see AUTHORS file)
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+The above license applies to all contributions after 2017-12-01, as well as
+all contributions that have been re-licensed (see AUTHORS file for the list of
+contributors who have re-licensed their code).
+--------------------------------------------------------------------------------
+dateutil - Extensions to the standard Python datetime module.
+
+Copyright (c) 2003-2011 - Gustavo Niemeyer <[email protected]>
+Copyright (c) 2012-2014 - Tomi Pieviläinen <[email protected]>
+Copyright (c) 2014-2016 - Yaron de Leeuw <[email protected]>
+Copyright (c) 2015-     - Paul Ganssle <[email protected]>
+Copyright (c) 2015-     - dateutil contributors (see AUTHORS file)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the copyright holder nor the names of its
+      contributors may be used to endorse or promote products derived from
+      this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The above BSD License Applies to all code, even that also covered by Apache 2.0.

asv_bench/benchmarks/ctors.py

@@ -3,26 +3,68 @@
 from pandas import Series, Index, DatetimeIndex, Timestamp, MultiIndex
 
 
+def no_change(arr):
+    return arr
+
+
+def list_of_str(arr):
+    return list(arr.astype(str))
+
+
+def gen_of_str(arr):
+    return (x for x in arr.astype(str))
+
+
+def arr_dict(arr):
+    return dict(zip(range(len(arr)), arr))
+
+
+def list_of_tuples(arr):
+    return [(i, -i) for i in arr]
+
+
+def gen_of_tuples(arr):
+    return ((i, -i) for i in arr)
+
+
+def list_of_lists(arr):
+    return [[i, -i] for i in arr]
+
+
+def list_of_tuples_with_none(arr):
+    return [(i, -i) for i in arr][:-1] + [None]
+
+
+def list_of_lists_with_none(arr):
+    return [[i, -i] for i in arr][:-1] + [None]
+
+
 class SeriesConstructors(object):
 
-    param_names = ["data_fmt", "with_index"]
-    params = [[lambda x: x,
+    param_names = ["data_fmt", "with_index", "dtype"]
+    params = [[no_change,
                list,
-               lambda arr: list(arr.astype(str)),
-               lambda arr: dict(zip(range(len(arr)), arr)),
-               lambda arr: [(i, -i) for i in arr],
-               lambda arr: [[i, -i] for i in arr],
-               lambda arr: ([(i, -i) for i in arr][:-1] + [None]),
-               lambda arr: ([[i, -i] for i in arr][:-1] + [None])],
-              [False, True]]
-
-    def setup(self, data_fmt, with_index):
+               list_of_str,
+               gen_of_str,
+               arr_dict,
+               list_of_tuples,
+               gen_of_tuples,
+               list_of_lists,
+               list_of_tuples_with_none,
+               list_of_lists_with_none],
+              [False, True],
+              ['float', 'int']]
+
+    def setup(self, data_fmt, with_index, dtype):
         N = 10**4
-        arr = np.random.randn(N)
+        if dtype == 'float':
+            arr = np.random.randn(N)
+        else:
+            arr = np.arange(N)
         self.data = data_fmt(arr)
         self.index = np.arange(N) if with_index else None
 
-    def time_series_constructor(self, data_fmt, with_index):
+    def time_series_constructor(self, data_fmt, with_index, dtype):
         Series(self.data, index=self.index)
 
 

asv_bench/benchmarks/dtypes.py

@@ -0,0 +1,39 @@
+from pandas.api.types import pandas_dtype
+
+import numpy as np
+from .pandas_vb_common import (
+    numeric_dtypes, datetime_dtypes, string_dtypes, extension_dtypes)
+
+
+_numpy_dtypes = [np.dtype(dtype)
+                 for dtype in (numeric_dtypes +
+                               datetime_dtypes +
+                               string_dtypes)]
+_dtypes = _numpy_dtypes + extension_dtypes
+
+
+class Dtypes(object):
+    params = (_dtypes +
+              list(map(lambda dt: dt.name, _dtypes)))
+    param_names = ['dtype']
+
+    def time_pandas_dtype(self, dtype):
+        pandas_dtype(dtype)
+
+
+class DtypesInvalid(object):
+    param_names = ['dtype']
+    params = ['scalar-string', 'scalar-int', 'list-string', 'array-string']
+    data_dict = {'scalar-string': 'foo',
+                 'scalar-int': 1,
+                 'list-string': ['foo'] * 1000,
+                 'array-string': np.array(['foo'] * 1000)}
+
+    def time_pandas_dtype_invalid(self, dtype):
+        try:
+            pandas_dtype(self.data_dict[dtype])
+        except TypeError:
+            pass
+
+
+from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/pandas_vb_common.py

@@ -2,6 +2,7 @@
 from importlib import import_module
 
 import numpy as np
+import pandas as pd
 
 # Compatibility import for lib
 for imp in ['pandas._libs.lib', 'pandas.lib']:
@@ -14,6 +15,15 @@
 numeric_dtypes = [np.int64, np.int32, np.uint32, np.uint64, np.float32,
                   np.float64, np.int16, np.int8, np.uint16, np.uint8]
 datetime_dtypes = [np.datetime64, np.timedelta64]
+string_dtypes = [np.object]
+extension_dtypes = [pd.Int8Dtype, pd.Int16Dtype,
+                    pd.Int32Dtype, pd.Int64Dtype,
+                    pd.UInt8Dtype, pd.UInt16Dtype,
+                    pd.UInt32Dtype, pd.UInt64Dtype,
+                    pd.CategoricalDtype,
+                    pd.IntervalDtype,
+                    pd.DatetimeTZDtype('ns', 'UTC'),
+                    pd.PeriodDtype('D')]
 
 
 def setup(*args, **kwargs):

asv_bench/benchmarks/stat_ops.py

@@ -106,6 +106,7 @@ def setup(self, method, use_bottleneck):
             from pandas.core import nanops
             nanops._USE_BOTTLENECK = use_bottleneck
         self.df = pd.DataFrame(np.random.randn(1000, 30))
+        self.df2 = pd.DataFrame(np.random.randn(1000, 30))
         self.s = pd.Series(np.random.randn(1000))
         self.s2 = pd.Series(np.random.randn(1000))
 
@@ -115,6 +116,12 @@ def time_corr(self, method, use_bottleneck):
     def time_corr_series(self, method, use_bottleneck):
         self.s.corr(self.s2, method=method)
 
+    def time_corrwith_cols(self, method, use_bottleneck):
+        self.df.corrwith(self.df2, method=method)
+
+    def time_corrwith_rows(self, method, use_bottleneck):
+        self.df.corrwith(self.df2, axis=1, method=method)
+
 
 class Covariance(object):
 

asv_bench/benchmarks/timeseries.py

@@ -12,7 +12,7 @@
 
 class DatetimeIndex(object):
 
-    params = ['dst', 'repeated', 'tz_aware', 'tz_naive']
+    params = ['dst', 'repeated', 'tz_aware', 'tz_local', 'tz_naive']
     param_names = ['index_type']
 
     def setup(self, index_type):
@@ -26,6 +26,10 @@ def setup(self, index_type):
                                           periods=N,
                                           freq='s',
                                           tz='US/Eastern'),
+                   'tz_local': date_range(start='2000',
+                                          periods=N,
+                                          freq='s',
+                                          tz=dateutil.tz.tzlocal()),
                    'tz_naive': date_range(start='2000',
                                           periods=N,
                                           freq='s')}

ci/azure/posix.yml

@@ -12,37 +12,37 @@ jobs:
         py35_np_120:
           ENV_FILE: ci/deps/azure-macos-35.yaml
           CONDA_PY: "35"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
 
       ${{ if eq(parameters.name, 'Linux') }}:
         py27_np_120:
           ENV_FILE: ci/deps/azure-27-compat.yaml
           CONDA_PY: "27"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
 
         py27_locale_slow_old_np:
           ENV_FILE: ci/deps/azure-27-locale.yaml
           CONDA_PY: "27"
-          PATTERN: "slow and not db"
+          PATTERN: "slow"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
           EXTRA_APT: "language-pack-zh-hans"
 
         py36_locale_slow:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "it_IT.UTF-8"
 
         py37_locale:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml
           CONDA_PY: "37"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
           TEST_ARGS: "-W error"
           PANDAS_TESTING_MODE: "deprecate"
           EXTRA_APT: "xsel"

ci/azure/windows.yml

@@ -38,7 +38,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest -m "not slow and not network and not db" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
+        pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

ci/code_checks.sh

@@ -148,6 +148,11 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --exclude=*.pyc --exclude=testing.py --exclude=test_util.py assert_raises_regex pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    # Check for the following code in testing: `unittest.mock`, `mock.Mock()` or `mock.patch`
+    MSG='Check that unittest.mock is not used (pytest builtin monkeypatch fixture should be used instead)' ; echo $MSG
+    invgrep -r -E --include '*.py' '(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)' pandas/tests/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     # Check that we use pytest.raises only as a context manager
     #
     # For any flake8-compliant code, the only way this regex gets
@@ -157,6 +162,14 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     # invgrep -R --include '*.py' -E '[[:space:]] pytest.raises' pandas/tests
     # RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for wrong space after code-block directive and before colon (".. code-block ::" instead of ".. code-block::")' ; echo $MSG
+    invgrep -R --include="*.rst" ".. code-block ::" doc/source
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for wrong space after ipython directive and before colon (".. ipython ::" instead of ".. ipython::")' ; echo $MSG
+    invgrep -R --include="*.rst" ".. ipython ::" doc/source
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check that no file in the repo contains tailing whitespaces' ; echo $MSG
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then

ci/deps/azure-macos-35.yaml

@@ -13,6 +13,7 @@ dependencies:
   - numexpr
   - numpy=1.12.0
   - openpyxl=2.5.5
+  - pyarrow
   - pytables
   - python=3.5*
   - pytz

ci/deps/azure-windows-36.yaml

@@ -6,7 +6,7 @@ dependencies:
   - blosc
   - bottleneck
   - boost-cpp<1.67
-  - fastparquet
+  - fastparquet>=0.2.1
   - matplotlib
   - numexpr
   - numpy=1.14*
@@ -18,7 +18,6 @@ dependencies:
   - python=3.6.6
   - pytz
   - scipy
-  - thrift=0.10*
   - xlrd
   - xlsxwriter
   - xlwt

ci/deps/travis-27.yaml

@@ -6,7 +6,7 @@ dependencies:
   - beautifulsoup4
   - bottleneck
   - cython=0.28.2
-  - fastparquet
+  - fastparquet>=0.2.1
   - gcsfs
   - html5lib
   - ipython