Merge remote-tracking branch 'upstream/master' into CI-TestCase-#29886

Author: MomIsBestFriend

.travis.yml

@@ -30,31 +30,34 @@ matrix:
       - python: 3.5
 
     include:
-    - dist: trusty
-      env:
+    - env:
         - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)"
 
-    - dist: trusty
-      env:
+    - env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
 
-    - dist: trusty
-      env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8"
+    - env:
+        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
+      services:
+        - mysql
+        - postgresql
 
-    - dist: trusty
-      env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
+    - env:
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
+      services:
+        - mysql
+        - postgresql
 
     # In allow_failures
-    - dist: trusty
-      env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
+    - env:
+        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
+      services:
+        - mysql
+        - postgresql
 
     allow_failures:
-      - dist: trusty
-        env:
-          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
+      - env:
+          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
 
 before_install:
   - echo "before_install"

LICENSES/MSGPACK_LICENSE

@@ -20,7 +20,6 @@ global-exclude *.gz
 global-exclude *.h5
 global-exclude *.html
 global-exclude *.json
-global-exclude *.msgpack
 global-exclude *.pickle
 global-exclude *.png
 global-exclude *.pyc

LICENSES/MSGPACK_NUMPY_LICENSE

@@ -124,7 +124,7 @@ Here are just a few of the things that pandas does well:
     and saving/loading data from the ultrafast [**HDF5 format**][hdfstore]
   - [**Time series**][timeseries]-specific functionality: date range
     generation and frequency conversion, moving window statistics,
-    moving window linear regressions, date shifting and lagging, etc.
+    date shifting and lagging.
 
 
    [missing-data]: https://pandas.pydata.org/pandas-docs/stable/missing_data.html#working-with-missing-data

MANIFEST.in

@@ -0,0 +1,23 @@
+import numpy as np
+
+import pandas as pd
+
+
+class BooleanArray:
+    def setup(self):
+        self.values_bool = np.array([True, False, True, False])
+        self.values_float = np.array([1.0, 0.0, 1.0, 0.0])
+        self.values_integer = np.array([1, 0, 1, 0])
+        self.values_integer_like = [1, 0, 1, 0]
+
+    def time_from_bool_array(self):
+        pd.array(self.values_bool, dtype="boolean")
+
+    def time_from_integer_array(self):
+        pd.array(self.values_integer, dtype="boolean")
+
+    def time_from_integer_like(self):
+        pd.array(self.values_integer_like, dtype="boolean")
+
+    def time_from_float_array(self):
+        pd.array(self.values_float, dtype="boolean")

README.md

@@ -0,0 +1,32 @@
+import numpy as np
+
+import pandas as pd
+
+
+class TimeLogicalOps:
+    def setup(self):
+        N = 10_000
+        left, right, lmask, rmask = np.random.randint(0, 2, size=(4, N)).astype("bool")
+        self.left = pd.arrays.BooleanArray(left, lmask)
+        self.right = pd.arrays.BooleanArray(right, rmask)
+
+    def time_or_scalar(self):
+        self.left | True
+        self.left | False
+
+    def time_or_array(self):
+        self.left | self.right
+
+    def time_and_scalar(self):
+        self.left & True
+        self.left & False
+
+    def time_and_array(self):
+        self.left & self.right
+
+    def time_xor_scalar(self):
+        self.left ^ True
+        self.left ^ False
+
+    def time_xor_array(self):
+        self.left ^ self.right

asv_bench/benchmarks/array.py

@@ -105,4 +105,16 @@ def time_frame_from_lists(self):
         self.df = DataFrame(self.data)
 
 
+class FromRange:
+
+    goal_time = 0.2
+
+    def setup(self):
+        N = 1_000_000
+        self.data = range(N)
+
+    def time_frame_from_range(self):
+        self.df = DataFrame(self.data)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/boolean.py

@@ -321,10 +321,9 @@ class Dropna:
 
     def setup(self, how, axis):
         self.df = DataFrame(np.random.randn(10000, 1000))
-        with warnings.catch_warnings(record=True):
-            self.df.ix[50:1000, 20:50] = np.nan
-            self.df.ix[2000:3000] = np.nan
-            self.df.ix[:, 60:70] = np.nan
+        self.df.iloc[50:1000, 20:50] = np.nan
+        self.df.iloc[2000:3000] = np.nan
+        self.df.iloc[:, 60:70] = np.nan
         self.df_mixed = self.df.copy()
         self.df_mixed["foo"] = "bar"
 
@@ -342,10 +341,9 @@ class Count:
 
     def setup(self, axis):
         self.df = DataFrame(np.random.randn(10000, 1000))
-        with warnings.catch_warnings(record=True):
-            self.df.ix[50:1000, 20:50] = np.nan
-            self.df.ix[2000:3000] = np.nan
-            self.df.ix[:, 60:70] = np.nan
+        self.df.iloc[50:1000, 20:50] = np.nan
+        self.df.iloc[2000:3000] = np.nan
+        self.df.iloc[:, 60:70] = np.nan
         self.df_mixed = self.df.copy()
         self.df_mixed["foo"] = "bar"
 

asv_bench/benchmarks/frame_ctor.py

@@ -67,22 +67,6 @@ def time_iloc_scalar(self, index, index_structure):
     def time_iloc_slice(self, index, index_structure):
         self.data.iloc[:800000]
 
-    def time_ix_array(self, index, index_structure):
-        with warnings.catch_warnings(record=True):
-            self.data.ix[self.array]
-
-    def time_ix_list_like(self, index, index_structure):
-        with warnings.catch_warnings(record=True):
-            self.data.ix[[800000]]
-
-    def time_ix_scalar(self, index, index_structure):
-        with warnings.catch_warnings(record=True):
-            self.data.ix[800000]
-
-    def time_ix_slice(self, index, index_structure):
-        with warnings.catch_warnings(record=True):
-            self.data.ix[:800000]
-
     def time_loc_array(self, index, index_structure):
         self.data.loc[self.array]
 
@@ -148,10 +132,6 @@ def setup(self):
         self.bool_indexer = self.df[self.col_scalar] > 0
         self.bool_obj_indexer = self.bool_indexer.astype(object)
 
-    def time_ix(self):
-        with warnings.catch_warnings(record=True):
-            self.df.ix[self.idx_scalar, self.col_scalar]
-
     def time_loc(self):
         self.df.loc[self.idx_scalar, self.col_scalar]
 
@@ -228,14 +208,6 @@ def setup(self):
         self.idx = IndexSlice[20000:30000, 20:30, 35:45, 30000:40000]
         self.mdt = self.mdt.set_index(["A", "B", "C", "D"]).sort_index()
 
-    def time_series_ix(self):
-        with warnings.catch_warnings(record=True):
-            self.s.ix[999]
-
-    def time_frame_ix(self):
-        with warnings.catch_warnings(record=True):
-            self.df.ix[999]
-
     def time_index_slice(self):
         self.mdt.loc[self.idx, :]
 
@@ -310,10 +282,6 @@ def setup_cache(self):
     def time_lookup_iloc(self, s):
         s.iloc
 
-    def time_lookup_ix(self, s):
-        with warnings.catch_warnings(record=True):
-            s.ix
-
     def time_lookup_loc(self, s):
         s.loc
 

asv_bench/benchmarks/frame_methods.py

@@ -26,5 +26,5 @@ def setup(self, format):
         ]
         self.f = os.path.join(*paths)
 
-    def time_read_msgpack(self, format):
+    def time_read_sas(self, format):
         read_sas(self.f, format=format)

asv_bench/benchmarks/indexing.py

@@ -44,15 +44,13 @@ jobs:
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
-        # Disabled for NumPy object-dtype warning.
-        # https://github.com/pandas-dev/pandas/issues/30043
-        # py37_np_dev:
-        #   ENV_FILE: ci/deps/azure-37-numpydev.yaml
-        #   CONDA_PY: "37"
-        #   PATTERN: "not slow and not network"
-        #   TEST_ARGS: "-W error"
-        #   PANDAS_TESTING_MODE: "deprecate"
-        #   EXTRA_APT: "xsel"
+        py37_np_dev:
+          ENV_FILE: ci/deps/azure-37-numpydev.yaml
+          CONDA_PY: "37"
+          PATTERN: "not slow and not network"
+          TEST_ARGS: "-W error"
+          PANDAS_TESTING_MODE: "deprecate"
+          EXTRA_APT: "xsel"
 
   steps:
     - script: |

asv_bench/benchmarks/io/msgpack.py

@@ -39,7 +39,7 @@ function invgrep {
 }
 
 if [[ "$GITHUB_ACTIONS" == "true" ]]; then
-    FLAKE8_FORMAT="##[error]%(path)s:%(row)s:%(col)s:%(code):%(text)s"
+    FLAKE8_FORMAT="##[error]%(path)s:%(row)s:%(col)s:%(code)s:%(text)s"
     INVGREP_PREPEND="##[error]"
 else
     FLAKE8_FORMAT="default"
@@ -94,18 +94,23 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
     # We don't lint all C files because we don't want to lint any that are built
     # from Cython files nor do we want to lint C files that we didn't modify for
-    # this particular codebase (e.g. src/headers, src/klib, src/msgpack). However,
+    # this particular codebase (e.g. src/headers, src/klib). However,
     # we can lint all header files since they aren't "generated" like C files are.
     MSG='Linting .c and .h' ; echo $MSG
-    cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime pandas/io/msgpack pandas/_libs/*.cpp pandas/util
+    cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime pandas/_libs/*.cpp
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     echo "isort --version-number"
     isort --version-number
 
     # Imports - Check formatting using isort see setup.cfg for settings
     MSG='Check import format using isort ' ; echo $MSG
-    isort --recursive --check-only pandas asv_bench
+    ISORT_CMD="isort --recursive --check-only pandas asv_bench"
+    if [[ "$GITHUB_ACTIONS" == "true" ]]; then
+        eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]}))
+    else
+        eval $ISORT_CMD
+    fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi
@@ -284,6 +289,10 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     pytest -q --doctest-modules pandas/core/arrays/string_.py
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Doctests arrays/boolean.py' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/arrays/boolean.py
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 ### DOCSTRINGS ###

asv_bench/benchmarks/io/sas.py

@@ -18,7 +18,7 @@ dependencies:
   - lxml
   - matplotlib=2.2.2
   - numpy=1.14.*
-  - openpyxl=2.4.8
+  - openpyxl=2.5.7
   - python-dateutil
   - python-blosc
   - pytz=2017.2