Merge branch 'master' of https://github.com/pandas-dev/pandas into 19356

� Conflicts:
�	doc/source/whatsnew/v1.2.0.rst
�	pandas/core/algorithms.py
�	pandas/tests/series/methods/test_isin.py

Author: phofl

.gitignore

@@ -12,6 +12,7 @@
 *.log
 *.swp
 *.pdb
+*.zip
 .project
 .pydevproject
 .settings

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         name: isort (cython)
         types: [cython]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.3
+    rev: v2.7.4
     hooks:
     -   id: pyupgrade
         args: [--py37-plus]

Dockerfile

@@ -43,5 +43,5 @@ RUN conda env update -n base -f "$pandas_home/environment.yml"
 
 # Build C extensions and pandas
 RUN cd "$pandas_home" \
-    && python setup.py build_ext --inplace -j 4 \
+    && python setup.py build_ext -j 4 \
     && python -m pip install -e .

Makefile

@@ -9,7 +9,7 @@ clean_pyc:
 	-find . -name '*.py[co]' -exec rm {} \;
 
 build: clean_pyc
-	python setup.py build_ext --inplace
+	python setup.py build_ext
 
 lint-diff:
 	git diff upstream/master --name-only -- "*.py" | xargs flake8

README.md

@@ -60,7 +60,7 @@ Here are just a few of the things that pandas does well:
     and saving/loading data from the ultrafast [**HDF5 format**][hdfstore]
   - [**Time series**][timeseries]-specific functionality: date range
     generation and frequency conversion, moving window statistics,
-    date shifting and lagging.
+    date shifting and lagging
 
 
    [missing-data]: https://pandas.pydata.org/pandas-docs/stable/missing_data.html#working-with-missing-data

asv_bench/benchmarks/algorithms.py

@@ -5,6 +5,7 @@
 from pandas._libs import lib
 
 import pandas as pd
+from pandas.core.algorithms import make_duplicates_of_left_unique_in_right
 
 from .pandas_vb_common import tm
 
@@ -174,4 +175,15 @@ def time_argsort(self, N):
         self.array.argsort()
 
 
+class RemoveDuplicates:
+    def setup(self):
+        N = 10 ** 5
+        na = np.arange(int(N / 2))
+        self.left = np.concatenate([na[: int(N / 4)], na[: int(N / 4)]])
+        self.right = np.concatenate([na, na])
+
+    def time_make_duplicates_of_left_unique_in_right(self):
+        make_duplicates_of_left_unique_in_right(self.left, self.right)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/categoricals.py

@@ -1,3 +1,5 @@
+import string
+import sys
 import warnings
 
 import numpy as np
@@ -67,6 +69,47 @@ def time_existing_series(self):
         pd.Categorical(self.series)
 
 
+class AsType:
+    def setup(self):
+        N = 10 ** 5
+
+        random_pick = np.random.default_rng().choice
+
+        categories = {
+            "str": list(string.ascii_letters),
+            "int": np.random.randint(2 ** 16, size=154),
+            "float": sys.maxsize * np.random.random((38,)),
+            "timestamp": [
+                pd.Timestamp(x, unit="s") for x in np.random.randint(2 ** 18, size=578)
+            ],
+        }
+
+        self.df = pd.DataFrame(
+            {col: random_pick(cats, N) for col, cats in categories.items()}
+        )
+
+        for col in ("int", "float", "timestamp"):
+            self.df[col + "_as_str"] = self.df[col].astype(str)
+
+        for col in self.df.columns:
+            self.df[col] = self.df[col].astype("category")
+
+    def astype_str(self):
+        [self.df[col].astype("str") for col in "int float timestamp".split()]
+
+    def astype_int(self):
+        [self.df[col].astype("int") for col in "int_as_str timestamp".split()]
+
+    def astype_float(self):
+        [
+            self.df[col].astype("float")
+            for col in "float_as_str int int_as_str timestamp".split()
+        ]
+
+    def astype_datetime(self):
+        self.df["float"].astype(pd.DatetimeTZDtype(tz="US/Pacific"))
+
+
 class Concat:
     def setup(self):
         N = 10 ** 5

asv_bench/benchmarks/groupby.py

@@ -486,7 +486,7 @@ def setup(self):
         tmp2 = (np.random.random(10000) * 10.0).astype(np.float32)
         tmp = np.concatenate((tmp1, tmp2))
         arr = np.repeat(tmp, 10)
-        self.df = DataFrame(dict(a=arr, b=arr))
+        self.df = DataFrame({"a": arr, "b": arr})
 
     def time_sum(self):
         self.df.groupby(["a"])["b"].sum()

asv_bench/benchmarks/join_merge.py

@@ -132,6 +132,9 @@ def time_join_dataframe_index_single_key_small(self, sort):
     def time_join_dataframe_index_shuffle_key_bigger_sort(self, sort):
         self.df_shuf.join(self.df_key2, on="key2", sort=sort)
 
+    def time_join_dataframes_cross(self, sort):
+        self.df.loc[:2000].join(self.df_key1, how="cross", sort=sort)
+
 
 class JoinIndex:
     def setup(self):
@@ -205,6 +208,9 @@ def time_merge_dataframe_integer_2key(self, sort):
     def time_merge_dataframe_integer_key(self, sort):
         merge(self.df, self.df2, on="key1", sort=sort)
 
+    def time_merge_dataframes_cross(self, sort):
+        merge(self.left.loc[:2000], self.right.loc[:2000], how="cross", sort=sort)
+
 
 class I8Merge:
 

asv_bench/benchmarks/reshape.py

@@ -103,7 +103,10 @@ def setup(self):
         nidvars = 20
         N = 5000
         self.letters = list("ABCD")
-        yrvars = [l + str(num) for l, num in product(self.letters, range(1, nyrs + 1))]
+        yrvars = [
+            letter + str(num)
+            for letter, num in product(self.letters, range(1, nyrs + 1))
+        ]
         columns = [str(i) for i in range(nidvars)] + yrvars
         self.df = DataFrame(np.random.randn(N, nidvars + len(yrvars)), columns=columns)
         self.df["id"] = self.df.index

asv_bench/benchmarks/rolling.py

@@ -225,4 +225,17 @@ def time_rolling_offset(self, method):
         getattr(self.groupby_roll_offset, method)()
 
 
+class GroupbyEWM:
+
+    params = ["cython", "numba"]
+    param_names = ["engine"]
+
+    def setup(self, engine):
+        df = pd.DataFrame({"A": range(50), "B": range(50)})
+        self.gb_ewm = df.groupby("A").ewm(com=1.0)
+
+    def time_groupby_mean(self, engine):
+        self.gb_ewm.mean(engine=engine)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/series_methods.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from pandas import NaT, Series, date_range
+from pandas import Categorical, NaT, Series, date_range
 
 from .pandas_vb_common import tm
 
@@ -36,6 +36,28 @@ def time_isin(self, dtypes):
         self.s.isin(self.values)
 
 
+class IsInDatetime64:
+    def setup(self):
+        dti = date_range(
+            start=datetime(2015, 10, 26), end=datetime(2016, 1, 1), freq="50s"
+        )
+        self.ser = Series(dti)
+        self.subset = self.ser._values[::3]
+        self.cat_subset = Categorical(self.subset)
+
+    def time_isin(self):
+        self.ser.isin(self.subset)
+
+    def time_isin_cat_values(self):
+        self.ser.isin(self.cat_subset)
+
+    def time_isin_mismatched_dtype(self):
+        self.ser.isin([1, 2])
+
+    def time_isin_empty(self):
+        self.ser.isin([])
+
+
 class IsInFloat64:
     def setup(self):
         self.small = Series([1, 2], dtype=np.float64)
@@ -90,6 +112,55 @@ def time_isin_long_series_long_values_floats(self):
         self.s_long_floats.isin(self.vals_long_floats)
 
 
+class IsInLongSeriesLookUpDominates:
+    params = [
+        ["int64", "int32", "float64", "float32", "object"],
+        [5, 1000],
+        ["random_hits", "random_misses", "monotone_hits", "monotone_misses"],
+    ]
+    param_names = ["dtype", "MaxNumber", "series_type"]
+
+    def setup(self, dtype, MaxNumber, series_type):
+        N = 10 ** 7
+        if series_type == "random_hits":
+            np.random.seed(42)
+            array = np.random.randint(0, MaxNumber, N)
+        if series_type == "random_misses":
+            np.random.seed(42)
+            array = np.random.randint(0, MaxNumber, N) + MaxNumber
+        if series_type == "monotone_hits":
+            array = np.repeat(np.arange(MaxNumber), N // MaxNumber)
+        if series_type == "monotone_misses":
+            array = np.arange(N) + MaxNumber
+        self.series = Series(array).astype(dtype)
+        self.values = np.arange(MaxNumber).astype(dtype)
+
+    def time_isin(self, dtypes, MaxNumber, series_type):
+        self.series.isin(self.values)
+
+
+class IsInLongSeriesValuesDominate:
+    params = [
+        ["int64", "int32", "float64", "float32", "object"],
+        ["random", "monotone"],
+    ]
+    param_names = ["dtype", "series_type"]
+
+    def setup(self, dtype, series_type):
+        N = 10 ** 7
+        if series_type == "random":
+            np.random.seed(42)
+            vals = np.random.randint(0, 10 * N, N)
+        if series_type == "monotone":
+            vals = np.arange(N)
+        self.values = vals.astype(dtype)
+        M = 10 ** 6 + 1
+        self.series = Series(np.arange(M)).astype(dtype)
+
+    def time_isin(self, dtypes, series_type):
+        self.series.isin(self.values)
+
+
 class NSort:
 
     params = ["first", "last", "all"]

azure-pipelines.yml

@@ -40,7 +40,7 @@ jobs:
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel setuptools && \
         pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
-        python setup.py build_ext -q -i -j2 && \
+        python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
         pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'

ci/azure/windows.yml

@@ -34,7 +34,7 @@ jobs:
     - bash: |
         source activate pandas-dev
         conda list
-        python setup.py build_ext -q -i -j 4
+        python setup.py build_ext -q -j 4
         python -m pip install --no-build-isolation -e .
       displayName: 'Build'
 

ci/deps/azure-39.yaml

@@ -15,3 +15,8 @@ dependencies:
   - numpy
   - python-dateutil
   - pytz
+
+  # optional dependencies
+  - pytables
+  - scipy
+  - pyarrow=1.0

ci/deps/travis-37-locale.yaml

@@ -34,7 +34,7 @@ dependencies:
   - pyarrow>=0.17
   - pytables>=3.5.1
   - scipy
-  - xarray=0.12.0
+  - xarray=0.12.3
   - xlrd
   - xlsxwriter
   - xlwt

ci/run_tests.sh

@@ -25,7 +25,7 @@ PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile -s
 if [[ $(uname) != "Linux"  && $(uname) != "Darwin" ]]; then
     # GH#37455 windows py38 build appears to be running out of memory
     #  skip collection of window tests
-    PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/window/"
+    PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/window/ --ignore=pandas/tests/plotting/"
 fi
 
 echo $PYTEST_CMD

ci/setup_env.sh

@@ -131,7 +131,7 @@ conda list pandas
 # Make sure any error below is reported as such
 
 echo "[Build extensions]"
-python setup.py build_ext -q -i -j2
+python setup.py build_ext -q -j2
 
 echo "[Updating pip]"
 python -m pip install --no-deps -U pip wheel setuptools