Merge branch 'master' of https://github.com/pandas-dev/pandas into 19356

Author: phofl

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
 
     - name: Setting conda path
-      run: echo "::add-path::${HOME}/miniconda3/bin"
+      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
 
     - name: Checkout
       uses: actions/checkout@v1
@@ -98,7 +98,7 @@ jobs:
     steps:
 
     - name: Setting conda path
-      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
 
     - name: Checkout
       uses: actions/checkout@v1

.travis.yml

@@ -35,11 +35,6 @@ matrix:
   fast_finish: true
 
   include:
-    - dist: bionic
-      python: 3.9-dev
-      env:
-        - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
-
     - env:
       - JOB="3.8, slow" ENV_FILE="ci/deps/travis-38-slow.yaml" PATTERN="slow" SQL="1"
       services:
@@ -94,7 +89,7 @@ install:
 script:
   - echo "script start"
   - echo "$JOB"
-  - if [ "$JOB" != "3.9-dev" ]; then source activate pandas-dev; fi
+  - source activate pandas-dev
   - ci/run_tests.sh
 
 after_script:

asv_bench/benchmarks/hash_functions.py

@@ -0,0 +1,164 @@
+import numpy as np
+
+import pandas as pd
+
+
+class IsinAlmostFullWithRandomInt:
+    params = [
+        [np.float64, np.int64, np.uint64, np.object],
+        range(10, 21),
+    ]
+    param_names = ["dtype", "exponent"]
+
+    def setup(self, dtype, exponent):
+        M = 3 * 2 ** (exponent - 2)
+        # 0.77-the maximal share of occupied buckets
+        np.random.seed(42)
+        self.s = pd.Series(np.random.randint(0, M, M)).astype(dtype)
+        self.values = np.random.randint(0, M, M).astype(dtype)
+        self.values_outside = self.values + M
+
+    def time_isin(self, dtype, exponent):
+        self.s.isin(self.values)
+
+    def time_isin_outside(self, dtype, exponent):
+        self.s.isin(self.values_outside)
+
+
+class IsinWithRandomFloat:
+    params = [
+        [np.float64, np.object],
+        [
+            1_300,
+            2_000,
+            7_000,
+            8_000,
+            70_000,
+            80_000,
+            750_000,
+            900_000,
+        ],
+    ]
+    param_names = ["dtype", "M"]
+
+    def setup(self, dtype, M):
+        np.random.seed(42)
+        self.values = np.random.rand(M)
+        self.s = pd.Series(self.values).astype(dtype)
+        np.random.shuffle(self.values)
+        self.values_outside = self.values + 0.1
+
+    def time_isin(self, dtype, M):
+        self.s.isin(self.values)
+
+    def time_isin_outside(self, dtype, M):
+        self.s.isin(self.values_outside)
+
+
+class IsinWithArangeSorted:
+    params = [
+        [np.float64, np.int64, np.uint64, np.object],
+        [
+            1_000,
+            2_000,
+            8_000,
+            100_000,
+            1_000_000,
+        ],
+    ]
+    param_names = ["dtype", "M"]
+
+    def setup(self, dtype, M):
+        self.s = pd.Series(np.arange(M)).astype(dtype)
+        self.values = np.arange(M).astype(dtype)
+
+    def time_isin(self, dtype, M):
+        self.s.isin(self.values)
+
+
+class IsinWithArange:
+    params = [
+        [np.float64, np.int64, np.uint64, np.object],
+        [
+            1_000,
+            2_000,
+            8_000,
+        ],
+        [-2, 0, 2],
+    ]
+    param_names = ["dtype", "M", "offset_factor"]
+
+    def setup(self, dtype, M, offset_factor):
+        offset = int(M * offset_factor)
+        np.random.seed(42)
+        tmp = pd.Series(np.random.randint(offset, M + offset, 10 ** 6))
+        self.s = tmp.astype(dtype)
+        self.values = np.arange(M).astype(dtype)
+
+    def time_isin(self, dtype, M, offset_factor):
+        self.s.isin(self.values)
+
+
+class Float64GroupIndex:
+    # GH28303
+    def setup(self):
+        self.df = pd.date_range(
+            start="1/1/2018", end="1/2/2018", periods=1e6
+        ).to_frame()
+        self.group_index = np.round(self.df.index.astype(int) / 1e9)
+
+    def time_groupby(self):
+        self.df.groupby(self.group_index).last()
+
+
+class UniqueAndFactorizeArange:
+    params = range(4, 16)
+    param_names = ["exponent"]
+
+    def setup(self, exponent):
+        a = np.arange(10 ** 4, dtype="float64")
+        self.a2 = (a + 10 ** exponent).repeat(100)
+
+    def time_factorize(self, exponent):
+        pd.factorize(self.a2)
+
+    def time_unique(self, exponent):
+        pd.unique(self.a2)
+
+
+class NumericSeriesIndexing:
+
+    params = [
+        (pd.Int64Index, pd.UInt64Index, pd.Float64Index),
+        (10 ** 4, 10 ** 5, 5 * 10 ** 5, 10 ** 6, 5 * 10 ** 6),
+    ]
+    param_names = ["index_dtype", "N"]
+
+    def setup(self, index, N):
+        vals = np.array(list(range(55)) + [54] + list(range(55, N - 1)))
+        indices = index(vals)
+        self.data = pd.Series(np.arange(N), index=indices)
+
+    def time_loc_slice(self, index, N):
+        # trigger building of mapping
+        self.data.loc[:800]
+
+
+class NumericSeriesIndexingShuffled:
+
+    params = [
+        (pd.Int64Index, pd.UInt64Index, pd.Float64Index),
+        (10 ** 4, 10 ** 5, 5 * 10 ** 5, 10 ** 6, 5 * 10 ** 6),
+    ]
+    param_names = ["index_dtype", "N"]
+
+    def setup(self, index, N):
+        vals = np.array(list(range(55)) + [54] + list(range(55, N - 1)))
+        np.random.seed(42)
+        np.random.shuffle(vals)
+        indices = index(vals)
+        self.data = pd.Series(np.arange(N), index=indices)
+
+    def time_loc_slice(self, index, N):
+        # trigger building of mapping
+        self.data.loc[:800]

ci/azure/posix.yml

@@ -61,6 +61,11 @@ jobs:
           PANDAS_TESTING_MODE: "deprecate"
           EXTRA_APT: "xsel"
 
+        py39:
+          ENV_FILE: ci/deps/azure-39.yaml
+          CONDA_PY: "39"
+          PATTERN: "not slow and not network and not clipboard"
+
   steps:
     - script: |
         if [ "$(uname)" == "Linux" ]; then

ci/build39.sh

@@ -225,7 +225,7 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS02, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
+    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS01, SS02, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA02,SA03
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 

ci/check_cache.sh

@@ -34,7 +34,7 @@ dependencies:
   - xlsxwriter
   - xlwt
   - moto
-  - pyarrow>=0.15
+  - pyarrow=1.0.0
   - pip
   - pip:
     - pyxlsb

ci/code_checks.sh

@@ -0,0 +1,17 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.9.*
+
+  # tools
+  - cython>=0.29.21
+  - pytest>=5.0.1
+  - pytest-xdist>=1.21
+  - hypothesis>=3.58.0
+  - pytest-azurepipelines
+
+  # pandas dependencies
+  - numpy
+  - python-dateutil
+  - pytz

ci/deps/azure-38-locale.yaml

@@ -1,10 +1,5 @@
 #!/bin/bash -e
 
-if [ "$JOB" == "3.9-dev" ]; then
-    /bin/bash ci/build39.sh
-    exit 0
-fi
-
 # edit the locale file if needed
 if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo "Adding locale to the first line of pandas/__init__.py"

ci/deps/azure-39.yaml

@@ -442,7 +442,7 @@ Some other important things to know about the docs:
 
      contributing_docstring.rst
 
-* The tutorials make heavy use of the `ipython directive
+* The tutorials make heavy use of the `IPython directive
   <https://matplotlib.org/sampledoc/ipython_directive.html>`_ sphinx extension.
   This directive lets you put code in the documentation which will be run
   during the doc build. For example::

ci/setup_env.sh

@@ -63,14 +63,14 @@ The first conventions every Python docstring should follow are defined in
 `PEP-257 <https://www.python.org/dev/peps/pep-0257/>`_.
 
 As PEP-257 is quite broad, other more specific standards also exist. In the
-case of pandas, the numpy docstring convention is followed. These conventions are
+case of pandas, the NumPy docstring convention is followed. These conventions are
 explained in this document:
 
 * `numpydoc docstring guide <https://numpydoc.readthedocs.io/en/latest/format.html>`_
   (which is based in the original `Guide to NumPy/SciPy documentation
   <https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt>`_)
 
-numpydoc is a Sphinx extension to support the numpy docstring convention.
+numpydoc is a Sphinx extension to support the NumPy docstring convention.
 
 The standard uses reStructuredText (reST). reStructuredText is a markup
 language that allows encoding styles in plain text files. Documentation
@@ -401,7 +401,7 @@ DataFrame:
 * pandas.Categorical
 * pandas.arrays.SparseArray
 
-If the exact type is not relevant, but must be compatible with a numpy
+If the exact type is not relevant, but must be compatible with a NumPy
 array, array-like can be specified. If Any type that can be iterated is
 accepted, iterable can be used:
 
@@ -819,7 +819,7 @@ positional arguments ``head(3)``.
         """
         A sample DataFrame method.
 
-        Do not import numpy and pandas.
+        Do not import NumPy and pandas.
 
         Try to use meaningful data, when it makes the example easier
         to understand.
@@ -854,7 +854,7 @@ Tips for getting your examples pass the doctests
 Getting the examples pass the doctests in the validation script can sometimes
 be tricky. Here are some attention points:
 
-* Import all needed libraries (except for pandas and numpy, those are already
+* Import all needed libraries (except for pandas and NumPy, those are already
   imported as ``import pandas as pd`` and ``import numpy as np``) and define
   all variables you use in the example.
 

doc/source/development/contributing.rst

@@ -219,7 +219,7 @@ and re-boxes it if necessary.
 
 If applicable, we highly recommend that you implement ``__array_ufunc__`` in your
 extension array to avoid coercion to an ndarray. See
-`the numpy documentation <https://numpy.org/doc/stable/reference/generated/numpy.lib.mixins.NDArrayOperatorsMixin.html>`__
+`the NumPy documentation <https://numpy.org/doc/stable/reference/generated/numpy.lib.mixins.NDArrayOperatorsMixin.html>`__
 for an example.
 
 As part of your implementation, we require that you defer to pandas when a pandas

doc/source/development/contributing_docstring.rst

@@ -16,6 +16,7 @@ Development
     code_style
     maintaining
     internals
+    test_writing
     extending
     developer
     policies