Merge branch 'master' into feature/groupby-repr-ellipses-1135

Author: benjaminarjun

asv_bench/benchmarks/groupby.py

@@ -14,7 +14,7 @@
 method_blacklist = {
     'object': {'median', 'prod', 'sem', 'cumsum', 'sum', 'cummin', 'mean',
                'max', 'skew', 'cumprod', 'cummax', 'rank', 'pct_change', 'min',
-               'var', 'mad', 'describe', 'std'},
+               'var', 'mad', 'describe', 'std', 'quantile'},
     'datetime': {'median', 'prod', 'sem', 'cumsum', 'sum', 'mean', 'skew',
                  'cumprod', 'cummax', 'pct_change', 'var', 'mad', 'describe',
                  'std'}
@@ -316,8 +316,9 @@ class GroupByMethods(object):
               ['all', 'any', 'bfill', 'count', 'cumcount', 'cummax', 'cummin',
                'cumprod', 'cumsum', 'describe', 'ffill', 'first', 'head',
                'last', 'mad', 'max', 'min', 'median', 'mean', 'nunique',
-               'pct_change', 'prod', 'rank', 'sem', 'shift', 'size', 'skew',
-               'std', 'sum', 'tail', 'unique', 'value_counts', 'var'],
+               'pct_change', 'prod', 'quantile', 'rank', 'sem', 'shift',
+               'size', 'skew', 'std', 'sum', 'tail', 'unique', 'value_counts',
+               'var'],
               ['direct', 'transformation']]
 
     def setup(self, dtype, method, application):

ci/code_checks.sh

@@ -241,8 +241,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL06, GL07, GL09, SS04, PR03, PR05, PR10, EX04, RT04, RT05, SS05, SA05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL06,GL07,GL09,SS04,PR03,PR04,PR05,EX04,RT04,RT05,SS05,SA05
+    MSG='Validate docstrings (GL06, GL07, GL09, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT04, RT05, SA05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL06,GL07,GL09,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT04,RT05,SA05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/azure-27-compat.yaml

@@ -18,9 +18,10 @@ dependencies:
   - xlsxwriter=0.5.2
   - xlwt=0.7.5
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
+  - isort
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1

ci/deps/azure-27-locale.yaml

@@ -20,10 +20,11 @@ dependencies:
   - xlsxwriter=0.5.2
   - xlwt=0.7.5
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1

ci/deps/azure-36-locale_slow.yaml

@@ -26,9 +26,10 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
+  - isort
   - pip:
     - hypothesis>=3.58.0

ci/deps/azure-37-locale.yaml

@@ -25,9 +25,10 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
+  - isort
   - pip:
     - hypothesis>=3.58.0
     - moto  # latest moto in conda-forge fails with 3.7, move to conda dependencies when this is fixed

ci/deps/azure-37-numpydev.yaml

@@ -6,10 +6,11 @@ dependencies:
   - pytz
   - Cython>=0.28.2
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"

ci/deps/azure-macos-35.yaml

@@ -21,10 +21,11 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  # universal
-  - pytest
-  - pytest-xdist
-  - pytest-mock
+  - isort
   - pip:
     - python-dateutil==2.5.3
+    # universal
+    - pytest>=4.0.2
+    - pytest-xdist
+    - pytest-mock
     - hypothesis>=3.58.0

ci/deps/azure-windows-27.yaml

@@ -25,8 +25,9 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
   - hypothesis>=3.58.0
+  - isort

ci/deps/azure-windows-36.yaml

@@ -23,7 +23,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort

ci/deps/travis-27.yaml

@@ -39,11 +39,12 @@ dependencies:
   - xlsxwriter=0.5.2
   - xlwt=0.7.5
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto==1.3.4
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - backports.lzma
     - pandas-gbq

ci/deps/travis-36-doc.yaml

@@ -41,5 +41,6 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
+  - isort

ci/deps/travis-36-locale.yaml

@@ -28,9 +28,10 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
+  - isort
   - pip:
     - hypothesis>=3.58.0

ci/deps/travis-36-slow.yaml

@@ -25,8 +25,9 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
   - hypothesis>=3.58.0
+  - isort

ci/deps/travis-36.yaml

@@ -33,11 +33,12 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-cov
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - brotlipy
     - coverage

ci/deps/travis-37.yaml

@@ -12,10 +12,11 @@ dependencies:
   - nomkl
   - pyarrow
   - pytz
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
   - s3fs
+  - isort
   - pip:
     - moto

doc/make.py

@@ -294,14 +294,16 @@ def main():
                            help='number of jobs used by sphinx-build')
     argparser.add_argument('--no-api',
                            default=False,
-                           help='ommit api and autosummary',
+                           help='omit api and autosummary',
                            action='store_true')
     argparser.add_argument('--single',
                            metavar='FILENAME',
                            type=str,
                            default=None,
-                           help=('filename of section or method name to '
-                                 'compile, e.g. "indexing", "DataFrame.join"'))
+                           help=('filename (relative to the "source" folder)'
+                                 ' of section or method name to compile, e.g. '
+                                 '"development/contributing.rst",'
+                                 ' "ecosystem.rst", "pandas.DataFrame.join"'))
     argparser.add_argument('--python-path',
                            type=str,
                            default=os.path.dirname(DOC_PATH),
@@ -323,7 +325,7 @@ def main():
     # the import of `python_path` correctly. The latter is used to resolve
     # the import within the module, injecting it into the global namespace
     os.environ['PYTHONPATH'] = args.python_path
-    sys.path.append(args.python_path)
+    sys.path.insert(0, args.python_path)
     globals()['pandas'] = importlib.import_module('pandas')
 
     # Set the matplotlib backend to the non-interactive Agg backend for all

doc/source/development/contributing.rst

@@ -178,6 +178,7 @@ We'll now kick off a three-step process:
    # Create and activate the build environment
    conda env create -f environment.yml
    conda activate pandas-dev
+   conda uninstall --force pandas
 
    # or with older versions of Anaconda:
    source activate pandas-dev
@@ -428,14 +429,14 @@ reducing the turn-around time for checking your changes.
     python make.py clean
     python make.py --no-api
 
-    # compile the docs with only a single
-    # section, that which is in indexing.rst
+    # compile the docs with only a single section, relative to the "source" folder.
+    # For example, compiling only this guide (docs/source/development/contributing.rst)
     python make.py clean
-    python make.py --single indexing
+    python make.py --single development/contributing.rst
 
     # compile the reference docs for a single function
     python make.py clean
-    python make.py --single DataFrame.join
+    python make.py --single pandas.DataFrame.join
 
 For comparison, a full documentation build may take 15 minutes, but a single
 section may take 15 seconds. Subsequent builds, which only process portions
@@ -731,7 +732,7 @@ extensions in `numpy.testing
 
 .. note::
 
-   The earliest supported pytest version is 3.6.0.
+   The earliest supported pytest version is 4.0.2.
 
 Writing tests
 ~~~~~~~~~~~~~

doc/source/install.rst

@@ -202,7 +202,7 @@ pandas is equipped with an exhaustive set of unit tests, covering about 97% of
 the code base as of this writing. To run it on your machine to verify that
 everything is working (and that you have all of the dependencies, soft and hard,
 installed), make sure you have `pytest
-<http://docs.pytest.org/en/latest/>`__ >= 3.6 and `Hypothesis
+<http://docs.pytest.org/en/latest/>`__ >= 4.0.2 and `Hypothesis
 <https://hypothesis.readthedocs.io/>`__ >= 3.58, then run:
 
 ::

doc/source/reference/groupby.rst

@@ -99,6 +99,7 @@ application to columns of a specific data type.
    DataFrameGroupBy.idxmax
    DataFrameGroupBy.idxmin
    DataFrameGroupBy.mad
+   DataFrameGroupBy.nunique
    DataFrameGroupBy.pct_change
    DataFrameGroupBy.plot
    DataFrameGroupBy.quantile

doc/source/reference/series.rst

@@ -409,6 +409,7 @@ strings and apply several methods to it. These can be accessed like
    :template: autosummary/accessor_method.rst
 
    Series.str.capitalize
+   Series.str.casefold
    Series.str.cat
    Series.str.center
    Series.str.contains

doc/source/user_guide/indexing.rst

@@ -435,7 +435,7 @@ Selection By Position
    This is sometimes called ``chained assignment`` and should be avoided.
    See :ref:`Returning a View versus Copy <indexing.view_versus_copy>`.
 
-Pandas provides a suite of methods in order to get **purely integer based indexing**. The semantics follow closely Python and NumPy slicing. These are ``0-based`` indexing. When slicing, the start bounds is *included*, while the upper bound is *excluded*. Trying to use a non-integer, even a **valid** label will raise an ``IndexError``.
+Pandas provides a suite of methods in order to get **purely integer based indexing**. The semantics follow closely Python and NumPy slicing. These are ``0-based`` indexing. When slicing, the start bound is *included*, while the upper bound is *excluded*. Trying to use a non-integer, even a **valid** label will raise an ``IndexError``.
 
 The ``.iloc`` attribute is the primary access method. The following are valid inputs:
 
@@ -545,7 +545,7 @@ Selection By Callable
 .. versionadded:: 0.18.1
 
 ``.loc``, ``.iloc``, and also ``[]`` indexing can accept a ``callable`` as indexer.
-The ``callable`` must be a function with one argument (the calling Series, DataFrame or Panel) and that returns valid output for indexing.
+The ``callable`` must be a function with one argument (the calling Series, DataFrame or Panel) that returns valid output for indexing.
 
 .. ipython:: python
 
@@ -569,7 +569,7 @@ You can use callable indexing in ``Series``.
    df1.A.loc[lambda s: s > 0]
 
 Using these methods / indexers, you can chain data selection operations
-without using temporary variable.
+without using a temporary variable.
 
 .. ipython:: python
 
@@ -907,7 +907,7 @@ of the DataFrame):
 
    df[df['A'] > 0]
 
-List comprehensions and ``map`` method of Series can also be used to produce
+List comprehensions and the ``map`` method of Series can also be used to produce
 more complex criteria:
 
 .. ipython:: python
@@ -1556,7 +1556,7 @@ See :ref:`Advanced Indexing <advanced>` for usage of MultiIndexes.
   ind
 
 ``set_names``, ``set_levels``, and ``set_codes`` also take an optional
-`level`` argument
+``level`` argument
 
 .. ipython:: python
 

doc/source/user_guide/text.rst

@@ -600,6 +600,7 @@ Method Summary
     :meth:`~Series.str.partition`;Equivalent to ``str.partition``
     :meth:`~Series.str.rpartition`;Equivalent to ``str.rpartition``
     :meth:`~Series.str.lower`;Equivalent to ``str.lower``
+    :meth:`~Series.str.casefold`;Equivalent to ``str.casefold``
     :meth:`~Series.str.upper`;Equivalent to ``str.upper``
     :meth:`~Series.str.find`;Equivalent to ``str.find``
     :meth:`~Series.str.rfind`;Equivalent to ``str.rfind``

doc/source/whatsnew/v0.24.2.rst

@@ -23,13 +23,16 @@ Fixed Regressions
 - Fixed regression in :meth:`DataFrame.all` and :meth:`DataFrame.any` where ``bool_only=True`` was ignored (:issue:`25101`)
 - Fixed issue in ``DataFrame`` construction with passing a mixed list of mixed types could segfault. (:issue:`25075`)
 - Fixed regression in :meth:`DataFrame.apply` causing ``RecursionError`` when ``dict``-like classes were passed as argument. (:issue:`25196`)
+- Fixed regression in :meth:`DataFrame.replace` where ``regex=True`` was only replacing patterns matching the start of the string (:issue:`25259`)
 
 - Fixed regression in :meth:`DataFrame.duplicated()`, where empty dataframe was not returning a boolean dtyped Series. (:issue:`25184`)
 - Fixed regression in :meth:`Series.min` and :meth:`Series.max` where ``numeric_only=True`` was ignored when the ``Series`` contained ```Categorical`` data (:issue:`25299`)
 - Fixed regression in subtraction between :class:`Series` objects with ``datetime64[ns]`` dtype incorrectly raising ``OverflowError`` when the `Series` on the right contains null values (:issue:`25317`)
 - Fixed regression in :class:`TimedeltaIndex` where `np.sum(index)` incorrectly returned a zero-dimensional object instead of a scalar (:issue:`25282`)
 - Fixed regression in ``IntervalDtype`` construction where passing an incorrect string with 'Interval' as a prefix could result in a ``RecursionError``. (:issue:`25338`)
 
+- Fixed regression in :class:`Categorical`, where constructing it from a categorical ``Series`` and an explicit ``categories=`` that differed from that in the ``Series`` created an invalid object which could trigger segfaults. (:issue:`25318`)
+
 .. _whatsnew_0242.enhancements:
 
 Enhancements
@@ -97,6 +100,7 @@ Bug Fixes
 
 - Bug in :meth:`Series.is_unique` where single occurrences of ``NaN`` were not considered unique (:issue:`25180`)
 - Bug in :func:`merge` when merging an empty ``DataFrame`` with an ``Int64`` column or a non-empty ``DataFrame`` with an ``Int64`` column that is all ``NaN`` (:issue:`25183`)
+- Bug in ``IntervalTree`` where a ``RecursionError`` occurs upon construction due to an overflow when adding endpoints, which also causes :class:`IntervalIndex` to crash during indexing operations (:issue:`25485`)
 -
 
 .. _whatsnew_0.242.contributors: