Merge branch 'master' into pandas-devGH-21980

Author: illegalnumbers

ci/doctests.sh

@@ -21,7 +21,7 @@ if [ "$DOCTEST" ]; then
 
     # DataFrame / Series docstrings
     pytest --doctest-modules -v pandas/core/frame.py \
-        -k"-assign -axes -combine -isin -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_records -to_stata -transform"
+        -k"-assign -axes -combine -isin -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_stata -transform"
 
     if [ $? -ne "0" ]; then
         RET=1

ci/lint.sh

@@ -20,14 +20,14 @@ if [ "$LINT" ]; then
 
     # pandas/_libs/src is C code, so no need to search there.
     echo "Linting *.py"
-    flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting *.py DONE"
 
     echo "Linting setup.py"
-    flake8 setup.py --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 setup.py --ignore=E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
@@ -41,21 +41,21 @@ if [ "$LINT" ]; then
     echo "Linting asv_bench/benchmarks/*.py DONE"
 
     echo "Linting scripts/*.py"
-    flake8 scripts --filename=*.py --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting scripts/*.py DONE"
 
     echo "Linting doc scripts"
-    flake8 doc/make.py doc/source/conf.py --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting doc scripts DONE"
 
     echo "Linting *.pyx"
-    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C407,C411
+    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411
     if [ $? -ne "0" ]; then
         RET=1
     fi

ci/travis-27.yaml

@@ -29,6 +29,7 @@ dependencies:
   - PyCrypto
   - pymysql=0.6.3
   - pytables
+  - blosc=1.14.3
   - python-blosc
   - python-dateutil=2.5.0
   - python=2.7*

ci/travis-36-doc.yaml

@@ -36,6 +36,7 @@ dependencies:
   - sphinx
   - sqlalchemy
   - statsmodels
+  - tzlocal
   - xarray
   - xlrd
   - xlsxwriter

doc/source/api.rst

@@ -2352,6 +2352,7 @@ Computations / Descriptive Stats
    Resampler.std
    Resampler.sum
    Resampler.var
+   Resampler.quantile
 
 Style
 -----

doc/source/groupby.rst

@@ -106,9 +106,8 @@ consider the following ``DataFrame``:
    .. versionadded:: 0.20
 
    A string passed to ``groupby`` may refer to either a column or an index level.
-   If a string matches both a column name and an index level name then a warning is
-   issued and the column takes precedence. This will result in an ambiguity error
-   in a future version.
+   If a string matches both a column name and an index level name, a
+   ``ValueError`` will be raised.
 
 .. ipython:: python
 
@@ -389,7 +388,7 @@ This is mainly syntactic sugar for the alternative and much more verbose:
 Additionally this method avoids recomputing the internal grouping information
 derived from the passed key.
 
-.. _groupby.iterating:
+.. _groupby.iterating-label:
 
 Iterating through groups
 ------------------------
@@ -415,8 +414,7 @@ In the case of grouping by multiple keys, the group name will be a tuple:
       ...:        print(group)
       ...:
 
-It's standard Python-fu but remember you can unpack the tuple in the for loop
-statement if you wish: ``for (k1, k2), group in grouped:``.
+See :ref:`timeseries.iterating-label`.
 
 Selecting a group
 -----------------

doc/source/timeseries.rst

@@ -703,6 +703,24 @@ regularity will result in a ``DatetimeIndex``, although frequency is lost:
 
    ts2[[0, 2, 6]].index
 
+.. _timeseries.iterating-label:
+
+Iterating through groups
+------------------------
+
+With the :ref:`Resampler` object in hand, iterating through the grouped data is very
+natural and functions similarly to :py:func:`itertools.groupby`:
+
+.. ipython:: python
+
+   resampled = df.resample('H')
+
+   for name, group in resampled:
+       print(name)
+       print(group)
+
+See :ref:`groupby.iterating-label`.
+
 .. _timeseries.components:
 
 Time/Date Components

doc/source/whatsnew/v0.24.0.txt

@@ -42,7 +42,7 @@ Pandas has gained the ability to hold integer dtypes with missing values. This l
 Here is an example of the usage.
 
 We can construct a ``Series`` with the specified dtype. The dtype string ``Int64`` is a pandas ``ExtensionDtype``. Specifying a list or array using the traditional missing value
-marker of ``np.nan`` will infer to integer dtype. The display of the ``Series`` will also use the ``NaN`` to indicate missing values in string outputs. (:issue:`20700`, :issue:`20747`)
+marker of ``np.nan`` will infer to integer dtype. The display of the ``Series`` will also use the ``NaN`` to indicate missing values in string outputs. (:issue:`20700`, :issue:`20747`, :issue:`22441`)
 
 .. ipython:: python
 
@@ -182,6 +182,8 @@ Other Enhancements
 - :func:`to_timedelta` now supports iso-formated timedelta strings (:issue:`21877`)
 - :class:`Series` and :class:`DataFrame` now support :class:`Iterable` in constructor (:issue:`2193`)
 - :class:`DatetimeIndex` gained :attr:`DatetimeIndex.timetz` attribute. Returns local time with timezone information. (:issue:`21358`)
+- :class:`Resampler` now is iterable like :class:`GroupBy` (:issue:`15314`).
+- :ref:`Series.resample` and :ref:`DataFrame.resample` have gained the :meth:`Resampler.quantile` (:issue:`15023`).
 
 .. _whatsnew_0240.api_breaking:
 
@@ -442,11 +444,13 @@ ExtensionType Changes
 - ``ExtensionArray`` has gained the abstract methods ``.dropna()`` (:issue:`21185`)
 - ``ExtensionDtype`` has gained the ability to instantiate from string dtypes, e.g. ``decimal`` would instantiate a registered ``DecimalDtype``; furthermore
   the ``ExtensionDtype`` has gained the method ``construct_array_type`` (:issue:`21185`)
+- Added ``ExtensionDtype._is_numeric`` for controlling whether an extension dtype is considered numeric (:issue:`22290`).
 - The ``ExtensionArray`` constructor, ``_from_sequence`` now take the keyword arg ``copy=False`` (:issue:`21185`)
 - Bug in :meth:`Series.get` for ``Series`` using ``ExtensionArray`` and integer index (:issue:`21257`)
+- :meth:`~Series.shift` now dispatches to :meth:`ExtensionArray.shift` (:issue:`22386`)
 - :meth:`Series.combine()` works correctly with :class:`~pandas.api.extensions.ExtensionArray` inside of :class:`Series` (:issue:`20825`)
 - :meth:`Series.combine()` with scalar argument now works for any function type (:issue:`21248`)
--
+- :meth:`Series.astype` and :meth:`DataFrame.astype` now dispatch to :meth:`ExtensionArray.astype` (:issue:`21185:`).
 
 .. _whatsnew_0240.api.incompatibilities:
 
@@ -517,8 +521,10 @@ Removal of prior version deprecations/changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - The ``LongPanel`` and ``WidePanel`` classes have been removed (:issue:`10892`)
+- :meth:`Series.repeat` has renamed the ``reps`` argument to ``repeats`` (:issue:`14645`)
 - Several private functions were removed from the (non-public) module ``pandas.core.common`` (:issue:`22001`)
 - Removal of the previously deprecated module ``pandas.core.datetools`` (:issue:`14105`, :issue:`14094`)
+- Strings passed into :meth:`DataFrame.groupby` that refer to both column and index levels will raise a ``ValueError`` (:issue:`14432`)
 -
 
 .. _whatsnew_0240.performance:
@@ -576,13 +582,17 @@ Datetimelike
 - Bug in :class:`DataFrame` comparisons against ``Timestamp``-like objects failing to raise ``TypeError`` for inequality checks with mismatched types (:issue:`8932`,:issue:`22163`)
 - Bug in :class:`DataFrame` with mixed dtypes including ``datetime64[ns]`` incorrectly raising ``TypeError`` on equality comparisons (:issue:`13128`,:issue:`22163`)
 - Bug in :meth:`DataFrame.eq` comparison against ``NaT`` incorrectly returning ``True`` or ``NaN`` (:issue:`15697`,:issue:`22163`)
-- Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`)
--
 
 Timedelta
 ^^^^^^^^^
 
 - Fixed bug where subtracting :class:`Timedelta` from an object-dtyped array would raise ``TypeError`` (:issue:`21980`)
+- Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`)
+- Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`22390`)
+- Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`22390`)
+- Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`22390`)
+- Bug in :class:`Index` with numeric dtype when multiplying or dividing an array with dtype ``timedelta64`` (:issue:`22390`)
+-
 -
 -
 
@@ -626,6 +636,7 @@ Numeric
   a ``TypeError`` was wrongly raised. For all three methods such calculation are now done correctly. (:issue:`16679`).
 - Bug in :class:`Series` comparison against datetime-like scalars and arrays (:issue:`22074`)
 - Bug in :class:`DataFrame` multiplication between boolean dtype and integer returning ``object`` dtype instead of integer dtype (:issue:`22047`,:issue:`22163`)
+- Bug in :meth:`DataFrame.apply` where, when supplied with a string argument and additional positional or keyword arguments (e.g. ``df.apply('sum', min_count=1)``), a ``TypeError`` was wrongly raised (:issue:`22376`)
 -
 
 Strings
@@ -654,6 +665,7 @@ Indexing
 - Fixed ``DataFrame[np.nan]`` when columns are non-unique (:issue:`21428`)
 - Bug when indexing :class:`DatetimeIndex` with nanosecond resolution dates and timezones (:issue:`11679`)
 - Bug where indexing with a Numpy array containing negative values would mutate the indexer (:issue:`21867`)
+- ``Float64Index.get_loc`` now raises ``KeyError`` when boolean key passed. (:issue:`19087`)
 
 Missing
 ^^^^^^^
@@ -674,6 +686,7 @@ I/O
 
 - :func:`read_html()` no longer ignores all-whitespace ``<tr>`` within ``<thead>`` when considering the ``skiprows`` and ``header`` arguments. Previously, users had to decrease their ``header`` and ``skiprows`` values on such tables to work around the issue. (:issue:`21641`)
 - :func:`read_excel()` will correctly show the deprecation warning for previously deprecated ``sheetname`` (:issue:`17994`)
+- :func:`read_csv()` will correctly parse timezone-aware datetimes (:issue:`22256`)
 -
 
 Plotting
@@ -691,7 +704,7 @@ Groupby/Resample/Rolling
   ``SeriesGroupBy`` when the grouping variable only contains NaNs and numpy version < 1.13 (:issue:`21956`).
 - Multiple bugs in :func:`pandas.core.Rolling.min` with ``closed='left'` and a
   datetime-like index leading to incorrect results and also segfault. (:issue:`21704`)
--
+- Bug in :meth:`Resampler.apply` when passing postiional arguments to applied func (:issue:`14615`).
 
 Sparse
 ^^^^^^
@@ -712,7 +725,7 @@ Reshaping
 - Bug in :func:`get_dummies` with Unicode attributes in Python 2 (:issue:`22084`)
 - Bug in :meth:`DataFrame.replace` raises ``RecursionError`` when replacing empty lists (:issue:`22083`)
 - Bug in :meth:`Series.replace` and meth:`DataFrame.replace` when dict is used as the `to_replace` value and one key in the dict is is another key's value, the results were inconsistent between using integer key and using string key (:issue:`20656`)
--
+- Bug in :meth:`DataFrame.drop_duplicates` for empty ``DataFrame`` which incorrectly raises an error (:issue:`20516`)
 
 Build Changes
 ^^^^^^^^^^^^^

pandas/_libs/algos.pxd

@@ -1,17 +1,20 @@
 from util cimport numeric
-from numpy cimport float64_t, double_t
+
 
 cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k) nogil
 
+
 cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil:
-    cdef numeric t
+    cdef:
+        numeric t
 
     # cython doesn't allow pointer dereference so use array syntax
     t = a[0]
     a[0] = b[0]
     b[0] = t
     return 0
 
+
 cdef enum TiebreakEnumType:
     TIEBREAK_AVERAGE
     TIEBREAK_MIN,

pandas/_libs/algos.pyx

@@ -45,7 +45,7 @@ tiebreakers = {
 }
 
 
-cdef inline are_diff(object left, object right):
+cdef inline bint are_diff(object left, object right):
     try:
         return fabs(left - right) > FP_ERR
     except TypeError:

pandas/_libs/algos_common_helper.pxi.in

@@ -68,12 +68,12 @@ cpdef map_indices_{{name}}(ndarray[{{c_type}}] index):
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def pad_{{name}}(ndarray[{{c_type}}] old, ndarray[{{c_type}}] new,
-                 limit=None):
-    cdef Py_ssize_t i, j, nleft, nright
-    cdef ndarray[int64_t, ndim=1] indexer
-    cdef {{c_type}} cur, next
-    cdef int lim, fill_count = 0
+def pad_{{name}}(ndarray[{{c_type}}] old, ndarray[{{c_type}}] new, limit=None):
+    cdef:
+        Py_ssize_t i, j, nleft, nright
+        ndarray[int64_t, ndim=1] indexer
+        {{c_type}} cur, next
+        int lim, fill_count = 0
 
     nleft = len(old)
     nright = len(new)
@@ -135,9 +135,10 @@ def pad_{{name}}(ndarray[{{c_type}}] old, ndarray[{{c_type}}] new,
 def pad_inplace_{{name}}(ndarray[{{c_type}}] values,
                          ndarray[uint8_t, cast=True] mask,
                          limit=None):
-    cdef Py_ssize_t i, N
-    cdef {{c_type}} val
-    cdef int lim, fill_count = 0
+    cdef:
+        Py_ssize_t i, N
+        {{c_type}} val
+        int lim, fill_count = 0
 
     N = len(values)
 
@@ -171,9 +172,10 @@ def pad_inplace_{{name}}(ndarray[{{c_type}}] values,
 def pad_2d_inplace_{{name}}(ndarray[{{c_type}}, ndim=2] values,
                             ndarray[uint8_t, ndim=2] mask,
                             limit=None):
-    cdef Py_ssize_t i, j, N, K
-    cdef {{c_type}} val
-    cdef int lim, fill_count = 0
+    cdef:
+        Py_ssize_t i, j, N, K
+        {{c_type}} val
+        int lim, fill_count = 0
 
     K, N = (<object> values).shape
 
@@ -233,10 +235,11 @@ D
 @cython.wraparound(False)
 def backfill_{{name}}(ndarray[{{c_type}}] old, ndarray[{{c_type}}] new,
                       limit=None):
-    cdef Py_ssize_t i, j, nleft, nright
-    cdef ndarray[int64_t, ndim=1] indexer
-    cdef {{c_type}} cur, prev
-    cdef int lim, fill_count = 0
+    cdef:
+        Py_ssize_t i, j, nleft, nright
+        ndarray[int64_t, ndim=1] indexer
+        {{c_type}} cur, prev
+        int lim, fill_count = 0
 
     nleft = len(old)
     nright = len(new)
@@ -299,9 +302,10 @@ def backfill_{{name}}(ndarray[{{c_type}}] old, ndarray[{{c_type}}] new,
 def backfill_inplace_{{name}}(ndarray[{{c_type}}] values,
                               ndarray[uint8_t, cast=True] mask,
                               limit=None):
-    cdef Py_ssize_t i, N
-    cdef {{c_type}} val
-    cdef int lim, fill_count = 0
+    cdef:
+        Py_ssize_t i, N
+        {{c_type}} val
+        int lim, fill_count = 0
 
     N = len(values)
 
@@ -335,9 +339,10 @@ def backfill_inplace_{{name}}(ndarray[{{c_type}}] values,
 def backfill_2d_inplace_{{name}}(ndarray[{{c_type}}, ndim=2] values,
                                  ndarray[uint8_t, ndim=2] mask,
                                  limit=None):
-    cdef Py_ssize_t i, j, N, K
-    cdef {{c_type}} val
-    cdef int lim, fill_count = 0
+    cdef:
+        Py_ssize_t i, j, N, K
+        {{c_type}} val
+        int lim, fill_count = 0
 
     K, N = (<object> values).shape
 
@@ -428,10 +433,10 @@ def is_monotonic_{{name}}(ndarray[{{c_type}}] arr, bint timelike):
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def arrmap_{{name}}(ndarray[{{c_type}}] index, object func):
-    cdef Py_ssize_t length = index.shape[0]
-    cdef Py_ssize_t i = 0
-
-    cdef ndarray[object] result = np.empty(length, dtype=np.object_)
+    cdef:
+        Py_ssize_t length = index.shape[0]
+        Py_ssize_t i = 0
+        ndarray[object] result = np.empty(length, dtype=np.object_)
 
     from pandas._libs.lib import maybe_convert_objects
 
@@ -535,6 +540,7 @@ def put2d_{{name}}_{{dest_type}}(ndarray[{{c_type}}, ndim=2, cast=True] values,
 
 cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.intp)).descr.type_num
 
+
 cpdef ensure_platform_int(object arr):
     # GH3033, GH1392
     # platform int is the size of the int pointer, e.g. np.intp
@@ -546,6 +552,7 @@ cpdef ensure_platform_int(object arr):
     else:
         return np.array(arr, dtype=np.intp)
 
+
 cpdef ensure_object(object arr):
     if util.is_array(arr):
         if (<ndarray> arr).descr.type_num == NPY_OBJECT: