Merge branch 'master' of github.com:pandas-dev/pandas into feature/pivot_table_groupby_observed

Author: benjaminr

asv_bench/benchmarks/index_object.py

@@ -1,7 +1,7 @@
 import numpy as np
 import pandas.util.testing as tm
 from pandas import (Series, date_range, DatetimeIndex, Index, RangeIndex,
-                    Float64Index)
+                    Float64Index, IntervalIndex)
 
 
 class SetOperations:
@@ -181,4 +181,18 @@ def time_get_loc(self):
         self.ind.get_loc(0)
 
 
+class IntervalIndexMethod(object):
+    # GH 24813
+    params = [10**3, 10**5]
+
+    def setup(self, N):
+        left = np.append(np.arange(N), np.array(0))
+        right = np.append(np.arange(1, N + 1), np.array(1))
+        self.intv = IntervalIndex.from_arrays(left, right)
+        self.intv._engine
+
+    def time_monotonic_inc(self, N):
+        self.intv.is_monotonic_increasing
+
+
 from .pandas_vb_common import setup  # noqa: F401

doc/source/reference/groupby.rst

@@ -50,6 +50,10 @@ Computations / Descriptive Stats
    GroupBy.bfill
    GroupBy.count
    GroupBy.cumcount
+   GroupBy.cummax
+   GroupBy.cummin
+   GroupBy.cumprod
+   GroupBy.cumsum
    GroupBy.ffill
    GroupBy.first
    GroupBy.head

doc/source/whatsnew/v0.25.0.rst

@@ -24,7 +24,7 @@ including other versions of pandas.
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
 - :func:`DataFrame.plot` keywords ``logy``, ``logx`` and ``loglog`` can now accept the value ``'sym'`` for symlog scaling. (:issue:`24867`)
-- Added support for ISO week year format ('%G-%V-%u') when parsing datetimes using :meth: `to_datetime` (:issue:`16607`)
+- Added support for ISO week year format ('%G-%V-%u') when parsing datetimes using :meth:`to_datetime` (:issue:`16607`)
 - Indexing of ``DataFrame`` and ``Series`` now accepts zerodim ``np.ndarray`` (:issue:`24919`)
 - :meth:`Timestamp.replace` now supports the ``fold`` argument to disambiguate DST transition times (:issue:`25017`)
 - :meth:`DataFrame.at_time` and :meth:`Series.at_time` now support :meth:`datetime.time` objects with timezones (:issue:`24043`)
@@ -53,16 +53,14 @@ Indexing a :class:`DataFrame` or :class:`Series` with a :class:`DatetimeIndex` w
 date string with a UTC offset would previously ignore the UTC offset. Now, the UTC offset
 is respected in indexing. (:issue:`24076`, :issue:`16785`)
 
-*Previous Behavior*:
+.. ipython:: python
 
-.. code-block:: ipython
+    df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
+    df
 
-    In [1]: df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
+*Previous Behavior*:
 
-    In [2]: df
-    Out[2]:
-                               0
-    2019-01-01 00:00:00-08:00  0
+.. code-block:: ipython
 
     In [3]: df['2019-01-01 00:00:00+04:00':'2019-01-01 01:00:00+04:00']
     Out[3]:
@@ -71,9 +69,8 @@ is respected in indexing. (:issue:`24076`, :issue:`16785`)
 
 *New Behavior*:
 
-.. ipython:: ipython
+.. ipython:: python
 
-    df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
     df['2019-01-01 12:00:00+04:00':'2019-01-01 13:00:00+04:00']
 
 .. _whatsnew_0250.api_breaking.groupby_apply_first_group_once:
@@ -84,10 +81,7 @@ GroupBy.apply on ``DataFrame`` evaluates first group only once
 The implementation of :meth:`DataFrameGroupBy.apply() <pandas.core.groupby.DataFrameGroupBy.apply>`
 previously evaluated the supplied function consistently twice on the first group
 to infer if it is safe to use a fast code path. Particularly for functions with
-side effects, this was an undesired behavior and may have led to surprises.
-
-(:issue:`2936`, :issue:`2656`, :issue:`7739`, :issue:`10519`, :issue:`12155`,
-:issue:`20084`, :issue:`21417`)
+side effects, this was an undesired behavior and may have led to surprises. (:issue:`2936`, :issue:`2656`, :issue:`7739`, :issue:`10519`, :issue:`12155`, :issue:`20084`, :issue:`21417`)
 
 Now every group is evaluated only a single time.
 
@@ -124,7 +118,7 @@ Concatenating Sparse Values
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 When passed DataFrames whose values are sparse, :func:`concat` will now return a
-Series or DataFrame with sparse values, rather than a ``SparseDataFrame`` (:issue:`25702`).
+:class:`Series` or :class:`DataFrame` with sparse values, rather than a :class:`SparseDataFrame` (:issue:`25702`).
 
 .. ipython:: python
 
@@ -161,7 +155,7 @@ cause a ``SparseSeries`` or ``SparseDataFrame`` to be returned, as before.
 Increased minimum versions for dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Due to dropping support for Python 2.7, a number of optional dependencies have updated minimum versions (issue:`25725`, :issue:`24942`, :issue:`25752`).
+Due to dropping support for Python 2.7, a number of optional dependencies have updated minimum versions (:issue:`25725`, :issue:`24942`, :issue:`25752`).
 Independently, some minimum supported versions of dependencies were updated (:issue:`23519`, :issue:`25554`).
 If installed, we now require:
 
@@ -215,7 +209,7 @@ Other API Changes
 ^^^^^^^^^^^^^^^^^
 
 - :class:`DatetimeTZDtype` will now standardize pytz timezones to a common timezone instance (:issue:`24713`)
-- ``Timestamp`` and ``Timedelta`` scalars now implement the :meth:`to_numpy` method as aliases to :meth:`Timestamp.to_datetime64` and :meth:`Timedelta.to_timedelta64`, respectively. (:issue:`24653`)
+- :class:`Timestamp` and :class:`Timedelta` scalars now implement the :meth:`to_numpy` method as aliases to :meth:`Timestamp.to_datetime64` and :meth:`Timedelta.to_timedelta64`, respectively. (:issue:`24653`)
 - :meth:`Timestamp.strptime` will now rise a ``NotImplementedError`` (:issue:`25016`)
 - Comparing :class:`Timestamp` with unsupported objects now returns :py:obj:`NotImplemented` instead of raising ``TypeError``. This implies that unsupported rich comparisons are delegated to the other object, and are now consistent with Python 3 behavior for ``datetime`` objects (:issue:`24011`)
 - Bug in :meth:`DatetimeIndex.snap` which didn't preserving the ``name`` of the input :class:`Index` (:issue:`25575`)
@@ -226,14 +220,14 @@ Other API Changes
 Deprecations
 ~~~~~~~~~~~~
 
-- Deprecated the `M (months)` and `Y (year)` `units` parameter of :func: `pandas.to_timedelta`, :func: `pandas.Timedelta` and :func: `pandas.TimedeltaIndex` (:issue:`16344`)
-- The functions :func:`pandas.to_datetime` and :func:`pandas.to_timedelta` have deprecated the ``box`` keyword. Instead, use :meth:`to_numpy` or :meth:`Timestamp.to_datetime64`/:meth:`Timedelta.to_timedelta64`. (:issue:`24416`)
+- Deprecated the ``units=M`` (months) and ``units=Y`` (year) parameters for ``units`` of :func:`pandas.to_timedelta`, :func:`pandas.Timedelta` and :func:`pandas.TimedeltaIndex` (:issue:`16344`)
+- The functions :func:`pandas.to_datetime` and :func:`pandas.to_timedelta` have deprecated the ``box`` keyword. Instead, use :meth:`to_numpy` or :meth:`Timestamp.to_datetime64` or :meth:`Timedelta.to_timedelta64`. (:issue:`24416`)
 
 .. _whatsnew_0250.prior_deprecations:
 
 Removal of prior version deprecations/changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- Removed (parts of) :class:`Panel` (:issue:`25047`,:issue:`25191`,:issue:`25231`)
+- Removed ``Panel`` (:issue:`25047`, :issue:`25191`, :issue:`25231`)
 -
 -
 -
@@ -243,15 +237,16 @@ Removal of prior version deprecations/changes
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Significant speedup in `SparseArray` initialization that benefits most operations, fixing performance regression introduced in v0.20.0 (:issue:`24985`)
-- `DataFrame.to_stata()` is now faster when outputting data with any string or non-native endian columns (:issue:`25045`)
+- Significant speedup in :class:`SparseArray` initialization that benefits most operations, fixing performance regression introduced in v0.20.0 (:issue:`24985`)
+- :meth:`DataFrame.to_stata()` is now faster when outputting data with any string or non-native endian columns (:issue:`25045`)
 - Improved performance of :meth:`Series.searchsorted`. The speedup is especially large when the dtype is
   int8/int16/int32 and the searched key is within the integer bounds for the dtype (:issue:`22034`)
 - Improved performance of :meth:`pandas.core.groupby.GroupBy.quantile` (:issue:`20405`)
 - Improved performance of :meth:`read_csv` by faster tokenizing and faster parsing of small float numbers (:issue:`25784`)
 - Improved performance of :meth:`read_csv` by faster parsing of N/A and boolean values (:issue:`25804`)
-- Improved performance of :meth:DataFrame.`to_csv` when write datetime dtype data (:issue:`25708`)
-- Improved performance of :meth:`read_csv` by much faster parsing of MM/YYYY and DD/MM/YYYY datetime formats (:issue:`25922`)
+- Imporved performance of :meth:`IntervalIndex.is_monotonic`, :meth:`IntervalIndex.is_monotonic_increasing` and :meth:`IntervalIndex.is_monotonic_decreasing` by removing conversion to :class:`MultiIndex` (:issue:`24813`)
+- Improved performance of :meth:`DataFrame.to_csv` when writing datetime dtypes (:issue:`25708`)
+- Improved performance of :meth:`read_csv` by much faster parsing of ``MM/YYYY`` and ``DD/MM/YYYY`` datetime formats (:issue:`25922`)
 
 .. _whatsnew_0250.bug_fixes:
 
@@ -271,8 +266,10 @@ Datetimelike
 ^^^^^^^^^^^^
 
 - Bug in :func:`to_datetime` which would raise an (incorrect) ``ValueError`` when called with a date far into the future and the ``format`` argument specified instead of raising ``OutOfBoundsDatetime`` (:issue:`23830`)
-- Bug in :func:`to_datetime` which would raise ``InvalidIndexError: Reindexing only valid with uniquely valued Index objects`` when called with ``cache=True``, with ``arg`` including at least two different elements from the set {None, numpy.nan, pandas.NaT} (:issue:`22305`)
--
+- Bug in :func:`to_datetime` which would raise ``InvalidIndexError: Reindexing only valid with uniquely valued Index objects`` when called with ``cache=True``, with ``arg`` including at least two different elements from the set ``{None, numpy.nan, pandas.NaT}`` (:issue:`22305`)
+- Bug in :class:`DataFrame` and :class:`Series` where timezone aware data with ``dtype='datetime64[ns]`` was not cast to naive (:issue:`25843`)
+- Improved :class:`Timestamp` type checking in various datetime functions to prevent exceptions when using a subclassed ``datetime`` (:issue:`25851`)
+- Bug in :class:`Series` and :class:`DataFrame` repr where ``np.datetime64('NaT')`` and ``np.timedelta64('NaT')`` with ``dtype=object`` would be represented as ``NaN`` (:issue:`25445`)
 -
 
 Timedelta
@@ -331,21 +328,21 @@ Indexing
 ^^^^^^^^
 
 - Improved exception message when calling :meth:`DataFrame.iloc` with a list of non-numeric objects (:issue:`25753`).
--
+- Bug in which :meth:`DataFrame.append` produced an erroneous warning indicating that a ``KeyError`` will be thrown in the future when the data to be appended contains new columns (:issue:`22252`).
 -
 
 
 Missing
 ^^^^^^^
 
-- Fixed misleading exception message in :meth:`Series.missing` if argument ``order`` is required, but omitted (:issue:`10633`, :issue:`24014`).
+- Fixed misleading exception message in :meth:`Series.interpolate` if argument ``order`` is required, but omitted (:issue:`10633`, :issue:`24014`).
 - Fixed class type displayed in exception message in :meth:`DataFrame.dropna` if invalid ``axis`` parameter passed (:issue:`25555`)
 -
 
 MultiIndex
 ^^^^^^^^^^
 
-- Bug in which incorrect exception raised by :meth:`pd.Timedelta` when testing the membership of :class:`MultiIndex` (:issue:`24570`)
+- Bug in which incorrect exception raised by :class:`Timedelta` when testing the membership of :class:`MultiIndex` (:issue:`24570`)
 -
 -
 
@@ -397,9 +394,9 @@ Groupby/Resample/Rolling
 Reshaping
 ^^^^^^^^^
 
-- Bug in :func:`pandas.merge` adds a string of ``None`` if ``None`` is assigned in suffixes instead of remain the column name as-is (:issue:`24782`).
+- Bug in :func:`pandas.merge` adds a string of ``None``, if ``None`` is assigned in suffixes instead of remain the column name as-is (:issue:`24782`).
 - Bug in :func:`merge` when merging by index name would sometimes result in an incorrectly numbered index (:issue:`24212`)
-- :func:`to_records` now accepts dtypes to its `column_dtypes` parameter (:issue:`24895`)
+- :func:`to_records` now accepts dtypes to its ``column_dtypes`` parameter (:issue:`24895`)
 - Bug in :func:`concat` where order of ``OrderedDict`` (and ``dict`` in Python 3.6+) is not respected, when passed in as  ``objs`` argument (:issue:`21510`)
 - Bug in :func:`pivot_table` where columns with ``NaN`` values are dropped even if ``dropna`` argument is ``False``, when the ``aggfunc`` argument contains a ``list`` (:issue:`22159`)
 - Bug in :func:`concat` where the resulting ``freq`` of two :class:`DatetimeIndex` with the same ``freq`` would be dropped (:issue:`3232`).
@@ -410,16 +407,14 @@ Reshaping
 Sparse
 ^^^^^^
 
-- Significant speedup in `SparseArray` initialization that benefits most operations, fixing performance regression introduced in v0.20.0 (:issue:`24985`)
+- Significant speedup in :class:`SparseArray` initialization that benefits most operations, fixing performance regression introduced in v0.20.0 (:issue:`24985`)
 - Bug in :class:`SparseFrame` constructor where passing ``None`` as the data would cause ``default_fill_value`` to be ignored (:issue:`16807`)
-- Bug in `SparseDataFrame` when adding a column in which the length of values does not match length of index, ``AssertionError`` is raised instead of raising ``ValueError`` (:issue:`25484`)
+- Bug in :class:`SparseDataFrame` when adding a column in which the length of values does not match length of index, ``AssertionError`` is raised instead of raising ``ValueError`` (:issue:`25484`)
 
 
 Other
 ^^^^^
 
-- Improved :class:`Timestamp` type checking in various datetime functions to prevent exceptions when using a subclassed `datetime` (:issue:`25851`)
-- Bug in :class:`Series` and :class:`DataFrame` repr where ``np.datetime64('NaT')`` and ``np.timedelta64('NaT')`` with ``dtype=object`` would be represented as ``NaN`` (:issue:`25445`)
 -
 -
 

mypy.ini

@@ -11,15 +11,9 @@ ignore_errors=True
 [mypy-pandas.compat.numpy.function]
 ignore_errors=True
 
-[mypy-pandas.core.accessor]
-ignore_errors=True
-
 [mypy-pandas.core.api]
 ignore_errors=True
 
-[mypy-pandas.core.apply]
-ignore_errors=True
-
 [mypy-pandas.core.arrays.array_]
 ignore_errors=True
 
@@ -32,15 +26,9 @@ ignore_errors=True
 [mypy-pandas.core.arrays.interval]
 ignore_errors=True
 
-[mypy-pandas.core.arrays.numpy_]
-ignore_errors=True
-
 [mypy-pandas.core.arrays.period]
 ignore_errors=True
 
-[mypy-pandas.core.arrays.sparse]
-ignore_errors=True
-
 [mypy-pandas.core.arrays.timedeltas]
 ignore_errors=True
 
@@ -98,9 +86,6 @@ ignore_errors=True
 [mypy-pandas.core.series]
 ignore_errors=True
 
-[mypy-pandas.core.sparse.frame]
-ignore_errors=True
-
 [mypy-pandas.core.util.hashing]
 ignore_errors=True
 

pandas/_libs/intervaltree.pxi.in

@@ -4,6 +4,8 @@ Template for intervaltree
 WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 """
 
+from pandas._libs.algos import is_monotonic
+
 ctypedef fused scalar_t:
     float64_t
     float32_t
@@ -101,6 +103,17 @@ cdef class IntervalTree(IntervalMixin):
 
         return self._is_overlapping
 
+    @property
+    def is_monotonic_increasing(self):
+        """
+        Return True if the IntervalTree is monotonic increasing (only equal or
+        increasing values), else False
+        """
+        values = [self.right, self.left]
+
+        sort_order = np.lexsort(values)
+        return is_monotonic(sort_order, False)[0]
+
     def get_loc(self, scalar_t key):
         """Return all positions corresponding to intervals that overlap with
         the given scalar key

pandas/_libs/tslibs/c_timestamp.pyx

@@ -23,8 +23,6 @@ cimport numpy as cnp
 from numpy cimport int64_t, int8_t
 cnp.import_array()
 
-from dateutil.tz import tzutc
-
 from cpython.datetime cimport (datetime,
                                PyDateTime_Check, PyDelta_Check,
                                PyDateTime_IMPORT)
@@ -38,9 +36,9 @@ from pandas._libs.tslibs.fields import get_start_end_field, get_date_name_field
 from pandas._libs.tslibs.nattype cimport c_NaT as NaT
 from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
 from pandas._libs.tslibs.np_datetime cimport (
-    reverse_ops, cmp_scalar, npy_datetimestruct, dt64_to_dtstruct)
+    reverse_ops, cmp_scalar)
 from pandas._libs.tslibs.timezones cimport (
-    get_timezone, get_utcoffset, is_utc, tz_compare)
+    get_timezone, is_utc, tz_compare)
 from pandas._libs.tslibs.timezones import UTC
 from pandas._libs.tslibs.tzconversion cimport tz_convert_single
 
@@ -381,5 +379,6 @@ cdef class _Timestamp(datetime):
 
     def timestamp(self):
         """Return POSIX timestamp as float."""
-        # py27 compat, see GH#17329
+        # GH 17329
+        # Note: Naive timestamps will not match datetime.stdlib
         return round(self.value / 1e9, 6)

pandas/_libs/tslibs/conversion.pyx

@@ -3,23 +3,20 @@ import cython
 
 import numpy as np
 cimport numpy as cnp
-from numpy cimport uint8_t, int64_t, int32_t, intp_t, ndarray
+from numpy cimport int64_t, int32_t, intp_t, ndarray
 cnp.import_array()
 
 import pytz
-from dateutil.tz import tzutc
 
 # stdlib datetime imports
 from datetime import time as datetime_time
 from cpython.datetime cimport (datetime, tzinfo,
                                PyDateTime_Check, PyDate_Check,
-                               PyDateTime_IMPORT, PyDelta_Check)
+                               PyDateTime_IMPORT)
 PyDateTime_IMPORT
 
 from pandas._libs.tslibs.c_timestamp cimport _Timestamp
 
-from pandas._libs.tslibs.ccalendar import DAY_SECONDS, HOUR_SECONDS
-
 from pandas._libs.tslibs.np_datetime cimport (
     check_dts_bounds, npy_datetimestruct, pandas_datetime_to_datetimestruct,
     _string_to_dts, npy_datetime, dt64_to_dtstruct, dtstruct_to_dt64,
@@ -42,7 +39,7 @@ from pandas._libs.tslibs.nattype cimport (
     NPY_NAT, checknull_with_nat, c_NaT as NaT)
 
 from pandas._libs.tslibs.tzconversion import (
-    tz_localize_to_utc, tz_convert, tz_convert_single)
+    tz_localize_to_utc, tz_convert_single)
 from pandas._libs.tslibs.tzconversion cimport _tz_convert_tzlocal_utc
 
 # ----------------------------------------------------------------------

pandas/_libs/tslibs/timedeltas.pyx

@@ -3,8 +3,6 @@ import collections
 import textwrap
 import warnings
 
-import sys
-
 import cython
 
 from cpython cimport Py_NE, Py_EQ, PyObject_RichCompare
@@ -14,7 +12,7 @@ cimport numpy as cnp
 from numpy cimport int64_t
 cnp.import_array()
 
-from cpython.datetime cimport (datetime, timedelta,
+from cpython.datetime cimport (timedelta,
                                PyDateTime_Check, PyDelta_Check,
                                PyDateTime_IMPORT)
 PyDateTime_IMPORT

pandas/_libs/tslibs/timestamps.pyx

@@ -22,7 +22,6 @@ from pandas._libs.tslibs.conversion import normalize_i8_timestamps
 from pandas._libs.tslibs.conversion cimport (
     _TSObject, convert_to_tsobject,
     convert_datetime_to_tsobject)
-from pandas._libs.tslibs.fields import get_start_end_field, get_date_name_field
 from pandas._libs.tslibs.nattype cimport NPY_NAT, c_NaT as NaT
 from pandas._libs.tslibs.np_datetime cimport (
     check_dts_bounds, npy_datetimestruct, dt64_to_dtstruct)