Merge remote-tracking branch 'upstream/master' into as_index_false

Author: rhshadrach

doc/source/reference/general_utility_functions.rst

@@ -40,6 +40,7 @@ Exceptions and warnings
    errors.EmptyDataError
    errors.OutOfBoundsDatetime
    errors.MergeError
+   errors.NullFrequencyError
    errors.NumbaUtilError
    errors.ParserError
    errors.ParserWarning

doc/source/reference/offset_frequency.rst

@@ -1044,6 +1044,7 @@ Properties
     Tick.nanos
     Tick.normalize
     Tick.rule_code
+    Tick.n
 
 Methods
 ~~~~~~~
@@ -1077,6 +1078,7 @@ Properties
     Day.nanos
     Day.normalize
     Day.rule_code
+    Day.n
 
 Methods
 ~~~~~~~
@@ -1110,6 +1112,7 @@ Properties
     Hour.nanos
     Hour.normalize
     Hour.rule_code
+    Hour.n
 
 Methods
 ~~~~~~~
@@ -1143,6 +1146,7 @@ Properties
     Minute.nanos
     Minute.normalize
     Minute.rule_code
+    Minute.n
 
 Methods
 ~~~~~~~
@@ -1176,6 +1180,7 @@ Properties
     Second.nanos
     Second.normalize
     Second.rule_code
+    Second.n
 
 Methods
 ~~~~~~~
@@ -1209,6 +1214,7 @@ Properties
     Milli.nanos
     Milli.normalize
     Milli.rule_code
+    Milli.n
 
 Methods
 ~~~~~~~
@@ -1242,6 +1248,7 @@ Properties
     Micro.nanos
     Micro.normalize
     Micro.rule_code
+    Micro.n
 
 Methods
 ~~~~~~~
@@ -1275,6 +1282,7 @@ Properties
     Nano.nanos
     Nano.normalize
     Nano.rule_code
+    Nano.n
 
 Methods
 ~~~~~~~

doc/source/user_guide/computation.rst

@@ -648,6 +648,24 @@ from present information back to past information. This allows the rolling windo
 Currently, this feature is only implemented for time-based windows.
 For fixed windows, the closed parameter cannot be set and the rolling window will always have both endpoints closed.
 
+.. _stats.iter_rolling_window:
+
+Iteration over window:
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.1.0
+
+``Rolling`` and ``Expanding`` objects now support iteration. Be noted that ``min_periods`` is ignored in iteration.
+
+.. ipython::
+
+   In [1]: df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+
+   In [2]: for i in df.rolling(2):
+      ...:     print(i)
+      ...:
+
+
 .. _stats.moments.ts-versus-resampling:
 
 Time-aware rolling vs. resampling

doc/source/whatsnew/v1.1.0.rst

@@ -234,6 +234,8 @@ Other enhancements
   compression library. Compression was also added to the low-level Stata-file writers
   :class:`~pandas.io.stata.StataWriter`, :class:`~pandas.io.stata.StataWriter117`,
   and :class:`~pandas.io.stata.StataWriterUTF8` (:issue:`26599`).
+- :meth:`HDFStore.put` now accepts `track_times` parameter. Parameter is passed to ``create_table`` method of ``PyTables`` (:issue:`32682`).
+- Make :class:`pandas.core.window.Rolling` and :class:`pandas.core.window.Expanding` iterable（:issue:`11704`)
 
 .. ---------------------------------------------------------------------------
 
@@ -641,6 +643,7 @@ Deprecations
 
 - :func:`pandas.api.types.is_categorical` is deprecated and will be removed in a future version; use `:func:pandas.api.types.is_categorical_dtype` instead (:issue:`33385`)
 - :meth:`Index.get_value` is deprecated and will be removed in a future version (:issue:`19728`)
+- :meth:`DateOffset.__call__` is deprecated and will be removed in a future version, use ``offset + other`` instead (:issue:`34171`)
 
 .. ---------------------------------------------------------------------------
 
@@ -661,6 +664,8 @@ Performance improvements
   sparse values from ``scipy.sparse`` matrices using the
   :meth:`DataFrame.sparse.from_spmatrix` constructor (:issue:`32821`,
   :issue:`32825`,  :issue:`32826`, :issue:`32856`, :issue:`32858`).
+- Performance improvement for groupby methods :meth:`~pandas.core.groupby.groupby.Groupby.first`
+  and :meth:`~pandas.core.groupby.groupby.Groupby.last` (:issue:`34178`)
 - Performance improvement in :func:`factorize` for nullable (integer and boolean) dtypes (:issue:`33064`).
 - Performance improvement in reductions (sum, prod, min, max) for nullable (integer and boolean) dtypes (:issue:`30982`, :issue:`33261`, :issue:`33442`).
 
@@ -869,6 +874,8 @@ Groupby/resample/rolling
 - Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`)
 - Bug in :meth:`Rolling.min` and :meth:`Rolling.max`: Growing memory usage after multiple calls when using a fixed window (:issue:`30726`)
 - Bug in :meth:`GroupBy.agg`, :meth:`GroupBy.transform`, and :meth:`GroupBy.resample` where subclasses are not preserved (:issue:`28330`)
+- Bug in :meth:`GroupBy.rolling.apply` ignores args and kwargs parameters (:issue:`33433`)
+
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/groupby.pyx

@@ -9,11 +9,9 @@ cimport numpy as cnp
 from numpy cimport (ndarray,
                     int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t,
                     uint32_t, uint64_t, float32_t, float64_t, complex64_t, complex128_t)
+from numpy.math cimport NAN
 cnp.import_array()
 
-cdef extern from "numpy/npy_math.h":
-    float64_t NAN "NPY_NAN"
-
 from pandas._libs.util cimport numeric, get_nat
 
 from pandas._libs.algos cimport (swap, TiebreakEnumType, TIEBREAK_AVERAGE,

pandas/_libs/hashtable.pyx

@@ -8,10 +8,9 @@ from libc.stdlib cimport malloc, free
 import numpy as np
 cimport numpy as cnp
 from numpy cimport ndarray, uint8_t, uint32_t, float64_t
+from numpy.math cimport NAN
 cnp.import_array()
 
-cdef extern from "numpy/npy_math.h":
-    float64_t NAN "NPY_NAN"
 
 from pandas._libs.khash cimport (
     khiter_t,
@@ -54,7 +53,7 @@ from pandas._libs.khash cimport (
 )
 
 
-cimport pandas._libs.util as util
+from pandas._libs cimport util
 
 from pandas._libs.missing cimport checknull
 

pandas/_libs/index.pyx

@@ -19,11 +19,10 @@ from numpy cimport (
 cnp.import_array()
 
 
-cimport pandas._libs.util as util
+from pandas._libs cimport util
 
-from pandas._libs.tslibs import Period, Timedelta
 from pandas._libs.tslibs.nattype cimport c_NaT as NaT
-from pandas._libs.tslibs.base cimport ABCTimestamp
+from pandas._libs.tslibs.base cimport ABCTimestamp, ABCTimedelta, ABCPeriod
 
 from pandas._libs.hashtable cimport HashTable
 
@@ -470,7 +469,7 @@ cdef class TimedeltaEngine(DatetimeEngine):
         return 'm8[ns]'
 
     cdef int64_t _unbox_scalar(self, scalar) except? -1:
-        if not (isinstance(scalar, Timedelta) or scalar is NaT):
+        if not (isinstance(scalar, ABCTimedelta) or scalar is NaT):
             raise TypeError(scalar)
         return scalar.value
 
@@ -480,7 +479,7 @@ cdef class PeriodEngine(Int64Engine):
     cdef int64_t _unbox_scalar(self, scalar) except? -1:
         if scalar is NaT:
             return scalar.value
-        if isinstance(scalar, Period):
+        if isinstance(scalar, ABCPeriod):
             # NB: we assume that we have the correct freq here.
             return scalar.ordinal
         raise TypeError(scalar)

pandas/_libs/internals.pyx

@@ -1,5 +1,6 @@
-import cython
 from collections import defaultdict
+
+import cython
 from cython import Py_ssize_t
 
 from cpython.slice cimport PySlice_GetIndicesEx

pandas/_libs/interval.pyx

@@ -33,7 +33,7 @@ from numpy cimport (
 cnp.import_array()
 
 
-cimport pandas._libs.util as util
+from pandas._libs cimport util
 
 from pandas._libs.hashtable cimport Int64Vector
 from pandas._libs.tslibs.util cimport (
@@ -42,8 +42,7 @@ from pandas._libs.tslibs.util cimport (
     is_timedelta64_object,
 )
 
-from pandas._libs.tslibs import Timestamp
-from pandas._libs.tslibs.timedeltas import Timedelta
+from pandas._libs.tslibs.base cimport ABCTimestamp, ABCTimedelta
 from pandas._libs.tslibs.timezones cimport tz_compare
 
 
@@ -329,7 +328,7 @@ cdef class Interval(IntervalMixin):
             raise ValueError(f"invalid option for 'closed': {closed}")
         if not left <= right:
             raise ValueError("left side of interval must be <= right side")
-        if (isinstance(left, Timestamp) and
+        if (isinstance(left, ABCTimestamp) and
                 not tz_compare(left.tzinfo, right.tzinfo)):
             # GH 18538
             raise ValueError("left and right must have the same time zone, got "
@@ -341,7 +340,7 @@ cdef class Interval(IntervalMixin):
     def _validate_endpoint(self, endpoint):
         # GH 23013
         if not (is_integer_object(endpoint) or is_float_object(endpoint) or
-                isinstance(endpoint, (Timestamp, Timedelta))):
+                isinstance(endpoint, (ABCTimestamp, ABCTimedelta))):
             raise ValueError("Only numeric, Timestamp and Timedelta endpoints "
                              "are allowed when constructing an Interval.")
 
@@ -371,7 +370,7 @@ cdef class Interval(IntervalMixin):
         right = self.right
 
         # TODO: need more general formatting methodology here
-        if isinstance(left, Timestamp) and isinstance(right, Timestamp):
+        if isinstance(left, ABCTimestamp) and isinstance(right, ABCTimestamp):
             left = left._short_repr
             right = right._short_repr
 

pandas/_libs/lib.pyx

@@ -1,6 +1,5 @@
 from collections import abc
 from decimal import Decimal
-
 import warnings
 
 import cython
@@ -63,7 +62,7 @@ cdef extern from "numpy/arrayobject.h":
 cdef extern from "src/parse_helper.h":
     int floatify(object, float64_t *result, int *maybe_int) except -1
 
-cimport pandas._libs.util as util
+from pandas._libs cimport util
 from pandas._libs.util cimport is_nan, UINT64_MAX, INT64_MAX, INT64_MIN
 
 from pandas._libs.tslib import array_to_datetime

pandas/_libs/missing.pyx

@@ -8,7 +8,7 @@ cimport numpy as cnp
 from numpy cimport ndarray, int64_t, uint8_t, float64_t
 cnp.import_array()
 
-cimport pandas._libs.util as util
+from pandas._libs cimport util
 
 
 from pandas._libs.tslibs.np_datetime cimport get_datetime64_value, get_timedelta64_value

pandas/_libs/parsers.pyx

@@ -34,7 +34,7 @@ cimport numpy as cnp
 from numpy cimport ndarray, uint8_t, uint64_t, int64_t, float64_t
 cnp.import_array()
 
-cimport pandas._libs.util as util
+from pandas._libs cimport util
 from pandas._libs.util cimport UINT64_MAX, INT64_MAX, INT64_MIN
 import pandas._libs.lib as lib
 

pandas/_libs/reduction.pyx

@@ -14,7 +14,7 @@ from numpy cimport (ndarray,
                     flatiter)
 cnp.import_array()
 
-cimport pandas._libs.util as util
+from pandas._libs cimport util
 from pandas._libs.lib import maybe_convert_objects, is_scalar
 
 
@@ -603,7 +603,7 @@ cdef class BlockSlider:
             arr.shape[1] = 0
 
 
-def compute_reduction(arr: np.ndarray, f, axis: int = 0, dummy=None, labels=None):
+def compute_reduction(arr: ndarray, f, axis: int = 0, dummy=None, labels=None):
     """
 
     Parameters

pandas/_libs/reshape.pyx

@@ -15,11 +15,12 @@ from numpy cimport (
     uint64_t,
 )
 
-cimport numpy as cnp
 import numpy as np
-from pandas._libs.lib cimport c_is_list_like
+cimport numpy as cnp
 cnp.import_array()
 
+from pandas._libs.lib cimport c_is_list_like
+
 ctypedef fused reshape_t:
     uint8_t
     uint16_t

pandas/_libs/testing.pyx

@@ -1,4 +1,8 @@
 import numpy as np
+from numpy cimport import_array
+import_array()
+
+from pandas._libs.util cimport is_array
 
 from pandas.core.dtypes.missing import isna, array_equivalent
 from pandas.core.dtypes.common import is_dtype_equal
@@ -116,8 +120,8 @@ cpdef assert_almost_equal(a, b,
         assert a == b, f"{a} != {b}"
         return True
 
-    a_is_ndarray = isinstance(a, np.ndarray)
-    b_is_ndarray = isinstance(b, np.ndarray)
+    a_is_ndarray = is_array(a)
+    b_is_ndarray = is_array(b)
 
     if obj is None:
         if a_is_ndarray or b_is_ndarray:

pandas/_libs/tslibs/__init__.py

@@ -13,7 +13,6 @@
     "ints_to_pytimedelta",
     "Timestamp",
     "tz_convert_single",
-    "NullFrequencyError",
 ]
 
 
@@ -22,5 +21,5 @@
 from .np_datetime import OutOfBoundsDatetime
 from .period import IncompatibleFrequency, Period
 from .timedeltas import Timedelta, delta_to_nanoseconds, ints_to_pytimedelta
-from .timestamps import NullFrequencyError, Timestamp
+from .timestamps import Timestamp
 from .tzconversion import tz_convert_single

pandas/_libs/tslibs/frequencies.pxd

@@ -1,7 +1,6 @@
 cpdef str get_rule_month(object source, str default=*)
 
 cpdef get_freq_code(freqstr)
-cpdef object get_freq(object freq)
 cpdef str get_base_alias(freqstr)
 cpdef int get_to_timestamp_base(int base)
 cpdef str get_freq_str(base, mult=*)

pandas/_libs/tslibs/frequencies.pyx

@@ -306,25 +306,6 @@ cpdef int get_to_timestamp_base(int base):
     return base
 
 
-cpdef object get_freq(object freq):
-    """
-    Return frequency code of given frequency str.
-    If input is not string, return input as it is.
-
-    Examples
-    --------
-    >>> get_freq('A')
-    1000
-
-    >>> get_freq('3A')
-    1000
-    """
-    if isinstance(freq, str):
-        base, mult = get_freq_code(freq)
-        freq = base
-    return freq
-
-
 # ----------------------------------------------------------------------
 # Frequency comparison
 

pandas/_libs/tslibs/np_datetime.pxd

@@ -53,8 +53,6 @@ cdef extern from "src/datetime/np_datetime.h":
                                            npy_datetimestruct *result) nogil
 
 
-cdef int reverse_ops[6]
-
 cdef bint cmp_scalar(int64_t lhs, int64_t rhs, int op) except -1
 
 cdef check_dts_bounds(npy_datetimestruct *dts)