Merge branch 'main' of https://github.com/pandas-dev/pandas into bug#60583

Author: Anurag-Varma

asv_bench/benchmarks/rolling.py

@@ -10,7 +10,19 @@ class Methods:
         ["DataFrame", "Series"],
         [("rolling", {"window": 10}), ("rolling", {"window": 1000}), ("expanding", {})],
         ["int", "float"],
-        ["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum", "sem"],
+        [
+            "median",
+            "mean",
+            "max",
+            "min",
+            "std",
+            "count",
+            "skew",
+            "kurt",
+            "sum",
+            "sem",
+            "nunique",
+        ],
     )
     param_names = ["constructor", "window_kwargs", "dtype", "method"]
 

ci/code_checks.sh

@@ -83,6 +83,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.resample.Resampler.quantile PR01,PR07" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.n GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.normalize GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.rule_code GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.startingMonth GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.n GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.normalize GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.rule_code GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.n GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.normalize GL08" \
@@ -185,6 +195,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.FY5253Quarter.variation GL08" \
         -i "pandas.tseries.offsets.FY5253Quarter.weekday GL08" \
         -i "pandas.tseries.offsets.FY5253Quarter.year_has_extra_week GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.n GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.normalize GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.rule_code GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.startingMonth GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.n GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.normalize GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.rule_code GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.Hour.is_on_offset GL08" \
         -i "pandas.tseries.offsets.Hour.n GL08" \
         -i "pandas.tseries.offsets.Hour.normalize GL08" \

doc/source/reference/offset_frequency.rst

@@ -776,6 +776,146 @@ Methods
     QuarterBegin.is_year_start
     QuarterBegin.is_year_end
 
+BHalfYearEnd
+------------
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd.freqstr
+    BHalfYearEnd.kwds
+    BHalfYearEnd.name
+    BHalfYearEnd.nanos
+    BHalfYearEnd.normalize
+    BHalfYearEnd.rule_code
+    BHalfYearEnd.n
+    BHalfYearEnd.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd.copy
+    BHalfYearEnd.is_on_offset
+    BHalfYearEnd.is_month_start
+    BHalfYearEnd.is_month_end
+    BHalfYearEnd.is_quarter_start
+    BHalfYearEnd.is_quarter_end
+    BHalfYearEnd.is_year_start
+    BHalfYearEnd.is_year_end
+
+BHalfYearBegin
+--------------
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin.freqstr
+    BHalfYearBegin.kwds
+    BHalfYearBegin.name
+    BHalfYearBegin.nanos
+    BHalfYearBegin.normalize
+    BHalfYearBegin.rule_code
+    BHalfYearBegin.n
+    BHalfYearBegin.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin.copy
+    BHalfYearBegin.is_on_offset
+    BHalfYearBegin.is_month_start
+    BHalfYearBegin.is_month_end
+    BHalfYearBegin.is_quarter_start
+    BHalfYearBegin.is_quarter_end
+    BHalfYearBegin.is_year_start
+    BHalfYearBegin.is_year_end
+
+HalfYearEnd
+-----------
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd.freqstr
+    HalfYearEnd.kwds
+    HalfYearEnd.name
+    HalfYearEnd.nanos
+    HalfYearEnd.normalize
+    HalfYearEnd.rule_code
+    HalfYearEnd.n
+    HalfYearEnd.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd.copy
+    HalfYearEnd.is_on_offset
+    HalfYearEnd.is_month_start
+    HalfYearEnd.is_month_end
+    HalfYearEnd.is_quarter_start
+    HalfYearEnd.is_quarter_end
+    HalfYearEnd.is_year_start
+    HalfYearEnd.is_year_end
+
+HalfYearBegin
+-------------
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin.freqstr
+    HalfYearBegin.kwds
+    HalfYearBegin.name
+    HalfYearBegin.nanos
+    HalfYearBegin.normalize
+    HalfYearBegin.rule_code
+    HalfYearBegin.n
+    HalfYearBegin.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin.copy
+    HalfYearBegin.is_on_offset
+    HalfYearBegin.is_month_start
+    HalfYearBegin.is_month_end
+    HalfYearBegin.is_quarter_start
+    HalfYearBegin.is_quarter_end
+    HalfYearBegin.is_year_start
+    HalfYearBegin.is_year_end
+
 BYearEnd
 --------
 .. autosummary::

doc/source/reference/window.rst

@@ -42,6 +42,7 @@ Rolling window functions
    Rolling.quantile
    Rolling.sem
    Rolling.rank
+   Rolling.nunique
 
 .. _api.functions_window:
 
@@ -86,6 +87,7 @@ Expanding window functions
    Expanding.quantile
    Expanding.sem
    Expanding.rank
+   Expanding.nunique
 
 .. _api.functions_ewm:
 

doc/source/user_guide/timeseries.rst

@@ -891,6 +891,10 @@ into ``freq`` keyword arguments. The available date offsets and associated frequ
     :class:`~pandas.tseries.offsets.BQuarterEnd`, ``'BQE``, "business quarter end"
     :class:`~pandas.tseries.offsets.BQuarterBegin`, ``'BQS'``, "business quarter begin"
     :class:`~pandas.tseries.offsets.FY5253Quarter`, ``'REQ'``, "retail (aka 52-53 week) quarter"
+    :class:`~pandas.tseries.offsets.HalfYearEnd`, ``'HYE'``, "calendar half year end"
+    :class:`~pandas.tseries.offsets.HalfYearBegin`, ``'HYS'``, "calendar half year begin"
+    :class:`~pandas.tseries.offsets.BHalfYearEnd`, ``'BHYE``, "business half year end"
+    :class:`~pandas.tseries.offsets.BHalfYearBegin`, ``'BHYS'``, "business half year begin"
     :class:`~pandas.tseries.offsets.YearEnd`, ``'YE'``, "calendar year end"
     :class:`~pandas.tseries.offsets.YearBegin`, ``'YS'`` or ``'BYS'``,"calendar year begin"
     :class:`~pandas.tseries.offsets.BYearEnd`, ``'BYE'``, "business year end"

doc/source/whatsnew/v2.3.0.rst

@@ -118,6 +118,7 @@ Conversion
 
 Strings
 ^^^^^^^
+- Bug in :meth:`DataFrame.sum` with ``axis=1``, :meth:`.DataFrameGroupBy.sum` or :meth:`.SeriesGroupBy.sum` with ``skipna=True``, and :meth:`.Resampler.sum` on :class:`StringDtype` with all NA values resulted in ``0`` and is now the empty string ``""`` (:issue:`60229`)
 - Bug in :meth:`Series.__pos__` and :meth:`DataFrame.__pos__` did not raise for :class:`StringDtype` with ``storage="pyarrow"`` (:issue:`60710`)
 - Bug in :meth:`Series.rank` for :class:`StringDtype` with ``storage="pyarrow"`` incorrectly returning integer results in case of ``method="average"`` and raising an error if it would truncate results (:issue:`59768`)
 - Bug in :meth:`Series.replace` with :class:`StringDtype` when replacing with a non-string value was not upcasting to ``object`` dtype (:issue:`60282`)

doc/source/whatsnew/v3.0.0.rst

@@ -62,6 +62,7 @@ Other enhancements
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
 - :meth:`DataFrame.plot.scatter` argument ``c`` now accepts a column of strings, where rows with the same string are colored identically (:issue:`16827` and :issue:`16485`)
 - :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` methods ``sum``, ``mean``, ``median``, ``prod``, ``min``, ``max``, ``std``, ``var`` and ``sem`` now accept ``skipna`` parameter (:issue:`15675`)
+- :class:`Rolling` and :class:`Expanding` now support ``nunique`` (:issue:`26958`)
 - :class:`Rolling` and :class:`Expanding` now support aggregations ``first`` and ``last`` (:issue:`33155`)
 - :func:`read_parquet` accepts ``to_pandas_kwargs`` which are forwarded to :meth:`pyarrow.Table.to_pandas` which enables passing additional keywords to customize the conversion to pandas, such as ``maps_as_pydicts`` to read the Parquet map data type as python dictionaries (:issue:`56842`)
 - :meth:`.DataFrameGroupBy.transform`, :meth:`.SeriesGroupBy.transform`, :meth:`.DataFrameGroupBy.agg`, :meth:`.SeriesGroupBy.agg`, :meth:`.SeriesGroupBy.apply`, :meth:`.DataFrameGroupBy.apply` now support ``kurt`` (:issue:`40139`)

pandas/_libs/window/aggregations.pyi

@@ -89,6 +89,12 @@ def roll_rank(
     method: WindowingRankType,
     ascending: bool,
 ) -> np.ndarray: ...  # np.ndarray[float]
+def roll_nunique(
+    values: np.ndarray,  # const float64_t[:]
+    start: np.ndarray,  # np.ndarray[np.int64]
+    end: np.ndarray,  # np.ndarray[np.int64]
+    minp: int,  # int64_t
+) -> np.ndarray: ...  # np.ndarray[float]
 def roll_apply(
     obj: object,
     start: np.ndarray,  # np.ndarray[np.int64]

pandas/_libs/window/aggregations.pyx

@@ -6,6 +6,7 @@ from libc.math cimport (
     sqrt,
 )
 from libcpp.deque cimport deque
+from libcpp.unordered_map cimport unordered_map
 
 from pandas._libs.algos cimport TiebreakEnumType
 
@@ -1470,6 +1471,66 @@ def roll_rank(const float64_t[:] values, ndarray[int64_t] start,
     return np.asarray(output)
 
 
+def roll_nunique(const float64_t[:] values, ndarray[int64_t] start,
+                 ndarray[int64_t] end, int64_t minp) -> np.ndarray:
+    """
+    Rolling number of unique elements in the window
+    """
+    cdef:
+        Py_ssize_t i, j, s, e, N = len(start)
+        int64_t nobs = 0
+        float64_t val
+        float64_t[::1] output
+        unordered_map[float64_t, int64_t] value_counts
+
+    is_monotonic_increasing_bounds = is_monotonic_increasing_start_end_bounds(
+        start, end
+    )
+    output = np.empty(N, dtype=np.float64)
+    value_counts = unordered_map[float64_t, int64_t]()
+
+    with nogil:
+        for i in range(N):
+            s = start[i]
+            e = end[i]
+
+            if i == 0 or not is_monotonic_increasing_bounds or s >= end[i - 1]:
+                if i != 0:
+                    nobs = 0
+                    value_counts.clear()
+
+                # setup
+                for j in range(s, e):
+                    val = values[j]
+                    if val == val:
+                        nobs += 1
+                        value_counts[val] += 1
+
+            else:
+                # calculate deletes
+                for j in range(start[i - 1], s):
+                    val = values[j]
+                    if val == val:
+                        value_counts[val] -= 1
+                        if value_counts[val] == 0:
+                            value_counts.erase(val)
+                        nobs -= 1
+
+                # calculate adds
+                for j in range(end[i - 1], e):
+                    val = values[j]
+                    if val == val:
+                        nobs += 1
+                        value_counts[val] += 1
+
+            if nobs >= minp:
+                output[i] = value_counts.size()
+            else:
+                output[i] = NaN
+
+    return np.asarray(output)
+
+
 def roll_apply(object obj,
                ndarray[int64_t] start, ndarray[int64_t] end,
                int64_t minp,

pandas/core/arrays/base.py

@@ -2628,7 +2628,15 @@ def _groupby_op(
             if op.how not in ["any", "all"]:
                 # Fail early to avoid conversion to object
                 op._get_cython_function(op.kind, op.how, np.dtype(object), False)
-            npvalues = self.to_numpy(object, na_value=np.nan)
+
+            arr = self
+            if op.how == "sum":
+                # https://github.com/pandas-dev/pandas/issues/60229
+                # All NA should result in the empty string.
+                assert "skipna" in kwargs
+                if kwargs["skipna"] and min_count == 0:
+                    arr = arr.fillna("")
+            npvalues = arr.to_numpy(object, na_value=np.nan)
         else:
             raise NotImplementedError(
                 f"function is not implemented for this dtype: {self.dtype}"

pandas/core/window/expanding.py

@@ -927,6 +927,41 @@ def rank(
             numeric_only=numeric_only,
         )
 
+    @doc(
+        template_header,
+        ".. versionadded:: 3.0.0 \n\n",
+        create_section_header("Parameters"),
+        kwargs_numeric_only,
+        create_section_header("Returns"),
+        template_returns,
+        create_section_header("See Also"),
+        template_see_also,
+        create_section_header("Examples"),
+        dedent(
+            """
+        >>> s = pd.Series([1, 4, 2, 3, 5, 3])
+        >>> s.expanding().nunique()
+        0    1.0
+        1    2.0
+        2    3.0
+        3    4.0
+        4    5.0
+        5    5.0
+        dtype: float64
+        """
+        ).replace("\n", "", 1),
+        window_method="expanding",
+        aggregation_description="nunique",
+        agg_method="nunique",
+    )
+    def nunique(
+        self,
+        numeric_only: bool = False,
+    ):
+        return super().nunique(
+            numeric_only=numeric_only,
+        )
+
     @doc(
         template_header,
         create_section_header("Parameters"),