BUG: transform with nunique should have dtype int64

GH35109

Author: rhshadrach

doc/source/whatsnew/v1.1.0.rst

@@ -1080,6 +1080,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.groupby` lost index, when one of the ``agg`` keys referenced an empty list (:issue:`32580`)
 - Bug in :meth:`Rolling.apply` where ``center=True`` was ignored when ``engine='numba'`` was specified (:issue:`34784`)
 - Bug in :meth:`DataFrame.ewm.cov` was throwing ``AssertionError`` for :class:`MultiIndex` inputs (:issue:`34440`)
+- Bug in :meth:`core.groupby.DataFrameGroupBy.transform` when ``func='nunique'`` and columns are of type ``datetime64``, the result would also be of type ``datetime64`` instead of ``int64`` (:issue:`35109`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/dtypes/cast.py

@@ -319,6 +319,9 @@ def maybe_cast_result_dtype(dtype: DtypeObj, how: str) -> DtypeObj:
         return np.dtype(np.int64)
     elif how in ["add", "cumsum", "sum"] and isinstance(dtype, BooleanDtype):
         return Int64Dtype()
+    elif how == "nunique":
+        return np.dtype(np.int64)
+
     return dtype
 
 

pandas/tests/groupby/test_nunique.py

@@ -167,3 +167,10 @@ def test_nunique_preserves_column_level_names():
     result = test.groupby([0, 0, 0]).nunique()
     expected = pd.DataFrame([2], columns=test.columns)
     tm.assert_frame_equal(result, expected)
+
+
+def test_nunique_transform_with_datetime():
+    df = pd.DataFrame(date_range("2008-12-31", "2009-01-02"), columns=["date"])
+    result = df.groupby([0, 0, 1])["date"].transform("nunique")
+    expected = pd.Series([2, 2, 1], name="date")
+    tm.assert_series_equal(result, expected)