BUG: agg() function on groupby dataframe changes dtype of datetime64[ns] column to float64

closes #12821
closes #12941

Author: 颜发才（Yan Facai） <[email protected]>

Closes #12992 from ningchi/agg_time_dtype and squashes the following commits:

607a170 [颜发才（Yan Facai）] add whatsnew entry
8d17eed [颜发才（Yan Facai）] BUG: fix GH12941, Operations on NaT returning float instead of datetime64[ns]
a949cee [颜发才（Yan Facai）] BUG: fix GH12821, agg() function on groupby dataframe changes dtype of datetime64[ns] column to float64
914ed71 [颜发才（Yan Facai）] add test function

Author: facaiy

.gitignore

@@ -18,6 +18,7 @@
 .vagrant
 .noseids
 .ipynb_checkpoints
+.tags
 
 # Compiled source #
 ###################

doc/source/whatsnew/v0.19.0.txt

@@ -960,5 +960,11 @@ Bug Fixes
 - Bug in ``Index.union`` returns an incorrect result with a named empty index (:issue:`13432`)
 - Bugs in ``Index.difference`` and ``DataFrame.join`` raise in Python3 when using mixed-integer indexes (:issue:`13432`, :issue:`12814`)
 - Bug in ``.to_excel()`` when DataFrame contains a MultiIndex which contains a label with a NaN value (:issue:`13511`)
+
+- Bug in ``agg()`` function on groupby dataframe changes dtype of ``datetime64[ns]`` column to ``float64`` (:issue:`12821`)
+
+- Bug in operations on ``NaT`` returning ``float`` instead of ``datetime64[ns]`` (:issue:`12941`)
+
 - Bug in ``pd.read_csv`` in Python 2.x with non-UTF8 encoded, multi-character separated data (:issue:`3404`)
+
 - Bug in ``Index`` raises ``KeyError`` displaying incorrect column when column is not in the df and columns contains duplicate values (:issue:`13822`)

pandas/core/nanops.py

@@ -17,6 +17,7 @@
                                  is_integer, is_complex, is_float_dtype,
                                  is_complex_dtype, is_integer_dtype,
                                  is_bool_dtype, is_object_dtype,
+                                 is_numeric_dtype,
                                  is_datetime64_dtype, is_timedelta64_dtype,
                                  is_datetime_or_timedelta_dtype,
                                  is_int_or_datetime_dtype, is_any_int_dtype)
@@ -638,11 +639,15 @@ def _maybe_null_out(result, axis, mask):
     if axis is not None and getattr(result, 'ndim', False):
         null_mask = (mask.shape[axis] - mask.sum(axis)) == 0
         if np.any(null_mask):
-            if np.iscomplexobj(result):
-                result = result.astype('c16')
+            if is_numeric_dtype(result):
+                if np.iscomplexobj(result):
+                    result = result.astype('c16')
+                else:
+                    result = result.astype('f8')
+                result[null_mask] = np.nan
             else:
-                result = result.astype('f8')
-            result[null_mask] = np.nan
+                # GH12941, use None to auto cast null
+                result[null_mask] = None
     elif result is not tslib.NaT:
         null_mask = mask.size - mask.sum()
         if null_mask == 0:

pandas/tests/frame/test_timeseries.py

@@ -341,3 +341,27 @@ def test_first_last_valid(self):
         empty = DataFrame()
         self.assertIsNone(empty.last_valid_index())
         self.assertIsNone(empty.first_valid_index())
+
+    def test_operation_on_NaT(self):
+        # Both NaT and Timestamp are in DataFrame.
+        df = pd.DataFrame({'foo': [pd.NaT, pd.NaT,
+                                   pd.Timestamp('2012-05-01')]})
+
+        res = df.min()
+        exp = pd.Series([pd.Timestamp('2012-05-01')], index=["foo"])
+        tm.assert_series_equal(res, exp)
+
+        res = df.max()
+        exp = pd.Series([pd.Timestamp('2012-05-01')], index=["foo"])
+        tm.assert_series_equal(res, exp)
+
+        # GH12941, only NaTs are in DataFrame.
+        df = pd.DataFrame({'foo': [pd.NaT, pd.NaT]})
+
+        res = df.min()
+        exp = pd.Series([pd.NaT], index=["foo"])
+        tm.assert_series_equal(res, exp)
+
+        res = df.max()
+        exp = pd.Series([pd.NaT], index=["foo"])
+        tm.assert_series_equal(res, exp)

pandas/tests/test_groupby.py

@@ -720,6 +720,32 @@ def test_agg_period_index(self):
         grouped = df.groupby(df.index.month)
         list(grouped)
 
+    def test_agg_dict_parameter_cast_result_dtypes(self):
+        # GH 12821
+
+        df = DataFrame(
+            {'class': ['A', 'A', 'B', 'B', 'C', 'C', 'D', 'D'],
+             'time': date_range('1/1/2011', periods=8, freq='H')})
+        df.loc[[0, 1, 2, 5], 'time'] = None
+
+        # test for `first` function
+        exp = df.loc[[0, 3, 4, 6]].set_index('class')
+        grouped = df.groupby('class')
+        assert_frame_equal(grouped.first(), exp)
+        assert_frame_equal(grouped.agg('first'), exp)
+        assert_frame_equal(grouped.agg({'time': 'first'}), exp)
+        assert_series_equal(grouped.time.first(), exp['time'])
+        assert_series_equal(grouped.time.agg('first'), exp['time'])
+
+        # test for `last` function
+        exp = df.loc[[0, 3, 4, 7]].set_index('class')
+        grouped = df.groupby('class')
+        assert_frame_equal(grouped.last(), exp)
+        assert_frame_equal(grouped.agg('last'), exp)
+        assert_frame_equal(grouped.agg({'time': 'last'}), exp)
+        assert_series_equal(grouped.time.last(), exp['time'])
+        assert_series_equal(grouped.time.agg('last'), exp['time'])
+
     def test_agg_must_agg(self):
         grouped = self.df.groupby('A')['C']
         self.assertRaises(Exception, grouped.agg, lambda x: x.describe())

pandas/types/cast.py

@@ -122,7 +122,8 @@ def trans(x):  # noqa
                         return new_result
 
         # a datetimelike
-        elif dtype.kind in ['M', 'm'] and result.dtype.kind in ['i']:
+        # GH12821, iNaT is casted to float
+        elif dtype.kind in ['M', 'm'] and result.dtype.kind in ['i', 'f']:
             try:
                 result = result.astype(dtype)
             except: