diff --git a/pandas/core/internals.py b/pandas/core/internals.py index abfc5c989056e..188b00348b4e1 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -4905,8 +4905,12 @@ def get_reindexed_values(self, empty_dtype, upcasted_na): if len(values) and values[0] is None: fill_value = None - if getattr(self.block, 'is_datetimetz', False): - pass + if getattr(self.block, 'is_datetimetz', False) \ + or com.is_datetimetz(empty_dtype): + # handle timezone-aware all NaT cases + num_elements = np.prod(self.shape) + return DatetimeIndex([fill_value] * num_elements, + dtype=empty_dtype) elif getattr(self.block, 'is_categorical', False): pass elif getattr(self.block, 'is_sparse', False): diff --git a/pandas/tools/tests/test_merge.py b/pandas/tools/tests/test_merge.py index 13f00afb5a489..a44db54e36f38 100644 --- a/pandas/tools/tests/test_merge.py +++ b/pandas/tools/tests/test_merge.py @@ -12,6 +12,7 @@ from pandas.compat import range, lrange, lzip, StringIO from pandas import compat from pandas.tseries.index import DatetimeIndex +from pandas.types.dtypes import DatetimeTZDtype from pandas.tools.merge import merge, concat, ordered_merge, MergeError from pandas import Categorical, Timestamp from pandas.util.testing import (assert_frame_equal, assert_series_equal, @@ -2522,6 +2523,112 @@ def test_concat_multiindex_with_tz(self): result = concat([df, df]) tm.assert_frame_equal(result, expected) + def test_concat_NaT_dataframes_all_NaT_axis_0(self): + # GH 12396 + expect = pd.DataFrame([pd.NaT, pd.NaT, pd.NaT], index=[0, 1, 0]) + + # non-timezone aware + first = pd.DataFrame([[pd.NaT], [pd.NaT]]) + second = pd.DataFrame([[pd.NaT]]) + + result = pd.concat([first, second], axis=0) + assert_frame_equal(result, expect) + + # one side timezone-aware + dtype = DatetimeTZDtype('ns', tz='UTC') + first = pd.DataFrame([[pd.NaT], [pd.NaT]], dtype=dtype) + + result = pd.concat([first, second], axis=0) + # upcasts for mixed case + expect = expect.apply(lambda x: x.astype(dtype)) + assert_frame_equal(result, expect) + + # both sides timezone-aware + second = pd.DataFrame([[pd.NaT]], dtype=dtype) + + # upcasts to tz-aware + result = pd.concat([first, second], axis=0) + assert_frame_equal(result, expect) + + def test_concat_NaT_dataframes_all_NaT_axis_1(self): + # GH 12396 + expect = pd.DataFrame([[pd.NaT, pd.NaT], [pd.NaT, pd.NaT]], + columns=[0, 1]) + + # non-timezone aware + first = pd.DataFrame([[pd.NaT], [pd.NaT]]) + second = pd.DataFrame([[pd.NaT]], columns=[1]) + + result = pd.concat([first, second], axis=1) + assert_frame_equal(result, expect) + + # one side timezone-aware + dtype = DatetimeTZDtype('ns', tz='UTC') + first = pd.DataFrame([[pd.NaT], [pd.NaT]], dtype=dtype) + + # upcasts result to tz-aware + expect.loc[:, 0] = expect.loc[:, 0].astype(dtype) + result = pd.concat([first, second], axis=1) + assert_frame_equal(result, expect) + + # both sides timezone-aware + second = pd.DataFrame([[pd.NaT]], dtype=dtype, columns=[1]) + + # upcasts to tz-aware + expect = expect.apply(lambda x: x.astype(dtype)) + result = pd.concat([first, second], axis=1) + assert_frame_equal(result, expect) + + def test_concat_NaT_dataframes_mixed_timestamps_and_NaT(self): + # GH 12396 + + # non-timezone aware + first = pd.DataFrame([[pd.NaT], [pd.NaT]]) + second = pd.DataFrame([[pd.Timestamp('2015/01/01')], + [pd.Timestamp('2016/01/01')]]) + + expect = pd.DataFrame([pd.NaT, pd.NaT, second.iloc[0, 0], + second.iloc[1, 0]], index=[0, 1, 0, 1]) + + result = pd.concat([first, second], axis=0) + assert_frame_equal(result, expect) + + # one side timezone-aware + dtype = DatetimeTZDtype('ns', tz='UTC') + second = second.apply(lambda x: x.astype(dtype)) + + result = pd.concat([first, second], axis=0) + expect = expect.apply(lambda x: x.astype(dtype)) + assert_frame_equal(result, expect) + + def test_concat_NaT_series_dataframe_all_NaT(self): + # GH 12396 + + # non-timezone aware + first = pd.Series([pd.NaT, pd.NaT]) + second = pd.DataFrame([[pd.Timestamp('2015/01/01')], + [pd.Timestamp('2016/01/01')]]) + + expect = pd.DataFrame([pd.NaT, pd.NaT, second.iloc[0, 0], + second.iloc[1, 0]], index=[0, 1, 0, 1]) + + result = pd.concat([first, second]) + assert_frame_equal(result, expect) + + # one side timezone-aware + dtype = DatetimeTZDtype('ns', tz='UTC') + second = second.apply(lambda x: x.astype(dtype)) + + result = pd.concat([first, second]) + + expect = expect.apply(lambda x: x.astype(dtype)) + assert_frame_equal(result, expect) + + # both sides timezone-aware + first = first.astype(dtype) + result = pd.concat([first, second]) + assert_frame_equal(result, expect) + def test_concat_keys_and_levels(self): df = DataFrame(np.random.randn(1, 3)) df2 = DataFrame(np.random.randn(1, 4))