BUG: DatetimeIndex.append does not preserve timezone #2260

Author: changhiskhan

pandas/io/tests/test_excel.py

@@ -490,7 +490,8 @@ def _check_extension_indexlabels(self, ext):
         os.remove(path)
 
     def test_excel_roundtrip_indexname(self):
-        _skip_if_no_excelsuite()
+        _skip_if_no_xlrd()
+        _skip_if_no_xlwt()
 
         path = '%s.xls' % tm.rands(10)
 
@@ -809,4 +810,3 @@ def test_to_excel_header_styling_xlsx(self):
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__,'-vvs','-x','--pdb', '--pdb-failure'],
                    exit=False)
-

pandas/tseries/index.py

@@ -605,37 +605,6 @@ def summary(self, name=None):
 
         return result
 
-    def append(self, other):
-        """
-        Append a collection of Index options together
-
-        Parameters
-        ----------
-        other : Index or list/tuple of indices
-
-        Returns
-        -------
-        appended : Index
-        """
-        name = self.name
-        to_concat = [self]
-
-        if isinstance(other, (list, tuple)):
-            to_concat = to_concat + list(other)
-        else:
-            to_concat.append(other)
-
-        for obj in to_concat:
-            if isinstance(obj, Index) and obj.name != name:
-                name = None
-                break
-
-        to_concat = self._ensure_compat_concat(to_concat)
-        to_concat = [x.values if isinstance(x, Index) else x
-                     for x in to_concat]
-
-        return Index(com._concat_compat(to_concat), name=name)
-
     def get_duplicates(self):
         values = Index.get_duplicates(self)
         return DatetimeIndex(values)
@@ -864,6 +833,36 @@ def union_many(self, others):
             this.offset = to_offset(this.inferred_freq)
         return this
 
+    def append(self, other):
+        """
+        Append a collection of Index options together
+
+        Parameters
+        ----------
+        other : Index or list/tuple of indices
+
+        Returns
+        -------
+        appended : Index
+        """
+        name = self.name
+        to_concat = [self]
+
+        if isinstance(other, (list, tuple)):
+            to_concat = to_concat + list(other)
+        else:
+            to_concat.append(other)
+
+        for obj in to_concat:
+            if isinstance(obj, Index) and obj.name != name:
+                name = None
+                break
+
+        to_concat = self._ensure_compat_concat(to_concat)
+        to_concat, factory = _process_concat_data(to_concat, name)
+
+        return factory(com._concat_compat(to_concat))
+
     def join(self, other, how='left', level=None, return_indexers=False):
         """
         See Index.join
@@ -1633,3 +1632,33 @@ def _in_range(start, end, rng_start, rng_end):
 def _time_to_micros(time):
     seconds = time.hour * 60 * 60 + 60 * time.minute + time.second
     return 1000000 * seconds + time.microsecond
+
+def _process_concat_data(to_concat, name):
+    klass = Index
+    kwargs = {}
+
+    all_dti = True
+    need_utc_convert = False
+    tz = None
+    for x in to_concat:
+        if not isinstance(x, DatetimeIndex):
+            all_dti = False
+        else:
+            if tz is None:
+                tz = x.tz
+            elif x.tz != tz:
+                need_utc_convert = True
+                tz = 'UTC'
+
+    if need_utc_convert:
+        to_concat = [x.tz_convert('UTC') for x in to_concat]
+
+    if all_dti:
+        klass = DatetimeIndex
+        kwargs = {'tz' : tz}
+
+    to_concat = [x.values if isinstance(x, Index) else x
+                 for x in to_concat]
+
+    factory_func = lambda x: klass(x, name=name, **kwargs)
+    return to_concat, factory_func

pandas/tseries/tests/test_timezones.py

@@ -668,6 +668,35 @@ def test_align_aware(self):
         self.assertEqual(df1.index.tz, new1.index.tz)
         self.assertEqual(df2.index.tz, new2.index.tz)
 
+    def test_append_aware(self):
+        rng1 = date_range('1/1/2011 01:00', periods=1, freq='H',
+                          tz='US/Eastern')
+        rng2 = date_range('1/1/2011 02:00', periods=1, freq='H',
+                          tz='US/Eastern')
+        ts1 = Series(np.random.randn(len(rng1)), index=rng1)
+        ts2 = Series(np.random.randn(len(rng2)), index=rng2)
+        ts_result = ts1.append(ts2)
+        self.assertEqual(ts_result.index.tz, rng1.tz)
+
+        rng1 = date_range('1/1/2011 01:00', periods=1, freq='H',
+                          tz='UTC')
+        rng2 = date_range('1/1/2011 02:00', periods=1, freq='H',
+                          tz='UTC')
+        ts1 = Series(np.random.randn(len(rng1)), index=rng1)
+        ts2 = Series(np.random.randn(len(rng2)), index=rng2)
+        ts_result = ts1.append(ts2)
+        utc = rng1.tz
+        self.assertEqual(utc, ts_result.index.tz)
+
+        rng1 = date_range('1/1/2011 01:00', periods=1, freq='H',
+                          tz='US/Eastern')
+        rng2 = date_range('1/1/2011 02:00', periods=1, freq='H',
+                          tz='US/Central')
+        ts1 = Series(np.random.randn(len(rng1)), index=rng1)
+        ts2 = Series(np.random.randn(len(rng2)), index=rng2)
+        ts_result = ts1.append(ts2)
+        self.assertEqual(utc, ts_result.index.tz)
+
     def test_equal_join_ensure_utc(self):
         rng = date_range('1/1/2011', periods=10, freq='H', tz='US/Eastern')
         ts = Series(np.random.randn(len(rng)), index=rng)

vb_suite/timeseries.py

@@ -175,3 +175,16 @@ def date_range(start=None, end=None, periods=None, freq=None):
 datetimeindex_normalize = \
     Benchmark('rng.normalize()', setup,
               start_date=datetime(2012, 9, 1))
+
+setup = common_setup + """
+from pandas.tseries.offsets import Second
+s1 = date_range('1/1/2000', periods=100, freq='S')
+curr = s1[-1]
+slst = []
+for i in range(100):
+    slst.append(curr + Second(), periods=100, freq='S')
+    curr = slst[-1][-1]
+"""
+
+dti_append_tz = \
+    Benchmark('s1.append(slst)', setup, start_date=datetime(2012, 9 ,1))