Merge branch 'test_branch'

merging with latest changes from master

Author: Ajay Saxena

ci/lint.sh

@@ -10,18 +10,24 @@ if [ "$LINT" ]; then
     # pandas/rpy is deprecated and will be removed.
     # pandas/src is C code, so no need to search there.
     echo "Linting  *.py"
-    flake8 pandas --filename '*.py' --exclude pandas/rpy,pandas/src
+    flake8 pandas --filename=*.py --exclude pandas/rpy,pandas/src
+    if [ $? -ne "0" ]; then
+        RET=1
+    fi
     echo "Linting *.py DONE"
 
     echo "Linting *.pyx"
-    flake8 pandas --filename '*.pyx' --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126
+    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126
+    if [ $? -ne "0" ]; then
+        RET=1
+    fi
     echo "Linting *.pyx DONE"
 
     echo "Linting *.pxi.in"
     for path in 'src'
     do
         echo "linting -> pandas/$path"
-        flake8 pandas/$path --filename '*.pxi.in' --select=E501,E302,E203,E111,E114,E221,E303,E231,E126
+        flake8 pandas/$path --filename=*.pxi.in --select=E501,E302,E203,E111,E114,E221,E303,E231,E126
         if [ $? -ne "0" ]; then
             RET=1
         fi

ci/requirements-3.5_OSX.pip

@@ -1 +1 @@
-python-dateutil>=2.5.0
+python-dateutil==2.5.3

doc/source/whatsnew.rst

@@ -18,6 +18,8 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.19.2.txt
+
 .. include:: whatsnew/v0.19.1.txt
 
 .. include:: whatsnew/v0.19.0.txt

doc/source/whatsnew/v0.19.2.txt

@@ -0,0 +1,27 @@
+.. _whatsnew_0192:
+
+v0.19.2 (December ??, 2016)
+---------------------------
+
+This is a minor bug-fix release from 0.19.1 and includes some small regression fixes,
+bug fixes and performance improvements.
+We recommend that all users upgrade to this version.
+
+.. contents:: What's new in v0.19.2
+    :local:
+    :backlinks: none
+
+
+.. _whatsnew_0192.performance:
+
+Performance Improvements
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+.. _whatsnew_0192.bug_fixes:
+
+Bug Fixes
+~~~~~~~~~
+
+- compat with ``dateutil==2.6.0`` for testing (:issue:`14621`)
+- allow ``nanoseconds`` in ``Timestamp.replace`` kwargs (:issue:`14621`)

pandas/core/generic.py

@@ -1066,7 +1066,7 @@ def to_json(self, path_or_buf=None, orient=None, date_format='epoch',
             Handler to call if object cannot otherwise be converted to a
             suitable format for JSON. Should receive a single argument which is
             the object to convert and return a serialisable object.
-        lines : boolean, defalut False
+        lines : boolean, default False
             If 'orient' is 'records' write out line delimited json format. Will
             throw ValueError if incorrect 'orient' since others are not list
             like.

pandas/core/series.py

@@ -1216,16 +1216,10 @@ def count(self, level=None):
                                  dtype='int64').__finalize__(self)
 
     def mode(self):
-        """Returns the mode(s) of the dataset.
+        """Return the mode(s) of the dataset.
 
-        Empty if nothing occurs at least 2 times.  Always returns Series even
-        if only one value.
-
-        Parameters
-        ----------
-        sort : bool, default True
-            If True, will lexicographically sort values, if False skips
-            sorting. Result ordering when ``sort=False`` is not defined.
+        Empty if nothing occurs at least 2 times. Always returns Series even
+        if only one value is returned.
 
         Returns
         -------

pandas/io/stata.py

@@ -1210,18 +1210,18 @@ def _read_old_header(self, first_char):
                 if tp in self.OLD_TYPE_MAPPING:
                     typlist.append(self.OLD_TYPE_MAPPING[tp])
                 else:
-                    typlist.append(tp - 127)  # string
+                    typlist.append(tp - 127)  # py2 string, py3 bytes
 
         try:
             self.typlist = [self.TYPE_MAP[typ] for typ in typlist]
         except:
             raise ValueError("cannot convert stata types [{0}]"
-                             .format(','.join(typlist)))
+                             .format(','.join(str(x) for x in typlist)))
         try:
             self.dtyplist = [self.DTYPE_MAP[typ] for typ in typlist]
         except:
             raise ValueError("cannot convert stata dtypes [{0}]"
-                             .format(','.join(typlist)))
+                             .format(','.join(str(x) for x in typlist)))
 
         if self.format_version > 108:
             self.varlist = [self._null_terminate(self.path_or_buf.read(33))

pandas/io/tests/test_gbq.py

@@ -824,6 +824,9 @@ def test_upload_data_if_table_exists_append(self):
                        private_key=_get_private_key_path())
 
     def test_upload_data_if_table_exists_replace(self):
+
+        raise nose.SkipTest("buggy test")
+
         destination_table = DESTINATION_TABLE + "4"
 
         test_size = 10

pandas/tests/indexes/test_base.py

@@ -913,7 +913,19 @@ def test_summary(self):
     def test_format(self):
         self._check_method_works(Index.format)
 
-        index = Index([datetime.now()])
+        # GH 14626
+        # our formatting is different by definition when we have
+        # ms vs us precision (e.g. trailing zeros);
+        # so don't compare this case
+        def datetime_now_without_trailing_zeros():
+            now = datetime.now()
+
+            while str(now).endswith("000"):
+                now = datetime.now()
+
+            return now
+
+        index = Index([datetime_now_without_trailing_zeros()])
 
         # windows has different precision on datetime.datetime.now (it doesn't
         # include us since the default for Timestamp shows these but Index

pandas/tseries/offsets.py

@@ -68,6 +68,7 @@ def wrapper(self, other):
                 other = other.tz_localize(None)
 
             result = func(self, other)
+
             if self._adjust_dst:
                 result = tslib._localize_pydatetime(result, tz)
 

pandas/tseries/tests/test_offsets.py

@@ -1,4 +1,5 @@
 import os
+from distutils.version import LooseVersion
 from datetime import date, datetime, timedelta
 from dateutil.relativedelta import relativedelta
 from pandas.compat import range, iteritems
@@ -4851,6 +4852,7 @@ def _test_all_offsets(self, n, **kwds):
 
     def _test_offset(self, offset_name, offset_n, tstart, expected_utc_offset):
         offset = DateOffset(**{offset_name: offset_n})
+
         t = tstart + offset
         if expected_utc_offset is not None:
             self.assertTrue(get_utc_offset_hours(t) == expected_utc_offset)
@@ -4890,17 +4892,23 @@ def _make_timestamp(self, string, hrs_offset, tz):
         return Timestamp(string + offset_string).tz_convert(tz)
 
     def test_fallback_plural(self):
-        """test moving from daylight savings to standard time"""
+        # test moving from daylight savings to standard time
+        import dateutil
         for tz, utc_offsets in self.timezone_utc_offsets.items():
             hrs_pre = utc_offsets['utc_offset_daylight']
             hrs_post = utc_offsets['utc_offset_standard']
-            self._test_all_offsets(
-                n=3, tstart=self._make_timestamp(self.ts_pre_fallback,
-                                                 hrs_pre, tz),
-                expected_utc_offset=hrs_post)
+
+            if dateutil.__version__ != LooseVersion('2.6.0'):
+                # buggy ambiguous behavior in 2.6.0
+                # GH 14621
+                # https://github.com/dateutil/dateutil/issues/321
+                self._test_all_offsets(
+                    n=3, tstart=self._make_timestamp(self.ts_pre_fallback,
+                                                     hrs_pre, tz),
+                    expected_utc_offset=hrs_post)
 
     def test_springforward_plural(self):
-        """test moving from standard to daylight savings"""
+        # test moving from standard to daylight savings
         for tz, utc_offsets in self.timezone_utc_offsets.items():
             hrs_pre = utc_offsets['utc_offset_standard']
             hrs_post = utc_offsets['utc_offset_daylight']

pandas/tseries/tests/test_timezones.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 import pytz
-
+from distutils.version import LooseVersion
 from pandas.types.dtypes import DatetimeTZDtype
 from pandas import (Index, Series, DataFrame, isnull, Timestamp)
 
@@ -518,8 +518,12 @@ def f():
 
         times = date_range("2013-10-26 23:00", "2013-10-27 01:00", freq="H",
                            tz=tz, ambiguous='infer')
-        self.assertEqual(times[0], Timestamp('2013-10-26 23:00', tz=tz))
-        self.assertEqual(times[-1], Timestamp('2013-10-27 01:00', tz=tz))
+        self.assertEqual(times[0], Timestamp('2013-10-26 23:00', tz=tz,
+                                             freq="H"))
+        if dateutil.__version__ != LooseVersion('2.6.0'):
+            # GH 14621
+            self.assertEqual(times[-1], Timestamp('2013-10-27 01:00', tz=tz,
+                                                  freq="H"))
 
     def test_ambiguous_nat(self):
         tz = self.tz('US/Eastern')
@@ -1163,6 +1167,85 @@ class TestTimeZones(tm.TestCase):
     def setUp(self):
         tm._skip_if_no_pytz()
 
+    def test_replace(self):
+        # GH 14621
+        # GH 7825
+        # replacing datetime components with and w/o presence of a timezone
+        dt = Timestamp('2016-01-01 09:00:00')
+        result = dt.replace(hour=0)
+        expected = Timestamp('2016-01-01 00:00:00')
+        self.assertEqual(result, expected)
+
+        for tz in self.timezones:
+            dt = Timestamp('2016-01-01 09:00:00', tz=tz)
+            result = dt.replace(hour=0)
+            expected = Timestamp('2016-01-01 00:00:00', tz=tz)
+            self.assertEqual(result, expected)
+
+        # we preserve nanoseconds
+        dt = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
+        result = dt.replace(hour=0)
+        expected = Timestamp('2016-01-01 00:00:00.000000123', tz=tz)
+        self.assertEqual(result, expected)
+
+        # test all
+        dt = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
+        result = dt.replace(year=2015, month=2, day=2, hour=0, minute=5,
+                            second=5, microsecond=5, nanosecond=5)
+        expected = Timestamp('2015-02-02 00:05:05.000005005', tz=tz)
+        self.assertEqual(result, expected)
+
+        # error
+        def f():
+            dt.replace(foo=5)
+        self.assertRaises(ValueError, f)
+
+        def f():
+            dt.replace(hour=0.1)
+        self.assertRaises(ValueError, f)
+
+        # assert conversion to naive is the same as replacing tzinfo with None
+        dt = Timestamp('2013-11-03 01:59:59.999999-0400', tz='US/Eastern')
+        self.assertEqual(dt.tz_localize(None), dt.replace(tzinfo=None))
+
+    def test_ambiguous_compat(self):
+        # validate that pytz and dateutil are compat for dst
+        # when the transition happens
+        tm._skip_if_no_dateutil()
+        tm._skip_if_no_pytz()
+
+        pytz_zone = 'Europe/London'
+        dateutil_zone = 'dateutil/Europe/London'
+        result_pytz = (Timestamp('2013-10-27 01:00:00')
+                       .tz_localize(pytz_zone, ambiguous=0))
+        result_dateutil = (Timestamp('2013-10-27 01:00:00')
+                           .tz_localize(dateutil_zone, ambiguous=0))
+        self.assertEqual(result_pytz.value, result_dateutil.value)
+        self.assertEqual(result_pytz.value, 1382835600000000000)
+
+        # dateutil 2.6 buggy w.r.t. ambiguous=0
+        if dateutil.__version__ != LooseVersion('2.6.0'):
+            # GH 14621
+            # https://github.com/dateutil/dateutil/issues/321
+            self.assertEqual(result_pytz.to_pydatetime().tzname(),
+                             result_dateutil.to_pydatetime().tzname())
+            self.assertEqual(str(result_pytz), str(result_dateutil))
+
+        # 1 hour difference
+        result_pytz = (Timestamp('2013-10-27 01:00:00')
+                       .tz_localize(pytz_zone, ambiguous=1))
+        result_dateutil = (Timestamp('2013-10-27 01:00:00')
+                           .tz_localize(dateutil_zone, ambiguous=1))
+        self.assertEqual(result_pytz.value, result_dateutil.value)
+        self.assertEqual(result_pytz.value, 1382832000000000000)
+
+        # dateutil < 2.6 is buggy w.r.t. ambiguous timezones
+        if dateutil.__version__ > LooseVersion('2.5.3'):
+            # GH 14621
+            self.assertEqual(str(result_pytz), str(result_dateutil))
+            self.assertEqual(result_pytz.to_pydatetime().tzname(),
+                             result_dateutil.to_pydatetime().tzname())
+
     def test_index_equals_with_tz(self):
         left = date_range('1/1/2011', periods=100, freq='H', tz='utc')
         right = date_range('1/1/2011', periods=100, freq='H', tz='US/Eastern')

pandas/tseries/tests/test_tslib.py

@@ -327,8 +327,9 @@ def test_repr(self):
 
         # dateutil zone change (only matters for repr)
         import dateutil
-        if dateutil.__version__ >= LooseVersion(
-                '2.3') and dateutil.__version__ <= LooseVersion('2.4.0'):
+        if (dateutil.__version__ >= LooseVersion('2.3') and
+            (dateutil.__version__ <= LooseVersion('2.4.0') or
+             dateutil.__version__ >= LooseVersion('2.6.0'))):
             timezones = ['UTC', 'Asia/Tokyo', 'US/Eastern',
                          'dateutil/US/Pacific']
         else: