From b7358de23ba0b1a88327e4df550e1783764b1192 Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 09:01:28 -0800 Subject: [PATCH 01/10] bug fixed --- pandas/types/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/types/common.py b/pandas/types/common.py index 754ff80924c07..7eb4a08619af0 100644 --- a/pandas/types/common.py +++ b/pandas/types/common.py @@ -22,9 +22,11 @@ _NS_DTYPE = np.dtype('M8[ns]') _TD_DTYPE = np.dtype('m8[ns]') _INT64_DTYPE = np.dtype(np.int64) +_NS_DTYPE_UTC = DatetimeTZDtype.__new__(DatetimeTZDtype, unit='ns', tz='UTC') + _DATELIKE_DTYPES = set([np.dtype(t) for t in ['M8[ns]', 'M8[ns]', - 'm8[ns]', 'm8[ns]']]) + 'm8[ns]', 'm8[ns]']] + [_NS_DTYPE_UTC]) _ensure_float64 = algos.ensure_float64 _ensure_float32 = algos.ensure_float32 From ba83fc85ab207780d232d6533bde9221d93245e1 Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 10:08:57 -0800 Subject: [PATCH 02/10] test --- test_dtype_utc.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 test_dtype_utc.py diff --git a/test_dtype_utc.py b/test_dtype_utc.py new file mode 100644 index 0000000000000..733c759f067bb --- /dev/null +++ b/test_dtype_utc.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Sat Dec 17 09:34:47 2016 + +@author: rodolfoxps +""" + +import pandas as pd +import datetime, pytz +from pandas.util.testing import assert_frame_equal + +def test_dtype_utc(self): + + data=pd.Series( [pd.NaT, pd.NaT, datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ] ) + + filled=data.fillna(method='bfill') + + + expected=pd.Series([datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , + datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , + datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ]) + + assert_frame_equal(filled, expected) \ No newline at end of file From bc68bf7a3ea2b575377fee11e52d1157b28dc51b Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 10:14:25 -0800 Subject: [PATCH 03/10] test modified --- test_dtype_utc.py => pandas/tests/test_dtype_utc.py | 1 - 1 file changed, 1 deletion(-) rename test_dtype_utc.py => pandas/tests/test_dtype_utc.py (99%) diff --git a/test_dtype_utc.py b/pandas/tests/test_dtype_utc.py similarity index 99% rename from test_dtype_utc.py rename to pandas/tests/test_dtype_utc.py index 733c759f067bb..a061cc9e7ee6a 100644 --- a/test_dtype_utc.py +++ b/pandas/tests/test_dtype_utc.py @@ -16,7 +16,6 @@ def test_dtype_utc(self): filled=data.fillna(method='bfill') - expected=pd.Series([datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ]) From 5a59eac272ad8eee8dbb1c53ef012dbf3f9e867d Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 10:20:06 -0800 Subject: [PATCH 04/10] test modified --- pandas/tests/test_dtype_utc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/tests/test_dtype_utc.py b/pandas/tests/test_dtype_utc.py index a061cc9e7ee6a..7a1c17063a155 100644 --- a/pandas/tests/test_dtype_utc.py +++ b/pandas/tests/test_dtype_utc.py @@ -5,7 +5,7 @@ @author: rodolfoxps """ - +import nose import pandas as pd import datetime, pytz from pandas.util.testing import assert_frame_equal @@ -20,4 +20,8 @@ def test_dtype_utc(self): datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ]) - assert_frame_equal(filled, expected) \ No newline at end of file + assert_frame_equal(filled, expected) + +if __name__ == '__main__': + nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], + exit=False) \ No newline at end of file From 59b91a1226974bf4221420bca910c708eb62fc16 Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 10:28:20 -0800 Subject: [PATCH 05/10] test modified --- pandas/tests/test_dtype_utc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/test_dtype_utc.py b/pandas/tests/test_dtype_utc.py index 7a1c17063a155..4b8efef674bc9 100644 --- a/pandas/tests/test_dtype_utc.py +++ b/pandas/tests/test_dtype_utc.py @@ -8,9 +8,9 @@ import nose import pandas as pd import datetime, pytz -from pandas.util.testing import assert_frame_equal +from pandas.util.testing import assert_series_equal -def test_dtype_utc(self): +def test_dtype_utc(): data=pd.Series( [pd.NaT, pd.NaT, datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ] ) @@ -20,7 +20,7 @@ def test_dtype_utc(self): datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ]) - assert_frame_equal(filled, expected) + assert_series_equal(filled, expected) if __name__ == '__main__': nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], From 19eecb280c89e2207bbcf46f389116a5b6085cc0 Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 12:06:42 -0800 Subject: [PATCH 06/10] fixed style errors using flake8 --- pandas/tests/test_dtype_utc.py | 29 ++++++++++++++++++----------- pandas/types/common.py | 3 ++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/pandas/tests/test_dtype_utc.py b/pandas/tests/test_dtype_utc.py index 4b8efef674bc9..2ede75cff19c6 100644 --- a/pandas/tests/test_dtype_utc.py +++ b/pandas/tests/test_dtype_utc.py @@ -7,21 +7,28 @@ """ import nose import pandas as pd -import datetime, pytz +import datetime +import pytz from pandas.util.testing import assert_series_equal + def test_dtype_utc(): - - data=pd.Series( [pd.NaT, pd.NaT, datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ] ) - - filled=data.fillna(method='bfill') - - expected=pd.Series([datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , - datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) , - datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, tzinfo=pytz.utc) ]) + + data = pd.Series([pd.NaT, pd.NaT, + datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, + tzinfo=pytz.utc)]) + + filled = data.fillna(method='bfill') + + expected = pd.Series([datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, + tzinfo=pytz.utc), + datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, + tzinfo=pytz.utc), + datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, + tzinfo=pytz.utc)]) assert_series_equal(filled, expected) - + if __name__ == '__main__': nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], - exit=False) \ No newline at end of file + exit=False) diff --git a/pandas/types/common.py b/pandas/types/common.py index 7eb4a08619af0..8f560ab623912 100644 --- a/pandas/types/common.py +++ b/pandas/types/common.py @@ -26,7 +26,8 @@ _DATELIKE_DTYPES = set([np.dtype(t) for t in ['M8[ns]', 'M8[ns]', - 'm8[ns]', 'm8[ns]']] + [_NS_DTYPE_UTC]) + 'm8[ns]', 'm8[ns]']] + + [_NS_DTYPE_UTC]) _ensure_float64 = algos.ensure_float64 _ensure_float32 = algos.ensure_float32 From 5d37ce8647732de854eb5188abd4efae76c3065b Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 16:04:13 -0800 Subject: [PATCH 07/10] added is_datetime64tz_dtype and changed evaluation from 'values' to dtype --- pandas/core/missing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/core/missing.py b/pandas/core/missing.py index f1191ff1c7009..bba8ec6795507 100644 --- a/pandas/core/missing.py +++ b/pandas/core/missing.py @@ -10,8 +10,8 @@ from pandas.compat import range, string_types from pandas.types.common import (is_numeric_v_string_like, is_float_dtype, is_datetime64_dtype, - is_integer_dtype, _ensure_float64, - is_scalar, + is_datetime64tz_dtype, is_integer_dtype, + _ensure_float64, is_scalar, _DATELIKE_DTYPES, needs_i8_conversion) from pandas.types.missing import isnull @@ -450,7 +450,7 @@ def pad_1d(values, limit=None, mask=None, dtype=None): _method = None if is_float_dtype(values): _method = getattr(algos, 'pad_inplace_%s' % dtype.name, None) - elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): + elif is_datetime64_dtype(dtype) or is_datetime64tz_dtype(dtype): _method = _pad_1d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) @@ -475,7 +475,7 @@ def backfill_1d(values, limit=None, mask=None, dtype=None): _method = None if is_float_dtype(values): _method = getattr(algos, 'backfill_inplace_%s' % dtype.name, None) - elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): + elif is_datetime64_dtype(dtype) or is_datetime64tz_dtype(dtype): _method = _backfill_1d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) @@ -501,7 +501,7 @@ def pad_2d(values, limit=None, mask=None, dtype=None): _method = None if is_float_dtype(values): _method = getattr(algos, 'pad_2d_inplace_%s' % dtype.name, None) - elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): + elif is_datetime64_dtype(dtype) or is_datetime64tz_dtype(dtype): _method = _pad_2d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) @@ -531,7 +531,7 @@ def backfill_2d(values, limit=None, mask=None, dtype=None): _method = None if is_float_dtype(values): _method = getattr(algos, 'backfill_2d_inplace_%s' % dtype.name, None) - elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): + elif is_datetime64_dtype(dtype) or is_datetime64tz_dtype(dtype): _method = _backfill_2d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) From e4ba7e0f029b93ade0bde7f7773b1fcd68652172 Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 16:09:26 -0800 Subject: [PATCH 08/10] removed all references to _DATELIKE_DTYPES from /pandas/core/missing.py --- pandas/core/missing.py | 1 - pandas/types/common.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pandas/core/missing.py b/pandas/core/missing.py index bba8ec6795507..e83a0518d97f6 100644 --- a/pandas/core/missing.py +++ b/pandas/core/missing.py @@ -12,7 +12,6 @@ is_float_dtype, is_datetime64_dtype, is_datetime64tz_dtype, is_integer_dtype, _ensure_float64, is_scalar, - _DATELIKE_DTYPES, needs_i8_conversion) from pandas.types.missing import isnull diff --git a/pandas/types/common.py b/pandas/types/common.py index 8f560ab623912..5d161efa838de 100644 --- a/pandas/types/common.py +++ b/pandas/types/common.py @@ -22,12 +22,10 @@ _NS_DTYPE = np.dtype('M8[ns]') _TD_DTYPE = np.dtype('m8[ns]') _INT64_DTYPE = np.dtype(np.int64) -_NS_DTYPE_UTC = DatetimeTZDtype.__new__(DatetimeTZDtype, unit='ns', tz='UTC') _DATELIKE_DTYPES = set([np.dtype(t) for t in ['M8[ns]', 'M8[ns]', - 'm8[ns]', 'm8[ns]']] + - [_NS_DTYPE_UTC]) + 'm8[ns]', 'm8[ns]']]) _ensure_float64 = algos.ensure_float64 _ensure_float32 = algos.ensure_float32 From e0c6c7c629b94c4b2d85a2a60a77c076c5344de4 Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sat, 17 Dec 2016 21:59:31 -0800 Subject: [PATCH 09/10] added line to whatsnew v0.19.2 and test to test_missing.py in series folder --- doc/source/whatsnew/v0.19.2.txt | 1 + pandas/tests/series/test_missing.py | 21 ++++++++++++++++++ pandas/tests/test_dtype_utc.py | 34 ----------------------------- 3 files changed, 22 insertions(+), 34 deletions(-) delete mode 100644 pandas/tests/test_dtype_utc.py diff --git a/doc/source/whatsnew/v0.19.2.txt b/doc/source/whatsnew/v0.19.2.txt index 4cd58f0148ae8..c9056054fb1d7 100644 --- a/doc/source/whatsnew/v0.19.2.txt +++ b/doc/source/whatsnew/v0.19.2.txt @@ -38,6 +38,7 @@ Other Enhancements Bug Fixes ~~~~~~~~~ +- Bug in fillna() in which timezone aware datetime64 values were incorrectly rounded (:issue:'14872') - Compat with ``dateutil==2.6.0``; segfault reported in the testing suite (:issue:`14621`) - Allow ``nanoseconds`` in ``Timestamp.replace`` as a kwarg (:issue:`14621`) - Bug in ``pd.read_csv`` in which aliasing was being done for ``na_values`` when passed in as a dictionary (:issue:`14203`) diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 5666a07cad4b8..5514445ef266d 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -17,6 +17,9 @@ from .common import TestData +import datetime +import pytz + def _skip_if_no_pchip(): try: @@ -908,6 +911,24 @@ def test_interp_timedelta64(self): index=pd.to_timedelta([1, 2, 4])) assert_series_equal(result, expected) + # GH 14872 + def test_dtype_utc(): + + data = pd.Series([pd.NaT, pd.NaT, + datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, + tzinfo=pytz.utc)]) + + filled = data.fillna(method='bfill') + + expected = pd.Series([ + datetime.datetime(2016, 12, 12, 22, 24, 6, + 100001, tzinfo=pytz.utc), + datetime.datetime(2016, 12, 12, 22, 24, 6, + 100001, tzinfo=pytz.utc), + datetime.datetime(2016, 12, 12, 22, 24, 6, + 100001, tzinfo=pytz.utc)]) + + assert_series_equal(filled, expected) if __name__ == '__main__': import nose diff --git a/pandas/tests/test_dtype_utc.py b/pandas/tests/test_dtype_utc.py deleted file mode 100644 index 2ede75cff19c6..0000000000000 --- a/pandas/tests/test_dtype_utc.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Sat Dec 17 09:34:47 2016 - -@author: rodolfoxps -""" -import nose -import pandas as pd -import datetime -import pytz -from pandas.util.testing import assert_series_equal - - -def test_dtype_utc(): - - data = pd.Series([pd.NaT, pd.NaT, - datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, - tzinfo=pytz.utc)]) - - filled = data.fillna(method='bfill') - - expected = pd.Series([datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, - tzinfo=pytz.utc), - datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, - tzinfo=pytz.utc), - datetime.datetime(2016, 12, 12, 22, 24, 6, 100001, - tzinfo=pytz.utc)]) - - assert_series_equal(filled, expected) - -if __name__ == '__main__': - nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], - exit=False) From 18802b4c96705ca0df78518f485d3975f1615387 Mon Sep 17 00:00:00 2001 From: Rodolfo Fernandez Date: Sun, 18 Dec 2016 08:39:03 -0800 Subject: [PATCH 10/10] added 'self' to test_dtype_utc function in pandas/tests/series/test_missing --- pandas/tests/series/test_missing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 5514445ef266d..58f50ffbf08f5 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -912,7 +912,7 @@ def test_interp_timedelta64(self): assert_series_equal(result, expected) # GH 14872 - def test_dtype_utc(): + def test_dtype_utc(self): data = pd.Series([pd.NaT, pd.NaT, datetime.datetime(2016, 12, 12, 22, 24, 6, 100001,