diff --git a/doc/source/whatsnew/v0.16.0.txt b/doc/source/whatsnew/v0.16.0.txt index 3fa48c7e9a1fc..83d24cc081755 100644 --- a/doc/source/whatsnew/v0.16.0.txt +++ b/doc/source/whatsnew/v0.16.0.txt @@ -186,7 +186,7 @@ Bug Fixes - Bug in Panel indexing with an object-like (:issue:`9140`) - Bug in the returned ``Series.dt.components`` index was reset to the default index (:issue:`9247`) - Bug in ``Categorical.__getitem__/__setitem__`` with listlike input getting incorrect results from indexer coercion (:issue:`9469`) - +- Bug in partial setting with a DatetimeIndex (:issue:`9478`) - Fixed bug in ``to_sql`` when mapping a ``Timestamp`` object column (datetime column with timezone info) to the according sqlalchemy type (:issue:`9085`). - Fixed bug in ``to_sql`` ``dtype`` argument not accepting an instantiated diff --git a/pandas/core/common.py b/pandas/core/common.py index f8f5928ca7d51..581ed31b9819b 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -1011,6 +1011,27 @@ def conv(r, dtype): return [conv(r, dtype) for r, dtype in zip(result, dtypes)] +def _infer_fill_value(val): + """ + infer the fill value for the nan/NaT from the provided scalar/ndarray/list-like + if we are a NaT, return the correct dtyped element to provide proper block construction + + """ + + if not is_list_like(val): + val = [val] + val = np.array(val,copy=False) + if is_datetimelike(val): + return np.array('NaT',dtype=val.dtype) + elif is_object_dtype(val.dtype): + dtype = lib.infer_dtype(_ensure_object(val)) + if dtype in ['datetime','datetime64']: + return np.array('NaT',dtype=_NS_DTYPE) + elif dtype in ['timedelta','timedelta64']: + return np.array('NaT',dtype=_TD_DTYPE) + return np.nan + + def _infer_dtype_from_scalar(val): """ interpret the dtype from a scalar, upcast floats and ints return the new value and the dtype """ diff --git a/pandas/core/index.py b/pandas/core/index.py index 2e23c38dd3457..75a4e0c9647df 100644 --- a/pandas/core/index.py +++ b/pandas/core/index.py @@ -345,7 +345,10 @@ def _get_attributes_dict(self): return dict([ (k,getattr(self,k,None)) for k in self._attributes]) def view(self, cls=None): - if cls is not None and not issubclass(cls, Index): + + # we need to see if we are subclassing an + # index type here + if cls is not None and not hasattr(cls,'_typ'): result = self._data.view(cls) else: result = self._shallow_copy() diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index e305eb828f410..1ce9decd178a0 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -8,7 +8,7 @@ from pandas.core.common import (_is_bool_indexer, is_integer_dtype, _asarray_tuplesafe, is_list_like, isnull, ABCSeries, ABCDataFrame, ABCPanel, is_float, - _values_from_object) + _values_from_object, _infer_fill_value) import pandas.lib as lib import numpy as np @@ -238,7 +238,9 @@ def _setitem_with_indexer(self, indexer, value): self.obj[key] = value return self.obj - self.obj[key] = np.nan + + # add a new item with the dtype setup + self.obj[key] = _infer_fill_value(value) new_indexer = _convert_from_missing_indexer_tuple( indexer, self.obj.axes) diff --git a/pandas/tests/test_index.py b/pandas/tests/test_index.py index ef41748e2cda9..65e42f128564e 100644 --- a/pandas/tests/test_index.py +++ b/pandas/tests/test_index.py @@ -297,6 +297,9 @@ def test_view(self): i_view = i.view() self.assertEqual(i_view.name, 'Foo') + # with arguments + self.assertRaises(TypeError, lambda : i.view('i8')) + def test_legacy_pickle_identity(self): # GH 8431 @@ -1469,6 +1472,12 @@ def test_view(self): i_view = i.view() self.assertEqual(i_view.name, 'Foo') + i_view = i.view('i8') + tm.assert_index_equal(i, Int64Index(i_view)) + + i_view = i.view(Int64Index) + tm.assert_index_equal(i, Int64Index(i_view)) + def test_coerce_list(self): # coerce things arr = Index([1, 2, 3, 4]) @@ -1856,7 +1865,21 @@ def test_slice_keep_name(self): idx = Int64Index([1, 2], name='asdf') self.assertEqual(idx.name, idx[1:].name) -class TestDatetimeIndex(Base, tm.TestCase): +class DatetimeLike(Base): + + def test_view(self): + + i = self.create_index() + + i_view = i.view('i8') + result = self._holder(i) + tm.assert_index_equal(result, i) + + i_view = i.view(self._holder) + result = self._holder(i) + tm.assert_index_equal(result, i) + +class TestDatetimeIndex(DatetimeLike, tm.TestCase): _holder = DatetimeIndex _multiprocess_can_split_ = True @@ -1926,7 +1949,7 @@ def test_time_overflow_for_32bit_machines(self): self.assertEqual(len(idx2), periods) -class TestPeriodIndex(Base, tm.TestCase): +class TestPeriodIndex(DatetimeLike, tm.TestCase): _holder = PeriodIndex _multiprocess_can_split_ = True @@ -1936,7 +1959,7 @@ def create_index(self): def test_pickle_compat_construction(self): pass -class TestTimedeltaIndex(Base, tm.TestCase): +class TestTimedeltaIndex(DatetimeLike, tm.TestCase): _holder = TimedeltaIndex _multiprocess_can_split_ = True diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py index c2d5910e7859f..53401b58d4c45 100644 --- a/pandas/tests/test_indexing.py +++ b/pandas/tests/test_indexing.py @@ -3158,6 +3158,19 @@ def f(): df.loc[3] = [6,7] assert_frame_equal(df,DataFrame([[6,7]],index=[3],columns=['A','B'],dtype='float64')) + def test_partial_setting_with_datetimelike_dtype(self): + + # GH9478 + # a datetimeindex alignment issue with partial setting + df = pd.DataFrame(np.arange(6.).reshape(3,2), columns=list('AB'), + index=pd.date_range('1/1/2000', periods=3, freq='1H')) + expected = df.copy() + expected['C'] = [expected.index[0]] + [pd.NaT,pd.NaT] + + mask = df.A < 1 + df.loc[mask, 'C'] = df.loc[mask].index + assert_frame_equal(df, expected) + def test_series_partial_set(self): # partial set with new index # Regression from GH4825