Merge branch 'master' into TST--split-tests-for-windows-to-sub-modules-pandas-dev#19228

Author: giba0

doc/source/whatsnew/v0.23.0.txt

@@ -822,6 +822,7 @@ Sparse
 - Bug in which creating a ``SparseDataFrame`` from a dense ``Series`` or an unsupported type raised an uncontrolled exception (:issue:`19374`)
 - Bug in :class:`SparseDataFrame.to_csv` causing exception (:issue:`19384`)
 - Bug in :class:`SparseSeries.memory_usage` which caused segfault by accessing non sparse elements (:issue:`19368`)
+- Bug in constructing a ``SparseArray``: if ``data`` is a scalar and ``index`` is defined it will coerce to ``float64`` regardless of scalar's dtype. (:issue:`19163`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/sparse/array.py

@@ -26,7 +26,8 @@
     is_scalar, is_dtype_equal)
 from pandas.core.dtypes.cast import (
     maybe_convert_platform, maybe_promote,
-    astype_nansafe, find_common_type)
+    astype_nansafe, find_common_type, infer_dtype_from_scalar,
+    construct_1d_arraylike_from_scalar)
 from pandas.core.dtypes.missing import isna, notna, na_value_for_dtype
 
 import pandas._libs.sparse as splib
@@ -162,9 +163,9 @@ def __new__(cls, data, sparse_index=None, index=None, kind='integer',
                 data = np.nan
             if not is_scalar(data):
                 raise Exception("must only pass scalars with an index ")
-            values = np.empty(len(index), dtype='float64')
-            values.fill(data)
-            data = values
+            dtype = infer_dtype_from_scalar(data)[0]
+            data = construct_1d_arraylike_from_scalar(
+                data, len(index), dtype)
 
         if isinstance(data, ABCSparseSeries):
             data = data.values

pandas/tests/indexes/datetimes/test_tools.py

@@ -8,7 +8,7 @@
 import dateutil
 import numpy as np
 from dateutil.parser import parse
-from datetime import datetime, date, time, timedelta
+from datetime import datetime, date, time
 from distutils.version import LooseVersion
 
 import pandas as pd
@@ -19,7 +19,6 @@
 
 from pandas.errors import OutOfBoundsDatetime
 from pandas.compat import lmap, PY3
-from pandas.compat.numpy import np_array_datetime64_compat
 from pandas.core.dtypes.common import is_datetime64_ns_dtype
 from pandas.util import testing as tm
 import pandas.util._test_decorators as td
@@ -803,6 +802,15 @@ def test_dataframe_dtypes(self, cache):
 
 
 class TestToDatetimeMisc(object):
+    def test_to_datetime_barely_out_of_bounds(self):
+        # GH#19529
+        # GH#19382 close enough to bounds that dropping nanos would result
+        # in an in-bounds datetime
+        arr = np.array(['2262-04-11 23:47:16.854775808'], dtype=object)
+
+        with pytest.raises(OutOfBoundsDatetime):
+            to_datetime(arr)
+
     @pytest.mark.parametrize('cache', [True, False])
     def test_to_datetime_iso8601(self, cache):
         result = to_datetime(["2012-01-01 00:00:00"], cache=cache)
@@ -1464,180 +1472,6 @@ def test_parsers_timezone_minute_offsets_roundtrip(self, cache):
             converted_time = dt_time.tz_localize('UTC').tz_convert(tz)
             assert dt_string_repr == repr(converted_time)
 
-    def test_parsers_iso8601(self):
-        # GH 12060
-        # test only the iso parser - flexibility to different
-        # separators and leadings 0s
-        # Timestamp construction falls back to dateutil
-        cases = {'2011-01-02': datetime(2011, 1, 2),
-                 '2011-1-2': datetime(2011, 1, 2),
-                 '2011-01': datetime(2011, 1, 1),
-                 '2011-1': datetime(2011, 1, 1),
-                 '2011 01 02': datetime(2011, 1, 2),
-                 '2011.01.02': datetime(2011, 1, 2),
-                 '2011/01/02': datetime(2011, 1, 2),
-                 '2011\\01\\02': datetime(2011, 1, 2),
-                 '2013-01-01 05:30:00': datetime(2013, 1, 1, 5, 30),
-                 '2013-1-1 5:30:00': datetime(2013, 1, 1, 5, 30)}
-        for date_str, exp in compat.iteritems(cases):
-            actual = tslib._test_parse_iso8601(date_str)
-            assert actual == exp
-
-        # separators must all match - YYYYMM not valid
-        invalid_cases = ['2011-01/02', '2011^11^11',
-                         '201401', '201111', '200101',
-                         # mixed separated and unseparated
-                         '2005-0101', '200501-01',
-                         '20010101 12:3456', '20010101 1234:56',
-                         # HHMMSS must have two digits in each component
-                         # if unseparated
-                         '20010101 1', '20010101 123', '20010101 12345',
-                         '20010101 12345Z',
-                         # wrong separator for HHMMSS
-                         '2001-01-01 12-34-56']
-        for date_str in invalid_cases:
-            with pytest.raises(ValueError):
-                tslib._test_parse_iso8601(date_str)
-                # If no ValueError raised, let me know which case failed.
-                raise Exception(date_str)
-
-
-class TestArrayToDatetime(object):
-    def test_coerce_out_of_bounds_utc(self):
-        # GH#19612
-        ts = Timestamp('1900-01-01', tz='US/Pacific')
-        dt = ts.to_pydatetime() - timedelta(days=365 * 300)  # ~1600AD
-        arr = np.array([dt])
-        result = tslib.array_to_datetime(arr, utc=True, errors='coerce')
-        expected = np.array(['NaT'], dtype='datetime64[ns]')
-        tm.assert_numpy_array_equal(result, expected)
-
-    def test_parsing_valid_dates(self):
-        arr = np.array(['01-01-2013', '01-02-2013'], dtype=object)
-        tm.assert_numpy_array_equal(
-            tslib.array_to_datetime(arr),
-            np_array_datetime64_compat(
-                [
-                    '2013-01-01T00:00:00.000000000-0000',
-                    '2013-01-02T00:00:00.000000000-0000'
-                ],
-                dtype='M8[ns]'
-            )
-        )
-
-        arr = np.array(['Mon Sep 16 2013', 'Tue Sep 17 2013'], dtype=object)
-        tm.assert_numpy_array_equal(
-            tslib.array_to_datetime(arr),
-            np_array_datetime64_compat(
-                [
-                    '2013-09-16T00:00:00.000000000-0000',
-                    '2013-09-17T00:00:00.000000000-0000'
-                ],
-                dtype='M8[ns]'
-            )
-        )
-
-    def test_parsing_timezone_offsets(self):
-        # All of these datetime strings with offsets are equivalent
-        # to the same datetime after the timezone offset is added
-        dt_strings = [
-            '01-01-2013 08:00:00+08:00',
-            '2013-01-01T08:00:00.000000000+0800',
-            '2012-12-31T16:00:00.000000000-0800',
-            '12-31-2012 23:00:00-01:00'
-        ]
-
-        expected_output = tslib.array_to_datetime(np.array(
-            ['01-01-2013 00:00:00'], dtype=object))
-
-        for dt_string in dt_strings:
-            tm.assert_numpy_array_equal(
-                tslib.array_to_datetime(
-                    np.array([dt_string], dtype=object)
-                ),
-                expected_output
-            )
-
-    def test_number_looking_strings_not_into_datetime(self):
-        # #4601
-        # These strings don't look like datetimes so they shouldn't be
-        # attempted to be converted
-        arr = np.array(['-352.737091', '183.575577'], dtype=object)
-        tm.assert_numpy_array_equal(
-            tslib.array_to_datetime(arr, errors='ignore'), arr)
-
-        arr = np.array(['1', '2', '3', '4', '5'], dtype=object)
-        tm.assert_numpy_array_equal(
-            tslib.array_to_datetime(arr, errors='ignore'), arr)
-
-    def test_coercing_dates_outside_of_datetime64_ns_bounds(self):
-        invalid_dates = [
-            date(1000, 1, 1),
-            datetime(1000, 1, 1),
-            '1000-01-01',
-            'Jan 1, 1000',
-            np.datetime64('1000-01-01'),
-        ]
-
-        for invalid_date in invalid_dates:
-            pytest.raises(ValueError,
-                          tslib.array_to_datetime,
-                          np.array([invalid_date], dtype='object'),
-                          errors='raise', )
-            tm.assert_numpy_array_equal(
-                tslib.array_to_datetime(
-                    np.array([invalid_date], dtype='object'),
-                    errors='coerce'),
-                np.array([tslib.iNaT], dtype='M8[ns]')
-            )
-
-        arr = np.array(['1/1/1000', '1/1/2000'], dtype=object)
-        tm.assert_numpy_array_equal(
-            tslib.array_to_datetime(arr, errors='coerce'),
-            np_array_datetime64_compat(
-                [
-                    tslib.iNaT,
-                    '2000-01-01T00:00:00.000000000-0000'
-                ],
-                dtype='M8[ns]'
-            )
-        )
-
-    def test_coerce_of_invalid_datetimes(self):
-        arr = np.array(['01-01-2013', 'not_a_date', '1'], dtype=object)
-
-        # Without coercing, the presence of any invalid dates prevents
-        # any values from being converted
-        tm.assert_numpy_array_equal(
-            tslib.array_to_datetime(arr, errors='ignore'), arr)
-
-        # With coercing, the invalid dates becomes iNaT
-        tm.assert_numpy_array_equal(
-            tslib.array_to_datetime(arr, errors='coerce'),
-            np_array_datetime64_compat(
-                [
-                    '2013-01-01T00:00:00.000000000-0000',
-                    tslib.iNaT,
-                    tslib.iNaT
-                ],
-                dtype='M8[ns]'
-            )
-        )
-
-    def test_to_datetime_barely_out_of_bounds(self):
-        # GH#19529
-        # GH#19382 close enough to bounds that dropping nanos would result
-        # in an in-bounds datetime
-        arr = np.array(['2262-04-11 23:47:16.854775808'], dtype=object)
-
-        with pytest.raises(OutOfBoundsDatetime):
-            to_datetime(arr)
-
-        with pytest.raises(OutOfBoundsDatetime):
-            # Essentially the same as above, but more directly calling
-            # the relevant function
-            tslib.array_to_datetime(arr)
-
 
 def test_normalize_date():
     value = date(2012, 9, 7)

pandas/tests/sparse/frame/test_frame.py

@@ -1257,3 +1257,14 @@ def test_quantile_multi(self):
 
         tm.assert_frame_equal(result, dense_expected)
         tm.assert_sp_frame_equal(result, sparse_expected)
+
+    def test_assign_with_sparse_frame(self):
+        # GH 19163
+        df = pd.DataFrame({"a": [1, 2, 3]})
+        res = df.to_sparse(fill_value=False).assign(newcol=False)
+        exp = df.assign(newcol=False).to_sparse(fill_value=False)
+
+        tm.assert_sp_frame_equal(res, exp)
+
+        for column in res.columns:
+            assert type(res[column]) is SparseSeries

pandas/tests/sparse/test_array.py

@@ -113,6 +113,21 @@ def test_constructor_spindex_dtype(self):
         assert arr.dtype == np.int64
         assert arr.fill_value == 0
 
+    @pytest.mark.parametrize('scalar,dtype', [
+        (False, bool),
+        (0.0, 'float64'),
+        (1, 'int64'),
+        ('z', 'object')])
+    def test_scalar_with_index_infer_dtype(self, scalar, dtype):
+        # GH 19163
+        arr = SparseArray(scalar, index=[1, 2, 3], fill_value=scalar)
+        exp = SparseArray([scalar, scalar, scalar], fill_value=scalar)
+
+        tm.assert_sp_array_equal(arr, exp)
+
+        assert arr.dtype == dtype
+        assert exp.dtype == dtype
+
     def test_sparseseries_roundtrip(self):
         # GH 13999
         for kind in ['integer', 'block']: