BUG: Fix initialization of DataFrame from dict with NaN as key

closes pandas-dev#18455

Author: toobaz

pandas/core/frame.py

@@ -416,44 +416,29 @@ def _init_dict(self, data, index, columns, dtype=None):
         Needs to handle a lot of exceptional cases.
         """
         if columns is not None:
-            columns = _ensure_index(columns)
+            arrays = Series(data, index=columns, dtype=object)
+            data_names = arrays.index
 
-            # GH10856
-            # raise ValueError if only scalars in dict
+            missing = arrays.isnull()
             if index is None:
-                extract_index(list(data.values()))
-
-            # prefilter if columns passed
-            data = {k: v for k, v in compat.iteritems(data) if k in columns}
-
-            if index is None:
-                index = extract_index(list(data.values()))
-
+                # GH10856
+                # raise ValueError if only scalars in dict
+                index = extract_index(arrays[~missing].tolist())
             else:
                 index = _ensure_index(index)
 
-            arrays = []
-            data_names = []
-            for k in columns:
-                if k not in data:
-                    # no obvious "empty" int column
-                    if dtype is not None and issubclass(dtype.type,
-                                                        np.integer):
-                        continue
-
-                    if dtype is None:
-                        # 1783
-                        v = np.empty(len(index), dtype=object)
-                    elif np.issubdtype(dtype, np.flexible):
-                        v = np.empty(len(index), dtype=object)
-                    else:
-                        v = np.empty(len(index), dtype=dtype)
-
-                    v.fill(np.nan)
+            # no obvious "empty" int column
+            if missing.any() and not (dtype is not None and
+                                      issubclass(dtype.type, np.integer)):
+                if dtype is None or np.issubdtype(dtype, np.flexible):
+                    # 1783
+                    nan_dtype = object
                 else:
-                    v = data[k]
-                data_names.append(k)
-                arrays.append(v)
+                    nan_dtype = dtype
+                v = np.empty(len(index), dtype=nan_dtype)
+                v.fill(np.nan)
+                arrays.loc[missing] = [v] * missing.sum()
+            arrays = arrays.tolist()
 
         else:
             keys = list(data.keys())

pandas/core/generic.py

@@ -6185,7 +6185,6 @@ def _where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
                 if not is_bool_dtype(dt):
                     raise ValueError(msg.format(dtype=dt))
 
-        cond = cond.astype(bool, copy=False)
         cond = -cond if inplace else cond
 
         # try to align with other

pandas/core/series.py

@@ -3208,7 +3208,11 @@ def _try_cast(arr, take_fast_path):
         return subarr
 
     elif isinstance(data, (list, tuple)) and len(data) > 0:
-        if dtype is not None:
+        if all(is_list_like(item) for item in data):
+            # Ensure nested lists are not interpreted as further dimensions:
+            subarr = np.empty(len(data), dtype='object')
+            subarr[:] = data
+        elif dtype is not None:
             try:
                 subarr = _try_cast(data, False)
             except Exception:

pandas/tests/frame/test_constructors.py

@@ -275,8 +275,50 @@ def test_constructor_dict(self):
         with tm.assert_raises_regex(ValueError, msg):
             DataFrame({'a': 0.7}, columns=['a'])
 
-        with tm.assert_raises_regex(ValueError, msg):
-            DataFrame({'a': 0.7}, columns=['b'])
+        # No reason to raise if item is not used:
+        result = DataFrame({'a': 0.7}, columns=['b'])
+        expected = DataFrame(columns=['b'])
+        tm.assert_frame_equal(result, expected)
+
+    @pytest.mark.parametrize("value", [2, np.nan, None, float('nan')])
+    def test_constructor_dict_nan_key(self, value):
+        # GH 18455
+        cols = [1, value, 3]
+        idx = ['a', value]
+        values = [[0, 3], [1, 4], [2, 5]]
+        data = {cols[c]: pd.Series(values[c], index=idx) for c in range(3)}
+        result = pd.DataFrame(data).sort_values(1).sort_values('a', axis=1)
+        expected = pd.DataFrame(np.arange(6).reshape(2, 3),
+                                index=idx, columns=cols)
+        tm.assert_frame_equal(result, expected)
+
+        result = pd.DataFrame(data, index=idx).sort_values('a', axis=1)
+        tm.assert_frame_equal(result, expected)
+
+        result = pd.DataFrame(data, index=idx, columns=cols)
+        tm.assert_frame_equal(result, expected)
+
+    @pytest.mark.xfail(reason='GH 18485 comparison fails on MI with NaNs)')
+    @pytest.mark.parametrize("value", [np.nan, None, float('nan')])
+    def test_constructor_dict_nan_tuple_key(self, value):
+        # GH 18455
+        cols = Index([(11, 21), (value, 22), (13, value)])
+        idx = Index([('a', value), (value, 2)])
+        values = [[0, 3], [1, 4], [2, 5]]
+        data = {cols[c]: pd.Series(values[c], index=idx) for c in range(3)}
+        result = (DataFrame(data)
+                  .sort_values((11, 21))
+                  .sort_values(('a', value), axis=1))
+        expected = pd.DataFrame(np.arange(6).reshape(2, 3),
+                                index=idx, columns=cols)
+        tm.assert_frame_equal(result, expected)
+
+        result = pd.DataFrame(data, index=idx).sort_values(('a', value),
+                                                           axis=1)
+        tm.assert_frame_equal(result, expected)
+
+        result = pd.DataFrame(data, index=idx, columns=cols)
+        tm.assert_frame_equal(result, expected)
 
     def test_constructor_multi_index(self):
         # GH 4078
@@ -723,15 +765,15 @@ def test_constructor_corner(self):
 
         # does not error but ends up float
         df = DataFrame(index=lrange(10), columns=['a', 'b'], dtype=int)
-        assert df.values.dtype == np.object_
+        assert df.values.dtype == np.dtype('float64')
 
         # #1783 empty dtype object
         df = DataFrame({}, columns=['foo', 'bar'])
         assert df.values.dtype == np.object_
 
         df = DataFrame({'b': 1}, index=lrange(10), columns=list('abc'),
                        dtype=int)
-        assert df.values.dtype == np.object_
+        assert df.values.dtype == np.dtype('float64')
 
     def test_constructor_scalar_inference(self):
         data = {'int': 1, 'bool': True,

pandas/tests/io/test_excel.py

@@ -516,7 +516,7 @@ def test_read_one_empty_col_with_header(self):
             )
         expected_header_none = DataFrame(pd.Series([0], dtype='int64'))
         tm.assert_frame_equal(actual_header_none, expected_header_none)
-        expected_header_zero = DataFrame(columns=[0], dtype='int64')
+        expected_header_zero = DataFrame(columns=[0])
         tm.assert_frame_equal(actual_header_zero, expected_header_zero)
 
     def test_set_column_names_in_parameter(self):

pandas/tests/series/test_constructors.py

@@ -713,6 +713,16 @@ def test_constructor_set(self):
         values = frozenset(values)
         pytest.raises(TypeError, Series, values)
 
+    @pytest.mark.parametrize('klass', [list, tuple, set, Series])
+    def test_constructor_complex_values(self, klass):
+        d = {1: klass([1, 2, 3]), 3: klass([4, 5, 6]), 5: klass([7, 8, 9])}
+        result = Series(d).sort_index()
+
+        exp_data = np.empty(len(d), dtype='object')
+        exp_data[:] = [d[i] for i in [1, 3, 5]]
+        expected = Series(exp_data, index=pd.Index([1, 3, 5]))
+        tm.assert_series_equal(result, expected)
+
     def test_fromDict(self):
         data = {'a': 0, 'b': 1, 'c': 2, 'd': 3}