BUG-20591 move logic to _convert_to_ndarrays

Author: JustinZhengBC

doc/source/whatsnew/v0.24.0.rst

@@ -330,7 +330,7 @@ Backwards incompatible API changes
 - :meth:`Series.str.cat` will now raise if `others` is a `set` (:issue:`23009`)
 - Passing scalar values to :class:`DatetimeIndex` or :class:`TimedeltaIndex` will now raise ``TypeError`` instead of ``ValueError`` (:issue:`23539`)
 - ``max_rows`` and ``max_cols`` parameters removed from :class:`HTMLFormatter` since truncation is handled by :class:`DataFrameFormatter` (:issue:`23818`)
-- :meth:`read_csv` will now throw a ``ValueError`` if a column with missing values is declared as having dtype ``bool`` (:issue:`20591`)
+- :meth:`read_csv` will now raise a ``ValueError`` if a column with missing values is declared as having dtype ``bool`` (:issue:`20591`)
 
 .. _whatsnew_0240.api_breaking.deps:
 

pandas/io/parsers.py

@@ -1669,6 +1669,16 @@ def _convert_to_ndarrays(self, dct, na_values, na_fvalues, verbose=False,
 
                 # type specified in dtype param
                 if cast_type and not is_dtype_equal(cvals, cast_type):
+                    try:
+                        if (is_bool_dtype(cast_type) and
+                                not is_categorical_dtype(cast_type)
+                                and set(values) - set(col_na_values)):
+                            raise ValueError("Bool column has NA values in "
+                                             "column {column}"
+                                             .format(column=c))
+                    except (AttributeError, TypeError):
+                        # invalid input to is_bool_dtype
+                        pass
                     cvals = self._cast_types(cvals, cast_type, c)
 
             result[c] = cvals
@@ -2435,21 +2445,6 @@ def _clean_mapping(mapping):
             clean_na_values = self.na_values
             clean_na_fvalues = self.na_fvalues
 
-        try:
-            if isinstance(clean_dtypes, dict):
-                for col, dt in clean_dtypes.items():
-                    if is_bool_dtype(dt) and data[col][data[col] == ''].size:
-                        raise ValueError("Bool column has NA values in "
-                                         "column {column}".format(column=col))
-            elif (isinstance(clean_dtypes, string_types) and
-                  is_bool_dtype(clean_dtypes)):
-                for col, values in data.items():
-                    if any(isna(values)):
-                        raise ValueError("Bool column has NA values in "
-                                         "column {column}".format(column=col))
-        except (AttributeError, TypeError):  # invalid input to is_bool_dtype
-            pass
-
         return self._convert_to_ndarrays(data, clean_na_values,
                                          clean_na_fvalues, self.verbose,
                                          clean_conv, clean_dtypes)

pandas/tests/io/parser/test_na_values.py

@@ -423,10 +423,16 @@ def test_na_values_with_dtype_str_and_na_filter(all_parsers, na_filter):
     tm.assert_frame_equal(result, expected)
 
 
-def test_cast_NA_to_bool_raises_error(all_parsers):
+@pytest.mark.parametrize("data", [
+    "false,1\n,1\ntrue,",
+    "false,1\nnull,1\ntrue,",
+    "false,1\nnan,1\ntrue,",
+])
+def test_cast_NA_to_bool_raises_error(all_parsers, data):
     parser = all_parsers
-    data = "false,1\n,1\ntrue,"
-    msg = "Bool column has NA values in column [0a]"
+    msg = ("(Bool column has NA values in column [0a])|"
+            "(cannot safely convert passed user dtype of "
+            " bool for object dtyped data in column 0)")
     with pytest.raises(ValueError, match=msg):
         parser.read_csv(StringIO(data), header=None, names=['a', 'b'],
                         dtype={'a': 'bool'})