ERR: [Python] infer_dtypes for better error messages on write_dataframe

ERR: nicer error message on passing duplicate columns, xref #53
ERR: nice error message on serializing python objects, closes #240

should be after #239

xref pandas-dev/pandas#14383

Author: Jeff Reback <[email protected]>

Closes #244 from jreback/infer_dtype and squashes the following commits:

091ee60 [Jeff Reback] fix pandas < 0.19.0 compat
6ef3a47 [Jeff Reback] fixed up mixed with embedded nulls tests
560b9e6 [Jeff Reback] ERR: nicer error message on passing duplicate columns, xref #53

Author: jreback

python/feather/api.py

@@ -15,6 +15,7 @@
 import six
 from distutils.version import LooseVersion
 import pandas as pd
+from feather.compat import pdapi
 
 import feather.ext as ext
 
@@ -27,14 +28,33 @@ def write_dataframe(df, path):
     Write a pandas.DataFrame to Feather format
     '''
     writer = ext.FeatherWriter(path)
-    
+
     if isinstance(df, pd.SparseDataFrame):
         df = df.to_dense()
 
+    if not df.columns.is_unique:
+        raise ValueError("cannot serialize duplicate column names")
+
     # TODO(wesm): pipeline conversion to Arrow memory layout
     for i, name in enumerate(df.columns):
         col = df.iloc[:, i]
 
+        if pdapi.is_object_dtype(col):
+            inferred_type = pd.lib.infer_dtype(col)
+            msg = ("cannot serialize column {n} "
+                   "named {name} with dtype {dtype}".format(
+                       n=i, name=name, dtype=inferred_type))
+
+            if inferred_type in ['mixed']:
+
+                # allow columns with nulls + an inferable type
+                inferred_type = pd.lib.infer_dtype(col[col.notnull()])
+                if inferred_type in ['mixed']:
+                    raise ValueError(msg)
+
+            elif inferred_type not in ['unicode', 'string']:
+                raise ValueError(msg)
+
         if not isinstance(name, six.string_types):
             name = str(name)
 

python/feather/tests/test_reader.py

@@ -82,6 +82,18 @@ def _check_pandas_roundtrip(self, df, expected=None, path=None,
 
         np.testing.assert_array_equal(self._get_null_counts(path, columns), null_counts)
 
+    def _assert_error_on_write(self, df, exc, path=None):
+        # check that we are raising the exception
+        # on writing
+
+        if path is None:
+            path = random_path()
+
+        self.test_files.append(path)
+        def f():
+            feather.write_dataframe(df, path)
+        self.assertRaises(exc, f)
+
     def test_num_rows_attr(self):
         df = pd.DataFrame({'foo': [1, 2, 3, 4, 5]})
         path = random_path()
@@ -237,10 +249,20 @@ def test_boolean_object_nulls(self):
 
     def test_strings(self):
         repeats = 1000
+
+        # we hvae mixed bytes, unicode, strings
         values = [b'foo', None, u'bar', 'qux', np.nan]
         df = pd.DataFrame({'strings': values * repeats})
+        self._assert_error_on_write(df, ValueError)
 
+        # embedded nulls are ok
         values = ['foo', None, 'bar', 'qux', None]
+        df = pd.DataFrame({'strings': values * repeats})
+        expected = pd.DataFrame({'strings': values * repeats})
+        self._check_pandas_roundtrip(df, expected, null_counts=[2 * repeats])
+
+        values = ['foo', None, 'bar', 'qux', np.nan]
+        df = pd.DataFrame({'strings': values * repeats})
         expected = pd.DataFrame({'strings': values * repeats})
         self._check_pandas_roundtrip(df, expected, null_counts=[2 * repeats])
 
@@ -300,3 +322,23 @@ def test_sparse_dataframe(self):
         df = pd.DataFrame(data).to_sparse(fill_value=1)
         expected = df.to_dense()
         self._check_pandas_roundtrip(df, expected)
+
+    def test_duplicate_columns(self):
+
+        # https://github.com/wesm/feather/issues/53
+        # not currently able to handle duplicate columns
+        df = pd.DataFrame(np.arange(12).reshape(4, 3),
+                          columns=list('aaa')).copy()
+        self._assert_error_on_write(df, ValueError)
+
+    def test_unsupported(self):
+        # https://github.com/wesm/feather/issues/240
+        # serializing actual python objects
+
+        # period
+        df = pd.DataFrame({'a': pd.period_range('2013', freq='M', periods=3)})
+        self._assert_error_on_write(df, ValueError)
+
+        # non-strings
+        df = pd.DataFrame({'a': ['a', 1, 2.0]})
+        self._assert_error_on_write(df, ValueError)