ENH: add encoding/decoding error handling

When encoding/decoding strings with errors allow you
to pass error handling strings. This works the same as
error handling for other encode/decode functions.
Defaults to 'strict', but you can pass 'ignore', 'replace',
etc.

Extends work done in #1706.

Author: chmullig

pandas/core/strings.py

@@ -519,35 +519,37 @@ def str_get(arr, i):
     return _na_map(f, arr)
 
 
-def str_decode(arr, encoding):
+def str_decode(arr, encoding, errors="strict"):
     """
     Decode character string to unicode using indicated encoding
 
     Parameters
     ----------
     encoding : string
+    errors : string
 
     Returns
     -------
     decoded : array
     """
-    f = lambda x: x.decode(encoding)
+    f = lambda x: x.decode(encoding, errors)
     return _na_map(f, arr)
 
 
-def str_encode(arr, encoding):
+def str_encode(arr, encoding, errors="strict"):
     """
-    Encode character string to unicode using indicated encoding
+    Encode character string to some other encoding using indicated encoding
 
     Parameters
     ----------
     encoding : string
+    errors : string
 
     Returns
     -------
     encoded : array
     """
-    f = lambda x: x.encode(encoding)
+    f = lambda x: x.encode(encoding, errors)
     return _na_map(f, arr)
 
 
@@ -675,13 +677,13 @@ def slice_replace(self, i=None, j=None):
         raise NotImplementedError
 
     @copy(str_decode)
-    def decode(self, encoding):
-        result = str_decode(self.series, encoding)
+    def decode(self, encoding, errors="strict"):
+        result = str_decode(self.series, encoding, errors)
         return self._wrap_result(result)
 
     @copy(str_encode)
-    def encode(self, encoding):
-        result = str_encode(self.series, encoding)
+    def encode(self, encoding, errors="strict"):
+        result = str_encode(self.series, encoding, errors)
         return self._wrap_result(result)
 
     count = _pat_wrapper(str_count, flags=True)

pandas/tests/test_strings.py

@@ -690,7 +690,7 @@ def test_match_findall_flags(self):
         self.assertEquals(result[0], True)
 
     def test_encode_decode(self):
-        base = Series([u'a', u'b', u'\xe4'])
+        base = Series([u'a', u'b', u'a\xe4'])
         series = base.str.encode('utf-8')
 
         f = lambda x: x.decode('utf-8')
@@ -699,6 +699,25 @@ def test_encode_decode(self):
 
         tm.assert_series_equal(result, exp)
 
+    def test_encode_decode_errors(self):
+        encodeBase = Series([u'a', u'b', u'a\x9d'])
+        with self.assertRaises(UnicodeEncodeError):
+            encodeBase.str.encode('cp1252')
+
+        f = lambda x: x.encode('cp1252', 'ignore')
+        result = encodeBase.str.encode('cp1252', 'ignore')
+        exp = encodeBase.map(f)
+        tm.assert_series_equal(result, exp)
+
+        decodeBase = Series(['a', 'b', 'a\x9d'])
+        with self.assertRaises(UnicodeDecodeError):
+            decodeBase.str.encode('cp1252')
+        f = lambda x: x.decode('cp1252', 'ignore')
+        result = decodeBase.str.decode('cp1252', 'ignore')
+        exp = decodeBase.map(f)
+
+        tm.assert_series_equal(result, exp)
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__,'-vvs','-x','--pdb', '--pdb-failure'],
                    exit=False)