ENH: handling of UTF-8 strings in DataFrame columns, close #1620

Author: wesm

RELEASE.rst

@@ -102,6 +102,7 @@ pandas 0.8.1
   - Handle None values in dict passed to concat (#1649)
   - Fix Series.interpolate with method='values' and DatetimeIndex (#1646)
   - Fix IndexError in left merges on a DataFrame with 0-length (#1628)
+  - Fix DataFrame column width display with UTF-8 encoded characters (#1620)
 
 pandas 0.8.0
 ============

pandas/core/format.py

@@ -136,6 +136,12 @@ def to_string(self):
     def _encode_diff(x):
         return len(x) - len(x.decode('utf-8'))
 
+def _strlen(x):
+    try:
+        return len(x.decode('utf-8'))
+    except UnicodeError:
+        return len(x)
+
 class DataFrameFormatter(object):
     """
     Render a DataFrame
@@ -204,7 +210,7 @@ def to_string(self, force_unicode=False):
                 if self.header:
                     fmt_values = self._format_col(i)
                     cheader = str_columns[i]
-                    max_len = max(max(len(x) for x in fmt_values),
+                    max_len = max(max(_strlen(x) for x in fmt_values),
                                   max(len(x) for x in cheader))
                     if self.justify == 'left':
                         cheader = [x.ljust(max_len) for x in cheader]
@@ -606,7 +612,7 @@ def _make_fixed_width(strings, justify='right'):
     if len(strings) == 0:
         return strings
 
-    max_len = max(len(x) for x in strings)
+    max_len = max(_strlen(x) for x in strings)
     conf_max = print_config.max_colwidth
     if conf_max is not None and max_len > conf_max:
         max_len = conf_max
@@ -617,7 +623,12 @@ def _make_fixed_width(strings, justify='right'):
         justfunc = lambda self, x: self.rjust(x)
 
     def just(x):
-        return justfunc(x[:max_len], max_len)
+        try:
+            eff_len = max_len + _encode_diff(x)
+        except UnicodeError:
+            eff_len = max_len
+
+        return justfunc(x[:eff_len], eff_len)
 
     return [just(x) for x in strings]
 

pandas/tests/test_frame.py

@@ -2574,6 +2574,18 @@ def test_repr(self):
 
         fmt.reset_printoptions()
 
+    def test_repr_unicode(self):
+        df = DataFrame({'A': ['\xc3\xa4\xc3\xa4\xc3\xa4\xc3\xa4',
+                              '\xc3\xbc\xc3\xbc\xc3\xbc\xc3\xbc']})
+
+        result = repr(df)
+        ex_top = '      A'
+        self.assertEqual(result.split('\n')[0].rstrip(), ex_top)
+
+        df = DataFrame({'A': [u'\xe4\xe4\xe4\xe4', u'\xfc\xfc\xfc\xfc']})
+        result = repr(df)
+        self.assertEqual(result.split('\n')[0].rstrip(), ex_top)
+
     def test_very_wide_info_repr(self):
         df = DataFrame(np.random.randn(10, 20),
                        columns=[tm.rands(10) for _ in xrange(20)])