BUG in clipboard (linux, python2) with unicode and separator (GH13747)

vendered updated version of Pyperclip

closes pandas-dev#13747
closes pandas-dev#14362
closes pandas-dev#12807
closes pandas-dev#12529

Author: Ajay Saxena <[email protected]>
Author: Ajay Saxena <[email protected]>

Closes pandas-dev#14599 from aileronajay/master and squashes the following commits:

2aafb66 [Ajay Saxena] moved comment inside test and added github issue labels to test
b74fbc1 [Ajay Saxena] ignore lint test for pyperclip files
9db42d8 [Ajay Saxena] whatsnew conflict
1dca292 [Ajay Saxena] conflict resolution
98b61e8 [Ajay Saxena] merge conflict
cedb690 [Ajay Saxena] merge conflict in whats new file
7af95da [Ajay Saxena] merging lastest changes
ac8ae60 [Ajay Saxena] skip clipboard test if clipboard primitives are absent
b03ed56 [Ajay Saxena] changed whatsnew file
c0aafd7 [Ajay Saxena] Merge branch 'test_branch'
9946fb7 [Ajay Saxena] Merge branch 'master' of https://github.com/pandas-dev/pandas into test_branch
ed1375f [Ajay Saxena] Merge branch 'test_branch'
0665fd4 [Ajay Saxena] fixed linting and test case as per code review
d202fd0 [Ajay Saxena] added test for valid encoding, modified setup.py so that pandas/util/clipboard can be found
dd57ae3 [Ajay Saxena] code review changes and read clipboard invalid encoding test
71d58d0 [Ajay Saxena] testing encoding in kwargs to to_clipboard and test case for the same
02f87b0 [Ajay Saxena] removed duplicate files
825bbe2 [Ajay Saxena] all files related to pyperclip are under pandas.util.clipboard
c5a87d8 [Ajay Saxena] Merge branch 'test_branch' of https://github.com/aileronajay/pandas into test_branch
f708c2e [Ajay Saxena] Merge branch 'master' of https://github.com/aileronajay/pandas
d565b1f [Ajay Saxena] updated pyperclip to the latest version
14d94a0 [Ajay Saxena] changed the pandas util clipboard file to return unicode if the python version is 2, else str
66d8ebf [Ajay Saxena] removed the disabled tag for clipboard test so that we can check if they pass after this change
edb8553 [Ajay Saxena] refactored the new unicode test to be in sync with the rest of the file
c83d000 [Ajay Saxena] added test case for unicode round trip
fb922d6 [Ajay Saxena] changes for GH 13747

(cherry picked from commit 4a1a330)

Author: Ajay Saxena

doc/source/whatsnew/v0.19.2.txt

@@ -53,4 +53,14 @@ Bug Fixes
 
 
 
+
+
+- Bug in clipboard functions on linux with python2 with unicode and separators (:issue:`13747`)
+- Bug in clipboard functions on Windows 10 and python 3 (:issue:`14362`, :issue:`12807`)
+- Bug in ``.to_clipboard()`` and Excel compat (:issue:`12529`)
+
+
+
+
+
 - Explicit check in ``to_stata`` and ``StataWriter`` for out-of-range values when writing doubles (:issue:`14618`)

pandas/io/clipboard.py

@@ -1,6 +1,6 @@
 """ io on the clipboard """
 from pandas import compat, get_option, option_context, DataFrame
-from pandas.compat import StringIO
+from pandas.compat import StringIO, PY2
 
 
 def read_clipboard(**kwargs):  # pragma: no cover
@@ -14,6 +14,14 @@ def read_clipboard(**kwargs):  # pragma: no cover
     -------
     parsed : DataFrame
     """
+    encoding = kwargs.pop('encoding', 'utf-8')
+
+    # only utf-8 is valid for passed value because that's what clipboard
+    # supports
+    if encoding is not None and encoding.lower().replace('-', '') != 'utf8':
+        raise NotImplementedError(
+            'reading from clipboard only supports utf-8 encoding')
+
     from pandas.util.clipboard import clipboard_get
     from pandas.io.parsers import read_table
     text = clipboard_get()
@@ -74,6 +82,12 @@ def to_clipboard(obj, excel=None, sep=None, **kwargs):  # pragma: no cover
       - Windows:
       - OS X:
     """
+    encoding = kwargs.pop('encoding', 'utf-8')
+
+    # testing if an invalid encoding is passed to clipboard
+    if encoding is not None and encoding.lower().replace('-', '') != 'utf8':
+        raise ValueError('clipboard only supports utf-8 encoding')
+
     from pandas.util.clipboard import clipboard_set
     if excel is None:
         excel = True
@@ -83,8 +97,12 @@ def to_clipboard(obj, excel=None, sep=None, **kwargs):  # pragma: no cover
             if sep is None:
                 sep = '\t'
             buf = StringIO()
-            obj.to_csv(buf, sep=sep, **kwargs)
-            clipboard_set(buf.getvalue())
+            # clipboard_set (pyperclip) expects unicode
+            obj.to_csv(buf, sep=sep, encoding='utf-8', **kwargs)
+            text = buf.getvalue()
+            if PY2:
+                text = text.decode('utf-8')
+            clipboard_set(text)
             return
         except:
             pass

pandas/io/tests/test_clipboard.py

@@ -9,16 +9,16 @@
 from pandas import read_clipboard
 from pandas import get_option
 from pandas.util import testing as tm
-from pandas.util.testing import makeCustomDataframe as mkdf, disabled
+from pandas.util.testing import makeCustomDataframe as mkdf
+from pandas.util.clipboard.exceptions import PyperclipException
 
 
 try:
-    import pandas.util.clipboard  # noqa
-except OSError:
-    raise nose.SkipTest("no clipboard found")
+    DataFrame({'A': [1, 2]}).to_clipboard()
+except PyperclipException:
+    raise nose.SkipTest("clipboard primitives not installed")
 
 
-@disabled
 class TestClipboard(tm.TestCase):
 
     @classmethod
@@ -52,20 +52,24 @@ def setUpClass(cls):
         # Test for non-ascii text: GH9263
         cls.data['nonascii'] = pd.DataFrame({'en': 'in English'.split(),
                                              'es': 'en español'.split()})
+        # unicode round trip test for GH 13747, GH 12529
+        cls.data['utf8'] = pd.DataFrame({'a': ['µasd', 'Ωœ∑´'],
+                                        'b': ['øπ∆˚¬', 'œ∑´®']})
         cls.data_types = list(cls.data.keys())
 
     @classmethod
     def tearDownClass(cls):
         super(TestClipboard, cls).tearDownClass()
         del cls.data_types, cls.data
 
-    def check_round_trip_frame(self, data_type, excel=None, sep=None):
+    def check_round_trip_frame(self, data_type, excel=None, sep=None,
+                               encoding=None):
         data = self.data[data_type]
-        data.to_clipboard(excel=excel, sep=sep)
+        data.to_clipboard(excel=excel, sep=sep, encoding=encoding)
         if sep is not None:
-            result = read_clipboard(sep=sep, index_col=0)
+            result = read_clipboard(sep=sep, index_col=0, encoding=encoding)
         else:
-            result = read_clipboard()
+            result = read_clipboard(encoding=encoding)
         tm.assert_frame_equal(data, result, check_dtype=False)
 
     def test_round_trip_frame_sep(self):
@@ -113,3 +117,16 @@ def test_read_clipboard_infer_excel(self):
         exp = pd.read_clipboard()
 
         tm.assert_frame_equal(res, exp)
+
+    def test_invalid_encoding(self):
+        # test case for testing invalid encoding
+        data = self.data['string']
+        with tm.assertRaises(ValueError):
+            data.to_clipboard(encoding='ascii')
+        with tm.assertRaises(NotImplementedError):
+            pd.read_clipboard(encoding='ascii')
+
+    def test_round_trip_valid_encodings(self):
+        for enc in ['UTF-8', 'utf-8', 'utf8']:
+            for dt in self.data_types:
+                self.check_round_trip_frame(dt, encoding=enc)