From fb922d6a0f3044268621cd0668798586a38c4daa Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Mon, 7 Nov 2016 00:40:43 -0500 Subject: [PATCH 01/20] changes for GH 13747 --- pandas/io/clipboard.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandas/io/clipboard.py b/pandas/io/clipboard.py index 2109e1c5d6d4c..73883deeb37c8 100644 --- a/pandas/io/clipboard.py +++ b/pandas/io/clipboard.py @@ -1,6 +1,6 @@ """ io on the clipboard """ from pandas import compat, get_option, option_context, DataFrame -from pandas.compat import StringIO +from pandas.compat import StringIO, PY2 def read_clipboard(**kwargs): # pragma: no cover @@ -83,8 +83,12 @@ def to_clipboard(obj, excel=None, sep=None, **kwargs): # pragma: no cover if sep is None: sep = '\t' buf = StringIO() - obj.to_csv(buf, sep=sep, **kwargs) - clipboard_set(buf.getvalue()) + # clipboard_set (pyperclip) expects unicode + obj.to_csv(buf, sep=sep, encoding='utf-8', **kwargs) + text = buf.getvalue() + if PY2: + text = text.decode('utf-8') + clipboard_set(text) return except: pass From c83d0003aa8108c821420c23137a703881eed70b Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Mon, 7 Nov 2016 23:04:10 -0500 Subject: [PATCH 02/20] added test case for unicode round trip --- pandas/io/tests/test_clipboard.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index a7da27a2f75dd..6aefdc53bf742 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -113,3 +113,12 @@ def test_read_clipboard_infer_excel(self): exp = pd.read_clipboard() tm.assert_frame_equal(res, exp) + + # unicode round trip test for GH 13747 + def test_round_trip_frame_unicode(self): + sep = ',' + df = pd.DataFrame({'a':['µasd','Ωœ∑´'], 'b':['øπ∆˚¬','œ∑´®']}) + df.to_clipboard(excel=None, sep =sep) + result = read_clipboard(sep = sep, index_col = 0) + tm.assert_frame_equal(df, result, check_dtype=False) + From edb855320eebdc7c536d4c028788e6725b3e4bc0 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Tue, 8 Nov 2016 00:28:19 -0500 Subject: [PATCH 03/20] refactored the new unicode test to be in sync with the rest of the file --- pandas/io/tests/test_clipboard.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index 6aefdc53bf742..e8b3622564aad 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -52,6 +52,9 @@ def setUpClass(cls): # Test for non-ascii text: GH9263 cls.data['nonascii'] = pd.DataFrame({'en': 'in English'.split(), 'es': 'en español'.split()}) + + # unicode round trip test for GH 13747 + cls.data['utf8'] = pd.DataFrame({'a':['µasd','Ωœ∑´'], 'b':['øπ∆˚¬','œ∑´®']}) cls.data_types = list(cls.data.keys()) @classmethod @@ -114,11 +117,3 @@ def test_read_clipboard_infer_excel(self): tm.assert_frame_equal(res, exp) - # unicode round trip test for GH 13747 - def test_round_trip_frame_unicode(self): - sep = ',' - df = pd.DataFrame({'a':['µasd','Ωœ∑´'], 'b':['øπ∆˚¬','œ∑´®']}) - df.to_clipboard(excel=None, sep =sep) - result = read_clipboard(sep = sep, index_col = 0) - tm.assert_frame_equal(df, result, check_dtype=False) - From 66d8ebfb4308e67f7ff9246ba2669228c61ddb8d Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Tue, 8 Nov 2016 17:32:45 -0500 Subject: [PATCH 04/20] removed the disabled tag for clipboard test so that we can check if they pass after this change --- pandas/io/tests/test_clipboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index e8b3622564aad..dd28fd14b8bba 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -18,7 +18,7 @@ raise nose.SkipTest("no clipboard found") -@disabled + class TestClipboard(tm.TestCase): @classmethod From 14d94a01a44f912d873ec168a8e566b798aacc7b Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Fri, 11 Nov 2016 22:39:03 +0000 Subject: [PATCH 05/20] changed the pandas util clipboard file to return unicode if the python version is 2, else str --- pandas/util/clipboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/util/clipboard.py b/pandas/util/clipboard.py index 02da0d5b8159f..850fb9edbd1e2 100644 --- a/pandas/util/clipboard.py +++ b/pandas/util/clipboard.py @@ -138,7 +138,7 @@ def _copyGtk(text): def _pasteQt(): - return str(cb.text()) + return text_type(cb.text()) def _copyQt(text): From d565b1ffdf2db4235eb13bf2b09ecb4557077f9e Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Fri, 11 Nov 2016 19:58:13 -0500 Subject: [PATCH 06/20] updated pyperclip to the latest version --- pandas/util/clipboard.py | 330 ++++++++++---------------------------- pandas/util/clipboards.py | 134 ++++++++++++++++ pandas/util/exceptions.py | 11 ++ pandas/util/windows.py | 151 +++++++++++++++++ 4 files changed, 382 insertions(+), 244 deletions(-) create mode 100644 pandas/util/clipboards.py create mode 100644 pandas/util/exceptions.py create mode 100644 pandas/util/windows.py diff --git a/pandas/util/clipboard.py b/pandas/util/clipboard.py index 02da0d5b8159f..b7f33681592ee 100644 --- a/pandas/util/clipboard.py +++ b/pandas/util/clipboard.py @@ -1,266 +1,108 @@ -# Pyperclip v1.5.15 -# A cross-platform clipboard module for Python. -# By Al Sweigart al@inventwithpython.com +""" +Pyperclip -# Usage: -# import pyperclip -# pyperclip.copy('The text to be copied to the clipboard.') -# spam = pyperclip.paste() +A cross-platform clipboard module for Python. (only handles plain text for now) +By Al Sweigart al@inventwithpython.com +BSD License -# On Windows, no additional modules are needed. -# On Mac, this module makes use of the pbcopy and pbpaste commands, which -# should come with the os. -# On Linux, this module makes use of the xclip or xsel commands, which should -# come with the os. Otherwise run "sudo apt-get install xclip" or -# "sudo apt-get install xsel" -# Otherwise on Linux, you will need the gtk or PyQt4 modules installed. -# The gtk module is not available for Python 3, and this module does not work -# with PyGObject yet. +Usage: + import pyperclip + pyperclip.copy('The text to be copied to the clipboard.') + spam = pyperclip.paste() + if not pyperclip.copy: + print("Copy functionality unavailable!") -# Copyright (c) 2015, Albert Sweigart -# All rights reserved. -# -# BSD-style license: -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the pyperclip nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY Albert Sweigart "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL Albert Sweigart BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +On Windows, no additional modules are needed. +On Mac, the module uses pbcopy and pbpaste, which should come with the os. +On Linux, install xclip or xsel via package manager. For example, in Debian: +sudo apt-get install xclip -# flake8: noqa +Otherwise on Linux, you will need the gtk or PyQt4 modules installed. + +gtk and PyQt4 modules are not available for Python 3, +and this module does not work with PyGObject yet. +""" +__version__ = '1.5.27' import platform import os -from subprocess import call, Popen, PIPE - -PY2 = '2' == platform.python_version_tuple()[0] -text_type = unicode if PY2 else str - - -class NoClipboardProgramError(OSError): - pass - - -def _pasteWindows(): - CF_UNICODETEXT = 13 - d = ctypes.windll - d.user32.OpenClipboard(0) - handle = d.user32.GetClipboardData(CF_UNICODETEXT) - data = ctypes.c_wchar_p(handle).value - d.user32.CloseClipboard() - return data - - -def _copyWindows(text): - GMEM_DDESHARE = 0x2000 - CF_UNICODETEXT = 13 - d = ctypes.windll # cdll expects 4 more bytes in user32.OpenClipboard(0) - if not isinstance(text, text_type): - text = text.decode('mbcs') - - d.user32.OpenClipboard(0) - - d.user32.EmptyClipboard() - hCd = d.kernel32.GlobalAlloc(GMEM_DDESHARE, - len(text.encode('utf-16-le')) + 2) - pchData = d.kernel32.GlobalLock(hCd) - ctypes.cdll.msvcrt.wcscpy(ctypes.c_wchar_p(pchData), text) - d.kernel32.GlobalUnlock(hCd) - d.user32.SetClipboardData(CF_UNICODETEXT, hCd) - d.user32.CloseClipboard() - - -def _pasteCygwin(): - CF_UNICODETEXT = 13 - d = ctypes.cdll - d.user32.OpenClipboard(0) - handle = d.user32.GetClipboardData(CF_UNICODETEXT) - data = ctypes.c_wchar_p(handle).value - d.user32.CloseClipboard() - return data - - -def _copyCygwin(text): - GMEM_DDESHARE = 0x2000 - CF_UNICODETEXT = 13 - d = ctypes.cdll - if not isinstance(text, text_type): - text = text.decode('mbcs') - d.user32.OpenClipboard(0) - d.user32.EmptyClipboard() - hCd = d.kernel32.GlobalAlloc(GMEM_DDESHARE, - len(text.encode('utf-16-le')) + 2) - pchData = d.kernel32.GlobalLock(hCd) - ctypes.cdll.msvcrt.wcscpy(ctypes.c_wchar_p(pchData), text) - d.kernel32.GlobalUnlock(hCd) - d.user32.SetClipboardData(CF_UNICODETEXT, hCd) - d.user32.CloseClipboard() - - -def _copyOSX(text): - p = Popen(['pbcopy', 'w'], stdin=PIPE, close_fds=True) - p.communicate(input=text.encode('utf-8')) - - -def _pasteOSX(): - p = Popen(['pbpaste', 'r'], stdout=PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode('utf-8') - - -def _pasteGtk(): - return gtk.Clipboard().wait_for_text() - - -def _copyGtk(text): - global cb - cb = gtk.Clipboard() - cb.set_text(text) - cb.store() - - -def _pasteQt(): - return str(cb.text()) - - -def _copyQt(text): - cb.setText(text) - - -def _copyXclip(text): - p = Popen(['xclip', '-selection', 'c'], stdin=PIPE, close_fds=True) - p.communicate(input=text.encode('utf-8')) - - -def _pasteXclip(): - p = Popen(['xclip', '-selection', 'c', '-o'], stdout=PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode('utf-8') - - -def _copyXsel(text): - p = Popen(['xsel', '-b', '-i'], stdin=PIPE, close_fds=True) - p.communicate(input=text.encode('utf-8')) - - -def _pasteXsel(): - p = Popen(['xsel', '-b', '-o'], stdout=PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode('utf-8') +import subprocess +from .clipboards import (init_osx_clipboard, + init_gtk_clipboard, init_qt_clipboard, + init_xclip_clipboard, init_xsel_clipboard, + init_klipper_clipboard, init_no_clipboard) +from .windows import init_windows_clipboard + +# `import PyQt4` sys.exit()s if DISPLAY is not in the environment. +# Thus, we need to detect the presence of $DISPLAY manually +# and not load PyQt4 if it is absent. +HAS_DISPLAY = os.getenv("DISPLAY", False) +CHECK_CMD = "where" if platform.system() == "Windows" else "which" + + +def _executable_exists(name): + return subprocess.call([CHECK_CMD, name], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + + +def determine_clipboard(): + # Determine the OS/platform and set + # the copy() and paste() functions accordingly. + if 'cygwin' in platform.system().lower(): + # FIXME: pyperclip currently does not support Cygwin, + # see https://github.com/asweigart/pyperclip/issues/55 + pass + elif os.name == 'nt' or platform.system() == 'Windows': + return init_windows_clipboard() + if os.name == 'mac' or platform.system() == 'Darwin': + return init_osx_clipboard() + if HAS_DISPLAY: + # Determine which command/module is installed, if any. + try: + import gtk # check if gtk is installed + except ImportError: + pass + else: + return init_gtk_clipboard() + try: + import PyQt4 # check if PyQt4 is installed + except ImportError: + pass + else: + return init_qt_clipboard() -def _copyKlipper(text): - p = Popen(['qdbus', 'org.kde.klipper', '/klipper', - 'setClipboardContents', text.encode('utf-8')], - stdin=PIPE, close_fds=True) - p.communicate(input=None) + if _executable_exists("xclip"): + return init_xclip_clipboard() + if _executable_exists("xsel"): + return init_xsel_clipboard() + if _executable_exists("klipper") and _executable_exists("qdbus"): + return init_klipper_clipboard() + return init_no_clipboard() -def _pasteKlipper(): - p = Popen(['qdbus', 'org.kde.klipper', '/klipper', - 'getClipboardContents'], stdout=PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode('utf-8') +def set_clipboard(clipboard): + global copy, paste -# Determine the OS/platform and set the copy() and paste() functions -# accordingly. -if 'cygwin' in platform.system().lower(): - _functions = 'Cygwin' # for debugging - import ctypes - paste = _pasteCygwin - copy = _copyCygwin -elif os.name == 'nt' or platform.system() == 'Windows': - _functions = 'Windows' # for debugging - import ctypes - paste = _pasteWindows - copy = _copyWindows -elif os.name == 'mac' or platform.system() == 'Darwin': - _functions = 'OS X pbcopy/pbpaste' # for debugging - paste = _pasteOSX - copy = _copyOSX -elif os.name == 'posix' or platform.system() == 'Linux': - # Determine which command/module is installed, if any. - xclipExists = call(['which', 'xclip'], - stdout=PIPE, stderr=PIPE) == 0 + clipboard_types = {'osx': init_osx_clipboard, + 'gtk': init_gtk_clipboard, + 'qt': init_qt_clipboard, + 'xclip': init_xclip_clipboard, + 'xsel': init_xsel_clipboard, + 'klipper': init_klipper_clipboard, + 'windows': init_windows_clipboard, + 'no': init_no_clipboard} - xselExists = call(['which', 'xsel'], - stdout=PIPE, stderr=PIPE) == 0 + copy, paste = clipboard_types[clipboard]() - xklipperExists = ( - call(['which', 'klipper'], stdout=PIPE, stderr=PIPE) == 0 and - call(['which', 'qdbus'], stdout=PIPE, stderr=PIPE) == 0 - ) - gtkInstalled = False - try: - # Check it gtk is installed. - import gtk - gtkInstalled = True - except ImportError: - pass +copy, paste = determine_clipboard() - if not gtkInstalled: - # Check for either PyQt4 or PySide - qtBindingInstalled = True - try: - from PyQt4 import QtGui - except ImportError: - try: - from PySide import QtGui - except ImportError: - qtBindingInstalled = False +__all__ = ["copy", "paste"] - # Set one of the copy & paste functions. - if xclipExists: - _functions = 'xclip command' # for debugging - paste = _pasteXclip - copy = _copyXclip - elif xklipperExists: - _functions = '(KDE Klipper) - qdbus (external)' # for debugging - paste = _pasteKlipper - copy = _copyKlipper - elif gtkInstalled: - _functions = 'gtk module' # for debugging - paste = _pasteGtk - copy = _copyGtk - elif qtBindingInstalled: - _functions = 'PyQt4 module' # for debugging - app = QtGui.QApplication([]) - cb = QtGui.QApplication.clipboard() - paste = _pasteQt - copy = _copyQt - elif xselExists: - # TODO: xsel doesn't seem to work on Raspberry Pi (my test Linux - # environment). Putting this as the last method tried. - _functions = 'xsel command' # for debugging - paste = _pasteXsel - copy = _copyXsel - else: - raise NoClipboardProgramError('Pyperclip requires the gtk, PyQt4, or ' - 'PySide module installed, or either the ' - 'xclip or xsel command.') -else: - raise RuntimeError('pyperclip does not support your system.') # pandas aliases clipboard_get = paste -clipboard_set = copy +clipboard_set = copy \ No newline at end of file diff --git a/pandas/util/clipboards.py b/pandas/util/clipboards.py new file mode 100644 index 0000000000000..5eac945628a5d --- /dev/null +++ b/pandas/util/clipboards.py @@ -0,0 +1,134 @@ +import sys +import subprocess +from .exceptions import PyperclipException + +EXCEPT_MSG = """ + Pyperclip could not find a copy/paste mechanism for your system. + For more information, please visit https://pyperclip.readthedocs.org """ +PY2 = sys.version_info[0] == 2 +text_type = unicode if PY2 else str + + +def init_osx_clipboard(): + def copy_osx(text): + p = subprocess.Popen(['pbcopy', 'w'], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=text.encode('utf-8')) + + def paste_osx(): + p = subprocess.Popen(['pbpaste', 'r'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + return stdout.decode('utf-8') + + return copy_osx, paste_osx + + +def init_gtk_clipboard(): + import gtk + + def copy_gtk(text): + global cb + cb = gtk.Clipboard() + cb.set_text(text) + cb.store() + + def paste_gtk(): + clipboardContents = gtk.Clipboard().wait_for_text() + # for python 2, returns None if the clipboard is blank. + if clipboardContents is None: + return '' + else: + return clipboardContents + + return copy_gtk, paste_gtk + + +def init_qt_clipboard(): + # $DISPLAY should exist + from PyQt4.QtGui import QApplication + + app = QApplication([]) + + def copy_qt(text): + cb = app.clipboard() + cb.setText(text) + + def paste_qt(): + cb = app.clipboard() + return text_type(cb.text()) + + return copy_qt, paste_qt + + +def init_xclip_clipboard(): + def copy_xclip(text): + p = subprocess.Popen(['xclip', '-selection', 'c'], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=text.encode('utf-8')) + + def paste_xclip(): + p = subprocess.Popen(['xclip', '-selection', 'c', '-o'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + return stdout.decode('utf-8') + + return copy_xclip, paste_xclip + + +def init_xsel_clipboard(): + def copy_xsel(text): + p = subprocess.Popen(['xsel', '-b', '-i'], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=text.encode('utf-8')) + + def paste_xsel(): + p = subprocess.Popen(['xsel', '-b', '-o'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + return stdout.decode('utf-8') + + return copy_xsel, paste_xsel + + +def init_klipper_clipboard(): + def copy_klipper(text): + p = subprocess.Popen( + ['qdbus', 'org.kde.klipper', '/klipper', 'setClipboardContents', + text.encode('utf-8')], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=None) + + def paste_klipper(): + p = subprocess.Popen( + ['qdbus', 'org.kde.klipper', '/klipper', 'getClipboardContents'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + + # Workaround for https://bugs.kde.org/show_bug.cgi?id=342874 + # TODO: https://github.com/asweigart/pyperclip/issues/43 + clipboardContents = stdout.decode('utf-8') + # even if blank, Klipper will append a newline at the end + assert len(clipboardContents) > 0 + # make sure that newline is there + assert clipboardContents.endswith('\n') + if clipboardContents.endswith('\n'): + clipboardContents = clipboardContents[:-1] + return clipboardContents + + return copy_klipper, paste_klipper + + +def init_no_clipboard(): + class ClipboardUnavailable(object): + def __call__(self, *args, **kwargs): + raise PyperclipException(EXCEPT_MSG) + + if PY2: + def __nonzero__(self): + return False + else: + def __bool__(self): + return False + + return ClipboardUnavailable(), ClipboardUnavailable() diff --git a/pandas/util/exceptions.py b/pandas/util/exceptions.py new file mode 100644 index 0000000000000..c5ba3e75d2a76 --- /dev/null +++ b/pandas/util/exceptions.py @@ -0,0 +1,11 @@ +import ctypes + + +class PyperclipException(RuntimeError): + pass + + +class PyperclipWindowsException(PyperclipException): + def __init__(self, message): + message += " (%s)" % ctypes.WinError() + super(PyperclipWindowsException, self).__init__(message) diff --git a/pandas/util/windows.py b/pandas/util/windows.py new file mode 100644 index 0000000000000..a12932a575f40 --- /dev/null +++ b/pandas/util/windows.py @@ -0,0 +1,151 @@ +""" +This module implements clipboard handling on Windows using ctypes. +""" +import time +import contextlib +import ctypes +from ctypes import c_size_t, sizeof, c_wchar_p, get_errno, c_wchar +from .exceptions import PyperclipWindowsException + + +class CheckedCall(object): + def __init__(self, f): + super(CheckedCall, self).__setattr__("f", f) + + def __call__(self, *args): + ret = self.f(*args) + if not ret and get_errno(): + raise PyperclipWindowsException("Error calling " + self.f.__name__) + return ret + + def __setattr__(self, key, value): + setattr(self.f, key, value) + + +def init_windows_clipboard(): + from ctypes.wintypes import (HGLOBAL, LPVOID, DWORD, LPCSTR, INT, HWND, + HINSTANCE, HMENU, BOOL, UINT, HANDLE) + + windll = ctypes.windll + + safeCreateWindowExA = CheckedCall(windll.user32.CreateWindowExA) + safeCreateWindowExA.argtypes = [DWORD, LPCSTR, LPCSTR, DWORD, INT, INT, + INT, INT, HWND, HMENU, HINSTANCE, LPVOID] + safeCreateWindowExA.restype = HWND + + safeDestroyWindow = CheckedCall(windll.user32.DestroyWindow) + safeDestroyWindow.argtypes = [HWND] + safeDestroyWindow.restype = BOOL + + OpenClipboard = windll.user32.OpenClipboard + OpenClipboard.argtypes = [HWND] + OpenClipboard.restype = BOOL + + safeCloseClipboard = CheckedCall(windll.user32.CloseClipboard) + safeCloseClipboard.argtypes = [] + safeCloseClipboard.restype = BOOL + + safeEmptyClipboard = CheckedCall(windll.user32.EmptyClipboard) + safeEmptyClipboard.argtypes = [] + safeEmptyClipboard.restype = BOOL + + safeGetClipboardData = CheckedCall(windll.user32.GetClipboardData) + safeGetClipboardData.argtypes = [UINT] + safeGetClipboardData.restype = HANDLE + + safeSetClipboardData = CheckedCall(windll.user32.SetClipboardData) + safeSetClipboardData.argtypes = [UINT, HANDLE] + safeSetClipboardData.restype = HANDLE + + safeGlobalAlloc = CheckedCall(windll.kernel32.GlobalAlloc) + safeGlobalAlloc.argtypes = [UINT, c_size_t] + safeGlobalAlloc.restype = HGLOBAL + + safeGlobalLock = CheckedCall(windll.kernel32.GlobalLock) + safeGlobalLock.argtypes = [HGLOBAL] + safeGlobalLock.restype = LPVOID + + safeGlobalUnlock = CheckedCall(windll.kernel32.GlobalUnlock) + safeGlobalUnlock.argtypes = [HGLOBAL] + safeGlobalUnlock.restype = BOOL + + GMEM_MOVEABLE = 0x0002 + CF_UNICODETEXT = 13 + + @contextlib.contextmanager + def window(): + """ + Context that provides a valid Windows hwnd. + """ + # we really just need the hwnd, so setting "STATIC" + # as predefined lpClass is just fine. + hwnd = safeCreateWindowExA(0, b"STATIC", None, 0, 0, 0, 0, 0, + None, None, None, None) + try: + yield hwnd + finally: + safeDestroyWindow(hwnd) + + @contextlib.contextmanager + def clipboard(hwnd): + """ + Context manager that opens the clipboard and prevents + other applications from modifying the clipboard content. + """ + # We may not get the clipboard handle immediately because + # some other application is accessing it (?) + # We try for at least 500ms to get the clipboard. + t = time.time() + 0.5 + success = False + while time.time() < t: + success = OpenClipboard(hwnd) + if success: + break + time.sleep(0.01) + if not success: + raise PyperclipWindowsException("Error calling OpenClipboard") + + try: + yield + finally: + safeCloseClipboard() + + def copy_windows(text): + # This function is heavily based on + # http://msdn.com/ms649016#_win32_Copying_Information_to_the_Clipboard + with window() as hwnd: + # http://msdn.com/ms649048 + # If an application calls OpenClipboard with hwnd set to NULL, + # EmptyClipboard sets the clipboard owner to NULL; + # this causes SetClipboardData to fail. + # => We need a valid hwnd to copy something. + with clipboard(hwnd): + safeEmptyClipboard() + + if text: + # http://msdn.com/ms649051 + # If the hMem parameter identifies a memory object, + # the object must have been allocated using the + # function with the GMEM_MOVEABLE flag. + count = len(text) + 1 + handle = safeGlobalAlloc(GMEM_MOVEABLE, + count * sizeof(c_wchar)) + locked_handle = safeGlobalLock(handle) + + ctypes.memmove(c_wchar_p(locked_handle), c_wchar_p(text), count * sizeof(c_wchar)) + + safeGlobalUnlock(handle) + safeSetClipboardData(CF_UNICODETEXT, handle) + + def paste_windows(): + with clipboard(None): + handle = safeGetClipboardData(CF_UNICODETEXT) + if not handle: + # GetClipboardData may return NULL with errno == NO_ERROR + # if the clipboard is empty. + # (Also, it may return a handle to an empty buffer, + # but technically that's not empty) + return "" + return c_wchar_p(handle).value + + return copy_windows, paste_windows From 825bbe21542fc1b84d31a0abf02e682bddbfc17d Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Fri, 11 Nov 2016 23:13:13 -0500 Subject: [PATCH 07/20] all files related to pyperclip are under pandas.util.clipboard --- pandas/util/clipboard/__init__.py | 108 ++++++++++++++++++++ pandas/util/clipboard/clipboards.py | 134 ++++++++++++++++++++++++ pandas/util/clipboard/exceptions.py | 11 ++ pandas/util/clipboard/windows.py | 151 ++++++++++++++++++++++++++++ 4 files changed, 404 insertions(+) create mode 100644 pandas/util/clipboard/__init__.py create mode 100644 pandas/util/clipboard/clipboards.py create mode 100644 pandas/util/clipboard/exceptions.py create mode 100644 pandas/util/clipboard/windows.py diff --git a/pandas/util/clipboard/__init__.py b/pandas/util/clipboard/__init__.py new file mode 100644 index 0000000000000..b7f33681592ee --- /dev/null +++ b/pandas/util/clipboard/__init__.py @@ -0,0 +1,108 @@ +""" +Pyperclip + +A cross-platform clipboard module for Python. (only handles plain text for now) +By Al Sweigart al@inventwithpython.com +BSD License + +Usage: + import pyperclip + pyperclip.copy('The text to be copied to the clipboard.') + spam = pyperclip.paste() + + if not pyperclip.copy: + print("Copy functionality unavailable!") + +On Windows, no additional modules are needed. +On Mac, the module uses pbcopy and pbpaste, which should come with the os. +On Linux, install xclip or xsel via package manager. For example, in Debian: +sudo apt-get install xclip + +Otherwise on Linux, you will need the gtk or PyQt4 modules installed. + +gtk and PyQt4 modules are not available for Python 3, +and this module does not work with PyGObject yet. +""" +__version__ = '1.5.27' + +import platform +import os +import subprocess +from .clipboards import (init_osx_clipboard, + init_gtk_clipboard, init_qt_clipboard, + init_xclip_clipboard, init_xsel_clipboard, + init_klipper_clipboard, init_no_clipboard) +from .windows import init_windows_clipboard + +# `import PyQt4` sys.exit()s if DISPLAY is not in the environment. +# Thus, we need to detect the presence of $DISPLAY manually +# and not load PyQt4 if it is absent. +HAS_DISPLAY = os.getenv("DISPLAY", False) +CHECK_CMD = "where" if platform.system() == "Windows" else "which" + + +def _executable_exists(name): + return subprocess.call([CHECK_CMD, name], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + + +def determine_clipboard(): + # Determine the OS/platform and set + # the copy() and paste() functions accordingly. + if 'cygwin' in platform.system().lower(): + # FIXME: pyperclip currently does not support Cygwin, + # see https://github.com/asweigart/pyperclip/issues/55 + pass + elif os.name == 'nt' or platform.system() == 'Windows': + return init_windows_clipboard() + if os.name == 'mac' or platform.system() == 'Darwin': + return init_osx_clipboard() + if HAS_DISPLAY: + # Determine which command/module is installed, if any. + try: + import gtk # check if gtk is installed + except ImportError: + pass + else: + return init_gtk_clipboard() + + try: + import PyQt4 # check if PyQt4 is installed + except ImportError: + pass + else: + return init_qt_clipboard() + + if _executable_exists("xclip"): + return init_xclip_clipboard() + if _executable_exists("xsel"): + return init_xsel_clipboard() + if _executable_exists("klipper") and _executable_exists("qdbus"): + return init_klipper_clipboard() + + return init_no_clipboard() + + +def set_clipboard(clipboard): + global copy, paste + + clipboard_types = {'osx': init_osx_clipboard, + 'gtk': init_gtk_clipboard, + 'qt': init_qt_clipboard, + 'xclip': init_xclip_clipboard, + 'xsel': init_xsel_clipboard, + 'klipper': init_klipper_clipboard, + 'windows': init_windows_clipboard, + 'no': init_no_clipboard} + + copy, paste = clipboard_types[clipboard]() + + +copy, paste = determine_clipboard() + +__all__ = ["copy", "paste"] + + +# pandas aliases +clipboard_get = paste +clipboard_set = copy \ No newline at end of file diff --git a/pandas/util/clipboard/clipboards.py b/pandas/util/clipboard/clipboards.py new file mode 100644 index 0000000000000..5eac945628a5d --- /dev/null +++ b/pandas/util/clipboard/clipboards.py @@ -0,0 +1,134 @@ +import sys +import subprocess +from .exceptions import PyperclipException + +EXCEPT_MSG = """ + Pyperclip could not find a copy/paste mechanism for your system. + For more information, please visit https://pyperclip.readthedocs.org """ +PY2 = sys.version_info[0] == 2 +text_type = unicode if PY2 else str + + +def init_osx_clipboard(): + def copy_osx(text): + p = subprocess.Popen(['pbcopy', 'w'], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=text.encode('utf-8')) + + def paste_osx(): + p = subprocess.Popen(['pbpaste', 'r'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + return stdout.decode('utf-8') + + return copy_osx, paste_osx + + +def init_gtk_clipboard(): + import gtk + + def copy_gtk(text): + global cb + cb = gtk.Clipboard() + cb.set_text(text) + cb.store() + + def paste_gtk(): + clipboardContents = gtk.Clipboard().wait_for_text() + # for python 2, returns None if the clipboard is blank. + if clipboardContents is None: + return '' + else: + return clipboardContents + + return copy_gtk, paste_gtk + + +def init_qt_clipboard(): + # $DISPLAY should exist + from PyQt4.QtGui import QApplication + + app = QApplication([]) + + def copy_qt(text): + cb = app.clipboard() + cb.setText(text) + + def paste_qt(): + cb = app.clipboard() + return text_type(cb.text()) + + return copy_qt, paste_qt + + +def init_xclip_clipboard(): + def copy_xclip(text): + p = subprocess.Popen(['xclip', '-selection', 'c'], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=text.encode('utf-8')) + + def paste_xclip(): + p = subprocess.Popen(['xclip', '-selection', 'c', '-o'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + return stdout.decode('utf-8') + + return copy_xclip, paste_xclip + + +def init_xsel_clipboard(): + def copy_xsel(text): + p = subprocess.Popen(['xsel', '-b', '-i'], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=text.encode('utf-8')) + + def paste_xsel(): + p = subprocess.Popen(['xsel', '-b', '-o'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + return stdout.decode('utf-8') + + return copy_xsel, paste_xsel + + +def init_klipper_clipboard(): + def copy_klipper(text): + p = subprocess.Popen( + ['qdbus', 'org.kde.klipper', '/klipper', 'setClipboardContents', + text.encode('utf-8')], + stdin=subprocess.PIPE, close_fds=True) + p.communicate(input=None) + + def paste_klipper(): + p = subprocess.Popen( + ['qdbus', 'org.kde.klipper', '/klipper', 'getClipboardContents'], + stdout=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + + # Workaround for https://bugs.kde.org/show_bug.cgi?id=342874 + # TODO: https://github.com/asweigart/pyperclip/issues/43 + clipboardContents = stdout.decode('utf-8') + # even if blank, Klipper will append a newline at the end + assert len(clipboardContents) > 0 + # make sure that newline is there + assert clipboardContents.endswith('\n') + if clipboardContents.endswith('\n'): + clipboardContents = clipboardContents[:-1] + return clipboardContents + + return copy_klipper, paste_klipper + + +def init_no_clipboard(): + class ClipboardUnavailable(object): + def __call__(self, *args, **kwargs): + raise PyperclipException(EXCEPT_MSG) + + if PY2: + def __nonzero__(self): + return False + else: + def __bool__(self): + return False + + return ClipboardUnavailable(), ClipboardUnavailable() diff --git a/pandas/util/clipboard/exceptions.py b/pandas/util/clipboard/exceptions.py new file mode 100644 index 0000000000000..c5ba3e75d2a76 --- /dev/null +++ b/pandas/util/clipboard/exceptions.py @@ -0,0 +1,11 @@ +import ctypes + + +class PyperclipException(RuntimeError): + pass + + +class PyperclipWindowsException(PyperclipException): + def __init__(self, message): + message += " (%s)" % ctypes.WinError() + super(PyperclipWindowsException, self).__init__(message) diff --git a/pandas/util/clipboard/windows.py b/pandas/util/clipboard/windows.py new file mode 100644 index 0000000000000..a12932a575f40 --- /dev/null +++ b/pandas/util/clipboard/windows.py @@ -0,0 +1,151 @@ +""" +This module implements clipboard handling on Windows using ctypes. +""" +import time +import contextlib +import ctypes +from ctypes import c_size_t, sizeof, c_wchar_p, get_errno, c_wchar +from .exceptions import PyperclipWindowsException + + +class CheckedCall(object): + def __init__(self, f): + super(CheckedCall, self).__setattr__("f", f) + + def __call__(self, *args): + ret = self.f(*args) + if not ret and get_errno(): + raise PyperclipWindowsException("Error calling " + self.f.__name__) + return ret + + def __setattr__(self, key, value): + setattr(self.f, key, value) + + +def init_windows_clipboard(): + from ctypes.wintypes import (HGLOBAL, LPVOID, DWORD, LPCSTR, INT, HWND, + HINSTANCE, HMENU, BOOL, UINT, HANDLE) + + windll = ctypes.windll + + safeCreateWindowExA = CheckedCall(windll.user32.CreateWindowExA) + safeCreateWindowExA.argtypes = [DWORD, LPCSTR, LPCSTR, DWORD, INT, INT, + INT, INT, HWND, HMENU, HINSTANCE, LPVOID] + safeCreateWindowExA.restype = HWND + + safeDestroyWindow = CheckedCall(windll.user32.DestroyWindow) + safeDestroyWindow.argtypes = [HWND] + safeDestroyWindow.restype = BOOL + + OpenClipboard = windll.user32.OpenClipboard + OpenClipboard.argtypes = [HWND] + OpenClipboard.restype = BOOL + + safeCloseClipboard = CheckedCall(windll.user32.CloseClipboard) + safeCloseClipboard.argtypes = [] + safeCloseClipboard.restype = BOOL + + safeEmptyClipboard = CheckedCall(windll.user32.EmptyClipboard) + safeEmptyClipboard.argtypes = [] + safeEmptyClipboard.restype = BOOL + + safeGetClipboardData = CheckedCall(windll.user32.GetClipboardData) + safeGetClipboardData.argtypes = [UINT] + safeGetClipboardData.restype = HANDLE + + safeSetClipboardData = CheckedCall(windll.user32.SetClipboardData) + safeSetClipboardData.argtypes = [UINT, HANDLE] + safeSetClipboardData.restype = HANDLE + + safeGlobalAlloc = CheckedCall(windll.kernel32.GlobalAlloc) + safeGlobalAlloc.argtypes = [UINT, c_size_t] + safeGlobalAlloc.restype = HGLOBAL + + safeGlobalLock = CheckedCall(windll.kernel32.GlobalLock) + safeGlobalLock.argtypes = [HGLOBAL] + safeGlobalLock.restype = LPVOID + + safeGlobalUnlock = CheckedCall(windll.kernel32.GlobalUnlock) + safeGlobalUnlock.argtypes = [HGLOBAL] + safeGlobalUnlock.restype = BOOL + + GMEM_MOVEABLE = 0x0002 + CF_UNICODETEXT = 13 + + @contextlib.contextmanager + def window(): + """ + Context that provides a valid Windows hwnd. + """ + # we really just need the hwnd, so setting "STATIC" + # as predefined lpClass is just fine. + hwnd = safeCreateWindowExA(0, b"STATIC", None, 0, 0, 0, 0, 0, + None, None, None, None) + try: + yield hwnd + finally: + safeDestroyWindow(hwnd) + + @contextlib.contextmanager + def clipboard(hwnd): + """ + Context manager that opens the clipboard and prevents + other applications from modifying the clipboard content. + """ + # We may not get the clipboard handle immediately because + # some other application is accessing it (?) + # We try for at least 500ms to get the clipboard. + t = time.time() + 0.5 + success = False + while time.time() < t: + success = OpenClipboard(hwnd) + if success: + break + time.sleep(0.01) + if not success: + raise PyperclipWindowsException("Error calling OpenClipboard") + + try: + yield + finally: + safeCloseClipboard() + + def copy_windows(text): + # This function is heavily based on + # http://msdn.com/ms649016#_win32_Copying_Information_to_the_Clipboard + with window() as hwnd: + # http://msdn.com/ms649048 + # If an application calls OpenClipboard with hwnd set to NULL, + # EmptyClipboard sets the clipboard owner to NULL; + # this causes SetClipboardData to fail. + # => We need a valid hwnd to copy something. + with clipboard(hwnd): + safeEmptyClipboard() + + if text: + # http://msdn.com/ms649051 + # If the hMem parameter identifies a memory object, + # the object must have been allocated using the + # function with the GMEM_MOVEABLE flag. + count = len(text) + 1 + handle = safeGlobalAlloc(GMEM_MOVEABLE, + count * sizeof(c_wchar)) + locked_handle = safeGlobalLock(handle) + + ctypes.memmove(c_wchar_p(locked_handle), c_wchar_p(text), count * sizeof(c_wchar)) + + safeGlobalUnlock(handle) + safeSetClipboardData(CF_UNICODETEXT, handle) + + def paste_windows(): + with clipboard(None): + handle = safeGetClipboardData(CF_UNICODETEXT) + if not handle: + # GetClipboardData may return NULL with errno == NO_ERROR + # if the clipboard is empty. + # (Also, it may return a handle to an empty buffer, + # but technically that's not empty) + return "" + return c_wchar_p(handle).value + + return copy_windows, paste_windows From 02f87b0c84f6bffc00db60be2fca7b0501ea3623 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Sat, 12 Nov 2016 12:02:47 -0500 Subject: [PATCH 08/20] removed duplicate files --- pandas/util/clipboard.py | 108 --------------------------- pandas/util/clipboards.py | 134 --------------------------------- pandas/util/exceptions.py | 11 --- pandas/util/windows.py | 151 -------------------------------------- 4 files changed, 404 deletions(-) delete mode 100644 pandas/util/clipboard.py delete mode 100644 pandas/util/clipboards.py delete mode 100644 pandas/util/exceptions.py delete mode 100644 pandas/util/windows.py diff --git a/pandas/util/clipboard.py b/pandas/util/clipboard.py deleted file mode 100644 index b7f33681592ee..0000000000000 --- a/pandas/util/clipboard.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Pyperclip - -A cross-platform clipboard module for Python. (only handles plain text for now) -By Al Sweigart al@inventwithpython.com -BSD License - -Usage: - import pyperclip - pyperclip.copy('The text to be copied to the clipboard.') - spam = pyperclip.paste() - - if not pyperclip.copy: - print("Copy functionality unavailable!") - -On Windows, no additional modules are needed. -On Mac, the module uses pbcopy and pbpaste, which should come with the os. -On Linux, install xclip or xsel via package manager. For example, in Debian: -sudo apt-get install xclip - -Otherwise on Linux, you will need the gtk or PyQt4 modules installed. - -gtk and PyQt4 modules are not available for Python 3, -and this module does not work with PyGObject yet. -""" -__version__ = '1.5.27' - -import platform -import os -import subprocess -from .clipboards import (init_osx_clipboard, - init_gtk_clipboard, init_qt_clipboard, - init_xclip_clipboard, init_xsel_clipboard, - init_klipper_clipboard, init_no_clipboard) -from .windows import init_windows_clipboard - -# `import PyQt4` sys.exit()s if DISPLAY is not in the environment. -# Thus, we need to detect the presence of $DISPLAY manually -# and not load PyQt4 if it is absent. -HAS_DISPLAY = os.getenv("DISPLAY", False) -CHECK_CMD = "where" if platform.system() == "Windows" else "which" - - -def _executable_exists(name): - return subprocess.call([CHECK_CMD, name], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 - - -def determine_clipboard(): - # Determine the OS/platform and set - # the copy() and paste() functions accordingly. - if 'cygwin' in platform.system().lower(): - # FIXME: pyperclip currently does not support Cygwin, - # see https://github.com/asweigart/pyperclip/issues/55 - pass - elif os.name == 'nt' or platform.system() == 'Windows': - return init_windows_clipboard() - if os.name == 'mac' or platform.system() == 'Darwin': - return init_osx_clipboard() - if HAS_DISPLAY: - # Determine which command/module is installed, if any. - try: - import gtk # check if gtk is installed - except ImportError: - pass - else: - return init_gtk_clipboard() - - try: - import PyQt4 # check if PyQt4 is installed - except ImportError: - pass - else: - return init_qt_clipboard() - - if _executable_exists("xclip"): - return init_xclip_clipboard() - if _executable_exists("xsel"): - return init_xsel_clipboard() - if _executable_exists("klipper") and _executable_exists("qdbus"): - return init_klipper_clipboard() - - return init_no_clipboard() - - -def set_clipboard(clipboard): - global copy, paste - - clipboard_types = {'osx': init_osx_clipboard, - 'gtk': init_gtk_clipboard, - 'qt': init_qt_clipboard, - 'xclip': init_xclip_clipboard, - 'xsel': init_xsel_clipboard, - 'klipper': init_klipper_clipboard, - 'windows': init_windows_clipboard, - 'no': init_no_clipboard} - - copy, paste = clipboard_types[clipboard]() - - -copy, paste = determine_clipboard() - -__all__ = ["copy", "paste"] - - -# pandas aliases -clipboard_get = paste -clipboard_set = copy \ No newline at end of file diff --git a/pandas/util/clipboards.py b/pandas/util/clipboards.py deleted file mode 100644 index 5eac945628a5d..0000000000000 --- a/pandas/util/clipboards.py +++ /dev/null @@ -1,134 +0,0 @@ -import sys -import subprocess -from .exceptions import PyperclipException - -EXCEPT_MSG = """ - Pyperclip could not find a copy/paste mechanism for your system. - For more information, please visit https://pyperclip.readthedocs.org """ -PY2 = sys.version_info[0] == 2 -text_type = unicode if PY2 else str - - -def init_osx_clipboard(): - def copy_osx(text): - p = subprocess.Popen(['pbcopy', 'w'], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text.encode('utf-8')) - - def paste_osx(): - p = subprocess.Popen(['pbpaste', 'r'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode('utf-8') - - return copy_osx, paste_osx - - -def init_gtk_clipboard(): - import gtk - - def copy_gtk(text): - global cb - cb = gtk.Clipboard() - cb.set_text(text) - cb.store() - - def paste_gtk(): - clipboardContents = gtk.Clipboard().wait_for_text() - # for python 2, returns None if the clipboard is blank. - if clipboardContents is None: - return '' - else: - return clipboardContents - - return copy_gtk, paste_gtk - - -def init_qt_clipboard(): - # $DISPLAY should exist - from PyQt4.QtGui import QApplication - - app = QApplication([]) - - def copy_qt(text): - cb = app.clipboard() - cb.setText(text) - - def paste_qt(): - cb = app.clipboard() - return text_type(cb.text()) - - return copy_qt, paste_qt - - -def init_xclip_clipboard(): - def copy_xclip(text): - p = subprocess.Popen(['xclip', '-selection', 'c'], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text.encode('utf-8')) - - def paste_xclip(): - p = subprocess.Popen(['xclip', '-selection', 'c', '-o'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode('utf-8') - - return copy_xclip, paste_xclip - - -def init_xsel_clipboard(): - def copy_xsel(text): - p = subprocess.Popen(['xsel', '-b', '-i'], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text.encode('utf-8')) - - def paste_xsel(): - p = subprocess.Popen(['xsel', '-b', '-o'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode('utf-8') - - return copy_xsel, paste_xsel - - -def init_klipper_clipboard(): - def copy_klipper(text): - p = subprocess.Popen( - ['qdbus', 'org.kde.klipper', '/klipper', 'setClipboardContents', - text.encode('utf-8')], - stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=None) - - def paste_klipper(): - p = subprocess.Popen( - ['qdbus', 'org.kde.klipper', '/klipper', 'getClipboardContents'], - stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - - # Workaround for https://bugs.kde.org/show_bug.cgi?id=342874 - # TODO: https://github.com/asweigart/pyperclip/issues/43 - clipboardContents = stdout.decode('utf-8') - # even if blank, Klipper will append a newline at the end - assert len(clipboardContents) > 0 - # make sure that newline is there - assert clipboardContents.endswith('\n') - if clipboardContents.endswith('\n'): - clipboardContents = clipboardContents[:-1] - return clipboardContents - - return copy_klipper, paste_klipper - - -def init_no_clipboard(): - class ClipboardUnavailable(object): - def __call__(self, *args, **kwargs): - raise PyperclipException(EXCEPT_MSG) - - if PY2: - def __nonzero__(self): - return False - else: - def __bool__(self): - return False - - return ClipboardUnavailable(), ClipboardUnavailable() diff --git a/pandas/util/exceptions.py b/pandas/util/exceptions.py deleted file mode 100644 index c5ba3e75d2a76..0000000000000 --- a/pandas/util/exceptions.py +++ /dev/null @@ -1,11 +0,0 @@ -import ctypes - - -class PyperclipException(RuntimeError): - pass - - -class PyperclipWindowsException(PyperclipException): - def __init__(self, message): - message += " (%s)" % ctypes.WinError() - super(PyperclipWindowsException, self).__init__(message) diff --git a/pandas/util/windows.py b/pandas/util/windows.py deleted file mode 100644 index a12932a575f40..0000000000000 --- a/pandas/util/windows.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -This module implements clipboard handling on Windows using ctypes. -""" -import time -import contextlib -import ctypes -from ctypes import c_size_t, sizeof, c_wchar_p, get_errno, c_wchar -from .exceptions import PyperclipWindowsException - - -class CheckedCall(object): - def __init__(self, f): - super(CheckedCall, self).__setattr__("f", f) - - def __call__(self, *args): - ret = self.f(*args) - if not ret and get_errno(): - raise PyperclipWindowsException("Error calling " + self.f.__name__) - return ret - - def __setattr__(self, key, value): - setattr(self.f, key, value) - - -def init_windows_clipboard(): - from ctypes.wintypes import (HGLOBAL, LPVOID, DWORD, LPCSTR, INT, HWND, - HINSTANCE, HMENU, BOOL, UINT, HANDLE) - - windll = ctypes.windll - - safeCreateWindowExA = CheckedCall(windll.user32.CreateWindowExA) - safeCreateWindowExA.argtypes = [DWORD, LPCSTR, LPCSTR, DWORD, INT, INT, - INT, INT, HWND, HMENU, HINSTANCE, LPVOID] - safeCreateWindowExA.restype = HWND - - safeDestroyWindow = CheckedCall(windll.user32.DestroyWindow) - safeDestroyWindow.argtypes = [HWND] - safeDestroyWindow.restype = BOOL - - OpenClipboard = windll.user32.OpenClipboard - OpenClipboard.argtypes = [HWND] - OpenClipboard.restype = BOOL - - safeCloseClipboard = CheckedCall(windll.user32.CloseClipboard) - safeCloseClipboard.argtypes = [] - safeCloseClipboard.restype = BOOL - - safeEmptyClipboard = CheckedCall(windll.user32.EmptyClipboard) - safeEmptyClipboard.argtypes = [] - safeEmptyClipboard.restype = BOOL - - safeGetClipboardData = CheckedCall(windll.user32.GetClipboardData) - safeGetClipboardData.argtypes = [UINT] - safeGetClipboardData.restype = HANDLE - - safeSetClipboardData = CheckedCall(windll.user32.SetClipboardData) - safeSetClipboardData.argtypes = [UINT, HANDLE] - safeSetClipboardData.restype = HANDLE - - safeGlobalAlloc = CheckedCall(windll.kernel32.GlobalAlloc) - safeGlobalAlloc.argtypes = [UINT, c_size_t] - safeGlobalAlloc.restype = HGLOBAL - - safeGlobalLock = CheckedCall(windll.kernel32.GlobalLock) - safeGlobalLock.argtypes = [HGLOBAL] - safeGlobalLock.restype = LPVOID - - safeGlobalUnlock = CheckedCall(windll.kernel32.GlobalUnlock) - safeGlobalUnlock.argtypes = [HGLOBAL] - safeGlobalUnlock.restype = BOOL - - GMEM_MOVEABLE = 0x0002 - CF_UNICODETEXT = 13 - - @contextlib.contextmanager - def window(): - """ - Context that provides a valid Windows hwnd. - """ - # we really just need the hwnd, so setting "STATIC" - # as predefined lpClass is just fine. - hwnd = safeCreateWindowExA(0, b"STATIC", None, 0, 0, 0, 0, 0, - None, None, None, None) - try: - yield hwnd - finally: - safeDestroyWindow(hwnd) - - @contextlib.contextmanager - def clipboard(hwnd): - """ - Context manager that opens the clipboard and prevents - other applications from modifying the clipboard content. - """ - # We may not get the clipboard handle immediately because - # some other application is accessing it (?) - # We try for at least 500ms to get the clipboard. - t = time.time() + 0.5 - success = False - while time.time() < t: - success = OpenClipboard(hwnd) - if success: - break - time.sleep(0.01) - if not success: - raise PyperclipWindowsException("Error calling OpenClipboard") - - try: - yield - finally: - safeCloseClipboard() - - def copy_windows(text): - # This function is heavily based on - # http://msdn.com/ms649016#_win32_Copying_Information_to_the_Clipboard - with window() as hwnd: - # http://msdn.com/ms649048 - # If an application calls OpenClipboard with hwnd set to NULL, - # EmptyClipboard sets the clipboard owner to NULL; - # this causes SetClipboardData to fail. - # => We need a valid hwnd to copy something. - with clipboard(hwnd): - safeEmptyClipboard() - - if text: - # http://msdn.com/ms649051 - # If the hMem parameter identifies a memory object, - # the object must have been allocated using the - # function with the GMEM_MOVEABLE flag. - count = len(text) + 1 - handle = safeGlobalAlloc(GMEM_MOVEABLE, - count * sizeof(c_wchar)) - locked_handle = safeGlobalLock(handle) - - ctypes.memmove(c_wchar_p(locked_handle), c_wchar_p(text), count * sizeof(c_wchar)) - - safeGlobalUnlock(handle) - safeSetClipboardData(CF_UNICODETEXT, handle) - - def paste_windows(): - with clipboard(None): - handle = safeGetClipboardData(CF_UNICODETEXT) - if not handle: - # GetClipboardData may return NULL with errno == NO_ERROR - # if the clipboard is empty. - # (Also, it may return a handle to an empty buffer, - # but technically that's not empty) - return "" - return c_wchar_p(handle).value - - return copy_windows, paste_windows From 71d58d03bf1938252499f4b60365696b66ea9bab Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Sat, 12 Nov 2016 13:10:26 -0500 Subject: [PATCH 09/20] testing encoding in kwargs to to_clipboard and test case for the same --- pandas/io/clipboard.py | 6 ++++++ pandas/io/tests/test_clipboard.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/pandas/io/clipboard.py b/pandas/io/clipboard.py index 73883deeb37c8..3e56372121dc2 100644 --- a/pandas/io/clipboard.py +++ b/pandas/io/clipboard.py @@ -74,6 +74,12 @@ def to_clipboard(obj, excel=None, sep=None, **kwargs): # pragma: no cover - Windows: - OS X: """ + encoding = kwargs.get('encoding') + + #testing if an invalid encoding is passed to clipboard + if encoding is not None and encoding is not 'utf-8': + raise ValueError('clipboard only supports utf-8 encoding') + from pandas.util.clipboard import clipboard_set if excel is None: excel = True diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index dd28fd14b8bba..d581321dc18c3 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -117,3 +117,11 @@ def test_read_clipboard_infer_excel(self): tm.assert_frame_equal(res, exp) + #test case for testing invalid encoding + def test_invalid_encoding(self): + data = self.data['string'] + with tm.assertRaises(ValueError): + data.to_clipboard(encoding='ascii') + + + From dd57ae322cfe6a2eb29f6aab73ca654c0689a810 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Sat, 12 Nov 2016 14:40:10 -0500 Subject: [PATCH 10/20] code review changes and read clipboard invalid encoding test --- pandas/io/clipboard.py | 10 ++++++++-- pandas/io/tests/test_clipboard.py | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pandas/io/clipboard.py b/pandas/io/clipboard.py index 3e56372121dc2..21b18ba26a637 100644 --- a/pandas/io/clipboard.py +++ b/pandas/io/clipboard.py @@ -14,6 +14,12 @@ def read_clipboard(**kwargs): # pragma: no cover ------- parsed : DataFrame """ + encoding = kwargs.pop('encoding','utf-8') + + #testing if an invalid encoding is passed to clipboard + if encoding is not None and encoding.lower().replace('-','') != 'utf8': + raise NotImplementedError('reading from clipboard only supports utf-8 encoding') + from pandas.util.clipboard import clipboard_get from pandas.io.parsers import read_table text = clipboard_get() @@ -74,10 +80,10 @@ def to_clipboard(obj, excel=None, sep=None, **kwargs): # pragma: no cover - Windows: - OS X: """ - encoding = kwargs.get('encoding') + encoding = kwargs.pop('encoding','utf-8') #testing if an invalid encoding is passed to clipboard - if encoding is not None and encoding is not 'utf-8': + if encoding is not None and encoding.lower().replace('-','') != 'utf8': raise ValueError('clipboard only supports utf-8 encoding') from pandas.util.clipboard import clipboard_set diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index d581321dc18c3..0bc78b7f35b94 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -122,6 +122,8 @@ def test_invalid_encoding(self): data = self.data['string'] with tm.assertRaises(ValueError): data.to_clipboard(encoding='ascii') + with tm.assertRaises(NotImplementedError): + pd.read_clipboard(encoding='ascii') From d202fd0021e5c1d28fc485baf6341ab60a1d1b77 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Sat, 12 Nov 2016 17:45:28 -0500 Subject: [PATCH 11/20] added test for valid encoding, modified setup.py so that pandas/util/clipboard can be found --- pandas/io/tests/test_clipboard.py | 12 +++++++++++- setup.py | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index 0bc78b7f35b94..9d0f1f48d9d1d 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -123,7 +123,17 @@ def test_invalid_encoding(self): with tm.assertRaises(ValueError): data.to_clipboard(encoding='ascii') with tm.assertRaises(NotImplementedError): - pd.read_clipboard(encoding='ascii') + pd.read_clipboard(encoding='ascii') + + def test_round_trip_valid_encodings(self): + for enc in ['UTF-8','utf-8','utf8']: + for dt in self.data_types: + data = self.data[dt] + data.to_clipboard(encoding=enc) + result = read_clipboard() + tm.assert_frame_equal(data, result, check_dtype=False) + + diff --git a/setup.py b/setup.py index a17dd502d7706..1731e786c966f 100755 --- a/setup.py +++ b/setup.py @@ -642,7 +642,8 @@ def pxd(name): 'pandas.io.tests.parser', 'pandas.io.tests.sas', 'pandas.stats.tests', - 'pandas.msgpack' + 'pandas.msgpack', + 'pandas.util.clipboard' ], package_data={'pandas.io': ['tests/data/legacy_hdf/*.h5', 'tests/data/legacy_pickle/*/*.pickle', From 0665fd4f7886cc80b1aa075604758b292bda2d5e Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Wed, 16 Nov 2016 01:10:23 -0500 Subject: [PATCH 12/20] fixed linting and test case as per code review --- pandas/io/clipboard.py | 18 ++++++++-------- pandas/io/tests/test_clipboard.py | 34 ++++++++++++------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/pandas/io/clipboard.py b/pandas/io/clipboard.py index 21b18ba26a637..bbcbae845d8d1 100644 --- a/pandas/io/clipboard.py +++ b/pandas/io/clipboard.py @@ -14,11 +14,13 @@ def read_clipboard(**kwargs): # pragma: no cover ------- parsed : DataFrame """ - encoding = kwargs.pop('encoding','utf-8') - - #testing if an invalid encoding is passed to clipboard - if encoding is not None and encoding.lower().replace('-','') != 'utf8': - raise NotImplementedError('reading from clipboard only supports utf-8 encoding') + encoding = kwargs.pop('encoding', 'utf-8') + + # only utf-8 is valid for passed value because that's what clipboard + # supports + if encoding is not None and encoding.lower().replace('-', '') != 'utf8': + raise NotImplementedError( + 'reading from clipboard only supports utf-8 encoding') from pandas.util.clipboard import clipboard_get from pandas.io.parsers import read_table @@ -80,10 +82,10 @@ def to_clipboard(obj, excel=None, sep=None, **kwargs): # pragma: no cover - Windows: - OS X: """ - encoding = kwargs.pop('encoding','utf-8') + encoding = kwargs.pop('encoding', 'utf-8') - #testing if an invalid encoding is passed to clipboard - if encoding is not None and encoding.lower().replace('-','') != 'utf8': + # testing if an invalid encoding is passed to clipboard + if encoding is not None and encoding.lower().replace('-', '') != 'utf8': raise ValueError('clipboard only supports utf-8 encoding') from pandas.util.clipboard import clipboard_set diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index 9d0f1f48d9d1d..307bd8d680813 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -9,7 +9,7 @@ from pandas import read_clipboard from pandas import get_option from pandas.util import testing as tm -from pandas.util.testing import makeCustomDataframe as mkdf, disabled +from pandas.util.testing import makeCustomDataframe as mkdf try: @@ -18,7 +18,6 @@ raise nose.SkipTest("no clipboard found") - class TestClipboard(tm.TestCase): @classmethod @@ -52,9 +51,9 @@ def setUpClass(cls): # Test for non-ascii text: GH9263 cls.data['nonascii'] = pd.DataFrame({'en': 'in English'.split(), 'es': 'en español'.split()}) - - # unicode round trip test for GH 13747 - cls.data['utf8'] = pd.DataFrame({'a':['µasd','Ωœ∑´'], 'b':['øπ∆˚¬','œ∑´®']}) + # unicode round trip test for GH 13747 + cls.data['utf8'] = pd.DataFrame({'a': ['µasd', 'Ωœ∑´'], + 'b': ['øπ∆˚¬', 'œ∑´®']}) cls.data_types = list(cls.data.keys()) @classmethod @@ -62,13 +61,14 @@ def tearDownClass(cls): super(TestClipboard, cls).tearDownClass() del cls.data_types, cls.data - def check_round_trip_frame(self, data_type, excel=None, sep=None): + def check_round_trip_frame(self, data_type, excel=None, sep=None, + encoding=None): data = self.data[data_type] - data.to_clipboard(excel=excel, sep=sep) + data.to_clipboard(excel=excel, sep=sep, encoding=encoding) if sep is not None: - result = read_clipboard(sep=sep, index_col=0) + result = read_clipboard(sep=sep, index_col=0, encoding=encoding) else: - result = read_clipboard() + result = read_clipboard(encoding=encoding) tm.assert_frame_equal(data, result, check_dtype=False) def test_round_trip_frame_sep(self): @@ -117,23 +117,15 @@ def test_read_clipboard_infer_excel(self): tm.assert_frame_equal(res, exp) - #test case for testing invalid encoding + # test case for testing invalid encoding def test_invalid_encoding(self): data = self.data['string'] with tm.assertRaises(ValueError): data.to_clipboard(encoding='ascii') with tm.assertRaises(NotImplementedError): - pd.read_clipboard(encoding='ascii') + pd.read_clipboard(encoding='ascii') def test_round_trip_valid_encodings(self): - for enc in ['UTF-8','utf-8','utf8']: + for enc in ['UTF-8', 'utf-8', 'utf8']: for dt in self.data_types: - data = self.data[dt] - data.to_clipboard(encoding=enc) - result = read_clipboard() - tm.assert_frame_equal(data, result, check_dtype=False) - - - - - + self.check_round_trip_frame(dt, encoding=enc) From b03ed56a33b8fe871ca7e9e9c16dbccf1b328298 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Wed, 16 Nov 2016 01:31:08 -0500 Subject: [PATCH 13/20] changed whatsnew file --- doc/source/whatsnew/v0.19.2.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.19.2.txt b/doc/source/whatsnew/v0.19.2.txt index dc11dd17bfdd7..14c2ea9cbbe9a 100644 --- a/doc/source/whatsnew/v0.19.2.txt +++ b/doc/source/whatsnew/v0.19.2.txt @@ -25,3 +25,4 @@ Bug Fixes - compat with ``dateutil==2.6.0`` for testing (:issue:`14621`) - allow ``nanoseconds`` in ``Timestamp.replace`` kwargs (:issue:`14621`) +- BUG in clipboard (linux, python2) with unicode and separator (:issue:`13747`) From ac8ae60b4f73126d38397fd77d3c664201e4c9ed Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Thu, 17 Nov 2016 15:02:33 -0500 Subject: [PATCH 14/20] skip clipboard test if clipboard primitives are absent --- pandas/io/tests/test_clipboard.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index 4480d8700e3a9..19a84dbc36b7e 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -10,12 +10,13 @@ from pandas import get_option from pandas.util import testing as tm from pandas.util.testing import makeCustomDataframe as mkdf +from pandas.util.clipboard.exceptions import PyperclipException try: - import pandas.util.clipboard # noqa -except OSError: - raise nose.SkipTest("no clipboard found") + DataFrame({'A': [1, 2]}).to_clipboard() +except PyperclipException: + raise nose.SkipTest("clipboard primitives not installed") class TestClipboard(tm.TestCase): From 7af95daa7e8490454cf243a87e9590f5313f5f01 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Thu, 17 Nov 2016 15:13:49 -0500 Subject: [PATCH 15/20] merging lastest changes --- .travis.yml | 56 +++++++++++++++++-- ci/install-2.7_NUMPY_DEV.sh | 18 ------ ci/install-3.6_DEV.sh | 16 ++++++ ci/install_travis.sh | 51 +++++++++-------- ci/requirements-2.7_NUMPY_DEV.build | 3 - ci/requirements-2.7_NUMPY_DEV.run | 2 - ci/requirements-3.4_SLOW.sh | 7 +++ ...Y_DEV.sh => requirements-3.5_NUMPY_DEV.sh} | 0 doc/source/reshaping.rst | 4 ++ doc/source/whatsnew/v0.19.2.txt | 23 ++++++++ doc/source/whatsnew/v0.20.0.txt | 1 + pandas/core/reshape.py | 5 ++ pandas/io/stata.py | 17 ++++++ pandas/io/tests/test_stata.py | 49 +++++++++++++++- pandas/src/datetime.pxd | 8 +-- pandas/tests/test_util.py | 43 ++++++++++++++ pandas/tools/pivot.py | 5 ++ pandas/tools/tests/test_tile.py | 12 ++++ pandas/tools/tests/test_util.py | 38 ++++++++++++- pandas/tools/tile.py | 4 +- pandas/tools/util.py | 2 +- pandas/util/move.c | 8 ++- setup.py | 3 +- 23 files changed, 310 insertions(+), 65 deletions(-) delete mode 100644 ci/install-2.7_NUMPY_DEV.sh create mode 100644 ci/install-3.6_DEV.sh delete mode 100644 ci/requirements-2.7_NUMPY_DEV.build delete mode 100644 ci/requirements-2.7_NUMPY_DEV.run create mode 100644 ci/requirements-3.4_SLOW.sh rename ci/{install-3.5_NUMPY_DEV.sh => requirements-3.5_NUMPY_DEV.sh} (100%) diff --git a/.travis.yml b/.travis.yml index 4eefd6ca83694..49765c9df96ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ matrix: compiler: clang osx_image: xcode6.4 env: + - PYTHON_VERSION=3.5 - JOB_NAME: "35_osx" - NOSE_ARGS="not slow and not network and not disabled" - BUILD_TYPE=conda @@ -43,6 +44,7 @@ matrix: - USE_CACHE=true - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_slow_nnet_LOCALE" - NOSE_ARGS="slow and not network and not disabled" - LOCALE_OVERRIDE="zh_CN.UTF-8" @@ -56,6 +58,7 @@ matrix: - language-pack-zh-hans - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_nslow" - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true @@ -69,6 +72,7 @@ matrix: - python-gtk2 - python: 3.4 env: + - PYTHON_VERSION=3.4 - JOB_NAME: "34_nslow" - NOSE_ARGS="not slow and not disabled" - FULL_DEPS=true @@ -81,6 +85,7 @@ matrix: - xsel - python: 3.5 env: + - PYTHON_VERSION=3.5 - JOB_NAME: "35_nslow" - NOSE_ARGS="not slow and not network and not disabled" - FULL_DEPS=true @@ -95,6 +100,7 @@ matrix: # In allow_failures - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_slow" - JOB_TAG=_SLOW - NOSE_ARGS="slow and not network and not disabled" @@ -104,6 +110,7 @@ matrix: # In allow_failures - python: 3.4 env: + - PYTHON_VERSION=3.4 - JOB_NAME: "34_slow" - JOB_TAG=_SLOW - NOSE_ARGS="slow and not network and not disabled" @@ -118,6 +125,7 @@ matrix: # In allow_failures - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_build_test_conda" - JOB_TAG=_BUILD_TEST - NOSE_ARGS="not slow and not disabled" @@ -125,9 +133,23 @@ matrix: - BUILD_TEST=true - CACHE_NAME="27_build_test_conda" - USE_CACHE=true +# In allow_failures + - python: 3.6-dev + env: + - PYTHON_VERSION=3.6 + - JOB_NAME: "36_dev" + - JOB_TAG=_DEV + - NOSE_ARGS="not slow and not network and not disabled" + - PANDAS_TESTING_MODE="deprecate" + addons: + apt: + packages: + - libatlas-base-dev + - gfortran # In allow_failures - python: 3.5 env: + - PYTHON_VERSION=3.5 - JOB_NAME: "35_numpy_dev" - JOB_TAG=_NUMPY_DEV - NOSE_ARGS="not slow and not network and not disabled" @@ -142,6 +164,7 @@ matrix: # In allow_failures - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_nslow_nnet_COMPAT" - NOSE_ARGS="not slow and not network and not disabled" - LOCALE_OVERRIDE="it_IT.UTF-8" @@ -156,6 +179,7 @@ matrix: # In allow_failures - python: 3.5 env: + - PYTHON_VERSION=3.5 - JOB_NAME: "35_ascii" - JOB_TAG=_ASCII - NOSE_ARGS="not slow and not network and not disabled" @@ -165,6 +189,7 @@ matrix: # In allow_failures - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "doc_build" - FULL_DEPS=true - DOC_BUILD=true @@ -174,6 +199,7 @@ matrix: allow_failures: - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_slow" - JOB_TAG=_SLOW - NOSE_ARGS="slow and not network and not disabled" @@ -182,6 +208,7 @@ matrix: - USE_CACHE=true - python: 3.4 env: + - PYTHON_VERSION=3.4 - JOB_NAME: "34_slow" - JOB_TAG=_SLOW - NOSE_ARGS="slow and not network and not disabled" @@ -195,6 +222,7 @@ matrix: - xsel - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_build_test_conda" - JOB_TAG=_BUILD_TEST - NOSE_ARGS="not slow and not disabled" @@ -202,14 +230,27 @@ matrix: - BUILD_TEST=true - CACHE_NAME="27_build_test_conda" - USE_CACHE=true - - python: 3.5 + - python: 3.6-dev env: - - JOB_NAME: "35_numpy_dev" - - JOB_TAG=_NUMPY_DEV + - PYTHON_VERSION=3.6 + - JOB_NAME: "36_dev" + - JOB_TAG=_DEV - NOSE_ARGS="not slow and not network and not disabled" - PANDAS_TESTING_MODE="deprecate" - - CACHE_NAME="35_numpy_dev" - - USE_CACHE=true + addons: + apt: + packages: + - libatlas-base-dev + - gfortran + - python: 3.5 + env: + - PYTHON_VERSION=3.5 + - JOB_NAME: "35_numpy_dev" + - JOB_TAG=_NUMPY_DEV + - NOSE_ARGS="not slow and not network and not disabled" + - PANDAS_TESTING_MODE="deprecate" + - CACHE_NAME="35_numpy_dev" + - USE_CACHE=true addons: apt: packages: @@ -217,6 +258,7 @@ matrix: - gfortran - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "27_nslow_nnet_COMPAT" - NOSE_ARGS="not slow and not network and not disabled" - LOCALE_OVERRIDE="it_IT.UTF-8" @@ -230,6 +272,7 @@ matrix: - language-pack-it - python: 3.5 env: + - PYTHON_VERSION=3.5 - JOB_NAME: "35_ascii" - JOB_TAG=_ASCII - NOSE_ARGS="not slow and not network and not disabled" @@ -238,6 +281,7 @@ matrix: - USE_CACHE=true - python: 2.7 env: + - PYTHON_VERSION=2.7 - JOB_NAME: "doc_build" - FULL_DEPS=true - DOC_BUILD=true @@ -249,7 +293,7 @@ before_install: - echo "before_install" - source ci/travis_process_gbq_encryption.sh - echo $VIRTUAL_ENV - - export PATH="$HOME/miniconda/bin:$PATH" + - export PATH="$HOME/miniconda3/bin:$PATH" - df -h - date - pwd diff --git a/ci/install-2.7_NUMPY_DEV.sh b/ci/install-2.7_NUMPY_DEV.sh deleted file mode 100644 index 22ac8f6547879..0000000000000 --- a/ci/install-2.7_NUMPY_DEV.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -source activate pandas - -echo "install numpy master wheel" - -# remove the system installed numpy -pip uninstall numpy -y - -# we need these for numpy - -# these wheels don't play nice with the conda libgfortran / openblas -# time conda install -n pandas libgfortran openblas || exit 1 - -# install numpy wheel from master -pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy - -true diff --git a/ci/install-3.6_DEV.sh b/ci/install-3.6_DEV.sh new file mode 100644 index 0000000000000..0b95f1cd45cad --- /dev/null +++ b/ci/install-3.6_DEV.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +echo "install 3.6 dev" + +conda config --set add_pip_as_python_dependency false +conda create -n pandas python=3.6 -c conda-forge/label/prerelease + +source activate pandas + +# ensure we have pip +python -m ensurepip + +# install deps +pip3.6 install nose cython numpy pytz python-dateutil + +true diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 98ce36acc096e..bdd2c01f611b2 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -31,10 +31,7 @@ edit_init home_dir=$(pwd) echo "home_dir: [$home_dir]" -python_major_version="${TRAVIS_PYTHON_VERSION:0:1}" -[ "$python_major_version" == "2" ] && python_major_version="" - -MINICONDA_DIR="$HOME/miniconda" +MINICONDA_DIR="$HOME/miniconda3" if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ] && [ "$USE_CACHE" ]; then echo "Miniconda install already present from cache: $MINICONDA_DIR" @@ -63,9 +60,9 @@ else rm -rf "$MINICONDA_DIR" # install miniconda if [ "${TRAVIS_OS_NAME}" == "osx" ]; then - wget http://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh || exit 1 + wget http://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh || exit 1 else - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh || exit 1 + wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh || exit 1 fi bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1 @@ -84,21 +81,25 @@ else # Useful for debugging any issues with conda conda info -a || exit 1 - - time conda create -n pandas python=$TRAVIS_PYTHON_VERSION nose coverage flake8 || exit 1 - fi -# build deps -REQ="ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.build" -# may have additional installation instructions for this build -INSTALL="ci/install-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.sh" +# may have installation instructions for this build +INSTALL="ci/install-${PYTHON_VERSION}${JOB_TAG}.sh" if [ -e ${INSTALL} ]; then time bash $INSTALL || exit 1 +else + + # create new env + time conda create -n pandas python=$PYTHON_VERSION nose coverage flake8 || exit 1 fi +# build deps +REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build" + # install deps -time conda install -n pandas --file=${REQ} || exit 1 +if [ -e ${REQ} ]; then + time conda install -n pandas --file=${REQ} || exit 1 +fi source activate pandas @@ -106,7 +107,7 @@ if [ "$BUILD_TEST" ]; then # build testing pip uninstall --yes cython - pip install cython==0.15.1 + pip install cython==0.19.1 ( python setup.py build_ext --inplace && python setup.py develop ) || true else @@ -117,14 +118,22 @@ else # we may have run installations echo "conda installs" - REQ="ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.run" - time conda install -n pandas --file=${REQ} || exit 1 + REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run" + if [ -e ${REQ} ]; then + time conda install -n pandas --file=${REQ} || exit 1 + fi # we may have additional pip installs echo "pip installs" - REQ="ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.pip" + REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip" if [ -e ${REQ} ]; then - pip install --upgrade -r $REQ + pip install --upgrade -r $REQ + fi + + # may have addtl installation instructions for this build + REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.sh" + if [ -e ${REQ} ]; then + time bash $REQ || exit 1 fi # remove any installed pandas package @@ -138,9 +147,5 @@ else fi -if [ "$JOB_NAME" == "34_slow" ]; then - conda install -c conda-forge/label/rc -c conda-forge matplotlib -fi - echo "done" exit 0 diff --git a/ci/requirements-2.7_NUMPY_DEV.build b/ci/requirements-2.7_NUMPY_DEV.build deleted file mode 100644 index d15edbfa3d2c1..0000000000000 --- a/ci/requirements-2.7_NUMPY_DEV.build +++ /dev/null @@ -1,3 +0,0 @@ -python-dateutil -pytz -cython diff --git a/ci/requirements-2.7_NUMPY_DEV.run b/ci/requirements-2.7_NUMPY_DEV.run deleted file mode 100644 index 0aa987baefb1d..0000000000000 --- a/ci/requirements-2.7_NUMPY_DEV.run +++ /dev/null @@ -1,2 +0,0 @@ -python-dateutil -pytz diff --git a/ci/requirements-3.4_SLOW.sh b/ci/requirements-3.4_SLOW.sh new file mode 100644 index 0000000000000..bc8fb79147d2c --- /dev/null +++ b/ci/requirements-3.4_SLOW.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source activate pandas + +echo "install 34_slow" + +conda install -n pandas -c conda-forge/label/rc -c conda-forge matplotlib diff --git a/ci/install-3.5_NUMPY_DEV.sh b/ci/requirements-3.5_NUMPY_DEV.sh similarity index 100% rename from ci/install-3.5_NUMPY_DEV.sh rename to ci/requirements-3.5_NUMPY_DEV.sh diff --git a/doc/source/reshaping.rst b/doc/source/reshaping.rst index 9ed2c42610b69..3a2c48834991f 100644 --- a/doc/source/reshaping.rst +++ b/doc/source/reshaping.rst @@ -323,6 +323,10 @@ Pivot tables .. _reshaping.pivot: +While ``pivot`` provides general purpose pivoting of DataFrames with various +data types (strings, numerics, etc.), Pandas also provides the ``pivot_table`` +function for pivoting with aggregation of numeric data. + The function ``pandas.pivot_table`` can be used to create spreadsheet-style pivot tables. See the :ref:`cookbook` for some advanced strategies diff --git a/doc/source/whatsnew/v0.19.2.txt b/doc/source/whatsnew/v0.19.2.txt index 14c2ea9cbbe9a..46ec375c90590 100644 --- a/doc/source/whatsnew/v0.19.2.txt +++ b/doc/source/whatsnew/v0.19.2.txt @@ -23,6 +23,29 @@ Performance Improvements Bug Fixes ~~~~~~~~~ +<<<<<<< HEAD - compat with ``dateutil==2.6.0`` for testing (:issue:`14621`) - allow ``nanoseconds`` in ``Timestamp.replace`` kwargs (:issue:`14621`) - BUG in clipboard (linux, python2) with unicode and separator (:issue:`13747`) +======= +- Compat with ``dateutil==2.6.0``; segfault reported in the testing suite (:issue:`14621`) +- Allow ``nanoseconds`` in ``Timestamp.replace`` as a kwarg (:issue:`14621`) + + + + + + + +- Bug in ``pd.cut`` with negative values and a single bin (:issue:`14652`) +- Bug in ``pd.to_numeric`` where a 0 was not unsigned on a ``downcast='unsigned'`` argument (:issue:`14401`) + + + + +- Bug in not propogating exceptions in parsing invalid datetimes, noted in python 3.6 (:issue:`14561`) + + + +- Explicit check in ``to_stata`` and ``StataWriter`` for out-of-range values when writing doubles (:issue:`14618`) +>>>>>>> 45543ec22bb01d5c46eff9384491d218cffddd64 diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index 660300e1814e8..8819a95f27b0d 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -80,3 +80,4 @@ Performance Improvements Bug Fixes ~~~~~~~~~ + diff --git a/pandas/core/reshape.py b/pandas/core/reshape.py index fa5d16bd85e98..055a0041b181a 100644 --- a/pandas/core/reshape.py +++ b/pandas/core/reshape.py @@ -357,6 +357,11 @@ def pivot_simple(index, columns, values): Returns ------- DataFrame + + See also + -------- + DataFrame.pivot_table : generalization of pivot that can handle + duplicate values for one index/column pair """ if (len(index) != len(columns)) or (len(columns) != len(values)): raise AssertionError('Length of index, columns, and values must be the' diff --git a/pandas/io/stata.py b/pandas/io/stata.py index 14bd670862b41..c35e07be2c31a 100644 --- a/pandas/io/stata.py +++ b/pandas/io/stata.py @@ -511,6 +511,9 @@ def _cast_to_stata_types(data): (np.uint16, np.int16, np.int32), (np.uint32, np.int32, np.int64)) + float32_max = struct.unpack('= 2 ** 53 or data[col].min() <= -2 ** 53: ws = precision_loss_doc % ('int64', 'float64') + elif dtype in (np.float32, np.float64): + value = data[col].max() + if np.isinf(value): + msg = 'Column {0} has a maximum value of infinity which is ' \ + 'outside the range supported by Stata.' + raise ValueError(msg.format(col)) + if dtype == np.float32 and value > float32_max: + data[col] = data[col].astype(np.float64) + elif dtype == np.float64: + if value > float64_max: + msg = 'Column {0} has a maximum value ({1}) outside the ' \ + 'range supported by Stata ({1})' + raise ValueError(msg.format(col, value, float64_max)) if ws: import warnings @@ -2048,6 +2064,7 @@ def _prepare_pandas(self, data): data = self._check_column_names(data) # Check columns for compatibility with stata, upcast if necessary + # Raise if outside the supported range data = _cast_to_stata_types(data) # Replace NaNs with Stata missing values diff --git a/pandas/io/tests/test_stata.py b/pandas/io/tests/test_stata.py index 1849b32a4a7c8..cd972868a6e32 100644 --- a/pandas/io/tests/test_stata.py +++ b/pandas/io/tests/test_stata.py @@ -11,8 +11,6 @@ import nose import numpy as np -from pandas.tslib import NaT - import pandas as pd import pandas.util.testing as tm from pandas import compat @@ -21,6 +19,7 @@ from pandas.io.parsers import read_csv from pandas.io.stata import (read_stata, StataReader, InvalidColumnName, PossiblePrecisionLoss, StataMissingValue) +from pandas.tslib import NaT from pandas.types.common import is_categorical_dtype @@ -1234,6 +1233,52 @@ def test_stata_111(self): original = original[['y', 'x', 'w', 'z']] tm.assert_frame_equal(original, df) + def test_out_of_range_double(self): + # GH 14618 + df = DataFrame({'ColumnOk': [0.0, + np.finfo(np.double).eps, + 4.49423283715579e+307], + 'ColumnTooBig': [0.0, + np.finfo(np.double).eps, + np.finfo(np.double).max]}) + with tm.assertRaises(ValueError) as cm: + with tm.ensure_clean() as path: + df.to_stata(path) + tm.assertTrue('ColumnTooBig' in cm.exception) + + df.loc[2, 'ColumnTooBig'] = np.inf + with tm.assertRaises(ValueError) as cm: + with tm.ensure_clean() as path: + df.to_stata(path) + tm.assertTrue('ColumnTooBig' in cm.exception) + tm.assertTrue('infinity' in cm.exception) + + def test_out_of_range_float(self): + original = DataFrame({'ColumnOk': [0.0, + np.finfo(np.float32).eps, + np.finfo(np.float32).max / 10.0], + 'ColumnTooBig': [0.0, + np.finfo(np.float32).eps, + np.finfo(np.float32).max]}) + original.index.name = 'index' + for col in original: + original[col] = original[col].astype(np.float32) + + with tm.ensure_clean() as path: + original.to_stata(path) + reread = read_stata(path) + original['ColumnTooBig'] = original['ColumnTooBig'].astype( + np.float64) + tm.assert_frame_equal(original, + reread.set_index('index')) + + original.loc[2, 'ColumnTooBig'] = np.inf + with tm.assertRaises(ValueError) as cm: + with tm.ensure_clean() as path: + original.to_stata(path) + tm.assertTrue('ColumnTooBig' in cm.exception) + tm.assertTrue('infinity' in cm.exception) + if __name__ == '__main__': nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], diff --git a/pandas/src/datetime.pxd b/pandas/src/datetime.pxd index 5f7de8244d17e..d3d471a33715d 100644 --- a/pandas/src/datetime.pxd +++ b/pandas/src/datetime.pxd @@ -126,8 +126,8 @@ cdef extern from "datetime/np_datetime_strings.h": -cdef inline _string_to_dts(object val, pandas_datetimestruct* dts, - int* out_local, int* out_tzoffset): +cdef inline int _string_to_dts(object val, pandas_datetimestruct* dts, + int* out_local, int* out_tzoffset) except? -1: cdef int result cdef char *tmp @@ -139,10 +139,11 @@ cdef inline _string_to_dts(object val, pandas_datetimestruct* dts, if result == -1: raise ValueError('Unable to parse %s' % str(val)) + return result cdef inline int _cstring_to_dts(char *val, int length, pandas_datetimestruct* dts, - int* out_local, int* out_tzoffset): + int* out_local, int* out_tzoffset) except? -1: cdef: npy_bool special PANDAS_DATETIMEUNIT out_bestunit @@ -195,4 +196,3 @@ cdef inline int64_t _date_to_datetime64(object val, dts.hour = dts.min = dts.sec = dts.us = 0 dts.ps = dts.as = 0 return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, dts) - diff --git a/pandas/tests/test_util.py b/pandas/tests/test_util.py index 9193880df7feb..f5828dab21e37 100644 --- a/pandas/tests/test_util.py +++ b/pandas/tests/test_util.py @@ -2,6 +2,9 @@ import nose from collections import OrderedDict +import sys +import unittest +from uuid import uuid4 from pandas.util._move import move_into_mutable_buffer, BadMove from pandas.util.decorators import deprecate_kwarg from pandas.util.validators import (validate_args, validate_kwargs, @@ -325,6 +328,46 @@ def test_exactly_one_ref(self): # materialize as bytearray to show that it is mutable self.assertEqual(bytearray(as_stolen_buf), b'test') + @unittest.skipIf( + sys.version_info[0] > 2, + 'bytes objects cannot be interned in py3', + ) + def test_interned(self): + salt = uuid4().hex + + def make_string(): + # We need to actually create a new string so that it has refcount + # one. We use a uuid so that we know the string could not already + # be in the intern table. + return ''.join(('testing: ', salt)) + + # This should work, the string has one reference on the stack. + move_into_mutable_buffer(make_string()) + + refcount = [None] # nonlocal + + def ref_capture(ob): + # Subtract two because those are the references owned by this + # frame: + # 1. The local variables of this stack frame. + # 2. The python data stack of this stack frame. + refcount[0] = sys.getrefcount(ob) - 2 + return ob + + with tm.assertRaises(BadMove): + # If we intern the string it will still have one reference but now + # it is in the intern table so if other people intern the same + # string while the mutable buffer holds the first string they will + # be the same instance. + move_into_mutable_buffer(ref_capture(intern(make_string()))) # noqa + + self.assertEqual( + refcount[0], + 1, + msg='The BadMove was probably raised for refcount reasons instead' + ' of interning reasons', + ) + def test_numpy_errstate_is_default(): # The defaults since numpy 1.6.0 diff --git a/pandas/tools/pivot.py b/pandas/tools/pivot.py index 9e064a1d1fc99..820a545363ee3 100644 --- a/pandas/tools/pivot.py +++ b/pandas/tools/pivot.py @@ -75,6 +75,11 @@ def pivot_table(data, values=None, index=None, columns=None, aggfunc='mean', Returns ------- table : DataFrame + + See also + -------- + DataFrame.pivot : pivot without aggregation that can handle + non-numeric data """ index = _convert_by(index) columns = _convert_by(columns) diff --git a/pandas/tools/tests/test_tile.py b/pandas/tools/tests/test_tile.py index 16731620a1dcd..e5b9c65b515d6 100644 --- a/pandas/tools/tests/test_tile.py +++ b/pandas/tools/tests/test_tile.py @@ -271,6 +271,18 @@ def test_series_retbins(self): np.array([0, 0, 1, 1], dtype=np.int8)) tm.assert_numpy_array_equal(bins, np.array([0, 1.5, 3])) + def test_single_bin(self): + # issue 14652 + expected = Series([0, 0]) + + s = Series([9., 9.]) + result = cut(s, 1, labels=False) + tm.assert_series_equal(result, expected) + + s = Series([-9., -9.]) + result = cut(s, 1, labels=False) + tm.assert_series_equal(result, expected) + def curpath(): pth, _ = os.path.split(os.path.abspath(__file__)) diff --git a/pandas/tools/tests/test_util.py b/pandas/tools/tests/test_util.py index 8c16308d79a31..f9647721e3c5b 100644 --- a/pandas/tools/tests/test_util.py +++ b/pandas/tools/tests/test_util.py @@ -4,9 +4,10 @@ import nose import numpy as np +from numpy import iinfo import pandas as pd -from pandas import date_range, Index +from pandas import (date_range, Index, _np_version_under1p9) import pandas.util.testing as tm from pandas.tools.util import cartesian_product, to_numeric @@ -401,6 +402,41 @@ def test_downcast(self): res = pd.to_numeric(data, downcast=downcast) tm.assert_numpy_array_equal(res, expected) + def test_downcast_limits(self): + # Test the limits of each downcast. Bug: #14401. + # Check to make sure numpy is new enough to run this test. + if _np_version_under1p9: + raise nose.SkipTest("Numpy version is under 1.9") + + i = 'integer' + u = 'unsigned' + dtype_downcast_min_max = [ + ('int8', i, [iinfo(np.int8).min, iinfo(np.int8).max]), + ('int16', i, [iinfo(np.int16).min, iinfo(np.int16).max]), + ('int32', i, [iinfo(np.int32).min, iinfo(np.int32).max]), + ('int64', i, [iinfo(np.int64).min, iinfo(np.int64).max]), + ('uint8', u, [iinfo(np.uint8).min, iinfo(np.uint8).max]), + ('uint16', u, [iinfo(np.uint16).min, iinfo(np.uint16).max]), + ('uint32', u, [iinfo(np.uint32).min, iinfo(np.uint32).max]), + # Test will be skipped until there is more uint64 support. + # ('uint64', u, [iinfo(uint64).min, iinfo(uint64).max]), + ('int16', i, [iinfo(np.int8).min, iinfo(np.int8).max + 1]), + ('int32', i, [iinfo(np.int16).min, iinfo(np.int16).max + 1]), + ('int64', i, [iinfo(np.int32).min, iinfo(np.int32).max + 1]), + ('int16', i, [iinfo(np.int8).min - 1, iinfo(np.int16).max]), + ('int32', i, [iinfo(np.int16).min - 1, iinfo(np.int32).max]), + ('int64', i, [iinfo(np.int32).min - 1, iinfo(np.int64).max]), + ('uint16', u, [iinfo(np.uint8).min, iinfo(np.uint8).max + 1]), + ('uint32', u, [iinfo(np.uint16).min, iinfo(np.uint16).max + 1]), + # Test will be skipped until there is more uint64 support. + # ('uint64', u, [iinfo(np.uint32).min, iinfo(np.uint32).max + 1]), + ] + + for dtype, downcast, min_max in dtype_downcast_min_max: + series = pd.to_numeric(pd.Series(min_max), downcast=downcast) + tm.assert_equal(series.dtype, dtype) + + if __name__ == '__main__': nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], exit=False) diff --git a/pandas/tools/tile.py b/pandas/tools/tile.py index 62bbfc2f630a5..ef75f2f84779b 100644 --- a/pandas/tools/tile.py +++ b/pandas/tools/tile.py @@ -98,8 +98,8 @@ def cut(x, bins, right=True, labels=None, retbins=False, precision=3, mn, mx = [mi + 0.0 for mi in rng] if mn == mx: # adjust end points before binning - mn -= .001 * mn - mx += .001 * mx + mn -= .001 * abs(mn) + mx += .001 * abs(mx) bins = np.linspace(mn, mx, bins + 1, endpoint=True) else: # adjust end points after binning bins = np.linspace(mn, mx, bins + 1, endpoint=True) diff --git a/pandas/tools/util.py b/pandas/tools/util.py index fec56328c1721..b50bf9dc448bc 100644 --- a/pandas/tools/util.py +++ b/pandas/tools/util.py @@ -205,7 +205,7 @@ def to_numeric(arg, errors='raise', downcast=None): if downcast in ('integer', 'signed'): typecodes = np.typecodes['Integer'] - elif downcast == 'unsigned' and np.min(values) > 0: + elif downcast == 'unsigned' and np.min(values) >= 0: typecodes = np.typecodes['UnsignedInteger'] elif downcast == 'float': typecodes = np.typecodes['Float'] diff --git a/pandas/util/move.c b/pandas/util/move.c index 68fcad793e16c..fb918c302b100 100644 --- a/pandas/util/move.c +++ b/pandas/util/move.c @@ -7,6 +7,9 @@ #define PyString_CheckExact PyBytes_CheckExact #define PyString_AS_STRING PyBytes_AS_STRING #define PyString_GET_SIZE PyBytes_GET_SIZE + +/* in python 3, we cannot intern bytes objects so this is always false */ +#define PyString_CHECK_INTERNED(cs) 0 #endif /* !COMPILING_IN_PY2 */ #ifndef Py_TPFLAGS_HAVE_GETCHARBUFFER @@ -113,8 +116,9 @@ stolenbuf_new(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } - if (Py_REFCNT(bytes_rvalue) != 1) { - /* there is a reference other than the caller's stack */ + if (Py_REFCNT(bytes_rvalue) != 1 || PyString_CHECK_INTERNED(bytes_rvalue)) { + /* there is a reference other than the caller's stack or the string is + interned */ PyErr_SetObject(badmove, bytes_rvalue); return NULL; } diff --git a/setup.py b/setup.py index 1731e786c966f..3e7cc1b5ec019 100755 --- a/setup.py +++ b/setup.py @@ -454,7 +454,8 @@ def pxd(name): tseries_depends = ['pandas/src/datetime/np_datetime.h', 'pandas/src/datetime/np_datetime_strings.h', - 'pandas/src/period_helper.h'] + 'pandas/src/period_helper.h', + 'pandas/src/datetime.pxd'] # some linux distros require it From cedb690e2f84ff8ffe4eef8b16279c2fddd51b92 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Thu, 17 Nov 2016 15:31:57 -0500 Subject: [PATCH 16/20] merge conflict in whats new file --- doc/source/whatsnew/v0.19.2.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/source/whatsnew/v0.19.2.txt b/doc/source/whatsnew/v0.19.2.txt index 46ec375c90590..93ed44b589f46 100644 --- a/doc/source/whatsnew/v0.19.2.txt +++ b/doc/source/whatsnew/v0.19.2.txt @@ -22,8 +22,6 @@ Performance Improvements Bug Fixes ~~~~~~~~~ - -<<<<<<< HEAD - compat with ``dateutil==2.6.0`` for testing (:issue:`14621`) - allow ``nanoseconds`` in ``Timestamp.replace`` kwargs (:issue:`14621`) - BUG in clipboard (linux, python2) with unicode and separator (:issue:`13747`) @@ -48,4 +46,3 @@ Bug Fixes - Explicit check in ``to_stata`` and ``StataWriter`` for out-of-range values when writing doubles (:issue:`14618`) ->>>>>>> 45543ec22bb01d5c46eff9384491d218cffddd64 From 98b61e8c2b524d37b91b8a04d814b3961ff55577 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Thu, 17 Nov 2016 15:37:28 -0500 Subject: [PATCH 17/20] merge conflict --- doc/source/whatsnew/v0.19.2.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.19.2.txt b/doc/source/whatsnew/v0.19.2.txt index 93ed44b589f46..97920c58e5e8b 100644 --- a/doc/source/whatsnew/v0.19.2.txt +++ b/doc/source/whatsnew/v0.19.2.txt @@ -22,10 +22,7 @@ Performance Improvements Bug Fixes ~~~~~~~~~ -- compat with ``dateutil==2.6.0`` for testing (:issue:`14621`) -- allow ``nanoseconds`` in ``Timestamp.replace`` kwargs (:issue:`14621`) -- BUG in clipboard (linux, python2) with unicode and separator (:issue:`13747`) -======= + - Compat with ``dateutil==2.6.0``; segfault reported in the testing suite (:issue:`14621`) - Allow ``nanoseconds`` in ``Timestamp.replace`` as a kwarg (:issue:`14621`) @@ -46,3 +43,4 @@ Bug Fixes - Explicit check in ``to_stata`` and ``StataWriter`` for out-of-range values when writing doubles (:issue:`14618`) +- BUG in clipboard (linux, python2) with unicode and separator (:issue:`13747`) \ No newline at end of file From 9db42d813a8384c5668ffe8651c26032316cec35 Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Thu, 17 Nov 2016 15:57:46 -0500 Subject: [PATCH 18/20] whatsnew conflict --- doc/source/whatsnew/v0.19.2.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.19.2.txt b/doc/source/whatsnew/v0.19.2.txt index f4a45a6938a95..47c5b1c148d4f 100644 --- a/doc/source/whatsnew/v0.19.2.txt +++ b/doc/source/whatsnew/v0.19.2.txt @@ -43,3 +43,4 @@ Bug Fixes - Explicit check in ``to_stata`` and ``StataWriter`` for out-of-range values when writing doubles (:issue:`14618`) +- BUG in clipboard (linux, python2) with unicode and separator (:issue:`13747`) From b74fbc12827f235d2e773a4905f00273ad75966a Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Thu, 17 Nov 2016 16:52:13 -0500 Subject: [PATCH 19/20] ignore lint test for pyperclip files --- pandas/util/clipboard/__init__.py | 2 ++ pandas/util/clipboard/clipboards.py | 2 ++ pandas/util/clipboard/exceptions.py | 1 + pandas/util/clipboard/windows.py | 1 + 4 files changed, 6 insertions(+) diff --git a/pandas/util/clipboard/__init__.py b/pandas/util/clipboard/__init__.py index b7f33681592ee..358c9b5f8035a 100644 --- a/pandas/util/clipboard/__init__.py +++ b/pandas/util/clipboard/__init__.py @@ -25,6 +25,8 @@ """ __version__ = '1.5.27' +# flake8: noqa + import platform import os import subprocess diff --git a/pandas/util/clipboard/clipboards.py b/pandas/util/clipboard/clipboards.py index 5eac945628a5d..182a685f956e6 100644 --- a/pandas/util/clipboard/clipboards.py +++ b/pandas/util/clipboard/clipboards.py @@ -1,3 +1,5 @@ +# flake8: noqa + import sys import subprocess from .exceptions import PyperclipException diff --git a/pandas/util/clipboard/exceptions.py b/pandas/util/clipboard/exceptions.py index c5ba3e75d2a76..615335f3a58da 100644 --- a/pandas/util/clipboard/exceptions.py +++ b/pandas/util/clipboard/exceptions.py @@ -1,3 +1,4 @@ +# flake8: noqa import ctypes diff --git a/pandas/util/clipboard/windows.py b/pandas/util/clipboard/windows.py index a12932a575f40..956d5b9d34025 100644 --- a/pandas/util/clipboard/windows.py +++ b/pandas/util/clipboard/windows.py @@ -1,3 +1,4 @@ +# flake8: noqa """ This module implements clipboard handling on Windows using ctypes. """ From 2aafb66cbf50f39e593a5e4b63667b62a47928bb Mon Sep 17 00:00:00 2001 From: Ajay Saxena Date: Thu, 17 Nov 2016 18:35:59 -0500 Subject: [PATCH 20/20] moved comment inside test and added github issue labels to test --- doc/source/whatsnew/v0.19.2.txt | 3 +++ pandas/io/tests/test_clipboard.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.19.2.txt b/doc/source/whatsnew/v0.19.2.txt index 47c5b1c148d4f..5236ebb38ae63 100644 --- a/doc/source/whatsnew/v0.19.2.txt +++ b/doc/source/whatsnew/v0.19.2.txt @@ -44,3 +44,6 @@ Bug Fixes - Explicit check in ``to_stata`` and ``StataWriter`` for out-of-range values when writing doubles (:issue:`14618`) - BUG in clipboard (linux, python2) with unicode and separator (:issue:`13747`) +- BUG: clipboard functions windows 10 py3.5 (:issue:`14362`) +- BUG: test_clipboard fails (:issue:`12807`) +- to_clipboard is no longer Excel compatible (:issue:`12529`) diff --git a/pandas/io/tests/test_clipboard.py b/pandas/io/tests/test_clipboard.py index 19a84dbc36b7e..3ce714274e2dc 100644 --- a/pandas/io/tests/test_clipboard.py +++ b/pandas/io/tests/test_clipboard.py @@ -52,7 +52,7 @@ def setUpClass(cls): # Test for non-ascii text: GH9263 cls.data['nonascii'] = pd.DataFrame({'en': 'in English'.split(), 'es': 'en español'.split()}) - # unicode round trip test for GH 13747 + # unicode round trip test for GH 13747, GH 12529 cls.data['utf8'] = pd.DataFrame({'a': ['µasd', 'Ωœ∑´'], 'b': ['øπ∆˚¬', 'œ∑´®']}) cls.data_types = list(cls.data.keys()) @@ -120,8 +120,8 @@ def test_read_clipboard_infer_excel(self): tm.assert_frame_equal(res, exp) - # test case for testing invalid encoding def test_invalid_encoding(self): + # test case for testing invalid encoding data = self.data['string'] with tm.assertRaises(ValueError): data.to_clipboard(encoding='ascii')