From 1c2d15266db8b64453b3a1c87206412bceb85bba Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 26 Feb 2021 22:11:46 -0600 Subject: [PATCH 1/8] DEPS: replace pyperclip with pyclip #39834 --- LICENSES/OTHER | 30 -- environment.yml | 1 - pandas/io/clipboard/__init__.py | 671 +----------------------- pandas/io/clipboards.py | 2 +- requirements-dev.txt | 1 - scripts/generate_pip_deps_from_conda.py | 2 +- setup.cfg | 3 - 7 files changed, 5 insertions(+), 705 deletions(-) diff --git a/LICENSES/OTHER b/LICENSES/OTHER index f0550b4ee208a..a1b367fe6061c 100644 --- a/LICENSES/OTHER +++ b/LICENSES/OTHER @@ -48,33 +48,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -Pyperclip v1.3 license ----------------------- - -Copyright (c) 2010, Albert Sweigart -All rights reserved. - -BSD-style license: - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the pyperclip nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY Albert Sweigart "AS IS" AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL Albert Sweigart BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/environment.yml b/environment.yml index 113780ed0264a..69582b79ba704 100644 --- a/environment.yml +++ b/environment.yml @@ -101,7 +101,6 @@ dependencies: - pyarrow>=0.15.0 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather - python-snappy # required by pyarrow - - pyqt>=5.9.2 # pandas.read_clipboard - pytables>=3.5.1 # pandas.read_hdf, DataFrame.to_hdf - s3fs>=0.4.0 # file IO when using 's3://...' path - fsspec>=0.7.4 # for generic remote file operations diff --git a/pandas/io/clipboard/__init__.py b/pandas/io/clipboard/__init__.py index e9c20ff42f51b..b02533b7e5366 100644 --- a/pandas/io/clipboard/__init__.py +++ b/pandas/io/clipboard/__init__.py @@ -1,672 +1,7 @@ -""" -Pyperclip +from functools import partial -A cross-platform clipboard module for Python, -with copy & paste functions for plain text. -By Al Sweigart al@inventwithpython.com -BSD License - -Usage: - import pyperclip - pyperclip.copy('The text to be copied to the clipboard.') - spam = pyperclip.paste() - - if not pyperclip.is_available(): - print("Copy functionality unavailable!") - -On Windows, no additional modules are needed. -On Mac, the pyobjc module is used, falling back to the pbcopy and pbpaste cli - commands. (These commands should come with OS X.). -On Linux, install xclip or xsel via package manager. For example, in Debian: - sudo apt-get install xclip - sudo apt-get install xsel - -Otherwise on Linux, you will need the PyQt5 modules installed. - -This module does not work with PyGObject yet. - -Cygwin is currently not supported. - -Security Note: This module runs programs with these names: - - which - - where - - pbcopy - - pbpaste - - xclip - - xsel - - klipper - - qdbus -A malicious user could rename or add programs with these names, tricking -Pyperclip into running them with whatever permissions the Python process has. - -""" -__version__ = "1.7.0" - -import contextlib -import ctypes -from ctypes import ( - c_size_t, - c_wchar, - c_wchar_p, - get_errno, - sizeof, -) -import distutils.spawn -import os -import platform -import subprocess -import time -import warnings - -# `import PyQt4` sys.exit()s if DISPLAY is not in the environment. -# Thus, we need to detect the presence of $DISPLAY manually -# and not load PyQt4 if it is absent. -HAS_DISPLAY = os.getenv("DISPLAY", False) - -EXCEPT_MSG = """ - Pyperclip could not find a copy/paste mechanism for your system. - For more information, please visit - https://pyperclip.readthedocs.io/en/latest/#not-implemented-error - """ - -ENCODING = "utf-8" - -# The "which" unix command finds where a command is. -if platform.system() == "Windows": - WHICH_CMD = "where" -else: - WHICH_CMD = "which" - - -def _executable_exists(name): - return ( - subprocess.call( - [WHICH_CMD, name], stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - == 0 - ) - - -# Exceptions -class PyperclipException(RuntimeError): - pass - - -class PyperclipWindowsException(PyperclipException): - def __init__(self, message): - message += f" ({ctypes.WinError()})" - super().__init__(message) - - -def _stringifyText(text) -> str: - acceptedTypes = (str, int, float, bool) - if not isinstance(text, acceptedTypes): - raise PyperclipException( - f"only str, int, float, and bool values " - f"can be copied to the clipboard, not {type(text).__name__}" - ) - return str(text) - - -def init_osx_pbcopy_clipboard(): - def copy_osx_pbcopy(text): - text = _stringifyText(text) # Converts non-str values to str. - p = subprocess.Popen(["pbcopy", "w"], stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text.encode(ENCODING)) - - def paste_osx_pbcopy(): - p = subprocess.Popen(["pbpaste", "r"], stdout=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - return stdout.decode(ENCODING) - - return copy_osx_pbcopy, paste_osx_pbcopy - - -def init_osx_pyobjc_clipboard(): - def copy_osx_pyobjc(text): - """Copy string argument to clipboard""" - text = _stringifyText(text) # Converts non-str values to str. - newStr = Foundation.NSString.stringWithString_(text).nsstring() - newData = newStr.dataUsingEncoding_(Foundation.NSUTF8StringEncoding) - board = AppKit.NSPasteboard.generalPasteboard() - board.declareTypes_owner_([AppKit.NSStringPboardType], None) - board.setData_forType_(newData, AppKit.NSStringPboardType) - - def paste_osx_pyobjc(): - """Returns contents of clipboard""" - board = AppKit.NSPasteboard.generalPasteboard() - content = board.stringForType_(AppKit.NSStringPboardType) - return content - - return copy_osx_pyobjc, paste_osx_pyobjc - - -def init_qt_clipboard(): - global QApplication - # $DISPLAY should exist - - # Try to import from qtpy, but if that fails try PyQt5 then PyQt4 - try: - from qtpy.QtWidgets import QApplication - except ImportError: - try: - from PyQt5.QtWidgets import QApplication - except ImportError: - from PyQt4.QtGui import QApplication - - app = QApplication.instance() - if app is None: - app = QApplication([]) - - def copy_qt(text): - text = _stringifyText(text) # Converts non-str values to str. - cb = app.clipboard() - cb.setText(text) - - def paste_qt() -> str: - cb = app.clipboard() - return str(cb.text()) - - return copy_qt, paste_qt - - -def init_xclip_clipboard(): - DEFAULT_SELECTION = "c" - PRIMARY_SELECTION = "p" - - def copy_xclip(text, primary=False): - text = _stringifyText(text) # Converts non-str values to str. - selection = DEFAULT_SELECTION - if primary: - selection = PRIMARY_SELECTION - p = subprocess.Popen( - ["xclip", "-selection", selection], stdin=subprocess.PIPE, close_fds=True - ) - p.communicate(input=text.encode(ENCODING)) - - def paste_xclip(primary=False): - selection = DEFAULT_SELECTION - if primary: - selection = PRIMARY_SELECTION - p = subprocess.Popen( - ["xclip", "-selection", selection, "-o"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=True, - ) - stdout, stderr = p.communicate() - # Intentionally ignore extraneous output on stderr when clipboard is empty - return stdout.decode(ENCODING) - - return copy_xclip, paste_xclip - - -def init_xsel_clipboard(): - DEFAULT_SELECTION = "-b" - PRIMARY_SELECTION = "-p" - - def copy_xsel(text, primary=False): - text = _stringifyText(text) # Converts non-str values to str. - selection_flag = DEFAULT_SELECTION - if primary: - selection_flag = PRIMARY_SELECTION - p = subprocess.Popen( - ["xsel", selection_flag, "-i"], stdin=subprocess.PIPE, close_fds=True - ) - p.communicate(input=text.encode(ENCODING)) - - def paste_xsel(primary=False): - selection_flag = DEFAULT_SELECTION - if primary: - selection_flag = PRIMARY_SELECTION - p = subprocess.Popen( - ["xsel", selection_flag, "-o"], stdout=subprocess.PIPE, close_fds=True - ) - stdout, stderr = p.communicate() - return stdout.decode(ENCODING) - - return copy_xsel, paste_xsel - - -def init_klipper_clipboard(): - def copy_klipper(text): - text = _stringifyText(text) # Converts non-str values to str. - p = subprocess.Popen( - [ - "qdbus", - "org.kde.klipper", - "/klipper", - "setClipboardContents", - text.encode(ENCODING), - ], - stdin=subprocess.PIPE, - close_fds=True, - ) - p.communicate(input=None) - - def paste_klipper(): - p = subprocess.Popen( - ["qdbus", "org.kde.klipper", "/klipper", "getClipboardContents"], - stdout=subprocess.PIPE, - close_fds=True, - ) - stdout, stderr = p.communicate() - - # Workaround for https://bugs.kde.org/show_bug.cgi?id=342874 - # TODO: https://github.com/asweigart/pyperclip/issues/43 - clipboardContents = stdout.decode(ENCODING) - # even if blank, Klipper will append a newline at the end - assert len(clipboardContents) > 0 - # make sure that newline is there - assert clipboardContents.endswith("\n") - if clipboardContents.endswith("\n"): - clipboardContents = clipboardContents[:-1] - return clipboardContents - - return copy_klipper, paste_klipper - - -def init_dev_clipboard_clipboard(): - def copy_dev_clipboard(text): - text = _stringifyText(text) # Converts non-str values to str. - if text == "": - warnings.warn( - "Pyperclip cannot copy a blank string to the clipboard on Cygwin." - "This is effectively a no-op." - ) - if "\r" in text: - warnings.warn("Pyperclip cannot handle \\r characters on Cygwin.") - - with open("/dev/clipboard", "wt") as fd: - fd.write(text) - - def paste_dev_clipboard() -> str: - with open("/dev/clipboard") as fd: - content = fd.read() - return content - - return copy_dev_clipboard, paste_dev_clipboard - - -def init_no_clipboard(): - class ClipboardUnavailable: - def __call__(self, *args, **kwargs): - raise PyperclipException(EXCEPT_MSG) - - def __bool__(self) -> bool: - return False - - return ClipboardUnavailable(), ClipboardUnavailable() - - -# Windows-related clipboard functions: -class CheckedCall: - def __init__(self, f): - super().__setattr__("f", f) - - def __call__(self, *args): - ret = self.f(*args) - if not ret and get_errno(): - raise PyperclipWindowsException("Error calling " + self.f.__name__) - return ret - - def __setattr__(self, key, value): - setattr(self.f, key, value) - - -def init_windows_clipboard(): - global HGLOBAL, LPVOID, DWORD, LPCSTR, INT - global HWND, HINSTANCE, HMENU, BOOL, UINT, HANDLE - from ctypes.wintypes import ( - BOOL, - DWORD, - HANDLE, - HGLOBAL, - HINSTANCE, - HMENU, - HWND, - INT, - LPCSTR, - LPVOID, - UINT, - ) - - windll = ctypes.windll - msvcrt = ctypes.CDLL("msvcrt") - - safeCreateWindowExA = CheckedCall(windll.user32.CreateWindowExA) - safeCreateWindowExA.argtypes = [ - DWORD, - LPCSTR, - LPCSTR, - DWORD, - INT, - INT, - INT, - INT, - HWND, - HMENU, - HINSTANCE, - LPVOID, - ] - safeCreateWindowExA.restype = HWND - - safeDestroyWindow = CheckedCall(windll.user32.DestroyWindow) - safeDestroyWindow.argtypes = [HWND] - safeDestroyWindow.restype = BOOL - - OpenClipboard = windll.user32.OpenClipboard - OpenClipboard.argtypes = [HWND] - OpenClipboard.restype = BOOL - - safeCloseClipboard = CheckedCall(windll.user32.CloseClipboard) - safeCloseClipboard.argtypes = [] - safeCloseClipboard.restype = BOOL - - safeEmptyClipboard = CheckedCall(windll.user32.EmptyClipboard) - safeEmptyClipboard.argtypes = [] - safeEmptyClipboard.restype = BOOL - - safeGetClipboardData = CheckedCall(windll.user32.GetClipboardData) - safeGetClipboardData.argtypes = [UINT] - safeGetClipboardData.restype = HANDLE - - safeSetClipboardData = CheckedCall(windll.user32.SetClipboardData) - safeSetClipboardData.argtypes = [UINT, HANDLE] - safeSetClipboardData.restype = HANDLE - - safeGlobalAlloc = CheckedCall(windll.kernel32.GlobalAlloc) - safeGlobalAlloc.argtypes = [UINT, c_size_t] - safeGlobalAlloc.restype = HGLOBAL - - safeGlobalLock = CheckedCall(windll.kernel32.GlobalLock) - safeGlobalLock.argtypes = [HGLOBAL] - safeGlobalLock.restype = LPVOID - - safeGlobalUnlock = CheckedCall(windll.kernel32.GlobalUnlock) - safeGlobalUnlock.argtypes = [HGLOBAL] - safeGlobalUnlock.restype = BOOL - - wcslen = CheckedCall(msvcrt.wcslen) - wcslen.argtypes = [c_wchar_p] - wcslen.restype = UINT - - GMEM_MOVEABLE = 0x0002 - CF_UNICODETEXT = 13 - - @contextlib.contextmanager - def window(): - """ - Context that provides a valid Windows hwnd. - """ - # we really just need the hwnd, so setting "STATIC" - # as predefined lpClass is just fine. - hwnd = safeCreateWindowExA( - 0, b"STATIC", None, 0, 0, 0, 0, 0, None, None, None, None - ) - try: - yield hwnd - finally: - safeDestroyWindow(hwnd) - - @contextlib.contextmanager - def clipboard(hwnd): - """ - Context manager that opens the clipboard and prevents - other applications from modifying the clipboard content. - """ - # We may not get the clipboard handle immediately because - # some other application is accessing it (?) - # We try for at least 500ms to get the clipboard. - t = time.time() + 0.5 - success = False - while time.time() < t: - success = OpenClipboard(hwnd) - if success: - break - time.sleep(0.01) - if not success: - raise PyperclipWindowsException("Error calling OpenClipboard") - - try: - yield - finally: - safeCloseClipboard() - - def copy_windows(text): - # This function is heavily based on - # http://msdn.com/ms649016#_win32_Copying_Information_to_the_Clipboard - - text = _stringifyText(text) # Converts non-str values to str. - - with window() as hwnd: - # http://msdn.com/ms649048 - # If an application calls OpenClipboard with hwnd set to NULL, - # EmptyClipboard sets the clipboard owner to NULL; - # this causes SetClipboardData to fail. - # => We need a valid hwnd to copy something. - with clipboard(hwnd): - safeEmptyClipboard() - - if text: - # http://msdn.com/ms649051 - # If the hMem parameter identifies a memory object, - # the object must have been allocated using the - # function with the GMEM_MOVEABLE flag. - count = wcslen(text) + 1 - handle = safeGlobalAlloc(GMEM_MOVEABLE, count * sizeof(c_wchar)) - locked_handle = safeGlobalLock(handle) - - ctypes.memmove( - c_wchar_p(locked_handle), - c_wchar_p(text), - count * sizeof(c_wchar), - ) - - safeGlobalUnlock(handle) - safeSetClipboardData(CF_UNICODETEXT, handle) - - def paste_windows(): - with clipboard(None): - handle = safeGetClipboardData(CF_UNICODETEXT) - if not handle: - # GetClipboardData may return NULL with errno == NO_ERROR - # if the clipboard is empty. - # (Also, it may return a handle to an empty buffer, - # but technically that's not empty) - return "" - return c_wchar_p(handle).value - - return copy_windows, paste_windows - - -def init_wsl_clipboard(): - def copy_wsl(text): - text = _stringifyText(text) # Converts non-str values to str. - p = subprocess.Popen(["clip.exe"], stdin=subprocess.PIPE, close_fds=True) - p.communicate(input=text.encode(ENCODING)) - - def paste_wsl(): - p = subprocess.Popen( - ["powershell.exe", "-command", "Get-Clipboard"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=True, - ) - stdout, stderr = p.communicate() - # WSL appends "\r\n" to the contents. - return stdout[:-2].decode(ENCODING) - - return copy_wsl, paste_wsl - - -# Automatic detection of clipboard mechanisms -# and importing is done in determine_clipboard(): -def determine_clipboard(): - """ - Determine the OS/platform and set the copy() and paste() functions - accordingly. - """ - global Foundation, AppKit, qtpy, PyQt4, PyQt5 - - # Setup for the CYGWIN platform: - if ( - "cygwin" in platform.system().lower() - ): # Cygwin has a variety of values returned by platform.system(), - # such as 'CYGWIN_NT-6.1' - # FIXME: pyperclip currently does not support Cygwin, - # see https://github.com/asweigart/pyperclip/issues/55 - if os.path.exists("/dev/clipboard"): - warnings.warn( - "Pyperclip's support for Cygwin is not perfect," - "see https://github.com/asweigart/pyperclip/issues/55" - ) - return init_dev_clipboard_clipboard() - - # Setup for the WINDOWS platform: - elif os.name == "nt" or platform.system() == "Windows": - return init_windows_clipboard() - - if platform.system() == "Linux": - if distutils.spawn.find_executable("wslconfig.exe"): - return init_wsl_clipboard() - - # Setup for the MAC OS X platform: - if os.name == "mac" or platform.system() == "Darwin": - try: - import AppKit - import Foundation # check if pyobjc is installed - except ImportError: - return init_osx_pbcopy_clipboard() - else: - return init_osx_pyobjc_clipboard() - - # Setup for the LINUX platform: - if HAS_DISPLAY: - if _executable_exists("xsel"): - return init_xsel_clipboard() - if _executable_exists("xclip"): - return init_xclip_clipboard() - if _executable_exists("klipper") and _executable_exists("qdbus"): - return init_klipper_clipboard() - - try: - # qtpy is a small abstraction layer that lets you write applications - # using a single api call to either PyQt or PySide. - # https://pypi.python.org/project/QtPy - import qtpy # check if qtpy is installed - except ImportError: - # If qtpy isn't installed, fall back on importing PyQt4. - try: - import PyQt5 # check if PyQt5 is installed - except ImportError: - try: - import PyQt4 # check if PyQt4 is installed - except ImportError: - pass # We want to fail fast for all non-ImportError exceptions. - else: - return init_qt_clipboard() - else: - return init_qt_clipboard() - else: - return init_qt_clipboard() - - return init_no_clipboard() - - -def set_clipboard(clipboard): - """ - Explicitly sets the clipboard mechanism. The "clipboard mechanism" is how - the copy() and paste() functions interact with the operating system to - implement the copy/paste feature. The clipboard parameter must be one of: - - pbcopy - - pbobjc (default on Mac OS X) - - qt - - xclip - - xsel - - klipper - - windows (default on Windows) - - no (this is what is set when no clipboard mechanism can be found) - """ - global copy, paste - - clipboard_types = { - "pbcopy": init_osx_pbcopy_clipboard, - "pyobjc": init_osx_pyobjc_clipboard, - "qt": init_qt_clipboard, # TODO - split this into 'qtpy', 'pyqt4', and 'pyqt5' - "xclip": init_xclip_clipboard, - "xsel": init_xsel_clipboard, - "klipper": init_klipper_clipboard, - "windows": init_windows_clipboard, - "no": init_no_clipboard, - } - - if clipboard not in clipboard_types: - allowed_clipboard_types = [repr(_) for _ in clipboard_types.keys()] - raise ValueError( - f"Argument must be one of {', '.join(allowed_clipboard_types)}" - ) - - # Sets pyperclip's copy() and paste() functions: - copy, paste = clipboard_types[clipboard]() - - -def lazy_load_stub_copy(text): - """ - A stub function for copy(), which will load the real copy() function when - called so that the real copy() function is used for later calls. - - This allows users to import pyperclip without having determine_clipboard() - automatically run, which will automatically select a clipboard mechanism. - This could be a problem if it selects, say, the memory-heavy PyQt4 module - but the user was just going to immediately call set_clipboard() to use a - different clipboard mechanism. - - The lazy loading this stub function implements gives the user a chance to - call set_clipboard() to pick another clipboard mechanism. Or, if the user - simply calls copy() or paste() without calling set_clipboard() first, - will fall back on whatever clipboard mechanism that determine_clipboard() - automatically chooses. - """ - global copy, paste - copy, paste = determine_clipboard() - return copy(text) - - -def lazy_load_stub_paste(): - """ - A stub function for paste(), which will load the real paste() function when - called so that the real paste() function is used for later calls. - - This allows users to import pyperclip without having determine_clipboard() - automatically run, which will automatically select a clipboard mechanism. - This could be a problem if it selects, say, the memory-heavy PyQt4 module - but the user was just going to immediately call set_clipboard() to use a - different clipboard mechanism. - - The lazy loading this stub function implements gives the user a chance to - call set_clipboard() to pick another clipboard mechanism. Or, if the user - simply calls copy() or paste() without calling set_clipboard() first, - will fall back on whatever clipboard mechanism that determine_clipboard() - automatically chooses. - """ - global copy, paste - copy, paste = determine_clipboard() - return paste() - - -def is_available() -> bool: - return copy != lazy_load_stub_copy and paste != lazy_load_stub_paste - - -# Initially, copy() and paste() are set to lazy loading wrappers which will -# set `copy` and `paste` to real functions the first time they're used, unless -# set_clipboard() or determine_clipboard() is called first. -copy, paste = lazy_load_stub_copy, lazy_load_stub_paste - - -__all__ = ["copy", "paste", "set_clipboard", "determine_clipboard"] +from pyclip import copy, paste # pandas aliases -clipboard_get = paste +clipboard_get = partial(paste, text=True) clipboard_set = copy diff --git a/pandas/io/clipboards.py b/pandas/io/clipboards.py index 54cb6b9f91137..32cc3580bd05b 100644 --- a/pandas/io/clipboards.py +++ b/pandas/io/clipboards.py @@ -118,7 +118,7 @@ def to_clipboard(obj, excel=True, sep=None, **kwargs): # pragma: no cover sep = "\t" buf = StringIO() - # clipboard_set (pyperclip) expects unicode + # clipboard_set (pyclip) expects unicode obj.to_csv(buf, sep=sep, encoding="utf-8", **kwargs) text = buf.getvalue() diff --git a/requirements-dev.txt b/requirements-dev.txt index be60c90aef8aa..3ca88d3fea3ff 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -65,7 +65,6 @@ odfpy fastparquet>=0.3.2 pyarrow>=0.15.0 python-snappy -pyqt5>=5.9.2 tables>=3.5.1 s3fs>=0.4.0 fsspec>=0.7.4 diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py index 1ad9ec03925a0..2a30f34d90689 100755 --- a/scripts/generate_pip_deps_from_conda.py +++ b/scripts/generate_pip_deps_from_conda.py @@ -20,7 +20,7 @@ import yaml EXCLUDE = {"python", "c-compiler", "cxx-compiler"} -RENAME = {"pytables": "tables", "pyqt": "pyqt5", "dask-core": "dask"} +RENAME = {"pytables": "tables", "dask-core": "dask"} def conda_package_to_pip(package): diff --git a/setup.cfg b/setup.cfg index ce055f550a868..cbfdffc297b72 100644 --- a/setup.cfg +++ b/setup.cfg @@ -189,6 +189,3 @@ check_untyped_defs = False [mypy-pandas._version] check_untyped_defs = False - -[mypy-pandas.io.clipboard] -check_untyped_defs = False From 5a9d9bcec239b206dbcedd1697a5fe696fce5744 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 27 Feb 2021 14:29:44 -0600 Subject: [PATCH 2/8] DEPS: optional import pyclip, remove test marker --- .github/workflows/database.yml | 2 +- .travis.yml | 4 ++-- azure-pipelines.yml | 2 +- ci/azure/posix.yml | 14 +++++++------- ci/deps/azure-38-locale.yaml | 1 + ci/deps/azure-38-numpydev.yaml | 1 + ci/deps/azure-macos-37.yaml | 1 + ci/deps/azure-windows-37.yaml | 1 + ci/deps/azure-windows-38.yaml | 3 +++ environment.yml | 1 + pandas/conftest.py | 1 - pandas/io/clipboard/__init__.py | 7 ------- pandas/io/clipboards.py | 13 ++++++++----- pandas/tests/io/test_clipboard.py | 27 ++++++++++----------------- requirements-dev.txt | 1 + 15 files changed, 38 insertions(+), 41 deletions(-) delete mode 100644 pandas/io/clipboard/__init__.py diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index b34373b82af1a..febb8305ba0cc 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -11,7 +11,7 @@ on: env: PYTEST_WORKERS: "auto" PANDAS_CI: 1 - PATTERN: ((not slow and not network and not clipboard) or (single and db)) + PATTERN: ((not slow and not network) or (single and db)) jobs: Linux_py37_locale: diff --git a/.travis.yml b/.travis.yml index 8ede978074a9c..f50c3f1cf10de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,13 +37,13 @@ matrix: include: - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not arm_slow)" allow_failures: # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719 - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not arm_slow)" before_install: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 464bad7884362..5ca495a07df77 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,7 +42,7 @@ jobs: pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ python setup.py build_ext -q -j2 && \ python -m pip install --no-build-isolation -e . && \ - pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml" + pytest -m 'not slow and not network' pandas --junitxml=test-data.xml" displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' - task: PublishTestResults@2 diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 4cb4eaf95f6f5..9e5b5c63a5bfb 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -18,12 +18,12 @@ jobs: py37_minimum_versions: ENV_FILE: ci/deps/azure-37-minimum_versions.yaml CONDA_PY: "37" - PATTERN: "not slow and not network and not clipboard" + PATTERN: "not slow and not network" py37: ENV_FILE: ci/deps/azure-37.yaml CONDA_PY: "37" - PATTERN: "not slow and not network and not clipboard" + PATTERN: "not slow and not network" py37_locale_slow: ENV_FILE: ci/deps/azure-37-locale_slow.yaml @@ -31,7 +31,7 @@ jobs: PATTERN: "slow" LANG: "it_IT.utf8" LC_ALL: "it_IT.utf8" - EXTRA_APT: "language-pack-it xsel" + EXTRA_APT: "language-pack-it" py37_slow: ENV_FILE: ci/deps/azure-37-slow.yaml @@ -41,7 +41,7 @@ jobs: py38: ENV_FILE: ci/deps/azure-38.yaml CONDA_PY: "38" - PATTERN: "not slow and not network and not clipboard" + PATTERN: "not slow and not network" py38_slow: ENV_FILE: ci/deps/azure-38-slow.yaml @@ -56,7 +56,7 @@ jobs: # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any LANG: "zh_CN.utf8" LC_ALL: "zh_CN.utf8" - EXTRA_APT: "language-pack-zh-hans xsel" + EXTRA_APT: "language-pack-zh-hans xclip" py38_np_dev: ENV_FILE: ci/deps/azure-38-numpydev.yaml @@ -64,12 +64,12 @@ jobs: PATTERN: "not slow and not network" TEST_ARGS: "-W error" PANDAS_TESTING_MODE: "deprecate" - EXTRA_APT: "xsel" + EXTRA_APT: "xclip" py39: ENV_FILE: ci/deps/azure-39.yaml CONDA_PY: "39" - PATTERN: "not slow and not network and not clipboard" + PATTERN: "not slow and not network" steps: - script: | diff --git a/ci/deps/azure-38-locale.yaml b/ci/deps/azure-38-locale.yaml index 26297a3066fa5..347efbcf7b018 100644 --- a/ci/deps/azure-38-locale.yaml +++ b/ci/deps/azure-38-locale.yaml @@ -39,3 +39,4 @@ dependencies: - pip - pip: - pyxlsb + - pyclip diff --git a/ci/deps/azure-38-numpydev.yaml b/ci/deps/azure-38-numpydev.yaml index f11a3bcb28ab2..ed4ffd7aeab93 100644 --- a/ci/deps/azure-38-numpydev.yaml +++ b/ci/deps/azure-38-numpydev.yaml @@ -20,3 +20,4 @@ dependencies: - "--pre" - "numpy" - "scipy" + - pyclip diff --git a/ci/deps/azure-macos-37.yaml b/ci/deps/azure-macos-37.yaml index d667adddda859..2fe874cfdad7d 100644 --- a/ci/deps/azure-macos-37.yaml +++ b/ci/deps/azure-macos-37.yaml @@ -34,3 +34,4 @@ dependencies: - cython>=0.29.21 - pyreadstat - pyxlsb + - pyclip diff --git a/ci/deps/azure-windows-37.yaml b/ci/deps/azure-windows-37.yaml index e7ac4c783b855..c507f0c8c90fc 100644 --- a/ci/deps/azure-windows-37.yaml +++ b/ci/deps/azure-windows-37.yaml @@ -40,3 +40,4 @@ dependencies: - pip - pip: - pyxlsb + - pyclip diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index 661d8813d32d2..046f2e8286bfa 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -34,3 +34,6 @@ dependencies: - xlrd<2.0 - xlsxwriter - xlwt + - pip + - pip: + - pyclip diff --git a/environment.yml b/environment.yml index 69582b79ba704..b4b78fdc1d80e 100644 --- a/environment.yml +++ b/environment.yml @@ -101,6 +101,7 @@ dependencies: - pyarrow>=0.15.0 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather - python-snappy # required by pyarrow + - pyclip # pandas.read_clipboard - pytables>=3.5.1 # pandas.read_hdf, DataFrame.to_hdf - s3fs>=0.4.0 # file IO when using 's3://...' path - fsspec>=0.7.4 # for generic remote file operations diff --git a/pandas/conftest.py b/pandas/conftest.py index 426cbf6a65aa5..b8e9dec9427fa 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -81,7 +81,6 @@ def pytest_configure(config): "markers", "db: tests requiring a database (mysql or postgres)" ) config.addinivalue_line("markers", "high_memory: mark a test as a high-memory only") - config.addinivalue_line("markers", "clipboard: mark a pd.read_clipboard test") config.addinivalue_line( "markers", "arm_slow: mark a test as slow for arm64 architecture" ) diff --git a/pandas/io/clipboard/__init__.py b/pandas/io/clipboard/__init__.py deleted file mode 100644 index b02533b7e5366..0000000000000 --- a/pandas/io/clipboard/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from functools import partial - -from pyclip import copy, paste - -# pandas aliases -clipboard_get = partial(paste, text=True) -clipboard_set = copy diff --git a/pandas/io/clipboards.py b/pandas/io/clipboards.py index 32cc3580bd05b..b832fc952b230 100644 --- a/pandas/io/clipboards.py +++ b/pandas/io/clipboards.py @@ -2,6 +2,8 @@ from io import StringIO import warnings +from pandas.compat._optional import import_optional_dependency + from pandas.core.dtypes.generic import ABCDataFrame from pandas import ( @@ -35,10 +37,11 @@ def read_clipboard(sep=r"\s+", **kwargs): # pragma: no cover if encoding is not None and encoding.lower().replace("-", "") != "utf8": raise NotImplementedError("reading from clipboard only supports utf-8 encoding") - from pandas.io.clipboard import clipboard_get from pandas.io.parsers import read_csv - text = clipboard_get() + pyclip = import_optional_dependency("pyclip") + + text = pyclip.paste(text=True) # Try to decode (if needed, as "text" might already be a string here). try: @@ -107,7 +110,7 @@ def to_clipboard(obj, excel=True, sep=None, **kwargs): # pragma: no cover if encoding is not None and encoding.lower().replace("-", "") != "utf8": raise ValueError("clipboard only supports utf-8 encoding") - from pandas.io.clipboard import clipboard_set + pyclip = import_optional_dependency("pyclip") if excel is None: excel = True @@ -122,7 +125,7 @@ def to_clipboard(obj, excel=True, sep=None, **kwargs): # pragma: no cover obj.to_csv(buf, sep=sep, encoding="utf-8", **kwargs) text = buf.getvalue() - clipboard_set(text) + pyclip.copy(text) return except TypeError: warnings.warn( @@ -137,4 +140,4 @@ def to_clipboard(obj, excel=True, sep=None, **kwargs): # pragma: no cover objstr = obj.to_string(**kwargs) else: objstr = str(obj) - clipboard_set(objstr) + pyclip.copy(objstr) diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index 45d9ad430aa43..53c7c36728f7a 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -10,10 +10,7 @@ ) import pandas._testing as tm -from pandas.io.clipboard import ( - clipboard_get, - clipboard_set, -) +pyclip = pytest.importorskip("pyclip") def build_kwargs(sep, excel): @@ -114,8 +111,7 @@ def df(request): def mock_clipboard(monkeypatch, request): """Fixture mocking clipboard IO. - This mocks pandas.io.clipboard.clipboard_get and - pandas.io.clipboard.clipboard_set. + This mocks pyclip.paste and pyclip.coopy. This uses a local dict for storing data. The dictionary key used is the test ID, available with ``request.node.name``. @@ -129,27 +125,25 @@ def mock_clipboard(monkeypatch, request): def _mock_set(data): _mock_data[request.node.name] = data - def _mock_get(): + def _mock_get(text=True): return _mock_data[request.node.name] - monkeypatch.setattr("pandas.io.clipboard.clipboard_set", _mock_set) - monkeypatch.setattr("pandas.io.clipboard.clipboard_get", _mock_get) + monkeypatch.setattr("pyclip.copy", _mock_set) + monkeypatch.setattr("pyclip.paste", _mock_get) yield _mock_data -@pytest.mark.clipboard def test_mock_clipboard(mock_clipboard): - import pandas.io.clipboard + import pyclip - pandas.io.clipboard.clipboard_set("abc") + pyclip.copy("abc") assert "abc" in set(mock_clipboard.values()) - result = pandas.io.clipboard.clipboard_get() + result = pyclip.paste(text=True) assert result == "abc" @pytest.mark.single -@pytest.mark.clipboard @pytest.mark.usefixtures("mock_clipboard") class TestClipboard: def check_round_trip_frame(self, data, excel=None, sep=None, encoding=None): @@ -257,9 +251,8 @@ def test_round_trip_valid_encodings(self, enc, df): @pytest.mark.single -@pytest.mark.clipboard @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑´...", "abcd..."]) def test_raw_roundtrip(data): # PR #25040 wide unicode wasn't copied correctly on PY3 on windows - clipboard_set(data) - assert data == clipboard_get() + pyclip.copy(data) + assert data == pyclip.paste(text=True) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3ca88d3fea3ff..db9f6a3a2dc5c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -65,6 +65,7 @@ odfpy fastparquet>=0.3.2 pyarrow>=0.15.0 python-snappy +pyclip tables>=3.5.1 s3fs>=0.4.0 fsspec>=0.7.4 From 0bc72264bbd0c6b40348749e8e9c302998d238cc Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 27 Feb 2021 14:35:25 -0600 Subject: [PATCH 3/8] fix typos, and environment --- environment.yml | 2 +- pandas/io/clipboards.py | 2 +- pandas/tests/io/test_clipboard.py | 2 +- requirements-dev.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index b4b78fdc1d80e..bb870f7379757 100644 --- a/environment.yml +++ b/environment.yml @@ -101,7 +101,6 @@ dependencies: - pyarrow>=0.15.0 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather - python-snappy # required by pyarrow - - pyclip # pandas.read_clipboard - pytables>=3.5.1 # pandas.read_hdf, DataFrame.to_hdf - s3fs>=0.4.0 # file IO when using 's3://...' path - fsspec>=0.7.4 # for generic remote file operations @@ -115,3 +114,4 @@ dependencies: - pip: - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@2488b7defbd3d753dd5fcfc890fc4a7e79d25103 - numpydoc < 1.2 # 2021-02-09 1.2dev breaking CI + - pyclip # pandas.read_clipboard diff --git a/pandas/io/clipboards.py b/pandas/io/clipboards.py index b832fc952b230..1e317733c5ba4 100644 --- a/pandas/io/clipboards.py +++ b/pandas/io/clipboards.py @@ -121,7 +121,7 @@ def to_clipboard(obj, excel=True, sep=None, **kwargs): # pragma: no cover sep = "\t" buf = StringIO() - # clipboard_set (pyclip) expects unicode + # pyclip.copy expects unicode obj.to_csv(buf, sep=sep, encoding="utf-8", **kwargs) text = buf.getvalue() diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index 53c7c36728f7a..c2146767a52bb 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -111,7 +111,7 @@ def df(request): def mock_clipboard(monkeypatch, request): """Fixture mocking clipboard IO. - This mocks pyclip.paste and pyclip.coopy. + This mocks pyclip.paste and pyclip.copy. This uses a local dict for storing data. The dictionary key used is the test ID, available with ``request.node.name``. diff --git a/requirements-dev.txt b/requirements-dev.txt index db9f6a3a2dc5c..ce8859745ad47 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -65,7 +65,6 @@ odfpy fastparquet>=0.3.2 pyarrow>=0.15.0 python-snappy -pyclip tables>=3.5.1 s3fs>=0.4.0 fsspec>=0.7.4 @@ -78,3 +77,4 @@ tabulate>=0.8.3 natsort git+https://github.com/pandas-dev/pydata-sphinx-theme.git@2488b7defbd3d753dd5fcfc890fc4a7e79d25103 numpydoc < 1.2 +pyclip From 92eba3c64704d864fe7ab8c678014e21b078fba4 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 28 Feb 2021 01:13:47 -0600 Subject: [PATCH 4/8] debug macos build --- ci/setup_env.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index c36422884f2ec..5d545ec8a4fea 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -57,8 +57,9 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -conda install pip conda # create conda to create a historical artifact for pip & setuptools -conda update -n base conda +# conda install pip conda # create conda to create a historical artifact for pip & setuptools +# conda update -n base conda +conda update conda echo "conda info -a" conda info -a From 23197c7a88a8c5b8afd037764b1915661ac84256 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 28 Feb 2021 01:18:14 -0600 Subject: [PATCH 5/8] debug macos build --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5ca495a07df77..c623e6a6ef687 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,7 +15,7 @@ jobs: - template: ci/azure/posix.yml parameters: name: macOS - vmImage: macOS-10.14 + vmImage: macOS-10.15 - template: ci/azure/posix.yml parameters: From a4d3313906af0b0d5e16c3c79a1f9d25bac16731 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 28 Feb 2021 09:41:02 -0600 Subject: [PATCH 6/8] revert changes in setup_env --- ci/setup_env.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 5d545ec8a4fea..c36422884f2ec 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -57,9 +57,8 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -# conda install pip conda # create conda to create a historical artifact for pip & setuptools -# conda update -n base conda -conda update conda +conda install pip conda # create conda to create a historical artifact for pip & setuptools +conda update -n base conda echo "conda info -a" conda info -a From c2c78ab1b3f6aaf04e1636d6cc651dc4e18a0e2f Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 3 Mar 2021 16:28:21 -0600 Subject: [PATCH 7/8] debug macOS build --- azure-pipelines.yml | 2 +- setup.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c623e6a6ef687..5ca495a07df77 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,7 +15,7 @@ jobs: - template: ci/azure/posix.yml parameters: name: macOS - vmImage: macOS-10.15 + vmImage: macOS-10.14 - template: ci/azure/posix.yml parameters: diff --git a/setup.py b/setup.py index 45548fed68322..3095138a679c2 100755 --- a/setup.py +++ b/setup.py @@ -350,6 +350,10 @@ def run(self): python_target = get_config_vars().get( "MACOSX_DEPLOYMENT_TARGET", current_system ) + print() + print(current_system) + print(python_target) + print() if ( LooseVersion(str(python_target)) < "10.9" and LooseVersion(current_system) >= "10.9" From e17051d521602aa9aa220cb0352c8d2d112d1ce0 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 3 Mar 2021 16:38:11 -0600 Subject: [PATCH 8/8] debug macOS build --- ci/azure/posix.yml | 1 + setup.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9e5b5c63a5bfb..3db886de6c6cd 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -13,6 +13,7 @@ jobs: ENV_FILE: ci/deps/azure-macos-37.yaml CONDA_PY: "37" PATTERN: "not slow and not network" + MACOSX_DEPLOYMENT_TARGET: 10.13 ${{ if eq(parameters.name, 'Linux') }}: py37_minimum_versions: diff --git a/setup.py b/setup.py index 3095138a679c2..45548fed68322 100755 --- a/setup.py +++ b/setup.py @@ -350,10 +350,6 @@ def run(self): python_target = get_config_vars().get( "MACOSX_DEPLOYMENT_TARGET", current_system ) - print() - print(current_system) - print(python_target) - print() if ( LooseVersion(str(python_target)) < "10.9" and LooseVersion(current_system) >= "10.9"