Skip to content

ENH: Add use_nullable_dtypes to read_clipboard #50502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jan 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions doc/source/whatsnew/v2.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ Configuration option, ``mode.dtype_backend``, to return pyarrow-backed dtypes
The ``use_nullable_dtypes`` keyword argument has been expanded to the following functions to enable automatic conversion to nullable dtypes (:issue:`36712`)

* :func:`read_csv`
* :func:`read_clipboard`
* :func:`read_fwf`
* :func:`read_excel`
* :func:`read_html`
Expand All @@ -49,6 +50,7 @@ Additionally a new global configuration, ``mode.dtype_backend`` can now be used
to select the nullable dtypes implementation.

* :func:`read_csv` (with ``engine="pyarrow"`` or ``engine="python"``)
* :func:`read_clipboard` (with ``engine="python"``)
* :func:`read_excel`
* :func:`read_html`
* :func:`read_xml`
Expand Down
23 changes: 21 additions & 2 deletions pandas/io/clipboards.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
)


def read_clipboard(sep: str = r"\s+", **kwargs): # pragma: no cover
def read_clipboard(
sep: str = r"\s+", use_nullable_dtypes: bool = False, **kwargs
): # pragma: no cover
r"""
Read text from clipboard and pass to read_csv.

Expand All @@ -24,6 +26,21 @@ def read_clipboard(sep: str = r"\s+", **kwargs): # pragma: no cover
A string or regex delimiter. The default of '\s+' denotes
one or more whitespace characters.

use_nullable_dtypes : bool = False
Whether or not to use nullable dtypes as default when reading data. If
set to True, nullable dtypes are used for all dtypes that have a nullable
implementation, even if no nulls are present.

The nullable dtype implementation can be configured by calling
``pd.set_option("mode.dtype_backend", "pandas")`` to use
numpy-backed nullable dtypes or
``pd.set_option("mode.dtype_backend", "pyarrow")`` to use
pyarrow-backed nullable dtypes (using ``pd.ArrowDtype``).
This is only implemented for the ``python``
engine.

.. versionadded:: 2.0

**kwargs
See read_csv for the full argument list.

Expand Down Expand Up @@ -85,7 +102,9 @@ def read_clipboard(sep: str = r"\s+", **kwargs): # pragma: no cover
stacklevel=find_stack_level(),
)

return read_csv(StringIO(text), sep=sep, **kwargs)
return read_csv(
StringIO(text), sep=sep, use_nullable_dtypes=use_nullable_dtypes, **kwargs
)


def to_clipboard(
Expand Down
2 changes: 2 additions & 0 deletions pandas/io/parsers/readers.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,8 @@
numpy-backed nullable dtypes or
``pd.set_option("mode.dtype_backend", "pyarrow")`` to use
pyarrow-backed nullable dtypes (using ``pd.ArrowDtype``).
This is only implemented for the ``pyarrow`` or ``python``
engines.

.. versionadded:: 2.0

Expand Down
64 changes: 64 additions & 0 deletions pandas/tests/io/test_clipboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,19 @@
PyperclipWindowsException,
)

import pandas as pd
from pandas import (
NA,
DataFrame,
Series,
get_option,
read_clipboard,
)
import pandas._testing as tm
from pandas.core.arrays import (
ArrowStringArray,
StringArray,
)

from pandas.io.clipboard import (
CheckedCall,
Expand Down Expand Up @@ -402,3 +409,60 @@ def test_raw_roundtrip(self, data):
# PR #25040 wide unicode wasn't copied correctly on PY3 on windows
clipboard_set(data)
assert data == clipboard_get()

@pytest.mark.parametrize("dtype_backend", ["pandas", "pyarrow"])
@pytest.mark.parametrize("engine", ["c", "python"])
def test_read_clipboard_nullable_dtypes(
self, request, mock_clipboard, string_storage, dtype_backend, engine
):
# GH#50502
if string_storage == "pyarrow" or dtype_backend == "pyarrow":
pa = pytest.importorskip("pyarrow")

if dtype_backend == "pyarrow" and engine == "c":
pytest.skip(reason="c engine not yet supported")

if string_storage == "python":
string_array = StringArray(np.array(["x", "y"], dtype=np.object_))
string_array_na = StringArray(np.array(["x", NA], dtype=np.object_))

else:
string_array = ArrowStringArray(pa.array(["x", "y"]))
string_array_na = ArrowStringArray(pa.array(["x", None]))

text = """a,b,c,d,e,f,g,h,i
x,1,4.0,x,2,4.0,,True,False
y,2,5.0,,,,,False,"""
mock_clipboard[request.node.name] = text

with pd.option_context("mode.string_storage", string_storage):
with pd.option_context("mode.dtype_backend", dtype_backend):
result = read_clipboard(
sep=",", use_nullable_dtypes=True, engine=engine
)

expected = DataFrame(
{
"a": string_array,
"b": Series([1, 2], dtype="Int64"),
"c": Series([4.0, 5.0], dtype="Float64"),
"d": string_array_na,
"e": Series([2, NA], dtype="Int64"),
"f": Series([4.0, NA], dtype="Float64"),
"g": Series([NA, NA], dtype="Int64"),
"h": Series([True, False], dtype="boolean"),
"i": Series([False, NA], dtype="boolean"),
}
)
if dtype_backend == "pyarrow":
from pandas.arrays import ArrowExtensionArray

expected = DataFrame(
{
col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
for col in expected.columns
}
)
expected["g"] = ArrowExtensionArray(pa.array([None, None]))

tm.assert_frame_equal(result, expected)