From c1b191ed7895dd3eb01c93b6aecc09a2e9e90e50 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 15 Nov 2022 08:58:53 +0000 Subject: [PATCH 1/7] fixup b005 errors --- doc/source/conf.py | 4 ++- pandas/core/ops/common.py | 7 ++++- pandas/tests/io/test_sql.py | 2 +- pandas/tests/util/test_str_methods.py | 41 +++++++++++++++++++++++++++ pandas/util/_str_methods.py | 18 ++++++++++++ setup.cfg | 2 -- 6 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 pandas/tests/util/test_str_methods.py create mode 100644 pandas/util/_str_methods.py diff --git a/doc/source/conf.py b/doc/source/conf.py index 24482d7c7461a..133bfe1560076 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -583,7 +583,9 @@ class AccessorCallableDocumenter(AccessorLevelDocumenter, MethodDocumenter): priority = 0.5 def format_name(self): - return MethodDocumenter.format_name(self).rstrip(".__call__") + return pandas.util._str_methods.removesuffix( + MethodDocumenter.format_name(self), ".__call__" + ) class PandasAutosummary(Autosummary): diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index f0e6aa3750cee..ca82479204096 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -9,6 +9,10 @@ from pandas._libs.lib import item_from_zerodim from pandas._libs.missing import is_matching_na from pandas._typing import F +from pandas.util._str_methods import ( + removeprefix, + removesuffix, +) from pandas.core.dtypes.generic import ( ABCDataFrame, @@ -52,7 +56,8 @@ def _unpack_zerodim_and_defer(method, name: str): ------- method """ - is_cmp = name.strip("__") in {"eq", "ne", "lt", "le", "gt", "ge"} + stripped_name = removesuffix(removeprefix(name, "__"), "__") + is_cmp = stripped_name in {"eq", "ne", "lt", "le", "gt", "ge"} @wraps(method) def new_method(self, other): diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f321ecc2f65ff..cb1bcf7a813eb 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1587,7 +1587,7 @@ def test_get_schema2(self, test_frame1): def _get_sqlite_column_type(self, schema, column): for col in schema.split("\n"): - if col.split()[0].strip('""') == column: + if col.split()[0].strip('"') == column: return col.split()[1] raise ValueError(f"Column {column} not found") diff --git a/pandas/tests/util/test_str_methods.py b/pandas/tests/util/test_str_methods.py new file mode 100644 index 0000000000000..e5fd6c4066477 --- /dev/null +++ b/pandas/tests/util/test_str_methods.py @@ -0,0 +1,41 @@ +import pytest + +from pandas.util._str_methods import ( + removeprefix, + removesuffix, +) + + +@pytest.mark.parametrize( + "string, prefix, expected", + ( + ("wildcat", "wild", "cat"), + ("blackbird", "black", "bird"), + ("housefly", "house", "fly"), + ("ladybug", "lady", "bug"), + ("rattlesnake", "rattle", "snake"), + ("baboon", "badger", "baboon"), + ("quetzal", "elk", "quetzal"), + ), +) +def test_remove_prefix(string, prefix, expected): + result = removeprefix(string, prefix) + assert result == expected + + +@pytest.mark.parametrize( + "string, suffix, expected", + ( + ("wildcat", "cat", "wild"), + ("blackbird", "bird", "black"), + ("housefly", "fly", "house"), + ("ladybug", "bug", "lady"), + ("rattlesnake", "snake", "rattle"), + ("seahorse", "horse", "sea"), + ("baboon", "badger", "baboon"), + ("quetzal", "elk", "quetzal"), + ), +) +def test_remove_suffix(string, suffix, expected): + result = removesuffix(string, suffix) + assert result == expected diff --git a/pandas/util/_str_methods.py b/pandas/util/_str_methods.py new file mode 100644 index 0000000000000..0cd5a4ef5cb84 --- /dev/null +++ b/pandas/util/_str_methods.py @@ -0,0 +1,18 @@ +""" +Python3.9 introduces removesuffix and remove prefix. + +They're reimplemented here for use in Python3.8. +""" +from __future__ import annotations + + +def removesuffix(string: str, suffix: str) -> str: + if string.endswith(suffix): + return string[: -len(suffix)] + return string + + +def removeprefix(string: str, prefix: str) -> str: + if string.startswith(prefix): + return string[len(prefix) :] + return string diff --git a/setup.cfg b/setup.cfg index 785143c7b647c..e252f1f013e30 100644 --- a/setup.cfg +++ b/setup.cfg @@ -192,8 +192,6 @@ ignore = # found modulo formatter (incorrect picks up mod operations) S001, # controversial - B005, - # controversial B006, # controversial B007, From 29597ca2ffcd4d4f49398b594e678f911c45fb05 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 16 Nov 2022 12:09:14 +0000 Subject: [PATCH 2/7] use sys.version_info check --- doc/source/conf.py | 11 ++++++++--- pandas/core/ops/common.py | 15 ++++++++++----- pandas/core/strings/object_array.py | 20 ++++++++++++-------- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 133bfe1560076..6671cefae9073 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -583,9 +583,14 @@ class AccessorCallableDocumenter(AccessorLevelDocumenter, MethodDocumenter): priority = 0.5 def format_name(self): - return pandas.util._str_methods.removesuffix( - MethodDocumenter.format_name(self), ".__call__" - ) + if sys.version_info < (3, 9): + # NOTE pyupgrade will remove this when we run it with --py39-plus + # so don't remove the unnecessary `else` statement below + from pandas.util._str_methods import removesuffix + + return removesuffix(MethodDocumenter.format_name(self), ".__call__") + else: + return MethodDocumenter.format_name(self).removesuffix(".__call__") class PandasAutosummary(Autosummary): diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index ca82479204096..8bba400cedc9a 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -4,15 +4,12 @@ from __future__ import annotations from functools import wraps +import sys from typing import Callable from pandas._libs.lib import item_from_zerodim from pandas._libs.missing import is_matching_na from pandas._typing import F -from pandas.util._str_methods import ( - removeprefix, - removesuffix, -) from pandas.core.dtypes.generic import ( ABCDataFrame, @@ -56,7 +53,15 @@ def _unpack_zerodim_and_defer(method, name: str): ------- method """ - stripped_name = removesuffix(removeprefix(name, "__"), "__") + if sys.version_info < (3, 9): + from pandas.util._str_methods import ( + removeprefix, + removesuffix, + ) + + stripped_name = removesuffix(removeprefix(name, "__"), "__") + else: + stripped_name = name.removeprefix("__").removesuffix("__") is_cmp = stripped_name in {"eq", "ne", "lt", "le", "gt", "ge"} @wraps(method) diff --git a/pandas/core/strings/object_array.py b/pandas/core/strings/object_array.py index 21e7ede3ed386..7694de0c1e6fa 100644 --- a/pandas/core/strings/object_array.py +++ b/pandas/core/strings/object_array.py @@ -3,6 +3,7 @@ from collections.abc import Callable # noqa: PDF001 import functools import re +import sys import textwrap from typing import ( TYPE_CHECKING, @@ -462,16 +463,19 @@ def removeprefix(text: str) -> str: return self._str_map(removeprefix) def _str_removesuffix(self, suffix: str) -> Series: - # this could be used on Python 3.9+ - # f = lambda x: x.removesuffix(suffix) - # return self._str_map(str.removesuffix) + if sys.version_info < (3, 9): + # NOTE pyupgrade will remove this when we run it with --py39-plus + # so don't remove the unnecessary `else` statement below - def removesuffix(text: str) -> str: - if text.endswith(suffix): - return text[: -len(suffix)] - return text + def removesuffix(text: str) -> str: + if text.endswith(suffix): + return text[: -len(suffix)] + return text + + return self._str_map(removesuffix) - return self._str_map(removesuffix) + else: + return self._str_map(lambda x: x.removesuffix(suffix)) def _str_extract(self, pat: str, flags: int = 0, expand: bool = True): regex = re.compile(pat, flags=flags) From 4fa9e0a994e1b6f13f8b224b80ac45830cc795f4 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 16 Nov 2022 12:16:33 +0000 Subject: [PATCH 3/7] replace redefined func --- pandas/core/strings/object_array.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/core/strings/object_array.py b/pandas/core/strings/object_array.py index 7694de0c1e6fa..99ede534da350 100644 --- a/pandas/core/strings/object_array.py +++ b/pandas/core/strings/object_array.py @@ -466,11 +466,7 @@ def _str_removesuffix(self, suffix: str) -> Series: if sys.version_info < (3, 9): # NOTE pyupgrade will remove this when we run it with --py39-plus # so don't remove the unnecessary `else` statement below - - def removesuffix(text: str) -> str: - if text.endswith(suffix): - return text[: -len(suffix)] - return text + from pandas.util._str_methods import removesuffix return self._str_map(removesuffix) From 4530f3467924966272696b4f86ec7613d67f07c7 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 16 Nov 2022 12:17:47 +0000 Subject: [PATCH 4/7] fixup --- pandas/core/strings/object_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/strings/object_array.py b/pandas/core/strings/object_array.py index 99ede534da350..fc4efcaffb980 100644 --- a/pandas/core/strings/object_array.py +++ b/pandas/core/strings/object_array.py @@ -468,7 +468,7 @@ def _str_removesuffix(self, suffix: str) -> Series: # so don't remove the unnecessary `else` statement below from pandas.util._str_methods import removesuffix - return self._str_map(removesuffix) + return self._str_map(functools.partial(removesuffix, suffix=suffix)) else: return self._str_map(lambda x: x.removesuffix(suffix)) From 1f3f0ed1e47cf347e99b20776416ac3b46bfc38d Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 16 Nov 2022 13:07:18 +0000 Subject: [PATCH 5/7] use version_info check in _str_methods too --- pandas/core/strings/object_array.py | 3 +- pandas/tests/util/test_str_methods.py | 69 +++++++++++++++------------ pandas/util/_str_methods.py | 26 ++++++---- 3 files changed, 57 insertions(+), 41 deletions(-) diff --git a/pandas/core/strings/object_array.py b/pandas/core/strings/object_array.py index fc4efcaffb980..424b0413d97d4 100644 --- a/pandas/core/strings/object_array.py +++ b/pandas/core/strings/object_array.py @@ -470,8 +470,7 @@ def _str_removesuffix(self, suffix: str) -> Series: return self._str_map(functools.partial(removesuffix, suffix=suffix)) - else: - return self._str_map(lambda x: x.removesuffix(suffix)) + return self._str_map(lambda x: x.removesuffix(suffix)) def _str_extract(self, pat: str, flags: int = 0, expand: bool = True): regex = re.compile(pat, flags=flags) diff --git a/pandas/tests/util/test_str_methods.py b/pandas/tests/util/test_str_methods.py index e5fd6c4066477..eea62c4bfc9c7 100644 --- a/pandas/tests/util/test_str_methods.py +++ b/pandas/tests/util/test_str_methods.py @@ -1,3 +1,5 @@ +import sys + import pytest from pandas.util._str_methods import ( @@ -5,37 +7,42 @@ removesuffix, ) +if sys.version_info < (3, 9): -@pytest.mark.parametrize( - "string, prefix, expected", - ( - ("wildcat", "wild", "cat"), - ("blackbird", "black", "bird"), - ("housefly", "house", "fly"), - ("ladybug", "lady", "bug"), - ("rattlesnake", "rattle", "snake"), - ("baboon", "badger", "baboon"), - ("quetzal", "elk", "quetzal"), - ), -) -def test_remove_prefix(string, prefix, expected): - result = removeprefix(string, prefix) - assert result == expected + @pytest.mark.parametrize( + "string, prefix, expected", + ( + ("wildcat", "wild", "cat"), + ("blackbird", "black", "bird"), + ("housefly", "house", "fly"), + ("ladybug", "lady", "bug"), + ("rattlesnake", "rattle", "snake"), + ("baboon", "badger", "baboon"), + ("quetzal", "elk", "quetzal"), + ), + ) + def test_remove_prefix(string, prefix, expected): + result = removeprefix(string, prefix) + assert result == expected + @pytest.mark.parametrize( + "string, suffix, expected", + ( + ("wildcat", "cat", "wild"), + ("blackbird", "bird", "black"), + ("housefly", "fly", "house"), + ("ladybug", "bug", "lady"), + ("rattlesnake", "snake", "rattle"), + ("seahorse", "horse", "sea"), + ("baboon", "badger", "baboon"), + ("quetzal", "elk", "quetzal"), + ), + ) + def test_remove_suffix(string, suffix, expected): + result = removesuffix(string, suffix) + assert result == expected -@pytest.mark.parametrize( - "string, suffix, expected", - ( - ("wildcat", "cat", "wild"), - ("blackbird", "bird", "black"), - ("housefly", "fly", "house"), - ("ladybug", "bug", "lady"), - ("rattlesnake", "snake", "rattle"), - ("seahorse", "horse", "sea"), - ("baboon", "badger", "baboon"), - ("quetzal", "elk", "quetzal"), - ), -) -def test_remove_suffix(string, suffix, expected): - result = removesuffix(string, suffix) - assert result == expected +else: + # NOTE: remove this file when pyupgrade --py39-plus removes + # the above block + pass diff --git a/pandas/util/_str_methods.py b/pandas/util/_str_methods.py index 0cd5a4ef5cb84..8f7aef80bc108 100644 --- a/pandas/util/_str_methods.py +++ b/pandas/util/_str_methods.py @@ -2,17 +2,27 @@ Python3.9 introduces removesuffix and remove prefix. They're reimplemented here for use in Python3.8. + +NOTE: when pyupgrade --py39-plus removes nearly everything in this file, +this file and the associated tests should be removed. """ from __future__ import annotations +import sys + +if sys.version_info < (3, 9): -def removesuffix(string: str, suffix: str) -> str: - if string.endswith(suffix): - return string[: -len(suffix)] - return string + def removesuffix(string: str, suffix: str) -> str: + if string.endswith(suffix): + return string[: -len(suffix)] + return string + def removeprefix(string: str, prefix: str) -> str: + if string.startswith(prefix): + return string[len(prefix) :] + return string -def removeprefix(string: str, prefix: str) -> str: - if string.startswith(prefix): - return string[len(prefix) :] - return string +else: + # NOTE: remove this file when pyupgrade --py39-plus removes + # the above block + pass From 3f0f76ffa18d25aebf797b55e4ff4b79458565f9 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 16 Nov 2022 15:39:36 +0000 Subject: [PATCH 6/7] move import --- pandas/tests/util/test_str_methods.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/tests/util/test_str_methods.py b/pandas/tests/util/test_str_methods.py index eea62c4bfc9c7..c07730f589824 100644 --- a/pandas/tests/util/test_str_methods.py +++ b/pandas/tests/util/test_str_methods.py @@ -2,12 +2,11 @@ import pytest -from pandas.util._str_methods import ( - removeprefix, - removesuffix, -) - if sys.version_info < (3, 9): + from pandas.util._str_methods import ( + removeprefix, + removesuffix, + ) @pytest.mark.parametrize( "string, prefix, expected", From 7481196736ba1b2a614f0659ad9f361ddc6ac170 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 17 Nov 2022 09:50:15 +0000 Subject: [PATCH 7/7] put unnecessary else statement --- pandas/core/strings/object_array.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/strings/object_array.py b/pandas/core/strings/object_array.py index 424b0413d97d4..2d77cd0da816f 100644 --- a/pandas/core/strings/object_array.py +++ b/pandas/core/strings/object_array.py @@ -469,8 +469,8 @@ def _str_removesuffix(self, suffix: str) -> Series: from pandas.util._str_methods import removesuffix return self._str_map(functools.partial(removesuffix, suffix=suffix)) - - return self._str_map(lambda x: x.removesuffix(suffix)) + else: + return self._str_map(lambda x: x.removesuffix(suffix)) def _str_extract(self, pat: str, flags: int = 0, expand: bool = True): regex = re.compile(pat, flags=flags)