From fb16d3dc3a6cd74e111cb1ea85f41d88a71988d5 Mon Sep 17 00:00:00 2001 From: Vitor Serpa <83547959+vrserpa@users.noreply.github.com> Date: Thu, 1 Jul 2021 14:09:57 -0300 Subject: [PATCH 1/5] Linting check of use of string join() with generator expression --- README.md | 1 + pandas_dev_flaker/_ast_helpers.py | 16 +++++ .../_plugins_tree/generator_join_strings.py | 29 ++++++++ tests/generator_join_strings_test.py | 71 +++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 pandas_dev_flaker/_plugins_tree/generator_join_strings.py create mode 100644 tests/generator_join_strings_test.py diff --git a/README.md b/README.md index 5ec40ee..6d51ced 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ a linter for pandas usage, please see [pandas-vet](https://github.com/deppen8/pa | PDF021 | found 'np.bool' or 'np.object' (use 'np.bool_' or 'np.object_' instead) | | PDF022 | found import from 'numpy.random' | | PDF023 | found assignment to single-letter variable | +| PDF024 | found string join() with generator expressions | ## contributing See `contributing.md` for how to get started. diff --git a/pandas_dev_flaker/_ast_helpers.py b/pandas_dev_flaker/_ast_helpers.py index 168e1b9..e003ff4 100644 --- a/pandas_dev_flaker/_ast_helpers.py +++ b/pandas_dev_flaker/_ast_helpers.py @@ -1,4 +1,5 @@ import ast +import sys from typing import Container, Dict, Sequence, Set @@ -30,3 +31,18 @@ def check_for_wrong_alias( return name_.asname != alias else: return False + + +def is_str_constant( + node: ast.Call, +) -> bool: + if isinstance(node.func, ast.Attribute): + if sys.version_info.major == 3 and sys.version_info.minor < 8: + return isinstance(node.func.value, ast.Str) + else: + return isinstance(node.func.value, ast.Constant) and isinstance( + node.func.value.value, + str, + ) + else: + return False diff --git a/pandas_dev_flaker/_plugins_tree/generator_join_strings.py b/pandas_dev_flaker/_plugins_tree/generator_join_strings.py new file mode 100644 index 0000000..346ff8d --- /dev/null +++ b/pandas_dev_flaker/_plugins_tree/generator_join_strings.py @@ -0,0 +1,29 @@ +""" +Based on + +https://github.com/asottile/pyupgrade/blob/5fb168667ae73f157dd579344708e1cdfb0c0341/pyupgrade/_plugins/generator_expressions_pep289.py +""" + +import ast +from typing import Iterator, Tuple + +from pandas_dev_flaker._ast_helpers import is_str_constant +from pandas_dev_flaker._data_tree import State, register + +MSG = "PDF024 found string join() with generator expressions" + + +@register(ast.Call) +def visit_Call( + state: State, + node: ast.Call, + parent: ast.AST, +) -> Iterator[Tuple[int, int, str]]: + if ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "join" + and is_str_constant(node) + and node.args + and isinstance(node.args[0], ast.GeneratorExp) + ): + yield (node.lineno, node.col_offset, MSG) diff --git a/tests/generator_join_strings_test.py b/tests/generator_join_strings_test.py new file mode 100644 index 0000000..20e4c0f --- /dev/null +++ b/tests/generator_join_strings_test.py @@ -0,0 +1,71 @@ +""" +Based on + +https://github.com/asottile/pyupgrade/blob/5fb168667ae73f157dd579344708e1cdfb0c0341/pyupgrade/_plugins/generator_expressions_pep289.py +""" + +import ast +import tokenize +from io import StringIO + +import pytest + +from pandas_dev_flaker.__main__ import run + + +def results(s): + return { + "{}:{}: {}".format(*r) + for r in run( + ast.parse(s), + list(tokenize.generate_tokens(StringIO(s).readline)), + ) + } + + +@pytest.mark.parametrize( + "source", + ( + pytest.param( + "''.join([str(i) for i in range(5)])", + id="String join() with list comprehension", + ), + pytest.param( + "''.join([\n" + " str(i) for i in range(5)\n" + " ]\n" + ")\n", + id="String join() with multiline list comprehension", + ), + ), +) +def test_list_comprehensions(source): + assert not results(source) + + +@pytest.mark.parametrize( + "source, expected", + ( + pytest.param( + "''.join(str(i) for i in range(5))", + "1:0: PDF024 found string join() with generator expressions", + id="String join() with generator expression", + ), + pytest.param( + "''.join((str(i) for i in range(5)))", + "1:0: PDF024 found string join() with generator expressions", + id="String join() with parenthesised generator expression", + ), + pytest.param( + "''.join((\n" + " str(i) for i in range(5)\n" + " )\n" + ")\n", + "1:0: PDF024 found string join() with generator expressions", + id="String join() with multiline generator expression", + ), + ), +) +def test_generator_expressions(source, expected): + (result,) = results(source) + assert result == expected From 68e0fa965608e73d494b759cb7f09ef1c575009c Mon Sep 17 00:00:00 2001 From: Vitor Serpa <83547959+vrserpa@users.noreply.github.com> Date: Mon, 12 Jul 2021 14:37:06 -0300 Subject: [PATCH 2/5] Fix coverage --- pandas_dev_flaker/_ast_helpers.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pandas_dev_flaker/_ast_helpers.py b/pandas_dev_flaker/_ast_helpers.py index e003ff4..8ed3c2f 100644 --- a/pandas_dev_flaker/_ast_helpers.py +++ b/pandas_dev_flaker/_ast_helpers.py @@ -36,13 +36,14 @@ def check_for_wrong_alias( def is_str_constant( node: ast.Call, ) -> bool: - if isinstance(node.func, ast.Attribute): - if sys.version_info.major == 3 and sys.version_info.minor < 8: - return isinstance(node.func.value, ast.Str) - else: - return isinstance(node.func.value, ast.Constant) and isinstance( - node.func.value.value, - str, - ) - else: - return False + python_version = float(sys.version_info.major) + 0.1 * float( + sys.version_info.minor, + ) + return isinstance(node.func, ast.Attribute) and ( + (python_version < 3.8 and isinstance(node.func.value, ast.Str)) + or ( + python_version >= 3.8 + and isinstance(node.func.value, ast.Constant) + and isinstance(node.func.value.value, str) + ) + ) From f24becab196ff4a0727393b3755c0f0935a88ee4 Mon Sep 17 00:00:00 2001 From: Vitor Serpa <83547959+vrserpa@users.noreply.github.com> Date: Mon, 12 Jul 2021 16:30:24 -0300 Subject: [PATCH 3/5] Fix python version comparison --- pandas_dev_flaker/_ast_helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas_dev_flaker/_ast_helpers.py b/pandas_dev_flaker/_ast_helpers.py index 8ed3c2f..4dae305 100644 --- a/pandas_dev_flaker/_ast_helpers.py +++ b/pandas_dev_flaker/_ast_helpers.py @@ -36,13 +36,13 @@ def check_for_wrong_alias( def is_str_constant( node: ast.Call, ) -> bool: - python_version = float(sys.version_info.major) + 0.1 * float( - sys.version_info.minor, - ) return isinstance(node.func, ast.Attribute) and ( - (python_version < 3.8 and isinstance(node.func.value, ast.Str)) + ( + sys.version_info[0:2] < (3, 8) + and isinstance(node.func.value, ast.Str) + ) or ( - python_version >= 3.8 + sys.version_info[0:2] >= (3, 8) and isinstance(node.func.value, ast.Constant) and isinstance(node.func.value.value, str) ) From c16ed24008554ef46c95cbf1970f8ef836cbedd5 Mon Sep 17 00:00:00 2001 From: Marco Edward Gorelli Date: Mon, 12 Jul 2021 20:41:49 +0100 Subject: [PATCH 4/5] Update pandas_dev_flaker/_ast_helpers.py --- pandas_dev_flaker/_ast_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas_dev_flaker/_ast_helpers.py b/pandas_dev_flaker/_ast_helpers.py index 4dae305..1ad95f7 100644 --- a/pandas_dev_flaker/_ast_helpers.py +++ b/pandas_dev_flaker/_ast_helpers.py @@ -38,7 +38,7 @@ def is_str_constant( ) -> bool: return isinstance(node.func, ast.Attribute) and ( ( - sys.version_info[0:2] < (3, 8) + sys.version_info < (3, 8) and isinstance(node.func.value, ast.Str) ) or ( From f39b9fe3713876527070c3d15fefe191065ce095 Mon Sep 17 00:00:00 2001 From: Marco Edward Gorelli Date: Mon, 12 Jul 2021 20:42:02 +0100 Subject: [PATCH 5/5] Update pandas_dev_flaker/_ast_helpers.py --- pandas_dev_flaker/_ast_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas_dev_flaker/_ast_helpers.py b/pandas_dev_flaker/_ast_helpers.py index 1ad95f7..1558f5d 100644 --- a/pandas_dev_flaker/_ast_helpers.py +++ b/pandas_dev_flaker/_ast_helpers.py @@ -42,7 +42,7 @@ def is_str_constant( and isinstance(node.func.value, ast.Str) ) or ( - sys.version_info[0:2] >= (3, 8) + sys.version_info >= (3, 8) and isinstance(node.func.value, ast.Constant) and isinstance(node.func.value.value, str) )