From bb8bc8faf3c8f8004a3d9400b4baddfc430a6dab Mon Sep 17 00:00:00 2001 From: Anh Trinh Date: Thu, 1 Feb 2024 04:20:09 +0700 Subject: [PATCH 1/2] Use ruff to detect banned import --- .pre-commit-config.yaml | 11 ++-- pandas/io/common.py | 2 +- pyproject.toml | 3 + scripts/tests/test_use_io_common_urlopen.py | 23 ------- scripts/use_io_common_urlopen.py | 67 --------------------- 5 files changed, 8 insertions(+), 98 deletions(-) delete mode 100644 scripts/tests/test_use_io_common_urlopen.py delete mode 100644 scripts/use_io_common_urlopen.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ccd01abc4affe..febae49797a1d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,10 @@ repos: files: ^pandas exclude: ^pandas/tests args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix] + - id: ruff + args: [--exit-non-zero-on-fix, --select=TID251] + files: ^pandas/ + exclude: ^pandas/tests/ - id: ruff-format # TODO: "." not needed in ruff 0.1.8 args: ["."] @@ -299,13 +303,6 @@ repos: files: ^pandas/core/ exclude: ^pandas/core/api\.py$ types: [python] - - id: use-io-common-urlopen - name: Use pandas.io.common.urlopen instead of urllib.request.urlopen - language: python - entry: python scripts/use_io_common_urlopen.py - files: ^pandas/ - exclude: ^pandas/tests/ - types: [python] - id: no-bool-in-core-generic name: Use bool_t instead of bool in pandas/core/generic.py entry: python scripts/no_bool_in_generic.py diff --git a/pandas/io/common.py b/pandas/io/common.py index 16d7cb76f9ce9..682780a409a8b 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -286,7 +286,7 @@ def urlopen(*args, **kwargs): """ import urllib.request - return urllib.request.urlopen(*args, **kwargs) + return urllib.request.urlopen(*args, **kwargs) # noqa: TID251 def is_fsspec_url(url: FilePath | BaseBuffer) -> bool: diff --git a/pyproject.toml b/pyproject.toml index 934f66136f601..7614ceecbd8ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -330,6 +330,9 @@ exclude = [ "env", ] +[tool.ruff.lint.flake8-tidy-imports.banned-api] +"urllib.request.urlopen".msg = "Use pandas.io.common.urlopen instead of urllib.request.urlopen" + [tool.ruff.per-file-ignores] # relative imports allowed for asv_bench "asv_bench/*" = ["TID", "NPY002"] diff --git a/scripts/tests/test_use_io_common_urlopen.py b/scripts/tests/test_use_io_common_urlopen.py deleted file mode 100644 index c2c4a7fe9cb58..0000000000000 --- a/scripts/tests/test_use_io_common_urlopen.py +++ /dev/null @@ -1,23 +0,0 @@ -import pytest - -from scripts.use_io_common_urlopen import use_io_common_urlopen - -PATH = "t.py" - - -def test_inconsistent_usage(capsys) -> None: - content = "from urllib.request import urlopen" - result_msg = ( - "t.py:1:0: Don't use urllib.request.urlopen, " - "use pandas.io.common.urlopen instead\n" - ) - with pytest.raises(SystemExit, match=None): - use_io_common_urlopen(content, PATH) - expected_msg, _ = capsys.readouterr() - assert result_msg == expected_msg - - -def test_consistent_usage() -> None: - # should not raise - content = "from pandas.io.common import urlopen" - use_io_common_urlopen(content, PATH) diff --git a/scripts/use_io_common_urlopen.py b/scripts/use_io_common_urlopen.py deleted file mode 100644 index ade97f53cd827..0000000000000 --- a/scripts/use_io_common_urlopen.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Check that pandas/core imports pandas.array as pd_array. - -This makes it easier to grep for usage of pandas array. - -This is meant to be run as a pre-commit hook - to run it manually, you can do: - - pre-commit run use-io-common-urlopen --all-files - -""" - -from __future__ import annotations - -import argparse -import ast -import sys -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Sequence - - -ERROR_MESSAGE = ( - "{path}:{lineno}:{col_offset}: " - "Don't use urllib.request.urlopen, use pandas.io.common.urlopen instead\n" -) - - -class Visitor(ast.NodeVisitor): - def __init__(self, path: str) -> None: - self.path = path - - def visit_ImportFrom(self, node: ast.ImportFrom) -> None: - # Check that pandas.io.common.urlopen is used instead of - # urllib.request.urlopen - if ( - node.module is not None - and node.module.startswith("urllib.request") - and any(i.name == "urlopen" for i in node.names) - ): - msg = ERROR_MESSAGE.format( - path=self.path, lineno=node.lineno, col_offset=node.col_offset - ) - sys.stdout.write(msg) - sys.exit(1) - super().generic_visit(node) - - -def use_io_common_urlopen(content: str, path: str) -> None: - tree = ast.parse(content) - visitor = Visitor(path) - visitor.visit(tree) - - -def main(argv: Sequence[str] | None = None) -> None: - parser = argparse.ArgumentParser() - parser.add_argument("paths", nargs="*") - args = parser.parse_args(argv) - - for path in args.paths: - with open(path, encoding="utf-8") as fd: - content = fd.read() - use_io_common_urlopen(content, path) - - -if __name__ == "__main__": - main() From 7a2c0e1b372b0e2f3cce8f8e316139c4a47dc63e Mon Sep 17 00:00:00 2001 From: Anh Trinh Date: Sun, 4 Feb 2024 05:21:37 +0700 Subject: [PATCH 2/2] Combine rules --- .pre-commit-config.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index febae49797a1d..b7e43404b86bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,10 +30,6 @@ repos: files: ^pandas exclude: ^pandas/tests args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix] - - id: ruff - args: [--exit-non-zero-on-fix, --select=TID251] - files: ^pandas/ - exclude: ^pandas/tests/ - id: ruff-format # TODO: "." not needed in ruff 0.1.8 args: ["."]