From 678252d79f56c54dfbb54aa369d3143f95377de7 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 9 Dec 2022 17:49:06 +0100 Subject: [PATCH 01/13] Test ruff for linting --- pyproject.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1f9ae620db22c..c85e3234d3fcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,6 +184,20 @@ exclude = ''' ) ''' +[tool.ruff] +line-length = 88 + +# Enable Pyflakes `E` and `F` codes by default. +select = ["E", "F"] +ignore = [ + # module level import not at top of file + "E402", + # do not assign a lambda expression, use a def + "E731", +] + +exclude = ["*.pyi"] + [tool.pylint.messages_control] max-line-length = 88 disable = [ From dacc80322d84bf72a77c900c5b7b7a80166d7dda Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 9 Dec 2022 20:11:10 +0100 Subject: [PATCH 02/13] further updates --- pyproject.toml | 74 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c85e3234d3fcd..6e6a661456d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,16 +187,76 @@ exclude = ''' [tool.ruff] line-length = 88 -# Enable Pyflakes `E` and `F` codes by default. -select = ["E", "F"] +select = [ + # pyflakes + "F", + # pycodestyle + "E", + "W", + # flake8-bugbear + "B", +] + ignore = [ - # module level import not at top of file - "E402", - # do not assign a lambda expression, use a def - "E731", + # space before : (needed for how black formats slicing) + # "E203", # not yet implemented + # module level import not at top of file + "E402", + # do not assign a lambda expression, use a def + "E731", + # line break before binary operator + # "W503", # not yet implemented + # line break after binary operator + # "W504", # not yet implemented + # controversial + "B006", + # controversial + "B007", + # controversial + "B008", + # setattr is used to side-step mypy + "B009", + # getattr is used to side-step mypy + "B010", + # tests use assert False + "B011", + # tests use comparisons but not their returned value + "B015", + # false positives + "B019", + # Use of functools.lru_cache or functools.cache on methods can lead to memory leaks. + "B020", + # Loop control variable overrides iterable it iterates + "B023", + # Functions defined inside a loop must not use variables redefined in the loop + # "B301", # not yet implemented + + # Additional checks that don't pass yet + # `zip()` without an explicit `strict=` parameter + "B905", + # Within an except clause, raise exceptions with ... + "B904", +] + +exclude = [ + "*.pyi", + "doc/sphinxext/*.py", + "doc/build/*.py", + "doc/temp/*.py", + ".eggs/*.py", + "versioneer.py", + # exclude asv benchmark environments from linting + "env", ] -exclude = ["*.pyi"] +[tool.ruff.per-file-ignores] +# ignoring multiline is not yet supported (https://github.com/charliermarsh/ruff/issues/1052) +"pandas/util/__init__.py" = ["F401"] +"pandas/tests/extension/base/__init__.py" = ["F401"] +# undefined name with global (https://github.com/charliermarsh/ruff/issues/960) +"pandas/io/clipboard/__init__.py" = ["F821"] +# undefined name with Literal (https://github.com/charliermarsh/ruff/issues/733 but should be fixed?) +"pandas/core/reshape/merge.py" = ["F821"] [tool.pylint.messages_control] max-line-length = 88 From 9aedb849087472cd98f6f70019de4bb7499d9424 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 11 Dec 2022 21:23:10 +0100 Subject: [PATCH 03/13] add pre-commit --- .pre-commit-config.yaml | 4 ++++ environment.yml | 1 + requirements-dev.txt | 1 + 3 files changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18f3644a0e0ae..2402bb4995668 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,10 @@ default_stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, pos ci: autofix_prs: false repos: +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.174 + hooks: + - id: ruff - repo: https://github.com/MarcoGorelli/absolufy-imports rev: v0.3.1 hooks: diff --git a/environment.yml b/environment.yml index 70884f4ca98a3..581c8630d9823 100644 --- a/environment.yml +++ b/environment.yml @@ -93,6 +93,7 @@ dependencies: - pre-commit>=2.15.0 - pycodestyle # used by flake8 - pyupgrade + - ruff=0.0.174 # documentation - gitpython # obtain contributors from git for whatsnew diff --git a/requirements-dev.txt b/requirements-dev.txt index caa3dd49add3b..c663afc408ef1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -70,6 +70,7 @@ mypy==0.990 pre-commit>=2.15.0 pycodestyle pyupgrade +ruff==0.0.174 gitpython gitdb natsort From 27963c01ffc2e663e89ff3b63f296837321232b3 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sun, 1 Jan 2023 11:43:07 +0000 Subject: [PATCH 04/13] fixup --- .pre-commit-config.yaml | 25 +-- .../development/contributing_codebase.rst | 2 +- environment.yml | 6 +- pandas/core/computation/pytables.py | 2 +- pandas/core/generic.py | 2 +- pandas/io/formats/latex.py | 2 +- pandas/io/formats/style.py | 2 +- pandas/io/parsers/python_parser.py | 8 +- pandas/plotting/_matplotlib/core.py | 21 ++- pandas/tests/arithmetic/test_datetime64.py | 8 +- pandas/tests/frame/test_constructors.py | 2 +- pandas/tests/indexes/period/test_indexing.py | 4 +- pandas/tests/io/formats/test_format.py | 2 +- pandas/tests/io/test_html.py | 4 +- pandas/tests/plotting/test_datetimelike.py | 26 ++-- pandas/util/_doctools.py | 8 +- requirements-dev.txt | 6 +- scripts/sync_flake8_versions.py | 144 ------------------ scripts/tests/test_sync_flake8_versions.py | 125 --------------- setup.cfg | 42 +---- 20 files changed, 59 insertions(+), 382 deletions(-) delete mode 100644 scripts/sync_flake8_versions.py delete mode 100644 scripts/tests/test_sync_flake8_versions.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9244903b1f9dd..b5e671a21cbaf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ ci: autofix_prs: false repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.174 + rev: v0.0.205 hooks: - id: ruff - repo: https://github.com/MarcoGorelli/absolufy-imports @@ -70,16 +70,6 @@ repos: --linelength=88, '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size' ] -- repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - # Need to patch os.remove rule in pandas-dev-flaker - exclude: ^ci/fix_wheels.py - additional_dependencies: &flake8_dependencies - - flake8==6.0.0 - - flake8-bugbear==22.7.1 - - pandas-dev-flaker==0.5.0 - repo: https://github.com/pycqa/pylint rev: v2.15.6 hooks: @@ -125,12 +115,6 @@ repos: rev: v0.6.7 hooks: - id: sphinx-lint -- repo: https://github.com/asottile/yesqa - rev: v1.4.0 - hooks: - - id: yesqa - additional_dependencies: *flake8_dependencies - stages: [manual] - repo: local hooks: # NOTE: we make `black` a local hook because if it's installed from @@ -243,13 +227,6 @@ repos: files: ^(environment.yml|requirements-dev.txt)$ pass_filenames: false additional_dependencies: [pyyaml, toml] - - id: sync-flake8-versions - name: Check flake8 version is synced across flake8, yesqa, and environment.yml - language: python - entry: python scripts/sync_flake8_versions.py - files: ^(\.pre-commit-config\.yaml|environment\.yml)$ - pass_filenames: false - additional_dependencies: [pyyaml, toml] - id: title-capitalization name: Validate correct capitalization among titles in documentation entry: python scripts/validate_rst_title_capitalization.py diff --git a/doc/source/development/contributing_codebase.rst b/doc/source/development/contributing_codebase.rst index b05f026bbbb44..c631fbb344235 100644 --- a/doc/source/development/contributing_codebase.rst +++ b/doc/source/development/contributing_codebase.rst @@ -43,7 +43,7 @@ Pre-commit ---------- Additionally, :ref:`Continuous Integration ` will run code formatting checks -like ``black``, ``flake8`` (including a `pandas-dev-flaker `_ plugin), +like ``black``, ``ruff``, ``isort``, and ``cpplint`` and more using `pre-commit hooks `_ Any warnings from these checks will cause the :ref:`Continuous Integration ` to fail; therefore, it is helpful to run the check yourself before submitting code. This diff --git a/environment.yml b/environment.yml index 6d495e25958d2..7096a6a454dbb 100644 --- a/environment.yml +++ b/environment.yml @@ -77,21 +77,17 @@ dependencies: # code checks - black=22.10.0 - cpplint - - flake8=6.0.0 - - flake8-bugbear=22.7.1 # used by flake8, find likely bugs - isort>=5.2.1 # check that imports are in the right order - mypy=0.991 - pre-commit>=2.15.0 - - pycodestyle # used by flake8 - pyupgrade - - ruff=0.0.174 + - ruff=0.0.205 # documentation - gitpython # obtain contributors from git for whatsnew - gitdb - natsort # DataFrame.sort_values doctest - numpydoc - - pandas-dev-flaker=0.5.0 - pydata-sphinx-theme<0.11 - pytest-cython # doctest - sphinx diff --git a/pandas/core/computation/pytables.py b/pandas/core/computation/pytables.py index 446f830d4100b..04a6f6c6277ee 100644 --- a/pandas/core/computation/pytables.py +++ b/pandas/core/computation/pytables.py @@ -242,7 +242,7 @@ def stringify(value): return TermValue(v, v, kind) elif kind == "bool": if isinstance(v, str): - v = not v.strip().lower() in [ + v = v.strip().lower() not in [ "false", "f", "no", diff --git a/pandas/core/generic.py b/pandas/core/generic.py index c893e9ce3d9a9..4660fa200d114 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -10829,7 +10829,7 @@ def _accum_func( if axis == 1: return self.T._accum_func( - name, func, axis=0, skipna=skipna, *args, **kwargs + name, func, axis=0, skipna=skipna, *args, **kwargs # noqa: B026 ).T def block_accum_func(blk_values): diff --git a/pandas/io/formats/latex.py b/pandas/io/formats/latex.py index b2622e61896a0..a97f3d4ef541e 100644 --- a/pandas/io/formats/latex.py +++ b/pandas/io/formats/latex.py @@ -54,7 +54,7 @@ def _split_into_full_short_caption( return full_caption, short_caption -class RowStringConverter(ABC): +class RowStringConverter: r"""Converter for dataframe rows into LaTeX strings. Parameters diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index fbafb772c59d6..0d2aca8b78112 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -2253,7 +2253,7 @@ def set_sticky( "props": props + "top:0px; z-index:2;", } ] - if not self.index.names[0] is None: + if self.index.names[0] is not None: styles[0]["props"] = ( props + f"top:0px; z-index:2; height:{pixel_size}px;" ) diff --git a/pandas/io/parsers/python_parser.py b/pandas/io/parsers/python_parser.py index aebf285e669bb..8a44fbe927768 100644 --- a/pandas/io/parsers/python_parser.py +++ b/pandas/io/parsers/python_parser.py @@ -998,12 +998,12 @@ def _rows_to_cols(self, content: list[list[Scalar]]) -> list[np.ndarray]: content_len = len(content) content = [] - for (i, l) in iter_content: - actual_len = len(l) + for (i, _content) in iter_content: + actual_len = len(_content) if actual_len > col_len: if callable(self.on_bad_lines): - new_l = self.on_bad_lines(l) + new_l = self.on_bad_lines(_content) if new_l is not None: content.append(new_l) elif self.on_bad_lines in ( @@ -1016,7 +1016,7 @@ def _rows_to_cols(self, content: list[list[Scalar]]) -> list[np.ndarray]: if self.on_bad_lines == self.BadLineHandleMethod.ERROR: break else: - content.append(l) + content.append(_content) for row_num, actual_len in bad_lines: msg = ( diff --git a/pandas/plotting/_matplotlib/core.py b/pandas/plotting/_matplotlib/core.py index 3a634a60e784e..68f896755e922 100644 --- a/pandas/plotting/_matplotlib/core.py +++ b/pandas/plotting/_matplotlib/core.py @@ -455,6 +455,7 @@ def generate(self) -> None: self._post_plot_logic_common(ax, self.data) self._post_plot_logic(ax, self.data) + @abstractmethod def _args_adjust(self) -> None: pass @@ -664,6 +665,7 @@ def _post_plot_logic_common(self, ax, data): else: # pragma no cover raise ValueError + @abstractmethod def _post_plot_logic(self, ax, data) -> None: """Post process for each axes. Overridden in child classes""" @@ -1278,6 +1280,9 @@ def _make_plot(self): err_kwds["ecolor"] = scatter.get_facecolor()[0] ax.errorbar(data[x].values, data[y].values, linestyle="none", **err_kwds) + def _args_adjust(self) -> None: + pass + class HexBinPlot(PlanePlot): @property @@ -1310,6 +1315,9 @@ def _make_plot(self) -> None: def _make_legend(self) -> None: pass + def _args_adjust(self) -> None: + pass + class LinePlot(MPLPlot): _default_rot = 0 @@ -1469,6 +1477,9 @@ def _update_stacker(cls, ax: Axes, stacking_id, values) -> None: elif (values <= 0).all(): ax._stacker_neg_prior[stacking_id] += values + def _args_adjust(self) -> None: + pass + def _post_plot_logic(self, ax: Axes, data) -> None: from matplotlib.ticker import FixedLocator @@ -1573,6 +1584,9 @@ def _plot( # type: ignore[override] res = [rect] return res + def _args_adjust(self) -> None: + pass + def _post_plot_logic(self, ax: Axes, data) -> None: LinePlot._post_plot_logic(self, ax, data) @@ -1855,5 +1869,8 @@ def blank_labeler(label, value): # leglabels is used for legend labels leglabels = labels if labels is not None else idx - for p, l in zip(patches, leglabels): - self._append_legend_handles_labels(p, l) + for _patch, _leglabel in zip(patches, leglabels): + self._append_legend_handles_labels(_patch, _leglabel) + + def _post_plot_logic(self, ax: Axes, data) -> None: + pass diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 25e31d8064c29..02188a6e57534 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -201,13 +201,13 @@ def test_nat_comparisons( expected, ): box = index_or_series - l, r = pair + lhs, rhs = pair if reverse: # add lhs / rhs switched data - l, r = r, l + lhs, rhs = rhs, lhs - left = Series(l, dtype=dtype) - right = box(r, dtype=dtype) + left = Series(lhs, dtype=dtype) + right = box(rhs, dtype=dtype) result = op(left, right) diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index 25f82eb7ff4b3..0c43defaa27bd 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -2478,7 +2478,7 @@ def test_dict_nocopy( if ( using_array_manager and not copy - and not (any_numpy_dtype in (tm.STRING_DTYPES + tm.BYTES_DTYPES)) + and any_numpy_dtype not in tm.STRING_DTYPES + tm.BYTES_DTYPES ): # TODO(ArrayManager) properly honor copy keyword for dict input td.mark_array_manager_not_yet_implemented(request) diff --git a/pandas/tests/indexes/period/test_indexing.py b/pandas/tests/indexes/period/test_indexing.py index 6cf942ad3d5d5..4fe1e471db434 100644 --- a/pandas/tests/indexes/period/test_indexing.py +++ b/pandas/tests/indexes/period/test_indexing.py @@ -778,8 +778,8 @@ def test_contains_freq_mismatch(self): rng = period_range("2007-01", freq="M", periods=10) assert Period("2007-01", freq="M") in rng - assert not Period("2007-01", freq="D") in rng - assert not Period("2007-01", freq="2M") in rng + assert Period("2007-01", freq="D") not in rng + assert Period("2007-01", freq="2M") not in rng def test_contains_nat(self): # see gh-13582 diff --git a/pandas/tests/io/formats/test_format.py b/pandas/tests/io/formats/test_format.py index f5aa481b2172e..d04e5183d980f 100644 --- a/pandas/tests/io/formats/test_format.py +++ b/pandas/tests/io/formats/test_format.py @@ -134,7 +134,7 @@ def has_horizontally_truncated_repr(df): return False # Make sure each row has this ... in the same place r = repr(df) - for ix, l in enumerate(r.splitlines()): + for ix, _ in enumerate(r.splitlines()): if not r.split()[cand_col] == "...": return False return True diff --git a/pandas/tests/io/test_html.py b/pandas/tests/io/test_html.py index 4d0f99b93ac5f..ff4fb6ae4c6cb 100644 --- a/pandas/tests/io/test_html.py +++ b/pandas/tests/io/test_html.py @@ -406,9 +406,7 @@ def test_invalid_table_attrs(self, banklist_data): ) def _bank_data(self, path, *args, **kwargs): - return self.read_html( - path, match="Metcalf", attrs={"id": "table"}, *args, **kwargs - ) + return self.read_html(path, match="Metcalf", attrs={"id": "table"}, **kwargs) @pytest.mark.slow def test_multiindex_header(self, banklist_data): diff --git a/pandas/tests/plotting/test_datetimelike.py b/pandas/tests/plotting/test_datetimelike.py index 3149fa9cb2095..f145c8d750e12 100644 --- a/pandas/tests/plotting/test_datetimelike.py +++ b/pandas/tests/plotting/test_datetimelike.py @@ -1021,10 +1021,10 @@ def test_time(self): # verify tick labels ticks = ax.get_xticks() labels = ax.get_xticklabels() - for t, l in zip(ticks, labels): - m, s = divmod(int(t), 60) + for _tick, _label in zip(ticks, labels): + m, s = divmod(int(_tick), 60) h, m = divmod(m, 60) - rs = l.get_text() + rs = _label.get_text() if len(rs) > 0: if s != 0: xp = time(h, m, s).strftime("%H:%M:%S") @@ -1045,10 +1045,10 @@ def test_time_change_xlim(self): # verify tick labels ticks = ax.get_xticks() labels = ax.get_xticklabels() - for t, l in zip(ticks, labels): - m, s = divmod(int(t), 60) + for _tick, _label in zip(ticks, labels): + m, s = divmod(int(_tick), 60) h, m = divmod(m, 60) - rs = l.get_text() + rs = _label.get_text() if len(rs) > 0: if s != 0: xp = time(h, m, s).strftime("%H:%M:%S") @@ -1062,10 +1062,10 @@ def test_time_change_xlim(self): # check tick labels again ticks = ax.get_xticks() labels = ax.get_xticklabels() - for t, l in zip(ticks, labels): - m, s = divmod(int(t), 60) + for _tick, _label in zip(ticks, labels): + m, s = divmod(int(_tick), 60) h, m = divmod(m, 60) - rs = l.get_text() + rs = _label.get_text() if len(rs) > 0: if s != 0: xp = time(h, m, s).strftime("%H:%M:%S") @@ -1086,13 +1086,13 @@ def test_time_musec(self): # verify tick labels ticks = ax.get_xticks() labels = ax.get_xticklabels() - for t, l in zip(ticks, labels): - m, s = divmod(int(t), 60) + for _tick, _label in zip(ticks, labels): + m, s = divmod(int(_tick), 60) - us = round((t - int(t)) * 1e6) + us = round((_tick - int(_tick)) * 1e6) h, m = divmod(m, 60) - rs = l.get_text() + rs = _label.get_text() if len(rs) > 0: if (us % 1000) != 0: xp = time(h, m, s, us).strftime("%H:%M:%S.%f") diff --git a/pandas/util/_doctools.py b/pandas/util/_doctools.py index 6144d1a179828..9e3ab80d1d40a 100644 --- a/pandas/util/_doctools.py +++ b/pandas/util/_doctools.py @@ -77,9 +77,9 @@ def plot(self, left, right, labels: Iterable[str] = (), vertical: bool = True): # left max_left_cols = max(self._shape(df)[1] for df in left) max_left_rows = max(self._shape(df)[0] for df in left) - for i, (l, label) in enumerate(zip(left, labels)): + for i, (_left, _label) in enumerate(zip(left, labels)): ax = fig.add_subplot(gs[i, 0:max_left_cols]) - self._make_table(ax, l, title=label, height=1.0 / max_left_rows) + self._make_table(ax, _left, title=_label, height=1.0 / max_left_rows) # right ax = plt.subplot(gs[:, max_left_cols:]) self._make_table(ax, right, title="Result", height=1.05 / vcells) @@ -90,10 +90,10 @@ def plot(self, left, right, labels: Iterable[str] = (), vertical: bool = True): gs = gridspec.GridSpec(1, hcells) # left i = 0 - for df, label in zip(left, labels): + for df, _label in zip(left, labels): sp = self._shape(df) ax = fig.add_subplot(gs[0, i : i + sp[1]]) - self._make_table(ax, df, title=label, height=height) + self._make_table(ax, df, title=_label, height=height) i += sp[1] # right ax = plt.subplot(gs[0, i:]) diff --git a/requirements-dev.txt b/requirements-dev.txt index d6945a735219d..23d1e6a6bc7bd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -54,19 +54,15 @@ flask asv>=0.5.1 black==22.10.0 cpplint -flake8==6.0.0 -flake8-bugbear==22.7.1 isort>=5.2.1 mypy==0.991 pre-commit>=2.15.0 -pycodestyle pyupgrade -ruff==0.0.174 +ruff==0.0.205 gitpython gitdb natsort numpydoc -pandas-dev-flaker==0.5.0 pydata-sphinx-theme<0.11 pytest-cython sphinx diff --git a/scripts/sync_flake8_versions.py b/scripts/sync_flake8_versions.py deleted file mode 100644 index 8852634c5d796..0000000000000 --- a/scripts/sync_flake8_versions.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Check that the flake8 (and pandas-dev-flaker) pins are the same in: - -- environment.yml -- .pre-commit-config.yaml, in the flake8 hook -- .pre-commit-config.yaml, in the additional dependencies of the yesqa hook - -The flake8 hook revision in .pre-commit-config.yaml is taken as the reference revision. - -Usage: either - -- ``python scripts/sync_flake8_versions.py``, or -- ``pre-commit run sync-flake8-versions --all-files``. -""" -from __future__ import annotations - -from dataclasses import ( - dataclass, - replace, -) -import sys -from typing import ( - Any, - Mapping, - Sequence, - TypeVar, -) - -import yaml - - -@dataclass -class Revision: - name: str - compare: str - version: str - - -@dataclass -class Revisions: - name: str - pre_commit: Revision | None = None - yesqa: Revision | None = None - environment: Revision | None = None - - -YamlMapping = Mapping[str, Any] -Repo = TypeVar("Repo", bound=YamlMapping) - -COMPARE = ("<=", "==", ">=", "<", ">", "=") - - -def _get_repo_hook(repos: Sequence[Repo], hook_name: str) -> tuple[Repo, YamlMapping]: - for repo in repos: - for hook in repo["hooks"]: - if hook["id"] == hook_name: - return repo, hook - raise RuntimeError(f"Repo with hook {hook_name} not found") # pragma: no cover - - -def _conda_to_pip_compat(dep): - if dep.compare == "=": - return replace(dep, compare="==") - else: - return dep - - -def _validate_additional_dependencies( - flake8_additional_dependencies, - environment_additional_dependencies, -) -> None: - for dep in flake8_additional_dependencies: - if dep not in environment_additional_dependencies: - sys.stdout.write( - f"Mismatch of '{dep.name}' version between 'environment.yml' " - "and additional dependencies of 'flake8' in '.pre-commit-config.yaml'\n" - ) - sys.exit(1) - - -def _validate_revisions(revisions): - if revisions.environment != revisions.pre_commit: - sys.stdout.write( - f"{revisions.name} in 'environment.yml' does not " - "match in 'flake8' from 'pre-commit'\n" - ) - sys.exit(1) - - -def _process_dependencies(deps): - for dep in deps: - if isinstance(dep, str): - for compare in COMPARE: - if compare in dep: - pkg, rev = dep.split(compare, maxsplit=1) - yield _conda_to_pip_compat(Revision(pkg, compare, rev)) - break - else: - yield from _process_dependencies(dep["pip"]) - - -def get_revisions( - precommit_config: YamlMapping, environment: YamlMapping -) -> tuple[Revisions, Revisions]: - flake8_revisions = Revisions(name="flake8") - pandas_dev_flaker_revisions = Revisions(name="pandas-dev-flaker") - - repos = precommit_config["repos"] - flake8_repo, flake8_hook = _get_repo_hook(repos, "flake8") - flake8_revisions.pre_commit = Revision("flake8", "==", flake8_repo["rev"]) - flake8_additional_dependencies = [] - for dep in _process_dependencies(flake8_hook.get("additional_dependencies", [])): - if dep.name == "pandas-dev-flaker": - pandas_dev_flaker_revisions.pre_commit = dep - else: - flake8_additional_dependencies.append(dep) - - environment_dependencies = environment["dependencies"] - environment_additional_dependencies = [] - for dep in _process_dependencies(environment_dependencies): - if dep.name == "flake8": - flake8_revisions.environment = dep - environment_additional_dependencies.append(dep) - elif dep.name == "pandas-dev-flaker": - pandas_dev_flaker_revisions.environment = dep - else: - environment_additional_dependencies.append(dep) - - _validate_additional_dependencies( - flake8_additional_dependencies, - environment_additional_dependencies, - ) - - for revisions in flake8_revisions, pandas_dev_flaker_revisions: - _validate_revisions(revisions) - - -if __name__ == "__main__": - with open(".pre-commit-config.yaml") as fd: - precommit_config = yaml.safe_load(fd) - with open("environment.yml") as fd: - environment = yaml.safe_load(fd) - get_revisions(precommit_config, environment) - sys.exit(0) diff --git a/scripts/tests/test_sync_flake8_versions.py b/scripts/tests/test_sync_flake8_versions.py deleted file mode 100644 index 743ece34e0b56..0000000000000 --- a/scripts/tests/test_sync_flake8_versions.py +++ /dev/null @@ -1,125 +0,0 @@ -import pytest - -from ..sync_flake8_versions import get_revisions - - -def test_wrong_env_flake8(capsys): - precommit_config = { - "repos": [ - { - "repo": "https://gitlab.com/pycqa/flake8", - "rev": "0.1.1", - "hooks": [ - { - "id": "flake8", - } - ], - }, - ] - } - environment = { - "dependencies": [ - "flake8=1.5.6", - ] - } - with pytest.raises(SystemExit, match=None): - get_revisions(precommit_config, environment) - result, _ = capsys.readouterr() - expected = ( - "flake8 in 'environment.yml' does not match in 'flake8' from 'pre-commit'\n" - ) - assert result == expected - - -def test_wrong_env_add_dep(capsys): - precommit_config = { - "repos": [ - { - "repo": "https://gitlab.com/pycqa/flake8", - "rev": "0.1.1", - "hooks": [ - { - "id": "flake8", - "additional_dependencies": [ - "flake8-bugs==1.1.1", - ], - } - ], - }, - { - "repo": "https://github.com/asottile/yesqa", - "rev": "v1.2.2", - "hooks": [ - { - "id": "yesqa", - "additional_dependencies": [ - "flake8==0.4.2", - "flake8-bugs==1.1.1", - ], - } - ], - }, - ] - } - environment = { - "dependencies": [ - "flake8=1.5.6", - "flake8-bugs=1.1.2", - ] - } - with pytest.raises(SystemExit, match=None): - get_revisions(precommit_config, environment) - result, _ = capsys.readouterr() - expected = ( - "Mismatch of 'flake8-bugs' version between 'environment.yml' " - "and additional dependencies of 'flake8' in '.pre-commit-config.yaml'\n" - ) - assert result == expected - - -def test_get_revisions_no_failure(capsys): - precommit_config = { - "repos": [ - { - "repo": "https://gitlab.com/pycqa/flake8", - "rev": "0.1.1", - "hooks": [ - { - "id": "flake8", - "additional_dependencies": [ - "pandas-dev-flaker==0.4.0", - "flake8-bugs==1.1.1", - ], - } - ], - }, - { - "repo": "https://github.com/asottile/yesqa", - "rev": "v1.2.2", - "hooks": [ - { - "id": "yesqa", - "additional_dependencies": [ - "flake8==0.1.1", - "pandas-dev-flaker==0.4.0", - "flake8-bugs==1.1.1", - ], - } - ], - }, - ] - } - environment = { - "dependencies": [ - "flake8=0.1.1", - "flake8-bugs=1.1.1", - { - "pip": [ - "git+https://github.com/pydata/pydata-sphinx-theme.git@master", - "pandas-dev-flaker==0.4.0", - ] - }, - ] - } - # should not raise - get_revisions(precommit_config, environment) diff --git a/setup.cfg b/setup.cfg index ef84dd7f9ce85..88b61086e1e0f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,7 @@ [flake8] max-line-length = 88 +# Although ruff is now the main linter for style checks, this section +# is still needed for validate_docstrings.py and flake8-pyi ignore = # space before : (needed for how black formats slicing) E203, @@ -12,34 +14,6 @@ ignore = # do not assign a lambda expression, use a def E731, # found modulo formatter (incorrect picks up mod operations) - S001, - # controversial - B006, - # controversial - B007, - # controversial - B008, - # setattr is used to side-step mypy - B009, - # getattr is used to side-step mypy - B010, - # tests use assert False - B011, - # tests use comparisons but not their returned value - B015, - # false positives - B019, - # Use of functools.lru_cache or functools.cache on methods can lead to memory leaks. - B020 - # Loop control variable overrides iterable it iterates - B023 - # Functions defined inside a loop must not use variables redefined in the loop - B301, - # single-letter variables - PDF023, - # "use 'pandas._testing' instead" in non-test code - PDF025, - # If test must be a simple comparison against sys.platform or sys.version_info Y002, # Use "_typeshed.Self" instead of class-bound TypeVar Y019, @@ -59,18 +33,6 @@ exclude = versioneer.py, # exclude asv benchmark environments from linting env -per-file-ignores = - # private import across modules - pandas/tests/*:PDF020 - # pytest.raises without match= - pandas/tests/extension/*:PDF009 - # os.remove - doc/make.py:PDF008 - # import from pandas._testing - pandas/testing.py:PDF014 - # can't use fixtures in asv - asv_bench/*:PDF016 - [flake8-rst] max-line-length = 84 From cdebc83ba0924759b4f4c617be5930fe480bc4e2 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sun, 1 Jan 2023 16:10:49 +0000 Subject: [PATCH 05/13] restore flake (still needed for validate_docstrings) --- environment.yml | 1 + requirements-dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 7096a6a454dbb..13c7c347309f1 100644 --- a/environment.yml +++ b/environment.yml @@ -77,6 +77,7 @@ dependencies: # code checks - black=22.10.0 - cpplint + - flake8=6.0.0 - isort>=5.2.1 # check that imports are in the right order - mypy=0.991 - pre-commit>=2.15.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 23d1e6a6bc7bd..e066c8f7f6ab4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -54,6 +54,7 @@ flask asv>=0.5.1 black==22.10.0 cpplint +flake8==6.0.0 isort>=5.2.1 mypy==0.991 pre-commit>=2.15.0 From 21495ea3a0a52ff5c50f83ca28a4dba8002ebd14 Mon Sep 17 00:00:00 2001 From: Marco Edward Gorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 2 Jan 2023 14:37:56 +0000 Subject: [PATCH 06/13] Update pandas/tests/io/test_html.py Co-authored-by: Joris Van den Bossche --- pandas/tests/io/test_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_html.py b/pandas/tests/io/test_html.py index ff4fb6ae4c6cb..f8284b5ab1c65 100644 --- a/pandas/tests/io/test_html.py +++ b/pandas/tests/io/test_html.py @@ -405,7 +405,7 @@ def test_invalid_table_attrs(self, banklist_data): url, match="First Federal Bank of Florida", attrs={"id": "tasdfable"} ) - def _bank_data(self, path, *args, **kwargs): + def _bank_data(self, path, **kwargs): return self.read_html(path, match="Metcalf", attrs={"id": "table"}, **kwargs) @pytest.mark.slow From db0cf966de3bbabaf2a2fc4634cc05c708563448 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 2 Jan 2023 14:18:58 +0000 Subject: [PATCH 07/13] remove outdated per-file-ignores --- .pre-commit-config.yaml | 2 +- pyproject.toml | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5e671a21cbaf..76bfd560476c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ ci: autofix_prs: false repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.205 + rev: v0.0.206 hooks: - id: ruff - repo: https://github.com/MarcoGorelli/absolufy-imports diff --git a/pyproject.toml b/pyproject.toml index d348b60b04320..27c7c310076d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -250,11 +250,6 @@ exclude = [ ] [tool.ruff.per-file-ignores] -# ignoring multiline is not yet supported (https://github.com/charliermarsh/ruff/issues/1052) -"pandas/util/__init__.py" = ["F401"] -"pandas/tests/extension/base/__init__.py" = ["F401"] -# undefined name with global (https://github.com/charliermarsh/ruff/issues/960) -"pandas/io/clipboard/__init__.py" = ["F821"] # undefined name with Literal (https://github.com/charliermarsh/ruff/issues/733 but should be fixed?) "pandas/core/reshape/merge.py" = ["F821"] From 407e9621a9f6eab404bb47390ebd6bf988cef776 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 2 Jan 2023 14:28:30 +0000 Subject: [PATCH 08/13] hey, all the per-file-ignores can be removed! --- pandas/core/reshape/merge.py | 2 +- pyproject.toml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index dceff32108c63..8d009d25a66ba 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -10,6 +10,7 @@ from typing import ( TYPE_CHECKING, Hashable, + Literal, Sequence, cast, ) @@ -31,7 +32,6 @@ DtypeObj, IndexLabel, JoinHow, - Literal, MergeHow, Shape, Suffixes, diff --git a/pyproject.toml b/pyproject.toml index 27c7c310076d6..1cf58d8f94d63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -249,10 +249,6 @@ exclude = [ "env", ] -[tool.ruff.per-file-ignores] -# undefined name with Literal (https://github.com/charliermarsh/ruff/issues/733 but should be fixed?) -"pandas/core/reshape/merge.py" = ["F821"] - [tool.pylint.messages_control] max-line-length = 88 disable = [ From 0fa40d83650d74884a39a932aaafe8ebeadda823 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 9 Jan 2023 18:37:23 +0000 Subject: [PATCH 09/13] update to .216, set target version --- .pre-commit-config.yaml | 2 +- pyproject.toml | 9 ++++++++- scripts/no_bool_in_generic.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa0b070a17354..faa7eec3c3089 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ ci: autofix_prs: false repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.206 + rev: v0.0.216 hooks: - id: ruff - repo: https://github.com/MarcoGorelli/absolufy-imports diff --git a/pyproject.toml b/pyproject.toml index 1cf58d8f94d63..a25863fa1ae3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,6 +186,8 @@ exclude = ''' [tool.ruff] line-length = 88 +update-check = false +target-version = "py38" select = [ # pyflakes @@ -193,8 +195,14 @@ select = [ # pycodestyle "E", "W", + # flake8-2020 + "YTT", # flake8-bugbear "B", + # flake8-quotes + "Q", + # pylint + "PLE", "PLR", "PLW", ] ignore = [ @@ -239,7 +247,6 @@ ignore = [ ] exclude = [ - "*.pyi", "doc/sphinxext/*.py", "doc/build/*.py", "doc/temp/*.py", diff --git a/scripts/no_bool_in_generic.py b/scripts/no_bool_in_generic.py index f63ae4ae1659c..92e2c0983b25b 100644 --- a/scripts/no_bool_in_generic.py +++ b/scripts/no_bool_in_generic.py @@ -19,7 +19,7 @@ def visit(tree: ast.Module) -> dict[int, list[int]]: - "Step through tree, recording when nodes are in annotations." + """Step through tree, recording when nodes are in annotations.""" in_annotation = False nodes: list[tuple[bool, ast.AST]] = [(in_annotation, tree)] to_replace = collections.defaultdict(list) From bb5d7e5bd9238ee58607b9817fa5e9cf5cdda786 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 10 Jan 2023 09:52:52 +0000 Subject: [PATCH 10/13] sync version with environment (todo - remove later) --- .pre-commit-config.yaml | 2 +- environment.yml | 2 +- pyproject.toml | 6 ++---- requirements-dev.txt | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index faa7eec3c3089..2d38a9601c52b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ ci: autofix_prs: false repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.216 + rev: v0.0.217 hooks: - id: ruff - repo: https://github.com/MarcoGorelli/absolufy-imports diff --git a/environment.yml b/environment.yml index 87423ce6844fa..668f55f0f4182 100644 --- a/environment.yml +++ b/environment.yml @@ -82,7 +82,7 @@ dependencies: - mypy=0.991 - pre-commit>=2.15.0 - pyupgrade - - ruff=0.0.205 + - ruff=0.0.217 # documentation - gitpython # obtain contributors from git for whatsnew diff --git a/pyproject.toml b/pyproject.toml index a25863fa1ae3e..a41204590bcf6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -232,16 +232,14 @@ ignore = [ "B015", # false positives "B019", - # Use of functools.lru_cache or functools.cache on methods can lead to memory leaks. - "B020", # Loop control variable overrides iterable it iterates + "B020", + # Function definition does not bind loop variable "B023", # Functions defined inside a loop must not use variables redefined in the loop # "B301", # not yet implemented # Additional checks that don't pass yet - # `zip()` without an explicit `strict=` parameter - "B905", # Within an except clause, raise exceptions with ... "B904", ] diff --git a/requirements-dev.txt b/requirements-dev.txt index ef3c022409cbe..fc2054cfed86e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -59,7 +59,7 @@ isort>=5.2.1 mypy==0.991 pre-commit>=2.15.0 pyupgrade -ruff==0.0.205 +ruff==0.0.217 gitpython gitdb natsort From 61437df513bb423efa0497948c096352bf4355cb Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 10 Jan 2023 11:25:52 +0000 Subject: [PATCH 11/13] CI didnt start, empty commit to retry? From 2e1a4aabffe715725394f301b43f15b5fd2ec08a Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 10 Jan 2023 11:38:24 +0000 Subject: [PATCH 12/13] remove ruff from environment.yml --- environment.yml | 1 - requirements-dev.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/environment.yml b/environment.yml index 668f55f0f4182..237ebaa726dd0 100644 --- a/environment.yml +++ b/environment.yml @@ -82,7 +82,6 @@ dependencies: - mypy=0.991 - pre-commit>=2.15.0 - pyupgrade - - ruff=0.0.217 # documentation - gitpython # obtain contributors from git for whatsnew diff --git a/requirements-dev.txt b/requirements-dev.txt index fc2054cfed86e..bece01d0ce4fc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -59,7 +59,6 @@ isort>=5.2.1 mypy==0.991 pre-commit>=2.15.0 pyupgrade -ruff==0.0.217 gitpython gitdb natsort From 2aaaa720c24f46546d3a3f81887f41dff31cb463 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 10 Jan 2023 13:07:04 +0000 Subject: [PATCH 13/13] back to 0.0.215 --- .pre-commit-config.yaml | 2 +- environment.yml | 1 + requirements-dev.txt | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d38a9601c52b..dfef9844ce6cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ ci: autofix_prs: false repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.217 + rev: v0.0.215 hooks: - id: ruff - repo: https://github.com/MarcoGorelli/absolufy-imports diff --git a/environment.yml b/environment.yml index 237ebaa726dd0..9e4c6db82b2ce 100644 --- a/environment.yml +++ b/environment.yml @@ -82,6 +82,7 @@ dependencies: - mypy=0.991 - pre-commit>=2.15.0 - pyupgrade + - ruff=0.0.215 # documentation - gitpython # obtain contributors from git for whatsnew diff --git a/requirements-dev.txt b/requirements-dev.txt index bece01d0ce4fc..1a2e39bb47786 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -59,6 +59,7 @@ isort>=5.2.1 mypy==0.991 pre-commit>=2.15.0 pyupgrade +ruff==0.0.215 gitpython gitdb natsort